[3/3] feat: tell viewers when html5 playback is buffering (#274)

* feat: report when html5 playback is buffering

* feat: tell viewers when html5 playback is buffering

* fix: judge html5 buffering by the bake and keep the notice in view

* feat: show html5 buffering as a single word

Author: dazzatronus

src/components/canvas/buffering-notice.ts

@@ -0,0 +1,80 @@
+export interface ContentBounds {
+	left: number;
+	right: number;
+	top: number;
+	bottom: number;
+}
+
+/** The part of the video inside a preview of this size, or null when none of it is. */
+export function visibleBounds(content: ContentBounds, width: number, height: number): ContentBounds | null {
+	const left = Math.max(content.left, 0);
+	const right = Math.min(content.right, width);
+	const top = Math.max(content.top, 0);
+	const bottom = Math.min(content.bottom, height);
+	return right > left && bottom > top ? { left, right, top, bottom } : null;
+}
+
+// Long enough that one late frame doesn't flash it, short enough that a real stutter gets explained.
+const SHOW_AFTER_MS = 500;
+const HIDE_AFTER_MS = 1500;
+const INSET_PX = 12;
+// A longer gap between ticks means ticking paused, as in an export or a hidden tab, and says nothing about stutter.
+const MAX_TICK_GAP_MS = 100;
+
+/**
+ * Labels stuttering html5 playback "Buffering" in the video's bottom-left corner. It holds only its own display
+ * state, and the canvas feeds it on every tick.
+ * @internal
+ */
+export class BufferingNotice {
+	public readonly element: HTMLDivElement;
+	private readonly now: () => number;
+	private bufferingSince: number | null = null;
+	private smoothSince: number | null = null;
+	private lastTickAt: number | null = null;
+
+	constructor(now: () => number = () => performance.now()) {
+		this.now = now;
+		this.element = document.createElement("div");
+		this.element.className = "ss-buffering-notice";
+		this.element.hidden = true;
+		// The toolbar's status line already tells assistive tech that playback is preparing.
+		this.element.setAttribute("aria-hidden", "true");
+		this.element.innerHTML = '<span class="ss-buffering-notice-spinner"></span>';
+		this.element.append("Buffering");
+	}
+
+	public mount(root: HTMLElement): void {
+		root.appendChild(this.element);
+	}
+
+	/** `idle` is paused or fully loaded, and `bounds` is null when none of the video is in view. Either hides it at once. */
+	public update(buffering: boolean, idle: boolean, bounds: ContentBounds | null): void {
+		const now = this.now();
+		// A change happened somewhere after the last tick, so its timer starts there.
+		const previous = this.lastTickAt !== null && now - this.lastTickAt <= MAX_TICK_GAP_MS ? this.lastTickAt : now;
+		this.lastTickAt = now;
+		if (idle || !bounds) {
+			this.bufferingSince = null;
+			this.smoothSince = null;
+			this.element.hidden = true;
+			return;
+		}
+		if (buffering) {
+			this.smoothSince = null;
+			this.bufferingSince ??= previous;
+			if (now - this.bufferingSince >= SHOW_AFTER_MS) this.element.hidden = false;
+		} else {
+			this.bufferingSince = null;
+			this.smoothSince ??= previous;
+			if (now - this.smoothSince >= HIDE_AFTER_MS) this.element.hidden = true;
+		}
+		if (this.element.hidden) return;
+		this.element.style.left = `${bounds.left + INSET_PX}px`;
+		this.element.style.top = `${bounds.bottom - INSET_PX}px`;
+	}
+
+	public dispose(): void {
+		this.element.remove();
+	}
+}

src/components/canvas/players/html5-player.ts

@@ -972,6 +972,12 @@ export class Html5Player extends Player {
 		return this.edit.getHtml5Prerender() === "on-load" || this.hasTriggeredCapture ? "pending" : null;
 	}
 
+	// A baked frame reaches the screen a tick or two after the playhead does, so the bake decides, not the screen.
+	public override isPlaybackBuffering(): boolean {
+		if (!this.edit.isPlaying || !this.isActive() || this.getPlaybackReadiness() !== "pending") return false;
+		return !this.bakingFrames?.[Math.floor(this.getPlaybackTime() * this.captureFps)];
+	}
+
 	public override getCaptureProgress(): number {
 		if (this.getPlaybackReadiness() !== "pending") return 1;
 		return this.captureFramesTotal > 0 ? this.captureFramesDone / this.captureFramesTotal : 0;

src/components/canvas/players/player.ts

@@ -585,6 +585,10 @@ export abstract class Player extends Entity {
 		return 1;
 	}
 
+	public isPlaybackBuffering(): boolean {
+		return false;
+	}
+
 	public isActive(): boolean {
 		// playbackTime is in seconds, matching clip start/end
 		return this.edit.playbackTime >= this.getStart() && this.edit.playbackTime < this.getEnd();

src/components/canvas/shotstack-canvas.ts

@@ -1,8 +1,9 @@
+import { BufferingNotice, visibleBounds } from "@canvas/buffering-notice";
 import type { Player } from "@canvas/players/player";
 import { AlignmentGuides } from "@canvas/system/alignment-guides";
 import { createWebGLErrorOverlay } from "@canvas/webgl-error-overlay";
 import { Edit } from "@core/edit-session";
-import { InternalEvent } from "@core/events/edit-events";
+import { EditEvent, InternalEvent } from "@core/events/edit-events";
 import { sanitizeColor } from "@core/shared/color-utils";
 import { ms } from "@core/timing/types";
 import type { UIController } from "@core/ui/ui-controller";
@@ -61,6 +62,7 @@ export class Canvas {
 	private onBackgroundClickBound: (event: pixi.FederatedPointerEvent) => void;
 	private onWheelBound: (e: WheelEvent) => void;
 	private canvasRoot: HTMLDivElement | null = null;
+	private readonly bufferingNotice = new BufferingNotice();
 
 	constructor(edit: Edit) {
 		this.application = new pixi.Application();
@@ -141,6 +143,7 @@ export class Canvas {
 
 		root.style.position = "relative";
 		root.appendChild(this.application.canvas);
+		this.bufferingNotice.mount(root);
 
 		// Auto-mount UIController to canvas root (toolbars sit inside canvas)
 		// mount() handles deferred positioning via double rAF
@@ -568,6 +571,7 @@ export class Canvas {
 		internalEvents.on(InternalEvent.PlayerRemovedFromTrack, this.onPlayerRemovedFromTrack);
 		internalEvents.on(InternalEvent.TrackContainerRemoved, this.onTrackContainerRemoved);
 		internalEvents.on(InternalEvent.ViewportSizeChanged, this.onViewportSizeChanged);
+		this.edit.events.on(EditEvent.PlaybackPause, this.onPlaybackPause);
 		internalEvents.on(InternalEvent.ViewportNeedsZoomToFit, this.onViewportNeedsZoomToFit);
 	}
 
@@ -603,6 +607,11 @@ export class Canvas {
 		this.removeTrackContainer(trackIndex);
 	};
 
+	// An export stops the ticker before it pauses, so no tick would be left to hide the notice.
+	private onPlaybackPause = (): void => {
+		this.bufferingNotice.update(false, true, null);
+	};
+
 	private onViewportSizeChanged = ({ width, height, backgroundColor }: { width: number; height: number; backgroundColor: string }): void => {
 		this.updateViewportForSize(width, height, backgroundColor);
 	};
@@ -648,6 +657,9 @@ export class Canvas {
 
 	private onTick(ticker: pixi.Ticker): void {
 		this.edit.update(ticker.deltaTime, ms(ticker.deltaMS));
+		const idle = !this.edit.isPlaying || this.edit.getPendingCaptures().pending === 0;
+		const shown = visibleBounds(this.getContentBounds(), this.viewportSize.width, this.viewportSize.height);
+		this.bufferingNotice.update(this.edit.isPlaybackBuffering(), idle, shown);
 
 		// Update canvas overlays (selection handles, guides, etc.)
 		this.uiController?.updateOverlays(ticker.deltaTime, ticker.deltaMS);
@@ -693,13 +705,15 @@ export class Canvas {
 	}
 
 	public dispose(): void {
+		this.bufferingNotice.dispose();
 		// Unsubscribe from Edit internal events
 		const internalEvents = this.edit.getInternalEvents();
 		internalEvents.off(InternalEvent.PlayerAddedToTrack, this.onPlayerAddedToTrack);
 		internalEvents.off(InternalEvent.PlayerMovedBetweenTracks, this.onPlayerMovedBetweenTracks);
 		internalEvents.off(InternalEvent.PlayerRemovedFromTrack, this.onPlayerRemovedFromTrack);
 		internalEvents.off(InternalEvent.TrackContainerRemoved, this.onTrackContainerRemoved);
 		internalEvents.off(InternalEvent.ViewportSizeChanged, this.onViewportSizeChanged);
+		this.edit.events.off(EditEvent.PlaybackPause, this.onPlaybackPause);
 		internalEvents.off(InternalEvent.ViewportNeedsZoomToFit, this.onViewportNeedsZoomToFit);
 
 		const root = document.querySelector<HTMLDivElement>(Canvas.CanvasSelector);

src/core/edit-session.ts

@@ -584,6 +584,14 @@ export class Edit {
 		return captureProgressOf(this.getPlayerByClipId(clipId));
 	}
 
+	/**
+	 * Whether a playing html5 clip is showing a held frame, a late still or nothing.
+	 * @internal
+	 */
+	public isPlaybackBuffering(): boolean {
+		return this.clips.some(player => player.isPlaybackBuffering());
+	}
+
 	/**
 	 * Transient generation state for a clip, or undefined when idle. Not part of the
 	 * edit: never saved, never undone, cleared on reload.

src/styles/index.css

@@ -22,6 +22,7 @@
 @import "./ui/background-color-picker.css";
 @import "./ui/font-picker.css";
 @import "./ui/webgl-error.css";
+@import "./ui/buffering-notice.css";
 
 /* Canvas Styles */
 @import "./canvas/canvas-pan.css";

src/styles/ui/buffering-notice.css

@@ -0,0 +1,48 @@
+/* Sits in the video's corner while html5 playback stutters. It's editor chrome, so it never reaches an export. */
+.ss-buffering-notice {
+	position: absolute;
+	z-index: 8;
+	transform: translateY(-100%);
+	display: flex;
+	align-items: center;
+	gap: 7px;
+	padding: 6px 11px;
+	border-radius: 999px;
+	background: rgb(17 24 39 / 72%);
+	-webkit-backdrop-filter: blur(4px);
+	backdrop-filter: blur(4px);
+	color: #fff;
+	font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif;
+	font-size: 12px;
+	font-weight: 600;
+	line-height: 1.3;
+	white-space: nowrap;
+	/* A label, not a control: clicks go through to the canvas. */
+	pointer-events: none;
+}
+
+.ss-buffering-notice[hidden] {
+	display: none;
+}
+
+.ss-buffering-notice-spinner {
+	flex: none;
+	width: 12px;
+	height: 12px;
+	border: 2px solid rgb(255 255 255 / 35%);
+	border-top-color: #fff;
+	border-radius: 50%;
+	animation: ss-buffering-spin 0.9s linear infinite;
+}
+
+@keyframes ss-buffering-spin {
+	to {
+		transform: rotate(360deg);
+	}
+}
+
+@media (prefers-reduced-motion: reduce) {
+	.ss-buffering-notice-spinner {
+		animation-duration: 3s;
+	}
+}

tests/buffering-notice.test.ts

@@ -0,0 +1,113 @@
+/** @jest-environment jsdom */
+import { BufferingNotice, type ContentBounds, visibleBounds } from "@canvas/buffering-notice";
+
+const wide: ContentBounds = { left: 100, right: 740, top: 50, bottom: 410 };
+
+function mountNotice() {
+	let now = 0;
+	const root = document.createElement("div");
+	document.body.appendChild(root);
+	const notice = new BufferingNotice(() => now);
+	notice.mount(root);
+	notice.update(false, false, wide);
+	const { element } = notice;
+	// Ticks at 50 ms, the way the canvas would call it every frame.
+	const play = (ms: number, buffering: boolean, idle = false, bounds: ContentBounds | null = wide) => {
+		for (let t = 0; t < ms; t += 50) {
+			now += 50;
+			notice.update(buffering, idle, bounds);
+		}
+	};
+	const pauseTicking = (ms: number) => {
+		now += ms;
+	};
+	return { notice, element, root, play, pauseTicking };
+}
+
+afterEach(() => {
+	document.body.replaceChildren();
+});
+
+it("appears after half a second of buffering, not for a single late frame", () => {
+	const { element, play } = mountNotice();
+
+	play(100, true);
+	play(100, false);
+	play(450, true);
+	expect(element.hidden).toBe(true);
+
+	play(50, true);
+	expect(element.hidden).toBe(false);
+});
+
+it("goes after 1.5 s of smooth playback, and at once when paused or fully loaded", () => {
+	const { element, play } = mountNotice();
+	play(500, true);
+
+	play(1450, false);
+	expect(element.hidden).toBe(false);
+	play(50, false);
+	expect(element.hidden).toBe(true);
+
+	play(500, true);
+	expect(element.hidden).toBe(false);
+	play(50, true, true);
+	expect(element.hidden).toBe(true);
+});
+
+it("says only Buffering, and leaves it to the toolbar's status to tell assistive tech", () => {
+	const { element, play } = mountNotice();
+
+	play(500, true);
+
+	expect(element.textContent).toBe("Buffering");
+	expect(element.getAttribute("aria-hidden")).toBe("true");
+});
+
+it("sits inside the video's bottom-left corner", () => {
+	const { element, play } = mountNotice();
+
+	play(500, true);
+
+	expect(element.style.left).toBe("112px");
+	expect(element.style.top).toBe("398px");
+});
+
+it("leaves the canvas when disposed", () => {
+	const { notice, root } = mountNotice();
+
+	notice.dispose();
+
+	expect(root.querySelector(".ss-buffering-notice")).toBeNull();
+});
+
+it("keeps to the part of the video inside the preview", () => {
+	const { element, play } = mountNotice();
+	const shown = visibleBounds({ left: -50, right: 600, top: -20, bottom: 900 }, 800, 500);
+	expect(shown).toEqual({ left: 0, right: 600, top: 0, bottom: 500 });
+
+	play(500, true, false, shown);
+
+	expect(element.style.left).toBe("12px");
+	expect(element.style.top).toBe("488px");
+});
+
+it("hides while none of the video is in view", () => {
+	const { element, play } = mountNotice();
+	play(500, true);
+	const offscreen = visibleBounds({ left: 900, right: 1500, top: 0, bottom: 300 }, 800, 500);
+	expect(offscreen).toBeNull();
+
+	play(50, true, false, offscreen);
+
+	expect(element.hidden).toBe(true);
+});
+
+it("dates no timer from before a pause in ticking", () => {
+	const { element, play, pauseTicking } = mountNotice();
+	pauseTicking(5000);
+
+	play(50, true);
+
+	expect(element.hidden).toBe(true);
+});