feat: hold the nearest baked html5 frame while playing

Author: dazzatronus

src/components/canvas/players/html5-player.ts

@@ -25,6 +25,8 @@ import { createCaptureLoadingGraphic, createPlaceholderGraphic } from "./placeho
 const IFRAME_OFFSCREEN_X = -10000;
 const IFRAME_LOAD_TIMEOUT_MS = 10_000;
 const DECODED_FRAME_LIMIT = 30;
+// One first-pass stride: a held frame is never more than 233 ms behind at 30 fps.
+const HELD_FRAME_LIMIT = 7;
 
 type HarnessWindow = Window & {
 	["__shotstackSeek"]?: (ms: number) => void;
@@ -99,6 +101,12 @@ export function captureFramesInFlight(logicalCores: number | undefined, frameByt
 	return Math.max(1, Math.min(byCores, Math.floor(CAPTURE_MEMORY_BUDGET_BYTES / frameBytes)));
 }
 
+/** The latest baked frame at most `maxBehind` frames before `idx`, or null. */
+export function heldFrame(frames: ArrayLike<Blob | undefined>, idx: number, maxBehind: number): number | null {
+	for (let i = idx; i >= Math.max(0, idx - maxBehind); i -= 1) if (frames[i]) return i;
+	return null;
+}
+
 export function bakeScaleFor(onScreenScale: number): number {
 	return BAKE_SCALES.find(scale => scale >= onScreenScale) ?? 1;
 }
@@ -160,6 +168,8 @@ export class Html5Player extends Player {
 	private stillTexture: pixi.Texture | null = null;
 	private liveStill: pixi.Sprite | null = null;
 	private liveStillFrame: number = -1;
+	// False while the live still is a held frame from before the playhead, so pausing on it renders the exact one.
+	private liveStillExact: boolean = true;
 	private liveStillInFlight: boolean = false;
 	private stillRequest: number | null = null;
 	// The frame the running bake draws next. Null between bakes.
@@ -1092,8 +1102,9 @@ export class Html5Player extends Player {
 	// One still renders at a time. Positions asked for meanwhile replace each other, so only the newest renders next.
 	private showStillAt(seconds: number): void {
 		const frame = Math.floor(seconds * this.captureFps);
-		if (frame === this.liveStillFrame) return;
+		if (frame === this.liveStillFrame && (this.liveStillExact || this.edit.isPlaying)) return;
 		this.liveStillFrame = frame;
+		this.liveStillExact = true;
 		this.stillRequest = seconds;
 		if (!this.liveStillInFlight) this.renderLiveStills().catch(err => console.warn("[Html5Player] still failed:", err));
 	}
@@ -1111,7 +1122,10 @@ export class Html5Player extends Player {
 						if (!this.disposed) console.warn("[Html5Player] still failed:", err);
 						return null;
 					});
-				const baked = this.bakingFrames?.[frame];
+				// Playing, a clip holds a recent baked frame rather than spend bake time on a still.
+				const held = this.bakingFrames ? heldFrame(this.bakingFrames, frame, this.edit.isPlaying ? HELD_FRAME_LIMIT : 0) : null;
+				const baked = held === null ? undefined : this.bakingFrames?.[held];
+				if (baked && held !== frame) this.liveStillExact = false;
 				let texture = baked ? await this.bakingTexture(baked) : await attempt();
 				// A page can load too slowly once, so a failed still gets one more try while nothing newer is waiting.
 				if (!texture && !baked && this.liveStillFrame === frame && this.stillRequest === null) texture = await attempt();
@@ -1143,6 +1157,7 @@ export class Html5Player extends Player {
 	private removeLiveStill(): void {
 		this.stillRequest = null;
 		this.liveStillFrame = -1;
+		this.liveStillExact = true;
 		if (!this.liveStill) return;
 		this.contentContainer.removeChild(this.liveStill);
 		this.liveStill.texture.destroy(true);

tests/html5-player.test.ts

@@ -1,7 +1,7 @@
 /** @jest-environment jsdom */
 
 import { BakeQueue } from "@canvas/players/bake-queue";
-import { Html5Player, bakeScaleFor, captureFramesInFlight } from "@canvas/players/html5-player";
+import { Html5Player, bakeScaleFor, captureFramesInFlight, heldFrame } from "@canvas/players/html5-player";
 import type { Edit } from "@core/edit-session";
 import { EditEvent, InternalEvent } from "@core/events/edit-events";
 import { sec } from "@core/timing/types";
@@ -1600,6 +1600,54 @@ it("shows a first bake's frames as they land", async () => {
 	await capture;
 });
 
+it.each([
+	[[0, 3], 2, 7, 0],
+	[[0, 3], 3, 7, 3],
+	[[0], 8, 7, null],
+	[[0, 3], 2, 0, null],
+	[[19], 20, 7, 19]
+])("finds the frame to hold among %p at %p within %p frames: %p", (present, idx, maxBehind, expected) => {
+	const frames = Array.from({ length: 20 }, (_, i) => (present.includes(i) ? new Blob([`${i}`]) : undefined));
+
+	expect(heldFrame(frames, idx, maxBehind)).toBe(expected);
+});
+
+const playingAhead = async (...baked: number[]) => {
+	const created = createStillPlayer();
+	const { player, harness, edit } = created;
+	Object.assign(edit, { isPlaying: true });
+	Object.assign(player, { getPlaybackTime: () => 0.4 }); // frame 12
+	baking(harness);
+	harness.beginCapture();
+	Object.assign(harness, { bakingFrames: bakedAt(...baked), bakingScale: 1 });
+	player.update(0, 0);
+	await until(stillShown(harness));
+	return created;
+};
+
+it("holds the nearest baked frame while playing ahead of the bake", async () => {
+	const { harness, seeks } = await playingAhead(8, 16);
+
+	expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toBe("baked 8");
+	expect(seeks.some(seek => seek.startsWith("still"))).toBe(false);
+});
+
+it("renders a still while playing when the nearest baked frame is more than 7 frames back", async () => {
+	const { harness } = await playingAhead(0);
+
+	expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<");
+});
+
+it("renders the exact still once playback pauses on a held frame", async () => {
+	const { player, harness, edit } = await playingAhead(8, 16);
+	const held = layer(harness)[0].texture;
+	Object.assign(edit, { isPlaying: false });
+	player.update(0, 0);
+	await until(() => layer(harness)[0].texture !== held);
+
+	expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<");
+});
+
 it("swaps the still for baked frames once the bake finishes", async () => {
 	const { player, harness } = createStillPlayer();
 	let finish: (frames: Blob[]) => void = () => {};