fix: keep the frame on screen when a held html5 frame would be older

Author: dazzatronus

src/components/canvas/players/html5-player.ts

@@ -25,7 +25,7 @@ import { createCaptureLoadingGraphic, createPlaceholderGraphic } from "./placeho
 const IFRAME_OFFSCREEN_X = -10000;
 const IFRAME_LOAD_TIMEOUT_MS = 10_000;
 const DECODED_FRAME_LIMIT = 30;
-// One first-pass stride: a held frame is never more than 233 ms behind at 30 fps.
+// How stale a held frame may be when it's chosen: 233 ms at 30 fps.
 const HELD_FRAME_LIMIT = 7;
 
 type HarnessWindow = Window & {
@@ -170,6 +170,8 @@ export class Html5Player extends Player {
 	private liveStillFrame: number = -1;
 	// False while the live still is a held frame from before the playhead, so pausing on it renders the exact one.
 	private liveStillExact: boolean = true;
+	// The frame the live still shows, which a held frame trails.
+	private liveStillShown: number = -1;
 	private liveStillInFlight: boolean = false;
 	private stillRequest: number | null = null;
 	// The frame the running bake draws next. Null between bakes.
@@ -1117,38 +1119,51 @@ export class Html5Player extends Player {
 				const frame = Math.floor(seconds * this.captureFps);
 				const hash = this.contentHash;
 				this.stillRequest = null;
-				const attempt = () =>
-					this.renderStill(seconds).catch(err => {
-						if (!this.disposed) console.warn("[Html5Player] still failed:", err);
-						return null;
-					});
-				// Playing, a clip holds a recent baked frame rather than spend bake time on a still.
+				// A still takes main-thread time from the bake, so a playing clip holds a recent baked frame.
 				const held = this.bakingFrames ? heldFrame(this.bakingFrames, frame, this.edit.isPlaying ? HELD_FRAME_LIMIT : 0) : null;
-				const baked = held === null ? undefined : this.bakingFrames?.[held];
-				if (baked && held !== frame) this.liveStillExact = false;
-				let texture = baked ? await this.bakingTexture(baked) : await attempt();
-				// A page can load too slowly once, so a failed still gets one more try while nothing newer is waiting.
-				if (!texture && !baked && this.liveStillFrame === frame && this.stillRequest === null) texture = await attempt();
-				const unbaked = this.mode === "editing" || this.mode === "stale" || this.mode === "capturing";
-				if (!texture || this.disposed || !unbaked || this.contentHash !== hash) {
-					texture?.destroy(true);
-				} else if (this.liveStill) {
-					this.liveStill.texture.destroy(true);
-					this.liveStill.texture = texture;
+				const shown = this.liveStillShown;
+				// Playback never steps back to an older frame, or decodes the frame it already shows.
+				if (this.edit.isPlaying && held !== null && this.liveStill && held <= shown && shown <= frame) {
+					this.liveStillExact = shown === frame;
 				} else {
-					this.removeLoadingGraphic();
-					this.liveStill = new pixi.Sprite(texture);
-					// A capture in progress shows its own copy of the clip.
-					this.liveStill.visible = !this.staticSprite;
-					this.contentContainer.addChild(this.liveStill);
-					if (this.clipConfiguration.width && this.clipConfiguration.height) this.applyFixedDimensions();
+					await this.renderLiveStill(seconds, frame, held, hash);
 				}
 			}
 		} finally {
 			this.liveStillInFlight = false;
 		}
 	}
 
+	private async renderLiveStill(seconds: number, frame: number, held: number | null, hash: string | null): Promise<void> {
+		const attempt = () =>
+			this.renderStill(seconds).catch(err => {
+				if (!this.disposed) console.warn("[Html5Player] still failed:", err);
+				return null;
+			});
+		const baked = held === null ? undefined : this.bakingFrames?.[held];
+		if (baked && held !== frame) this.liveStillExact = false;
+		let texture = baked ? await this.bakingTexture(baked) : await attempt();
+		// A page can load too slowly once, so a failed still gets one more try while nothing newer is waiting.
+		if (!texture && !baked && this.liveStillFrame === frame && this.stillRequest === null) texture = await attempt();
+		const unbaked = this.mode === "editing" || this.mode === "stale" || this.mode === "capturing";
+		if (!texture || this.disposed || !unbaked || this.contentHash !== hash) {
+			texture?.destroy(true);
+			return;
+		}
+		this.liveStillShown = baked && held !== null ? held : frame;
+		if (this.liveStill) {
+			this.liveStill.texture.destroy(true);
+			this.liveStill.texture = texture;
+			return;
+		}
+		this.removeLoadingGraphic();
+		this.liveStill = new pixi.Sprite(texture);
+		// A capture in progress shows its own copy of the clip.
+		this.liveStill.visible = !this.staticSprite;
+		this.contentContainer.addChild(this.liveStill);
+		if (this.clipConfiguration.width && this.clipConfiguration.height) this.applyFixedDimensions();
+	}
+
 	// The resolution makes a scaled frame's texture report the clip's full size.
 	private async bakingTexture(blob: Blob): Promise<pixi.Texture> {
 		return pixi.Texture.from({ resource: await createImageBitmap(blob), resolution: this.bakingScale });
@@ -1157,7 +1172,7 @@ export class Html5Player extends Player {
 	private removeLiveStill(): void {
 		this.stillRequest = null;
 		this.liveStillFrame = -1;
-		this.liveStillExact = true;
+		this.liveStillShown = -1;
 		if (!this.liveStill) return;
 		this.contentContainer.removeChild(this.liveStill);
 		this.liveStill.texture.destroy(true);

tests/html5-player.test.ts

@@ -1604,6 +1604,7 @@ it.each([
 	[[0, 3], 2, 7, 0],
 	[[0, 3], 3, 7, 3],
 	[[0], 8, 7, null],
+	[[1], 8, 7, 1],
 	[[0, 3], 2, 0, null],
 	[[19], 20, 7, 19]
 ])("finds the frame to hold among %p at %p within %p frames: %p", (present, idx, maxBehind, expected) => {
@@ -1632,8 +1633,36 @@ it("holds the nearest baked frame while playing ahead of the bake", async () =>
 	expect(seeks.some(seek => seek.startsWith("still"))).toBe(false);
 });
 
+it("holds a baked frame exactly 7 frames back while playing", async () => {
+	const { harness } = await playingAhead(5);
+
+	expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toBe("baked 5");
+});
+
 it("renders a still while playing when the nearest baked frame is more than 7 frames back", async () => {
-	const { harness } = await playingAhead(0);
+	const { harness } = await playingAhead(4);
+
+	expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<");
+});
+
+it("decodes a held frame once while playing on through it", async () => {
+	const { player } = await playingAhead(8);
+	const decode = jest.spyOn(globalThis, "createImageBitmap");
+	[0.4, 0.4, 13 / 30, 13 / 30, 14 / 30].forEach(t => {
+		Object.assign(player, { getPlaybackTime: () => t });
+		player.update(0, 0);
+	});
+	await flush(50);
+
+	expect(decode).not.toHaveBeenCalled();
+});
+
+it("never steps back to an older baked frame than the still on screen while playing", async () => {
+	const { player, harness } = await playingAhead(0); // a still of frame 12
+	harness.bakingFrames![11] = new Blob(["baked 11"]);
+	Object.assign(player, { getPlaybackTime: () => 13 / 30 });
+	player.update(0, 0);
+	await flush(50);
 
 	expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<");
 });