diff --git a/src/components/canvas/players/html5-player.ts b/src/components/canvas/players/html5-player.ts
index 1cc8ba84..ba0a6216 100644
--- a/src/components/canvas/players/html5-player.ts
+++ b/src/components/canvas/players/html5-player.ts
@@ -25,6 +25,8 @@ import { createCaptureLoadingGraphic, createPlaceholderGraphic } from "./placeho
const IFRAME_OFFSCREEN_X = -10000;
const IFRAME_LOAD_TIMEOUT_MS = 10_000;
const DECODED_FRAME_LIMIT = 30;
+// How stale a held frame may be when it's chosen: 233 ms at 30 fps.
+const HELD_FRAME_LIMIT = 7;
type HarnessWindow = Window & {
["__shotstackSeek"]?: (ms: number) => void;
@@ -99,6 +101,12 @@ export function captureFramesInFlight(logicalCores: number | undefined, frameByt
return Math.max(1, Math.min(byCores, Math.floor(CAPTURE_MEMORY_BUDGET_BYTES / frameBytes)));
}
+/** The latest baked frame at most `maxBehind` frames before `idx`, or null. */
+export function heldFrame(frames: ArrayLike, idx: number, maxBehind: number): number | null {
+ for (let i = idx; i >= Math.max(0, idx - maxBehind); i -= 1) if (frames[i]) return i;
+ return null;
+}
+
export function bakeScaleFor(onScreenScale: number): number {
return BAKE_SCALES.find(scale => scale >= onScreenScale) ?? 1;
}
@@ -160,6 +168,10 @@ export class Html5Player extends Player {
private stillTexture: pixi.Texture | null = null;
private liveStill: pixi.Sprite | null = null;
private liveStillFrame: number = -1;
+ // False while the live still is a held frame from before the playhead, so pausing on it renders the exact one.
+ private liveStillExact: boolean = true;
+ // The frame the live still shows, which a held frame trails.
+ private liveStillShown: number = -1;
private liveStillInFlight: boolean = false;
private stillRequest: number | null = null;
// The frame the running bake draws next. Null between bakes.
@@ -1092,8 +1104,9 @@ export class Html5Player extends Player {
// One still renders at a time. Positions asked for meanwhile replace each other, so only the newest renders next.
private showStillAt(seconds: number): void {
const frame = Math.floor(seconds * this.captureFps);
- if (frame === this.liveStillFrame) return;
+ if (frame === this.liveStillFrame && (this.liveStillExact || this.edit.isPlaying)) return;
this.liveStillFrame = frame;
+ this.liveStillExact = true;
this.stillRequest = seconds;
if (!this.liveStillInFlight) this.renderLiveStills().catch(err => console.warn("[Html5Player] still failed:", err));
}
@@ -1106,28 +1119,14 @@ export class Html5Player extends Player {
const frame = Math.floor(seconds * this.captureFps);
const hash = this.contentHash;
this.stillRequest = null;
- const attempt = () =>
- this.renderStill(seconds).catch(err => {
- if (!this.disposed) console.warn("[Html5Player] still failed:", err);
- return null;
- });
- const baked = this.bakingFrames?.[frame];
- let texture = baked ? await this.bakingTexture(baked) : await attempt();
- // A page can load too slowly once, so a failed still gets one more try while nothing newer is waiting.
- if (!texture && !baked && this.liveStillFrame === frame && this.stillRequest === null) texture = await attempt();
- const unbaked = this.mode === "editing" || this.mode === "stale" || this.mode === "capturing";
- if (!texture || this.disposed || !unbaked || this.contentHash !== hash) {
- texture?.destroy(true);
- } else if (this.liveStill) {
- this.liveStill.texture.destroy(true);
- this.liveStill.texture = texture;
+ // A still takes main-thread time from the bake, so a playing clip holds a recent baked frame.
+ const held = this.bakingFrames ? heldFrame(this.bakingFrames, frame, this.edit.isPlaying ? HELD_FRAME_LIMIT : 0) : null;
+ const shown = this.liveStillShown;
+ // Playback never steps back to an older frame, or decodes the frame it already shows.
+ if (this.edit.isPlaying && held !== null && this.liveStill && held <= shown && shown <= frame) {
+ this.liveStillExact = shown === frame;
} else {
- this.removeLoadingGraphic();
- this.liveStill = new pixi.Sprite(texture);
- // A capture in progress shows its own copy of the clip.
- this.liveStill.visible = !this.staticSprite;
- this.contentContainer.addChild(this.liveStill);
- if (this.clipConfiguration.width && this.clipConfiguration.height) this.applyFixedDimensions();
+ await this.renderLiveStill(seconds, frame, held, hash);
}
}
} finally {
@@ -1135,6 +1134,36 @@ export class Html5Player extends Player {
}
}
+ private async renderLiveStill(seconds: number, frame: number, held: number | null, hash: string | null): Promise {
+ const attempt = () =>
+ this.renderStill(seconds).catch(err => {
+ if (!this.disposed) console.warn("[Html5Player] still failed:", err);
+ return null;
+ });
+ const baked = held === null ? undefined : this.bakingFrames?.[held];
+ if (baked && held !== frame) this.liveStillExact = false;
+ let texture = baked ? await this.bakingTexture(baked) : await attempt();
+ // A page can load too slowly once, so a failed still gets one more try while nothing newer is waiting.
+ if (!texture && !baked && this.liveStillFrame === frame && this.stillRequest === null) texture = await attempt();
+ const unbaked = this.mode === "editing" || this.mode === "stale" || this.mode === "capturing";
+ if (!texture || this.disposed || !unbaked || this.contentHash !== hash) {
+ texture?.destroy(true);
+ return;
+ }
+ this.liveStillShown = baked && held !== null ? held : frame;
+ if (this.liveStill) {
+ this.liveStill.texture.destroy(true);
+ this.liveStill.texture = texture;
+ return;
+ }
+ this.removeLoadingGraphic();
+ this.liveStill = new pixi.Sprite(texture);
+ // A capture in progress shows its own copy of the clip.
+ this.liveStill.visible = !this.staticSprite;
+ this.contentContainer.addChild(this.liveStill);
+ if (this.clipConfiguration.width && this.clipConfiguration.height) this.applyFixedDimensions();
+ }
+
// The resolution makes a scaled frame's texture report the clip's full size.
private async bakingTexture(blob: Blob): Promise {
return pixi.Texture.from({ resource: await createImageBitmap(blob), resolution: this.bakingScale });
@@ -1143,6 +1172,7 @@ export class Html5Player extends Player {
private removeLiveStill(): void {
this.stillRequest = null;
this.liveStillFrame = -1;
+ this.liveStillShown = -1;
if (!this.liveStill) return;
this.contentContainer.removeChild(this.liveStill);
this.liveStill.texture.destroy(true);
diff --git a/tests/html5-player.test.ts b/tests/html5-player.test.ts
index 98e815ef..4abb328c 100644
--- a/tests/html5-player.test.ts
+++ b/tests/html5-player.test.ts
@@ -1,7 +1,7 @@
/** @jest-environment jsdom */
import { BakeQueue } from "@canvas/players/bake-queue";
-import { Html5Player, bakeScaleFor, captureFramesInFlight } from "@canvas/players/html5-player";
+import { Html5Player, bakeScaleFor, captureFramesInFlight, heldFrame } from "@canvas/players/html5-player";
import type { Edit } from "@core/edit-session";
import { EditEvent, InternalEvent } from "@core/events/edit-events";
import { sec } from "@core/timing/types";
@@ -1600,6 +1600,83 @@ it("shows a first bake's frames as they land", async () => {
await capture;
});
+it.each([
+ [[0, 3], 2, 7, 0],
+ [[0, 3], 3, 7, 3],
+ [[0], 8, 7, null],
+ [[1], 8, 7, 1],
+ [[0, 3], 2, 0, null],
+ [[19], 20, 7, 19]
+])("finds the frame to hold among %p at %p within %p frames: %p", (present, idx, maxBehind, expected) => {
+ const frames = Array.from({ length: 20 }, (_, i) => (present.includes(i) ? new Blob([`${i}`]) : undefined));
+
+ expect(heldFrame(frames, idx, maxBehind)).toBe(expected);
+});
+
+const playingAhead = async (...baked: number[]) => {
+ const created = createStillPlayer();
+ const { player, harness, edit } = created;
+ Object.assign(edit, { isPlaying: true });
+ Object.assign(player, { getPlaybackTime: () => 0.4 }); // frame 12
+ baking(harness);
+ harness.beginCapture();
+ Object.assign(harness, { bakingFrames: bakedAt(...baked), bakingScale: 1 });
+ player.update(0, 0);
+ await until(stillShown(harness));
+ return created;
+};
+
+it("holds the nearest baked frame while playing ahead of the bake", async () => {
+ const { harness, seeks } = await playingAhead(8, 16);
+
+ expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toBe("baked 8");
+ expect(seeks.some(seek => seek.startsWith("still"))).toBe(false);
+});
+
+it("holds a baked frame exactly 7 frames back while playing", async () => {
+ const { harness } = await playingAhead(5);
+
+ expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toBe("baked 5");
+});
+
+it("renders a still while playing when the nearest baked frame is more than 7 frames back", async () => {
+ const { harness } = await playingAhead(4);
+
+ expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<");
+});
+
+it("decodes a held frame once while playing on through it", async () => {
+ const { player } = await playingAhead(8);
+ const decode = jest.spyOn(globalThis, "createImageBitmap");
+ [0.4, 0.4, 13 / 30, 13 / 30, 14 / 30].forEach(t => {
+ Object.assign(player, { getPlaybackTime: () => t });
+ player.update(0, 0);
+ });
+ await flush(50);
+
+ expect(decode).not.toHaveBeenCalled();
+});
+
+it("never steps back to an older baked frame than the still on screen while playing", async () => {
+ const { player, harness } = await playingAhead(0); // a still of frame 12
+ harness.bakingFrames![11] = new Blob(["baked 11"]);
+ Object.assign(player, { getPlaybackTime: () => 13 / 30 });
+ player.update(0, 0);
+ await flush(50);
+
+ expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<");
+});
+
+it("renders the exact still once playback pauses on a held frame", async () => {
+ const { player, harness, edit } = await playingAhead(8, 16);
+ const held = layer(harness)[0].texture;
+ Object.assign(edit, { isPlaying: false });
+ player.update(0, 0);
+ await until(() => layer(harness)[0].texture !== held);
+
+ expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<");
+});
+
it("swaps the still for baked frames once the bake finishes", async () => {
const { player, harness } = createStillPlayer();
let finish: (frames: Blob[]) => void = () => {};