Skip to content

Commit 4f50166

Browse files
authored
[2/3] feat: hold the nearest baked html5 frame while playing (#272)
* feat: hold the nearest baked html5 frame while playing * fix: keep the frame on screen when a held html5 frame would be older
1 parent 7e837e6 commit 4f50166

2 files changed

Lines changed: 130 additions & 23 deletions

File tree

‎src/components/canvas/players/html5-player.ts‎

Lines changed: 52 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ import { createCaptureLoadingGraphic, createPlaceholderGraphic } from "./placeho
2525
const IFRAME_OFFSCREEN_X = -10000;
2626
const IFRAME_LOAD_TIMEOUT_MS = 10_000;
2727
const DECODED_FRAME_LIMIT = 30;
28+
// How stale a held frame may be when it's chosen: 233 ms at 30 fps.
29+
const HELD_FRAME_LIMIT = 7;
2830

2931
type HarnessWindow = Window & {
3032
["__shotstackSeek"]?: (ms: number) => void;
@@ -99,6 +101,12 @@ export function captureFramesInFlight(logicalCores: number | undefined, frameByt
99101
return Math.max(1, Math.min(byCores, Math.floor(CAPTURE_MEMORY_BUDGET_BYTES / frameBytes)));
100102
}
101103

104+
/** The latest baked frame at most `maxBehind` frames before `idx`, or null. */
105+
export function heldFrame(frames: ArrayLike<Blob | undefined>, idx: number, maxBehind: number): number | null {
106+
for (let i = idx; i >= Math.max(0, idx - maxBehind); i -= 1) if (frames[i]) return i;
107+
return null;
108+
}
109+
102110
export function bakeScaleFor(onScreenScale: number): number {
103111
return BAKE_SCALES.find(scale => scale >= onScreenScale) ?? 1;
104112
}
@@ -160,6 +168,10 @@ export class Html5Player extends Player {
160168
private stillTexture: pixi.Texture | null = null;
161169
private liveStill: pixi.Sprite | null = null;
162170
private liveStillFrame: number = -1;
171+
// False while the live still is a held frame from before the playhead, so pausing on it renders the exact one.
172+
private liveStillExact: boolean = true;
173+
// The frame the live still shows, which a held frame trails.
174+
private liveStillShown: number = -1;
163175
private liveStillInFlight: boolean = false;
164176
private stillRequest: number | null = null;
165177
// The frame the running bake draws next. Null between bakes.
@@ -1092,8 +1104,9 @@ export class Html5Player extends Player {
10921104
// One still renders at a time. Positions asked for meanwhile replace each other, so only the newest renders next.
10931105
private showStillAt(seconds: number): void {
10941106
const frame = Math.floor(seconds * this.captureFps);
1095-
if (frame === this.liveStillFrame) return;
1107+
if (frame === this.liveStillFrame && (this.liveStillExact || this.edit.isPlaying)) return;
10961108
this.liveStillFrame = frame;
1109+
this.liveStillExact = true;
10971110
this.stillRequest = seconds;
10981111
if (!this.liveStillInFlight) this.renderLiveStills().catch(err => console.warn("[Html5Player] still failed:", err));
10991112
}
@@ -1106,35 +1119,51 @@ export class Html5Player extends Player {
11061119
const frame = Math.floor(seconds * this.captureFps);
11071120
const hash = this.contentHash;
11081121
this.stillRequest = null;
1109-
const attempt = () =>
1110-
this.renderStill(seconds).catch(err => {
1111-
if (!this.disposed) console.warn("[Html5Player] still failed:", err);
1112-
return null;
1113-
});
1114-
const baked = this.bakingFrames?.[frame];
1115-
let texture = baked ? await this.bakingTexture(baked) : await attempt();
1116-
// A page can load too slowly once, so a failed still gets one more try while nothing newer is waiting.
1117-
if (!texture && !baked && this.liveStillFrame === frame && this.stillRequest === null) texture = await attempt();
1118-
const unbaked = this.mode === "editing" || this.mode === "stale" || this.mode === "capturing";
1119-
if (!texture || this.disposed || !unbaked || this.contentHash !== hash) {
1120-
texture?.destroy(true);
1121-
} else if (this.liveStill) {
1122-
this.liveStill.texture.destroy(true);
1123-
this.liveStill.texture = texture;
1122+
// A still takes main-thread time from the bake, so a playing clip holds a recent baked frame.
1123+
const held = this.bakingFrames ? heldFrame(this.bakingFrames, frame, this.edit.isPlaying ? HELD_FRAME_LIMIT : 0) : null;
1124+
const shown = this.liveStillShown;
1125+
// Playback never steps back to an older frame, or decodes the frame it already shows.
1126+
if (this.edit.isPlaying && held !== null && this.liveStill && held <= shown && shown <= frame) {
1127+
this.liveStillExact = shown === frame;
11241128
} else {
1125-
this.removeLoadingGraphic();
1126-
this.liveStill = new pixi.Sprite(texture);
1127-
// A capture in progress shows its own copy of the clip.
1128-
this.liveStill.visible = !this.staticSprite;
1129-
this.contentContainer.addChild(this.liveStill);
1130-
if (this.clipConfiguration.width && this.clipConfiguration.height) this.applyFixedDimensions();
1129+
await this.renderLiveStill(seconds, frame, held, hash);
11311130
}
11321131
}
11331132
} finally {
11341133
this.liveStillInFlight = false;
11351134
}
11361135
}
11371136

1137+
private async renderLiveStill(seconds: number, frame: number, held: number | null, hash: string | null): Promise<void> {
1138+
const attempt = () =>
1139+
this.renderStill(seconds).catch(err => {
1140+
if (!this.disposed) console.warn("[Html5Player] still failed:", err);
1141+
return null;
1142+
});
1143+
const baked = held === null ? undefined : this.bakingFrames?.[held];
1144+
if (baked && held !== frame) this.liveStillExact = false;
1145+
let texture = baked ? await this.bakingTexture(baked) : await attempt();
1146+
// A page can load too slowly once, so a failed still gets one more try while nothing newer is waiting.
1147+
if (!texture && !baked && this.liveStillFrame === frame && this.stillRequest === null) texture = await attempt();
1148+
const unbaked = this.mode === "editing" || this.mode === "stale" || this.mode === "capturing";
1149+
if (!texture || this.disposed || !unbaked || this.contentHash !== hash) {
1150+
texture?.destroy(true);
1151+
return;
1152+
}
1153+
this.liveStillShown = baked && held !== null ? held : frame;
1154+
if (this.liveStill) {
1155+
this.liveStill.texture.destroy(true);
1156+
this.liveStill.texture = texture;
1157+
return;
1158+
}
1159+
this.removeLoadingGraphic();
1160+
this.liveStill = new pixi.Sprite(texture);
1161+
// A capture in progress shows its own copy of the clip.
1162+
this.liveStill.visible = !this.staticSprite;
1163+
this.contentContainer.addChild(this.liveStill);
1164+
if (this.clipConfiguration.width && this.clipConfiguration.height) this.applyFixedDimensions();
1165+
}
1166+
11381167
// The resolution makes a scaled frame's texture report the clip's full size.
11391168
private async bakingTexture(blob: Blob): Promise<pixi.Texture> {
11401169
return pixi.Texture.from({ resource: await createImageBitmap(blob), resolution: this.bakingScale });
@@ -1143,6 +1172,7 @@ export class Html5Player extends Player {
11431172
private removeLiveStill(): void {
11441173
this.stillRequest = null;
11451174
this.liveStillFrame = -1;
1175+
this.liveStillShown = -1;
11461176
if (!this.liveStill) return;
11471177
this.contentContainer.removeChild(this.liveStill);
11481178
this.liveStill.texture.destroy(true);

‎tests/html5-player.test.ts‎

Lines changed: 78 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/** @jest-environment jsdom */
22

33
import { BakeQueue } from "@canvas/players/bake-queue";
4-
import { Html5Player, bakeScaleFor, captureFramesInFlight } from "@canvas/players/html5-player";
4+
import { Html5Player, bakeScaleFor, captureFramesInFlight, heldFrame } from "@canvas/players/html5-player";
55
import type { Edit } from "@core/edit-session";
66
import { EditEvent, InternalEvent } from "@core/events/edit-events";
77
import { sec } from "@core/timing/types";
@@ -1600,6 +1600,83 @@ it("shows a first bake's frames as they land", async () => {
16001600
await capture;
16011601
});
16021602

1603+
it.each([
1604+
[[0, 3], 2, 7, 0],
1605+
[[0, 3], 3, 7, 3],
1606+
[[0], 8, 7, null],
1607+
[[1], 8, 7, 1],
1608+
[[0, 3], 2, 0, null],
1609+
[[19], 20, 7, 19]
1610+
])("finds the frame to hold among %p at %p within %p frames: %p", (present, idx, maxBehind, expected) => {
1611+
const frames = Array.from({ length: 20 }, (_, i) => (present.includes(i) ? new Blob([`${i}`]) : undefined));
1612+
1613+
expect(heldFrame(frames, idx, maxBehind)).toBe(expected);
1614+
});
1615+
1616+
const playingAhead = async (...baked: number[]) => {
1617+
const created = createStillPlayer();
1618+
const { player, harness, edit } = created;
1619+
Object.assign(edit, { isPlaying: true });
1620+
Object.assign(player, { getPlaybackTime: () => 0.4 }); // frame 12
1621+
baking(harness);
1622+
harness.beginCapture();
1623+
Object.assign(harness, { bakingFrames: bakedAt(...baked), bakingScale: 1 });
1624+
player.update(0, 0);
1625+
await until(stillShown(harness));
1626+
return created;
1627+
};
1628+
1629+
it("holds the nearest baked frame while playing ahead of the bake", async () => {
1630+
const { harness, seeks } = await playingAhead(8, 16);
1631+
1632+
expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toBe("baked 8");
1633+
expect(seeks.some(seek => seek.startsWith("still"))).toBe(false);
1634+
});
1635+
1636+
it("holds a baked frame exactly 7 frames back while playing", async () => {
1637+
const { harness } = await playingAhead(5);
1638+
1639+
expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toBe("baked 5");
1640+
});
1641+
1642+
it("renders a still while playing when the nearest baked frame is more than 7 frames back", async () => {
1643+
const { harness } = await playingAhead(4);
1644+
1645+
expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<");
1646+
});
1647+
1648+
it("decodes a held frame once while playing on through it", async () => {
1649+
const { player } = await playingAhead(8);
1650+
const decode = jest.spyOn(globalThis, "createImageBitmap");
1651+
[0.4, 0.4, 13 / 30, 13 / 30, 14 / 30].forEach(t => {
1652+
Object.assign(player, { getPlaybackTime: () => t });
1653+
player.update(0, 0);
1654+
});
1655+
await flush(50);
1656+
1657+
expect(decode).not.toHaveBeenCalled();
1658+
});
1659+
1660+
it("never steps back to an older baked frame than the still on screen while playing", async () => {
1661+
const { player, harness } = await playingAhead(0); // a still of frame 12
1662+
harness.bakingFrames![11] = new Blob(["baked 11"]);
1663+
Object.assign(player, { getPlaybackTime: () => 13 / 30 });
1664+
player.update(0, 0);
1665+
await flush(50);
1666+
1667+
expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<");
1668+
});
1669+
1670+
it("renders the exact still once playback pauses on a held frame", async () => {
1671+
const { player, harness, edit } = await playingAhead(8, 16);
1672+
const held = layer(harness)[0].texture;
1673+
Object.assign(edit, { isPlaying: false });
1674+
player.update(0, 0);
1675+
await until(() => layer(harness)[0].texture !== held);
1676+
1677+
expect(await frameMarkup(layer(harness)[0].texture.source.blob)).toContain(">t=0.4<");
1678+
});
1679+
16031680
it("swaps the still for baked frames once the bake finishes", async () => {
16041681
const { player, harness } = createStillPlayer();
16051682
let finish: (frames: Blob[]) => void = () => {};

0 commit comments

Comments
 (0)