fix: keep a frame's embedded images when Safari draws an html5 clip

Author: dazzatronus

src/components/canvas/players/html5-player.ts

@@ -40,10 +40,33 @@ const CAPTURE_PROGRESS_STEPS = 20;
 const BAKE_SCALES = [0.25, 0.5, 0.75, 1];
 // A remembered frame holds its body markup, so a window bounds the memory. 300 frames covers a 10 s loop at 30 fps.
 const REPEAT_WINDOW_FRAMES = 300;
+// Part of every bake's cache key, so raising it redraws every cached bake.
+const FRAME_DRAWING_VERSION = 1;
 const ANIMATION_OVERRIDE = `<style>*,*::before,*::after{animation:none!important;transition:none!important}</style>`;
 
+// A base64 bitmap in a frame's markup: a CSS url(), an <img> source, or a canvas painted as a background.
+const NESTED_BITMAP = /data:image\/(?!svg)[\w.+-]+;base64,[A-Za-z0-9+/=]+/g;
+
+// WebKit lays an SVG image out as offscreen, so a large bitmap inside one decodes asynchronously and a draw before
+// the decode lands leaves it out. Nothing signals that decode, but an <img decoding="sync"> decodes synchronously, so
+// one per bitmap, painted ahead of the page, has every bitmap ready within the same draw. opacity:0 would skip its paint.
+// Other engines decode in time and would pay for the extra decodes on every frame. Every iOS browser runs WebKit and
+// reports Apple as its vendor.
 function foreignObjectSvg(width: number, height: number, styles: string, body: string): string {
-	return `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}"><foreignObject width="${width}" height="${height}">${styles}${ANIMATION_OVERRIDE}${body}</foreignObject></svg>`;
+	const decoders =
+		navigator.vendor === "Apple Computer, Inc."
+			? [...new Set(`${styles}${body}`.match(NESTED_BITMAP))]
+					.map(
+						src =>
+							`<img xmlns="http://www.w3.org/1999/xhtml" decoding="sync" src="${src}" style="position:absolute;left:0;top:0;width:1px;height:1px;filter:opacity(0)!important"/>`
+					)
+					.join("")
+			: "";
+	return (
+		`<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}">` +
+		`${decoders && `<foreignObject width="1" height="1">${decoders}</foreignObject>`}` +
+		`<foreignObject width="${width}" height="${height}">${styles}${ANIMATION_OVERRIDE}${body}</foreignObject></svg>`
+	);
 }
 
 function forceLayout(el: HTMLElement): void {
@@ -207,7 +230,7 @@ export class Html5Player extends Player {
 			width: this.renderedWidth,
 			height: this.renderedHeight
 		});
-		return `${content}:${frameCount}@${captureFps}`;
+		return `${content}:${frameCount}@${captureFps}:v${FRAME_DRAWING_VERSION}`;
 	}
 
 	// Canvas pixels per clip pixel: a bake at this scale or above looks as sharp as a full-size one.

tests/html5-player.test.ts

@@ -382,6 +382,40 @@ it("paints canvas pixels into the capture as the canvas's own background", () =>
 	expect(canvas.getAttribute("style")).toBe("border: 4px solid gold;");
 });
 
+it("decodes every image a capture embeds before the page paints in WebKit, without drawing it", () => {
+	const { harness, iframe } = createPlayer();
+	const doc = iframe.contentDocument!;
+	jest.spyOn(navigator, "vendor", "get").mockReturnValue("Apple Computer, Inc.");
+	const photo = "data:image/jpeg;base64,/9j/4AAQ";
+	const style = doc.createElement("style");
+	style.textContent = `.hero{background:url(${photo}) center/cover} .again{background-image:url('${photo}')} .grain{background:url('data:image/svg+xml;utf8,<svg/>')}`;
+	doc.head.appendChild(style);
+	const canvas = doc.createElement("canvas");
+	doc.body.replaceChildren(Object.assign(doc.createElement("div"), { className: "hero" }), canvas);
+	jest.spyOn(Object.getPrototypeOf(canvas) as HTMLCanvasElement, "toDataURL").mockReturnValue("data:image/png;base64,AAAA");
+
+	const parsed = new DOMParser().parseFromString(harness.captureIframeAsForeignObjectSvg(1080, 1920), "image/svg+xml");
+	const [decoders, page] = Array.from(parsed.documentElement.children);
+	const images = Array.from(decoders.querySelectorAll("img"));
+
+	expect(parsed.querySelector("parsererror")).toBeNull();
+	expect(images.map(img => img.getAttribute("src"))).toEqual([photo, "data:image/png;base64,AAAA"]);
+	expect(images.every(img => img.getAttribute("decoding") === "sync" && img.getAttribute("style")!.includes("filter:opacity(0)"))).toBe(true);
+	expect(page.querySelector(".hero")).not.toBeNull();
+});
+
+it("leaves embedded images to the page outside WebKit", () => {
+	const { harness, iframe } = createPlayer();
+	const doc = iframe.contentDocument!;
+	jest.spyOn(navigator, "vendor", "get").mockReturnValue("Google Inc.");
+	doc.body.replaceChildren(Object.assign(doc.createElement("img"), { src: "data:image/jpeg;base64,/9j/4AAQ" }));
+
+	const parsed = new DOMParser().parseFromString(harness.captureIframeAsForeignObjectSvg(1080, 1920), "image/svg+xml");
+
+	expect(parsed.documentElement.children).toHaveLength(1);
+	expect(parsed.querySelectorAll("img")).toHaveLength(1);
+});
+
 it("keeps a failed capture inside its canvas layer and retries only after the asset changes", async () => {
 	const { player, harness, iframe, events, edit } = createPlayer();
 	jest.spyOn(console, "warn").mockImplementation(() => {});