feat: give html5 and rich-caption clips their own look on the timeline

Author: dazzatronus

src/components/timeline/components/clip/clip-component.ts

@@ -31,6 +31,58 @@ export interface ClipComponentOptions {
 }
 
 /** Renders a single clip element */
+function fileName(src: string): string {
+	return (src.split("/").pop() || src).split("?")[0];
+}
+
+function truncateLabel(text: string): string {
+	return text.substring(0, 20) + (text.length > 20 ? "..." : "");
+}
+
+const BLOCK_ELEMENTS =
+	"address, article, aside, blockquote, dd, div, dl, dt, figcaption, figure, footer, h1, h2, h3, h4, h5, h6, header, li, main, nav, ol, p, pre, section, table, td, th, tr, ul";
+
+/** Each block's text on an html5 page, in reading order, with inline pieces such as animated words joined. */
+function pageLines(html: string): string[] {
+	// Parsing runs none of the page's scripts.
+	const doc = new DOMParser().parseFromString(html, "text/html");
+	doc.querySelectorAll("script, style, template, noscript").forEach(element => element.remove());
+	const lines: string[] = [];
+	let line = "";
+	const endLine = (): void => {
+		const text = line.replace(/\s+/g, " ").trim();
+		if (text) lines.push(text);
+		line = "";
+	};
+	const read = (node: Node): void => {
+		if (node.nodeType === Node.TEXT_NODE) {
+			const text = node.textContent ?? "";
+			// Line and word spans often sit flush against each other. Letters split one per span stay together.
+			if (text.trim().length > 1 && /\S$/.test(line) && /^\S/.test(text)) line += " ";
+			line += text;
+			return;
+		}
+		if (node.nodeType !== Node.ELEMENT_NODE) return;
+		const block = (node as Element).matches(BLOCK_ELEMENTS) || node.nodeName === "BR";
+		if (block) endLine();
+		node.childNodes.forEach(read);
+		if (block) endLine();
+	};
+	read(doc.body);
+	endLine();
+	return lines;
+}
+
+/**
+ * The line that reads as the page's headline. Pages often open with counters, bullets and all-caps labels, and a
+ * headline is the first line of two or more words in sentence case.
+ */
+export function pageHeadline(html: string): string | null {
+	const lines = pageLines(html);
+	const sentence = lines.find(text => /\p{Ll}/u.test(text) && (text.match(/\p{L}{2,}/gu)?.length ?? 0) >= 2);
+	return sentence ?? lines.find(text => /\p{L}/u.test(text)) ?? null;
+}
+
 export class ClipComponent {
 	public readonly element: HTMLElement;
 	private readonly options: ClipComponentOptions;
@@ -44,6 +96,9 @@ export class ClipComponent {
 	// Cached element references (avoid querySelector every frame)
 	private iconEl: HTMLElement | null = null;
 	private labelEl: HTMLElement | null = null;
+	// Parsing a page for its label is too slow to repeat on every draw.
+	private html5LabelSource: string | null = null;
+	private html5Label = "";
 	private badgeEl: HTMLElement | null = null;
 
 	constructor(clip: ClipState, options: ClipComponentOptions) {
@@ -368,6 +423,8 @@ export class ClipComponent {
 			shape: "◇",
 			caption: "≡",
 			html: "<>",
+			html5: "<>",
+			"rich-caption": "≡",
 			luma: "◐",
 			svg: "◇"
 		};
@@ -407,14 +464,27 @@ export class ClipComponent {
 			return "Shape";
 		}
 
-		if ("src" in asset && typeof asset.src === "string") {
+		if (assetType === "html5") {
+			const html = "html" in asset && typeof asset.html === "string" ? asset.html : "";
+			if (html !== this.html5LabelSource) {
+				this.html5LabelSource = html;
+				this.html5Label = truncateLabel(pageHeadline(html) ?? "HTML");
+			}
+			return this.html5Label;
+		}
+
+		// Its words come from parsing or transcribing the source at load, so the label names the source.
+		if (assetType === "rich-caption" && "src" in asset && typeof asset.src === "string") {
 			const { src } = asset;
-			const filename = src.split("/").pop() || src;
-			return filename.split("?")[0];
+			return `Captions · ${src.startsWith("alias://") ? src.slice("alias://".length) : fileName(src)}`;
+		}
+
+		if ("src" in asset && typeof asset.src === "string") {
+			return fileName(asset.src);
 		}
 
 		if ("text" in asset && typeof asset.text === "string") {
-			return asset.text.substring(0, 20) + (asset.text.length > 20 ? "..." : "");
+			return truncateLabel(asset.text);
 		}
 
 		return assetType || "Clip";

src/styles/timeline/timeline.css

@@ -256,13 +256,15 @@
 .ss-track[data-asset-type="shape"] {
 	background: rgba(255, 243, 184, 0.3);
 }
-.ss-track[data-asset-type="caption"] {
+.ss-track[data-asset-type="caption"],
+.ss-track[data-asset-type="rich-caption"] {
 	background: rgba(252, 228, 236, 0.3);
 }
 .ss-track[data-asset-type="svg"] {
 	background: rgba(251, 233, 231, 0.3);
 }
-.ss-track[data-asset-type="html"] {
+.ss-track[data-asset-type="html"],
+.ss-track[data-asset-type="html5"] {
 	background: rgba(179, 229, 252, 0.3);
 }
 .ss-track[data-asset-type="luma"] {
@@ -364,7 +366,8 @@
 	--clip-fg: #f57f17;
 	--clip-border: #f9a825;
 }
-.ss-clip[data-asset-type="caption"] {
+.ss-clip[data-asset-type="caption"],
+.ss-clip[data-asset-type="rich-caption"] {
 	--clip-bg: #fce4ec;
 	--clip-fg: #880e4f;
 	--clip-border: #c2185b;
@@ -374,7 +377,8 @@
 	--clip-fg: #bf360c;
 	--clip-border: #e64a19;
 }
-.ss-clip[data-asset-type="html"] {
+.ss-clip[data-asset-type="html"],
+.ss-clip[data-asset-type="html5"] {
 	--clip-bg: #b3e5fc;
 	--clip-fg: #006064;
 	--clip-border: #0097a7;

tests/timeline-clip-labels.test.ts

@@ -0,0 +1,98 @@
+/** @jest-environment jsdom */
+import { Timeline } from "@timeline/timeline";
+import { AssetLoadTracker } from "@core/events/asset-load-tracker";
+import { EventEmitter } from "@core/events/event-emitter";
+import { sec } from "@core/timing/types";
+
+async function mountClips(clips: Array<{ asset: Record<string, unknown> }>) {
+	const events = new EventEmitter();
+	const config = { timeline: { tracks: clips.map((clip, i) => ({ clips: [{ id: `clip-${i}`, start: 0, length: 5, ...clip }] })) } };
+	const edit = {
+		events,
+		assetLoader: { loadTracker: new AssetLoadTracker() },
+		getInternalEvents: () => events,
+		getResolvedEdit: () => config,
+		getEdit: () => config,
+		isClipSelected: () => false,
+		getClipGenerationState: () => undefined,
+		getClipError: () => null,
+		getPendingCaptures: () => ({ pending: 0, total: 0, progress: 1 }),
+		getClipCaptureProgress: () => null,
+		playbackTime: sec(0),
+		totalDuration: sec(5),
+		isPlaying: false
+	};
+	const container = document.createElement("div");
+	document.body.appendChild(container);
+	const timeline = new Timeline(edit as never, container);
+	["html5", "rich-caption"].forEach(type => timeline.registerClipRenderer(type, { render: () => {} }));
+	await timeline.load();
+	return [...container.querySelectorAll<HTMLElement>(".ss-clip")].map(clip => ({
+		icon: clip.querySelector(".ss-clip-icon")?.textContent,
+		label: clip.querySelector(".ss-clip-label")?.textContent
+	}));
+}
+
+afterEach(() => {
+	document.body.replaceChildren();
+});
+
+it("labels an html5 clip with its headline, under the html icon", async () => {
+	const [clip] = await mountClips([
+		{
+			asset: {
+				type: "html5",
+				html: "<style>h1 { color: red }</style><div><h1>Lights out &amp; away</h1><p>We go racing</p></div><script>run()</script>"
+			}
+		}
+	]);
+
+	expect(clip).toEqual({ icon: "<>", label: "Lights out & away" });
+});
+
+it("labels an html5 clip with its first sentence, past counters, bullets and all-caps labels", async () => {
+	const clips = await mountClips([
+		{
+			asset: {
+				type: "html5",
+				html: '<b>01</b><div class="k">/ LEARN</div><h1><span class="mask"><span class="wd">Learns</span></span> <span class="wd">your brand.</span></h1>'
+			}
+		},
+		{ asset: { type: "html5", html: "<b>●</b><div>SENNA · ONLINE</div><h2>Meet Senna.</h2><p>The AI agent that runs your ads.</p>" } }
+	]);
+
+	expect(clips.map(clip => clip.label)).toEqual(["Learns your brand.", "Meet Senna."]);
+});
+
+it("keeps words apart across spans with no space between them, and letters split one per span together", async () => {
+	const clips = await mountClips([
+		{ asset: { type: "html5", html: '<h1><span class="ln">Your best ad</span><span class="ln">burns out</span></h1>' } },
+		{ asset: { type: "html5", html: "<h1><span>G</span><span>o</span> <span>f</span><span>a</span><span>s</span><span>t</span></h1>" } }
+	]);
+
+	expect(clips.map(clip => clip.label)).toEqual(["Your best ad burns o...", "Go fast"]);
+});
+
+it("labels an html5 clip whose lines are all labels with the first one that has letters", async () => {
+	const [clip] = await mountClips([{ asset: { type: "html5", html: "<div>00:00.000</div><div><span>LAP</span> <b>01</b></div>" } }]);
+
+	expect(clip.label).toBe("LAP 01");
+});
+
+it("labels an html5 clip with no visible text as HTML", async () => {
+	const [clip] = await mountClips([{ asset: { type: "html5", html: "<div class='grain'></div>", css: ".grain { opacity: .2 }" } }]);
+
+	expect(clip.label).toBe("HTML");
+});
+
+it("labels a rich caption with what it captions, under the caption icon", async () => {
+	const clips = await mountClips([
+		{ asset: { type: "rich-caption", src: "alias://voiceover" } },
+		{ asset: { type: "rich-caption", src: "https://cdn.example.com/subs/race.srt?token=abc" } }
+	]);
+
+	expect(clips).toEqual([
+		{ icon: "≡", label: "Captions · voiceover" },
+		{ icon: "≡", label: "Captions · race.srt" }
+	]);
+});