feat(mcp): captions, caption translation and transcription over MCP

electron/ai-edition/agent-tools.test.ts

@@ -216,6 +216,10 @@ describe("the mutating-tool table", () => {
 				"setTrim",
 				"setWordText",
 				"setZoom",
+				"setCaptionSettings",
+				"putCaptionTranslation",
+				"removeCaptionTranslation",
+				"setVocabulary",
 			].sort(),
 		);
 		expect(isMutatingTool("getCurrentDocument")).toBe(false);

electron/ai-edition/agent-tools.ts

@@ -87,6 +87,7 @@ import { SETTING_BOUNDS } from "../../src/lib/projectDefaults";
 import type { SegmentationSupport } from "../../src/native/contracts";
 import { annotationFootageRect, zoomScaleLimit } from "../../src/native/sceneDescription";
 import { ASPECT_RATIO_PRESETS } from "../../src/utils/aspectRatioUtils";
+import { CAPTION_TOOL_NAMES, executeCaptionTool, MUTATING_CAPTION_TOOLS } from "./caption-tools";
 import { editorSettingsSnapshot, executeSetEditorSettings } from "./editor-settings-tool";
 import { measureTextInShippedFonts } from "./text-measure";
 
@@ -1068,6 +1069,7 @@ export const OPENSCREEN_TOOL_NAMES = [
 	"removeClip",
 	"setAspectRatio",
 	"setEditorSettings",
+	...CAPTION_TOOL_NAMES,
 ] as const;
 
 /**
@@ -1145,6 +1147,7 @@ export const MUTATING_TOOL_NAMES: ReadonlySet<string> = new Set([
 	"removeClip",
 	"setAspectRatio",
 	"setEditorSettings",
+	...MUTATING_CAPTION_TOOLS,
 ]);
 
 export function isMutatingTool(name: string): boolean {
@@ -1323,6 +1326,17 @@ export function documentSnapshotForModel(
 			// probe threw. Defaulting that to `false` would put our failure back in
 			// the answer as their fact, one layer lower down.
 			hasCursorTelemetry: availability?.[a.id] ?? null,
+			// Whether getTranscriptWords can read it; transcribe makes one when it is false.
+			hasTranscript: document.transcripts.some((t) => t.assetId === a.id),
+			// A verdict a past run left on the media ("no-audio", "unsupported-audio").
+			...(a.transcriptionFailure
+				? {
+						transcriptionFailure: {
+							kind: a.transcriptionFailure.kind,
+							message: a.transcriptionFailure.message,
+						},
+					}
+				: {}),
 		})),
 		// ponytail: `index`, `reason` and `origin` are here because without them a
 		// clip cannot be DESIGNATED. "Put the demo first" is unanswerable when the
@@ -1413,6 +1427,8 @@ export function documentSnapshotForModel(
 			fadeOutMs: t.fadeOutMs,
 		})),
 		hasTranscript: document.transcripts.length > 0 || document.transcript !== null,
+		// The names and terms Whisper is primed with on every transcription (setVocabulary).
+		vocabulary: document.project.vocabulary ?? null,
 	};
 }
 
@@ -1783,6 +1799,14 @@ export function executeAgentTool(
 	}
 
 	switch (name) {
+		case "getCaptions":
+		case "setCaptionSettings":
+		case "getCaptionTranslationUnits":
+		case "putCaptionTranslation":
+		case "removeCaptionTranslation":
+		case "setVocabulary":
+			return executeCaptionTool(document, name, args);
+
 		case "setAspectRatio": {
 			// The format alone, kept for clients that know it: setEditorSettings does the work.
 			const parsed = setAspectRatioArgs.safeParse(args);

electron/ai-edition/caption-tools.test.ts

@@ -0,0 +1,236 @@
+// The caption tools end to end on a document: turn them on, style and place them,
+// translate them as a client would, and check the export draws what was asked.
+
+import { describe, expect, it } from "vitest";
+import { CAPTION_STYLES, getCaptionSettings } from "../../src/lib/ai-edition/captions";
+import {
+	type AxcutDocument,
+	createEmptyDocument,
+	documentSchema,
+} from "../../src/lib/ai-edition/schema";
+import { buildSceneDescription } from "../../src/native/sceneDescription";
+import { type AgentToolExecution, executeAgentTool } from "./agent-tools";
+
+function fixtureDocument(): AxcutDocument {
+	const base = createEmptyDocument({
+		title: "Captions",
+		projectId: "proj_1",
+		createdAt: "2026-01-01T00:00:00.000Z",
+	});
+	const words = [
+		["w1", "Hello", 0, 0.4],
+		["w2", "there.", 0.4, 0.8],
+		["w3", "Second", 2, 2.4],
+		["w4", "phrase.", 2.4, 2.8],
+	] as const;
+	return documentSchema.parse({
+		...base,
+		project: { ...base.project, primaryAssetId: "asset_1" },
+		assets: [
+			{ id: "asset_1", kind: "video", label: "Rec", originalPath: "/tmp/rec.mp4", durationSec: 10 },
+		],
+		// One segment per word, as Whisper's transcripts are stored.
+		transcripts: [
+			{
+				assetId: "asset_1",
+				language: "en",
+				segments: words.map(([id, text, startSec, endSec]) => ({
+					id: `seg_${id}`,
+					kind: "speech",
+					startSec,
+					endSec,
+					text,
+					wordIds: [id],
+				})),
+				words: words.map(([id, text, startSec, endSec]) => ({
+					id,
+					segmentId: `seg_${id}`,
+					startSec,
+					endSec,
+					text,
+				})),
+			},
+		],
+		timeline: {
+			...base.timeline,
+			clips: [
+				{
+					id: "clip_1",
+					assetId: "asset_1",
+					sourceStartSec: 0,
+					sourceEndSec: 10,
+					timelineStartSec: 0,
+					timelineEndSec: 10,
+					wordRefs: [],
+					origin: "user",
+					reason: "",
+				},
+			],
+		},
+	});
+}
+
+function call(document: AxcutDocument, name: string, args: unknown = {}) {
+	const execution: AgentToolExecution = executeAgentTool(document, name, JSON.stringify(args));
+	return { ...execution, result: JSON.parse(execution.resultJson) as Record<string, unknown> };
+}
+
+function captionTexts(document: AxcutDocument): string[] {
+	return buildSceneDescription(document).annotations.map((a) => a.text?.content ?? "");
+}
+
+describe("setCaptionSettings", () => {
+	it("turns the captions on, and the export draws the transcript", () => {
+		const doc = fixtureDocument();
+		expect(captionTexts(doc)).toEqual([]);
+		const on = call(doc, "setCaptionSettings", { enabled: true });
+		expect(on.ok).toBe(true);
+		expect(on.document && captionTexts(on.document).join(" ")).toContain("Hello there.");
+	});
+
+	it("applies a style, then a plate, then the explicit fields over both", () => {
+		const doc = fixtureDocument();
+		const set = call(doc, "setCaptionSettings", {
+			style: "bold",
+			plate: "light",
+			fontSize: 72,
+			anchorV: "top",
+			insetY: 8,
+			minWordsPerLine: 3,
+			maxWordsPerLine: 5,
+		});
+		const settings = getCaptionSettings(set.document);
+		expect(settings.fontFamily).toBe(CAPTION_STYLES.bold.fontFamily);
+		expect(settings.fontSize).toBe(72);
+		expect(settings.backgroundColor).toBe("#ffffff");
+		// The bold style's yellow would vanish on a light plate: the plate recoloured it.
+		expect(settings.color).not.toBe(CAPTION_STYLES.bold.color);
+		expect(settings).toMatchObject({ anchorV: "top", insetY: 8, minWordsPerLine: 3 });
+		expect(set.result.plate).toBe("light");
+	});
+
+	it("refuses what the pane cannot express, and changes nothing", () => {
+		const doc = fixtureDocument();
+		expect(call(doc, "setCaptionSettings", {}).ok).toBe(false);
+		expect(call(doc, "setCaptionSettings", { minWordsPerLine: 9, maxWordsPerLine: 3 }).ok).toBe(
+			false,
+		);
+		expect(call(doc, "setCaptionSettings", { fontSize: 500 }).ok).toBe(false);
+		expect(call(doc, "setCaptionSettings", { fontFamily: "Comic Sans" }).ok).toBe(false);
+		const missing = call(doc, "setCaptionSettings", { language: "fr" });
+		expect(missing.ok).toBe(false);
+		expect(String(missing.result.error)).toContain("putCaptionTranslation");
+	});
+
+	it("reports the lane the captions really read", () => {
+		const set = call(fixtureDocument(), "setCaptionSettings", { lane: "voiceover" });
+		expect(set.result.resolvedLane).toBe("recording");
+		expect(set.result.laneNote).toBeDefined();
+	});
+});
+
+describe("getCaptions", () => {
+	it("reads the settings back, warns when there is nothing to caption", () => {
+		const doc = call(fixtureDocument(), "setCaptionSettings", { enabled: true }).document;
+		if (!doc) throw new Error("not applied");
+		const read = call(doc, "getCaptions").result;
+		expect(read).toMatchObject({ style: "classic", plate: "dark", resolvedLane: "recording" });
+		expect(read.warning).toBeUndefined();
+		const silent = call({ ...doc, transcripts: [], transcript: null }, "getCaptions").result;
+		expect(silent.warning).toBeDefined();
+	});
+});
+
+describe("translating the captions", () => {
+	it("reads phrase units, stores the client's translation, and the export shows it", () => {
+		let doc = fixtureDocument();
+		const units = call(doc, "getCaptionTranslationUnits", { language: "fr" }).result
+			.assets as Array<{
+			units: Array<{ id: string; text: string }>;
+		}>;
+		// Phrases, not words: one unit per sentence.
+		expect(units[0]?.units.map((u) => u.text)).toEqual(["Hello there.", "Second phrase."]);
+		const [first, second] = units[0]?.units ?? [];
+
+		const put = call(doc, "putCaptionTranslation", {
+			language: "fr",
+			label: "Français",
+			assetId: "asset_1",
+			units: [{ id: first.id, text: "Bonjour." }],
+		});
+		expect(put.result).toMatchObject({
+			written: 1,
+			coverage: { translated: 1, total: 2 },
+			shown: false,
+		});
+		doc = put.document ?? doc;
+
+		// Only what is left to translate comes back.
+		const left = call(doc, "getCaptionTranslationUnits", { language: "fr" }).result
+			.assets as Array<{
+			units: Array<{ id: string }>;
+		}>;
+		expect(left[0]?.units.map((u) => u.id)).toEqual([second.id]);
+
+		doc = call(doc, "setCaptionSettings", { enabled: true, language: "fr" }).document ?? doc;
+		expect(captionTexts(doc).join(" ")).toContain("Bonjour.");
+		// The transcript itself is untouched.
+		expect(doc.transcripts[0]?.words[0]?.text).toBe("Hello");
+
+		const read = call(doc, "getCaptions").result.translations as Array<Record<string, unknown>>;
+		expect(read).toEqual([
+			expect.objectContaining({
+				language: "fr",
+				label: "Français",
+				coverage: [{ assetId: "asset_1", translated: 1, total: 2 }],
+			}),
+		]);
+	});
+
+	it("refuses unit ids it does not know, and nothing is written", () => {
+		const put = call(fixtureDocument(), "putCaptionTranslation", {
+			language: "fr",
+			assetId: "asset_1",
+			units: [{ id: "seg_w1", text: "Bonjour" }],
+		});
+		expect(put.ok).toBe(false);
+		expect(put.document).toBeUndefined();
+		const twice = call(fixtureDocument(), "putCaptionTranslation", {
+			language: "fr",
+			assetId: "asset_1",
+			units: [
+				{ id: "u:seg_w1", text: "Bonjour." },
+				{ id: "u:seg_w1", text: "Salut." },
+			],
+		});
+		expect(twice.ok).toBe(false);
+	});
+
+	it("removing the language on screen puts the original back", () => {
+		let doc = fixtureDocument();
+		doc =
+			call(doc, "putCaptionTranslation", {
+				language: "fr",
+				assetId: "asset_1",
+				units: [{ id: "u:seg_w1", text: "Bonjour." }],
+			}).document ?? doc;
+		doc = call(doc, "setCaptionSettings", { language: "fr" }).document ?? doc;
+		const removed = call(doc, "removeCaptionTranslation", { language: "fr" });
+		expect(removed.result.shownLanguage).toBeNull();
+		expect(call(removed.document ?? doc, "removeCaptionTranslation", { language: "fr" }).ok).toBe(
+			false,
+		);
+	});
+});
+
+describe("setVocabulary", () => {
+	it("stores the names Whisper is primed with, and getCurrentDocument reads them", () => {
+		const set = call(fixtureDocument(), "setVocabulary", { vocabulary: "  OpenScreen, Vulkan " });
+		expect(set.document?.project.vocabulary).toBe("OpenScreen, Vulkan");
+		const read = set.document && call(set.document, "getCurrentDocument").result;
+		expect(read?.vocabulary).toBe("OpenScreen, Vulkan");
+		expect(call(fixtureDocument(), "setVocabulary", { vocabulary: "x".repeat(601) }).ok).toBe(
+			false,
+		);
+	});
+});