From 7ac2d54fb18db7c10055c920c71be1e63db9e41e Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:18:40 +0200 Subject: [PATCH 01/16] feat(mcp): transcribe an asset over MCP with the on-device Whisper The project open in the editor goes through the editor's own transcription queue (a job already running is waited for); any other project runs the main process's SttManager outside the runner queue and lands the transcript as an edit. A restore keeps a transcript made after its checkpoint. Part of #1180 --- electron/cli/cliMain.ts | 10 +- electron/ipc/handlers.ts | 3 + electron/mcp/app-channel.ts | 9 +- electron/mcp/editor-document-host.test.ts | 22 ++ electron/mcp/editor-document-host.ts | 39 ++- electron/mcp/openscreen-mcp-server.ts | 270 +++++++++++++++++- electron/mcp/transcribe-tool.test.ts | 216 ++++++++++++++ .../ai-edition/v4/MediaStage.test.ts | 8 +- src/components/ai-edition/v4/MediaStage.tsx | 21 +- src/lib/ai-edition/document/transcribe.ts | 56 +--- src/lib/ai-edition/document/transcript.ts | 43 +++ src/lib/ai-edition/store/mcpDocumentHost.ts | 36 ++- .../store/transcriptionStore.test.ts | 65 +++++ .../ai-edition/store/transcriptionStore.ts | 67 ++--- .../ai-edition/transcription/fromStt.test.ts | 50 ++++ src/lib/ai-edition/transcription/fromStt.ts | 80 ++++++ src/lib/ai-edition/transcription/status.ts | 21 ++ src/lib/captioning/transcribe.ts | 26 +- src/native/contracts.ts | 21 +- 19 files changed, 908 insertions(+), 155 deletions(-) create mode 100644 electron/mcp/transcribe-tool.test.ts create mode 100644 src/lib/ai-edition/transcription/fromStt.test.ts create mode 100644 src/lib/ai-edition/transcription/fromStt.ts diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.ts index 786f651b57..9e1d68825d 100644 --- a/electron/cli/cliMain.ts +++ b/electron/cli/cliMain.ts @@ -11,7 +11,7 @@ import { isDiagnosticModeEnabled } from "../diagnostics/main-log-buffer"; import { getSelectedDesktopSource, registerIpcHandlers } from "../ipc/handlers"; import { openVideoInHiddenWindow } from "../mcp/hidden-video"; import { serveMcpOverStdio } from "../mcp/stdio-server"; -import { registerSttIpc } from "../stt"; +import { registerSttIpc, shutdownStt } from "../stt"; import { createRunnerWindow } from "../windows"; import { CLI_USAGE, type CliCommand } from "./args"; import { runInfoCommand, runPackCommand } from "./projectCommands"; @@ -392,7 +392,8 @@ export function runCli(command: CliCommand): void { userDataDir, runners, openVideo: openVideoInHiddenWindow, - onClose: () => app.exit(0), + // The Whisper helper is a process of its own: it must not outlive this one. + onClose: () => void shutdownStt().finally(() => app.exit(0)), }); milestone("mcp server listening on stdio"); return; @@ -496,7 +497,7 @@ export function runCli(command: CliCommand): void { printSources(output, result.sources); } else if (command.kind === "captions") { output.info( - `Added ${result.captionCount ?? 0} caption annotation(s) → ${result.projectPath}`, + `Captions on: ${result.captionCount ?? 0} caption(s) from the transcript → ${result.projectPath}`, ); } else if (command.kind === "export") { output.info(`Exported ${result.format ?? ""} → ${result.outputPath}`); @@ -512,7 +513,8 @@ export function runCli(command: CliCommand): void { } // Give the renderer a beat to resolve the invoke before exiting. - setTimeout(() => app.exit(result.success ? 0 : 1), 50); + // and stop the Whisper helper `captions` started, which would outlive this process. + setTimeout(() => void shutdownStt().finally(() => app.exit(result.success ? 0 : 1)), 50); }) .catch((error) => { output.error(error instanceof Error ? (error.stack ?? error.message) : String(error)); diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 05bb98aa22..8a3fa45d32 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -157,6 +157,7 @@ import { shouldEnumerateRecordingSources, shouldPersistSelectedSource, } from "../recording-source-settings"; +import { getSttManager } from "../stt"; import { createRunnerWindow } from "../windows"; import { getSourcesByType, shouldEnumerateSourceTypesSeparately } from "./desktopSourceTypes"; import { registerNativeBridgeHandlers } from "./nativeBridge"; @@ -5061,6 +5062,8 @@ export function registerIpcHandlers( editsAllowed: () => mcpSettings.getSettings().allowEdits, cursor: agentCursorTelemetryReader, cameraSegmentation: () => loudnessService.probeSegmentation(), + // The editor's own engine, in this process: `openscreen mcp` transcribes without the app. + transcribe: (request) => getSttManager().transcribe(request), version: app.getVersion(), // importMedia: the editor's own probe, and the grant a picked file gets, so the editor // can play a file a client imported into the project it has open. diff --git a/electron/mcp/app-channel.ts b/electron/mcp/app-channel.ts index 09d7f6be46..9da5585cf7 100644 --- a/electron/mcp/app-channel.ts +++ b/electron/mcp/app-channel.ts @@ -44,6 +44,7 @@ import { LIST_PROJECTS_TOOL, type McpToolDeps, type McpToolRunner, + TRANSCRIBE_TOOL, } from "./openscreen-mcp-server"; import { PROJECT_EDIT_TOOLS } from "./project-tools"; @@ -52,6 +53,8 @@ const DISCOVERY_FILE = "mcp-app.json"; const CONNECT_TIMEOUT_MS = 3_000; // Above the editor's own 30 s for a snapshot plus 30 s for an apply. const CALL_TIMEOUT_MS = 90_000; +// Whisper over a whole recording: above the editor's own hour for it. +const TRANSCRIBE_TIMEOUT_MS = 65 * 60_000; const MAX_TOKEN_LINE_BYTES = 256; const MAX_ANSWER_BYTES = 256 * 1024 * 1024; @@ -260,7 +263,9 @@ export async function connectToApp( /** A call that changes nothing, so running it again here after a lost answer is safe. */ function isReadOnly(name: string): boolean { if (name === LIST_PROJECTS_TOOL) return true; - if (name === "createCheckpoint" || name === "restoreCheckpoint") return false; + if (name === "createCheckpoint" || name === "restoreCheckpoint" || name === TRANSCRIBE_TOOL) { + return false; + } return !isMutatingTool(name) && !PROJECT_EDIT_TOOLS.has(name); } @@ -297,7 +302,7 @@ export function routeToApp(options: { return (await connection.client.callTool( { name, arguments: (args ?? {}) as Record }, undefined, - { timeout: CALL_TIMEOUT_MS }, + { timeout: name === TRANSCRIBE_TOOL ? TRANSCRIBE_TIMEOUT_MS : CALL_TIMEOUT_MS }, )) as CallToolResult; } catch { // Closed or timed out: a hung app must not hold every later call too. diff --git a/electron/mcp/editor-document-host.test.ts b/electron/mcp/editor-document-host.test.ts index 5b11cd19a0..4777b64401 100644 --- a/electron/mcp/editor-document-host.test.ts +++ b/electron/mcp/editor-document-host.test.ts @@ -114,6 +114,28 @@ describe("EditorDocumentHost", () => { expect(second.sent).toHaveLength(0); }); + it("waits past the usual timeout for a transcription, and passes its outcome through", async () => { + vi.useFakeTimers(); + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const pending = host.transcribe("p", "asset_1", "fr"); + await vi.advanceTimersByTimeAsync(5 * 60_000); + expect(editor.sent[0]).toMatchObject({ op: "transcribe", projectId: "p", language: "fr" }); + await answer(editor, fromRenderer, { status: "transcribed", language: "fr", words: 4 }); + await expect(pending).resolves.toEqual({ status: "transcribed", language: "fr", words: 4 }); + }); + + it("answers a transcription at once when its editor goes away", async () => { + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const pending = host.transcribe("p", "asset_1", undefined); + await vi.waitFor(() => expect(editor.sent).toHaveLength(1)); + editor.destroy(); + await expect(pending).resolves.toBe("no-editor"); + }); + it("reports an unanswered apply as a timeout, not as a failure", async () => { vi.useFakeTimers(); const { host, fromRenderer } = setup(); diff --git a/electron/mcp/editor-document-host.ts b/electron/mcp/editor-document-host.ts index 55d6d61569..4a3be27dd7 100644 --- a/electron/mcp/editor-document-host.ts +++ b/electron/mcp/editor-document-host.ts @@ -19,12 +19,16 @@ import { type AiEditionMcpHostRequest, type AiEditionMcpHostResponse, type AiEditionMcpHostSnapshot, + type AiEditionMcpTranscription, } from "../../src/native/contracts"; import type { McpApplyResult, McpDocumentHost } from "./openscreen-mcp-server"; // Long enough for a save of a large project to land, short enough that a hung // renderer does not hold the client's call open indefinitely. const REQUEST_TIMEOUT_MS = 30_000; +// A transcription runs Whisper over the whole recording, on the CPU at worst. The editor +// going away answers sooner: every pending request is dropped then. +const TRANSCRIBE_TIMEOUT_MS = 60 * 60_000; const TIMED_OUT = Symbol("timed-out"); @@ -35,7 +39,8 @@ interface Pending { type RequestBody = | { op: "snapshot" } - | { op: "apply"; document: unknown; expectedRevision: number }; + | { op: "apply"; document: unknown; expectedRevision: number } + | { op: "transcribe"; projectId: string; assetId: string; language?: string }; export class EditorDocumentHost implements McpDocumentHost { private editor: WebContents | null = null; @@ -46,10 +51,10 @@ export class EditorDocumentHost implements McpDocumentHost { if (active === true) { this.editor = event.sender; event.sender.once("destroyed", () => { - if (this.editor === event.sender) this.editor = null; + if (this.editor === event.sender) this.leave(); }); } else if (this.editor === event.sender) { - this.editor = null; + this.leave(); } }); ipcMain.on(AI_EDITION_MCP_RESPONSE_CHANNEL, (event: IpcMainEvent, message: unknown) => { @@ -64,7 +69,17 @@ export class EditorDocumentHost implements McpDocumentHost { }); } - private request(body: RequestBody): Promise { + /** The editor stopped answering: what it was asked will never be answered. */ + private leave(): void { + this.editor = null; + for (const [requestId, pending] of this.pending) { + clearTimeout(pending.timer); + this.pending.delete(requestId); + pending.resolve(undefined); + } + } + + private request(body: RequestBody, timeoutMs = REQUEST_TIMEOUT_MS): Promise { const editor = this.editor; if (!editor || editor.isDestroyed()) return Promise.resolve(undefined); const requestId = randomUUID(); @@ -72,7 +87,7 @@ export class EditorDocumentHost implements McpDocumentHost { const timer = setTimeout(() => { this.pending.delete(requestId); resolve(TIMED_OUT); - }, REQUEST_TIMEOUT_MS); + }, timeoutMs); this.pending.set(requestId, { resolve, timer }); try { editor.send(AI_EDITION_MCP_REQUEST_CHANNEL, { @@ -104,4 +119,18 @@ export class EditorDocumentHost implements McpDocumentHost { if (typeof result !== "string") return "no-editor"; return result as McpApplyResult; } + + async transcribe( + projectId: string, + assetId: string, + language: string | undefined, + ): Promise { + const result = await this.request( + { op: "transcribe", projectId, assetId, ...(language ? { language } : {}) }, + TRANSCRIBE_TIMEOUT_MS, + ); + if (result === TIMED_OUT) return "timeout"; + if (!result || typeof result !== "object" || !("status" in result)) return "no-editor"; + return result as AiEditionMcpTranscription; + } } diff --git a/electron/mcp/openscreen-mcp-server.ts b/electron/mcp/openscreen-mcp-server.ts index dd4688a1f9..59adac7f76 100644 --- a/electron/mcp/openscreen-mcp-server.ts +++ b/electron/mcp/openscreen-mcp-server.ts @@ -49,8 +49,20 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { z } from "zod"; +import { + applyTranscriptionResult, + withTranscript, +} from "../../src/lib/ai-edition/document/transcript"; import { type AxcutDocument, documentSchema } from "../../src/lib/ai-edition/schema"; -import type { SegmentationSupport } from "../../src/native/contracts"; +import { + segmentsFromStt, + transcriptFromSegments, +} from "../../src/lib/ai-edition/transcription/fromStt"; +import { + classifyTranscriptionError, + withTranscriptionFailure, +} from "../../src/lib/ai-edition/transcription/status"; +import type { AiEditionMcpTranscription, SegmentationSupport } from "../../src/native/contracts"; import { type AgentToolExecution, consentRequired, @@ -70,6 +82,7 @@ import { type ProjectSummary, type ProjectVersion, } from "../ai-edition/document-service"; +import type { SttTranscribeRequest, SttTranscribeResponse } from "../stt/transcriptionContract"; import type { MediaTools } from "./media-tools"; import { IMPORT_MEDIA_TOOL, @@ -107,6 +120,16 @@ export interface McpDocumentHost { /** `null` when no editor window is open or it has no project loaded. */ snapshot(): Promise; apply(document: AxcutDocument, expectedRevision: number): Promise; + /** + * The editor's own transcription of an asset of the project it holds (`projectId`): a + * job of its own already under way is waited for rather than run twice. Absent where + * there is no editor to ask. + */ + transcribe?( + projectId: string, + assetId: string, + language: string | undefined, + ): Promise; } /** Every project on disk — the subset of `DocumentService` the server needs. */ @@ -147,10 +170,34 @@ export interface McpToolDeps extends ImportMediaDeps { cursor?: CursorTelemetryReader; /** Whether this machine can cut out the camera background; absent where nothing can ask. */ cameraSegmentation?: () => SegmentationSupport; + /** The main process's on-device Whisper (`SttManager.transcribe`). Absent: no transcribe tool. */ + transcribe?(request: SttTranscribeRequest): Promise; version: string; } export const LIST_PROJECTS_TOOL = "listProjects"; +export const TRANSCRIBE_TOOL = "transcribe"; + +const transcribeArgs = z.object({ + assetId: z + .string() + .min(1) + .optional() + .describe("The asset to transcribe, from getCurrentDocument. Default: the primary asset."), + language: z + .string() + .regex(/^(auto|[a-z]{2,3})$/, 'an ISO 639-1 code such as "en", or "auto"') + .optional() + .describe( + 'Transcribe again in this language (ISO 639-1, such as "en" or "fr"), or "auto" to let Whisper detect it. Omit it to keep a transcript the asset already has.', + ), +}); + +const TRANSCRIBE_DESCRIPTION = + "Transcribe one asset's speech with OpenScreen's on-device Whisper, as the editor does when it opens a project, and store the transcript in the project: getTranscript, getTranscriptWords, setWordText, removeWords and the captions all read it. A take from stopRecording has none until this runs. Without `language`, an asset that already has a transcript keeps it, and a transcription the editor is already running is waited for rather than run twice; with `language`, the asset is transcribed again in it, and the word corrections that still match are carried over. The project's vocabulary (setVocabulary) primes Whisper. It runs on this computer and can take minutes on a long recording, more the first time, when the speech model downloads. Media with no audio fails with failure.kind `no-audio`, which the asset then remembers (assets[].transcriptionFailure in getCurrentDocument). The result gives the language and the word count; it needs no checkpoint, and restoreCheckpoint keeps a transcript made after its checkpoint."; + +const TRANSCRIBE_NOTE = + "Speech: transcribe makes an asset's transcript with OpenScreen's on-device Whisper. Everything that reads speech needs it (getTranscript, getTranscriptWords, removeWords, the captions); a fresh take has none until you call it."; /** The in-app server's edit switch, as AI settings label it (`mcpServer.editsLabel`). */ export const MCP_EDITS_SETTING = "'MCP edits' under MCP server in AI settings"; @@ -234,6 +281,8 @@ export function mcpInstructions(options: { /** The media tools' preamble. Both servers register them; only the app's channel for * `openscreen mcp` (app-channel.ts), which no agent reads, goes without. */ media?: string; + /** The transcribe tool is served. */ + transcribe?: boolean; }): string { const { transport, editsAllowed, media } = options; const preamble = [ @@ -245,6 +294,7 @@ export function mcpInstructions(options: { `startRecording, exportVideo and createProject are refused too while ${MCP_EDITS_SETTING} is off.`, ] : []), + ...(options.transcribe ? [TRANSCRIBE_NOTE] : []), "", ].join("\n"); return ( @@ -396,7 +446,7 @@ export function createToolRunner(deps: McpToolDeps): McpToolRunner { return { ok: true, resultJson: JSON.stringify({ ok: true, restored: id }), - document: checkpoint.document, + document: keepTranscriptsMadeSince(checkpoint.document, document), sidecars: [...checkpoint.sidecars].map(([path, content]) => ({ path, content })), }; } @@ -486,8 +536,9 @@ export function createToolRunner(deps: McpToolDeps): McpToolRunner { if ((await deps.projects.projectOpenInEditor()) === projectId) { return textResult(new ProjectOpenInEditorError(projectId).message, true); } - // A restore IS the way back, so it needs no checkpoint of its own. - if (name !== "restoreCheckpoint" && !hasCheckpoint(projectId)) { + // A restore IS the way back, so it needs no checkpoint of its own; a transcript is + // no edit to take back (restoreCheckpoint keeps it), so it needs none either. + if (name !== "restoreCheckpoint" && name !== TRANSCRIBE_TOOL && !hasCheckpoint(projectId)) { return textResult(CHECKPOINT_REQUIRED_MESSAGES[deps.transport], true); } // Refused if anything read the project since (the editor opening it would @@ -556,7 +607,139 @@ export function createToolRunner(deps: McpToolDeps): McpToolRunner { return runOnFile(projectId, read, name, args, call); } - return (name, args, options = {}) => { + /** Whisper runs one transcription at a time; this runner's wait their turn here. */ + let whisperQueue: Promise = Promise.resolve(); + + /** Lands a change computed outside the queue, as an edit of the project as it is by then. */ + function landEdit(projectId: string, editsAllowed: boolean, edit: RunnerEdit) { + return runner(TRANSCRIBE_TOOL, { projectId }, { editsAllowed, edit }); + } + + /** + * The transcribe tool. Whisper takes minutes, so it runs OUTSIDE the queue: other calls + * go on meanwhile, and only the landing of the transcript is queued, as an edit of the + * project as it is by then. The project open in the editor is transcribed by the editor's + * own job instead, so the two never run the same transcription side by side. + */ + async function transcribe(rawArgs: unknown, editsAllowed: boolean): Promise { + const { projectId, args } = splitProjectId(rawArgs); + const parsed = transcribeArgs.safeParse(args ?? {}); + if (!parsed.success) return textResult(JSON.stringify({ error: parsed.error.message }), true); + if (!editsAllowed) { + return textResult(consentRequired(TRANSCRIBE_TOOL, args, MCP_EDITS_SETTING).resultJson, true); + } + const { language } = parsed.data; + const snapshot = await deps.host.snapshot(); + const openId = openProjectId(snapshot); + const targetId = projectId ?? openId; + if (!targetId) return textResult(NO_PROJECT_MESSAGES[deps.transport], true); + const onEditor = openId === targetId; + let document: AxcutDocument; + if (onEditor) { + const read = documentSchema.safeParse(snapshot?.document); + if (!read.success) { + return textResult("The project open in the editor could not be read.", true); + } + document = read.data; + } else { + const read = await deps.projects.getProjectForUpdate(targetId).catch((error) => { + if (error instanceof DocumentNotFoundError) return null; + throw error; + }); + if (!read) { + return textResult( + `No project has the id "${targetId}". Call listProjects for the valid ids.`, + true, + ); + } + document = read.document; + } + const assetId = + parsed.data.assetId ?? document.project.primaryAssetId ?? document.assets[0]?.id; + const asset = document.assets.find((a) => a.id === assetId); + if (!asset) { + return textResult( + JSON.stringify({ + error: `Unknown asset: ${assetId ?? "(none)"}. The project has: ${document.assets.map((a) => a.id).join(", ") || "no assets"}.`, + }), + true, + ); + } + if (onEditor && deps.host.transcribe) { + return editorTranscription( + await deps.host.transcribe(targetId, asset.id, language), + asset.id, + ); + } + const existing = document.transcripts.find((t) => t.assetId === asset.id); + if (language === undefined && existing) { + return textResult(JSON.stringify({ ...transcriptSummary(existing), unchanged: true })); + } + // An editor this process cannot reach would save over the transcript: say so now, + // not after minutes of Whisper. + if ((await deps.projects.projectOpenInEditor()) === targetId) { + return textResult(new ProjectOpenInEditorError(targetId).message, true); + } + const whisper = deps.transcribe; + if (!whisper) return textResult("Transcription is not available here.", true); + const started = whisperQueue.then(() => + whisper({ + sourcePath: asset.originalPath, + language: language === "auto" ? undefined : language, + prompt: document.project.vocabulary, + }), + ); + whisperQueue = started.catch(() => undefined); + let response: SttTranscribeResponse; + try { + response = await started; + } catch (error) { + const failure = classifyTranscriptionError(error); + // Remembered on the asset as the editor remembers it: best effort, the answer stands. + await landEdit(targetId, editsAllowed, async (current) => { + const next = withTranscriptionFailure(current, asset.id, failure); + return { ok: true, resultJson: "", ...(next === current ? {} : { document: next }) }; + }); + return textResult( + JSON.stringify({ + error: `The transcription failed: ${failure.message}`, + failure: { kind: failure.kind, message: failure.message }, + }), + true, + ); + } + const transcript = transcriptFromSegments( + asset.id, + segmentsFromStt(response).segments, + response.detectedLanguage ?? language, + ); + return landEdit(targetId, editsAllowed, async (current) => { + if (!current.assets.some((a) => a.id === asset.id)) { + return { + ok: false, + resultJson: JSON.stringify({ + error: `Asset ${asset.id} was removed from the project while it was transcribed.`, + }), + }; + } + const landed = applyTranscriptionResult(current, transcript); + return { + ok: true, + document: landed.document, + resultJson: JSON.stringify({ + ...transcriptSummary(transcript), + ...(landed.dropped > 0 ? { correctionsLost: landed.dropped } : {}), + }), + }; + }); + } + + const runner: McpToolRunner = (name, args, options = {}) => { + if (name === TRANSCRIBE_TOOL && !options.edit) { + return transcribe(args, options.editsAllowed ?? deps.editsAllowed()).catch((error) => + textResult(`Tool failed: ${errorMessage(error)}`, true), + ); + } // The setting is read when the call's turn comes: it may change while it waits. const runNow = async () => { const editsAllowed = options.editsAllowed ?? deps.editsAllowed(); @@ -573,6 +756,71 @@ export function createToolRunner(deps: McpToolDeps): McpToolRunner { queue = next.catch(() => undefined); return next.catch((error) => textResult(`Tool failed: ${errorMessage(error)}`, true)); }; + return runner; +} + +/** What a transcription made, in the terms a caller acts on. */ +function transcriptSummary(transcript: AxcutDocument["transcripts"][number]) { + return { + assetId: transcript.assetId, + language: transcript.language, + words: transcript.words.length, + ...(transcript.words.length === 0 ? { note: "No speech was found in this media." } : {}), + }; +} + +/** The editor's answer about a transcription it ran or waited for, as this tool's result. */ +function editorTranscription( + outcome: AiEditionMcpTranscription | "no-editor" | "timeout", + assetId: string, +): CallToolResult { + if (outcome === "timeout") { + return textResult( + "OpenScreen did not finish this transcription in time; it may still be running in the editor. Call getCurrentDocument later: assets[].hasTranscript says when it has landed.", + true, + ); + } + if (outcome === "no-editor" || outcome.status === "unavailable") { + return textResult( + "The editor closed the project before its transcription finished. Call transcribe again with its projectId.", + true, + ); + } + if (outcome.status === "failed") { + return textResult( + JSON.stringify({ + error: `The transcription failed: ${outcome.failure.message}`, + failure: outcome.failure, + }), + true, + ); + } + return textResult( + JSON.stringify({ + assetId, + language: outcome.language, + words: outcome.words, + ...(outcome.words === 0 ? { note: "No speech was found in this media." } : {}), + }), + ); +} + +/** + * A transcript is the media read, not an edit to take back: a restore keeps the one an + * asset was given since the checkpoint, where the checkpoint had none for it. + */ +function keepTranscriptsMadeSince( + checkpoint: AxcutDocument, + current: AxcutDocument, +): AxcutDocument { + let restored = checkpoint; + for (const transcript of current.transcripts) { + const { assetId } = transcript; + if (!restored.assets.some((a) => a.id === assetId)) continue; + if (restored.transcripts.some((t) => t.assetId === assetId)) continue; + restored = withTranscript(restored, transcript); + } + return restored; } export function createOpenScreenMcpServer( @@ -588,6 +836,7 @@ export function createOpenScreenMcpServer( transport: deps.transport, editsAllowed: deps.editsAllowed(), media: media?.preamble, + transcribe: deps.transcribe !== undefined, }), }, ); @@ -632,6 +881,17 @@ export function createOpenScreenMcpServer( (args: unknown) => runTool(tool.name, args), ); } + if (deps.transcribe) { + server.registerTool( + TRANSCRIBE_TOOL, + { + description: TRANSCRIBE_DESCRIPTION, + inputSchema: transcribeArgs.extend({ projectId: projectIdArg }), + annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, + }, + (args: unknown) => runTool(TRANSCRIBE_TOOL, args), + ); + } for (const tool of checkpointTools(media !== undefined)) { server.registerTool( tool.name, diff --git a/electron/mcp/transcribe-tool.test.ts b/electron/mcp/transcribe-tool.test.ts new file mode 100644 index 0000000000..32bc4670ec --- /dev/null +++ b/electron/mcp/transcribe-tool.test.ts @@ -0,0 +1,216 @@ +// The transcribe tool through the runner both servers share: a project on file is +// transcribed by the injected Whisper and lands like an edit; the project open in the +// editor is handed to the editor's own job instead. + +import { mkdtempSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + type AxcutDocument, + createEmptyDocument, + documentSchema, +} from "../../src/lib/ai-edition/schema"; +import { DocumentService } from "../ai-edition/document-service"; +import type { SttTranscribeRequest, SttTranscribeResponse } from "../stt/transcriptionContract"; +import { + createToolRunner, + type McpApplyResult, + type McpDocumentHost, + mcpInstructions, + TRANSCRIBE_TOOL, +} from "./openscreen-mcp-server"; + +function fixtureDocument(projectId: string): AxcutDocument { + const base = createEmptyDocument({ + title: projectId, + projectId, + createdAt: "2026-01-01T00:00:00.000Z", + }); + return documentSchema.parse({ + ...base, + project: { ...base.project, primaryAssetId: "asset_1", vocabulary: "OpenScreen" }, + assets: [ + { id: "asset_1", kind: "video", label: "Rec", originalPath: "/tmp/rec.mp4", durationSec: 30 }, + ], + }); +} + +const SPEECH: SttTranscribeResponse = { + segments: [{ text: "Hello world", startSec: 0, endSec: 1 }], + wordSegments: [ + { word: "Hello", startSec: 0, endSec: 0.5 }, + { word: "world", startSec: 0.5, endSec: 1 }, + ], + detectedLanguage: "en", + backend: "whispercpp-cpu", +}; + +/** No editor holds a project unless a test opens one. */ +class Host implements McpDocumentHost { + document: AxcutDocument | null = null; + transcribe = vi.fn(async () => ({ status: "transcribed" as const, language: "fr", words: 3 })); + async snapshot() { + return this.document ? { document: structuredClone(this.document), revision: 1 } : null; + } + async apply(document: AxcutDocument): Promise { + this.document = document; + return "applied"; + } +} + +let dir: string; +let projects: DocumentService; + +beforeEach(() => { + dir = mkdtempSync(path.join(os.tmpdir(), "openscreen-mcp-transcribe-")); + projects = new DocumentService(path.join(dir, "projects"), dir); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +function runnerWith(whisper: (request: SttTranscribeRequest) => Promise) { + const host = new Host(); + const transcribe = vi.fn(whisper); + const run = createToolRunner({ + transport: "stdio", + host, + projects, + editsAllowed: () => true, + transcribe, + version: "0.0.0", + }); + return { run, host, transcribe }; +} + +function payload(result: CallToolResult): Record { + const part = result.content[0]; + return JSON.parse(part?.type === "text" ? part.text : "{}"); +} + +describe("transcribe, on a project's file", () => { + it("transcribes with the project's vocabulary and lands the words, with no checkpoint", async () => { + await projects.saveProject(fixtureDocument("proj_1")); + const { run, transcribe } = runnerWith(async () => SPEECH); + + const result = await run(TRANSCRIBE_TOOL, { projectId: "proj_1" }); + + expect(result.isError).toBeUndefined(); + expect(payload(result)).toMatchObject({ assetId: "asset_1", language: "en", words: 2 }); + expect(transcribe).toHaveBeenCalledWith({ + sourcePath: "/tmp/rec.mp4", + language: undefined, + prompt: "OpenScreen", + }); + const saved = await projects.getProject("proj_1"); + expect(saved.transcripts[0]?.words.map((w) => w.text)).toEqual(["Hello", "world"]); + }); + + it("keeps a transcript without a language, and redoes it with one", async () => { + await projects.saveProject(fixtureDocument("proj_1")); + const { run, transcribe } = runnerWith(async () => SPEECH); + await run(TRANSCRIBE_TOOL, { projectId: "proj_1" }); + + const kept = await run(TRANSCRIBE_TOOL, { projectId: "proj_1" }); + expect(payload(kept)).toMatchObject({ unchanged: true, words: 2 }); + expect(transcribe).toHaveBeenCalledTimes(1); + + await run(TRANSCRIBE_TOOL, { projectId: "proj_1", language: "fr" }); + expect(transcribe).toHaveBeenLastCalledWith(expect.objectContaining({ language: "fr" })); + await run(TRANSCRIBE_TOOL, { projectId: "proj_1", language: "auto" }); + expect(transcribe).toHaveBeenLastCalledWith(expect.objectContaining({ language: undefined })); + }); + + it("reports a silent recording as the editor does, and remembers it on the asset", async () => { + await projects.saveProject(fixtureDocument("proj_1")); + const { run } = runnerWith(async () => { + throw new Error("No decodable audio in /tmp/rec.mp4"); + }); + + const result = await run(TRANSCRIBE_TOOL, { projectId: "proj_1" }); + + expect(result.isError).toBe(true); + expect(payload(result).failure).toMatchObject({ kind: "no-audio" }); + const saved = await projects.getProject("proj_1"); + expect(saved.assets[0]?.transcriptionFailure?.kind).toBe("no-audio"); + }); + + it("refuses an unknown asset, and while MCP edits are off", async () => { + await projects.saveProject(fixtureDocument("proj_1")); + const { run, transcribe } = runnerWith(async () => SPEECH); + const unknown = await run(TRANSCRIBE_TOOL, { projectId: "proj_1", assetId: "nope" }); + expect(String(payload(unknown).error)).toContain("asset_1"); + + const off = await run(TRANSCRIBE_TOOL, { projectId: "proj_1" }, { editsAllowed: false }); + expect(payload(off).code).toBe("consent_required"); + expect(transcribe).not.toHaveBeenCalled(); + }); + + it("lets other calls run while Whisper works", async () => { + await projects.saveProject(fixtureDocument("proj_1")); + const whisper: { finish: ((response: SttTranscribeResponse) => void) | null } = { + finish: null, + }; + const { run } = runnerWith( + () => + new Promise((resolve) => { + whisper.finish = resolve; + }), + ); + const pending = run(TRANSCRIBE_TOOL, { projectId: "proj_1" }); + await vi.waitFor(() => expect(whisper.finish).not.toBeNull()); + // Whisper is still working, and a read goes through all the same. + const read = await run("getCurrentDocument", { projectId: "proj_1" }); + expect(read.isError).toBeUndefined(); + whisper.finish?.(SPEECH); + expect((await pending).isError).toBeUndefined(); + }); + + it("survives a restore to a checkpoint made before it", async () => { + await projects.saveProject(fixtureDocument("proj_1")); + const { run } = runnerWith(async () => SPEECH); + const checkpoint = payload(await run("createCheckpoint", { projectId: "proj_1" })); + await run(TRANSCRIBE_TOOL, { projectId: "proj_1" }); + + await run("restoreCheckpoint", { projectId: "proj_1", checkpointId: checkpoint.checkpointId }); + + expect((await projects.getProject("proj_1")).transcripts).toHaveLength(1); + }); +}); + +describe("the instructions", () => { + it("name the transcribe tool only where it is served", () => { + expect(mcpInstructions({ transport: "stdio", editsAllowed: true, transcribe: true })).toContain( + "transcribe makes an asset's transcript", + ); + expect(mcpInstructions({ transport: "stdio", editsAllowed: true })).not.toContain( + "transcribe makes", + ); + }); +}); + +describe("transcribe, on the project open in the editor", () => { + it("hands it to the editor's own job rather than running Whisper beside it", async () => { + const { run, host, transcribe } = runnerWith(async () => SPEECH); + host.document = fixtureDocument("proj_1"); + + const result = await run(TRANSCRIBE_TOOL, { language: "fr" }); + + expect(host.transcribe).toHaveBeenCalledWith("proj_1", "asset_1", "fr"); + expect(transcribe).not.toHaveBeenCalled(); + expect(payload(result)).toMatchObject({ assetId: "asset_1", language: "fr", words: 3 }); + }); + + it("says so when the editor closes the project mid-run", async () => { + const { run, host } = runnerWith(async () => SPEECH); + host.document = fixtureDocument("proj_1"); + host.transcribe.mockResolvedValueOnce("no-editor" as never); + + const result = await run(TRANSCRIBE_TOOL, {}); + + expect(result.isError).toBe(true); + }); +}); diff --git a/src/components/ai-edition/v4/MediaStage.test.ts b/src/components/ai-edition/v4/MediaStage.test.ts index 51d9c789ea..390b3751f3 100644 --- a/src/components/ai-edition/v4/MediaStage.test.ts +++ b/src/components/ai-edition/v4/MediaStage.test.ts @@ -1,15 +1,11 @@ import { describe, expect, it, vi } from "vitest"; +import { VOCABULARY_MAX_CHARS, withVocabulary } from "@/lib/ai-edition/document/transcript"; import { type AxcutTranscript, createEmptyDocument, parseDocumentFile, } from "@/lib/ai-edition/schema"; -import { - addSelectedAssetToTimeline, - detectedLanguageForTranscript, - VOCABULARY_MAX_CHARS, - withVocabulary, -} from "./MediaStage"; +import { addSelectedAssetToTimeline, detectedLanguageForTranscript } from "./MediaStage"; function transcript(language: string, text = ""): AxcutTranscript { return { diff --git a/src/components/ai-edition/v4/MediaStage.tsx b/src/components/ai-edition/v4/MediaStage.tsx index 7923ab04af..9e93158cc0 100644 --- a/src/components/ai-edition/v4/MediaStage.tsx +++ b/src/components/ai-edition/v4/MediaStage.tsx @@ -3,12 +3,8 @@ import { useMemo, useRef, useState } from "react"; import { toast } from "sonner"; import { Tooltip } from "@/components/ui/tooltip"; import { useI18n, useScopedT } from "@/contexts/I18nContext"; -import type { - AxcutAsset, - AxcutDocument, - AxcutTranscript, - TranscriptLanguageCode, -} from "@/lib/ai-edition/schema"; +import { VOCABULARY_MAX_CHARS, withVocabulary } from "@/lib/ai-edition/document/transcript"; +import type { AxcutAsset, AxcutTranscript, TranscriptLanguageCode } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { useAssetTranscriptions, @@ -53,19 +49,6 @@ export function detectedLanguageForTranscript(transcript: AxcutTranscript | null return transcript.language; } -/** - * Whisper reads at most the last 223 tokens of its prompt; 600 characters of - * names and terms stays under that for Latin scripts (issue #1023). - */ -export const VOCABULARY_MAX_CHARS = 600; - -/** The document with its names and terms set, or null when nothing changed. */ -export function withVocabulary(document: AxcutDocument, text: string): AxcutDocument | null { - const vocabulary = text.trim().slice(0, VOCABULARY_MAX_CHARS) || undefined; - if (vocabulary === document.project.vocabulary) return null; - return { ...document, project: { ...document.project, vocabulary } }; -} - export async function addSelectedAssetToTimeline( selected: Pick | null, onAddToTimeline: (assetId: string) => Promise, diff --git a/src/lib/ai-edition/document/transcribe.ts b/src/lib/ai-edition/document/transcribe.ts index cffddf51dd..4f3f1c7016 100644 --- a/src/lib/ai-edition/document/transcribe.ts +++ b/src/lib/ai-edition/document/transcribe.ts @@ -13,7 +13,8 @@ import { } from "@/lib/captioning"; import type { SttRendererStatus } from "@/lib/captioning/transcribe"; import { STT_NATIVE_EXTRACTION_UNAVAILABLE } from "../../../../electron/stt/transcriptionContract"; -import type { AxcutDocument, AxcutTranscript, AxcutTranscriptSegment, AxcutWord } from "../schema"; +import type { AxcutDocument, AxcutTranscript } from "../schema"; +import { transcriptFromSegments } from "../transcription/fromStt"; /** * What the caller can show while a transcription runs. `completedSec` / @@ -119,54 +120,13 @@ export async function transcribeAsset( }); }); - const segments: AxcutTranscriptSegment[] = []; - const words: AxcutWord[] = []; - - for (let segIndex = 0; segIndex < result.segments.length; segIndex++) { - const seg = result.segments[segIndex]; - const segId = `seg_${segIndex + 1}`; - const wordIds: string[] = []; - - const tokens = seg.text.trim().split(/\s+/).filter(Boolean); - if (tokens.length === 0) continue; - - // Never end before the start: the schema rejects it, and one such word made the whole - // project unreadable (see `raiseInvertedTranscriptEnds`, which repairs old files). - const segEndSec = Math.max(seg.endSec, seg.startSec); - const wordDuration = (segEndSec - seg.startSec) / tokens.length; - for (let w = 0; w < tokens.length; w++) { - const wordId = `word_${words.length + 1}`; - const startSec = seg.startSec + w * wordDuration; - const endSec = startSec + wordDuration; - words.push({ - id: wordId, - segmentId: segId, - startSec, - endSec, - text: tokens[w], - }); - wordIds.push(wordId); - } - - segments.push({ - id: segId, - kind: "speech" as const, - startSec: seg.startSec, - endSec: segEndSec, - text: seg.text, - wordIds, - }); - } - - return { + // Prefer the model-reported language (covers both forced picks and auto-detect), + // then the input option. + return transcriptFromSegments( assetId, - // Prefer the model-reported language (covers both forced picks and - // auto-detect); fall back to the input option, then "auto" when - // nothing was detected (very rare on tiny.en — usually a no-audio run). - language: result.detectedLanguage ?? options.language ?? "auto", - segments, - words, - }; + result.segments, + result.detectedLanguage ?? options.language, + ); } // `withTranscript` used to live here. It moved to `document/transcript.ts`, next to diff --git a/src/lib/ai-edition/document/transcript.ts b/src/lib/ai-edition/document/transcript.ts index 1bfdb7ccf4..b702d416d1 100644 --- a/src/lib/ai-edition/document/transcript.ts +++ b/src/lib/ai-edition/document/transcript.ts @@ -257,3 +257,46 @@ export function carryOverWordEdits( return { transcript, carried, dropped: edits.length - carried }; } + +/** + * A finished transcription landing on the document: it replaces the asset's transcript, + * carrying the user's word corrections over (`carryOverWordEdits`), and clears the failure + * the asset remembered from an earlier run. What could not be carried is counted in + * `dropped`. The editor's queue and the MCP `transcribe` tool both land a run through here. + */ +export function applyTranscriptionResult( + document: AxcutDocument, + transcript: AxcutTranscript, +): { document: AxcutDocument; dropped: number } { + const merged = carryOverWordEdits( + document.transcripts.find((t) => t.assetId === transcript.assetId), + transcript, + ); + return { + document: withTranscript( + { + ...document, + assets: document.assets.map((a) => + a.id === transcript.assetId && a.transcriptionFailure + ? { ...a, transcriptionFailure: null } + : a, + ), + }, + merged.transcript, + ), + dropped: merged.dropped, + }; +} + +/** + * Whisper reads at most the last 223 tokens of its prompt; 600 characters of + * names and terms stays under that for Latin scripts (issue #1023). + */ +export const VOCABULARY_MAX_CHARS = 600; + +/** The document with its names and terms set, or null when nothing changed. */ +export function withVocabulary(document: AxcutDocument, text: string): AxcutDocument | null { + const vocabulary = text.trim().slice(0, VOCABULARY_MAX_CHARS) || undefined; + if (vocabulary === document.project.vocabulary) return null; + return { ...document, project: { ...document.project, vocabulary } }; +} diff --git a/src/lib/ai-edition/store/mcpDocumentHost.ts b/src/lib/ai-edition/store/mcpDocumentHost.ts index d1c82ff163..d99956b385 100644 --- a/src/lib/ai-edition/store/mcpDocumentHost.ts +++ b/src/lib/ai-edition/store/mcpDocumentHost.ts @@ -1,7 +1,13 @@ import { useEffect } from "react"; -import type { AiEditionMcpHostRequest, AiEditionMcpHostResponse } from "@/native/contracts"; +import type { + AiEditionMcpHostRequest, + AiEditionMcpHostResponse, + AiEditionMcpTranscription, +} from "@/native/contracts"; +import { findAssetTranscript } from "../transcription/status"; import { applyAgentDocumentIfCurrent } from "./agentDocumentApply"; import { useProjectStore } from "./projectStore"; +import { useTranscriptionStore } from "./transcriptionStore"; /** * The editor's side of the MCP server (electron/mcp/): hand it the live document @@ -16,6 +22,7 @@ export async function answerMcpHostRequest( if (request.op === "snapshot") { return document ? { document, revision } : null; } + if (request.op === "transcribe") return transcribeForMcp(request); if (!document) return "no-live-document"; // The revision counter restarts when a project is closed, so a quick close // and reopen of ANOTHER project can land on the revision the call was given. @@ -29,6 +36,33 @@ export async function answerMcpHostRequest( } } +/** + * The MCP `transcribe` tool on the project this editor holds: through the editor's own + * queue (`ensureTranscript`), so a job it already runs is waited for, never doubled. + */ +async function transcribeForMcp( + request: Extract, +): Promise { + const held = () => { + const document = useProjectStore.getState().document; + return document?.project.id === request.projectId ? document : null; + }; + if (!held()) return { status: "unavailable" }; + await useTranscriptionStore.getState().ensureTranscript(request.assetId, request.language); + const job = useTranscriptionStore.getState().jobs[request.assetId]; + const transcript = findAssetTranscript(held(), request.assetId); + // A transcript the asset kept answers a call that asked for no language, whatever an + // earlier job left behind. + if (job?.status === "failed" && job.failure && (request.language || !transcript)) { + return { + status: "failed", + failure: { kind: job.failure.kind, message: job.failure.message }, + }; + } + if (!transcript) return { status: "unavailable" }; + return { status: "transcribed", language: transcript.language, words: transcript.words.length }; +} + /** Makes this editor window the one the MCP server reads and writes, while mounted. */ export function useMcpDocumentHost(): void { useEffect(() => window.electronAPI?.onAiEditionMcpRequest?.(answerMcpHostRequest), []); diff --git a/src/lib/ai-edition/store/transcriptionStore.test.ts b/src/lib/ai-edition/store/transcriptionStore.test.ts index f58291e414..78e5766a0d 100644 --- a/src/lib/ai-edition/store/transcriptionStore.test.ts +++ b/src/lib/ai-edition/store/transcriptionStore.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { toastText } from "@/i18n/toastText"; import type { AxcutDocument, AxcutTranscript } from "../schema"; +import { answerMcpHostRequest } from "./mcpDocumentHost"; import { useProjectStore } from "./projectStore"; import { useTranscriptionStore, whenTranscriptionIdle } from "./transcriptionStore"; @@ -452,6 +453,70 @@ describe("useTranscriptionStore", () => { expect(useProjectStore.getState().document?.transcripts).toHaveLength(1); }); + it("ensureTranscript waits for the editor's own run, and the MCP host reports it", async () => { + const firstRun = deferred(); + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => { + await firstRun.promise; + return transcriptFor(assetId); + }, + ); + loadDocument(makeDoc(["asset_1"])); + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await Promise.resolve(); + expect(useTranscriptionStore.getState().jobs.asset_1?.status).toBe("running"); + + const answered = answerMcpHostRequest({ + requestId: "r", + op: "transcribe", + projectId: "proj_1", + assetId: "asset_1", + }); + firstRun.release(); + + await expect(answered).resolves.toEqual({ status: "transcribed", language: "en", words: 1 }); + expect(transcribeMocks.transcribeAsset).toHaveBeenCalledTimes(1); + }); + + it("ensureTranscript keeps a transcript unless a language is asked for", async () => { + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => transcriptFor(assetId), + ); + loadDocument({ ...makeDoc(["asset_1"]), transcripts: [transcriptFor("asset_1")] }); + + await useTranscriptionStore.getState().ensureTranscript("asset_1"); + expect(transcribeMocks.transcribeAsset).not.toHaveBeenCalled(); + + await useTranscriptionStore.getState().ensureTranscript("asset_1", "fr"); + expect(transcribeMocks.transcribeAsset).toHaveBeenCalledWith( + expect.anything(), + "asset_1", + expect.objectContaining({ language: "fr" }), + ); + }); + + it("the MCP host reports the failure of the run it asked for", async () => { + transcribeMocks.transcribeAsset.mockRejectedValue(new Error("No decodable audio in x")); + loadDocument(makeDoc(["asset_1"])); + + await expect( + answerMcpHostRequest({ + requestId: "r", + op: "transcribe", + projectId: "proj_1", + assetId: "asset_1", + }), + ).resolves.toMatchObject({ status: "failed", failure: { kind: "no-audio" } }); + await expect( + answerMcpHostRequest({ + requestId: "r", + op: "transcribe", + projectId: "another", + assetId: "asset_1", + }), + ).resolves.toEqual({ status: "unavailable" }); + }); + it("still honours a manual request without the auto pass", async () => { // biome-ignore lint/suspicious/noExplicitAny: test-only stub of the preload bridge delete (window as any).electronAPI; diff --git a/src/lib/ai-edition/store/transcriptionStore.ts b/src/lib/ai-edition/store/transcriptionStore.ts index 316f30791c..341bd72f80 100644 --- a/src/lib/ai-edition/store/transcriptionStore.ts +++ b/src/lib/ai-edition/store/transcriptionStore.ts @@ -27,7 +27,7 @@ import { toast } from "sonner"; import { create } from "zustand"; import { toastText as translateToast } from "@/i18n/toastText"; import { transcribeAsset } from "../document/transcribe"; -import { carryOverWordEdits, withTranscript } from "../document/transcript"; +import { applyTranscriptionResult } from "../document/transcript"; import type { AxcutDocument } from "../schema"; import { type AssetTranscriptionView, @@ -44,6 +44,7 @@ import { type TranscriptionProgress, transcriptionFailureHintKey, transcriptRelevantAssetIds, + withTranscriptionFailure, } from "../transcription/status"; import { useProjectStore } from "./projectStore"; @@ -87,6 +88,14 @@ interface TranscriptionState { * leave that button unable to transcribe the talking clip next to it. */ requestTimelineTranscripts: () => Promise; + /** + * What a caller outside the editor asks for (the MCP `transcribe` tool): a transcript of + * this asset, without a second run beside the editor's own. Without `language`, an asset + * that has one is left alone and a job already queued or running is waited for; with + * it, the job is waited for only when it runs in that language, else a run is asked for. + * Resolves once the run settles. + */ + ensureTranscript: (assetId: string, language?: string) => Promise; reset: () => void; } @@ -189,6 +198,17 @@ export const useTranscriptionStore = create((set, get) => ({ ).then(() => undefined); }, + ensureTranscript(assetId, language) { + const document = useProjectStore.getState().document; + if (document && document.project.id !== get().projectId) get().sync(document); + const job = get().jobs[assetId]; + if (job && job.status !== "failed" && (language === undefined || language === job.language)) { + return waitForSettle(assetId); + } + if (language === undefined && findAssetTranscript(document, assetId)) return Promise.resolve(); + return get().request(assetId, language); + }, + reset() { abortActiveRun(); const pending = Object.keys(get().jobs); @@ -318,8 +338,7 @@ async function persistPermanentFailure( assetId: string, failure: TranscriptionFailure, ): Promise { - const kind = failure.kind; - if (!isPermanentFailure(kind)) return; + if (!isPermanentFailure(failure.kind)) return; const project = useProjectStore.getState(); const doc = project.document; if (!doc || doc.project.id !== projectId) return; @@ -328,21 +347,7 @@ async function persistPermanentFailure( // false rather than throwing, and a note on the asset is not worth a second // message on top of the one the user already got. const persisted = await project.saveDocument( - { - ...doc, - assets: doc.assets.map((a) => - a.id === assetId - ? { - ...a, - transcriptionFailure: { - kind, - message: failure.message, - at: new Date().toISOString(), - }, - } - : a, - ), - }, + withTranscriptionFailure(doc, assetId, failure), // Bookkeeping, not an edit — it must not become an undo step. { history: false }, ); @@ -416,33 +421,17 @@ async function runJob(assetId: string, job: TranscriptionJob): Promise { // hand would go with it. Carry those corrections onto the new words first — // strictly, so nothing is invented (see `carryOverWordEdits`). What could not be // carried is lost; telling the user so is the UI's job, and there is no surface - // for it yet. - const merged = carryOverWordEdits( - current.transcripts.find((t) => t.assetId === assetId), - transcript, - ); - if (merged.dropped > 0) { + // for it yet. The same save clears the verdict remembered on the asset. + const landed = applyTranscriptionResult(current, transcript); + if (landed.dropped > 0) { console.warn( - `[transcription] ${merged.dropped} word correction(s) on asset ${assetId} could not be carried over to the new transcript.`, + `[transcription] ${landed.dropped} word correction(s) on asset ${assetId} could not be carried over to the new transcript.`, ); } - // One save: the transcript, and (on a successful retry) the removal of - // the verdict remembered on the asset. // `history: false`: a transcript landing from a background job is not an edit // the user made, and making it the target of the next Ctrl+Z would both surprise // them and throw the transcript away. - await useProjectStore.getState().saveDocument( - withTranscript( - { - ...current, - assets: current.assets.map((a) => - a.id === assetId && a.transcriptionFailure ? { ...a, transcriptionFailure: null } : a, - ), - }, - merged.transcript, - ), - { history: false }, - ); + await useProjectStore.getState().saveDocument(landed.document, { history: false }); dropJob(assetId, runId); if (job.manual) toast.success(toastText("mediaStage.transcriptReady")); } catch (error) { diff --git a/src/lib/ai-edition/transcription/fromStt.test.ts b/src/lib/ai-edition/transcription/fromStt.test.ts new file mode 100644 index 0000000000..85f6dc03d1 --- /dev/null +++ b/src/lib/ai-edition/transcription/fromStt.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { segmentsFromStt, transcriptFromSegments } from "./fromStt"; + +describe("segmentsFromStt", () => { + it("takes the engine's words, one segment each", () => { + const { segments, granularity } = segmentsFromStt({ + segments: [{ text: "Hi you", startSec: 0, endSec: 1 }], + wordSegments: [ + { word: "Hi", startSec: 0, endSec: 0.4 }, + { word: "you", startSec: 0.4, endSec: 1 }, + ], + }); + expect(granularity).toBe("word"); + expect(segments.map((s) => s.text)).toEqual(["Hi", "you"]); + }); + + it("falls back to the phrases when whisper kept no word", () => { + const { segments, granularity } = segmentsFromStt({ + segments: [{ text: "Hi you", startSec: 0, endSec: 1 }], + wordSegments: [], + }); + expect(granularity).toBe("phrase"); + expect(segments).toEqual([{ text: "Hi you", startSec: 0, endSec: 1 }]); + }); +}); + +describe("transcriptFromSegments", () => { + it("spreads each segment's tokens over its span, keeping ids by position", () => { + const transcript = transcriptFromSegments( + "asset_1", + [ + { text: " ", startSec: 0, endSec: 1 }, + { text: "two words", startSec: 2, endSec: 3 }, + ], + "fr", + ); + expect(transcript.language).toBe("fr"); + expect(transcript.segments.map((s) => s.id)).toEqual(["seg_2"]); + expect(transcript.words).toEqual([ + { id: "word_1", segmentId: "seg_2", startSec: 2, endSec: 2.5, text: "two" }, + { id: "word_2", segmentId: "seg_2", startSec: 2.5, endSec: 3, text: "words" }, + ]); + }); + + it("never ends a word before it starts, and says auto when no language came back", () => { + const transcript = transcriptFromSegments("a", [{ text: "x", startSec: 2, endSec: 1 }], null); + expect(transcript.language).toBe("auto"); + expect(transcript.words[0]).toMatchObject({ startSec: 2, endSec: 2 }); + }); +}); diff --git a/src/lib/ai-edition/transcription/fromStt.ts b/src/lib/ai-edition/transcription/fromStt.ts new file mode 100644 index 0000000000..6466c027c2 --- /dev/null +++ b/src/lib/ai-edition/transcription/fromStt.ts @@ -0,0 +1,80 @@ +// What the speech engine answers, turned into the document's transcript. Pure, so the +// editor's queue (through the renderer) and the MCP `transcribe` tool (in the main +// process, `electron/mcp/`) build the very same words from the very same answer. + +import type { SttTranscribeResponse } from "../../../../electron/stt/transcriptionContract"; +import type { CaptionSegment, CaptionTimestampGranularity } from "../../captioning/transcribe"; +import type { AxcutTranscript, AxcutTranscriptSegment, AxcutWord } from "../schema"; + +/** + * The engine's words as timed segments, one per word. When whisper dropped every word + * (OOV-heavy speech), its phrase spans instead, so there is still something to edit. + */ +export function segmentsFromStt( + response: Pick, +): { segments: CaptionSegment[]; granularity: CaptionTimestampGranularity } { + const words = response.wordSegments ?? []; + if (words.length > 0) { + return { + segments: words.map((w) => ({ startSec: w.startSec, endSec: w.endSec, text: w.word })), + granularity: "word", + }; + } + return { + segments: (response.segments ?? []).map((s) => ({ + startSec: s.startSec, + endSec: s.endSec, + text: s.text, + })), + granularity: "phrase", + }; +} + +/** + * Timed segments as an asset's transcript: each segment's tokens become its words, + * spread evenly over its span. + * + * `language` is what the engine settled on, or failing that what was asked for, then + * "auto" (very rare: usually a run with no audio). + */ +export function transcriptFromSegments( + assetId: string, + segments: readonly CaptionSegment[], + language: string | null | undefined, +): AxcutTranscript { + const transcriptSegments: AxcutTranscriptSegment[] = []; + const words: AxcutWord[] = []; + + for (const [index, seg] of segments.entries()) { + const tokens = seg.text.trim().split(/\s+/).filter(Boolean); + if (tokens.length === 0) continue; + const segId = `seg_${index + 1}`; + const wordIds: string[] = []; + // Never end before the start: the schema rejects it, and one such word made the whole + // project unreadable (see `raiseInvertedTranscriptEnds`, which repairs old files). + const segEndSec = Math.max(seg.endSec, seg.startSec); + const wordDuration = (segEndSec - seg.startSec) / tokens.length; + for (let w = 0; w < tokens.length; w++) { + const wordId = `word_${words.length + 1}`; + const startSec = seg.startSec + w * wordDuration; + words.push({ + id: wordId, + segmentId: segId, + startSec, + endSec: startSec + wordDuration, + text: tokens[w], + }); + wordIds.push(wordId); + } + transcriptSegments.push({ + id: segId, + kind: "speech" as const, + startSec: seg.startSec, + endSec: segEndSec, + text: seg.text, + wordIds, + }); + } + + return { assetId, language: language || "auto", segments: transcriptSegments, words }; +} diff --git a/src/lib/ai-edition/transcription/status.ts b/src/lib/ai-edition/transcription/status.ts index fe28e88edd..ea317e60e8 100644 --- a/src/lib/ai-edition/transcription/status.ts +++ b/src/lib/ai-edition/transcription/status.ts @@ -126,6 +126,27 @@ export function isPermanentFailure(kind: TranscriptionFailureKind): kind is Pers /** The failure kinds `assetSchema.transcriptionFailure` accepts. */ export type PersistableFailureKind = Exclude; +/** + * The document with a deterministic failure remembered on its asset, so the next open + * shows "no audio" at once instead of re-extracting the audio to find out again. A + * transient failure is not remembered: the document comes back unchanged. + */ +export function withTranscriptionFailure( + document: AxcutDocument, + assetId: string, + failure: TranscriptionFailure, + at = new Date().toISOString(), +): AxcutDocument { + const kind = failure.kind; + if (!isPermanentFailure(kind)) return document; + return { + ...document, + assets: document.assets.map((a) => + a.id === assetId ? { ...a, transcriptionFailure: { kind, message: failure.message, at } } : a, + ), + }; +} + /** * Map an exception out of `transcribeAsset` onto a failure the UI can explain. * The deterministic cases come from whichever layer decoded the audio, and there diff --git a/src/lib/captioning/transcribe.ts b/src/lib/captioning/transcribe.ts index 132c1cea16..806b669141 100644 --- a/src/lib/captioning/transcribe.ts +++ b/src/lib/captioning/transcribe.ts @@ -1,5 +1,6 @@ import type { TrimRegion } from "@/components/video-editor/types"; import type { SttWordSegment } from "../../../electron/stt/transcriptionContract"; +import { segmentsFromStt } from "../ai-edition/transcription/fromStt"; export interface CaptionSegment { startSec: number; @@ -138,30 +139,7 @@ function runTranscription( // quiet music intros / room tone. VAD or nothing. return api .transcribe({ ...payload, language: forcedLanguage, prompt: options?.prompt }) - .then((result) => { - const words = result.wordSegments ?? []; - let segments: CaptionSegment[]; - let granularity: CaptionTimestampGranularity; - if (words.length > 0) { - segments = words.map((w) => ({ - startSec: w.startSec, - endSec: w.endSec, - text: w.word, - })); - granularity = "word"; - } else { - // ponytail: whisper dropped every word for a segment (e.g. OOV - // heavy); fall back to raw phrase spans so the user still gets - // captions to edit. - segments = (result.segments ?? []).map((s) => ({ - startSec: s.startSec, - endSec: s.endSec, - text: s.text, - })); - granularity = "phrase"; - } - return { segments, granularity, detectedLanguage: result.detectedLanguage }; - }) + .then((result) => ({ ...segmentsFromStt(result), detectedLanguage: result.detectedLanguage })) .catch((error: unknown) => { // A run the caller cancelled surfaces as an abort, not as an engine // failure: the store drops it silently instead of toasting the user diff --git a/src/native/contracts.ts b/src/native/contracts.ts index 24b193cf24..7576b2cc73 100644 --- a/src/native/contracts.ts +++ b/src/native/contracts.ts @@ -339,7 +339,24 @@ export interface AiEditionMcpStatus { */ export type AiEditionMcpHostRequest = | { requestId: string; op: "snapshot" } - | { requestId: string; op: "apply"; document: unknown; expectedRevision: number }; + | { requestId: string; op: "apply"; document: unknown; expectedRevision: number } + | { + requestId: string; + op: "transcribe"; + projectId: string; + assetId: string; + /** Absent: keep a transcript the asset has. */ + language?: string; + }; + +/** + * How the editor's own transcription of an asset ended, for the MCP `transcribe` tool. + * `unavailable`: the editor no longer holds that project, or the run was dropped. + */ +export type AiEditionMcpTranscription = + | { status: "transcribed"; language: string; words: number } + | { status: "failed"; failure: { kind: string; message: string } } + | { status: "unavailable" }; export interface AiEditionMcpHostSnapshot { document: unknown; @@ -350,7 +367,7 @@ export type AiEditionMcpApplyResult = "applied" | "conflict" | "save-failed" | " export interface AiEditionMcpHostResponse { requestId: string; - result: AiEditionMcpHostSnapshot | null | AiEditionMcpApplyResult; + result: AiEditionMcpHostSnapshot | null | AiEditionMcpApplyResult | AiEditionMcpTranscription; } /** Editor → main: `true` when an editor starts answering MCP requests, `false` when it stops. */ From f3d83a77a07a4496fe7cac250b3dcc6874f5ea81 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:18:50 +0200 Subject: [PATCH 02/16] feat(mcp): read, style and translate the captions through the agent tools getCaptions, setCaptionSettings (through patchCaptionSettings, as the Captions pane does), getCaptionTranslationUnits, putCaptionTranslation, removeCaptionTranslation and setVocabulary, for the in-app agent and MCP clients alike. The caller translates the captions with its own model; the transcript is never rewritten. getCurrentDocument reports each asset's transcript state and the project vocabulary. Part of #1180 --- electron/ai-edition/agent-tools.test.ts | 4 + electron/ai-edition/agent-tools.ts | 24 ++ electron/ai-edition/caption-tools.test.ts | 236 +++++++++++ electron/ai-edition/caption-tools.ts | 378 ++++++++++++++++++ .../ai-edition/deep-agent/service.test.ts | 13 + electron/ai-edition/deep-agent/service.ts | 40 +- src/components/ai-edition/CaptionsPane.tsx | 6 +- src/lib/ai-edition/captions/index.ts | 2 + src/lib/ai-edition/captions/settings.ts | 4 + 9 files changed, 704 insertions(+), 3 deletions(-) create mode 100644 electron/ai-edition/caption-tools.test.ts create mode 100644 electron/ai-edition/caption-tools.ts diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index 26fd98cb5b..9cfc65449b 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -216,6 +216,10 @@ describe("the mutating-tool table", () => { "setTrim", "setWordText", "setZoom", + "setCaptionSettings", + "putCaptionTranslation", + "removeCaptionTranslation", + "setVocabulary", ].sort(), ); expect(isMutatingTool("getCurrentDocument")).toBe(false); diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 2df9ab4918..3a60dd38d8 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -87,6 +87,7 @@ import { SETTING_BOUNDS } from "../../src/lib/projectDefaults"; import type { SegmentationSupport } from "../../src/native/contracts"; import { annotationFootageRect, zoomScaleLimit } from "../../src/native/sceneDescription"; import { ASPECT_RATIO_PRESETS } from "../../src/utils/aspectRatioUtils"; +import { CAPTION_TOOL_NAMES, executeCaptionTool, MUTATING_CAPTION_TOOLS } from "./caption-tools"; import { editorSettingsSnapshot, executeSetEditorSettings } from "./editor-settings-tool"; import { measureTextInShippedFonts } from "./text-measure"; @@ -1068,6 +1069,7 @@ export const OPENSCREEN_TOOL_NAMES = [ "removeClip", "setAspectRatio", "setEditorSettings", + ...CAPTION_TOOL_NAMES, ] as const; /** @@ -1145,6 +1147,7 @@ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ "removeClip", "setAspectRatio", "setEditorSettings", + ...MUTATING_CAPTION_TOOLS, ]); export function isMutatingTool(name: string): boolean { @@ -1323,6 +1326,17 @@ export function documentSnapshotForModel( // probe threw. Defaulting that to `false` would put our failure back in // the answer as their fact, one layer lower down. hasCursorTelemetry: availability?.[a.id] ?? null, + // Whether getTranscriptWords can read it; transcribe makes one when it is false. + hasTranscript: document.transcripts.some((t) => t.assetId === a.id), + // A verdict a past run left on the media ("no-audio", "unsupported-audio"). + ...(a.transcriptionFailure + ? { + transcriptionFailure: { + kind: a.transcriptionFailure.kind, + message: a.transcriptionFailure.message, + }, + } + : {}), })), // ponytail: `index`, `reason` and `origin` are here because without them a // clip cannot be DESIGNATED. "Put the demo first" is unanswerable when the @@ -1413,6 +1427,8 @@ export function documentSnapshotForModel( fadeOutMs: t.fadeOutMs, })), hasTranscript: document.transcripts.length > 0 || document.transcript !== null, + // The names and terms Whisper is primed with on every transcription (setVocabulary). + vocabulary: document.project.vocabulary ?? null, }; } @@ -1783,6 +1799,14 @@ export function executeAgentTool( } switch (name) { + case "getCaptions": + case "setCaptionSettings": + case "getCaptionTranslationUnits": + case "putCaptionTranslation": + case "removeCaptionTranslation": + case "setVocabulary": + return executeCaptionTool(document, name, args); + case "setAspectRatio": { // The format alone, kept for clients that know it: setEditorSettings does the work. const parsed = setAspectRatioArgs.safeParse(args); diff --git a/electron/ai-edition/caption-tools.test.ts b/electron/ai-edition/caption-tools.test.ts new file mode 100644 index 0000000000..4e96e447a9 --- /dev/null +++ b/electron/ai-edition/caption-tools.test.ts @@ -0,0 +1,236 @@ +// The caption tools end to end on a document: turn them on, style and place them, +// translate them as a client would, and check the export draws what was asked. + +import { describe, expect, it } from "vitest"; +import { CAPTION_STYLES, getCaptionSettings } from "../../src/lib/ai-edition/captions"; +import { + type AxcutDocument, + createEmptyDocument, + documentSchema, +} from "../../src/lib/ai-edition/schema"; +import { buildSceneDescription } from "../../src/native/sceneDescription"; +import { type AgentToolExecution, executeAgentTool } from "./agent-tools"; + +function fixtureDocument(): AxcutDocument { + const base = createEmptyDocument({ + title: "Captions", + projectId: "proj_1", + createdAt: "2026-01-01T00:00:00.000Z", + }); + const words = [ + ["w1", "Hello", 0, 0.4], + ["w2", "there.", 0.4, 0.8], + ["w3", "Second", 2, 2.4], + ["w4", "phrase.", 2.4, 2.8], + ] as const; + return documentSchema.parse({ + ...base, + project: { ...base.project, primaryAssetId: "asset_1" }, + assets: [ + { id: "asset_1", kind: "video", label: "Rec", originalPath: "/tmp/rec.mp4", durationSec: 10 }, + ], + // One segment per word, as Whisper's transcripts are stored. + transcripts: [ + { + assetId: "asset_1", + language: "en", + segments: words.map(([id, text, startSec, endSec]) => ({ + id: `seg_${id}`, + kind: "speech", + startSec, + endSec, + text, + wordIds: [id], + })), + words: words.map(([id, text, startSec, endSec]) => ({ + id, + segmentId: `seg_${id}`, + startSec, + endSec, + text, + })), + }, + ], + timeline: { + ...base.timeline, + clips: [ + { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + wordRefs: [], + origin: "user", + reason: "", + }, + ], + }, + }); +} + +function call(document: AxcutDocument, name: string, args: unknown = {}) { + const execution: AgentToolExecution = executeAgentTool(document, name, JSON.stringify(args)); + return { ...execution, result: JSON.parse(execution.resultJson) as Record }; +} + +function captionTexts(document: AxcutDocument): string[] { + return buildSceneDescription(document).annotations.map((a) => a.text?.content ?? ""); +} + +describe("setCaptionSettings", () => { + it("turns the captions on, and the export draws the transcript", () => { + const doc = fixtureDocument(); + expect(captionTexts(doc)).toEqual([]); + const on = call(doc, "setCaptionSettings", { enabled: true }); + expect(on.ok).toBe(true); + expect(on.document && captionTexts(on.document).join(" ")).toContain("Hello there."); + }); + + it("applies a style, then a plate, then the explicit fields over both", () => { + const doc = fixtureDocument(); + const set = call(doc, "setCaptionSettings", { + style: "bold", + plate: "light", + fontSize: 72, + anchorV: "top", + insetY: 8, + minWordsPerLine: 3, + maxWordsPerLine: 5, + }); + const settings = getCaptionSettings(set.document); + expect(settings.fontFamily).toBe(CAPTION_STYLES.bold.fontFamily); + expect(settings.fontSize).toBe(72); + expect(settings.backgroundColor).toBe("#ffffff"); + // The bold style's yellow would vanish on a light plate: the plate recoloured it. + expect(settings.color).not.toBe(CAPTION_STYLES.bold.color); + expect(settings).toMatchObject({ anchorV: "top", insetY: 8, minWordsPerLine: 3 }); + expect(set.result.plate).toBe("light"); + }); + + it("refuses what the pane cannot express, and changes nothing", () => { + const doc = fixtureDocument(); + expect(call(doc, "setCaptionSettings", {}).ok).toBe(false); + expect(call(doc, "setCaptionSettings", { minWordsPerLine: 9, maxWordsPerLine: 3 }).ok).toBe( + false, + ); + expect(call(doc, "setCaptionSettings", { fontSize: 500 }).ok).toBe(false); + expect(call(doc, "setCaptionSettings", { fontFamily: "Comic Sans" }).ok).toBe(false); + const missing = call(doc, "setCaptionSettings", { language: "fr" }); + expect(missing.ok).toBe(false); + expect(String(missing.result.error)).toContain("putCaptionTranslation"); + }); + + it("reports the lane the captions really read", () => { + const set = call(fixtureDocument(), "setCaptionSettings", { lane: "voiceover" }); + expect(set.result.resolvedLane).toBe("recording"); + expect(set.result.laneNote).toBeDefined(); + }); +}); + +describe("getCaptions", () => { + it("reads the settings back, warns when there is nothing to caption", () => { + const doc = call(fixtureDocument(), "setCaptionSettings", { enabled: true }).document; + if (!doc) throw new Error("not applied"); + const read = call(doc, "getCaptions").result; + expect(read).toMatchObject({ style: "classic", plate: "dark", resolvedLane: "recording" }); + expect(read.warning).toBeUndefined(); + const silent = call({ ...doc, transcripts: [], transcript: null }, "getCaptions").result; + expect(silent.warning).toBeDefined(); + }); +}); + +describe("translating the captions", () => { + it("reads phrase units, stores the client's translation, and the export shows it", () => { + let doc = fixtureDocument(); + const units = call(doc, "getCaptionTranslationUnits", { language: "fr" }).result + .assets as Array<{ + units: Array<{ id: string; text: string }>; + }>; + // Phrases, not words: one unit per sentence. + expect(units[0]?.units.map((u) => u.text)).toEqual(["Hello there.", "Second phrase."]); + const [first, second] = units[0]?.units ?? []; + + const put = call(doc, "putCaptionTranslation", { + language: "fr", + label: "Français", + assetId: "asset_1", + units: [{ id: first.id, text: "Bonjour." }], + }); + expect(put.result).toMatchObject({ + written: 1, + coverage: { translated: 1, total: 2 }, + shown: false, + }); + doc = put.document ?? doc; + + // Only what is left to translate comes back. + const left = call(doc, "getCaptionTranslationUnits", { language: "fr" }).result + .assets as Array<{ + units: Array<{ id: string }>; + }>; + expect(left[0]?.units.map((u) => u.id)).toEqual([second.id]); + + doc = call(doc, "setCaptionSettings", { enabled: true, language: "fr" }).document ?? doc; + expect(captionTexts(doc).join(" ")).toContain("Bonjour."); + // The transcript itself is untouched. + expect(doc.transcripts[0]?.words[0]?.text).toBe("Hello"); + + const read = call(doc, "getCaptions").result.translations as Array>; + expect(read).toEqual([ + expect.objectContaining({ + language: "fr", + label: "Français", + coverage: [{ assetId: "asset_1", translated: 1, total: 2 }], + }), + ]); + }); + + it("refuses unit ids it does not know, and nothing is written", () => { + const put = call(fixtureDocument(), "putCaptionTranslation", { + language: "fr", + assetId: "asset_1", + units: [{ id: "seg_w1", text: "Bonjour" }], + }); + expect(put.ok).toBe(false); + expect(put.document).toBeUndefined(); + const twice = call(fixtureDocument(), "putCaptionTranslation", { + language: "fr", + assetId: "asset_1", + units: [ + { id: "u:seg_w1", text: "Bonjour." }, + { id: "u:seg_w1", text: "Salut." }, + ], + }); + expect(twice.ok).toBe(false); + }); + + it("removing the language on screen puts the original back", () => { + let doc = fixtureDocument(); + doc = + call(doc, "putCaptionTranslation", { + language: "fr", + assetId: "asset_1", + units: [{ id: "u:seg_w1", text: "Bonjour." }], + }).document ?? doc; + doc = call(doc, "setCaptionSettings", { language: "fr" }).document ?? doc; + const removed = call(doc, "removeCaptionTranslation", { language: "fr" }); + expect(removed.result.shownLanguage).toBeNull(); + expect(call(removed.document ?? doc, "removeCaptionTranslation", { language: "fr" }).ok).toBe( + false, + ); + }); +}); + +describe("setVocabulary", () => { + it("stores the names Whisper is primed with, and getCurrentDocument reads them", () => { + const set = call(fixtureDocument(), "setVocabulary", { vocabulary: " OpenScreen, Vulkan " }); + expect(set.document?.project.vocabulary).toBe("OpenScreen, Vulkan"); + const read = set.document && call(set.document, "getCurrentDocument").result; + expect(read?.vocabulary).toBe("OpenScreen, Vulkan"); + expect(call(fixtureDocument(), "setVocabulary", { vocabulary: "x".repeat(601) }).ok).toBe( + false, + ); + }); +}); diff --git a/electron/ai-edition/caption-tools.ts b/electron/ai-edition/caption-tools.ts new file mode 100644 index 0000000000..5053fa125a --- /dev/null +++ b/electron/ai-edition/caption-tools.ts @@ -0,0 +1,378 @@ +// The caption tools: how the captions look and where they sit, which language they show, +// and the translation layers behind that choice. Pure, like the rest of the executor +// (`agent-tools.ts` routes these names here): no IPC, no fs. +// +// The writes are the Captions pane's own (`useCaptions`): `patchCaptionSettings` at the +// document's output aspect, `putCaptionTranslation`, `removeCaptionTranslation`. The +// caller translates by itself: an MCP client and the in-app agent are both language +// models, so neither needs the app's own provider for it. The transcript is never touched. + +import { z } from "zod"; +import { + CAPTION_FONT_SIZE_MAX, + CAPTION_FONT_SIZE_MIN, + CAPTION_INSET_X_MAX, + CAPTION_INSET_Y_MAX, + CAPTION_PLATE_OPACITY_MAX, + CAPTION_PLATE_OPACITY_MIN, + CAPTION_STYLES, + type CaptionSettings, + type CaptionSettingsPatch, + type CaptionStyleId, + captionPlateOf, + captionPlatePatch, + captionStyleOf, + captionTranslationUnits, + getCaptionSettings, + getCaptionTranslations, + patchCaptionSettings, + putCaptionTranslation, + removeCaptionTranslation, + translationCoverage, +} from "../../src/lib/ai-edition/captions"; +import { resolveCaptionLane } from "../../src/lib/ai-edition/captions/settings"; +import { + CAPTION_WORDS_PER_LINE_MAX, + CAPTION_WORDS_PER_LINE_MIN, +} from "../../src/lib/ai-edition/captions/wordsPerLine"; +import { resolveAspectRatioValue } from "../../src/lib/ai-edition/document/outputFormat"; +import { VOCABULARY_MAX_CHARS, withVocabulary } from "../../src/lib/ai-edition/document/transcript"; +import type { AxcutDocument, AxcutTranscript } from "../../src/lib/ai-edition/schema"; +import { getEditorSettings } from "../../src/lib/ai-edition/store/editorSettings"; +import { lanePlacements } from "../../src/lib/ai-edition/timeline/aggregated-transcript"; +import { transcriptHasSpeech } from "../../src/lib/ai-edition/transcription/status"; +import { TEXT_FONT_FAMILIES } from "../../src/lib/textFonts"; +import type { AgentToolExecution } from "./agent-tools"; + +const STYLE_IDS = Object.keys(CAPTION_STYLES) as [CaptionStyleId, ...CaptionStyleId[]]; +const hexColor = z.string().regex(/^#[0-9a-fA-F]{6}$/, "a #rrggbb colour"); +const wordsPerLine = z + .number() + .int() + .min(CAPTION_WORDS_PER_LINE_MIN) + .max(CAPTION_WORDS_PER_LINE_MAX); +// A BCP 47 tag ("fr", "pt-BR"): it is the layer's storage key. +const languageCode = z + .string() + .regex(/^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})*$/, "a language code such as fr or pt-BR"); + +export const getCaptionsArgs = z.object({}); + +export const setCaptionSettingsArgs = z.object({ + enabled: z.boolean().optional(), + style: z.enum(STYLE_IDS).optional(), + fontFamily: z.enum(TEXT_FONT_FAMILIES).optional(), + fontWeight: z.enum(["normal", "bold"]).optional(), + fontSize: z.number().min(CAPTION_FONT_SIZE_MIN).max(CAPTION_FONT_SIZE_MAX).optional(), + color: hexColor.optional(), + plate: z.enum(["none", "dark", "light"]).optional(), + plateOpacity: z.number().min(CAPTION_PLATE_OPACITY_MIN).max(CAPTION_PLATE_OPACITY_MAX).optional(), + anchorV: z.enum(["bottom", "top"]).optional(), + anchorH: z.enum(["left", "center", "right"]).optional(), + insetY: z.number().min(0).max(CAPTION_INSET_Y_MAX).optional(), + insetX: z.number().min(0).max(CAPTION_INSET_X_MAX).optional(), + minWordsPerLine: wordsPerLine.optional(), + maxWordsPerLine: wordsPerLine.optional(), + lane: z.enum(["recording", "voiceover"]).optional(), + language: languageCode.nullable().optional(), +}); + +export const getCaptionTranslationUnitsArgs = z.object({ + language: languageCode, + assetId: z.string().min(1).optional(), + includeTranslated: z.boolean().optional(), +}); + +export const putCaptionTranslationArgs = z.object({ + language: languageCode, + label: z.string().trim().min(1).max(40).optional(), + assetId: z.string().min(1), + units: z.array(z.object({ id: z.string().min(1), text: z.string().trim().min(1) })).min(1), +}); + +export const removeCaptionTranslationArgs = z.object({ language: languageCode }); + +export const setVocabularyArgs = z.object({ vocabulary: z.string().max(VOCABULARY_MAX_CHARS) }); + +export const CAPTION_TOOL_NAMES = [ + "getCaptions", + "setCaptionSettings", + "getCaptionTranslationUnits", + "putCaptionTranslation", + "removeCaptionTranslation", + "setVocabulary", +] as const; + +export const MUTATING_CAPTION_TOOLS = [ + "setCaptionSettings", + "putCaptionTranslation", + "removeCaptionTranslation", + "setVocabulary", +] as const; + +function failure(message: string): AgentToolExecution { + return { ok: false, resultJson: JSON.stringify({ error: message }) }; +} + +/** The output aspect, as the Captions pane reads it: it sets the default insets. */ +function aspectOf(document: AxcutDocument): number { + return resolveAspectRatioValue(document, getEditorSettings(document).aspectRatio); +} + +/** The transcripts the captions are read from: those of the media on the resolved lane. */ +function captionedTranscripts( + document: AxcutDocument, + settings: CaptionSettings, +): AxcutTranscript[] { + const placed = new Set( + lanePlacements( + resolveCaptionLane(document, settings), + document.timeline.clips, + document.audioTracks ?? [], + ).map((placement) => placement.assetId), + ); + return document.transcripts.filter((t) => placed.has(t.assetId)); +} + +/** Everything a caller needs to read the captions back, in the editor's own terms. */ +function captionsReport(document: AxcutDocument): Record { + const settings = getCaptionSettings(document, aspectOf(document)); + const transcripts = captionedTranscripts(document, settings); + const translations = getCaptionTranslations(document); + const nothingToCaption = transcripts.every((t) => !transcriptHasSpeech(t)); + return { + settings, + style: captionStyleOf(settings) ?? "custom", + plate: captionPlateOf(settings), + resolvedLane: resolveCaptionLane(document, settings), + captionedAssets: transcripts.map((t) => ({ + assetId: t.assetId, + language: t.language, + words: t.words.length, + })), + ...(settings.enabled && nothingToCaption + ? { + warning: + "Captions are on, but no transcript with speech is on their lane, so nothing shows: the media needs a transcript first (assets[].hasTranscript in getCurrentDocument).", + } + : {}), + translations: Object.values(translations).map((layer) => ({ + language: layer.language, + label: layer.label, + updatedAt: layer.updatedAt, + coverage: transcripts.map((t) => ({ + assetId: t.assetId, + ...translationCoverage(t, translations, layer.language), + })), + })), + }; +} + +function hasAnyField(data: Record): boolean { + return Object.values(data).some((value) => value !== undefined); +} + +export function executeCaptionTool( + document: AxcutDocument, + name: (typeof CAPTION_TOOL_NAMES)[number], + args: unknown, +): AgentToolExecution { + switch (name) { + case "getCaptions": + return { + ok: true, + resultJson: JSON.stringify({ + ...captionsReport(document), + choices: { + styles: STYLE_IDS, + fonts: TEXT_FONT_FAMILIES, + plates: ["none", "dark", "light"], + }, + }), + }; + + case "setCaptionSettings": { + const parsed = setCaptionSettingsArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { style, plate, plateOpacity, lane, ...fields } = parsed.data; + if (!hasAnyField(parsed.data)) return failure("Pass at least one setting to change."); + const aspect = aspectOf(document); + const current = getCaptionSettings(document, aspect); + if (fields.language) { + const layers = Object.keys(getCaptionTranslations(document)); + if (!layers.includes(fields.language)) { + return failure( + `No "${fields.language}" translation to show. ${layers.length ? `This project has: ${layers.join(", ")}.` : "This project has none."} Write one with putCaptionTranslation first, or pass null for the transcript's own language.`, + ); + } + } + // The pane's order: a style sets the look as a whole, a plate then recolours the + // text that would vanish on it, and an explicit field wins over both. + const patch: CaptionSettingsPatch = { ...(style ? CAPTION_STYLES[style] : {}) }; + if (plate) Object.assign(patch, captionPlatePatch({ ...current, ...patch }, plate)); + for (const [key, value] of Object.entries(fields)) { + if (value !== undefined) Object.assign(patch, { [key]: value }); + } + if (plateOpacity !== undefined) patch.backgroundOpacity = plateOpacity; + if (lane !== undefined) patch.captionLane = lane; + const min = patch.minWordsPerLine ?? current.minWordsPerLine; + const max = patch.maxWordsPerLine ?? current.maxWordsPerLine; + if (min > max) { + return failure( + `minWordsPerLine (${min}) cannot exceed maxWordsPerLine (${max}). Nothing was modified.`, + ); + } + const next = patchCaptionSettings(document, patch, aspect); + const report = captionsReport(next); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + settings: report.settings, + style: report.style, + plate: report.plate, + resolvedLane: report.resolvedLane, + ...(lane && lane !== report.resolvedLane + ? { + laneNote: `The captions read the ${report.resolvedLane} lane: the project has no voiceover track to caption.`, + } + : {}), + ...(report.warning ? { warning: report.warning } : {}), + }), + summary: + fields.enabled === true + ? "turned the captions on" + : fields.enabled === false + ? "turned the captions off" + : "restyled the captions", + }; + } + + case "getCaptionTranslationUnits": { + const parsed = getCaptionTranslationUnitsArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { language, assetId, includeTranslated } = parsed.data; + const settings = getCaptionSettings(document, aspectOf(document)); + const transcripts = assetId + ? document.transcripts.filter((t) => t.assetId === assetId) + : captionedTranscripts(document, settings); + if (transcripts.length === 0) { + return failure( + assetId + ? `Asset ${assetId} has no transcript to translate yet.` + : "No transcript is on the captions' lane to translate yet.", + ); + } + const translations = getCaptionTranslations(document); + return { + ok: true, + resultJson: JSON.stringify({ + language, + assets: transcripts.map((t) => { + const done = translations[language]?.byAsset[t.assetId] ?? {}; + const units = captionTranslationUnits(t) + .filter((unit) => includeTranslated || !done[unit.id]?.trim()) + .map((unit) => ({ + id: unit.id, + startSec: Math.round(unit.startSec * 1000) / 1000, + endSec: Math.round(unit.endSec * 1000) / 1000, + text: unit.text, + ...(done[unit.id] ? { translation: done[unit.id] } : {}), + })); + return { + assetId: t.assetId, + sourceLanguage: t.language, + coverage: translationCoverage(t, translations, language), + units, + }; + }), + }), + }; + } + + case "putCaptionTranslation": { + const parsed = putCaptionTranslationArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { language, label, assetId, units } = parsed.data; + const transcript = document.transcripts.find((t) => t.assetId === assetId); + if (!transcript) return failure(`Asset ${assetId} has no transcript to translate.`); + const ids = units.map((unit) => unit.id); + const known = new Set(captionTranslationUnits(transcript).map((unit) => unit.id)); + const unknown = ids.filter((id) => !known.has(id)); + if (unknown.length > 0) { + return failure( + `Unknown unit id(s) for asset ${assetId}: ${unknown.slice(0, 5).join(", ")}${unknown.length > 5 ? ` and ${unknown.length - 5} more` : ""}. Take the ids from getCaptionTranslationUnits. Nothing was modified.`, + ); + } + if (new Set(ids).size !== ids.length) { + return failure("A unit id is given twice. Nothing was modified."); + } + const next = putCaptionTranslation(document, { + language, + label: label ?? getCaptionTranslations(document)[language]?.label ?? language, + assetId, + segments: Object.fromEntries(units.map((unit) => [unit.id, unit.text])), + }); + const shown = getCaptionSettings(next, aspectOf(next)).language === language; + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + language, + assetId, + written: ids.length, + coverage: translationCoverage(transcript, getCaptionTranslations(next), language), + shown, + ...(shown + ? {} + : { + next: "The captions still show another language: setCaptionSettings with this language shows it.", + }), + }), + summary: `translated ${ids.length} caption unit${ids.length === 1 ? "" : "s"} into ${language}`, + }; + } + + case "removeCaptionTranslation": { + const parsed = removeCaptionTranslationArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { language } = parsed.data; + const layers = Object.keys(getCaptionTranslations(document)); + if (!layers.includes(language)) { + return failure( + `No "${language}" translation to remove. ${layers.length ? `This project has: ${layers.join(", ")}.` : "This project has none."}`, + ); + } + // As the pane deletes one: captions showing it fall back to the transcript's language. + const aspect = aspectOf(document); + const cleared = removeCaptionTranslation(document, language); + const next = + getCaptionSettings(cleared, aspect).language === language + ? patchCaptionSettings(cleared, { language: null }, aspect) + : cleared; + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + removed: language, + shownLanguage: getCaptionSettings(next, aspect).language, + }), + summary: `removed the ${language} caption translation`, + }; + } + + case "setVocabulary": { + const parsed = setVocabularyArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const next = withVocabulary(document, parsed.data.vocabulary); + const vocabulary = (next ?? document).project.vocabulary ?? ""; + const resultJson = JSON.stringify({ + vocabulary, + note: "Whisper reads it on the next transcription. An existing transcript is unchanged: transcribe with a language redoes it.", + }); + return next + ? { ok: true, document: next, resultJson, summary: "set the project vocabulary" } + : { ok: true, resultJson }; + } + } +} diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 30d9734a70..8fa62ea514 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -92,6 +92,17 @@ const ARGS: Record = { removeClip: { clipId: "clip_1" }, setAspectRatio: { aspectRatio: "16:9" }, setEditorSettings: { padding: 30 }, + getCaptions: {}, + setCaptionSettings: { enabled: true, style: "bold" }, + getCaptionTranslationUnits: { language: "fr" }, + putCaptionTranslation: { + language: "fr", + assetId: "asset_1", + units: [{ id: "u:seg_1", text: "Bonjour" }], + }, + // The fixture has no translation layer, so this one is refused. + removeCaptionTranslation: { language: "fr" }, + setVocabulary: { vocabulary: "OpenScreen, Whisper" }, }; /** ponytail: the fixture starts from a FIXED instant. `createEmptyDocument` reads @@ -378,6 +389,8 @@ describe("one description of the tools, not two", () => { "getTranscript", "getTranscriptWords", "getCursorTrack", + "getCaptions", + "getCaptionTranslationUnits", ]); }); }); diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index e003a7460c..44571c2dd9 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -20,7 +20,20 @@ import { anthropicPromptCachingMiddleware, createAgent, tool } from "langchain"; import { z } from "zod"; import type { CursorTelemetryPoint } from "../../../src/components/video-editor/types"; +import { + CAPTION_FONT_SIZE_MAX, + CAPTION_FONT_SIZE_MIN, + CAPTION_INSET_X_MAX, + CAPTION_INSET_Y_MAX, + CAPTION_PLATE_OPACITY_MAX, + CAPTION_PLATE_OPACITY_MIN, +} from "../../../src/lib/ai-edition/captions"; +import { + CAPTION_WORDS_PER_LINE_MAX, + CAPTION_WORDS_PER_LINE_MIN, +} from "../../../src/lib/ai-edition/captions/wordsPerLine"; import { AUDIO_FADE_MAX_MS } from "../../../src/lib/ai-edition/document/audioTracks"; +import { VOCABULARY_MAX_CHARS } from "../../../src/lib/ai-edition/document/transcript"; import type { AxcutDocument } from "../../../src/lib/ai-edition/schema"; import { collectAutoZoomSuggestionsForDocument } from "../../../src/lib/ai-edition/timeline/apply-auto-zooms"; // ponytail: the legend is DERIVED from the depth→scale table, not retyped. Both @@ -72,6 +85,14 @@ import { setWordTextArgs, setZoomArgs, } from "../agent-tools"; +import { + getCaptionsArgs, + getCaptionTranslationUnitsArgs, + putCaptionTranslationArgs, + removeCaptionTranslationArgs, + setCaptionSettingsArgs, + setVocabularyArgs, +} from "../caption-tools"; import { setEditorSettingsArgs } from "../editor-settings-tool"; import { createOpenScreenChatModel, @@ -113,7 +134,7 @@ const consentPromptBlock = (setting: string) => "", "PROJECT EDITS ARE CURRENTLY DISABLED by the user, who asked to be consulted before the timeline changes.", "- Read freely: getCurrentDocument and getTranscript work as usual.", - "- Do NOT call any tool that writes (removeWords, addTrim, setTrim, setClipRange, moveClip, replaceTimeline, add*/set* effects, remove*). Every one of them will be refused, so calling them wastes the turn and tells the user nothing.", + "- Do NOT call any tool that writes (removeWords, addTrim, setTrim, setClipRange, moveClip, replaceTimeline, add*/set* effects and settings, putCaptionTranslation, remove*). Every one of them will be refused, so calling them wastes the turn and tells the user nothing.", "- Instead: say precisely what you would change — which tool, which times, which ids — and ask the user to confirm. Be specific enough that they can say yes to it.", `- Never state or imply that an edit was applied. If the user confirms and you are still refused, tell them to turn on ${setting} first.`, ].join("\n"); @@ -145,6 +166,7 @@ const BASE_SYSTEM_PROMPT = [ "- A zoom animates in just before its span and out just after it (zoomInFromSec, zoomOutUntilSec in getCurrentDocument). Leave those windows untrimmed, or the export jumps at the cut.", "- addAudio lays an imported voiceover or music file over a span. It plays an asset the project already has (kind 'audio'); importing or recording one is the editor's job, not a tool you have — so when the project has none, say so rather than naming an id that does not exist.", "- moveClip changes the order of placed clips, one call per clip that moves, preserving ids, source ranges, trims and anchored effects. replaceTimeline rebuilds the timeline from kept intervals and sorts them, so it cannot reorder anything.", + "- Captions are the transcript drawn on the video, never typed in: setCaptionSettings turns them on, styles and places them, getCaptions reads them back. To caption in another language, read the phrases with getCaptionTranslationUnits, translate them yourself, store them with putCaptionTranslation, then show that language with setCaptionSettings. A translation never rewrites the transcript. For text the speaker did not say, use addAnnotation.", "- Deleting is a first-class action, not a workaround: removeTrim, removeModifier, removeClip. Never fake a deletion by re-adding an element or zeroing it out (span 0, speed 1×) — that leaves it in the document and misreports what you did.", "If nothing in the list does what was asked, say so; do not approximate it with a bigger tool.", "", @@ -244,6 +266,16 @@ export const TOOL_DESCRIPTIONS: Record = { "Delete a modifier (zoom / speed / annotation / camera-fullscreen / audio) by id; the kind is resolved from the id. Touching rows with the same styling render as one pill and are deleted together, so read removedIds in the result for the complete set that disappeared. This is how you 'remove'/'delete' one — never neutralise it (span 0, speed 1×), which leaves it in the document. For a trim use removeTrim; for a clip use removeClip.", removeClip: "Delete a placed clip by id; remaining clips close the gap and effects anchored to it are dropped. Use only when the user asks to remove a clip — to shorten one, use setClipRange.", + getCaptions: + "Read the captions: whether they show (enabled), how they look (font, weight, size, colour, plate and its opacity), where they sit (anchorV/insetY, anchorH/insetX), how many words show at once, the lane they read (resolvedLane: the recording's speech, or the voiceover tracks'), and the language they show (null: the transcript's own words). captionedAssets lists the transcripts they are drawn from, and `warning` says when they are on with nothing to show. translations lists each language layer with its coverage per asset (translated of total phrase units). choices lists what setCaptionSettings accepts for style, fontFamily and plate.", + setCaptionSettings: `Turn the captions on or off, and set how they look and where they sit, as the editor's Captions pane does. Only the fields you pass change. \`style\` applies a named look first (font, weight, size, colour and plate together); \`plate\` then picks what is behind the text (none, dark, light) and recolours text that would vanish on it; any other field you pass wins over both. fontSize is px at a 1080-high frame (${CAPTION_FONT_SIZE_MIN}–${CAPTION_FONT_SIZE_MAX}), color a #rrggbb hex, plateOpacity ${CAPTION_PLATE_OPACITY_MIN}–${CAPTION_PLATE_OPACITY_MAX}. anchorV pins the caption block to the bottom or top edge, and insetY is its distance from that edge in % of the frame height (0–${CAPTION_INSET_Y_MAX}); anchorH pins it left, centre or right, and insetX (0–${CAPTION_INSET_X_MAX} % of the frame width) is ignored when it is centred. minWordsPerLine and maxWordsPerLine (${CAPTION_WORDS_PER_LINE_MIN}–${CAPTION_WORDS_PER_LINE_MAX}) bound how many words show at once. lane picks whose speech is captioned: the recording's, or the voiceover tracks' (it falls back to the recording when there is none, and resolvedLane says so). language shows a translation layer written with putCaptionTranslation; null shows the transcript's own words. Captions are drawn from the transcript, so they need one; enabled captions are burnt into the export.`, + getCaptionTranslationUnits: + "Read what to translate for one caption language: per asset the captions read (or the one assetId names), the transcript grouped into phrase units, each with its id, its span in the asset's source seconds and its text. A unit is a sentence or a run of speech between pauses, never a single word, so translate each one as a whole. By default only the units with no text yet in that language, so a translation sent in batches resumes where it stopped; includeTranslated returns the done ones too, with their translation. Translate them yourself, then store them with putCaptionTranslation.", + putCaptionTranslation: + "Store your translation of caption units for one asset in a language layer: `units` lists each unit as { id, text }: its id from getCaptionTranslationUnits and your translation of it. It merges into the layer, which the first call creates, so send long transcripts in batches. language is the layer's code (fr, pt-BR); label is the name the editor's language picker shows, such as Français (default: the code). Unknown unit ids refuse the whole call. The transcript is never changed: this is a separate layer the captions can show. It does not switch the captions to it: setCaptionSettings with this language does.", + removeCaptionTranslation: + "Delete one caption language layer. The transcript is untouched, and captions that showed this language go back to the transcript's own.", + setVocabulary: `Set the project's vocabulary: the names and technical terms Whisper is primed with on every transcription of this project, so it spells them right (at most ${VOCABULARY_MAX_CHARS} characters, read as a plain list such as "OpenScreen, Vulkan, whisper.cpp"). An empty string clears it. It changes no existing transcript: transcribe the asset again for that. getCurrentDocument reports it as vocabulary.`, }; // ponytail: mutable document holder so a write-tool that updates the snapshot @@ -479,6 +511,12 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray = [ ["removeClip", removeClipArgs], ["setAspectRatio", setAspectRatioArgs], ["setEditorSettings", setEditorSettingsArgs], + ["getCaptions", getCaptionsArgs], + ["setCaptionSettings", setCaptionSettingsArgs], + ["getCaptionTranslationUnits", getCaptionTranslationUnitsArgs], + ["putCaptionTranslation", putCaptionTranslationArgs], + ["removeCaptionTranslation", removeCaptionTranslationArgs], + ["setVocabulary", setVocabularyArgs], ]; /** diff --git a/src/components/ai-edition/CaptionsPane.tsx b/src/components/ai-edition/CaptionsPane.tsx index ba8710b7d2..d34aed06b1 100644 --- a/src/components/ai-edition/CaptionsPane.tsx +++ b/src/components/ai-edition/CaptionsPane.tsx @@ -20,6 +20,8 @@ import type { CaptionStyleId, } from "@/lib/ai-edition/captions"; import { + CAPTION_FONT_SIZE_MAX, + CAPTION_FONT_SIZE_MIN, CAPTION_INSET_X_MAX, CAPTION_INSET_Y_MAX, CAPTION_PLATE_OPACITY_MAX, @@ -526,8 +528,8 @@ export function CaptionsPane({ onClose }: { onClose?: () => void } = {}) { Date: Tue, 13 Oct 2026 15:18:53 +0200 Subject: [PATCH 03/16] feat(cli): openscreen captions writes a transcript and turns captions on It used to write v1 auto-caption annotations, which the Captions pane offers to clear. It now lands a transcript through the editor's transcribeAsset (or keeps the one the project has), clears those annotations and enables the editor's captions with --min-words / --max-words as the words per line. Part of #1180 --- electron/cli/args.ts | 2 +- src/cli/CliCaptionsRunner.tsx | 169 +++++++---------------------- src/cli/CliExportRunner.tsx | 161 ++------------------------- src/cli/captionAnnotations.test.ts | 53 --------- src/cli/captionAnnotations.ts | 84 -------------- src/cli/captionProject.test.ts | 108 ++++++++++++++++++ src/cli/captionProject.ts | 59 ++++++++++ src/cli/loadCliProject.ts | 166 ++++++++++++++++++++++++++++ src/cli/vendor/leadingSilence.ts | 80 -------------- src/lib/cliContracts.ts | 2 +- 10 files changed, 381 insertions(+), 503 deletions(-) delete mode 100644 src/cli/captionAnnotations.test.ts delete mode 100644 src/cli/captionAnnotations.ts create mode 100644 src/cli/captionProject.test.ts create mode 100644 src/cli/captionProject.ts create mode 100644 src/cli/loadCliProject.ts delete mode 100644 src/cli/vendor/leadingSilence.ts diff --git a/electron/cli/args.ts b/electron/cli/args.ts index 9b621b6607..6c13230108 100644 --- a/electron/cli/args.ts +++ b/electron/cli/args.ts @@ -84,7 +84,7 @@ Usage: openscreen record [options] Record the screen headlessly openscreen sources [--json] [-o ] List displays, windows, microphones and cameras openscreen pack --out Copy project + media into one portable folder - openscreen captions Add auto-captions (on-device Whisper) to a project + openscreen captions Transcribe (on-device Whisper) and turn captions on [--min-words <1-12>] [--max-words <1-12>] openscreen info [--json] Inspect a project file openscreen mcp Serve the agent tools over MCP on stdio diff --git a/src/cli/CliCaptionsRunner.tsx b/src/cli/CliCaptionsRunner.tsx index a87f729e2e..0466374409 100644 --- a/src/cli/CliCaptionsRunner.tsx +++ b/src/cli/CliCaptionsRunner.tsx @@ -1,149 +1,58 @@ -// Hidden-window runner for `openscreen captions`: transcribes the project's -// audio with the on-device Whisper worker and writes the resulting caption -// annotations back into the project. Mirrors VideoEditor.generateAutoCaptions. +// Hidden-window runner for `openscreen captions`: transcribes the project's primary media +// with the editor's own on-device Whisper path (`transcribeAsset`) and turns its captions +// on (`captionProject.ts`). The main process writes the project back. -import { useEffect, useRef, useState } from "react"; -import { - normalizeProjectEditor, - resolveProjectMedia, - toFileUrl, - validateProjectData, -} from "@/components/video-editor/projectPersistence"; -import type { AnnotationRegion, TrimRegion } from "@/components/video-editor/types"; -import { extractMono16kFromVideoUrl } from "@/lib/captioning/extractMono16k"; -import { type SttRendererStatus, transcribeMono16kToSegments } from "@/lib/captioning/transcribe"; +import { useEffect, useRef } from "react"; +import { type TranscribeStatus, transcribeAsset } from "@/lib/ai-edition/document/transcribe"; +import { classifyTranscriptionError } from "@/lib/ai-edition/transcription/status"; import type { CliCaptionsRequest, CliDoneResult } from "@/lib/cliContracts"; -import { nativeBridgeClient } from "@/native"; -import { captionSegmentsToAnnotationRegions } from "./captionAnnotations"; -import { - shiftTrimRegionsMsForCaptionBuffer, - trimLeadingSilenceMono16k, -} from "./vendor/leadingSilence"; - -/** Highest trailing number across existing region ids, so new ids never collide. */ -function nextNumericIdFrom(regions: { id: string }[]): number { - let max = 0; - for (const region of regions) { - const match = /(\d+)$/.exec(region.id); - if (match) max = Math.max(max, Number(match[1])); - } - return max + 1; -} +import { captionProject } from "./captionProject"; +import { loadCliProject } from "./loadCliProject"; async function runCaptions(request: CliCaptionsRequest): Promise { - const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(request.projectPath); - if (!loaded.success || loaded.project === undefined) { - throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file"); - } - if (!validateProjectData(loaded.project)) { - throw new Error("Project file is not a valid .openscreen project"); - } - const project = loaded.project; - const media = resolveProjectMedia(project); - if (!media) { - throw new Error("Project file does not reference any recorded media"); - } - const editor = normalizeProjectEditor(project.editor ?? {}); - const trimRegions: TrimRegion[] = editor.trimRegions; - - window.electronAPI.cliLog("info", "Extracting audio…"); - const videoUrl = toFileUrl(media.screenVideoPath); - const { samples, durationSec } = await extractMono16kFromVideoUrl(videoUrl); - if (!Number.isFinite(durationSec) || durationSec <= 0 || samples.length < 800) { - throw new Error("The project's video has no usable audio track to transcribe"); - } - - const { samples: speechSamples, trimSec } = trimLeadingSilenceMono16k(samples); - if (speechSamples.length < 800) { - throw new Error("No speech detected in the project's audio"); - } - - const trimMs = Math.round(trimSec * 1000); - const trimRegionsForTranscribe = shiftTrimRegionsMsForCaptionBuffer(trimRegions, trimMs); - - // `onStatus` now fires once per transcribed chunk, not once per phase, so log - // only on a phase change — otherwise a long transcription spams the CLI with - // one identical line per chunk. - let loggedPhase: SttRendererStatus["phase"] | null = null; - const transcribeOptions = { - onStatus: ({ phase }: SttRendererStatus) => { - if (phase === loggedPhase) return; - loggedPhase = phase; - window.electronAPI.cliLog( - "info", - phase === "model" ? "Loading caption model…" : "Transcribing…", - ); - }, + const { document } = await loadCliProject(request.projectPath); + + // `onStatus` fires once per transcribed chunk, so log only on a phase change: a long + // transcription would otherwise print one identical line per chunk. + let loggedPhase: TranscribeStatus["phase"] | null = null; + const onStatus = ({ phase }: TranscribeStatus) => { + if (phase === loggedPhase) return; + loggedPhase = phase; + window.electronAPI.cliLog( + "info", + phase === "extracting-audio" + ? "Extracting audio…" + : phase === "loading-model" + ? "Loading the speech model…" + : "Transcribing…", + ); }; - let { segments: segmentsRaw, granularity } = await transcribeMono16kToSegments(speechSamples, { - trimRegions: trimRegionsForTranscribe, - ...transcribeOptions, - }); - let transcribedFromTrimmedBuffer = true; - - // Leading-silence trimming can return empty even when the full source has - // speech. Retry once against the untrimmed buffer before giving up. - if (segmentsRaw.length === 0 && trimSec > 0) { - ({ segments: segmentsRaw, granularity } = await transcribeMono16kToSegments(samples, { - trimRegions, - ...transcribeOptions, - })); - transcribedFromTrimmedBuffer = false; - } - - const segments = - transcribedFromTrimmedBuffer && trimSec > 0 - ? segmentsRaw.map((segment) => ({ - ...segment, - startSec: segment.startSec + trimSec, - endSec: segment.endSec + trimSec, - })) - : segmentsRaw; - - // Re-running the command replaces earlier auto-captions instead of stacking - // duplicates; manually added annotations are preserved. - const manualAnnotations: AnnotationRegion[] = editor.annotationRegions.filter( - (annotation) => annotation.annotationSource !== "auto-caption", + const result = await captionProject( + document, + { + minWordsPerLine: request.minWordsPerCaption, + maxWordsPerLine: request.maxWordsPerCaption, + }, + (doc, assetId) => + transcribeAsset(doc, assetId, { onStatus }).catch((error: unknown) => { + throw new Error(`Transcription failed: ${classifyTranscriptionError(error).message}`); + }), ); - const startNumericId = nextNumericIdFrom([...editor.annotationRegions, ...editor.zoomRegions]); - const startZIndex = manualAnnotations.reduce((max, a) => Math.max(max, a.zIndex + 1), 1); - - let regions = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, { - minWordsPerCaption: request.minWordsPerCaption, - maxWordsPerCaption: request.maxWordsPerCaption, - timestampGranularity: granularity, - }); - if (regions.length === 0 && segments.length > 0) { - regions = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, { - minWordsPerCaption: 1, - maxWordsPerCaption: Number.MAX_SAFE_INTEGER, - timestampGranularity: granularity, - }); - } - if (regions.length === 0) { - throw new Error("Transcription produced no caption segments"); + if (!result.transcribed) { + window.electronAPI.cliLog("info", "Using the transcript the project already has."); } - const updatedProject = { - ...project, - editor: { - ...editor, - annotationRegions: [...manualAnnotations, ...regions], - }, - }; - return { success: true, projectPath: request.projectPath, - projectData: updatedProject, - captionCount: regions.length, + projectData: result.document, + captionCount: result.captionCount, }; } export function CliCaptionsRunner() { const startedRef = useRef(false); - const [status] = useState("Generating captions…"); useEffect(() => { if (startedRef.current) return; @@ -166,7 +75,7 @@ export function CliCaptionsRunner() { return (
- {status} + Generating captions…
); } diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx index e206e1b839..685f80f14c 100644 --- a/src/cli/CliExportRunner.tsx +++ b/src/cli/CliExportRunner.tsx @@ -6,22 +6,14 @@ // builds the clip list + scene JSON and relays progress. import { useEffect, useRef, useState } from "react"; -import { - type EditorProjectData, - normalizeProjectEditor, - resolveProjectMedia, - toFileUrl, - validateProjectData, -} from "@/components/video-editor/projectPersistence"; +import { toFileUrl } from "@/components/video-editor/projectPersistence"; import { prepareClickSound } from "@/lib/ai-edition/clickSound"; -import { migrateProjectDataToAxcutDocument } from "@/lib/ai-edition/document/migrate"; import { collectEffectiveClipDims, pickExtremeDims, resolveAspectRatioValue, } from "@/lib/ai-edition/document/outputFormat"; -import { applyProbedDuration } from "@/lib/ai-edition/document/timeline"; -import { type AxcutDocument, isAxcutDocumentFile } from "@/lib/ai-edition/schema"; +import type { AxcutDocument } from "@/lib/ai-edition/schema"; import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings"; import { appendAutoZoomSuggestions, @@ -38,38 +30,7 @@ import { exportGifNative, exportMultiNative, nativeBridgeClient } from "@/native import type { CompositorClipInput } from "@/native/contracts"; import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription"; import { type ExportProject, loadDocumentProject } from "./exportProject"; - -function probeVideoDimensions( - url: string, -): Promise<{ width: number; height: number; durationMs: number }> { - return new Promise((resolve, reject) => { - const video = document.createElement("video"); - video.preload = "metadata"; - video.muted = true; - const cleanup = () => { - clearTimeout(timer); - video.removeAttribute("src"); - video.load(); - }; - // A stalled load fires neither event; without a deadline the CLI hangs. - const timer = setTimeout(() => { - cleanup(); - reject(new Error(`Timed out reading video metadata: ${url}`)); - }, 30_000); - video.onloadedmetadata = () => { - const width = video.videoWidth; - const height = video.videoHeight; - const durationMs = Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : 0; - cleanup(); - resolve({ width, height, durationMs }); - }; - video.onerror = () => { - cleanup(); - reject(new Error(`Failed to load video metadata: ${url}`)); - }; - video.src = url; - }); -} +import { loadCliProject, probeVideoDimensions } from "./loadCliProject"; function replaceExtension(filePath: string, newExtension: string): string { return filePath.replace(/\.(openscreen|json)$/i, "") + newExtension; @@ -115,126 +76,18 @@ function gifOutputDims( return { width: even(sourceDims.width), height: even(sourceDims.height) }; } -/** - * A legacy v2 project (`{ version, media, editor }`), as written before the editor saved the - * document itself: migrated to an AxcutDocument, with what the migration cannot know probed. - */ -async function loadLegacyProject(project: EditorProjectData): Promise { - const media = resolveProjectMedia(project); - if (!media) { - throw new Error("Project file does not reference any recorded media"); - } - // Prefer the main process's approved session paths: they carry the - // packed-project sibling fallback when the stored absolute paths are stale. - try { - const sessionResult = await window.electronAPI.getCurrentRecordingSession(); - const session = sessionResult?.session; - if (session?.screenVideoPath) { - media.screenVideoPath = session.screenVideoPath; - if (media.webcamVideoPath && session.webcamVideoPath) { - media.webcamVideoPath = session.webcamVideoPath; - } - } - } catch { - // Fall back to the paths stored in the project file. - } - const editor = normalizeProjectEditor(project.editor ?? {}); - - const probed = await probeVideoDimensions(toFileUrl(media.screenVideoPath)); - - // Migrate the .openscreen project onto the AxcutDocument the native - // compositor consumes. The migration is pure and carries zooms, annotations, - // trims and the legacy editor settings; the clip's duration is unknown until - // probed, so applyProbedDuration must run or the export is a single frame. - let axcutDocument = migrateProjectDataToAxcutDocument({ - ...project, - media, - editor, - }); - const primaryAssetId = axcutDocument.project.primaryAssetId ?? axcutDocument.assets[0]?.id; - if (!primaryAssetId) { - throw new Error("Project migration produced no media asset"); - } - if (probed.durationMs > 0) { - axcutDocument = applyProbedDuration(axcutDocument, primaryAssetId, probed.durationMs / 1000); - } - // The screen's dimensions, for the same reason as the camera's below: the pure migration - // cannot know them, and the scene's screen box, the output frame and Auto all read them off - // `asset.video`. Without them the CLI laid every recording out as 1920x1080 while sizing the - // export from the probe, two answers to one question. - if (probed.width > 0 && probed.height > 0) { - axcutDocument = { - ...axcutDocument, - assets: axcutDocument.assets.map((asset) => - asset.id === primaryAssetId - ? { - ...asset, - video: { - codec: "unknown", - fps: 0, - ...asset.video, - width: probed.width, - height: probed.height, - }, - } - : asset, - ), - }; - } - - // The camera's dimensions decide the PiP's layout box, and this is the one caller the - // document cannot answer for: there is no editor session here to have probed and saved - // them, so without this the CLI lays the box out from a hardcoded 4:3 and a 16:9 camera - // exports framed differently from the same project opened in the app. - // - // Failure-tolerant on purpose, unlike the screen probe above which is allowed to reject: - // a camera file that has gone missing should cost the export its camera, not the export. - if (media.webcamVideoPath) { - const camera = await probeVideoDimensions(toFileUrl(media.webcamVideoPath)).catch(() => null); - if (camera) { - axcutDocument = { - ...axcutDocument, - assets: axcutDocument.assets.map((asset) => - asset.cameraTrack - ? { - ...asset, - cameraTrack: { - ...asset.cameraTrack, - width: camera.width, - height: camera.height, - }, - } - : asset, - ), - }; - } - } - return { - document: axcutDocument, - editor, - screenVideoPath: media.screenVideoPath, - durationMs: probed.durationMs, - sourceDims: { width: probed.width, height: probed.height }, - }; -} - /** The project to export: by id when the MCP server names one, else the file at its path. */ async function loadExportProject(request: CliExportRequest): Promise { - const probe = (videoPath: string) => probeVideoDimensions(toFileUrl(videoPath)); if (request.projectId) { const stored = await nativeBridgeClient.aiEdition.get(request.projectId); if (!stored.success || !stored.document) { throw new Error(stored.error ?? `No project has the id "${request.projectId}"`); } - return loadDocumentProject(stored.document, probe); - } - const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(request.projectPath); - if (!loaded.success || loaded.project === undefined) { - throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file"); + return loadDocumentProject(stored.document, (videoPath) => + probeVideoDimensions(toFileUrl(videoPath)), + ); } - if (isAxcutDocumentFile(loaded.project)) return loadDocumentProject(loaded.project, probe); - if (validateProjectData(loaded.project)) return loadLegacyProject(loaded.project); - throw new Error("Project file is not a valid .openscreen project"); + return loadCliProject(request.projectPath); } async function runExport(request: CliExportRequest): Promise { diff --git a/src/cli/captionAnnotations.test.ts b/src/cli/captionAnnotations.test.ts deleted file mode 100644 index 79f336a69f..0000000000 --- a/src/cli/captionAnnotations.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { DEFAULT_CAPTION_SETTINGS } from "@/lib/ai-edition/captions/settings"; -import type { CaptionSegment } from "@/lib/captioning/transcribe"; -import { captionSegmentsToAnnotationRegions } from "./captionAnnotations"; - -const words = (...texts: string[]): CaptionSegment[] => - texts.map((text, i) => ({ text, startSec: i * 0.5, endSec: i * 0.5 + 0.4 })); - -describe("captionSegmentsToAnnotationRegions", () => { - it("numbers ids and z-indexes from the given start, with non-empty spans", () => { - const regions = captionSegmentsToAnnotationRegions(words("one", "two", "three", "four"), 7, 3); - - expect(regions.length).toBeGreaterThan(0); - expect(regions.map((r) => r.id)).toEqual(regions.map((_, i) => `annotation-${7 + i}`)); - expect(regions.map((r) => r.zIndex)).toEqual(regions.map((_, i) => 3 + i)); - for (const region of regions) { - expect(region.endMs).toBeGreaterThan(region.startMs); - expect(region.annotationSource).toBe("auto-caption"); - expect(region.content.trim()).not.toBe(""); - } - }); - - it("groups phrase-granularity segments one line at a time", () => { - const regions = captionSegmentsToAnnotationRegions( - [ - { text: "hello there", startSec: 0, endSec: 1 }, - { text: "second line", startSec: 1.2, endSec: 2 }, - ], - 1, - 1, - { timestampGranularity: "phrase" }, - ); - - expect(regions.map((r) => r.content)).toEqual(["hello there", "second line"]); - }); - - it("looks like the editor's captions, not a faint annotation", () => { - const [region] = captionSegmentsToAnnotationRegions(words("one", "two"), 1, 1); - expect(region.style).toMatchObject({ - fontSize: DEFAULT_CAPTION_SETTINGS.fontSize, - fontWeight: DEFAULT_CAPTION_SETTINGS.fontWeight, - color: DEFAULT_CAPTION_SETTINGS.color, - backgroundColor: "rgba(0, 0, 0, 0.55)", - }); - // The editor's caption box: 16:9 safe column, bottom edge 1.5% off the frame's, room for - // a caption that wraps. The text is bottom-anchored in it (see sceneDescription). - expect(region.position.x).toBe(16); - expect(region.size.width).toBe(68); - expect(region.position.y + region.size.height).toBeCloseTo(98.5); - // Three 48 px lines of 1.5 em plus the plate padding: a wrapped caption is not clipped. - expect(region.size.height).toBeCloseTo(((48 * (3 * 1.5 + 0.2)) / 1080) * 100); - }); -}); diff --git a/src/cli/captionAnnotations.ts b/src/cli/captionAnnotations.ts deleted file mode 100644 index 91e3baa969..0000000000 --- a/src/cli/captionAnnotations.ts +++ /dev/null @@ -1,84 +0,0 @@ -// CLI-only: the v4 editor renders captions natively, but `openscreen captions` -// still writes caption *annotations* into .openscreen projects. Only the -// annotation conversion lives here — the segment grouping/dedupe helpers come -// from the live captioning module so the CLI can't drift from the editor. - -import type { AnnotationRegion, AnnotationTextStyle } from "@/components/video-editor/types"; -import { - captionBackgroundCss, - captionBoxRect, - DEFAULT_CAPTION_SETTINGS, -} from "@/lib/ai-edition/captions/settings"; -import { - type CaptionSegmentLayoutOptions, - dedupeAdjacentCaptionRepeats, - finalizeCaptionSegmentsForPlayback, - groupPhraseCaptionSegmentsIntoLines, - groupTimedCaptionWordsIntoLines, -} from "@/lib/captioning/annotationsFromCaptions"; -import type { CaptionSegment } from "@/lib/captioning/transcribe"; - -// The look and place of the editor's own captions (`DEFAULT_CAPTION_SETTINGS`): bold 48 px -// white on a 55% black plate, in the landscape safe column, just off the bottom edge. The CLI -// used to write 24 px regular text on no plate, which read as faint over any bright frame. -// `.openscreen` projects carry no transcript, so the CLI cannot write the editor's transcript- -// driven captions; matching their appearance is what keeps the two from looking different. -// -// Placed with the editor's own caption box: room for three lines, pinned by its BOTTOM edge -// 1.5% off the frame's. The scene description bottom-anchors `auto-caption` text in it, so -// the plate ends at the inset whether the caption holds one line or wraps to three. -const CAPTION_BOX = captionBoxRect(DEFAULT_CAPTION_SETTINGS, 16 / 9); - -const CAPTION_POSITION = { x: CAPTION_BOX.x, y: CAPTION_BOX.y }; - -const CAPTION_SIZE = { width: CAPTION_BOX.width, height: CAPTION_BOX.height }; - -const CAPTION_STYLE: AnnotationTextStyle = { - color: DEFAULT_CAPTION_SETTINGS.color, - backgroundColor: captionBackgroundCss(DEFAULT_CAPTION_SETTINGS), - fontSize: DEFAULT_CAPTION_SETTINGS.fontSize, - fontFamily: DEFAULT_CAPTION_SETTINGS.fontFamily, - fontWeight: DEFAULT_CAPTION_SETTINGS.fontWeight, - fontStyle: "normal", - textDecoration: "none", - textAlign: "center", -}; - -export function captionSegmentsToAnnotationRegions( - segments: CaptionSegment[], - startNumericId: number, - startZIndex: number, - layout?: CaptionSegmentLayoutOptions, -): AnnotationRegion[] { - // Don't echo-collapse raw word tokens before grouping: repeated words ("I … I") share a - // normalized key and would merge spans while keeping only the first token's text. - const minW = layout?.minWordsPerCaption ?? DEFAULT_CAPTION_SETTINGS.minWordsPerLine; - const maxW = layout?.maxWordsPerCaption ?? DEFAULT_CAPTION_SETTINGS.maxWordsPerLine; - const granularity = layout?.timestampGranularity ?? "word"; - - const grouped = - granularity === "phrase" - ? groupPhraseCaptionSegmentsIntoLines(segments, minW, maxW) - : groupTimedCaptionWordsIntoLines(segments, minW, maxW); - - const finalized = finalizeCaptionSegmentsForPlayback(dedupeAdjacentCaptionRepeats(grouped)); - - let nid = startNumericId; - let z = startZIndex; - return finalized.map((seg) => { - const startMs = Math.round(seg.startSec * 1000); - const endMs = Math.max(Math.round(seg.endSec * 1000), startMs + 1); - return { - id: `annotation-${nid++}`, - startMs, - endMs, - type: "text", - content: seg.text, - annotationSource: "auto-caption", - position: { ...CAPTION_POSITION }, - size: { ...CAPTION_SIZE }, - style: { ...CAPTION_STYLE }, - zIndex: z++, - }; - }); -} diff --git a/src/cli/captionProject.test.ts b/src/cli/captionProject.test.ts new file mode 100644 index 0000000000..a7f09f032e --- /dev/null +++ b/src/cli/captionProject.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it, vi } from "vitest"; +import { getCaptionSettings } from "@/lib/ai-edition/captions"; +import { + type AxcutDocument, + type AxcutTranscript, + createEmptyDocument, + documentSchema, +} from "@/lib/ai-edition/schema"; +import { captionProject } from "./captionProject"; + +function project(extra: Partial = {}): AxcutDocument { + const base = createEmptyDocument({ + title: "CLI", + projectId: "proj_1", + createdAt: "2026-01-01T00:00:00.000Z", + }); + return documentSchema.parse({ + ...base, + project: { ...base.project, primaryAssetId: "asset_1" }, + assets: [ + { id: "asset_1", kind: "video", label: "Rec", originalPath: "/tmp/rec.mp4", durationSec: 10 }, + ], + timeline: { + ...base.timeline, + clips: [ + { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + wordRefs: [], + origin: "user", + reason: "", + }, + ], + }, + ...extra, + }); +} + +function transcript(text: string): AxcutTranscript { + const words = text.split(" "); + return { + assetId: "asset_1", + language: "en", + segments: words.map((word, i) => ({ + id: `seg_${i + 1}`, + kind: "speech" as const, + startSec: i, + endSec: i + 1, + text: word, + wordIds: [`word_${i + 1}`], + })), + words: words.map((word, i) => ({ + id: `word_${i + 1}`, + segmentId: `seg_${i + 1}`, + startSec: i, + endSec: i + 1, + text: word, + })), + }; +} + +const LINE = { minWordsPerLine: 1, maxWordsPerLine: 2 }; + +describe("captionProject", () => { + it("transcribes the primary media and turns the editor's captions on", async () => { + const transcribe = vi.fn(async () => transcript("one two three four")); + const result = await captionProject(project(), LINE, transcribe); + + expect(transcribe).toHaveBeenCalledWith(expect.anything(), "asset_1"); + expect(result.transcribed).toBe(true); + expect(result.document.transcripts).toHaveLength(1); + expect(getCaptionSettings(result.document)).toMatchObject({ + enabled: true, + minWordsPerLine: 1, + maxWordsPerLine: 2, + }); + expect(result.captionCount).toBe(2); + }); + + it("reuses a transcript the project has, and clears the old caption annotations", async () => { + const withLegacy = { + ...project({ transcripts: [transcript("hello there")] }), + annotations: [ + { id: "annotation-1", annotationSource: "auto-caption" }, + ] as unknown as AxcutDocument["annotations"], + }; + const transcribe = vi.fn(); + const result = await captionProject(withLegacy, LINE, transcribe); + + expect(transcribe).not.toHaveBeenCalled(); + expect(result.transcribed).toBe(false); + expect(result.document.annotations).toEqual([]); + }); + + it("fails on media with no speech, writing nothing", async () => { + const silent: AxcutTranscript = { + assetId: "asset_1", + language: "auto", + segments: [], + words: [], + }; + await expect(captionProject(project(), LINE, async () => silent)).rejects.toThrow(/No speech/); + }); +}); diff --git a/src/cli/captionProject.ts b/src/cli/captionProject.ts new file mode 100644 index 0000000000..44ac480423 --- /dev/null +++ b/src/cli/captionProject.ts @@ -0,0 +1,59 @@ +// What `openscreen captions` does to a project, kept out of the runner window so it is +// testable: the editor's own captions, not annotations. The primary media gets a +// transcript (the one it has, or a fresh one), then the captions are turned on with the +// requested line length, as the Captions pane would. They are the transcript drawn at +// render time, so `openscreen export` burns them in and the editor shows them editable. + +import { + deriveCaptionCues, + getCaptionSettings, + getCaptionTranslations, + patchCaptionSettings, +} from "@/lib/ai-edition/captions"; +import { resolveAspectRatioValue } from "@/lib/ai-edition/document/outputFormat"; +import { applyTranscriptionResult } from "@/lib/ai-edition/document/transcript"; +import type { AxcutDocument, AxcutTranscript } from "@/lib/ai-edition/schema"; +import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings"; +import { findAssetTranscript, transcriptHasSpeech } from "@/lib/ai-edition/transcription/status"; + +export interface CaptionProjectOptions { + minWordsPerLine: number; + maxWordsPerLine: number; +} + +export async function captionProject( + document: AxcutDocument, + options: CaptionProjectOptions, + transcribe: (document: AxcutDocument, assetId: string) => Promise, +): Promise<{ document: AxcutDocument; captionCount: number; transcribed: boolean }> { + const assetId = document.project.primaryAssetId ?? document.assets[0]?.id; + if (!assetId) throw new Error("Project file does not reference any recorded media"); + const existing = findAssetTranscript(document, assetId); + const transcript = existing ?? (await transcribe(document, assetId)); + if (!transcriptHasSpeech(transcript)) { + throw new Error("No speech detected in the project's audio"); + } + const transcribed = existing ? document : applyTranscriptionResult(document, transcript).document; + // The v1 caption annotations this command used to write: the Captions pane offers to + // clear them, and next to real captions they would show every line twice. + const cleared: AxcutDocument = { + ...transcribed, + annotations: transcribed.annotations.filter((a) => a.annotationSource !== "auto-caption"), + }; + const aspect = resolveAspectRatioValue(cleared, getEditorSettings(cleared).aspectRatio); + const next = patchCaptionSettings( + cleared, + { + enabled: true, + minWordsPerLine: options.minWordsPerLine, + maxWordsPerLine: options.maxWordsPerLine, + }, + aspect, + ); + const captionCount = deriveCaptionCues( + next, + getCaptionSettings(next, aspect), + getCaptionTranslations(next), + ).length; + return { document: next, captionCount, transcribed: !existing }; +} diff --git a/src/cli/loadCliProject.ts b/src/cli/loadCliProject.ts new file mode 100644 index 0000000000..395514348b --- /dev/null +++ b/src/cli/loadCliProject.ts @@ -0,0 +1,166 @@ +// What `openscreen export` and `openscreen captions` read from a project file, in the +// hidden runner window: either shape the file was saved in, as one AxcutDocument. The +// renderer half of `exportProject.ts`, since probing a video needs a DOM. + +import { + type EditorProjectData, + normalizeProjectEditor, + resolveProjectMedia, + toFileUrl, + validateProjectData, +} from "@/components/video-editor/projectPersistence"; +import { migrateProjectDataToAxcutDocument } from "@/lib/ai-edition/document/migrate"; +import { applyProbedDuration } from "@/lib/ai-edition/document/timeline"; +import { isAxcutDocumentFile } from "@/lib/ai-edition/schema"; +import { nativeBridgeClient } from "@/native"; +import { type ExportProject, loadDocumentProject } from "./exportProject"; + +export function probeVideoDimensions( + url: string, +): Promise<{ width: number; height: number; durationMs: number }> { + return new Promise((resolve, reject) => { + const video = document.createElement("video"); + video.preload = "metadata"; + video.muted = true; + const cleanup = () => { + clearTimeout(timer); + video.removeAttribute("src"); + video.load(); + }; + // A stalled load fires neither event; without a deadline the CLI hangs. + const timer = setTimeout(() => { + cleanup(); + reject(new Error(`Timed out reading video metadata: ${url}`)); + }, 30_000); + video.onloadedmetadata = () => { + const width = video.videoWidth; + const height = video.videoHeight; + const durationMs = Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : 0; + cleanup(); + resolve({ width, height, durationMs }); + }; + video.onerror = () => { + cleanup(); + reject(new Error(`Failed to load video metadata: ${url}`)); + }; + video.src = url; + }); +} + +/** + * A legacy v2 project (`{ version, media, editor }`), as written before the editor saved the + * document itself: migrated to an AxcutDocument, with what the migration cannot know probed. + */ +async function loadLegacyProject(project: EditorProjectData): Promise { + const media = resolveProjectMedia(project); + if (!media) { + throw new Error("Project file does not reference any recorded media"); + } + // Prefer the main process's approved session paths: they carry the + // packed-project sibling fallback when the stored absolute paths are stale. + try { + const sessionResult = await window.electronAPI.getCurrentRecordingSession(); + const session = sessionResult?.session; + if (session?.screenVideoPath) { + media.screenVideoPath = session.screenVideoPath; + if (media.webcamVideoPath && session.webcamVideoPath) { + media.webcamVideoPath = session.webcamVideoPath; + } + } + } catch { + // Fall back to the paths stored in the project file. + } + const editor = normalizeProjectEditor(project.editor ?? {}); + + const probed = await probeVideoDimensions(toFileUrl(media.screenVideoPath)); + + // Migrate the .openscreen project onto the AxcutDocument the native + // compositor consumes. The migration is pure and carries zooms, annotations, + // trims and the legacy editor settings; the clip's duration is unknown until + // probed, so applyProbedDuration must run or the export is a single frame. + let axcutDocument = migrateProjectDataToAxcutDocument({ + ...project, + media, + editor, + }); + const primaryAssetId = axcutDocument.project.primaryAssetId ?? axcutDocument.assets[0]?.id; + if (!primaryAssetId) { + throw new Error("Project migration produced no media asset"); + } + if (probed.durationMs > 0) { + axcutDocument = applyProbedDuration(axcutDocument, primaryAssetId, probed.durationMs / 1000); + } + // The screen's dimensions, for the same reason as the camera's below: the pure migration + // cannot know them, and the scene's screen box, the output frame and Auto all read them off + // `asset.video`. Without them the CLI laid every recording out as 1920x1080 while sizing the + // export from the probe, two answers to one question. + if (probed.width > 0 && probed.height > 0) { + axcutDocument = { + ...axcutDocument, + assets: axcutDocument.assets.map((asset) => + asset.id === primaryAssetId + ? { + ...asset, + video: { + codec: "unknown", + fps: 0, + ...asset.video, + width: probed.width, + height: probed.height, + }, + } + : asset, + ), + }; + } + + // The camera's dimensions decide the PiP's layout box, and this is the one caller the + // document cannot answer for: there is no editor session here to have probed and saved + // them, so without this the CLI lays the box out from a hardcoded 4:3 and a 16:9 camera + // exports framed differently from the same project opened in the app. + // + // Failure-tolerant on purpose, unlike the screen probe above which is allowed to reject: + // a camera file that has gone missing should cost the export its camera, not the export. + if (media.webcamVideoPath) { + const camera = await probeVideoDimensions(toFileUrl(media.webcamVideoPath)).catch(() => null); + if (camera) { + axcutDocument = { + ...axcutDocument, + assets: axcutDocument.assets.map((asset) => + asset.cameraTrack + ? { + ...asset, + cameraTrack: { + ...asset.cameraTrack, + width: camera.width, + height: camera.height, + }, + } + : asset, + ), + }; + } + } + return { + document: axcutDocument, + editor, + screenVideoPath: media.screenVideoPath, + durationMs: probed.durationMs, + sourceDims: { width: probed.width, height: probed.height }, + }; +} + +/** The project at `projectPath`, whichever shape it was saved in. */ +export async function loadCliProject(projectPath: string): Promise { + const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(projectPath); + if (!loaded.success || loaded.project === undefined) { + throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file"); + } + if (isAxcutDocumentFile(loaded.project)) { + return loadDocumentProject(loaded.project, (videoPath) => + probeVideoDimensions(toFileUrl(videoPath)), + ); + } + if (validateProjectData(loaded.project)) return loadLegacyProject(loaded.project); + throw new Error("Project file is not a valid .openscreen project"); +} diff --git a/src/cli/vendor/leadingSilence.ts b/src/cli/vendor/leadingSilence.ts deleted file mode 100644 index 1f6b1b8469..0000000000 --- a/src/cli/vendor/leadingSilence.ts +++ /dev/null @@ -1,80 +0,0 @@ -// Vendored for the CLI: @/lib/captioning/leadingSilence was deleted in the 1.8 line. The CLI's captions command still trims leading silence before transcription, so the module lives on here verbatim. - -/** Caption path is always mono 16 kHz after `extractMono16kFromVideoUrl`. */ -import type { TrimRegion } from "@/components/video-editor/types"; - -const SAMPLE_RATE = 16_000; - -/** Window length for peak detection (~50 ms). */ -const WINDOW_SAMPLES = 800; - -/** Coarse hop so long intros scan quickly (~50 ms steps). */ -const HOP_SAMPLES = 800; - -/** Max |sample| in a window below this counts as silence (float PCM ~[-1, 1]). */ -const PEAK_THRESHOLD = 0.012; - -/** Keep a little audio before the first peak so word onsets are not clipped. */ -const PRE_ROLL_SEC = 0.12; - -/** Do not scan more than this much audio for leading silence (performance + pathological files). */ -const MAX_LEADING_SCAN_SEC = 15 * 60; - -/** - * Drops quiet audio at the beginning so Whisper is not fed a long silent prefix (which can skew - * the first phrase and wastes work). Returned `trimSec` must be added back to every segment time. - */ -export function trimLeadingSilenceMono16k(samples: Float32Array): { - samples: Float32Array; - trimSec: number; -} { - if (samples.length < WINDOW_SAMPLES) { - return { samples, trimSec: 0 }; - } - - const maxIndex = Math.min( - samples.length - WINDOW_SAMPLES, - Math.floor(MAX_LEADING_SCAN_SEC * SAMPLE_RATE), - ); - - let firstSpeechSample = -1; - for (let i = 0; i <= maxIndex; i += HOP_SAMPLES) { - let peak = 0; - for (let j = 0; j < WINDOW_SAMPLES; j++) { - peak = Math.max(peak, Math.abs(samples[i + j]!)); - } - if (peak > PEAK_THRESHOLD) { - firstSpeechSample = i; - break; - } - } - - if (firstSpeechSample <= 0) { - return { samples, trimSec: 0 }; - } - - const preRollSamples = Math.round(PRE_ROLL_SEC * SAMPLE_RATE); - const start = Math.max(0, firstSpeechSample - preRollSamples); - return { - samples: samples.subarray(start), - trimSec: start / SAMPLE_RATE, - }; -} - -/** - * When audio is trimmed from the front, Whisper times are relative to the shortened buffer. - * Shift trim regions by the same offset so `segmentOverlapsTrim` still uses consistent coordinates. - */ -export function shiftTrimRegionsMsForCaptionBuffer( - regions: TrimRegion[], - trimMs: number, -): TrimRegion[] { - if (trimMs <= 0) return regions; - return regions - .map((r) => ({ - ...r, - startMs: Math.max(0, r.startMs - trimMs), - endMs: Math.max(0, r.endMs - trimMs), - })) - .filter((r) => r.endMs > r.startMs); -} diff --git a/src/lib/cliContracts.ts b/src/lib/cliContracts.ts index fa400a872d..df1e6e87d2 100644 --- a/src/lib/cliContracts.ts +++ b/src/lib/cliContracts.ts @@ -132,6 +132,6 @@ export interface CliDoneResult { projectData?: unknown; /** Sources: enumeration payload printed by the main process. */ sources?: CliSourcesResult; - /** Captions: number of caption annotations generated. */ + /** Captions: how many captions the transcript now shows. */ captionCount?: number; } From 5e14b7936a25b1c95501836be6b6d977404a8cec Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:19:02 +0200 Subject: [PATCH 04/16] docs: captions, translation and transcription over MCP and in the CLI Part of #1180 --- skills/openscreen-demo/SKILL.md | 7 ++++++ .../architecture/mcp-server.md | 25 +++++++++++++++---- .../transcription-and-captions.md | 20 ++++++++++++++- website/docs/ai-editing.md | 7 +++--- website/docs/cli.md | 15 +++++------ 5 files changed, 58 insertions(+), 16 deletions(-) diff --git a/skills/openscreen-demo/SKILL.md b/skills/openscreen-demo/SKILL.md index 3918a420bf..073270e0bb 100644 --- a/skills/openscreen-demo/SKILL.md +++ b/skills/openscreen-demo/SKILL.md @@ -88,6 +88,12 @@ Use the document tools, always with the `projectId`. Read the project first with - **Hide what is private:** an email address, a token or a name that shows on screen gets `addAnnotation` with `type: "blur"` over the span it is visible. Its x, y, width and height are percent of the recording's picture, not of the frame, so it stays on what it hides when a zoom moves the picture. Check it with `getFrames`. - **Music or a voiceover:** `importMedia` with the file's absolute path adds it to the project's media, then `addAudio` plays it over a span. `removeMedia` takes back a file the timeline no longer uses. - **Name:** `setProjectInfo` renames the project, if the `title` given to `stopRecording` no longer fits. +- **Captions, for a narrated take** (recorded with `microphone`): + 1. `transcribe` with the `projectId`. Whisper runs on this machine and can take a minute; a take has no transcript before this. + 2. `setCaptionSettings` with `enabled: true` and a `style`, plus `anchorV`, `insetY` or `maxWordsPerLine` if the callouts need room. + 3. To caption in another language: `getCaptionTranslationUnits` with the `language`, translate each unit yourself as a whole phrase, store them with `putCaptionTranslation`, then `setCaptionSettings` with that `language`. + 4. `getCaptions` reads it all back. The export burns the captions in. + A misheard name is fixed with `setWordText` (or `setVocabulary`, then `transcribe` again with the `language`), a flubbed phrase cut with `getTranscriptWords` and `removeWords`. - **Undo:** `restoreCheckpoint` with the `checkpointId` takes the project back to the raw take in one step. It also puts back the take's own pointer track, undoing any `setPointerTrack` made since. - **Open in the app:** while OpenScreen is running, your edits go through it, so a project the user has open in its editor changes there live and they can undo each edit. If an edit is refused as "open in the OpenScreen editor", the app did not answer: retry once, then ask the user to close the project there or restart OpenScreen. @@ -137,5 +143,6 @@ Only to a pull request the user asked for a video on, or one you open for them i - **`The recording did not start`:** the window title did not match, or capture is not permitted. Check `listSources` and the permissions in step 1. - **Recording, exporting or an edit refused, naming *MCP edits*:** you are connected to the server in the app, which records, exports and edits only once the user allows it. Ask them to turn on *MCP edits* under *AI settings* → *MCP server*. - **No cursor in the video:** the app was driven by a script. See step 4. +- **`transcribe` fails with `no-audio`:** the take was recorded with neither `microphone` nor `systemAudio`. Record again with one of them. - **No zoom on an early click:** the first 2.5 s stay wide on purpose. Start the first action after that. - **`unknown flag: --attach`:** `gh` is older than 2.99.0. See step 8. diff --git a/technical-documentation/architecture/mcp-server.md b/technical-documentation/architecture/mcp-server.md index e8ebb85f13..18425581b0 100644 --- a/technical-documentation/architecture/mcp-server.md +++ b/technical-documentation/architecture/mcp-server.md @@ -12,7 +12,7 @@ The same tools are served over two transports, by two different processes: | Edits, recording, export | Off until the user turns on **MCP edits** | Always allowed | | Project open in the editor | Edited live through the editor, one undo step per call | The same while the app runs: the call is forwarded to it. With no app answering, refused ([the marker](#the-open-in-editor-marker)) | | Other projects | Read from and saved to their file, a checkpoint first | Same | -| Tools | `listProjects`, the agent's tools, the project tools (`setProjectInfo`, `importMedia`, `removeMedia`), the checkpoint tools, and the media tools: `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames`, `createProject` | The same | +| Tools | `listProjects`, the agent's tools, `transcribe`, the project tools (`setProjectInfo`, `importMedia`, `removeMedia`), the checkpoint tools, and the media tools: `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames`, `createProject` | The same | | A take | Recorded by the app's recording bar, as its Record button records; the editor opens it ([below](#the-media-tools)) | Recorded by a hidden runner of its own process, and made a project there | | Checkpoints | In the app's memory, shared with forwarded `openscreen mcp` calls, lost when the app quits | In the app's while it runs; otherwise in this process's, lost when it exits, which it does when the client disconnects | @@ -29,16 +29,18 @@ The same tools are served over two transports, by two different processes: | [`electron/mcp/app-channel.ts`](../../electron/mcp/app-channel.ts) | The local channel `openscreen mcp` forwards its document calls through to the running app. See [`openscreen mcp` and the running app](#openscreen-mcp-and-the-running-app). | | [`electron/mcp/mcp-controller.ts`](../../electron/mcp/mcp-controller.ts) | In-app server lifecycle: start when enabled, restart on a port or token change, status for the settings UI. | | [`electron/mcp/mcp-settings-store.ts`](../../electron/mcp/mcp-settings-store.ts) | `mcp-server.json` (enabled, port, allowEdits) and `mcp-token.enc` (bearer token, `safeStorage`). | -| [`src/lib/ai-edition/store/mcpDocumentHost.ts`](../../src/lib/ai-edition/store/mcpDocumentHost.ts) | Renderer side: answers snapshot / apply requests from the project store. Mounted by `NewEditorShell`. | +| [`src/lib/ai-edition/store/mcpDocumentHost.ts`](../../src/lib/ai-edition/store/mcpDocumentHost.ts) | Renderer side: answers snapshot / apply requests from the project store, and transcribe requests through the editor's transcription queue. Mounted by `NewEditorShell`. | +| [`electron/ai-edition/caption-tools.ts`](../../electron/ai-edition/caption-tools.ts) | The caption and vocabulary tools, routed there by `executeAgentTool`. | | [`src/components/ai-edition/McpServerSettings.tsx`](../../src/components/ai-edition/McpServerSettings.tsx) | The settings section: server toggle, the **MCP edits** toggle, port, token, copyable `claude mcp add` / `codex mcp add` commands. | ## One tool surface, two agents Nothing about the tools is reimplemented. The server registers `TOOL_ARG_SCHEMAS` (names and zod schemas) and `TOOL_DESCRIPTIONS`, and builds its `instructions` around `buildSystemPrompt`, all exported from [`deep-agent/service.ts`](../../electron/ai-edition/deep-agent/service.ts), where `buildTools` builds the in-app agent from the same table. Every call goes through `runDocumentTool`, the function the in-app agent's `documentTool` also calls: the cursor-telemetry read the zoom tools need (for `addAutoZooms`, the timeline wand's plan, `collectAutoZoomSuggestionsForDocument`, over every take on the timeline), then `executeAgentTool`. -So a tool added to the agent appears over MCP with no further work, and the MCP test asserts the listed tools equal `listProjects`, then `OPENSCREEN_TOOL_NAMES`, then the three project tools, then the two checkpoint tools, then the [media tools](#the-media-tools). What the server adds: +So a tool added to the agent appears over MCP with no further work, and the MCP test asserts the listed tools equal `listProjects`, then `OPENSCREEN_TOOL_NAMES`, then the three project tools, then the two checkpoint tools, then the [media tools](#the-media-tools). The caption tools are agent tools like the others (`getCaptions`, `setCaptionSettings`, `getCaptionTranslationUnits`, `putCaptionTranslation`, `removeCaptionTranslation`, `setVocabulary`): the caller translates the captions with its own model and stores them with `putCaptionTranslation`, so neither agent needs the app's provider for it. What the server adds: - **`listProjects`**, a tool of its own: every project's id, title, `updatedAt`, asset count, and `open` for the one in an editor (this process's, or the app's seen through the marker). +- **`transcribe`**, registered where `McpToolDeps.transcribe` is wired (both transports, by `registerIpcHandlers`), see [Transcription](#transcription). - **An optional `projectId` on every agent tool and both checkpoint tools**, added to its schema with `.extend()` at registration and stripped again before the call reaches the executor, which never sees it. See [Which project a call acts on](#which-project-a-call-acts-on). - **`createCheckpoint` / `restoreCheckpoint`**, see [Checkpoints](#checkpoints). - **The project tools**, see [The project and its media](#the-project-and-its-media). @@ -124,7 +126,7 @@ A client chains several edits in one turn, and each is its own undo step, so rev - **A project that is not open cannot be edited without one.** Ctrl+Z cannot reach an edit to it, so a write there is refused until a checkpoint of that project is in memory, and the client is told to call `createCheckpoint` first. Over stdio that is every project. `stopRecording` creates one for each new take and returns its id. - **They cover the cursor sidecar.** `setPointerTrack` rewrites `