Skip to content

Commit 7d94d17

Browse files
wuzixuan06EtienneLescot
authored andcommitted
feat: add waitForRecording MCP tool for user-controlled takes
1 parent 5bb1b3f commit 7d94d17

19 files changed

Lines changed: 947 additions & 65 deletions

‎electron/electron-env.d.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@ interface Window {
4646
meta: import("../src/native/contracts").CompositorSharedFrameMeta,
4747
) => void,
4848
) => () => void;
49+
reportRecordingOutcome?: (
50+
outcome: import("../src/lib/recordingSession").RecordingOutcome,
51+
) => Promise<{ success: boolean }>;
52+
4953
getSources: (opts: Electron.SourcesOptions) => Promise<ProcessedDesktopSource[]>;
5054
switchToEditor: () => Promise<void>;
5155
switchToHud: () => Promise<void>;

‎electron/ipc/handlers.ts‎

Lines changed: 47 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
shell,
1919
systemPreferences,
2020
} from "electron";
21+
import { z } from "zod";
2122
import { DEFAULT_WEBCAM_QUALITY, type WebcamQualityId } from "../../src/hooks/webcamCaptureTarget";
2223
import {
2324
type AxcutDocument,
@@ -88,6 +89,7 @@ import { McpController } from "../mcp/mcp-controller";
8889
import { McpSettingsStore } from "../mcp/mcp-settings-store";
8990
import { createMediaTools, type MediaToolDeps, type RecordRequest } from "../mcp/media-tools";
9091
import { createToolRunner, type McpToolDeps } from "../mcp/openscreen-mcp-server";
92+
import { RecordingCompletion } from "../mcp/recording-completion";
9193
import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks";
9294
import {
9395
readCursorRecordingFile as readCursorRecordingFileFrom,
@@ -1985,15 +1987,17 @@ export function registerIpcHandlers(
19851987
) {
19861988
/** The HUD is recording a take, the user's or an MCP client's. Set by `setAppRecording` only. */
19871989
let appRecording = false;
1990+
const recordingCompletion = new RecordingCompletion();
19881991
/**
19891992
* Every start and stop of a take goes through here: the browser path's `set-recording-state`
19901993
* and the native paths, which start and stop in this process. The MCP recorder waits on it
19911994
* for a client's take to start, and the tray shows it. A native path that called
19921995
* `onRecordingStateChange` itself left the MCP recorder waiting on a take already running.
19931996
*/
1994-
const setAppRecording = (recording: boolean, sourceName: string) => {
1997+
const setAppRecording = (recording: boolean, sourceName: string, recordingId?: number) => {
1998+
if (recording && recordingId !== undefined) recordingCompletion.started(recordingId);
19951999
appRecording = recording;
1996-
mcpAppRecorder.recordingChanged(recording);
2000+
mcpAppRecorder.recordingChanged(recording, recordingId);
19972001
onRecordingStateChange?.(recording, sourceName);
19982002
};
19992003
const appSettings = new AppSettingsStore(app.getPath("userData"));
@@ -2782,7 +2786,7 @@ export function registerIpcHandlers(
27822786
// "Screen" }`, so the tray confidently displayed the name of a
27832787
// window the capture had never been told about.
27842788
linuxNativeCaptureSourceLabel = linuxSourceLabel(session.grantedSourceKind);
2785-
setAppRecording(true, linuxNativeCaptureSourceLabel);
2789+
setAppRecording(true, linuxNativeCaptureSourceLabel, recordingId);
27862790

27872791
return { success: true, recordingId, path: outputPath };
27882792
} catch (error) {
@@ -3078,7 +3082,7 @@ export function registerIpcHandlers(
30783082
});
30793083

30803084
const source = selectedSource || { name: "Screen" };
3081-
setAppRecording(true, source.name);
3085+
setAppRecording(true, source.name, recordingId);
30823086

30833087
// Reported at start, not at stop: the helper decides the camera is a
30843088
// lost cause during its own init — before it announces "Recording
@@ -3350,7 +3354,7 @@ export function registerIpcHandlers(
33503354
: 0;
33513355

33523356
const source = selectedSource || { name: "Screen" };
3353-
setAppRecording(true, source.name);
3357+
setAppRecording(true, source.name, recordingId);
33543358

33553359
return {
33563360
success: true,
@@ -4145,16 +4149,15 @@ export function registerIpcHandlers(
41454149
ipcMain.handle(
41464150
"set-recording-state",
41474151
async (_, recording: boolean, recordingId?: number, cursorCaptureMode?: CursorCaptureMode) => {
4152+
const source = selectedSource || { name: "Screen" };
4153+
setAppRecording(recording, source.name, recordingId);
41484154
const normalizedCursorCaptureMode =
41494155
normalizeCursorCaptureMode(cursorCaptureMode) ?? "editable-overlay";
41504156
if (recording && normalizedCursorCaptureMode === "editable-overlay") {
41514157
await startCursorRecording(recordingId);
41524158
} else {
41534159
await stopCursorRecording();
41544160
}
4155-
4156-
const source = selectedSource || { name: "Screen" };
4157-
setAppRecording(recording, source.name);
41584161
},
41594162
);
41604163

@@ -4896,10 +4899,43 @@ export function registerIpcHandlers(
48964899
setCurrentRecordingSessionState(normalizedSession);
48974900
currentVideoPath = normalizedSession?.screenVideoPath ?? null;
48984901
currentProjectPath = null;
4899-
if (normalizedSession) mcpAppRecorder.takeSaved(normalizedSession.screenVideoPath);
4902+
if (normalizedSession) {
4903+
mcpAppRecorder.takeSaved(normalizedSession.screenVideoPath);
4904+
recordingCompletion.saved(normalizedSession.createdAt, normalizedSession.screenVideoPath);
4905+
}
49004906
return { success: true, session: currentRecordingSession };
49014907
});
49024908

4909+
// A renderer acknowledgement of the actual hand-off, after the duration and
4910+
// automatic zoom saves. An imported/opened project never emits this signal.
4911+
const recordingOutcomeSchema = z.discriminatedUnion("status", [
4912+
z.object({
4913+
recordingId: z.number().finite(),
4914+
status: z.literal("ready"),
4915+
projectId: z.string(),
4916+
}),
4917+
z.object({
4918+
recordingId: z.number().finite(),
4919+
status: z.enum(["failed", "cancelled"]),
4920+
message: z.string(),
4921+
}),
4922+
]);
4923+
ipcMain.handle("recording-outcome", async (_, input: unknown) => {
4924+
const parsed = recordingOutcomeSchema.safeParse(input);
4925+
if (!parsed.success) return { success: false };
4926+
const outcome = parsed.data;
4927+
if (outcome.status !== "ready") {
4928+
recordingCompletion.failed(outcome.recordingId, outcome.message);
4929+
mcpAppRecorder.recordingFinished(outcome.recordingId);
4930+
return { success: true };
4931+
}
4932+
await recordingCompletion.projectReady(outcome.recordingId, outcome.projectId, (id) =>
4933+
aiEditionDocuments.getProject(id),
4934+
);
4935+
mcpAppRecorder.recordingFinished(outcome.recordingId);
4936+
return { success: true };
4937+
});
4938+
49034939
ipcMain.handle("get-current-recording-session", () => {
49044940
if (!currentRecordingSession) {
49054941
return { success: false };
@@ -5183,6 +5219,8 @@ export function registerIpcHandlers(
51835219
host: mcpToolDeps.host,
51845220
});
51855221
const mcpMedia = createMediaTools({
5222+
recordingActive: () => recordingCompletion.isActive(),
5223+
waitForRecording: (timeoutMs, signal) => recordingCompletion.wait(timeoutMs, signal),
51865224
transport: "http",
51875225
// Recording and exporting change things too: the same switch as edits.
51885226
editsAllowed: mcpToolDeps.editsAllowed,

‎electron/ipc/recordingStateRouting.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ describe("recording state routing", () => {
3636
const setter = source.slice(source.indexOf("const setAppRecording"));
3737
const body = setter.slice(0, setter.indexOf("};"));
3838
expect(body).toContain("appRecording = recording");
39-
expect(body).toContain("mcpAppRecorder.recordingChanged(recording)");
39+
expect(body).toContain("mcpAppRecorder.recordingChanged(recording, recordingId)");
4040
expect(body).toContain("onRecordingStateChange?.(recording, sourceName)");
4141
});
4242

‎electron/mcp/app-recorder.test.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,20 @@ describe("an MCP take in the app", () => {
157157
expect(editor.applied).toEqual([]);
158158
});
159159

160+
it("releases only the client take whose app lifecycle finished", async () => {
161+
const { recorder, calls } = setup();
162+
const starting = recorder.start(REQUEST);
163+
await new Promise((resolve) => setTimeout(resolve, 5));
164+
recorder.recordingChanged(true, 7);
165+
await starting;
166+
recorder.recordingFinished(6);
167+
await expect(recorder.start(REQUEST)).rejects.toThrow("already recording");
168+
recorder.recordingChanged(false, 7);
169+
recorder.recordingFinished(7);
170+
expect(calls.filter((call) => call === "restore")).toHaveLength(1);
171+
await expect(started(recorder, calls)).resolves.toBeDefined();
172+
});
173+
160174
it("refuses a take while one records, or while the editor holds unsaved changes", async () => {
161175
const busy = setup({ isRecording: () => true });
162176
await expect(busy.recorder.start(REQUEST)).rejects.toThrow("already recording");

‎electron/mcp/app-recorder.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,9 @@ export interface AppRecorderDeps {
103103

104104
export interface AppRecorder extends TakeRecorder {
105105
/** The app's recording state changed (`set-recording-state`). */
106-
recordingChanged(recording: boolean): void;
106+
recordingChanged(recording: boolean, recordingId?: number): void;
107+
/** A completed or failed app take no longer owns the recording bar. */
108+
recordingFinished(recordingId: number): void;
107109
/** The recording bar handed a saved take to the editor (`set-current-recording-session`). */
108110
takeSaved(videoPath: string): void;
109111
}
@@ -145,6 +147,7 @@ interface Deferred<T> {
145147
}
146148

147149
interface ClientTake {
150+
recordingId?: number;
148151
phase: "starting" | "recording" | "saving";
149152
/** Resolves with when capture started. */
150153
started: Deferred<number>;
@@ -214,10 +217,16 @@ export function createAppRecorder(deps: AppRecorderDeps): AppRecorder {
214217
}
215218

216219
return {
217-
recordingChanged(recording) {
220+
recordingFinished(recordingId) {
221+
if (current?.recordingId !== recordingId) return;
222+
current.restore();
223+
current = null;
224+
},
225+
recordingChanged(recording, recordingId) {
218226
if (!current) return;
219227
if (recording && current.phase === "starting") {
220228
current.phase = "recording";
229+
current.recordingId = recordingId;
221230
current.started.resolve(Date.now());
222231
} else if (!recording && current.phase === "recording") {
223232
// The take is captured: the bar's own settings can come back while it saves.

‎electron/mcp/media-tools.test.ts‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import {
2626
createToolRunner,
2727
type McpToolDeps,
2828
} from "./openscreen-mcp-server";
29+
import { RecordingCompletion } from "./recording-completion";
2930

3031
/** The runner windows: records what was asked, answers like a runner would. */
3132
class FakeRunners {
@@ -1069,3 +1070,146 @@ describe("the media tools in the app", () => {
10691070
expect(exportPath.description).not.toContain("working directory, ending");
10701071
});
10711072
});
1073+
1074+
describe("waitForRecording through MCP", () => {
1075+
const clients: Client[] = [];
1076+
afterEach(async () => {
1077+
for (const client of clients.splice(0)) await client.close();
1078+
});
1079+
async function observing(
1080+
completion: RecordingCompletion,
1081+
overrides: Partial<MediaToolDeps> = {},
1082+
) {
1083+
const deps: McpToolDeps = {
1084+
transport: "http",
1085+
host: { snapshot: async () => null, apply: async () => "no-editor" },
1086+
projects: documents,
1087+
editsAllowed: () => false,
1088+
version: "test",
1089+
};
1090+
let subscribed: () => void = () => undefined;
1091+
const subscription = new Promise<void>((resolve) => {
1092+
subscribed = resolve;
1093+
});
1094+
const media = createMediaTools({
1095+
...stdioDeps,
1096+
transport: "http",
1097+
recordingActive: () => completion.isActive(),
1098+
editsAllowed: () => false,
1099+
waitForRecording: (timeout, signal) => {
1100+
const waiting = completion.wait(timeout, signal);
1101+
subscribed();
1102+
return waiting;
1103+
},
1104+
...overrides,
1105+
});
1106+
const server = createOpenScreenMcpServer(deps, createToolRunner(deps), media);
1107+
const [clientSide, serverSide] = InMemoryTransport.createLinkedPair();
1108+
await server.connect(serverSide);
1109+
const observer = new Client({ name: "observer", version: "1" });
1110+
await observer.connect(clientSide);
1111+
clients.push(observer);
1112+
return { observer, subscription };
1113+
}
1114+
1115+
it("returns the stopRecording fields for a user take while MCP edits is off, without recording commands", async () => {
1116+
const project = await documents.createProject("User take");
1117+
const completion = new RecordingCompletion();
1118+
completion.started(42, 1_000);
1119+
const { observer, subscription } = await observing(completion);
1120+
const waiting = observer.callTool({ name: "waitForRecording", arguments: {} });
1121+
await subscription;
1122+
completion.saved(42, videoPath);
1123+
completion.completed(42, {
1124+
projectId: project.project.id,
1125+
videoPath,
1126+
durationSec: 12,
1127+
autoZooms: 2,
1128+
});
1129+
const result = await waiting;
1130+
expect(result.isError).toBeFalsy();
1131+
expect(JSON.parse(textOf(result as CallToolResult))).toEqual({
1132+
projectId: project.project.id,
1133+
videoPath,
1134+
durationSec: 12,
1135+
autoZooms: 2,
1136+
recordingStartedAt: 1_000,
1137+
checkpointId: expect.any(String),
1138+
});
1139+
expect(runners.started).toEqual([]);
1140+
});
1141+
1142+
it("releases a client take after the user finishes it, so startRecording can record again", async () => {
1143+
const completion = new RecordingCompletion();
1144+
let id = 0;
1145+
const recorder: TakeRecorder = {
1146+
start: async () => {
1147+
completion.started(++id);
1148+
return {
1149+
startedAt: id,
1150+
stop: async () => {
1151+
throw new Error("wait must never stop the take");
1152+
},
1153+
};
1154+
},
1155+
};
1156+
const { observer, subscription } = await observing(completion, {
1157+
recorder,
1158+
editsAllowed: () => true,
1159+
});
1160+
expect(
1161+
(await observer.callTool({ name: "startRecording", arguments: {} })).isError,
1162+
).toBeFalsy();
1163+
const waiting = observer.callTool({ name: "waitForRecording", arguments: {} });
1164+
await subscription;
1165+
completion.saved(1, videoPath);
1166+
completion.completed(1, { projectId: "proj_take", videoPath, durationSec: 12, autoZooms: 0 });
1167+
await waiting;
1168+
expect(
1169+
(await observer.callTool({ name: "startRecording", arguments: {} })).isError,
1170+
).toBeFalsy();
1171+
expect(id).toBe(2);
1172+
});
1173+
1174+
it.each([
1175+
0,
1176+
-1,
1177+
1.5,
1178+
3601,
1179+
"10",
1180+
])("validates timeoutSec=%s before observing", async (timeoutSec) => {
1181+
const completion = new RecordingCompletion();
1182+
const { observer } = await observing(completion);
1183+
const result = await observer.callTool({ name: "waitForRecording", arguments: { timeoutSec } });
1184+
expect(result.isError).toBe(true);
1185+
expect(textOf(result as CallToolResult)).toContain("validation");
1186+
});
1187+
1188+
it("reports no active recording as a tool error", async () => {
1189+
const { observer } = await observing(new RecordingCompletion());
1190+
const result = await observer.callTool({ name: "waitForRecording", arguments: {} });
1191+
expect(result.isError).toBe(true);
1192+
expect(textOf(result as CallToolResult)).toContain("No recording");
1193+
});
1194+
1195+
it("propagates MCP cancellation and leaves the user's take available", async () => {
1196+
const completion = new RecordingCompletion();
1197+
completion.started(42);
1198+
const { observer, subscription } = await observing(completion);
1199+
const abort = new AbortController();
1200+
const waiting = expect(
1201+
observer.callTool({ name: "waitForRecording", arguments: {} }, undefined, {
1202+
signal: abort.signal,
1203+
}),
1204+
).rejects.toThrow();
1205+
await subscription;
1206+
abort.abort();
1207+
await waiting;
1208+
// Flush the MCP cancellation notification, then the same take still completes.
1209+
await new Promise((resolve) => setTimeout(resolve, 0));
1210+
const retry = completion.wait(1000, new AbortController().signal);
1211+
completion.saved(42, videoPath);
1212+
completion.completed(42, { projectId: "proj_take", videoPath, durationSec: 12, autoZooms: 0 });
1213+
await expect(retry).resolves.toMatchObject({ projectId: "proj_take" });
1214+
});
1215+
});

0 commit comments

Comments
 (0)