Skip to content

Commit e06e69b

Browse files
author
wuzixuan06
committed
feat: add waitForRecording MCP tool for user-controlled takes
1 parent d8bf2d8 commit e06e69b

19 files changed

Lines changed: 948 additions & 65 deletions

‎electron/electron-env.d.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@ interface Window {
4646
meta: import("../src/native/contracts").CompositorSharedFrameMeta,
4747
) => void,
4848
) => () => void;
49+
reportRecordingOutcome?: (
50+
outcome: import("../src/lib/recordingSession").RecordingOutcome,
51+
) => Promise<{ success: boolean }>;
52+
4953
getSources: (opts: Electron.SourcesOptions) => Promise<ProcessedDesktopSource[]>;
5054
switchToEditor: () => Promise<void>;
5155
switchToHud: () => Promise<void>;

‎electron/ipc/handlers.ts‎

Lines changed: 47 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
shell,
1919
systemPreferences,
2020
} from "electron";
21+
import { z } from "zod";
2122
import { DEFAULT_WEBCAM_QUALITY, type WebcamQualityId } from "../../src/hooks/webcamCaptureTarget";
2223
import {
2324
type AxcutDocument,
@@ -88,6 +89,7 @@ import { McpController } from "../mcp/mcp-controller";
8889
import { McpSettingsStore } from "../mcp/mcp-settings-store";
8990
import { createMediaTools, type MediaToolDeps, type RecordRequest } from "../mcp/media-tools";
9091
import { createToolRunner, type McpToolDeps } from "../mcp/openscreen-mcp-server";
92+
import { RecordingCompletion } from "../mcp/recording-completion";
9193
import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks";
9294
import {
9395
readCursorRecordingFile as readCursorRecordingFileFrom,
@@ -1966,15 +1968,17 @@ export function registerIpcHandlers(
19661968
) {
19671969
/** The HUD is recording a take, the user's or an MCP client's. Set by `setAppRecording` only. */
19681970
let appRecording = false;
1971+
const recordingCompletion = new RecordingCompletion();
19691972
/**
19701973
* Every start and stop of a take goes through here: the browser path's `set-recording-state`
19711974
* and the native paths, which start and stop in this process. The MCP recorder waits on it
19721975
* for a client's take to start, and the tray shows it. A native path that called
19731976
* `onRecordingStateChange` itself left the MCP recorder waiting on a take already running.
19741977
*/
1975-
const setAppRecording = (recording: boolean, sourceName: string) => {
1978+
const setAppRecording = (recording: boolean, sourceName: string, recordingId?: number) => {
1979+
if (recording && recordingId !== undefined) recordingCompletion.started(recordingId);
19761980
appRecording = recording;
1977-
mcpAppRecorder.recordingChanged(recording);
1981+
mcpAppRecorder.recordingChanged(recording, recordingId);
19781982
onRecordingStateChange?.(recording, sourceName);
19791983
};
19801984
const appSettings = new AppSettingsStore(app.getPath("userData"));
@@ -2760,7 +2764,7 @@ export function registerIpcHandlers(
27602764
// "Screen" }`, so the tray confidently displayed the name of a
27612765
// window the capture had never been told about.
27622766
linuxNativeCaptureSourceLabel = linuxSourceLabel(session.grantedSourceKind);
2763-
setAppRecording(true, linuxNativeCaptureSourceLabel);
2767+
setAppRecording(true, linuxNativeCaptureSourceLabel, recordingId);
27642768

27652769
return { success: true, recordingId, path: outputPath };
27662770
} catch (error) {
@@ -3056,7 +3060,7 @@ export function registerIpcHandlers(
30563060
});
30573061

30583062
const source = selectedSource || { name: "Screen" };
3059-
setAppRecording(true, source.name);
3063+
setAppRecording(true, source.name, recordingId);
30603064

30613065
// Reported at start, not at stop: the helper decides the camera is a
30623066
// lost cause during its own init — before it announces "Recording
@@ -3318,7 +3322,7 @@ export function registerIpcHandlers(
33183322
: 0;
33193323

33203324
const source = selectedSource || { name: "Screen" };
3321-
setAppRecording(true, source.name);
3325+
setAppRecording(true, source.name, recordingId);
33223326

33233327
return {
33243328
success: true,
@@ -4113,16 +4117,15 @@ export function registerIpcHandlers(
41134117
ipcMain.handle(
41144118
"set-recording-state",
41154119
async (_, recording: boolean, recordingId?: number, cursorCaptureMode?: CursorCaptureMode) => {
4120+
const source = selectedSource || { name: "Screen" };
4121+
setAppRecording(recording, source.name, recordingId);
41164122
const normalizedCursorCaptureMode =
41174123
normalizeCursorCaptureMode(cursorCaptureMode) ?? "editable-overlay";
41184124
if (recording && normalizedCursorCaptureMode === "editable-overlay") {
41194125
await startCursorRecording(recordingId);
41204126
} else {
41214127
await stopCursorRecording();
41224128
}
4123-
4124-
const source = selectedSource || { name: "Screen" };
4125-
setAppRecording(recording, source.name);
41264129
},
41274130
);
41284131

@@ -4864,10 +4867,43 @@ export function registerIpcHandlers(
48644867
setCurrentRecordingSessionState(normalizedSession);
48654868
currentVideoPath = normalizedSession?.screenVideoPath ?? null;
48664869
currentProjectPath = null;
4867-
if (normalizedSession) mcpAppRecorder.takeSaved(normalizedSession.screenVideoPath);
4870+
if (normalizedSession) {
4871+
mcpAppRecorder.takeSaved(normalizedSession.screenVideoPath);
4872+
recordingCompletion.saved(normalizedSession.createdAt, normalizedSession.screenVideoPath);
4873+
}
48684874
return { success: true, session: currentRecordingSession };
48694875
});
48704876

4877+
// A renderer acknowledgement of the actual hand-off, after the duration and
4878+
// automatic zoom saves. An imported/opened project never emits this signal.
4879+
const recordingOutcomeSchema = z.discriminatedUnion("status", [
4880+
z.object({
4881+
recordingId: z.number().finite(),
4882+
status: z.literal("ready"),
4883+
projectId: z.string(),
4884+
}),
4885+
z.object({
4886+
recordingId: z.number().finite(),
4887+
status: z.enum(["failed", "cancelled"]),
4888+
message: z.string(),
4889+
}),
4890+
]);
4891+
ipcMain.handle("recording-outcome", async (_, input: unknown) => {
4892+
const parsed = recordingOutcomeSchema.safeParse(input);
4893+
if (!parsed.success) return { success: false };
4894+
const outcome = parsed.data;
4895+
if (outcome.status !== "ready") {
4896+
recordingCompletion.failed(outcome.recordingId, outcome.message);
4897+
mcpAppRecorder.recordingFinished(outcome.recordingId);
4898+
return { success: true };
4899+
}
4900+
await recordingCompletion.projectReady(outcome.recordingId, outcome.projectId, (id) =>
4901+
aiEditionDocuments.getProject(id),
4902+
);
4903+
mcpAppRecorder.recordingFinished(outcome.recordingId);
4904+
return { success: true };
4905+
});
4906+
48714907
ipcMain.handle("get-current-recording-session", () => {
48724908
if (!currentRecordingSession) {
48734909
return { success: false };
@@ -5151,6 +5187,8 @@ export function registerIpcHandlers(
51515187
host: mcpToolDeps.host,
51525188
});
51535189
const mcpMedia = createMediaTools({
5190+
recordingActive: () => recordingCompletion.isActive(),
5191+
waitForRecording: (timeoutMs, signal) => recordingCompletion.wait(timeoutMs, signal),
51545192
transport: "http",
51555193
// Recording and exporting change things too: the same switch as edits.
51565194
editsAllowed: mcpToolDeps.editsAllowed,

‎electron/ipc/recordingStateRouting.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ describe("recording state routing", () => {
3636
const setter = source.slice(source.indexOf("const setAppRecording"));
3737
const body = setter.slice(0, setter.indexOf("};"));
3838
expect(body).toContain("appRecording = recording");
39-
expect(body).toContain("mcpAppRecorder.recordingChanged(recording)");
39+
expect(body).toContain("mcpAppRecorder.recordingChanged(recording, recordingId)");
4040
expect(body).toContain("onRecordingStateChange?.(recording, sourceName)");
4141
});
4242

‎electron/mcp/app-recorder.test.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,20 @@ describe("an MCP take in the app", () => {
157157
expect(editor.applied).toEqual([]);
158158
});
159159

160+
it("releases only the client take whose app lifecycle finished", async () => {
161+
const { recorder, calls } = setup();
162+
const starting = recorder.start(REQUEST);
163+
await new Promise((resolve) => setTimeout(resolve, 5));
164+
recorder.recordingChanged(true, 7);
165+
await starting;
166+
recorder.recordingFinished(6);
167+
await expect(recorder.start(REQUEST)).rejects.toThrow("already recording");
168+
recorder.recordingChanged(false, 7);
169+
recorder.recordingFinished(7);
170+
expect(calls.filter((call) => call === "restore")).toHaveLength(1);
171+
await expect(started(recorder, calls)).resolves.toBeDefined();
172+
});
173+
160174
it("refuses a take while one records, or while the editor holds unsaved changes", async () => {
161175
const busy = setup({ isRecording: () => true });
162176
await expect(busy.recorder.start(REQUEST)).rejects.toThrow("already recording");

‎electron/mcp/app-recorder.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,9 @@ export interface AppRecorderDeps {
103103

104104
export interface AppRecorder extends TakeRecorder {
105105
/** The app's recording state changed (`set-recording-state`). */
106-
recordingChanged(recording: boolean): void;
106+
recordingChanged(recording: boolean, recordingId?: number): void;
107+
/** A completed or failed app take no longer owns the recording bar. */
108+
recordingFinished(recordingId: number): void;
107109
/** The recording bar handed a saved take to the editor (`set-current-recording-session`). */
108110
takeSaved(videoPath: string): void;
109111
}
@@ -145,6 +147,7 @@ interface Deferred<T> {
145147
}
146148

147149
interface ClientTake {
150+
recordingId?: number;
148151
phase: "starting" | "recording" | "saving";
149152
/** Resolves with when capture started. */
150153
started: Deferred<number>;
@@ -214,10 +217,16 @@ export function createAppRecorder(deps: AppRecorderDeps): AppRecorder {
214217
}
215218

216219
return {
217-
recordingChanged(recording) {
220+
recordingFinished(recordingId) {
221+
if (current?.recordingId !== recordingId) return;
222+
current.restore();
223+
current = null;
224+
},
225+
recordingChanged(recording, recordingId) {
218226
if (!current) return;
219227
if (recording && current.phase === "starting") {
220228
current.phase = "recording";
229+
current.recordingId = recordingId;
221230
current.started.resolve(Date.now());
222231
} else if (!recording && current.phase === "recording") {
223232
// The take is captured: the bar's own settings can come back while it saves.

‎electron/mcp/media-tools.test.ts‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import {
2626
createToolRunner,
2727
type McpToolDeps,
2828
} from "./openscreen-mcp-server";
29+
import { RecordingCompletion } from "./recording-completion";
2930

3031
/** The runner windows: records what was asked, answers like a runner would. */
3132
class FakeRunners {
@@ -1033,3 +1034,146 @@ describe("the media tools in the app", () => {
10331034
expect(exportPath.description).not.toContain("working directory, ending");
10341035
});
10351036
});
1037+
1038+
describe("waitForRecording through MCP", () => {
1039+
const clients: Client[] = [];
1040+
afterEach(async () => {
1041+
for (const client of clients.splice(0)) await client.close();
1042+
});
1043+
async function observing(
1044+
completion: RecordingCompletion,
1045+
overrides: Partial<MediaToolDeps> = {},
1046+
) {
1047+
const deps: McpToolDeps = {
1048+
transport: "http",
1049+
host: { snapshot: async () => null, apply: async () => "no-editor" },
1050+
projects: documents,
1051+
editsAllowed: () => false,
1052+
version: "test",
1053+
};
1054+
let subscribed: () => void = () => undefined;
1055+
const subscription = new Promise<void>((resolve) => {
1056+
subscribed = resolve;
1057+
});
1058+
const media = createMediaTools({
1059+
...stdioDeps,
1060+
transport: "http",
1061+
recordingActive: () => completion.isActive(),
1062+
editsAllowed: () => false,
1063+
waitForRecording: (timeout, signal) => {
1064+
const waiting = completion.wait(timeout, signal);
1065+
subscribed();
1066+
return waiting;
1067+
},
1068+
...overrides,
1069+
});
1070+
const server = createOpenScreenMcpServer(deps, createToolRunner(deps), media);
1071+
const [clientSide, serverSide] = InMemoryTransport.createLinkedPair();
1072+
await server.connect(serverSide);
1073+
const observer = new Client({ name: "observer", version: "1" });
1074+
await observer.connect(clientSide);
1075+
clients.push(observer);
1076+
return { observer, subscription };
1077+
}
1078+
1079+
it("returns the stopRecording fields for a user take while MCP edits is off, without recording commands", async () => {
1080+
const project = await documents.createProject("User take");
1081+
const completion = new RecordingCompletion();
1082+
completion.started(42, 1_000);
1083+
const { observer, subscription } = await observing(completion);
1084+
const waiting = observer.callTool({ name: "waitForRecording", arguments: {} });
1085+
await subscription;
1086+
completion.saved(42, videoPath);
1087+
completion.completed(42, {
1088+
projectId: project.project.id,
1089+
videoPath,
1090+
durationSec: 12,
1091+
autoZooms: 2,
1092+
});
1093+
const result = await waiting;
1094+
expect(result.isError).toBeFalsy();
1095+
expect(JSON.parse(textOf(result as CallToolResult))).toEqual({
1096+
projectId: project.project.id,
1097+
videoPath,
1098+
durationSec: 12,
1099+
autoZooms: 2,
1100+
recordingStartedAt: 1_000,
1101+
checkpointId: expect.any(String),
1102+
});
1103+
expect(runners.started).toEqual([]);
1104+
});
1105+
1106+
it("releases a client take after the user finishes it, so startRecording can record again", async () => {
1107+
const completion = new RecordingCompletion();
1108+
let id = 0;
1109+
const recorder: TakeRecorder = {
1110+
start: async () => {
1111+
completion.started(++id);
1112+
return {
1113+
startedAt: id,
1114+
stop: async () => {
1115+
throw new Error("wait must never stop the take");
1116+
},
1117+
};
1118+
},
1119+
};
1120+
const { observer, subscription } = await observing(completion, {
1121+
recorder,
1122+
editsAllowed: () => true,
1123+
});
1124+
expect(
1125+
(await observer.callTool({ name: "startRecording", arguments: {} })).isError,
1126+
).toBeFalsy();
1127+
const waiting = observer.callTool({ name: "waitForRecording", arguments: {} });
1128+
await subscription;
1129+
completion.saved(1, videoPath);
1130+
completion.completed(1, { projectId: "proj_take", videoPath, durationSec: 12, autoZooms: 0 });
1131+
await waiting;
1132+
expect(
1133+
(await observer.callTool({ name: "startRecording", arguments: {} })).isError,
1134+
).toBeFalsy();
1135+
expect(id).toBe(2);
1136+
});
1137+
1138+
it.each([
1139+
0,
1140+
-1,
1141+
1.5,
1142+
3601,
1143+
"10",
1144+
])("validates timeoutSec=%s before observing", async (timeoutSec) => {
1145+
const completion = new RecordingCompletion();
1146+
const { observer } = await observing(completion);
1147+
const result = await observer.callTool({ name: "waitForRecording", arguments: { timeoutSec } });
1148+
expect(result.isError).toBe(true);
1149+
expect(textOf(result as CallToolResult)).toContain("validation");
1150+
});
1151+
1152+
it("reports no active recording as a tool error", async () => {
1153+
const { observer } = await observing(new RecordingCompletion());
1154+
const result = await observer.callTool({ name: "waitForRecording", arguments: {} });
1155+
expect(result.isError).toBe(true);
1156+
expect(textOf(result as CallToolResult)).toContain("No recording");
1157+
});
1158+
1159+
it("propagates MCP cancellation and leaves the user's take available", async () => {
1160+
const completion = new RecordingCompletion();
1161+
completion.started(42);
1162+
const { observer, subscription } = await observing(completion);
1163+
const abort = new AbortController();
1164+
const waiting = expect(
1165+
observer.callTool({ name: "waitForRecording", arguments: {} }, undefined, {
1166+
signal: abort.signal,
1167+
}),
1168+
).rejects.toThrow();
1169+
await subscription;
1170+
abort.abort();
1171+
await waiting;
1172+
// Flush the MCP cancellation notification, then the same take still completes.
1173+
await new Promise((resolve) => setTimeout(resolve, 0));
1174+
const retry = completion.wait(1000, new AbortController().signal);
1175+
completion.saved(42, videoPath);
1176+
completion.completed(42, { projectId: "proj_take", videoPath, durationSec: 12, autoZooms: 0 });
1177+
await expect(retry).resolves.toMatchObject({ projectId: "proj_take" });
1178+
});
1179+
});

0 commit comments

Comments
 (0)