Skip to content

Commit f22c9b4

Browse files
stym06stym06
authored
Merge branch 'main' into codex/macos-floating-camera-preview
2 parents 20975d3 + 3c9bd5d commit f22c9b4

15 files changed

Lines changed: 370 additions & 45 deletions

File tree

‎electron/ipc/handlers.ts‎

Lines changed: 6 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ import {
4848
import {
4949
AI_EDITION_OPEN_PROJECT_CHANNEL,
5050
type CursorRecordingData,
51-
type CursorRecordingSample,
5251
type ProjectFileResult,
5352
type ProjectPathResult,
5453
} from "../../src/native/contracts";
@@ -103,6 +102,7 @@ import {
103102
type LinuxCaptureSourceKind,
104103
LinuxNativeCaptureSession,
105104
} from "../native-bridge/capture/linuxNativeCaptureSession";
105+
import { compactCursorTelemetryPauseRanges } from "../native-bridge/cursor/recording/cursorPauseRanges";
106106
import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory";
107107
import {
108108
isMacCursorHelperUnavailable,
@@ -1434,45 +1434,11 @@ function shiftPendingCursorTelemetry(offsetMs: number) {
14341434
function compactPendingCursorTelemetryPauseRanges(
14351435
ranges: Array<{ startMs: number; endMs: number }>,
14361436
) {
1437-
if (!pendingCursorRecordingData || ranges.length === 0) {
1438-
return;
1439-
}
1440-
1441-
const normalizedRanges = ranges
1442-
.map((range) => ({
1443-
startMs: Math.max(0, Math.min(range.startMs, range.endMs)),
1444-
endMs: Math.max(0, Math.max(range.startMs, range.endMs)),
1445-
}))
1446-
.filter((range) => Number.isFinite(range.startMs) && Number.isFinite(range.endMs))
1447-
.filter((range) => range.endMs > range.startMs)
1448-
.sort((a, b) => a.startMs - b.startMs);
1449-
1450-
if (normalizedRanges.length === 0) {
1451-
return;
1452-
}
1453-
1454-
pendingCursorRecordingData = {
1455-
...pendingCursorRecordingData,
1456-
samples: pendingCursorRecordingData.samples
1457-
.map((sample) => {
1458-
let pausedBeforeSampleMs = 0;
1459-
for (const range of normalizedRanges) {
1460-
if (sample.timeMs >= range.startMs && sample.timeMs <= range.endMs) {
1461-
return null;
1462-
}
1463-
if (sample.timeMs > range.endMs) {
1464-
pausedBeforeSampleMs += range.endMs - range.startMs;
1465-
}
1466-
}
1467-
1468-
return {
1469-
...sample,
1470-
timeMs: Math.max(0, sample.timeMs - pausedBeforeSampleMs),
1471-
};
1472-
})
1473-
.filter((sample): sample is CursorRecordingSample => Boolean(sample))
1474-
.sort((a, b) => a.timeMs - b.timeMs),
1475-
};
1437+
if (!pendingCursorRecordingData) return;
1438+
pendingCursorRecordingData = compactCursorTelemetryPauseRanges(
1439+
pendingCursorRecordingData,
1440+
ranges,
1441+
);
14761442
}
14771443

14781444
function completeNativeMacCursorPauseRange(endMs = Date.now()) {

‎electron/native-bridge/capture/linuxNativeCaptureSession.test.ts‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,45 @@ function flushStdout() {
8585
return new Promise((resolve) => setTimeout(resolve, 0));
8686
}
8787

88+
function emitCaptureStarted(timestampMs: number) {
89+
helper.emitEvent({
90+
event: "capture-started",
91+
timestampMs,
92+
path: "/tmp/recording.mp4",
93+
width: 800,
94+
height: 600,
95+
fps: 30,
96+
});
97+
}
98+
99+
function emitCursorSample(timestampMs: number) {
100+
helper.emitEvent({
101+
event: "cursor-sample",
102+
timestampMs,
103+
x: 400,
104+
y: 300,
105+
width: 800,
106+
height: 600,
107+
visible: true,
108+
});
109+
}
110+
111+
async function finishCapture(session: LinuxNativeCaptureSession, timestampMs: number) {
112+
const stopping = session.stop();
113+
await flushStdout();
114+
helper.emitEvent({
115+
event: "capture-stopped",
116+
timestampMs,
117+
path: "/tmp/recording.mp4",
118+
durationMs: timestampMs - 1_000,
119+
frames: 27,
120+
dropped: 0,
121+
});
122+
await flushStdout();
123+
helper.emit("exit", 0, null);
124+
return stopping;
125+
}
126+
88127
/** Drives a session to a finished recording, optionally granting a source kind. */
89128
async function record(sourceKind?: string) {
90129
const session = newSession();
@@ -435,6 +474,59 @@ describe("LinuxNativeCaptureSession", () => {
435474
await expect(session.waitUntilCapturing()).resolves.toBeUndefined();
436475
});
437476

477+
it("removes cursor samples from repeated pauses and compacts later timestamps", async () => {
478+
const now = vi.spyOn(Date, "now").mockReturnValue(900);
479+
const session = newSession();
480+
await startReady(session);
481+
482+
emitCaptureStarted(1_000);
483+
emitCursorSample(1_100);
484+
await flushStdout();
485+
486+
now.mockReturnValue(1_200);
487+
session.pause();
488+
emitCursorSample(1_250);
489+
now.mockReturnValue(1_400);
490+
session.resume();
491+
emitCursorSample(1_500);
492+
493+
now.mockReturnValue(1_600);
494+
session.pause();
495+
emitCursorSample(1_650);
496+
now.mockReturnValue(1_800);
497+
session.resume();
498+
emitCursorSample(1_900);
499+
await flushStdout();
500+
501+
now.mockReturnValue(2_000);
502+
const result = await finishCapture(session, 2_000);
503+
504+
expect(result.cursor.samples.map((sample) => sample.timeMs)).toEqual([100, 300, 500]);
505+
expect(helper.stdinWrites.filter((line) => line === "pause\n")).toHaveLength(2);
506+
expect(helper.stdinWrites.filter((line) => line === "resume\n")).toHaveLength(2);
507+
});
508+
509+
it("closes the cursor pause range when the recording stops while paused", async () => {
510+
const now = vi.spyOn(Date, "now").mockReturnValue(900);
511+
const session = newSession();
512+
await startReady(session);
513+
514+
emitCaptureStarted(1_000);
515+
emitCursorSample(1_100);
516+
await flushStdout();
517+
518+
now.mockReturnValue(1_200);
519+
session.pause();
520+
emitCursorSample(1_300);
521+
await flushStdout();
522+
523+
now.mockReturnValue(1_500);
524+
const result = await finishCapture(session, 1_500);
525+
526+
expect(result.cursor.samples.map((sample) => sample.timeMs)).toEqual([100]);
527+
expect(helper.stdinWrites).not.toContain("resume\n");
528+
});
529+
438530
/**
439531
* A non-fatal `error` event after the first frame does not end the capture;
440532
* a later wait is still satisfied.

‎electron/native-bridge/capture/linuxNativeCaptureSession.ts‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import { type ChildProcessByStdio, spawn } from "node:child_process";
22
import type { Readable, Writable } from "node:stream";
33
import type { CursorRecordingData } from "../../../src/native/contracts";
4+
import {
5+
type CursorPauseRange,
6+
compactCursorTelemetryPauseRanges,
7+
} from "../cursor/recording/cursorPauseRanges";
48
import {
59
NdjsonLineReader,
610
PipeWireCursorAccumulator,
@@ -114,6 +118,9 @@ export class LinuxNativeCaptureSession {
114118
private lastError: string | null = null;
115119
private paused = false;
116120
private helperOutputText = "";
121+
private cursorRecordingStartMs = 0;
122+
private cursorPauseStartedAtMs: number | null = null;
123+
private cursorPauseRanges: CursorPauseRange[] = [];
117124

118125
constructor(private readonly config: LinuxCaptureConfig) {
119126
this.cursor = new PipeWireCursorAccumulator(config.maxCursorSamples);
@@ -134,6 +141,10 @@ export class LinuxNativeCaptureSession {
134141

135142
this.cursor.reset(Date.now());
136143
this.lines.reset();
144+
this.cursorRecordingStartMs = 0;
145+
this.cursorPauseStartedAtMs = null;
146+
this.cursorPauseRanges = [];
147+
this.paused = false;
137148

138149
const request = {
139150
outputPath: this.config.outputPath,
@@ -305,16 +316,27 @@ export class LinuxNativeCaptureSession {
305316
if (this.paused) {
306317
return;
307318
}
308-
this.paused = true;
309319
this.write("pause");
320+
this.paused = true;
321+
this.cursorPauseStartedAtMs = Date.now();
310322
}
311323

312324
resume() {
313325
if (!this.paused) {
314326
return;
315327
}
316-
this.paused = false;
317328
this.write("resume");
329+
this.completeCursorPauseRange();
330+
this.paused = false;
331+
}
332+
333+
private completeCursorPauseRange(endMs = Date.now()) {
334+
if (this.cursorPauseStartedAtMs === null || this.cursorRecordingStartMs <= 0) return;
335+
this.cursorPauseRanges.push({
336+
startMs: Math.max(0, this.cursorPauseStartedAtMs - this.cursorRecordingStartMs),
337+
endMs: Math.max(0, endMs - this.cursorRecordingStartMs),
338+
});
339+
this.cursorPauseStartedAtMs = null;
318340
}
319341

320342
get isPaused() {
@@ -375,6 +397,7 @@ export class LinuxNativeCaptureSession {
375397
throw new Error("The Linux capture helper is not running.");
376398
}
377399

400+
this.completeCursorPauseRange();
378401
const exited = new Promise<void>((resolve, reject) => {
379402
this.stoppedResolve = resolve;
380403
this.stoppedReject = reject;
@@ -518,6 +541,7 @@ export class LinuxNativeCaptureSession {
518541
// telemetry is re-based onto it and anything from during the
519542
// picker is dropped rather than left pinned to the start.
520543
this.cursor.rebase(payload.timestampMs);
544+
this.cursorRecordingStartMs = payload.timestampMs;
521545
this.capturing = true;
522546
console.info(
523547
"[capture-linux] capture started",
@@ -538,7 +562,10 @@ export class LinuxNativeCaptureSession {
538562
durationMs: payload.durationMs,
539563
frames: payload.frames,
540564
droppedFrames: payload.dropped,
541-
cursor: this.cursor.toRecordingData(),
565+
cursor: compactCursorTelemetryPauseRanges(
566+
this.cursor.toRecordingData(),
567+
this.cursorPauseRanges,
568+
),
542569
...(this.sourceKind ? { sourceKind: this.sourceKind } : {}),
543570
...(this.videoEncoder ? { videoEncoder: this.videoEncoder } : {}),
544571
};

‎electron/native-bridge/cursor/recording/cursorPauseRanges.test.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import { describe, expect, it } from "vitest";
2+
import type { CursorRecordingData } from "../../../../src/native/contracts";
3+
import { compactCursorTelemetryPauseRanges } from "./cursorPauseRanges";
4+
5+
function recording(...times: number[]): CursorRecordingData {
6+
return {
7+
version: 2,
8+
provider: "none",
9+
samples: times.map((timeMs) => ({ timeMs, cx: 0.5, cy: 0.5 })),
10+
assets: [],
11+
};
12+
}
13+
14+
describe("compactCursorTelemetryPauseRanges", () => {
15+
it("removes paused samples and shifts later samples across repeated pauses", () => {
16+
const compacted = compactCursorTelemetryPauseRanges(recording(100, 250, 500, 650, 900), [
17+
{ startMs: 200, endMs: 400 },
18+
{ startMs: 600, endMs: 800 },
19+
]);
20+
21+
expect(compacted.samples.map((sample) => sample.timeMs)).toEqual([100, 300, 500]);
22+
});
23+
24+
it("ignores invalid ranges without replacing the recording", () => {
25+
const input = recording(100, 200);
26+
27+
expect(
28+
compactCursorTelemetryPauseRanges(input, [
29+
{ startMs: 100, endMs: 100 },
30+
{ startMs: Number.NaN, endMs: 200 },
31+
]),
32+
).toBe(input);
33+
});
34+
});

‎electron/native-bridge/cursor/recording/cursorPauseRanges.ts‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
import type { CursorRecordingData, CursorRecordingSample } from "../../../../src/native/contracts";
2+
3+
export interface CursorPauseRange {
4+
startMs: number;
5+
endMs: number;
6+
}
7+
8+
export function compactCursorTelemetryPauseRanges(
9+
recording: CursorRecordingData,
10+
ranges: CursorPauseRange[],
11+
): CursorRecordingData {
12+
if (ranges.length === 0) return recording;
13+
14+
const normalizedRanges = ranges
15+
.map((range) => ({
16+
startMs: Math.max(0, Math.min(range.startMs, range.endMs)),
17+
endMs: Math.max(0, Math.max(range.startMs, range.endMs)),
18+
}))
19+
.filter((range) => Number.isFinite(range.startMs) && Number.isFinite(range.endMs))
20+
.filter((range) => range.endMs > range.startMs)
21+
.sort((a, b) => a.startMs - b.startMs);
22+
23+
if (normalizedRanges.length === 0) return recording;
24+
25+
return {
26+
...recording,
27+
samples: recording.samples
28+
.map((sample) => {
29+
let pausedBeforeSampleMs = 0;
30+
for (const range of normalizedRanges) {
31+
if (sample.timeMs >= range.startMs && sample.timeMs <= range.endMs) {
32+
return null;
33+
}
34+
if (sample.timeMs > range.endMs) {
35+
pausedBeforeSampleMs += range.endMs - range.startMs;
36+
}
37+
}
38+
39+
return { ...sample, timeMs: Math.max(0, sample.timeMs - pausedBeforeSampleMs) };
40+
})
41+
.filter((sample): sample is CursorRecordingSample => Boolean(sample))
42+
.sort((a, b) => a.timeMs - b.timeMs),
43+
};
44+
}

‎electron/stt/index.test.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ const fakeWhisperServer = {
2121
shutdown: vi.fn(),
2222
};
2323

24+
const netFetch = vi.hoisted(() => vi.fn(async () => new Response("model")));
25+
vi.mock("electron", () => ({ app: { getPath: () => "/tmp" }, net: { fetch: netFetch } }));
26+
2427
vi.mock("./whisperServer", () => {
2528
class FakeWhisperServerManager {
2629
start = fakeWhisperServer.start;
@@ -482,6 +485,19 @@ describe("SttManager", () => {
482485
expect(fakeWhisperServer.start).toHaveBeenCalledOnce();
483486
});
484487

488+
it("downloads the models through Chromium's network stack, which honours the system proxy", async () => {
489+
const { ensureModels } = await import("./modelManager");
490+
const mocked = vi.mocked(ensureModels);
491+
mocked.mockClear();
492+
netFetch.mockClear();
493+
await new SttManager().init({ modelsBaseDir: "/tmp/fake-stt-models" });
494+
495+
const fetcher = mocked.mock.calls[0]?.[0].fetcher;
496+
expect(fetcher).toBeDefined();
497+
await fetcher?.(new URL("https://huggingface.co/model.bin"), { method: "GET" });
498+
expect(netFetch).toHaveBeenCalledWith("https://huggingface.co/model.bin", { method: "GET" });
499+
});
500+
485501
describe("word aligner", () => {
486502
/** Chunks report what `alignerFor("fr")` gave them, as the helper would use it. */
487503
function recordAligner() {
@@ -524,6 +540,9 @@ describe("SttManager", () => {
524540
expect(asked.length).toBeGreaterThan(1);
525541
expect(asked.every((x) => x === null)).toBe(true);
526542
expect(ensure).toHaveBeenCalledOnce();
543+
netFetch.mockClear();
544+
await ensure.mock.calls[0]?.[0].fetcher?.("https://github.com/aligner.gguf");
545+
expect(netFetch).toHaveBeenCalledWith("https://github.com/aligner.gguf", undefined);
527546

528547
land("/fake/fr.gguf");
529548
await new Promise((resolve) => setTimeout(resolve, 0));

0 commit comments

Comments
 (0)