Skip to content

Commit f0d8c29

Browse files
committed
fix: align macOS mouse pointer with captured content
The capture helper now reports the filter content rect as the capture frame on macOS 14 and later, and keeps the source frame only on macOS 13. Window recordings, including windows chosen in Apple's picker, seed normalization from the source frame before the first cursor sample, then replace it with the helper's capture bounds and rebase samples already stored. Display recordings keep normalizing against the display bounds. Fixes #1130
1 parent abe64d5 commit f0d8c29

3 files changed

Lines changed: 299 additions & 17 deletions

File tree

‎electron/ipc/handlers.ts‎

Lines changed: 41 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -879,7 +879,9 @@ const nativeMacCaptureOutputs = new WeakMap<ChildProcessWithoutNullStreams, stri
879879
let nativeMacRecordingWarning: { screenVideoPath: string; message: string } | null = null;
880880
/** True while stop-native-mac-recording runs: the take is ending on purpose. */
881881
let nativeMacStopInFlight = false;
882-
// Global frame of the region captured by the SCK helper (see getSelectedSourceBounds).
882+
// Global frame the macOS cursor is normalised against (see getSelectedSourceBounds).
883+
// Seeded from the source frame before the first sample, then replaced by the
884+
// helper's `recording-started.captureBounds` for a window take.
883885
let activeMacCaptureBounds: Rectangle | null = null;
884886
let linuxNativeCaptureSession: LinuxNativeCaptureSession | null = null;
885887
/**
@@ -1127,24 +1129,33 @@ function selectedSourceFromPick(
11271129
};
11281130
}
11291131

1132+
/** Legacy `window:` ids and a window chosen in Apple's picker. Display picks are not. */
1133+
function isMacWindowCaptureSourceId(sourceId: string | null | undefined) {
1134+
return (
1135+
typeof sourceId === "string" &&
1136+
(sourceId.startsWith("window:") || sourceId.startsWith(`${MAC_PICKER_SOURCE_PREFIX}window:`))
1137+
);
1138+
}
1139+
11301140
function getSelectedSourceBounds() {
1141+
// A window take is normalised against the rect the helper is recording. That
1142+
// rect is seeded from the source frame before the first cursor sample and
1143+
// replaced by `recording-started.captureBounds`, the ScreenCaptureKit content
1144+
// rect. The picker's frame and the display bounds must not stay in charge for
1145+
// the take: either one leaves the pointer above the click (#1130).
1146+
if (isMacWindowCaptureSourceId(selectedSource?.id) && activeMacCaptureBounds) {
1147+
return activeMacCaptureBounds;
1148+
}
1149+
11311150
// A pick from Apple's picker carries its own frame; there is no desktopCapturer
1132-
// display to look up for it.
1151+
// display to look up for it. Display picks keep this frame for the whole take.
11331152
if (isMacPickerSourceId(selectedSource?.id)) {
11341153
const pick = macPickerSession?.getSelection();
11351154
if (pick) {
11361155
return pick.bounds;
11371156
}
11381157
}
11391158

1140-
// Single-window capture records only the window's region, not the whole display.
1141-
// Normalizing the cursor against display bounds leaves a fixed offset in the export,
1142-
// so prefer the helper-reported window frame when capturing a window.
1143-
const isWindowSource = selectedSource?.id?.startsWith("window:") === true;
1144-
if (isWindowSource && activeMacCaptureBounds) {
1145-
return activeMacCaptureBounds;
1146-
}
1147-
11481159
const cursor = screen.getCursorScreenPoint();
11491160
const sourceDisplayId = Number(selectedSource?.display_id);
11501161
const sourceDisplay = Number.isFinite(sourceDisplayId)
@@ -1646,8 +1657,16 @@ function tryParseNativeHelperEvent(line: string) {
16461657

16471658
function dispatchNativeMacHelperEvent(event: Record<string, unknown>) {
16481659
const bounds = event.captureBounds as Rectangle | undefined;
1649-
if (bounds && bounds.width > 0 && bounds.height > 0) {
1660+
// Display takes never adopt this rect: their cursor stays on the display bounds.
1661+
// Adopting it there would move a capture whose alignment is already right.
1662+
if (
1663+
isMacWindowCaptureSourceId(selectedSource?.id) &&
1664+
bounds &&
1665+
bounds.width > 0 &&
1666+
bounds.height > 0
1667+
) {
16501668
activeMacCaptureBounds = bounds;
1669+
cursorRecordingSession?.updateCaptureBounds?.(bounds);
16511670
}
16521671
nativeMacCaptureEvents.emit("helper-event", event);
16531672
}
@@ -3222,6 +3241,9 @@ export function registerIpcHandlers(
32223241
? (screen.getAllDisplays().find((display) => display.id === request.source.displayId) ??
32233242
null)
32243243
: getSelectedDisplay();
3244+
// Drop the previous take before this lookup. A window with no frame of its
3245+
// own would otherwise seed the cursor from that take's capture rect.
3246+
activeMacCaptureBounds = null;
32253247
const bounds =
32263248
pick?.bounds ?? request.source.bounds ?? sourceDisplay?.bounds ?? getSelectedSourceBounds();
32273249
const captureExcludedWindowSourceIds: string[] = [];
@@ -3285,7 +3307,14 @@ export function registerIpcHandlers(
32853307
nativeMacIsPaused = false;
32863308
nativeMacStopInFlight = false;
32873309
nativeMacRecordingWarning = null;
3288-
activeMacCaptureBounds = null;
3310+
// Before the cursor session samples. A legacy window otherwise normalises
3311+
// its opening samples against the display, and a picker window would keep
3312+
// the picker's frame after the helper has reported the content rect it is
3313+
// actually recording (#1130). Display takes stay unset, so the display
3314+
// bounds remain authoritative.
3315+
if (isMacWindowCaptureSourceId(selectedSource?.id) && bounds.width > 0 && bounds.height > 0) {
3316+
activeMacCaptureBounds = bounds;
3317+
}
32893318

32903319
const cursorStartTimeMs = Date.now();
32913320
if (cursorCaptureMode === "editable-overlay") {
Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
import { EventEmitter } from "node:events";
2+
import { PassThrough } from "node:stream";
3+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
4+
5+
/**
6+
* The cast on `actual` is written out in the factory rather than shared in a
7+
* helper: `vi.mock` calls are HOISTED above every top-level statement, so a
8+
* module-scope helper is still in its temporal dead zone when the factory runs.
9+
*/
10+
type WithDefault = { default?: Record<string, unknown> };
11+
12+
const mocks = vi.hoisted(() => ({
13+
cursor: { x: 0, y: 0 },
14+
nearestDisplay: {
15+
scaleFactor: 2,
16+
bounds: { x: 0, y: 0, width: 1920, height: 1080 },
17+
},
18+
isTrustedAccessibilityClient: vi.fn(() => true),
19+
accessSync: vi.fn(),
20+
}));
21+
22+
vi.mock("node:child_process", async (importOriginal) => {
23+
const actual = await importOriginal<typeof import("node:child_process")>();
24+
const spawn = vi.fn();
25+
return { ...actual, spawn, default: { ...((actual as WithDefault).default ?? {}), spawn } };
26+
});
27+
28+
vi.mock("node:fs", async (importOriginal) => {
29+
const actual = await importOriginal<typeof import("node:fs")>();
30+
return {
31+
...actual,
32+
accessSync: mocks.accessSync,
33+
default: { ...((actual as WithDefault).default ?? {}), accessSync: mocks.accessSync },
34+
};
35+
});
36+
37+
vi.mock("electron", () => ({
38+
systemPreferences: { isTrustedAccessibilityClient: mocks.isTrustedAccessibilityClient },
39+
screen: {
40+
getCursorScreenPoint: () => mocks.cursor,
41+
getDisplayNearestPoint: () => mocks.nearestDisplay,
42+
},
43+
}));
44+
45+
import { spawn } from "node:child_process";
46+
import type { Rectangle } from "electron";
47+
import { MacNativeCursorRecordingSession } from "./macNativeCursorRecordingSession";
48+
49+
/** Minimal stand-in for the cursor helper: stdio pipes plus kill bookkeeping. */
50+
class FakeHelper extends EventEmitter {
51+
stdout = new PassThrough();
52+
stderr = new PassThrough();
53+
killed = false;
54+
55+
kill() {
56+
this.killed = true;
57+
return true;
58+
}
59+
60+
/** Feeds one NDJSON line, the way the real helper emits them. */
61+
emitEvent(event: Record<string, unknown>) {
62+
this.stdout.write(`${JSON.stringify(event)}\n`);
63+
}
64+
}
65+
66+
const spawnMock = vi.mocked(spawn);
67+
let helper: FakeHelper;
68+
69+
beforeEach(() => {
70+
helper = new FakeHelper();
71+
spawnMock.mockReset();
72+
spawnMock.mockReturnValue(helper as unknown as ReturnType<typeof spawn>);
73+
mocks.cursor = { x: 0, y: 0 };
74+
mocks.nearestDisplay = {
75+
scaleFactor: 2,
76+
bounds: { x: 0, y: 0, width: 1920, height: 1080 },
77+
};
78+
mocks.isTrustedAccessibilityClient.mockReset();
79+
mocks.isTrustedAccessibilityClient.mockReturnValue(true);
80+
mocks.accessSync.mockReset();
81+
vi.spyOn(console, "warn").mockImplementation(() => undefined);
82+
vi.spyOn(console, "error").mockImplementation(() => undefined);
83+
});
84+
85+
afterEach(() => {
86+
vi.restoreAllMocks();
87+
});
88+
89+
const WINDOW_BOUNDS: Rectangle = { x: 200, y: 100, width: 1000, height: 800 };
90+
91+
async function recordWithHelper(
92+
getDisplayBounds: () => Rectangle | null,
93+
sample: (emit: (event: Record<string, unknown>) => void) => void,
94+
) {
95+
const session = new MacNativeCursorRecordingSession({
96+
getDisplayBounds,
97+
maxSamples: 20,
98+
sampleIntervalMs: 33,
99+
startTimeMs: 1_000,
100+
});
101+
const pending = session.start();
102+
await Promise.resolve();
103+
helper.emitEvent({ type: "ready", timestampMs: 1_000 });
104+
await pending;
105+
sample((event) => helper.emitEvent(event));
106+
return session.stop();
107+
}
108+
109+
function click(timestampMs: number) {
110+
return { type: "sample", timestampMs, leftButtonPressed: true };
111+
}
112+
113+
describe("MacNativeCursorRecordingSession capture bounds", () => {
114+
it("places click hotspots on the top and bottom of a window with a non-zero origin", async () => {
115+
// 10px inside each edge. A display-sized frame, or any fixed y shift, misses both.
116+
mocks.cursor = { x: 400, y: 110 };
117+
const data = await recordWithHelper(
118+
() => WINDOW_BOUNDS,
119+
(emit) => {
120+
emit(click(1_100));
121+
mocks.cursor = { x: 400, y: 890 };
122+
emit(click(1_200));
123+
},
124+
);
125+
126+
expect(data.samples[0]).toMatchObject({
127+
cx: 0.2,
128+
cy: 10 / 800,
129+
visible: true,
130+
interactionType: "click",
131+
});
132+
expect(data.samples[1]).toMatchObject({
133+
cx: 0.2,
134+
cy: 790 / 800,
135+
visible: true,
136+
interactionType: "click",
137+
});
138+
});
139+
140+
it("replaces the source frame with later bounds and leaves a display frame unshifted", async () => {
141+
// The picker frame sits below the pixels. The helper's capture rect replaces it;
142+
// the next sample must not keep the old origin. A full display then maps as itself.
143+
let bounds: Rectangle = { x: 200, y: 180, width: 1000, height: 800 };
144+
mocks.cursor = { x: 700, y: 500 };
145+
const data = await recordWithHelper(
146+
() => bounds,
147+
(emit) => {
148+
emit(click(1_100));
149+
bounds = { x: 200, y: 120, width: 1000, height: 860 };
150+
emit(click(1_200));
151+
bounds = { x: 0, y: 0, width: 1920, height: 1080 };
152+
mocks.cursor = { x: 960, y: 540 };
153+
emit(click(1_300));
154+
},
155+
);
156+
157+
expect(data.samples[0].cy).toBeCloseTo((500 - 180) / 800, 5);
158+
expect(data.samples[1].cx).toBeCloseTo((700 - 200) / 1000, 5);
159+
expect(data.samples[1].cy).toBeCloseTo((500 - 120) / 860, 5);
160+
expect(data.samples[2]).toMatchObject({ cx: 0.5, cy: 0.5 });
161+
});
162+
163+
it("rebases samples collected before the authoritative bounds arrive", async () => {
164+
let bounds: Rectangle = { x: 0, y: 0, width: 1920, height: 1080 };
165+
mocks.cursor = { x: 400, y: 300 };
166+
const session = new MacNativeCursorRecordingSession({
167+
getDisplayBounds: () => bounds,
168+
maxSamples: 20,
169+
sampleIntervalMs: 33,
170+
startTimeMs: 1_000,
171+
});
172+
const pending = session.start();
173+
await Promise.resolve();
174+
helper.emitEvent({ type: "ready", timestampMs: 1_000 });
175+
await pending;
176+
helper.emitEvent(click(1_100));
177+
178+
bounds = WINDOW_BOUNDS;
179+
session.updateCaptureBounds(bounds);
180+
181+
const data = await session.stop();
182+
expect(data.samples[0]).toMatchObject({
183+
cx: 0.2,
184+
cy: 0.25,
185+
visible: true,
186+
interactionType: "click",
187+
});
188+
});
189+
190+
it("falls back to the display under the pointer when no capture bounds exist", async () => {
191+
// No helper on disk: position-only sampling, and nothing has reported a capture rect.
192+
mocks.accessSync.mockImplementation(() => {
193+
throw new Error("ENOENT");
194+
});
195+
mocks.nearestDisplay = {
196+
scaleFactor: 2,
197+
bounds: { x: 0, y: 0, width: 1440, height: 900 },
198+
};
199+
mocks.cursor = { x: 720, y: 450 };
200+
const session = new MacNativeCursorRecordingSession({
201+
getDisplayBounds: () => null,
202+
maxSamples: 5,
203+
sampleIntervalMs: 60_000,
204+
startTimeMs: 1_000,
205+
});
206+
207+
await session.start();
208+
const data = await session.stop();
209+
210+
expect(spawnMock).not.toHaveBeenCalled();
211+
expect(data.provider).toBe("none");
212+
expect(data.samples[0]).toMatchObject({ cx: 0.5, cy: 0.5, visible: true });
213+
});
214+
215+
it("clamps a brief exit and hides the pointer only after a sustained one", async () => {
216+
// Outside the window, above its top edge. The first two stay visible at the
217+
// clamp; the third crosses the hide threshold. One sample back inside clears it.
218+
mocks.cursor = { x: 1_400, y: 40 };
219+
const data = await recordWithHelper(
220+
() => WINDOW_BOUNDS,
221+
(emit) => {
222+
emit({ type: "sample", timestampMs: 1_100 });
223+
emit({ type: "sample", timestampMs: 1_133 });
224+
emit({ type: "sample", timestampMs: 1_166 });
225+
mocks.cursor = { x: 700, y: 500 };
226+
emit({ type: "sample", timestampMs: 1_200 });
227+
},
228+
);
229+
230+
expect(data.samples.slice(0, 3).map((sample) => sample.visible)).toEqual([true, true, false]);
231+
expect(data.samples[0]).toMatchObject({ cx: 1, cy: 0 });
232+
expect(data.samples[2]).toMatchObject({ cx: 1, cy: 0, visible: false });
233+
expect(data.samples[3]).toMatchObject({ cx: 0.5, cy: 0.5, visible: true });
234+
});
235+
});

‎electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
119119
let filter: SCContentFilter
120120
let width: Int
121121
let height: Int
122-
// Global frame (points, top-left origin) of the captured region. Used by the
123-
// renderer to normalize cursor positions into the captured window's space.
122+
// Global frame (points, top-left origin) of the captured region. The cursor is
123+
// normalised against this, so it is the filter's content rect — the same
124+
// rectangle the output buffer is sized from — wherever that exists.
124125
let captureFrame: CGRect
125126
}
126127

@@ -623,7 +624,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
623624
filter: picked.filter,
624625
width: size.width,
625626
height: size.height,
626-
captureFrame: picked.frame
627+
captureFrame: captureContentFrame(for: picked.filter, fallback: picked.frame)
627628
)
628629
}
629630

@@ -716,7 +717,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
716717
filter: filter,
717718
width: size.width,
718719
height: size.height,
719-
captureFrame: display.frame
720+
captureFrame: captureContentFrame(for: filter, fallback: display.frame)
720721
)
721722
case "window":
722723
guard let windowId = request.source.windowId else {
@@ -741,7 +742,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
741742
filter: filter,
742743
width: size.width,
743744
height: size.height,
744-
captureFrame: window.frame
745+
captureFrame: captureContentFrame(for: filter, fallback: window.frame)
745746
)
746747
default:
747748
throw HelperError.invalidSourceType(request.source.type)
@@ -1150,6 +1151,23 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
11501151
return status == .complete
11511152
}
11521153

1154+
/// On-screen rectangle of the pixels this filter will rasterise, in points.
1155+
///
1156+
/// `captureSize` already takes its dimensions from `contentRect`: that product is
1157+
/// the buffer ScreenCaptureKit fills. The cursor is a global Electron point
1158+
/// normalised against a frame reported separately, so the frame has to be that
1159+
/// same rectangle, origin included. `SCWindow.frame` and the frame stored with a
1160+
/// system-picker choice describe the window, which on macOS 26 sits below the
1161+
/// pixels the stream writes; the pointer then draws above the click (#1130).
1162+
/// No constant is added here to compensate. `contentRect` is screen points and
1163+
/// exists from macOS 14. On 13 the source frame is the only measurement available.
1164+
private func captureContentFrame(for filter: SCContentFilter, fallback: CGRect) -> CGRect {
1165+
guard #available(macOS 14.0, *) else {
1166+
return fallback
1167+
}
1168+
return filter.contentRect
1169+
}
1170+
11531171
/// Output-buffer size for a filter, capped by the resolution the app asked for.
11541172
///
11551173
/// The filter is asked first, because `contentRect × pointPixelScale` is by definition the

0 commit comments

Comments
 (0)