From d1a4a683a9b8d16d1599b7c31ec6519d8f28e101 Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Fri, 16 Oct 2026 14:43:21 +0530 Subject: [PATCH 1/8] feat: add floating camera preview on macOS --- electron/cameraPreview.test.ts | 103 +++++++++++++++++ electron/cameraPreview.ts | 84 ++++++++++++++ electron/electron-env.d.ts | 6 + electron/ipc/handlers.ts | 33 +++++- .../ScreenCaptureRecorder.swift | 6 +- electron/preload.ts | 17 +++ electron/windows.ts | 65 +++++++++++ src/App.tsx | 7 +- .../launch/FloatingCameraPreview.module.css | 88 ++++++++++++++ .../launch/FloatingCameraPreview.test.tsx | 37 ++++++ .../launch/FloatingCameraPreview.tsx | 59 ++++++++++ src/components/launch/HudControls.tsx | 34 +++++- src/components/launch/LaunchWindow.test.tsx | 45 ++++++++ src/components/launch/LaunchWindow.tsx | 14 +++ src/hooks/useFloatingCameraPreview.test.tsx | 108 ++++++++++++++++++ src/hooks/useFloatingCameraPreview.ts | 63 ++++++++++ .../useScreenRecorder.cameraPrefs.test.tsx | 97 ++++++++++++++++ ...eenRecorder.nativeMacStartWarning.test.tsx | 7 +- ...eScreenRecorder.nativeStopFailure.test.tsx | 7 +- src/hooks/useScreenRecorder.noCamera.test.tsx | 11 +- .../useScreenRecorder.prefsRace.test.tsx | 7 +- .../useScreenRecorder.restartFailure.test.tsx | 7 +- src/hooks/useScreenRecorder.ts | 27 ++++- src/hooks/webcamCaptureTarget.test.ts | 4 + src/hooks/webcamCaptureTarget.ts | 2 + src/i18n/locales/ar/launch.json | 7 ++ src/i18n/locales/cs/launch.json | 7 ++ src/i18n/locales/de/launch.json | 7 ++ src/i18n/locales/en/launch.json | 7 ++ src/i18n/locales/es/launch.json | 7 ++ src/i18n/locales/fr/launch.json | 7 ++ src/i18n/locales/it/launch.json | 7 ++ src/i18n/locales/ja-JP/launch.json | 7 ++ src/i18n/locales/ko-KR/launch.json | 7 ++ src/i18n/locales/pt-BR/launch.json | 7 ++ src/i18n/locales/ru/launch.json | 7 ++ src/i18n/locales/tr/launch.json | 7 ++ src/i18n/locales/vi/launch.json | 7 ++ src/i18n/locales/zh-CN/launch.json | 7 ++ src/i18n/locales/zh-TW/launch.json | 7 ++ src/lib/cameraPreview.test.ts | 35 ++++++ src/lib/cameraPreview.ts | 37 ++++++ .../architecture/recording.md | 27 +++++ .../testing/manual-e2e-checklist.md | 4 + 44 files changed, 1120 insertions(+), 26 deletions(-) create mode 100644 electron/cameraPreview.test.ts create mode 100644 electron/cameraPreview.ts create mode 100644 src/components/launch/FloatingCameraPreview.module.css create mode 100644 src/components/launch/FloatingCameraPreview.test.tsx create mode 100644 src/components/launch/FloatingCameraPreview.tsx create mode 100644 src/hooks/useFloatingCameraPreview.test.tsx create mode 100644 src/hooks/useFloatingCameraPreview.ts create mode 100644 src/hooks/useScreenRecorder.cameraPrefs.test.tsx create mode 100644 src/lib/cameraPreview.test.ts create mode 100644 src/lib/cameraPreview.ts diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts new file mode 100644 index 0000000000..b393f7fce5 --- /dev/null +++ b/electron/cameraPreview.test.ts @@ -0,0 +1,103 @@ +import { EventEmitter } from "node:events"; +import type { BrowserWindow, WebContents } from "electron"; +import { describe, expect, it, vi } from "vitest"; +import { CameraPreviewController } from "./cameraPreview"; + +function setup() { + const owner = Object.assign(new EventEmitter(), { send: vi.fn() }); + const overlay = Object.assign(new EventEmitter(), { + webContents: { send: vi.fn() }, + showInactive: vi.fn(), + hide: vi.fn(), + destroy: vi.fn(), + }); + const create = vi.fn(() => overlay as unknown as BrowserWindow); + const resize = vi.fn(); + const preview = new CameraPreviewController(create, resize); + const sender = owner as unknown as WebContents; + const overlaySender = overlay.webContents as unknown as WebContents; + return { owner, overlay, create, resize, preview, sender, overlaySender }; +} + +const jpeg = "data:image/jpeg;base64,/9j/"; + +describe("floating camera preview lifecycle", () => { + it("waits for the window to paint, then shows without taking focus", () => { + const { preview, sender, overlay } = setup(); + preview.configure(sender, true, true); + expect(overlay.showInactive).not.toHaveBeenCalled(); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + }); + + it("hides and restores the same window, retaining its capture exclusion id", () => { + const { preview, sender, overlaySender, overlay, owner, create } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + preview.hide(overlaySender); + expect(owner.send).toHaveBeenCalledWith("camera-preview:hidden"); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.destroy).not.toHaveBeenCalled(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + }); + + it("does not flash a preview disabled before its renderer is ready", () => { + const { preview, sender, overlay } = setup(); + preview.configure(sender, true, true); + preview.configure(sender, true, false); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).not.toHaveBeenCalled(); + }); + + it("ignores other renderers and rejects oversized or non-JPEG frames", () => { + const { preview, sender, overlay, create } = setup(); + const stranger = {} as WebContents; + preview.configure(sender, true, true); + preview.configure(stranger, false, false); + preview.hide(stranger); + for (const invalid of [undefined, "https://example.com/camera", jpeg + "x".repeat(128_000)]) { + preview.frame(sender, invalid); + } + preview.frame(stranger, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + expect(overlay.destroy).not.toHaveBeenCalled(); + expect(create).toHaveBeenCalledOnce(); + }); + + it("allows only the preview to resize to a supported diameter without replacing its window", () => { + const { preview, sender, overlaySender, resize, create, overlay } = setup(); + preview.configure(sender, true, true); + preview.resize(sender, 320); + preview.resize(overlaySender, Number.NaN); + preview.resize(overlaySender, 10000); + expect(resize).not.toHaveBeenCalled(); + preview.resize(overlaySender, 320); + expect(resize).toHaveBeenCalledWith(overlay, 320); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.destroy).not.toHaveBeenCalled(); + }); + + it("destroys the auxiliary window when the HUD is destroyed", () => { + const { preview, sender, owner, overlay } = setup(); + preview.configure(sender, true, true); + owner.emit("destroyed"); + expect(overlay.destroy).toHaveBeenCalledOnce(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + }); + + it("releases listeners on camera-off rather than accumulating them between takes", () => { + const { preview, sender, owner, overlay } = setup(); + for (let i = 0; i < 15; i++) { + preview.configure(sender, true, false); + overlay.emit("ready-to-show"); + preview.configure(sender, false, false); + } + expect(owner.listenerCount("destroyed")).toBe(0); + expect(overlay.destroy).toHaveBeenCalledTimes(15); + }); +}); diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts new file mode 100644 index 0000000000..dafa598132 --- /dev/null +++ b/electron/cameraPreview.ts @@ -0,0 +1,84 @@ +import type { BrowserWindow, WebContents } from "electron"; +import { CAMERA_PREVIEW_SIZES } from "../src/lib/cameraPreview"; + +/** One stable window per HUD: hiding it must not change the native capture exclusion id. */ +export class CameraPreviewController { + private window: BrowserWindow | null = null; + private owner: WebContents | null = null; + private visible = false; + private ready = false; + private onOwnerDestroyed: (() => void) | null = null; + + constructor( + private readonly createWindow: () => BrowserWindow, + private readonly resizeWindow: (window: BrowserWindow, diameter: number) => void, + ) {} + + configure(owner: WebContents, enabled: boolean, visible: boolean): void { + if (this.owner && this.owner !== owner) return; + if (!enabled) { + this.dispose(); + return; + } + if (!this.owner) { + this.owner = owner; + this.onOwnerDestroyed = () => { + if (this.owner === owner) this.dispose(); + }; + owner.once("destroyed", this.onOwnerDestroyed); + } + if (!this.window) { + this.window = this.createWindow(); + const previewWindow = this.window; + this.window.once("ready-to-show", () => { + if (this.window !== previewWindow) return; + this.ready = true; + if (this.visible) this.window?.showInactive(); + }); + } + this.visible = visible; + if (visible && this.ready) this.window.showInactive(); + else if (!visible) this.window.hide(); + } + + hide(sender: WebContents): void { + if (sender !== this.window?.webContents) return; + this.visible = false; + this.window.hide(); + this.owner?.send("camera-preview:hidden"); + } + + resize(sender: WebContents, diameter: unknown): void { + if ( + sender !== this.window?.webContents || + typeof diameter !== "number" || + !CAMERA_PREVIEW_SIZES.some((size) => size === diameter) + ) + return; + this.resizeWindow(this.window, diameter); + } + + frame(sender: WebContents, jpeg: unknown): void { + if (sender !== this.owner || !this.visible || !this.window) return; + // A 320px JPEG is usually under 20KB. Never relay arbitrary URLs or unbounded IPC data. + if ( + typeof jpeg !== "string" || + jpeg.length > 128_000 || + !jpeg.startsWith("data:image/jpeg;base64,") + ) + return; + this.window.webContents.send("camera-preview:frame", jpeg); + } + + dispose(): void { + if (this.owner && this.onOwnerDestroyed) { + this.owner.removeListener("destroyed", this.onOwnerDestroyed); + } + this.onOwnerDestroyed = null; + this.ready = false; + this.window?.destroy(); + this.window = null; + this.owner = null; + this.visible = false; + } +} diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 9a4095c196..64a10b7102 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -58,6 +58,12 @@ interface Window { usesSystemSourcePicker?: () => Promise; /** Name of the last source picked in Apple's picker, for display only: it is not selected. */ getLastPickedSource?: () => Promise; + configureCameraPreview?: (enabled: boolean, visible: boolean) => Promise; + sendCameraPreviewFrame?: (jpeg: string) => Promise; + resizeCameraPreview?: (diameter: number) => Promise; + hideCameraPreview?: () => Promise; + onCameraPreviewFrame?: (callback: (jpeg: string) => void) => () => void; + onCameraPreviewHidden?: (callback: () => void) => () => void; openNotes: () => Promise<{ opened: boolean; reason?: string; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index b54ffe87a7..2a17586b9b 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -70,6 +70,7 @@ import { DocumentService } from "../ai-edition/document-service"; import { LlmConfigStore } from "../ai-edition/llm-config-store"; import { StylePresetService } from "../ai-edition/style-preset-service"; import { AppSettingsStore } from "../app-settings"; +import { CameraPreviewController } from "../cameraPreview"; import { createRunnerHost } from "../cli/runnerHost"; import { hideCountdownOverlay } from "../countdownOverlay"; import { isDiagnosticModeEnabled, mainLogFile } from "../diagnostics/main-log-file"; @@ -169,7 +170,12 @@ import { shouldPersistSelectedSource, } from "../recording-source-settings"; import { getSttManager } from "../stt"; -import { createRunnerWindow } from "../windows"; +import { + createCameraPreviewWindow, + createRunnerWindow, + getCameraPreviewWindow, + resizeCameraPreviewWindow, +} from "../windows"; import { assertWindowsUpdateIdle, isWindowsUpdatePending, @@ -2186,7 +2192,7 @@ export function registerIpcHandlers( * pick would record them unless the picker is told to leave them out. */ function macPickerExcludedWindowIds() { - const appWindowSourceIds = [getMainWindow(), getNotesWindow()] + const appWindowSourceIds = [getMainWindow(), getNotesWindow(), getCameraPreviewWindow()] .filter((window): window is BrowserWindow => !!window && !window.isDestroyed()) .map((window) => window.getMediaSourceId()); return collectMacCaptureExcludedWindowIds(appWindowSourceIds); @@ -2525,6 +2531,27 @@ export function registerIpcHandlers( return { opened: true }; }); + const cameraPreview = new CameraPreviewController( + createCameraPreviewWindow, + resizeCameraPreviewWindow, + ); + ipcMain.handle("camera-preview:configure", (event, enabled: unknown, visible: unknown) => { + if ( + process.platform !== "darwin" || + event.sender !== getMainWindow()?.webContents || + new URL(event.sender.getURL()).searchParams.get("windowType") !== "hud-overlay" + ) + return; + cameraPreview.configure(event.sender, enabled === true, visible === true); + }); + ipcMain.handle("camera-preview:frame", (event, jpeg: unknown) => + cameraPreview.frame(event.sender, jpeg), + ); + ipcMain.handle("camera-preview:hide", (event) => cameraPreview.hide(event.sender)); + ipcMain.handle("camera-preview:resize", (event, diameter: unknown) => + cameraPreview.resize(event.sender, diameter), + ); + ipcMain.handle("open-notes", async () => { const notesSelectorWin = getNotesWindow(); if (notesSelectorWin) { @@ -3288,7 +3315,7 @@ export function registerIpcHandlers( pick?.bounds ?? request.source.bounds ?? sourceDisplay?.bounds ?? getSelectedSourceBounds(); const captureExcludedWindowSourceIds: string[] = []; if (request.source.type === "display") { - for (const window of [getMainWindow(), getNotesWindow()]) { + for (const window of [getMainWindow(), getNotesWindow(), getCameraPreviewWindow()]) { if (window && !window.isDestroyed()) { captureExcludedWindowSourceIds.push(window.getMediaSourceId()); } diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift index 4d47d274d3..039b2484db 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift @@ -217,9 +217,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { if let picked { target = makeCaptureTarget(picked: picked) } else { + // Keep hidden recording overlays in the filter, so showing a camera preview + // during the take never exposes it while the exclusion-refresh poll catches up. let content = try await SCShareableContent.excludingDesktopWindows( false, - onScreenWindowsOnly: true + onScreenWindowsOnly: false ) target = try makeCaptureTarget(from: content) if recordsSystemAudioFromDisplay, let display = content.displays.first { @@ -700,7 +702,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { guard let self, let stream = self.stream, let content = try? await SCShareableContent.excludingDesktopWindows( false, - onScreenWindowsOnly: true + onScreenWindowsOnly: false ), let display = content.displays.first(where: { $0.displayID == displayID }) else { diff --git a/electron/preload.ts b/electron/preload.ts index 021932a332..f6a4ef7e05 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -170,6 +170,23 @@ contextBridge.exposeInMainWorld("electronAPI", { getLastPickedSource: (): Promise => { return ipcRenderer.invoke("get-last-picked-source"); }, + configureCameraPreview: (enabled: boolean, visible: boolean): Promise => + ipcRenderer.invoke("camera-preview:configure", enabled, visible), + sendCameraPreviewFrame: (jpeg: string): Promise => + ipcRenderer.invoke("camera-preview:frame", jpeg), + resizeCameraPreview: (diameter: number): Promise => + ipcRenderer.invoke("camera-preview:resize", diameter), + hideCameraPreview: (): Promise => ipcRenderer.invoke("camera-preview:hide"), + onCameraPreviewFrame: (callback: (jpeg: string) => void) => { + const listener = (_event: Electron.IpcRendererEvent, jpeg: string) => callback(jpeg); + ipcRenderer.on("camera-preview:frame", listener); + return () => ipcRenderer.removeListener("camera-preview:frame", listener); + }, + onCameraPreviewHidden: (callback: () => void) => { + const listener = () => callback(); + ipcRenderer.on("camera-preview:hidden", listener); + return () => ipcRenderer.removeListener("camera-preview:hidden", listener); + }, openNotes: () => { return ipcRenderer.invoke("open-notes"); }, diff --git a/electron/windows.ts b/electron/windows.ts index f0ff7c89dc..92bd9c59d2 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { app, BrowserWindow, ipcMain, screen } from "electron"; +import { cameraPreviewBounds, resizedCameraPreviewBounds } from "../src/lib/cameraPreview"; import { fitEditorRect, loadEditorWindowState, @@ -879,6 +880,70 @@ export function createPermissionsWindow(): BrowserWindow { return win; } +let cameraPreviewWindow: BrowserWindow | null = null; + +export function getCameraPreviewWindow(): BrowserWindow | null { + return cameraPreviewWindow && !cameraPreviewWindow.isDestroyed() ? cameraPreviewWindow : null; +} + +/** The camera stream belongs to the HUD; this window only displays preview frames. */ +export function createCameraPreviewWindow(): BrowserWindow { + const initialBounds = () => { + const bar = + hudOverlayWindow && !hudOverlayWindow.isDestroyed() && hudContentRect + ? hudContentScreenRect(hudOverlayWindow.getBounds(), hudContentRect) + : undefined; + const workArea = bar + ? screen.getDisplayMatching(bar).workArea + : screen.getPrimaryDisplay().workArea; + return cameraPreviewBounds(workArea, bar); + }; + const win = new BrowserWindow({ + ...initialBounds(), + title: "OpenScreen - Camera Preview", + hasShadow: false, + frame: false, + transparent: true, + backgroundColor: "#00000000", + resizable: false, + maximizable: false, + minimizable: false, + closable: false, + alwaysOnTop: true, + skipTaskbar: true, + show: false, + webPreferences: { + preload: path.join(__dirname, "preload.mjs"), + additionalArguments: [ASSET_BASE_URL_ARG], + nodeIntegration: false, + contextIsolation: true, + backgroundThrottling: false, + }, + }); + win.once("ready-to-show", () => win.setBounds(initialBounds())); + applyContentProtection(win, "Camera Preview"); + followAcrossSpaces(win); + cameraPreviewWindow = win; + win.on("closed", () => { + if (cameraPreviewWindow === win) cameraPreviewWindow = null; + }); + if (VITE_DEV_SERVER_URL) { + win.loadURL(VITE_DEV_SERVER_URL + "?windowType=camera-preview"); + } else { + win.loadFile(path.join(RENDERER_DIST, "index.html"), { + query: { windowType: "camera-preview" }, + }); + } + return win; +} + +export function resizeCameraPreviewWindow(win: BrowserWindow, diameter: number): void { + const bounds = win.getBounds(); + win.setBounds( + resizedCameraPreviewBounds(bounds, screen.getDisplayMatching(bounds).workArea, diameter), + ); +} + // Frameless Notes Window for taking notes during a recording. export function createNotesWindow(): BrowserWindow { const win = new BrowserWindow({ diff --git a/src/App.tsx b/src/App.tsx index 681bb59939..c326d3b835 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -6,6 +6,7 @@ import { installBrowserShims } from "./native/browserShim"; installBrowserShims(); import { CountdownOverlay } from "./components/launch/CountdownOverlay.tsx"; +import { FloatingCameraPreview } from "./components/launch/FloatingCameraPreview"; import { LaunchWindow } from "./components/launch/LaunchWindow"; import { NotesWindow } from "./components/launch/NotesWindow.tsx"; import { SourceSelector } from "./components/launch/SourceSelector"; @@ -52,7 +53,7 @@ export default function App() { setWindowType(type); } - if (type === "hud-overlay" || type === "source-selector" || type === "countdown-overlay") { + if (["hud-overlay", "source-selector", "countdown-overlay", "camera-preview"].includes(type)) { document.body.style.background = "transparent"; document.documentElement.style.background = "transparent"; document.getElementById("root")?.style.setProperty("background", "transparent"); @@ -60,7 +61,7 @@ export default function App() { // HUD is a fixed-size BrowserWindow; pin the document shell and hide overflow // so the renderer can't introduce scrollbars (see issue #305). - if (type === "hud-overlay") { + if (type === "hud-overlay" || type === "camera-preview") { document.documentElement.style.height = "100%"; document.documentElement.style.overflow = "hidden"; document.body.style.height = "100%"; @@ -85,6 +86,8 @@ export default function App() { switch (windowType) { case "hud-overlay": return ; + case "camera-preview": + return ; case "source-selector": return ; case "countdown-overlay": diff --git a/src/components/launch/FloatingCameraPreview.module.css b/src/components/launch/FloatingCameraPreview.module.css new file mode 100644 index 0000000000..9239c7e430 --- /dev/null +++ b/src/components/launch/FloatingCameraPreview.module.css @@ -0,0 +1,88 @@ +.preview { + -webkit-app-region: drag; + position: relative; + display: grid; + place-items: center; + width: calc(100% - 16px); + height: calc(100% - 16px); + margin: 8px; + overflow: hidden; + border-radius: 50%; + background: #09090b; + color: #a1a1aa; + font-size: 12px; + cursor: grab; +} + +.preview img { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + object-fit: cover; + object-position: center; + transform: scaleX(-1); +} + +.hide { + -webkit-app-region: no-drag; + position: absolute; + top: 9%; + right: 13%; + display: grid; + place-items: center; + width: 28px; + height: 28px; + border-radius: 50%; + background: #09090bbf; + color: #f4f4f5; + cursor: pointer; + opacity: 0; + transition: opacity 120ms ease; +} + +.preview:hover .hide, +.preview:focus-within .hide { + opacity: 1; +} + +.hide:focus-visible { + outline: 2px solid #10b981; +} + +.sizeControls { + -webkit-app-region: no-drag; + position: absolute; + bottom: 9%; + left: 50%; + display: flex; + gap: 4px; + transform: translateX(-50%); + opacity: 0; + transition: opacity 120ms ease; +} + +.sizeControls button { + display: grid; + place-items: center; + width: 28px; + height: 28px; + border-radius: 50%; + background: #09090bbf; + color: #f4f4f5; + cursor: pointer; +} + +.sizeControls button:disabled { + opacity: 0.4; + cursor: default; +} + +.preview:hover .sizeControls, +.preview:focus-within .sizeControls { + opacity: 1; +} + +.sizeControls button:focus-visible { + outline: 2px solid #10b981; +} diff --git a/src/components/launch/FloatingCameraPreview.test.tsx b/src/components/launch/FloatingCameraPreview.test.tsx new file mode 100644 index 0000000000..08194f9946 --- /dev/null +++ b/src/components/launch/FloatingCameraPreview.test.tsx @@ -0,0 +1,37 @@ +// @vitest-environment jsdom +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, expect, it, vi } from "vitest"; +import { FloatingCameraPreview } from "./FloatingCameraPreview"; + +vi.mock("../../contexts/I18nContext", () => ({ useScopedT: () => (key: string) => key })); +afterEach(cleanup); + +it("displays incoming frames and offers three bounded preview sizes", async () => { + let frame: ((jpeg: string) => void) | undefined; + const resize = vi.fn().mockResolvedValue(undefined); + const unsubscribe = vi.fn(); + window.electronAPI = { + onCameraPreviewFrame: (callback: (jpeg: string) => void) => { + frame = callback; + return unsubscribe; + }, + resizeCameraPreview: resize, + } as unknown as Window["electronAPI"]; + const view = render(); + act(() => frame?.("data:image/jpeg;base64,frame")); + expect(screen.getByAltText("floatingCamera.title").getAttribute("src")).toBe( + "data:image/jpeg;base64,frame", + ); + const larger = screen.getByRole("button", { name: "floatingCamera.larger" }); + const smaller = screen.getByRole("button", { name: "floatingCamera.smaller" }); + fireEvent.click(larger); + await waitFor(() => expect((larger as HTMLButtonElement).disabled).toBe(true)); + expect(resize).toHaveBeenLastCalledWith(320); + fireEvent.click(smaller); + await waitFor(() => expect((larger as HTMLButtonElement).disabled).toBe(false)); + fireEvent.click(smaller); + await waitFor(() => expect((smaller as HTMLButtonElement).disabled).toBe(true)); + expect(resize).toHaveBeenLastCalledWith(160); + view.unmount(); + expect(unsubscribe).toHaveBeenCalledOnce(); +}); diff --git a/src/components/launch/FloatingCameraPreview.tsx b/src/components/launch/FloatingCameraPreview.tsx new file mode 100644 index 0000000000..cdd21f3514 --- /dev/null +++ b/src/components/launch/FloatingCameraPreview.tsx @@ -0,0 +1,59 @@ +import { Minus, Plus, X } from "lucide-react"; +import { useEffect, useRef, useState } from "react"; +import { useScopedT } from "../../contexts/I18nContext"; +import { CAMERA_PREVIEW_SIZES } from "../../lib/cameraPreview"; +import styles from "./FloatingCameraPreview.module.css"; + +export function FloatingCameraPreview() { + const t = useScopedT("launch"); + const image = useRef(null); + const [ready, setReady] = useState(false); + const [sizeIndex, setSizeIndex] = useState(1); + const resize = (next: number) => { + void window.electronAPI + ?.resizeCameraPreview?.(CAMERA_PREVIEW_SIZES[next]) + .then(() => setSizeIndex(next)) + .catch((error: unknown) => console.warn("Could not resize camera preview:", error)); + }; + useEffect( + () => + window.electronAPI?.onCameraPreviewFrame?.((jpeg) => { + if (image.current) image.current.src = jpeg; + setReady(true); + }), + [], + ); + + return ( +
+ + {!ready && {t("deviceSettings.previewUnavailable")}} + +
+ + +
+
+ ); +} diff --git a/src/components/launch/HudControls.tsx b/src/components/launch/HudControls.tsx index c417b754c2..873c2db809 100644 --- a/src/components/launch/HudControls.tsx +++ b/src/components/launch/HudControls.tsx @@ -1,4 +1,4 @@ -import { Check, CircleArrowDown, Languages, NotepadText, Settings } from "lucide-react"; +import { Check, CircleArrowDown, Eye, EyeOff, Languages, NotepadText, Settings } from "lucide-react"; import { createContext, memo, @@ -301,6 +301,38 @@ export const HudMicButton = memo(function HudMicButton({ ); }); +export function HudCameraPreviewButton({ + visible, + disabled, + label, + onClick, +}: { + visible: boolean; + disabled: boolean; + label: string; + onClick: () => void; +}) { + return ( + + + + ); +} + export const HudCameraButton = memo(function HudCameraButton({ enabled, locked, diff --git a/src/components/launch/LaunchWindow.test.tsx b/src/components/launch/LaunchWindow.test.tsx index 5da604fa7e..20d2cc3f51 100644 --- a/src/components/launch/LaunchWindow.test.tsx +++ b/src/components/launch/LaunchWindow.test.tsx @@ -2547,3 +2547,48 @@ describe("LaunchWindow toasts", () => { }); }); }); + +describe("macOS floating camera preview control", () => { + beforeEach(() => { + platformState.value = "darwin"; + resetLaunchMocks(); + }); + + it("keeps hide/show available during recording without changing camera capture", () => { + recorderState.value.webcamEnabled = true; + recorderState.value.recording = true; + renderLaunchWindow(); + const button = screen.getByTestId("launch-camera-preview-button"); + expect(button).toHaveAttribute("aria-pressed", "true"); + fireEvent.click(button); + expect(button).toHaveAttribute("aria-pressed", "false"); + fireEvent.click(button); + expect(button).toHaveAttribute("aria-pressed", "true"); + expect(recorderState.value.setWebcamEnabled).not.toHaveBeenCalled(); + }); + + it("keeps the preview button visible but disabled when the camera is off", () => { + const view = renderLaunchWindow(); + const button = screen.getByTestId("launch-camera-preview-button"); + expect(button).toHaveAttribute("aria-disabled", "true"); + expect(button).toHaveAttribute("aria-pressed", "false"); + fireEvent.click(button); + expect(button).toHaveAttribute("aria-pressed", "false"); + expect(recorderState.value.setWebcamEnabled).not.toHaveBeenCalled(); + recorderState.value.webcamEnabled = true; + view.rerender( + + + , + ); + expect(button).not.toHaveAttribute("aria-disabled"); + expect(button).toHaveAttribute("aria-pressed", "true"); + }); + + it("does not offer the preview on Windows", () => { + platformState.value = "win32"; + recorderState.value.webcamEnabled = true; + renderLaunchWindow(); + expect(screen.queryByTestId("launch-camera-preview-button")).not.toBeInTheDocument(); + }); +}); diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index c094d62302..ba38189876 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -8,6 +8,7 @@ import { nativeBridgeClient } from "@/native"; import { canRecordMicrophone, getPlatform } from "@/utils/platformUtils"; import { type CameraDevice } from "../../hooks/useCameraDevices"; import { useCameraHudSync } from "../../hooks/useCameraHudSync"; +import { useFloatingCameraPreview } from "../../hooks/useFloatingCameraPreview"; import { isPlaceholderMicrophoneLabel, type MicrophoneDevice, @@ -22,6 +23,7 @@ import { requestCameraAccess } from "../../lib/requestCameraAccess"; import { Toaster } from "../ui/sonner"; import { HudCameraButton, + HudCameraPreviewButton, HudCursorButton, HudDivider, HudDragHandle, @@ -113,6 +115,7 @@ export function LaunchWindow() { const languageCode = locale.split("-")[0].toUpperCase(); const { + webcamPreviewStream, recording, paused, saving, @@ -146,6 +149,9 @@ export function LaunchWindow() { } = useScreenRecorder(); // Choosing a device and switching one on are deliberately separate concerns. + const cameraPreview = useFloatingCameraPreview(webcamPreviewStream, webcamEnabled && !saving); + const cameraPreviewShown = webcamEnabled && !saving && cameraPreview.visible; + // The mic and camera buttons are plain on/off toggles that use whatever device // is currently selected (the system default until the user says otherwise); // picking a different device — and checking it actually works — happens in the @@ -1310,6 +1316,14 @@ export function LaunchWindow() { tooltip={t("tooltips.camera")} onClick={toggleWebcam} /> + {cameraPreview.supported && ( + + )} ({ getPlatform: () => platform })); +let platform = "darwin"; +let hidden: (() => void) | undefined; +const configure = vi.fn().mockResolvedValue(undefined); +const send = vi.fn().mockResolvedValue(undefined); +const unsubscribe = vi.fn(); +const stop = vi.fn(); +const stream = { getTracks: () => [{ stop }] } as unknown as MediaStream; + +beforeEach(() => { + vi.useFakeTimers(); + platform = "darwin"; + vi.clearAllMocks(); + Object.defineProperty(window, "electronAPI", { + configurable: true, + value: { + configureCameraPreview: configure, + sendCameraPreviewFrame: send, + onCameraPreviewHidden: (callback: () => void) => { + hidden = callback; + return unsubscribe; + }, + }, + }); + vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(undefined); + vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => undefined); + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({ + drawImage: vi.fn(), + } as unknown as CanvasRenderingContext2D); + vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue( + "data:image/jpeg;base64,frame", + ); + vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockReturnValue(2); + vi.spyOn(HTMLVideoElement.prototype, "videoWidth", "get").mockReturnValue(1280); + vi.spyOn(HTMLVideoElement.prototype, "videoHeight", "get").mockReturnValue(720); +}); + +afterEach(() => { + cleanup(); + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +describe("floating camera publisher", () => { + it("shares frames without acquiring or stopping the recording camera", async () => { + const { unmount } = renderHook(() => useFloatingCameraPreview(stream, true)); + expect(configure).toHaveBeenLastCalledWith(true, true); + expect(send).toHaveBeenCalledOnce(); + await act(() => vi.advanceTimersByTimeAsync(300)); + expect(send).toHaveBeenCalledTimes(4); + unmount(); + await vi.advanceTimersByTimeAsync(500); + expect(send).toHaveBeenCalledTimes(4); + expect(stop).not.toHaveBeenCalled(); + expect(configure).toHaveBeenLastCalledWith(false, false); + expect(unsubscribe).toHaveBeenCalledOnce(); + }); + + it("hides without disabling the camera, then resumes from the HUD control", () => { + const { result } = renderHook(() => useFloatingCameraPreview(stream, true)); + act(() => hidden?.()); + expect(configure).toHaveBeenLastCalledWith(true, false); + expect(result.current.visible).toBe(false); + act(() => result.current.toggle()); + expect(configure).toHaveBeenLastCalledWith(true, true); + expect(stop).not.toHaveBeenCalled(); + }); + + it("applies backpressure while a previous frame is still being delivered", async () => { + let delivered: (() => void) | undefined; + send.mockImplementationOnce( + () => + new Promise((resolve) => { + delivered = resolve; + }), + ); + renderHook(() => useFloatingCameraPreview(stream, true)); + await act(() => vi.advanceTimersByTimeAsync(1000)); + expect(send).toHaveBeenCalledOnce(); + await act(async () => { + delivered?.(); + }); + await act(() => vi.advanceTimersByTimeAsync(100)); + expect(send).toHaveBeenCalledTimes(2); + }); + + it("waits for the recorder stream and never opens a preview on other platforms", () => { + const { rerender, unmount } = renderHook( + ({ enabled }) => useFloatingCameraPreview(null, enabled), + { initialProps: { enabled: true } }, + ); + expect(configure).toHaveBeenLastCalledWith(true, false); + expect(send).not.toHaveBeenCalled(); + rerender({ enabled: false }); + expect(configure).toHaveBeenLastCalledWith(false, false); + unmount(); + configure.mockClear(); + platform = "win32"; + renderHook(() => useFloatingCameraPreview(stream, true)); + expect(configure).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); +}); diff --git a/src/hooks/useFloatingCameraPreview.ts b/src/hooks/useFloatingCameraPreview.ts new file mode 100644 index 0000000000..d6bb2d0124 --- /dev/null +++ b/src/hooks/useFloatingCameraPreview.ts @@ -0,0 +1,63 @@ +import { useEffect, useState } from "react"; +import { getPlatform } from "../utils/platformUtils"; + +/** A small self-view from the recorder's stream; never acquires or stops the camera. */ +export function useFloatingCameraPreview(stream: MediaStream | null, enabled: boolean) { + const supported = getPlatform() === "darwin"; + const [visible, setVisible] = useState(true); + + useEffect(() => window.electronAPI?.onCameraPreviewHidden?.(() => setVisible(false)), []); + + useEffect(() => { + if (!supported) return; + void window.electronAPI + ?.configureCameraPreview?.(enabled, enabled && visible && !!stream) + .catch((error: unknown) => console.warn("Could not configure camera preview:", error)); + }, [enabled, visible, stream, supported]); + + useEffect(() => { + if (!supported || !enabled || !visible || !stream) return; + const video = document.createElement("video"); + video.muted = true; + video.playsInline = true; + video.srcObject = stream; + void video.play().catch(() => undefined); + const canvas = document.createElement("canvas"); + canvas.width = 320; + canvas.height = 180; + const context = canvas.getContext("2d"); + let cancelled = false; + let timer: ReturnType; + const draw = async () => { + try { + if (context && video.readyState >= 2 && video.videoWidth > 0) { + canvas.height = Math.max(1, Math.round((320 * video.videoHeight) / video.videoWidth)); + context.drawImage(video, 0, 0, canvas.width, canvas.height); + // Await delivery: a slow renderer must not build a queue of stale frames. + await window.electronAPI?.sendCameraPreviewFrame?.(canvas.toDataURL("image/jpeg", 0.7)); + } + } catch (error) { + console.warn("Could not update camera preview:", error); + } finally { + if (!cancelled) timer = setTimeout(() => void draw(), 100); + } + }; + void draw(); + return () => { + cancelled = true; + clearTimeout(timer); + video.pause(); + video.srcObject = null; + }; + }, [enabled, visible, stream, supported]); + + useEffect( + () => () => { + if (supported) + void window.electronAPI?.configureCameraPreview?.(false, false).catch(() => undefined); + }, + [supported], + ); + + return { supported, visible, toggle: () => setVisible((value) => !value) }; +} diff --git a/src/hooks/useScreenRecorder.cameraPrefs.test.tsx b/src/hooks/useScreenRecorder.cameraPrefs.test.tsx new file mode 100644 index 0000000000..b7f4f41297 --- /dev/null +++ b/src/hooks/useScreenRecorder.cameraPrefs.test.tsx @@ -0,0 +1,97 @@ +// @vitest-environment jsdom +import { act, cleanup, renderHook, waitFor } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useCameraHudSync } from "./useCameraHudSync"; +import { useScreenRecorder } from "./useScreenRecorder"; + +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); +vi.mock("sonner", () => ({ toast: { error: vi.fn(), warning: vi.fn(), info: vi.fn() } })); +vi.mock("./useCameraDevices", () => ({ + useCameraDevices: (_enabled: boolean, preferred?: string) => ({ + selectedDeviceId: preferred ?? "cam-1", + selectedDevice: { label: preferred === "cam-2" ? "Camera 2" : "Camera 1" }, + isReady: true, + }), +})); + +type Prefs = Awaited>; +const initial: Prefs = { + micEnabled: false, + micDeviceId: null, + micDeviceName: null, + camEnabled: true, + camDeviceId: null, + camDeviceName: null, + camQuality: "2160p", + systemAudioEnabled: false, + cursorCaptureMode: "editable-overlay", + hideDesktopIcons: false, + autoZoomEnabled: true, +}; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +function setup() { + let publish: ((prefs: Prefs) => void) | undefined; + const stops: Array> = []; + const getUserMedia = vi.fn(async () => { + const stop = vi.fn(); + stops.push(stop); + const track = { stop, onended: null }; + return { getTracks: () => [track], getVideoTracks: () => [track] } as unknown as MediaStream; + }); + Object.defineProperty(navigator, "mediaDevices", { configurable: true, value: { getUserMedia } }); + window.electronAPI = { + getPlatform: () => "darwin", + getRecordingPrefs: async () => initial, + onRecordingPrefsChanged: (callback: (prefs: Prefs) => void) => { + publish = callback; + return () => undefined; + }, + hideCountdownOverlay: vi.fn().mockResolvedValue(undefined), + } as unknown as Window["electronAPI"]; + const view = renderHook(() => { + const recorder = useScreenRecorder(); + useCameraHudSync(recorder); + return recorder; + }); + return { view, getUserMedia, stops, publish: (prefs: Prefs) => publish?.(prefs) }; +} + +describe("camera preview across recording preference broadcasts", () => { + it("keeps the resolved default camera stream alive when audio toggles publish unchanged camera preferences", async () => { + const { view, getUserMedia, stops, publish } = setup(); + await waitFor(() => expect(view.result.current.webcamPreviewStream).not.toBeNull()); + const stream = view.result.current.webcamPreviewStream; + const calls = getUserMedia.mock.calls.length; + const stop = stops.at(-1); + await act(async () => { + view.result.current.setSystemAudioEnabled(true); + publish({ ...initial, systemAudioEnabled: true }); + }); + await act(async () => { + view.result.current.setMicrophoneEnabled(true); + publish({ ...initial, micEnabled: true, systemAudioEnabled: true }); + }); + expect(view.result.current.webcamPreviewStream).toBe(stream); + expect(getUserMedia).toHaveBeenCalledTimes(calls); + expect(stop).not.toHaveBeenCalled(); + }); + + it("still applies a changed camera selection and an explicit reset to the default", async () => { + const { view, publish } = setup(); + await waitFor(() => expect(view.result.current.webcamPreviewStream).not.toBeNull()); + const first = view.result.current.webcamPreviewStream; + await act(async () => publish({ ...initial, camDeviceId: "cam-2", camDeviceName: "Camera 2" })); + expect(view.result.current.webcamDeviceId).toBe("cam-2"); + expect(view.result.current.webcamPreviewStream).not.toBe(first); + await act(async () => publish(initial)); + expect(view.result.current.webcamDeviceId).toBe("cam-1"); + }); +}); diff --git a/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx b/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx index 817a008f93..65176bcb40 100644 --- a/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx +++ b/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx @@ -2,9 +2,10 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx b/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx index 928ac5886d..caad8e673e 100644 --- a/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx +++ b/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx @@ -2,9 +2,10 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.noCamera.test.tsx b/src/hooks/useScreenRecorder.noCamera.test.tsx index cd75f16b8d..4958a8fbf1 100644 --- a/src/hooks/useScreenRecorder.noCamera.test.tsx +++ b/src/hooks/useScreenRecorder.noCamera.test.tsx @@ -2,9 +2,10 @@ import { act, renderHook, waitFor } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, @@ -101,7 +102,9 @@ describe("useScreenRecorder on a machine without a camera (#967)", () => { expect(ok).toBe(true); expect(view.result.current.webcamEnabled).toBe(true); - await waitFor(() => expect(getUserMedia).toHaveBeenCalled()); + await waitFor(() => expect(view.result.current.webcamPreviewStream).not.toBeNull()); + expect(view.result.current.webcamPreviewStream).toBe(await getUserMedia.mock.results[0].value); + expect(getUserMedia).toHaveBeenCalledTimes(1); expect(setRecordingPrefs).not.toHaveBeenCalled(); }); diff --git a/src/hooks/useScreenRecorder.prefsRace.test.tsx b/src/hooks/useScreenRecorder.prefsRace.test.tsx index ae84c2cd71..edb9c58591 100644 --- a/src/hooks/useScreenRecorder.prefsRace.test.tsx +++ b/src/hooks/useScreenRecorder.prefsRace.test.tsx @@ -2,9 +2,10 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.restartFailure.test.tsx b/src/hooks/useScreenRecorder.restartFailure.test.tsx index 564936f41b..889f0eb70e 100644 --- a/src/hooks/useScreenRecorder.restartFailure.test.tsx +++ b/src/hooks/useScreenRecorder.restartFailure.test.tsx @@ -2,9 +2,10 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 8ed63698cd..471892cfa8 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -88,6 +88,7 @@ const AUDIO_BITRATE_VOICE = 128_000; const AUDIO_BITRATE_SYSTEM = 192_000; type UseScreenRecorderReturn = { + webcamPreviewStream: MediaStream | null; recording: boolean; paused: boolean; saving: boolean; @@ -285,6 +286,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { useEffect(() => { let cancelled = false; let receivedNewerSnapshot = false; + let appliedCameraSelection: { + deviceId: string | undefined; + deviceName: string | undefined; + } | null = null; const applyPrefs = (prefs: { micEnabled: boolean; micDeviceId?: string | null; @@ -307,8 +312,20 @@ export function useScreenRecorder(): UseScreenRecorderReturn { new URLSearchParams(window.location.search).get("windowType") === "cli-record"; if (!isCliRecord) { setWebcamEnabledState(prefs.camEnabled); - setWebcamDeviceId(prefs.camDeviceId ?? undefined); - setWebcamDeviceName(prefs.camDeviceName ?? undefined); + const deviceId = prefs.camDeviceId ?? undefined; + const deviceName = prefs.camDeviceName ?? undefined; + // The HUD resolves a stored default (null) to a concrete device locally. + // An audio-only save republishes that same null: applying it again would + // reset the resolved device, stop its stream, and visibly blink the preview. + if ( + !appliedCameraSelection || + appliedCameraSelection.deviceId !== deviceId || + appliedCameraSelection.deviceName !== deviceName + ) { + setWebcamDeviceId(deviceId); + setWebcamDeviceName(deviceName); + appliedCameraSelection = { deviceId, deviceName }; + } } setWebcamQuality(webcamQualityFrom(prefs.camQuality)); setSystemAudioEnabled(prefs.systemAudioEnabled); @@ -351,6 +368,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const screenStream = useRef(null); const microphoneStream = useRef(null); const webcamStream = useRef(null); + const [webcamPreviewStream, setWebcamPreviewStream] = useState(null); const mixingContext = useRef(null); const recordingId = useRef(0); const accumulatedDurationMs = useRef(0); @@ -472,6 +490,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { track.stop(); }); webcamStream.current = null; + setWebcamPreviewStream(null); webcamReady.current = true; }, []); @@ -519,6 +538,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { stream.getVideoTracks().forEach((track) => { track.onended = () => { webcamStream.current = null; + setWebcamPreviewStream(null); if (!restarting.current) { setWebcamEnabledState(false); toast.error(t("recording.cameraDisconnected")); @@ -526,6 +546,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; }); webcamStream.current = stream; + setWebcamPreviewStream(stream); webcamReady.current = true; } catch (cameraError) { if (isRecoverableWebcamConstraintError(cameraError, webcamDeviceName)) { @@ -565,6 +586,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { track.stop(); }); webcamStream.current = null; + setWebcamPreviewStream(null); } }; }, [webcamEnabled, webcamDeviceId, webcamDeviceName, webcamQuality, t]); @@ -2520,6 +2542,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; return { + webcamPreviewStream, recording, paused, saving, diff --git a/src/hooks/webcamCaptureTarget.test.ts b/src/hooks/webcamCaptureTarget.test.ts index 25f83efdd4..0de5da0477 100644 --- a/src/hooks/webcamCaptureTarget.test.ts +++ b/src/hooks/webcamCaptureTarget.test.ts @@ -16,10 +16,12 @@ describe("webcamVideoConstraints", () => { expect(webcamVideoConstraints(undefined, "2160p")).toMatchObject({ width: { ideal: 3840 }, height: { ideal: 2160 }, + aspectRatio: { ideal: 16 / 9 }, }); expect(webcamVideoConstraints(undefined, "1080p")).toMatchObject({ width: { ideal: 1920 }, height: { ideal: 1080 }, + aspectRatio: { ideal: 16 / 9 }, }); }); @@ -33,6 +35,8 @@ describe("webcamVideoConstraints", () => { expect(constraints.height).not.toHaveProperty("exact"); expect(constraints.width).not.toHaveProperty("min"); expect(constraints.height).not.toHaveProperty("min"); + expect(constraints.aspectRatio).not.toHaveProperty("exact"); + expect(constraints.aspectRatio).not.toHaveProperty("min"); }); it("still pins the chosen device and caps the frame rate", () => { diff --git a/src/hooks/webcamCaptureTarget.ts b/src/hooks/webcamCaptureTarget.ts index 906412b19f..fa960469fc 100644 --- a/src/hooks/webcamCaptureTarget.ts +++ b/src/hooks/webcamCaptureTarget.ts @@ -74,6 +74,8 @@ export function webcamVideoConstraints( ...(deviceId ? { deviceId: { exact: deviceId } } : {}), width: { ideal: preset.width }, height: { ideal: preset.height }, + // Width/height alone can select a portrait mode on cameras offering both orientations. + aspectRatio: { ideal: preset.width / preset.height }, frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE }, }; } diff --git a/src/i18n/locales/ar/launch.json b/src/i18n/locales/ar/launch.json index ef613d15d4..538b34e262 100644 --- a/src/i18n/locales/ar/launch.json +++ b/src/i18n/locales/ar/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "معاينة الكاميرا", + "show": "إظهار معاينة الكاميرا", + "hide": "إخفاء معاينة الكاميرا", + "smaller": "تصغير المعاينة", + "larger": "تكبير المعاينة" + }, "tooltips": { "hideHUD": "إخفاء واجهة العرض", "hideHUDTip": "إخفاء الشريط. يمكنك إظهاره مجددًا من أيقونة علبة النظام.", diff --git a/src/i18n/locales/cs/launch.json b/src/i18n/locales/cs/launch.json index d9912da515..a5a63939a2 100644 --- a/src/i18n/locales/cs/launch.json +++ b/src/i18n/locales/cs/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Náhled kamery", + "show": "Zobrazit náhled kamery", + "hide": "Skrýt náhled kamery", + "smaller": "Zmenšit náhled", + "larger": "Zvětšit náhled" + }, "tooltips": { "hideHUD": "Skrýt HUD", "hideHUDTip": "Skrýt panel. Znovu ho zobrazíte z ikony v oznamovací oblasti.", diff --git a/src/i18n/locales/de/launch.json b/src/i18n/locales/de/launch.json index b4dab82a91..204b6e8589 100644 --- a/src/i18n/locales/de/launch.json +++ b/src/i18n/locales/de/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Kameravorschau", + "show": "Kameravorschau anzeigen", + "hide": "Kameravorschau ausblenden", + "smaller": "Vorschau verkleinern", + "larger": "Vorschau vergrößern" + }, "tooltips": { "hideHUD": "HUD ausblenden", "hideHUDTip": "Aufnahmeleiste ausblenden. Über das Symbol im Infobereich wieder einblenden.", diff --git a/src/i18n/locales/en/launch.json b/src/i18n/locales/en/launch.json index 97263f1849..4987f2396e 100644 --- a/src/i18n/locales/en/launch.json +++ b/src/i18n/locales/en/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Camera preview", + "show": "Show camera preview", + "hide": "Hide camera preview", + "smaller": "Decrease preview size", + "larger": "Increase preview size" + }, "tooltips": { "hideHUD": "Hide recording bar", "hideHUDTip": "Hide the recording bar. Show it again from the tray icon.", diff --git a/src/i18n/locales/es/launch.json b/src/i18n/locales/es/launch.json index 3114921ec5..fc230bba7f 100644 --- a/src/i18n/locales/es/launch.json +++ b/src/i18n/locales/es/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Vista previa de cámara", + "show": "Mostrar vista previa de cámara", + "hide": "Ocultar vista previa de cámara", + "smaller": "Reducir vista previa", + "larger": "Ampliar vista previa" + }, "tooltips": { "hideHUD": "Ocultar HUD", "hideHUDTip": "Ocultar la barra. Vuelve a mostrarla desde el icono de la bandeja del sistema.", diff --git a/src/i18n/locales/fr/launch.json b/src/i18n/locales/fr/launch.json index c956fb37ad..b768831edf 100644 --- a/src/i18n/locales/fr/launch.json +++ b/src/i18n/locales/fr/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Aperçu de la caméra", + "show": "Afficher l’aperçu de la caméra", + "hide": "Masquer l’aperçu de la caméra", + "smaller": "Réduire l’aperçu", + "larger": "Agrandir l’aperçu" + }, "tooltips": { "hideHUD": "Masquer le HUD", "hideHUDTip": "Masquer la barre. Réaffichez-la depuis l'icône de la zone de notification.", diff --git a/src/i18n/locales/it/launch.json b/src/i18n/locales/it/launch.json index 9897b92eb5..cad6d15516 100644 --- a/src/i18n/locales/it/launch.json +++ b/src/i18n/locales/it/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Anteprima della videocamera", + "show": "Mostra anteprima della videocamera", + "hide": "Nascondi anteprima della videocamera", + "smaller": "Riduci anteprima", + "larger": "Ingrandisci anteprima" + }, "tooltips": { "hideHUD": "Nascondi", "hideHUDTip": "Nascondi la barra. Puoi mostrarla di nuovo dall'icona nell'area di notifica.", diff --git a/src/i18n/locales/ja-JP/launch.json b/src/i18n/locales/ja-JP/launch.json index 345da320dd..aa6d4b5677 100644 --- a/src/i18n/locales/ja-JP/launch.json +++ b/src/i18n/locales/ja-JP/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "カメラプレビュー", + "show": "カメラプレビューを表示", + "hide": "カメラプレビューを非表示", + "smaller": "プレビューを縮小", + "larger": "プレビューを拡大" + }, "tooltips": { "hideHUD": "HUDを隠す", "hideHUDTip": "録画バーを隠します。通知領域のアイコンから再表示できます。", diff --git a/src/i18n/locales/ko-KR/launch.json b/src/i18n/locales/ko-KR/launch.json index 5ce2b8e721..455a84146a 100644 --- a/src/i18n/locales/ko-KR/launch.json +++ b/src/i18n/locales/ko-KR/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "카메라 미리보기", + "show": "카메라 미리보기 표시", + "hide": "카메라 미리보기 숨기기", + "smaller": "미리보기 축소", + "larger": "미리보기 확대" + }, "tooltips": { "hideHUD": "HUD 숨기기", "hideHUDTip": "녹화 바를 숨깁니다. 알림 영역 아이콘에서 다시 표시할 수 있습니다.", diff --git a/src/i18n/locales/pt-BR/launch.json b/src/i18n/locales/pt-BR/launch.json index 8866150881..893576b96f 100644 --- a/src/i18n/locales/pt-BR/launch.json +++ b/src/i18n/locales/pt-BR/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Prévia da câmera", + "show": "Mostrar prévia da câmera", + "hide": "Ocultar prévia da câmera", + "smaller": "Diminuir prévia", + "larger": "Aumentar prévia" + }, "tooltips": { "hideHUD": "Ocultar HUD", "hideHUDTip": "Ocultar a barra. Mostre-a de novo pelo ícone da bandeja do sistema.", diff --git a/src/i18n/locales/ru/launch.json b/src/i18n/locales/ru/launch.json index cdc103c0b4..a34023430a 100644 --- a/src/i18n/locales/ru/launch.json +++ b/src/i18n/locales/ru/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Предпросмотр камеры", + "show": "Показать предпросмотр камеры", + "hide": "Скрыть предпросмотр камеры", + "smaller": "Уменьшить предпросмотр", + "larger": "Увеличить предпросмотр" + }, "tooltips": { "hideHUD": "Скрыть HUD", "hideHUDTip": "Скрыть панель. Вернуть её можно через значок в области уведомлений.", diff --git a/src/i18n/locales/tr/launch.json b/src/i18n/locales/tr/launch.json index 530157dd2c..929709e362 100644 --- a/src/i18n/locales/tr/launch.json +++ b/src/i18n/locales/tr/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Kamera önizlemesi", + "show": "Kamera önizlemesini göster", + "hide": "Kamera önizlemesini gizle", + "smaller": "Önizlemeyi küçült", + "larger": "Önizlemeyi büyüt" + }, "tooltips": { "hideHUD": "Kontrol panelini gizle", "hideHUDTip": "Araç çubuğunu gizle. Bildirim alanındaki simgeden yeniden gösterebilirsiniz.", diff --git a/src/i18n/locales/vi/launch.json b/src/i18n/locales/vi/launch.json index 53f452e27c..fe3ce29fa3 100644 --- a/src/i18n/locales/vi/launch.json +++ b/src/i18n/locales/vi/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "Xem trước camera", + "show": "Hiện bản xem trước camera", + "hide": "Ẩn bản xem trước camera", + "smaller": "Thu nhỏ bản xem trước", + "larger": "Phóng to bản xem trước" + }, "tooltips": { "hideHUD": "Ẩn HUD", "hideHUDTip": "Ẩn thanh ghi hình. Hiện lại từ biểu tượng ở khay hệ thống.", diff --git a/src/i18n/locales/zh-CN/launch.json b/src/i18n/locales/zh-CN/launch.json index a4bbf2c576..b53025f9c4 100644 --- a/src/i18n/locales/zh-CN/launch.json +++ b/src/i18n/locales/zh-CN/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "摄像头预览", + "show": "显示摄像头预览", + "hide": "隐藏摄像头预览", + "smaller": "缩小预览", + "larger": "放大预览" + }, "tooltips": { "hideHUD": "隐藏录制栏", "hideHUDTip": "隐藏录制栏，可通过系统托盘图标重新显示", diff --git a/src/i18n/locales/zh-TW/launch.json b/src/i18n/locales/zh-TW/launch.json index 2e460592ed..03d134c201 100644 --- a/src/i18n/locales/zh-TW/launch.json +++ b/src/i18n/locales/zh-TW/launch.json @@ -1,4 +1,11 @@ { + "floatingCamera": { + "title": "攝影機預覽", + "show": "顯示攝影機預覽", + "hide": "隱藏攝影機預覽", + "smaller": "縮小預覽", + "larger": "放大預覽" + }, "tooltips": { "hideHUD": "隱藏控制面板", "hideHUDTip": "隱藏控制面板。可從系統匣圖示重新顯示。", diff --git a/src/lib/cameraPreview.test.ts b/src/lib/cameraPreview.test.ts new file mode 100644 index 0000000000..c98f8548fb --- /dev/null +++ b/src/lib/cameraPreview.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { cameraPreviewBounds, resizedCameraPreviewBounds } from "./cameraPreview"; + +const workArea = { x: 0, y: 25, width: 1512, height: 900 }; +const bar = { x: 300, y: 840, width: 900, height: 64 }; + +describe("camera preview placement", () => { + it("starts just above the recording bar's right end", () => { + const preview = cameraPreviewBounds(workArea, bar); + expect(preview.x + preview.width).toBe(bar.x + bar.width); + expect(preview.y + preview.height).toBe(bar.y - 8); + }); + + it("keeps the bottom centre fixed when resized near the bar", () => { + const before = cameraPreviewBounds(workArea, bar); + const after = resizedCameraPreviewBounds(before, workArea, 320); + expect(after.width).toBe(336); + expect(after.height).toBe(336); + expect(after.y + after.height).toBe(before.y + before.height); + expect(after.x + after.width / 2).toBe(before.x + before.width / 2); + }); + + it("keeps a preview on-screen on negative-origin displays and near the screen edge", () => { + const display = { x: -1280, y: -100, width: 1280, height: 720 }; + const preview = resizedCameraPreviewBounds( + { x: -1280, y: -100, width: 176, height: 176 }, + display, + 320, + ); + expect(preview.x).toBeGreaterThanOrEqual(display.x); + expect(preview.y).toBeGreaterThanOrEqual(display.y); + expect(preview.x + preview.width).toBeLessThanOrEqual(0); + expect(preview.y + preview.height).toBeLessThanOrEqual(620); + }); +}); diff --git a/src/lib/cameraPreview.ts b/src/lib/cameraPreview.ts new file mode 100644 index 0000000000..613d0e4109 --- /dev/null +++ b/src/lib/cameraPreview.ts @@ -0,0 +1,37 @@ +export const CAMERA_PREVIEW_SIZES = [160, 224, 320] as const; +export const CAMERA_PREVIEW_DEFAULT_SIZE = 224; +const WINDOW_PADDING = 16; + +type Rect = { x: number; y: number; width: number; height: number }; + +function clampBounds(x: number, y: number, diameter: number, workArea: Rect): Rect { + const size = Math.min(diameter + WINDOW_PADDING, workArea.width, workArea.height); + return { + x: Math.round(Math.max(workArea.x, Math.min(x, workArea.x + workArea.width - size))), + y: Math.round(Math.max(workArea.y, Math.min(y, workArea.y + workArea.height - size))), + width: size, + height: size, + }; +} + +/** Place the circle just above the bar's right end, including the transparent window padding. */ +export function cameraPreviewBounds(workArea: Rect, bar?: Rect): Rect { + const size = CAMERA_PREVIEW_DEFAULT_SIZE + WINDOW_PADDING; + return clampBounds( + bar ? bar.x + bar.width - size : workArea.x + workArea.width / 2 + 160, + (bar?.y ?? workArea.y + workArea.height - 84) - size - 8, + CAMERA_PREVIEW_DEFAULT_SIZE, + workArea, + ); +} + +/** Grow upward from the bottom centre, retaining its distance from the recording bar. */ +export function resizedCameraPreviewBounds(bounds: Rect, workArea: Rect, diameter: number): Rect { + const size = diameter + WINDOW_PADDING; + return clampBounds( + bounds.x + (bounds.width - size) / 2, + bounds.y + bounds.height - size, + diameter, + workArea, + ); +} diff --git a/technical-documentation/architecture/recording.md b/technical-documentation/architecture/recording.md index 2f643ec222..7e70f78197 100644 --- a/technical-documentation/architecture/recording.md +++ b/technical-documentation/architecture/recording.md @@ -22,6 +22,33 @@ The HUD starts and controls a recording. The source selector chooses a display o Electron applies `setContentProtection(true)` to the HUD window (`applyContentProtection` in `electron/windows.ts`). This keeps the controller out of captures and also makes it invisible in screenshots. It is skipped on macOS 26 and later, which never displays a protected window, and on Windows before 11 22H2 (build 22621), where it leaves the click-through HUD unable to take a click (#1105). On those Windows versions the HUD can appear in recordings; on macOS the ScreenCaptureKit helper still excludes it by window id. For a testing session only, `OPENSCREEN_DISABLE_CONTENT_PROTECTION=1` disables the protection; the code warns that the HUD then appears in captures. The tray icon is the reliable way to refocus OpenScreen or stop a recording when the click-through HUD is not convenient or is not visible. +## Presenter camera preview (macOS) + +When the HUD camera is enabled, a draggable circular self-view floats above other windows. +It starts just above the bar’s right end, centres the camera image to fill the circle +without stretching or letterboxing, and offers hover/focus minus/plus controls for +160, 224 and 320px diameters. +Resizing retains the window id and anchors its bottom centre, clamped to its display. +Camera constraints explicitly prefer the landscape aspect ratio of the selected quality +preset; width/height preferences alone can select a portrait mode on some cameras. +The eye button on the HUD and the circle's hover/focus hide button toggle the preview without +changing camera recording. The eye button stays visible but disabled when the camera is off. +Audio preference broadcasts retain the resolved camera selection unless the stored camera +identity changes, so toggling audio does not reopen its stream. This first implementation is macOS-only; Windows and Linux keep +the existing recording controls. + +`useScreenRecorder` exposes its existing camera stream to `useFloatingCameraPreview`, which +publishes 320px JPEG frames at up to 10 fps with one IPC request in flight. The preview never +opens a second camera session or stops the recorder's tracks. The main process accepts frames +only from the owning HUD and limits the payload size. `CameraPreviewController` owns the +auxiliary window and destroys it when the camera is disabled or its HUD is destroyed. + +Hiding the circle retains the window and its capture exclusion id. Both macOS source-picker +and direct-display requests exclude it alongside the HUD and Notes. The direct recorder +includes hidden windows when resolving exclusions, keeping the circle excluded even when +first shown during a take. The native helper must be rebuilt after this change. The window +uses the same content-protection and Spaces policy as the HUD. + ## Capture backends | Platform | Backend | Code | Produces | diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 22b5601ee5..ae447caa48 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -1076,3 +1076,7 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-22 | PR #1177, maintainer presentation follow-up on `54ee4e15` (dev build, Electron 41.2.1) | macOS 15.8, Apple Silicon | **Partial: Agent usage presentation** | Real Electron driven through OS computer-use with an isolated `/tmp` profile and a local scripted Anthropic-compatible provider (synthetic counts, no paid provider). A two-call turn showed muted `≥10.3k tokens` beside the spinner, one expandable Thinking label while reasoning streamed, then `12.3k tokens · 45s` beneath the reply. Captured [running](https://raw.githubusercontent.com/auberginewly/openscreen-community/2cf6e8435c9214981243088aebc470380e6f8611/technical-documentation/testing/screenshots/pr-1177/running.png) and [completed](https://raw.githubusercontent.com/auberginewly/openscreen-community/2cf6e8435c9214981243088aebc470380e6f8611/technical-documentation/testing/screenshots/pr-1177/completed.png) screenshots. Component tests also cover partial totals with details confined to the tooltip, unavailable usage showing duration only, and unavailable usage without duration rendering nothing. Skipped: capture/preview/export, native binaries, paid-provider billing, restart/session switching (covered in the preceding pass), Windows/Linux and the rest of this checklist; chat presentation only. | | 2026-10-27 | Dev `feat/agent-execution-steps-1169` on `main` @ `5bb1b3fb`, Electron 41.2.1; Vite 5191, private `/tmp/openscreen-execution-ui-*` profile and local Anthropic-shaped fixture provider | macOS 15.8, Apple Silicon | **Partial — execution steps and Markdown** | Real Electron driven through OS computer-use. Sent a multiline Markdown request with a heading, list and indented code fence; observed two separate successful `getCurrentDocument` rows and one failed `getTranscript` row, with two provider-exposed reasoning blocks in order. Expanded reasoning and failure details. Completed answer rendered heading, list, code and GFM table. Corrected code-block contrast to use theme colors. New conversations remained empty; switching during the second turn cancelled it and saved a stopped assistant with five steps in its original session. Fully quit/relaunched, reopened Studio and verified both completed/stopped traces, failure labels and Markdown restored without usable old rewind controls. Final observer implementation was exercised in a third real SDK/HTTP turn with no duplicate rows. Automated tests additionally cover schema-invalid/unknown calls, out-of-order completions, failed-turn persistence, source copying and unsafe Markdown. Screenshots retained in the task evidence. Skipped: paid/remote providers, capture/audio/webcam, native preview/export, Windows/Linux runtime and the rest of this checklist; chat-only slice, not release-promotion QA. | | 2026-10-17 | Local source build and packaged 2.1.1 AppImage, `integration/linux-x11-capture`, based on `main` 2.1.1; rebuilt Linux capture helper, complete unchanged compositor/runtime directory from official 2.1.1 AppImage | Ubuntu 24.04.5, Xfce/X11, ScreenCast portal interface absent | **Partial — native X11 CLI capture/export passed** | Recorded an isolated GTK window through `record --window` for 12 s, with real OS mouse movement/press/release events through XTest. Saved 301 cursor samples, 7 click events and 4 cursor sprites; positions outside the moved window remained hidden. `export --auto-zoom` generated one region and produced a fully decodable H.264/AAC MP4, 1624×1080, 30 fps, 12.033 s. 158 of 360 decoded frames differed from the same export without zoom; inspected the rendered hold. Packaged AppImage also recorded the GTK window on an isolated X11 desktop with XTest: 286 cursor samples, 3 clicks and 1 sprite; CLI auto-zoom export decoded as 1624×1080 H.264 at 30 fps. Final AppImage also passed explicit `--display 0` monitor capture (309 samples, 3 clicks) and window capture (325 samples, 3 clicks). The source-index `--display` switch is removed from Chromium before GTK initializes, avoiding an X-server argument collision. A window-unmap test exposed finalized partial files being reported as success; nonzero helper exits now reject both pending and subsequent stops, with unit regression coverage and a CLI failure check. No ScreenCast portal was installed or activated. **Not covered:** the full HUD/editor checklist, Wayland, Windows, macOS, mixed-DPI monitor capture (explicitly rejected; window capture remains available), or changing window size during a take (stops with an error). Computer-use MCP was unavailable; the native CLI slice used real X11 events rather than injected DOM events. | +| 2026-10-16 | dev `main` @ `b149c55` + uncommitted #1278 camera preview; Electron 41.2.1; capture helper compiled from this working tree using `swiftc` + macOS 26.5 SDK, SHA256 `06f142f075e4dd776b653f174379b2795a60d750557655f9bca6e2d2596c8f99`; remaining native payload copied as a complete directory from installed OpenScreen | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — camera preview slice passed** | Driven through `mcp__cua_repl` against the Electron app, with the legacy source picker and content protection disabled for UI visibility. Verified live camera frames in a circle with no card/header, HUD show/hide and circle hide without disabling camera capture, and show/hide while recording. Exercised OS dragging; final position was not independently measured. Stopping closed the preview and opened the editor; screen MP4 and separate webcam WebM were finalized. A second take started with the preview hidden, then showed it during capture using the rebuilt helper; a decoded late screen frame contains no floating circle (sample `/tmp/openscreen-camera-preview-capture.png`). This sample is not an exhaustive frame-by-frame exclusion test. Test takes: `recording-1792129915224` (borrowed helper), `recording-1792130142748` (rebuilt helper). **Skipped:** Apple's picker, full-screen Spaces, restart/cancel, disconnect/reconnect on physical hardware, export, all other checklist sections, Windows and Linux (implementation is macOS-first). SwiftPM cannot start with this machine's Command Line Tools (`BuildServerProtocol` dyld error); direct `swiftc` compilation passed, but XCTest is unavailable without full Xcode. | +| 2026-10-16 | dev `main` @ `b149c55` + #1278 audio-toggle and persistent preview-control fixes; Electron 41.2.1 | macOS 26.7, Apple Silicon | **Partial — HUD follow-up slice passed** | `mcp__cua_repl`: toggled system audio on/off and microphone off/on with camera and preview enabled; restored original audio preferences. Camera off leaves Show camera preview visible and disabled; camera on re-enables it. Regression test reproduces and fixes audio snapshots reopening the resolved default camera, asserting identical stream and no stopped tracks across both audio changes; explicit camera changes and resets still apply. **Skipped:** recording, export, all remaining checklist sections, and Windows/Linux for this follow-up. | +| 2026-10-16 | dev `main` @ `b149c55` + #1278 preview framing, placement and resizing | macOS 26.7, Apple Silicon | **Partial — preview UI slice passed** | `mcp__cua_repl`: inspected live fitted camera frames (portrait source, with black space at the sides instead of a zoomed crop), increased to 320px, decreased through 224px to 160px, verified limit buttons disable, and restored 224px. Window sizes changed visibly while remaining circular. Default placement above the HUD and resize anchoring are covered by geometry tests; their screen coordinates were not independently measured in the desktop pass. **Skipped:** recording/export, Apple’s picker, display disconnection, Windows/Linux, and all other checklist sections. | +| 2026-10-16 | dev `main` @ `b149c55` + #1278 normal camera framing follow-up | macOS 26.7, MacBook Pro Camera | **Partial — preview framing slice passed** | `mcp__cua_repl`: verified the circle fills with centred, unstretched live video and no side bars after preferring the preset’s landscape aspect ratio and removing letterboxing. The earlier webcam sidecar was 1080×1920 despite landscape width/height preferences. Temporarily hid and restored the recording bar for inspection, then left the camera and preview enabled. **Skipped:** recording/export, other cameras, Windows/Linux and all other checklist sections. | From bdef6c3cf182b866d62f350582ebb107933ded2a Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Fri, 16 Oct 2026 14:50:53 +0530 Subject: [PATCH 2/8] refactor: scope camera preview changes to macOS --- .../useScreenRecorder.cameraPrefs.test.tsx | 2 ++ ...eenRecorder.nativeMacStartWarning.test.tsx | 7 ++-- ...eScreenRecorder.nativeStopFailure.test.tsx | 7 ++-- .../useScreenRecorder.prefsRace.test.tsx | 7 ++-- .../useScreenRecorder.restartFailure.test.tsx | 7 ++-- src/hooks/useScreenRecorder.ts | 4 +++ src/hooks/webcamCaptureTarget.test.ts | 4 --- src/hooks/webcamCaptureTarget.ts | 2 -- .../architecture/recording.md | 34 ++++++------------- .../testing/manual-e2e-checklist.md | 5 +-- 10 files changed, 29 insertions(+), 50 deletions(-) diff --git a/src/hooks/useScreenRecorder.cameraPrefs.test.tsx b/src/hooks/useScreenRecorder.cameraPrefs.test.tsx index b7f4f41297..d60d1a56d2 100644 --- a/src/hooks/useScreenRecorder.cameraPrefs.test.tsx +++ b/src/hooks/useScreenRecorder.cameraPrefs.test.tsx @@ -34,10 +34,12 @@ const initial: Prefs = { afterEach(() => { cleanup(); + window.history.replaceState({}, "", "/"); vi.restoreAllMocks(); }); function setup() { + window.history.replaceState({}, "", "/?windowType=hud-overlay"); let publish: ((prefs: Prefs) => void) | undefined; const stops: Array> = []; const getUserMedia = vi.fn(async () => { diff --git a/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx b/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx index 65176bcb40..817a008f93 100644 --- a/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx +++ b/src/hooks/useScreenRecorder.nativeMacStartWarning.test.tsx @@ -2,10 +2,9 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => { - const translate = (key: string) => key; - return { useScopedT: () => translate }; -}); +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: () => (key: string) => key, +})); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx b/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx index caad8e673e..928ac5886d 100644 --- a/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx +++ b/src/hooks/useScreenRecorder.nativeStopFailure.test.tsx @@ -2,10 +2,9 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => { - const translate = (key: string) => key; - return { useScopedT: () => translate }; -}); +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: () => (key: string) => key, +})); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.prefsRace.test.tsx b/src/hooks/useScreenRecorder.prefsRace.test.tsx index edb9c58591..ae84c2cd71 100644 --- a/src/hooks/useScreenRecorder.prefsRace.test.tsx +++ b/src/hooks/useScreenRecorder.prefsRace.test.tsx @@ -2,10 +2,9 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => { - const translate = (key: string) => key; - return { useScopedT: () => translate }; -}); +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: () => (key: string) => key, +})); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.restartFailure.test.tsx b/src/hooks/useScreenRecorder.restartFailure.test.tsx index 889f0eb70e..564936f41b 100644 --- a/src/hooks/useScreenRecorder.restartFailure.test.tsx +++ b/src/hooks/useScreenRecorder.restartFailure.test.tsx @@ -2,10 +2,9 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => { - const translate = (key: string) => key; - return { useScopedT: () => translate }; -}); +vi.mock("@/contexts/I18nContext", () => ({ + useScopedT: () => (key: string) => key, +})); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 471892cfa8..2f0ecedb45 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -317,7 +317,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // The HUD resolves a stored default (null) to a concrete device locally. // An audio-only save republishes that same null: applying it again would // reset the resolved device, stop its stream, and visibly blink the preview. + const isMacHud = + window.electronAPI?.getPlatform?.() === "darwin" && + new URLSearchParams(window.location.search).get("windowType") === "hud-overlay"; if ( + !isMacHud || !appliedCameraSelection || appliedCameraSelection.deviceId !== deviceId || appliedCameraSelection.deviceName !== deviceName diff --git a/src/hooks/webcamCaptureTarget.test.ts b/src/hooks/webcamCaptureTarget.test.ts index 0de5da0477..25f83efdd4 100644 --- a/src/hooks/webcamCaptureTarget.test.ts +++ b/src/hooks/webcamCaptureTarget.test.ts @@ -16,12 +16,10 @@ describe("webcamVideoConstraints", () => { expect(webcamVideoConstraints(undefined, "2160p")).toMatchObject({ width: { ideal: 3840 }, height: { ideal: 2160 }, - aspectRatio: { ideal: 16 / 9 }, }); expect(webcamVideoConstraints(undefined, "1080p")).toMatchObject({ width: { ideal: 1920 }, height: { ideal: 1080 }, - aspectRatio: { ideal: 16 / 9 }, }); }); @@ -35,8 +33,6 @@ describe("webcamVideoConstraints", () => { expect(constraints.height).not.toHaveProperty("exact"); expect(constraints.width).not.toHaveProperty("min"); expect(constraints.height).not.toHaveProperty("min"); - expect(constraints.aspectRatio).not.toHaveProperty("exact"); - expect(constraints.aspectRatio).not.toHaveProperty("min"); }); it("still pins the chosen device and caps the frame rate", () => { diff --git a/src/hooks/webcamCaptureTarget.ts b/src/hooks/webcamCaptureTarget.ts index fa960469fc..906412b19f 100644 --- a/src/hooks/webcamCaptureTarget.ts +++ b/src/hooks/webcamCaptureTarget.ts @@ -74,8 +74,6 @@ export function webcamVideoConstraints( ...(deviceId ? { deviceId: { exact: deviceId } } : {}), width: { ideal: preset.width }, height: { ideal: preset.height }, - // Width/height alone can select a portrait mode on cameras offering both orientations. - aspectRatio: { ideal: preset.width / preset.height }, frameRate: { ideal: WEBCAM_TARGET_FRAME_RATE, max: WEBCAM_TARGET_FRAME_RATE }, }; } diff --git a/technical-documentation/architecture/recording.md b/technical-documentation/architecture/recording.md index 7e70f78197..41d3d6470f 100644 --- a/technical-documentation/architecture/recording.md +++ b/technical-documentation/architecture/recording.md @@ -24,30 +24,16 @@ Electron applies `setContentProtection(true)` to the HUD window (`applyContentPr ## Presenter camera preview (macOS) -When the HUD camera is enabled, a draggable circular self-view floats above other windows. -It starts just above the bar’s right end, centres the camera image to fill the circle -without stretching or letterboxing, and offers hover/focus minus/plus controls for -160, 224 and 320px diameters. -Resizing retains the window id and anchors its bottom centre, clamped to its display. -Camera constraints explicitly prefer the landscape aspect ratio of the selected quality -preset; width/height preferences alone can select a portrait mode on some cameras. -The eye button on the HUD and the circle's hover/focus hide button toggle the preview without -changing camera recording. The eye button stays visible but disabled when the camera is off. -Audio preference broadcasts retain the resolved camera selection unless the stored camera -identity changes, so toggling audio does not reopen its stream. This first implementation is macOS-only; Windows and Linux keep -the existing recording controls. - -`useScreenRecorder` exposes its existing camera stream to `useFloatingCameraPreview`, which -publishes 320px JPEG frames at up to 10 fps with one IPC request in flight. The preview never -opens a second camera session or stops the recorder's tracks. The main process accepts frames -only from the owning HUD and limits the payload size. `CameraPreviewController` owns the -auxiliary window and destroys it when the camera is disabled or its HUD is destroyed. - -Hiding the circle retains the window and its capture exclusion id. Both macOS source-picker -and direct-display requests exclude it alongside the HUD and Notes. The direct recorder -includes hidden windows when resolving exclusions, keeping the circle excluded even when -first shown during a take. The native helper must be rebuilt after this change. The window -uses the same content-protection and Spaces policy as the HUD. +The HUD camera has a draggable circular self-view with show/hide and three size choices. +Its preview button stays visible but disabled when the camera is off. The preview shares +`useScreenRecorder`'s stream and stops with its owning HUD; it does not acquire another camera. +`useFloatingCameraPreview` publishes small JPEG frames to the auxiliary window, with bounded +payloads and one IPC request in flight. Audio preference broadcasts in the macOS HUD preserve +its resolved camera selection so the preview does not blink when audio is toggled. + +Hiding or resizing retains the window id. macOS capture excludes that id alongside the HUD +and Notes, including windows hidden when capture starts. This requires rebuilding the native +helper. Windows and Linux keep their existing recording controls. ## Capture backends diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index ae447caa48..fc651b7cd4 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -1076,7 +1076,4 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-22 | PR #1177, maintainer presentation follow-up on `54ee4e15` (dev build, Electron 41.2.1) | macOS 15.8, Apple Silicon | **Partial: Agent usage presentation** | Real Electron driven through OS computer-use with an isolated `/tmp` profile and a local scripted Anthropic-compatible provider (synthetic counts, no paid provider). A two-call turn showed muted `≥10.3k tokens` beside the spinner, one expandable Thinking label while reasoning streamed, then `12.3k tokens · 45s` beneath the reply. Captured [running](https://raw.githubusercontent.com/auberginewly/openscreen-community/2cf6e8435c9214981243088aebc470380e6f8611/technical-documentation/testing/screenshots/pr-1177/running.png) and [completed](https://raw.githubusercontent.com/auberginewly/openscreen-community/2cf6e8435c9214981243088aebc470380e6f8611/technical-documentation/testing/screenshots/pr-1177/completed.png) screenshots. Component tests also cover partial totals with details confined to the tooltip, unavailable usage showing duration only, and unavailable usage without duration rendering nothing. Skipped: capture/preview/export, native binaries, paid-provider billing, restart/session switching (covered in the preceding pass), Windows/Linux and the rest of this checklist; chat presentation only. | | 2026-10-27 | Dev `feat/agent-execution-steps-1169` on `main` @ `5bb1b3fb`, Electron 41.2.1; Vite 5191, private `/tmp/openscreen-execution-ui-*` profile and local Anthropic-shaped fixture provider | macOS 15.8, Apple Silicon | **Partial — execution steps and Markdown** | Real Electron driven through OS computer-use. Sent a multiline Markdown request with a heading, list and indented code fence; observed two separate successful `getCurrentDocument` rows and one failed `getTranscript` row, with two provider-exposed reasoning blocks in order. Expanded reasoning and failure details. Completed answer rendered heading, list, code and GFM table. Corrected code-block contrast to use theme colors. New conversations remained empty; switching during the second turn cancelled it and saved a stopped assistant with five steps in its original session. Fully quit/relaunched, reopened Studio and verified both completed/stopped traces, failure labels and Markdown restored without usable old rewind controls. Final observer implementation was exercised in a third real SDK/HTTP turn with no duplicate rows. Automated tests additionally cover schema-invalid/unknown calls, out-of-order completions, failed-turn persistence, source copying and unsafe Markdown. Screenshots retained in the task evidence. Skipped: paid/remote providers, capture/audio/webcam, native preview/export, Windows/Linux runtime and the rest of this checklist; chat-only slice, not release-promotion QA. | | 2026-10-17 | Local source build and packaged 2.1.1 AppImage, `integration/linux-x11-capture`, based on `main` 2.1.1; rebuilt Linux capture helper, complete unchanged compositor/runtime directory from official 2.1.1 AppImage | Ubuntu 24.04.5, Xfce/X11, ScreenCast portal interface absent | **Partial — native X11 CLI capture/export passed** | Recorded an isolated GTK window through `record --window` for 12 s, with real OS mouse movement/press/release events through XTest. Saved 301 cursor samples, 7 click events and 4 cursor sprites; positions outside the moved window remained hidden. `export --auto-zoom` generated one region and produced a fully decodable H.264/AAC MP4, 1624×1080, 30 fps, 12.033 s. 158 of 360 decoded frames differed from the same export without zoom; inspected the rendered hold. Packaged AppImage also recorded the GTK window on an isolated X11 desktop with XTest: 286 cursor samples, 3 clicks and 1 sprite; CLI auto-zoom export decoded as 1624×1080 H.264 at 30 fps. Final AppImage also passed explicit `--display 0` monitor capture (309 samples, 3 clicks) and window capture (325 samples, 3 clicks). The source-index `--display` switch is removed from Chromium before GTK initializes, avoiding an X-server argument collision. A window-unmap test exposed finalized partial files being reported as success; nonzero helper exits now reject both pending and subsequent stops, with unit regression coverage and a CLI failure check. No ScreenCast portal was installed or activated. **Not covered:** the full HUD/editor checklist, Wayland, Windows, macOS, mixed-DPI monitor capture (explicitly rejected; window capture remains available), or changing window size during a take (stops with an error). Computer-use MCP was unavailable; the native CLI slice used real X11 events rather than injected DOM events. | -| 2026-10-16 | dev `main` @ `b149c55` + uncommitted #1278 camera preview; Electron 41.2.1; capture helper compiled from this working tree using `swiftc` + macOS 26.5 SDK, SHA256 `06f142f075e4dd776b653f174379b2795a60d750557655f9bca6e2d2596c8f99`; remaining native payload copied as a complete directory from installed OpenScreen | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — camera preview slice passed** | Driven through `mcp__cua_repl` against the Electron app, with the legacy source picker and content protection disabled for UI visibility. Verified live camera frames in a circle with no card/header, HUD show/hide and circle hide without disabling camera capture, and show/hide while recording. Exercised OS dragging; final position was not independently measured. Stopping closed the preview and opened the editor; screen MP4 and separate webcam WebM were finalized. A second take started with the preview hidden, then showed it during capture using the rebuilt helper; a decoded late screen frame contains no floating circle (sample `/tmp/openscreen-camera-preview-capture.png`). This sample is not an exhaustive frame-by-frame exclusion test. Test takes: `recording-1792129915224` (borrowed helper), `recording-1792130142748` (rebuilt helper). **Skipped:** Apple's picker, full-screen Spaces, restart/cancel, disconnect/reconnect on physical hardware, export, all other checklist sections, Windows and Linux (implementation is macOS-first). SwiftPM cannot start with this machine's Command Line Tools (`BuildServerProtocol` dyld error); direct `swiftc` compilation passed, but XCTest is unavailable without full Xcode. | -| 2026-10-16 | dev `main` @ `b149c55` + #1278 audio-toggle and persistent preview-control fixes; Electron 41.2.1 | macOS 26.7, Apple Silicon | **Partial — HUD follow-up slice passed** | `mcp__cua_repl`: toggled system audio on/off and microphone off/on with camera and preview enabled; restored original audio preferences. Camera off leaves Show camera preview visible and disabled; camera on re-enables it. Regression test reproduces and fixes audio snapshots reopening the resolved default camera, asserting identical stream and no stopped tracks across both audio changes; explicit camera changes and resets still apply. **Skipped:** recording, export, all remaining checklist sections, and Windows/Linux for this follow-up. | -| 2026-10-16 | dev `main` @ `b149c55` + #1278 preview framing, placement and resizing | macOS 26.7, Apple Silicon | **Partial — preview UI slice passed** | `mcp__cua_repl`: inspected live fitted camera frames (portrait source, with black space at the sides instead of a zoomed crop), increased to 320px, decreased through 224px to 160px, verified limit buttons disable, and restored 224px. Window sizes changed visibly while remaining circular. Default placement above the HUD and resize anchoring are covered by geometry tests; their screen coordinates were not independently measured in the desktop pass. **Skipped:** recording/export, Apple’s picker, display disconnection, Windows/Linux, and all other checklist sections. | -| 2026-10-16 | dev `main` @ `b149c55` + #1278 normal camera framing follow-up | macOS 26.7, MacBook Pro Camera | **Partial — preview framing slice passed** | `mcp__cua_repl`: verified the circle fills with centred, unstretched live video and no side bars after preferring the preset’s landscape aspect ratio and removing letterboxing. The earlier webcam sidecar was 1080×1920 despite landscape width/height preferences. Temporarily hid and restored the recording bar for inspection, then left the camera and preview enabled. **Skipped:** recording/export, other cameras, Windows/Linux and all other checklist sections. | +| 2026-10-16 | dev `main` @ `b149c55` + #1278 macOS camera preview; Electron 41.2.1; capture helper built from the working tree with `swiftc` + macOS 26.5 SDK (SHA256 `06f142f075e4dd776b653f174379b2795a60d750557655f9bca6e2d2596c8f99`) | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — preview slice passed** | `mcp__cua_repl`: live circular self-view, show/hide during recording, three sizes and limit buttons, disabled preview button when camera is off, and recording finalization into screen MP4 + webcam WebM. Exercised OS dragging without independently measuring the final position. A take started with the preview hidden and showed it during capture; a decoded late screen frame contained no floating circle. Default placement and resize anchoring are covered by geometry tests. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras, restart/cancel, export, exhaustive frame-by-frame exclusion, Windows/Linux and the remaining checklist. SwiftPM/XCTest unavailable locally; direct helper compilation passed. | From be5cf0d21aed544cf84e9d8fd6387cbdc9f91b57 Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Fri, 16 Oct 2026 15:04:42 +0530 Subject: [PATCH 3/8] fix: clear closed camera preview windows --- electron/cameraPreview.test.ts | 44 +++++++++++++++++++++++++++++++++- electron/cameraPreview.ts | 6 +++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts index b393f7fce5..c20e3f8a9b 100644 --- a/electron/cameraPreview.test.ts +++ b/electron/cameraPreview.test.ts @@ -9,7 +9,9 @@ function setup() { webContents: { send: vi.fn() }, showInactive: vi.fn(), hide: vi.fn(), - destroy: vi.fn(), + destroy: vi.fn(() => { + overlay.emit("closed"); + }), }); const create = vi.fn(() => overlay as unknown as BrowserWindow); const resize = vi.fn(); @@ -81,6 +83,46 @@ describe("floating camera preview lifecycle", () => { expect(overlay.destroy).not.toHaveBeenCalled(); }); + it("recreates an unexpectedly closed preview and waits for the replacement to be ready", () => { + const { preview, sender, overlay, create } = setup(); + const replacement = setup().overlay; + create + .mockReturnValueOnce(overlay as unknown as BrowserWindow) + .mockReturnValueOnce(replacement as unknown as BrowserWindow); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + overlay.emit("closed"); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledTimes(2); + expect(replacement.showInactive).not.toHaveBeenCalled(); + replacement.emit("ready-to-show"); + expect(replacement.showInactive).toHaveBeenCalledOnce(); + }); + + it("ignores a stale closed event from a previously disposed preview", () => { + const { preview, sender, overlay, create } = setup(); + // Delay closure to exercise a late event after the replacement already exists. + overlay.destroy.mockImplementationOnce(() => undefined); + const replacement = setup().overlay; + create + .mockReturnValueOnce(overlay as unknown as BrowserWindow) + .mockReturnValueOnce(replacement as unknown as BrowserWindow); + preview.configure(sender, true, true); + preview.configure(sender, false, false); + preview.configure(sender, true, true); + replacement.emit("ready-to-show"); + + overlay.emit("closed"); + preview.frame(sender, jpeg); + expect(replacement.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledTimes(2); + expect(replacement.showInactive).toHaveBeenCalledTimes(2); + }); + it("destroys the auxiliary window when the HUD is destroyed", () => { const { preview, sender, owner, overlay } = setup(); preview.configure(sender, true, true); diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts index dafa598132..abbc07e396 100644 --- a/electron/cameraPreview.ts +++ b/electron/cameraPreview.ts @@ -30,6 +30,12 @@ export class CameraPreviewController { if (!this.window) { this.window = this.createWindow(); const previewWindow = this.window; + previewWindow.once("closed", () => { + if (this.window !== previewWindow) return; + this.window = null; + this.ready = false; + this.visible = false; + }); this.window.once("ready-to-show", () => { if (this.window !== previewWindow) return; this.ready = true; From fd92c3dc65f801f39403a75f27166f566451bd7b Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Mon, 19 Oct 2026 20:55:34 +0530 Subject: [PATCH 4/8] fix: preserve camera framing in presenter preview --- electron/cameraPreview.test.ts | 18 +++++- electron/cameraPreview.ts | 41 ++++++++++--- electron/electron-env.d.ts | 4 +- electron/ipc/handlers.ts | 4 +- electron/preload.ts | 8 +-- electron/windows.ts | 13 +++- .../launch/FloatingCameraPreview.module.css | 11 ++-- src/components/launch/LaunchWindow.test.tsx | 22 ++++--- src/components/launch/LaunchWindow.tsx | 4 +- src/hooks/useFloatingCameraPreview.test.tsx | 1 + src/hooks/useFloatingCameraPreview.ts | 7 ++- src/lib/cameraPreview.test.ts | 11 +++- src/lib/cameraPreview.ts | 59 +++++++++++++------ .../architecture/recording.md | 5 +- .../testing/manual-e2e-checklist.md | 1 + 15 files changed, 151 insertions(+), 58 deletions(-) diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts index c20e3f8a9b..d4ba4ffe4c 100644 --- a/electron/cameraPreview.test.ts +++ b/electron/cameraPreview.test.ts @@ -70,7 +70,7 @@ describe("floating camera preview lifecycle", () => { expect(create).toHaveBeenCalledOnce(); }); - it("allows only the preview to resize to a supported diameter without replacing its window", () => { + it("allows only the preview to resize to a supported width without replacing its window", () => { const { preview, sender, overlaySender, resize, create, overlay } = setup(); preview.configure(sender, true, true); preview.resize(sender, 320); @@ -78,11 +78,25 @@ describe("floating camera preview lifecycle", () => { preview.resize(overlaySender, 10000); expect(resize).not.toHaveBeenCalled(); preview.resize(overlaySender, 320); - expect(resize).toHaveBeenCalledWith(overlay, 320); + expect(resize).toHaveBeenCalledWith(overlay, 320, 16 / 9); expect(create).toHaveBeenCalledOnce(); expect(overlay.destroy).not.toHaveBeenCalled(); }); + it("sizes the same window to the delivered frame and preserves that ratio on resize", () => { + const { preview, sender, overlaySender, resize, overlay } = setup(); + preview.configure(sender, true, true); + preview.frame(sender, jpeg, 9 / 16); + expect(resize).toHaveBeenLastCalledWith(overlay, 224, 9 / 16); + preview.resize(overlaySender, 320); + expect(resize).toHaveBeenLastCalledWith(overlay, 320, 9 / 16); + const calls = resize.mock.calls.length; + for (const ratio of [0, -1, Number.NaN, Number.POSITIVE_INFINITY, 100, "16:9"]) { + preview.frame(sender, jpeg, ratio); + } + expect(resize).toHaveBeenCalledTimes(calls); + }); + it("recreates an unexpectedly closed preview and waits for the replacement to be ready", () => { const { preview, sender, overlay, create } = setup(); const replacement = setup().overlay; diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts index abbc07e396..8567e48e20 100644 --- a/electron/cameraPreview.ts +++ b/electron/cameraPreview.ts @@ -1,17 +1,27 @@ import type { BrowserWindow, WebContents } from "electron"; -import { CAMERA_PREVIEW_SIZES } from "../src/lib/cameraPreview"; +import { + CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO, + CAMERA_PREVIEW_DEFAULT_SIZE, + CAMERA_PREVIEW_SIZES, +} from "../src/lib/cameraPreview"; /** One stable window per HUD: hiding it must not change the native capture exclusion id. */ export class CameraPreviewController { private window: BrowserWindow | null = null; private owner: WebContents | null = null; private visible = false; + private width: number = CAMERA_PREVIEW_DEFAULT_SIZE; + private aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO; private ready = false; private onOwnerDestroyed: (() => void) | null = null; constructor( private readonly createWindow: () => BrowserWindow, - private readonly resizeWindow: (window: BrowserWindow, diameter: number) => void, + private readonly resizeWindow: ( + window: BrowserWindow, + width: number, + aspectRatio: number, + ) => void, ) {} configure(owner: WebContents, enabled: boolean, visible: boolean): void { @@ -39,6 +49,7 @@ export class CameraPreviewController { this.window.once("ready-to-show", () => { if (this.window !== previewWindow) return; this.ready = true; + this.resizeWindow(previewWindow, this.width, this.aspectRatio); if (this.visible) this.window?.showInactive(); }); } @@ -54,17 +65,18 @@ export class CameraPreviewController { this.owner?.send("camera-preview:hidden"); } - resize(sender: WebContents, diameter: unknown): void { + resize(sender: WebContents, width: unknown): void { if ( sender !== this.window?.webContents || - typeof diameter !== "number" || - !CAMERA_PREVIEW_SIZES.some((size) => size === diameter) + typeof width !== "number" || + !CAMERA_PREVIEW_SIZES.some((size) => size === width) ) return; - this.resizeWindow(this.window, diameter); + this.width = width; + this.resizeWindow(this.window, this.width, this.aspectRatio); } - frame(sender: WebContents, jpeg: unknown): void { + frame(sender: WebContents, jpeg: unknown, aspectRatio?: unknown): void { if (sender !== this.owner || !this.visible || !this.window) return; // A 320px JPEG is usually under 20KB. Never relay arbitrary URLs or unbounded IPC data. if ( @@ -73,6 +85,19 @@ export class CameraPreviewController { !jpeg.startsWith("data:image/jpeg;base64,") ) return; + if (aspectRatio !== undefined) { + if ( + typeof aspectRatio !== "number" || + !Number.isFinite(aspectRatio) || + aspectRatio < 1 / 8 || + aspectRatio > 8 + ) + return; + if (this.aspectRatio !== aspectRatio) { + this.aspectRatio = aspectRatio; + this.resizeWindow(this.window, this.width, aspectRatio); + } + } this.window.webContents.send("camera-preview:frame", jpeg); } @@ -86,5 +111,7 @@ export class CameraPreviewController { this.window = null; this.owner = null; this.visible = false; + this.width = CAMERA_PREVIEW_DEFAULT_SIZE; + this.aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO; } } diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 64a10b7102..7604f5cb44 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -59,8 +59,8 @@ interface Window { /** Name of the last source picked in Apple's picker, for display only: it is not selected. */ getLastPickedSource?: () => Promise; configureCameraPreview?: (enabled: boolean, visible: boolean) => Promise; - sendCameraPreviewFrame?: (jpeg: string) => Promise; - resizeCameraPreview?: (diameter: number) => Promise; + sendCameraPreviewFrame?: (jpeg: string, aspectRatio: number) => Promise; + resizeCameraPreview?: (width: number) => Promise; hideCameraPreview?: () => Promise; onCameraPreviewFrame?: (callback: (jpeg: string) => void) => () => void; onCameraPreviewHidden?: (callback: () => void) => () => void; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 2a17586b9b..1cf6ae0a06 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -2544,8 +2544,8 @@ export function registerIpcHandlers( return; cameraPreview.configure(event.sender, enabled === true, visible === true); }); - ipcMain.handle("camera-preview:frame", (event, jpeg: unknown) => - cameraPreview.frame(event.sender, jpeg), + ipcMain.handle("camera-preview:frame", (event, jpeg: unknown, aspectRatio: unknown) => + cameraPreview.frame(event.sender, jpeg, aspectRatio), ); ipcMain.handle("camera-preview:hide", (event) => cameraPreview.hide(event.sender)); ipcMain.handle("camera-preview:resize", (event, diameter: unknown) => diff --git a/electron/preload.ts b/electron/preload.ts index f6a4ef7e05..dc6eb1ea51 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -172,10 +172,10 @@ contextBridge.exposeInMainWorld("electronAPI", { }, configureCameraPreview: (enabled: boolean, visible: boolean): Promise => ipcRenderer.invoke("camera-preview:configure", enabled, visible), - sendCameraPreviewFrame: (jpeg: string): Promise => - ipcRenderer.invoke("camera-preview:frame", jpeg), - resizeCameraPreview: (diameter: number): Promise => - ipcRenderer.invoke("camera-preview:resize", diameter), + sendCameraPreviewFrame: (jpeg: string, aspectRatio: number): Promise => + ipcRenderer.invoke("camera-preview:frame", jpeg, aspectRatio), + resizeCameraPreview: (width: number): Promise => + ipcRenderer.invoke("camera-preview:resize", width), hideCameraPreview: (): Promise => ipcRenderer.invoke("camera-preview:hide"), onCameraPreviewFrame: (callback: (jpeg: string) => void) => { const listener = (_event: Electron.IpcRendererEvent, jpeg: string) => callback(jpeg); diff --git a/electron/windows.ts b/electron/windows.ts index 92bd9c59d2..ea773af28b 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -937,10 +937,19 @@ export function createCameraPreviewWindow(): BrowserWindow { return win; } -export function resizeCameraPreviewWindow(win: BrowserWindow, diameter: number): void { +export function resizeCameraPreviewWindow( + win: BrowserWindow, + width: number, + aspectRatio: number, +): void { const bounds = win.getBounds(); win.setBounds( - resizedCameraPreviewBounds(bounds, screen.getDisplayMatching(bounds).workArea, diameter), + resizedCameraPreviewBounds( + bounds, + screen.getDisplayMatching(bounds).workArea, + width, + aspectRatio, + ), ); } diff --git a/src/components/launch/FloatingCameraPreview.module.css b/src/components/launch/FloatingCameraPreview.module.css index 9239c7e430..ee65ec55e7 100644 --- a/src/components/launch/FloatingCameraPreview.module.css +++ b/src/components/launch/FloatingCameraPreview.module.css @@ -7,7 +7,7 @@ height: calc(100% - 16px); margin: 8px; overflow: hidden; - border-radius: 50%; + border-radius: 12px; background: #09090b; color: #a1a1aa; font-size: 12px; @@ -19,16 +19,15 @@ inset: 0; width: 100%; height: 100%; - object-fit: cover; + object-fit: contain; object-position: center; - transform: scaleX(-1); } .hide { -webkit-app-region: no-drag; position: absolute; - top: 9%; - right: 13%; + top: 8px; + right: 8px; display: grid; place-items: center; width: 28px; @@ -53,7 +52,7 @@ .sizeControls { -webkit-app-region: no-drag; position: absolute; - bottom: 9%; + bottom: 8px; left: 50%; display: flex; gap: 4px; diff --git a/src/components/launch/LaunchWindow.test.tsx b/src/components/launch/LaunchWindow.test.tsx index 20d2cc3f51..5c065f74ee 100644 --- a/src/components/launch/LaunchWindow.test.tsx +++ b/src/components/launch/LaunchWindow.test.tsx @@ -2567,22 +2567,26 @@ describe("macOS floating camera preview control", () => { expect(recorderState.value.setWebcamEnabled).not.toHaveBeenCalled(); }); - it("keeps the preview button visible but disabled when the camera is off", () => { + it("hides the preview control when the camera is off", () => { const view = renderLaunchWindow(); - const button = screen.getByTestId("launch-camera-preview-button"); - expect(button).toHaveAttribute("aria-disabled", "true"); - expect(button).toHaveAttribute("aria-pressed", "false"); - fireEvent.click(button); - expect(button).toHaveAttribute("aria-pressed", "false"); - expect(recorderState.value.setWebcamEnabled).not.toHaveBeenCalled(); + expect(screen.queryByTestId("launch-camera-preview-button")).not.toBeInTheDocument(); recorderState.value.webcamEnabled = true; view.rerender( , ); - expect(button).not.toHaveAttribute("aria-disabled"); - expect(button).toHaveAttribute("aria-pressed", "true"); + expect(screen.getByTestId("launch-camera-preview-button")).toHaveAttribute( + "aria-pressed", + "true", + ); + recorderState.value.webcamEnabled = false; + view.rerender( + + + , + ); + expect(screen.queryByTestId("launch-camera-preview-button")).not.toBeInTheDocument(); }); it("does not offer the preview on Windows", () => { diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index ba38189876..dcd36a7f48 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -1316,10 +1316,10 @@ export function LaunchWindow() { tooltip={t("tooltips.camera")} onClick={toggleWebcam} /> - {cameraPreview.supported && ( + {cameraPreview.supported && webcamEnabled && ( diff --git a/src/hooks/useFloatingCameraPreview.test.tsx b/src/hooks/useFloatingCameraPreview.test.tsx index 39c982a0b9..0c0279c0db 100644 --- a/src/hooks/useFloatingCameraPreview.test.tsx +++ b/src/hooks/useFloatingCameraPreview.test.tsx @@ -51,6 +51,7 @@ describe("floating camera publisher", () => { const { unmount } = renderHook(() => useFloatingCameraPreview(stream, true)); expect(configure).toHaveBeenLastCalledWith(true, true); expect(send).toHaveBeenCalledOnce(); + expect(send).toHaveBeenCalledWith("data:image/jpeg;base64,frame", 1280 / 720); await act(() => vi.advanceTimersByTimeAsync(300)); expect(send).toHaveBeenCalledTimes(4); unmount(); diff --git a/src/hooks/useFloatingCameraPreview.ts b/src/hooks/useFloatingCameraPreview.ts index d6bb2d0124..3d1f1108ba 100644 --- a/src/hooks/useFloatingCameraPreview.ts +++ b/src/hooks/useFloatingCameraPreview.ts @@ -30,11 +30,14 @@ export function useFloatingCameraPreview(stream: MediaStream | null, enabled: bo let timer: ReturnType; const draw = async () => { try { - if (context && video.readyState >= 2 && video.videoWidth > 0) { + if (context && video.readyState >= 2 && video.videoWidth > 0 && video.videoHeight > 0) { canvas.height = Math.max(1, Math.round((320 * video.videoHeight) / video.videoWidth)); context.drawImage(video, 0, 0, canvas.width, canvas.height); // Await delivery: a slow renderer must not build a queue of stale frames. - await window.electronAPI?.sendCameraPreviewFrame?.(canvas.toDataURL("image/jpeg", 0.7)); + await window.electronAPI?.sendCameraPreviewFrame?.( + canvas.toDataURL("image/jpeg", 0.7), + video.videoWidth / video.videoHeight, + ); } } catch (error) { console.warn("Could not update camera preview:", error); diff --git a/src/lib/cameraPreview.test.ts b/src/lib/cameraPreview.test.ts index c98f8548fb..0a2ebd6b3c 100644 --- a/src/lib/cameraPreview.test.ts +++ b/src/lib/cameraPreview.test.ts @@ -15,11 +15,20 @@ describe("camera preview placement", () => { const before = cameraPreviewBounds(workArea, bar); const after = resizedCameraPreviewBounds(before, workArea, 320); expect(after.width).toBe(336); - expect(after.height).toBe(336); + expect(after.height).toBe(196); expect(after.y + after.height).toBe(before.y + before.height); expect(after.x + after.width / 2).toBe(before.x + before.width / 2); }); + it("preserves the full portrait, landscape and square frame at every preview size", () => { + for (const ratio of [9 / 16, 16 / 9, 1]) { + for (const width of [160, 224, 320]) { + const bounds = resizedCameraPreviewBounds(bar, workArea, width, ratio); + expect((bounds.width - 16) / (bounds.height - 16)).toBeCloseTo(ratio, 2); + } + } + }); + it("keeps a preview on-screen on negative-origin displays and near the screen edge", () => { const display = { x: -1280, y: -100, width: 1280, height: 720 }; const preview = resizedCameraPreviewBounds( diff --git a/src/lib/cameraPreview.ts b/src/lib/cameraPreview.ts index 613d0e4109..8525b32477 100644 --- a/src/lib/cameraPreview.ts +++ b/src/lib/cameraPreview.ts @@ -1,37 +1,62 @@ export const CAMERA_PREVIEW_SIZES = [160, 224, 320] as const; export const CAMERA_PREVIEW_DEFAULT_SIZE = 224; +export const CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO = 16 / 9; const WINDOW_PADDING = 16; type Rect = { x: number; y: number; width: number; height: number }; -function clampBounds(x: number, y: number, diameter: number, workArea: Rect): Rect { - const size = Math.min(diameter + WINDOW_PADDING, workArea.width, workArea.height); +function previewSize(width: number, aspectRatio: number, workArea: Rect) { + const contentWidth = Math.min( + width, + workArea.width - WINDOW_PADDING, + (workArea.height - WINDOW_PADDING) * aspectRatio, + ); + return { + width: Math.round(contentWidth) + WINDOW_PADDING, + height: Math.round(contentWidth / aspectRatio) + WINDOW_PADDING, + }; +} + +function clampBounds( + x: number, + y: number, + size: { width: number; height: number }, + workArea: Rect, +): Rect { return { - x: Math.round(Math.max(workArea.x, Math.min(x, workArea.x + workArea.width - size))), - y: Math.round(Math.max(workArea.y, Math.min(y, workArea.y + workArea.height - size))), - width: size, - height: size, + x: Math.round(Math.max(workArea.x, Math.min(x, workArea.x + workArea.width - size.width))), + y: Math.round(Math.max(workArea.y, Math.min(y, workArea.y + workArea.height - size.height))), + ...size, }; } -/** Place the circle just above the bar's right end, including the transparent window padding. */ +/** Place the complete camera frame just above the bar's right end. */ export function cameraPreviewBounds(workArea: Rect, bar?: Rect): Rect { - const size = CAMERA_PREVIEW_DEFAULT_SIZE + WINDOW_PADDING; - return clampBounds( - bar ? bar.x + bar.width - size : workArea.x + workArea.width / 2 + 160, - (bar?.y ?? workArea.y + workArea.height - 84) - size - 8, + const size = previewSize( CAMERA_PREVIEW_DEFAULT_SIZE, + CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO, + workArea, + ); + return clampBounds( + bar ? bar.x + bar.width - size.width : workArea.x + workArea.width / 2 + 160, + (bar?.y ?? workArea.y + workArea.height - 84) - size.height - 8, + size, workArea, ); } -/** Grow upward from the bottom centre, retaining its distance from the recording bar. */ -export function resizedCameraPreviewBounds(bounds: Rect, workArea: Rect, diameter: number): Rect { - const size = diameter + WINDOW_PADDING; +/** Resize without cropping, retaining the bottom centre and distance from the bar. */ +export function resizedCameraPreviewBounds( + bounds: Rect, + workArea: Rect, + width: number, + aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO, +): Rect { + const size = previewSize(width, aspectRatio, workArea); return clampBounds( - bounds.x + (bounds.width - size) / 2, - bounds.y + bounds.height - size, - diameter, + bounds.x + (bounds.width - size.width) / 2, + bounds.y + bounds.height - size.height, + size, workArea, ); } diff --git a/technical-documentation/architecture/recording.md b/technical-documentation/architecture/recording.md index 41d3d6470f..1396dd6ea7 100644 --- a/technical-documentation/architecture/recording.md +++ b/technical-documentation/architecture/recording.md @@ -24,8 +24,9 @@ Electron applies `setContentProtection(true)` to the HUD window (`applyContentPr ## Presenter camera preview (macOS) -The HUD camera has a draggable circular self-view with show/hide and three size choices. -Its preview button stays visible but disabled when the camera is off. The preview shares +The HUD camera has a draggable rounded rectangular self-view with show/hide and three width choices. +It shows the complete camera frame at the shared recording stream’s aspect ratio, including when resized. +Its preview button is hidden when the camera is off. The preview shares `useScreenRecorder`'s stream and stops with its owning HUD; it does not acquire another camera. `useFloatingCameraPreview` publishes small JPEG frames to the auxiliary window, with bounded payloads and one IPC request in flight. Audio preference broadcasts in the macOS HUD preserve diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index fc651b7cd4..0f045b2c09 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -1077,3 +1077,4 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-27 | Dev `feat/agent-execution-steps-1169` on `main` @ `5bb1b3fb`, Electron 41.2.1; Vite 5191, private `/tmp/openscreen-execution-ui-*` profile and local Anthropic-shaped fixture provider | macOS 15.8, Apple Silicon | **Partial — execution steps and Markdown** | Real Electron driven through OS computer-use. Sent a multiline Markdown request with a heading, list and indented code fence; observed two separate successful `getCurrentDocument` rows and one failed `getTranscript` row, with two provider-exposed reasoning blocks in order. Expanded reasoning and failure details. Completed answer rendered heading, list, code and GFM table. Corrected code-block contrast to use theme colors. New conversations remained empty; switching during the second turn cancelled it and saved a stopped assistant with five steps in its original session. Fully quit/relaunched, reopened Studio and verified both completed/stopped traces, failure labels and Markdown restored without usable old rewind controls. Final observer implementation was exercised in a third real SDK/HTTP turn with no duplicate rows. Automated tests additionally cover schema-invalid/unknown calls, out-of-order completions, failed-turn persistence, source copying and unsafe Markdown. Screenshots retained in the task evidence. Skipped: paid/remote providers, capture/audio/webcam, native preview/export, Windows/Linux runtime and the rest of this checklist; chat-only slice, not release-promotion QA. | | 2026-10-17 | Local source build and packaged 2.1.1 AppImage, `integration/linux-x11-capture`, based on `main` 2.1.1; rebuilt Linux capture helper, complete unchanged compositor/runtime directory from official 2.1.1 AppImage | Ubuntu 24.04.5, Xfce/X11, ScreenCast portal interface absent | **Partial — native X11 CLI capture/export passed** | Recorded an isolated GTK window through `record --window` for 12 s, with real OS mouse movement/press/release events through XTest. Saved 301 cursor samples, 7 click events and 4 cursor sprites; positions outside the moved window remained hidden. `export --auto-zoom` generated one region and produced a fully decodable H.264/AAC MP4, 1624×1080, 30 fps, 12.033 s. 158 of 360 decoded frames differed from the same export without zoom; inspected the rendered hold. Packaged AppImage also recorded the GTK window on an isolated X11 desktop with XTest: 286 cursor samples, 3 clicks and 1 sprite; CLI auto-zoom export decoded as 1624×1080 H.264 at 30 fps. Final AppImage also passed explicit `--display 0` monitor capture (309 samples, 3 clicks) and window capture (325 samples, 3 clicks). The source-index `--display` switch is removed from Chromium before GTK initializes, avoiding an X-server argument collision. A window-unmap test exposed finalized partial files being reported as success; nonzero helper exits now reject both pending and subsequent stops, with unit regression coverage and a CLI failure check. No ScreenCast portal was installed or activated. **Not covered:** the full HUD/editor checklist, Wayland, Windows, macOS, mixed-DPI monitor capture (explicitly rejected; window capture remains available), or changing window size during a take (stops with an error). Computer-use MCP was unavailable; the native CLI slice used real X11 events rather than injected DOM events. | | 2026-10-16 | dev `main` @ `b149c55` + #1278 macOS camera preview; Electron 41.2.1; capture helper built from the working tree with `swiftc` + macOS 26.5 SDK (SHA256 `06f142f075e4dd776b653f174379b2795a60d750557655f9bca6e2d2596c8f99`) | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — preview slice passed** | `mcp__cua_repl`: live circular self-view, show/hide during recording, three sizes and limit buttons, disabled preview button when camera is off, and recording finalization into screen MP4 + webcam WebM. Exercised OS dragging without independently measuring the final position. A take started with the preview hidden and showed it during capture; a decoded late screen frame contained no floating circle. Default placement and resize anchoring are covered by geometry tests. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras, restart/cancel, export, exhaustive frame-by-frame exclusion, Windows/Linux and the remaining checklist. SwiftPM/XCTest unavailable locally; direct helper compilation passed. | +| 2026-10-19 | dev `codex/macos-floating-camera-preview`, rebased on `main` @ `3620cf8`, plus #1288 review changes; Electron 41.2.1; native helper rebuilt directly with `swiftc` + macOS 26.5 SDK | macOS 26.7, Apple M5 Pro, MacBook Pro Camera | **Partial — preview review slice passed** | Computer-use verified the complete rounded rectangular camera frame, aspect-preserving resize, camera-off hiding the HUD preview control, and show/hide during a finalized camera take. In the same take, after 5 s settling per condition, 30 s process-tree CPU deltas were **17.5% hidden / 22.3% shown** (100%=one core; includes Electron children and capture helper). Native `recording-stopped` reported **0 droppedVideoFrames over the entire take**, covering both intervals; this counts screen-writer backpressure/append failures, not upstream capture or webcam drops. Screen 1 at 3024×1964, saved webcam verified 1920×1080 (30 fps requested); mic/system audio off, system cursor. One sequential pair, no statistical claim. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras/platforms, restart/cancel, export, exhaustive exclusion and remaining checklist. | From 14a61255f56d7736b74d9eb70644627bf382ab42 Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Mon, 19 Oct 2026 21:13:18 +0530 Subject: [PATCH 5/8] fix: retain camera preview window across camera toggles --- electron/cameraPreview.test.ts | 36 +++++++++++++++---- electron/cameraPreview.ts | 10 ++---- electron/windows.ts | 1 - .../testing/manual-e2e-checklist.md | 1 + 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts index d4ba4ffe4c..df490f1e50 100644 --- a/electron/cameraPreview.test.ts +++ b/electron/cameraPreview.test.ts @@ -125,7 +125,7 @@ describe("floating camera preview lifecycle", () => { .mockReturnValueOnce(overlay as unknown as BrowserWindow) .mockReturnValueOnce(replacement as unknown as BrowserWindow); preview.configure(sender, true, true); - preview.configure(sender, false, false); + preview.dispose(); preview.configure(sender, true, true); replacement.emit("ready-to-show"); @@ -146,14 +146,36 @@ describe("floating camera preview lifecycle", () => { expect(overlay.webContents.send).not.toHaveBeenCalled(); }); - it("releases listeners on camera-off rather than accumulating them between takes", () => { - const { preview, sender, owner, overlay } = setup(); + it("creates a hidden exclusion window even when the camera starts off", () => { + const { preview, sender, owner, overlay, create } = setup(); + preview.configure(sender, false, true); + overlay.emit("ready-to-show"); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.hide).toHaveBeenCalledOnce(); + expect(overlay.showInactive).not.toHaveBeenCalled(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + owner.emit("destroyed"); + expect(overlay.destroy).toHaveBeenCalledOnce(); + }); + + it("retains the exclusion window and one owner listener across camera toggles", () => { + const { preview, sender, owner, overlay, create } = setup(); for (let i = 0; i < 15; i++) { - preview.configure(sender, true, false); - overlay.emit("ready-to-show"); - preview.configure(sender, false, false); + preview.configure(sender, true, true); + if (i === 0) overlay.emit("ready-to-show"); + preview.configure(sender, false, true); } + expect(create).toHaveBeenCalledOnce(); + expect(overlay.hide).toHaveBeenCalledTimes(15); + expect(overlay.showInactive).toHaveBeenCalledTimes(15); + expect(overlay.destroy).not.toHaveBeenCalled(); + expect(owner.listenerCount("destroyed")).toBe(1); + owner.emit("destroyed"); expect(owner.listenerCount("destroyed")).toBe(0); - expect(overlay.destroy).toHaveBeenCalledTimes(15); + expect(overlay.destroy).toHaveBeenCalledOnce(); }); }); diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts index 8567e48e20..c3be9b7bd4 100644 --- a/electron/cameraPreview.ts +++ b/electron/cameraPreview.ts @@ -26,10 +26,6 @@ export class CameraPreviewController { configure(owner: WebContents, enabled: boolean, visible: boolean): void { if (this.owner && this.owner !== owner) return; - if (!enabled) { - this.dispose(); - return; - } if (!this.owner) { this.owner = owner; this.onOwnerDestroyed = () => { @@ -53,9 +49,9 @@ export class CameraPreviewController { if (this.visible) this.window?.showInactive(); }); } - this.visible = visible; - if (visible && this.ready) this.window.showInactive(); - else if (!visible) this.window.hide(); + this.visible = enabled && visible; + if (this.visible && this.ready) this.window.showInactive(); + else if (!this.visible) this.window.hide(); } hide(sender: WebContents): void { diff --git a/electron/windows.ts b/electron/windows.ts index ea773af28b..a1df99bfe9 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -908,7 +908,6 @@ export function createCameraPreviewWindow(): BrowserWindow { resizable: false, maximizable: false, minimizable: false, - closable: false, alwaysOnTop: true, skipTaskbar: true, show: false, diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 0f045b2c09..0c15ad48f6 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -1078,3 +1078,4 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-17 | Local source build and packaged 2.1.1 AppImage, `integration/linux-x11-capture`, based on `main` 2.1.1; rebuilt Linux capture helper, complete unchanged compositor/runtime directory from official 2.1.1 AppImage | Ubuntu 24.04.5, Xfce/X11, ScreenCast portal interface absent | **Partial — native X11 CLI capture/export passed** | Recorded an isolated GTK window through `record --window` for 12 s, with real OS mouse movement/press/release events through XTest. Saved 301 cursor samples, 7 click events and 4 cursor sprites; positions outside the moved window remained hidden. `export --auto-zoom` generated one region and produced a fully decodable H.264/AAC MP4, 1624×1080, 30 fps, 12.033 s. 158 of 360 decoded frames differed from the same export without zoom; inspected the rendered hold. Packaged AppImage also recorded the GTK window on an isolated X11 desktop with XTest: 286 cursor samples, 3 clicks and 1 sprite; CLI auto-zoom export decoded as 1624×1080 H.264 at 30 fps. Final AppImage also passed explicit `--display 0` monitor capture (309 samples, 3 clicks) and window capture (325 samples, 3 clicks). The source-index `--display` switch is removed from Chromium before GTK initializes, avoiding an X-server argument collision. A window-unmap test exposed finalized partial files being reported as success; nonzero helper exits now reject both pending and subsequent stops, with unit regression coverage and a CLI failure check. No ScreenCast portal was installed or activated. **Not covered:** the full HUD/editor checklist, Wayland, Windows, macOS, mixed-DPI monitor capture (explicitly rejected; window capture remains available), or changing window size during a take (stops with an error). Computer-use MCP was unavailable; the native CLI slice used real X11 events rather than injected DOM events. | | 2026-10-16 | dev `main` @ `b149c55` + #1278 macOS camera preview; Electron 41.2.1; capture helper built from the working tree with `swiftc` + macOS 26.5 SDK (SHA256 `06f142f075e4dd776b653f174379b2795a60d750557655f9bca6e2d2596c8f99`) | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — preview slice passed** | `mcp__cua_repl`: live circular self-view, show/hide during recording, three sizes and limit buttons, disabled preview button when camera is off, and recording finalization into screen MP4 + webcam WebM. Exercised OS dragging without independently measuring the final position. A take started with the preview hidden and showed it during capture; a decoded late screen frame contained no floating circle. Default placement and resize anchoring are covered by geometry tests. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras, restart/cancel, export, exhaustive frame-by-frame exclusion, Windows/Linux and the remaining checklist. SwiftPM/XCTest unavailable locally; direct helper compilation passed. | | 2026-10-19 | dev `codex/macos-floating-camera-preview`, rebased on `main` @ `3620cf8`, plus #1288 review changes; Electron 41.2.1; native helper rebuilt directly with `swiftc` + macOS 26.5 SDK | macOS 26.7, Apple M5 Pro, MacBook Pro Camera | **Partial — preview review slice passed** | Computer-use verified the complete rounded rectangular camera frame, aspect-preserving resize, camera-off hiding the HUD preview control, and show/hide during a finalized camera take. In the same take, after 5 s settling per condition, 30 s process-tree CPU deltas were **17.5% hidden / 22.3% shown** (100%=one core; includes Electron children and capture helper). Native `recording-stopped` reported **0 droppedVideoFrames over the entire take**, covering both intervals; this counts screen-writer backpressure/append failures, not upstream capture or webcam drops. Screen 1 at 3024×1964, saved webcam verified 1920×1080 (30 fps requested); mic/system audio off, system cursor. One sequential pair, no statistical claim. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras/platforms, restart/cancel, export, exhaustive exclusion and remaining checklist. | +| 2026-10-19 | dev #1288 on `8c2608d9` + preview lifetime/quit fixes; Electron 41.2.1 | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — lifecycle slice passed** | Computer-use camera off/on removed/restored the HUD preview control. Normal Cmd+Q exited the OpenScreen Electron process with the preview enabled. Unit tests verify the same preview window survives initial camera-off and repeated toggles, with one owner-destruction listener and cleanup on HUD destruction. **Skipped:** recording, export, capture-exclusion measurements, other platforms and remaining checklist; no native capture changes in this slice. | From e02d74fa666aca23df608a3e3365ed3895b9af4d Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Wed, 21 Oct 2026 11:35:59 +0530 Subject: [PATCH 6/8] fix: reserve camera preview before capture exclusions --- electron/cameraPreview.test.ts | 28 +++++++++++++++++++ electron/cameraPreview.ts | 15 +++++++--- electron/ipc/handlers.ts | 13 +++++++++ .../testing/manual-e2e-checklist.md | 1 + 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts index df490f1e50..2a6add0ff7 100644 --- a/electron/cameraPreview.test.ts +++ b/electron/cameraPreview.test.ts @@ -1,11 +1,13 @@ import { EventEmitter } from "node:events"; import type { BrowserWindow, WebContents } from "electron"; import { describe, expect, it, vi } from "vitest"; +import { collectMacCaptureExcludedWindowIds } from "../src/lib/nativeMacRecording"; import { CameraPreviewController } from "./cameraPreview"; function setup() { const owner = Object.assign(new EventEmitter(), { send: vi.fn() }); const overlay = Object.assign(new EventEmitter(), { + getMediaSourceId: () => "window:123:0", webContents: { send: vi.fn() }, showInactive: vi.fn(), hide: vi.fn(), @@ -24,6 +26,32 @@ function setup() { const jpeg = "data:image/jpeg;base64,/9j/"; describe("floating camera preview lifecycle", () => { + it("reserves the same exclusion ID before the renderer enables the preview", () => { + const { preview, sender, overlay, create } = setup(); + const reserved = preview.ensure(sender); + const excludedIds = collectMacCaptureExcludedWindowIds([reserved!.getMediaSourceId()]); + expect(excludedIds).toContain(123); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).not.toHaveBeenCalled(); + preview.configure(sender, true, true); + expect(preview.ensure(sender)).toBe(reserved); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + expect(excludedIds).toContain(123); + }); + + it("ensuring an existing preview preserves visibility and refuses a different owner", () => { + const { preview, sender, overlay, create } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + expect(preview.ensure(sender)).toBe(overlay); + expect(preview.ensure({} as WebContents)).toBeNull(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + expect(overlay.hide).not.toHaveBeenCalled(); + expect(create).toHaveBeenCalledOnce(); + }); + it("waits for the window to paint, then shows without taking focus", () => { const { preview, sender, overlay } = setup(); preview.configure(sender, true, true); diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts index c3be9b7bd4..9f1ff2ec73 100644 --- a/electron/cameraPreview.ts +++ b/electron/cameraPreview.ts @@ -24,8 +24,9 @@ export class CameraPreviewController { ) => void, ) {} - configure(owner: WebContents, enabled: boolean, visible: boolean): void { - if (this.owner && this.owner !== owner) return; + /** Reserve the capture-exclusion window before renderer configuration arrives. */ + ensure(owner: WebContents): BrowserWindow | null { + if (this.owner && this.owner !== owner) return null; if (!this.owner) { this.owner = owner; this.onOwnerDestroyed = () => { @@ -49,9 +50,15 @@ export class CameraPreviewController { if (this.visible) this.window?.showInactive(); }); } + return this.window; + } + + configure(owner: WebContents, enabled: boolean, visible: boolean): void { + const previewWindow = this.ensure(owner); + if (!previewWindow) return; this.visible = enabled && visible; - if (this.visible && this.ready) this.window.showInactive(); - else if (!this.visible) this.window.hide(); + if (this.visible && this.ready) previewWindow.showInactive(); + else if (!this.visible) previewWindow.hide(); } hide(sender: WebContents): void { diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 1cf6ae0a06..eaab74500d 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -2192,6 +2192,7 @@ export function registerIpcHandlers( * pick would record them unless the picker is told to leave them out. */ function macPickerExcludedWindowIds() { + ensureMacCameraPreviewWindow(); const appWindowSourceIds = [getMainWindow(), getNotesWindow(), getCameraPreviewWindow()] .filter((window): window is BrowserWindow => !!window && !window.isDestroyed()) .map((window) => window.getMediaSourceId()); @@ -2535,6 +2536,17 @@ export function registerIpcHandlers( createCameraPreviewWindow, resizeCameraPreviewWindow, ); + function ensureMacCameraPreviewWindow() { + const hud = getMainWindow(); + if ( + process.platform === "darwin" && + hud && + !hud.isDestroyed() && + new URL(hud.webContents.getURL()).searchParams.get("windowType") === "hud-overlay" + ) { + cameraPreview.ensure(hud.webContents); + } + } ipcMain.handle("camera-preview:configure", (event, enabled: unknown, visible: unknown) => { if ( process.platform !== "darwin" || @@ -3268,6 +3280,7 @@ export function registerIpcHandlers( normalizeCursorCaptureMode(request.cursor?.mode) ?? "editable-overlay"; // A source from Apple's picker records through the session that holds the pick, // and needs no Screen Recording grant -- so nothing here may go near one. + ensureMacCameraPreviewWindow(); const pickerSession = isMacPickerSourceId(request.source.sourceId) ? macPickerSession : null; // Normally caught by `get-selected-source` first; this is the last word (#965). pickerSession?.forgetSelectionUnlessExcluding(macPickerExcludedWindowIds()); diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 0c15ad48f6..987dda5323 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -1079,3 +1079,4 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-16 | dev `main` @ `b149c55` + #1278 macOS camera preview; Electron 41.2.1; capture helper built from the working tree with `swiftc` + macOS 26.5 SDK (SHA256 `06f142f075e4dd776b653f174379b2795a60d750557655f9bca6e2d2596c8f99`) | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — preview slice passed** | `mcp__cua_repl`: live circular self-view, show/hide during recording, three sizes and limit buttons, disabled preview button when camera is off, and recording finalization into screen MP4 + webcam WebM. Exercised OS dragging without independently measuring the final position. A take started with the preview hidden and showed it during capture; a decoded late screen frame contained no floating circle. Default placement and resize anchoring are covered by geometry tests. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras, restart/cancel, export, exhaustive frame-by-frame exclusion, Windows/Linux and the remaining checklist. SwiftPM/XCTest unavailable locally; direct helper compilation passed. | | 2026-10-19 | dev `codex/macos-floating-camera-preview`, rebased on `main` @ `3620cf8`, plus #1288 review changes; Electron 41.2.1; native helper rebuilt directly with `swiftc` + macOS 26.5 SDK | macOS 26.7, Apple M5 Pro, MacBook Pro Camera | **Partial — preview review slice passed** | Computer-use verified the complete rounded rectangular camera frame, aspect-preserving resize, camera-off hiding the HUD preview control, and show/hide during a finalized camera take. In the same take, after 5 s settling per condition, 30 s process-tree CPU deltas were **17.5% hidden / 22.3% shown** (100%=one core; includes Electron children and capture helper). Native `recording-stopped` reported **0 droppedVideoFrames over the entire take**, covering both intervals; this counts screen-writer backpressure/append failures, not upstream capture or webcam drops. Screen 1 at 3024×1964, saved webcam verified 1920×1080 (30 fps requested); mic/system audio off, system cursor. One sequential pair, no statistical claim. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras/platforms, restart/cancel, export, exhaustive exclusion and remaining checklist. | | 2026-10-19 | dev #1288 on `8c2608d9` + preview lifetime/quit fixes; Electron 41.2.1 | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — lifecycle slice passed** | Computer-use camera off/on removed/restored the HUD preview control. Normal Cmd+Q exited the OpenScreen Electron process with the preview enabled. Unit tests verify the same preview window survives initial camera-off and repeated toggles, with one owner-destruction listener and cleanup on HUD destruction. **Skipped:** recording, export, capture-exclusion measurements, other platforms and remaining checklist; no native capture changes in this slice. | +| 2026-10-21 | dev #1288 on `20975d30` + synchronous preview reservation before capture exclusions; Electron 41.2.1; existing helper rebuilt from this branch on Oct 19 | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — capture-exclusion ordering slice passed** | Computer-use started native display recording with the preview hidden, showed it during the take and finalized into the editor. Helper diagnostics resolved both requested app window IDs and reported zero screen-writer drops; a decoded late screen frame contained no floating preview. Unit regression covers reserving the exclusion window before any renderer configuration and reusing it when enabled later without altering an existing preview’s visibility. **Skipped:** Apple’s picker (ordering path checked in code), forced renderer-effect race on desktop (covered by unit regression), other platforms, export, exhaustive frame-by-frame exclusion and remaining checklist. | From f1314f51ec5d7fb3a1cc8692aa5f28ec673b4289 Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Thu, 29 Oct 2026 14:50:24 +0530 Subject: [PATCH 7/8] feat: extend camera preview to Windows and Linux window capture --- electron/cameraPreview.test.ts | 26 +++++ electron/cameraPreview.ts | 19 +++- electron/electron-env.d.ts | 2 + electron/ipc/handlers.ts | 36 ++++++- electron/native/wgc-capture/CMakeLists.txt | 8 ++ electron/native/wgc-capture/src/main.cpp | 5 + .../native/wgc-capture/src/webcam_preview.h | 54 +++++++++++ .../wgc-capture/src/webcam_preview_bitmap.h | 48 +++++++++ .../src/webcam_preview_bitmap_test.cpp | 30 ++++++ electron/preload.ts | 7 ++ electron/windows.ts | 1 + electron/windowsCameraPreview.test.ts | 97 +++++++++++++++++++ electron/windowsCameraPreview.ts | 52 ++++++++++ scripts/build-windows-wgc-helper.mjs | 7 ++ src/components/launch/LaunchWindow.test.tsx | 10 +- src/components/launch/LaunchWindow.tsx | 17 +++- src/hooks/useFloatingCameraPreview.test.tsx | 40 +++++++- src/hooks/useFloatingCameraPreview.ts | 20 ++-- .../useScreenRecorder.cameraPrefs.test.tsx | 12 ++- ...nRecorder.nativeLinuxWebcamCursor.test.tsx | 12 ++- src/hooks/useScreenRecorder.ts | 9 +- src/i18n/locales/ar/launch.json | 1 + src/i18n/locales/cs/launch.json | 1 + src/i18n/locales/de/launch.json | 1 + src/i18n/locales/en/launch.json | 1 + src/i18n/locales/es/launch.json | 1 + src/i18n/locales/fr/launch.json | 1 + src/i18n/locales/it/launch.json | 1 + src/i18n/locales/ja-JP/launch.json | 1 + src/i18n/locales/ko-KR/launch.json | 1 + src/i18n/locales/pt-BR/launch.json | 1 + src/i18n/locales/ru/launch.json | 1 + src/i18n/locales/tr/launch.json | 1 + src/i18n/locales/vi/launch.json | 1 + src/i18n/locales/zh-CN/launch.json | 1 + src/i18n/locales/zh-TW/launch.json | 1 + .../architecture/recording.md | 24 ++++- .../testing/manual-e2e-checklist.md | 1 + 38 files changed, 516 insertions(+), 36 deletions(-) create mode 100644 electron/native/wgc-capture/src/webcam_preview.h create mode 100644 electron/native/wgc-capture/src/webcam_preview_bitmap.h create mode 100644 electron/native/wgc-capture/src/webcam_preview_bitmap_test.cpp create mode 100644 electron/windowsCameraPreview.test.ts create mode 100644 electron/windowsCameraPreview.ts diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts index 2a6add0ff7..bd25330edd 100644 --- a/electron/cameraPreview.test.ts +++ b/electron/cameraPreview.test.ts @@ -206,4 +206,30 @@ describe("floating camera preview lifecycle", () => { expect(owner.listenerCount("destroyed")).toBe(0); expect(overlay.destroy).toHaveBeenCalledOnce(); }); + it("blocks preview display and frames during a monitor take even across HUD toggles", () => { + const { preview, sender, overlay, owner } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + preview.setCaptureAllowed(false); + expect(owner.send).toHaveBeenLastCalledWith("camera-preview:availability", false); + preview.configure(sender, true, true); + preview.frame(sender, jpeg); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + preview.setCaptureAllowed(true); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + }); + + it("does not flash a blocked preview on ready or restore one the user hid", () => { + const { preview, sender, overlay, overlaySender } = setup(); + preview.setCaptureAllowed(false); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).not.toHaveBeenCalled(); + preview.hide(overlaySender); + preview.setCaptureAllowed(true); + expect(overlay.showInactive).not.toHaveBeenCalled(); + }); }); diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts index 9f1ff2ec73..f91474d8ee 100644 --- a/electron/cameraPreview.ts +++ b/electron/cameraPreview.ts @@ -10,6 +10,7 @@ export class CameraPreviewController { private window: BrowserWindow | null = null; private owner: WebContents | null = null; private visible = false; + private captureAllowed = true; private width: number = CAMERA_PREVIEW_DEFAULT_SIZE; private aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO; private ready = false; @@ -29,6 +30,7 @@ export class CameraPreviewController { if (this.owner && this.owner !== owner) return null; if (!this.owner) { this.owner = owner; + owner.send("camera-preview:availability", this.captureAllowed); this.onOwnerDestroyed = () => { if (this.owner === owner) this.dispose(); }; @@ -47,7 +49,7 @@ export class CameraPreviewController { if (this.window !== previewWindow) return; this.ready = true; this.resizeWindow(previewWindow, this.width, this.aspectRatio); - if (this.visible) this.window?.showInactive(); + if (this.visible && this.captureAllowed) this.window?.showInactive(); }); } return this.window; @@ -57,8 +59,16 @@ export class CameraPreviewController { const previewWindow = this.ensure(owner); if (!previewWindow) return; this.visible = enabled && visible; - if (this.visible && this.ready) previewWindow.showInactive(); - else if (!this.visible) previewWindow.hide(); + if (this.visible && this.captureAllowed && this.ready) previewWindow.showInactive(); + else if (!this.visible || !this.captureAllowed) previewWindow.hide(); + } + + /** Linux portal monitor captures cannot exclude app windows. */ + setCaptureAllowed(allowed: boolean): void { + this.captureAllowed = allowed; + if (!allowed) this.window?.hide(); + else if (this.visible && this.ready) this.window?.showInactive(); + this.owner?.send("camera-preview:availability", allowed); } hide(sender: WebContents): void { @@ -80,7 +90,7 @@ export class CameraPreviewController { } frame(sender: WebContents, jpeg: unknown, aspectRatio?: unknown): void { - if (sender !== this.owner || !this.visible || !this.window) return; + if (sender !== this.owner || !this.visible || !this.captureAllowed || !this.window) return; // A 320px JPEG is usually under 20KB. Never relay arbitrary URLs or unbounded IPC data. if ( typeof jpeg !== "string" || @@ -114,6 +124,7 @@ export class CameraPreviewController { this.window = null; this.owner = null; this.visible = false; + this.captureAllowed = true; this.width = CAMERA_PREVIEW_DEFAULT_SIZE; this.aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO; } diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 7604f5cb44..151eb90f04 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -58,6 +58,8 @@ interface Window { usesSystemSourcePicker?: () => Promise; /** Name of the last source picked in Apple's picker, for display only: it is not selected. */ getLastPickedSource?: () => Promise; + prepareCameraPreviewCapture?: () => Promise; + onCameraPreviewAvailability?: (callback: (available: boolean) => void) => () => void; configureCameraPreview?: (enabled: boolean, visible: boolean) => Promise; sendCameraPreviewFrame?: (jpeg: string, aspectRatio: number) => Promise; resizeCameraPreview?: (width: number) => Promise; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index eaab74500d..c7076e3435 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -181,6 +181,10 @@ import { isWindowsUpdatePending, WINDOWS_UPDATE_PENDING_MESSAGE, } from "../windows-update"; +import { + createWindowsCameraPreviewMailbox, + watchWindowsCameraPreview, +} from "../windowsCameraPreview"; import { getSourcesByType, shouldEnumerateSourceTypesSeparately } from "./desktopSourceTypes"; import { registerNativeBridgeHandlers } from "./nativeBridge"; import { createNativeMacMidCaptureErrorWatch } from "./nativeMacMidCaptureErrorWatch"; @@ -2019,6 +2023,7 @@ export function registerIpcHandlers( */ const setAppRecording = (recording: boolean, sourceName: string, preparing = false) => { appRecording = recording; + if (!recording && !preparing) cameraPreview.setCaptureAllowed(true); if (!preparing) mcpAppRecorder.recordingChanged(recording); onRecordingStateChange?.(recording, sourceName, preparing); }; @@ -2549,13 +2554,17 @@ export function registerIpcHandlers( } ipcMain.handle("camera-preview:configure", (event, enabled: unknown, visible: unknown) => { if ( - process.platform !== "darwin" || event.sender !== getMainWindow()?.webContents || new URL(event.sender.getURL()).searchParams.get("windowType") !== "hud-overlay" ) return; cameraPreview.configure(event.sender, enabled === true, visible === true); }); + ipcMain.handle("camera-preview:prepare-capture", (event) => { + if (process.platform === "linux" && event.sender === getMainWindow()?.webContents) { + cameraPreview.setCaptureAllowed(false); + } + }); ipcMain.handle("camera-preview:frame", (event, jpeg: unknown, aspectRatio: unknown) => cameraPreview.frame(event.sender, jpeg, aspectRatio), ); @@ -2755,6 +2764,7 @@ export function registerIpcHandlers( }); lastLinuxCaptureSession = session; + cameraPreview.setCaptureAllowed(false); await session.start(); // The picker is up now. No timeout: a human is reading a dialog. await session.waitUntilSourceSelected(); @@ -2768,6 +2778,7 @@ export function registerIpcHandlers( } preparingLinuxCaptureToken = null; + cameraPreview.setCaptureAllowed(session.grantedSourceKind === "window"); preparedLinuxCapture = { session, outputPath, request }; return { success: true, @@ -2779,6 +2790,7 @@ export function registerIpcHandlers( // The portal may have come or gone since the probe. resetLinuxCaptureBackendProbe(); discardPreparedLinuxCapture("prepare failed"); + cameraPreview.setCaptureAllowed(true); return { success: false, error: String(error) }; } }, @@ -2787,6 +2799,7 @@ export function registerIpcHandlers( /** Drops a prepared session, e.g. when the countdown was cancelled. */ ipcMain.handle("cancel-native-linux-prepare", async () => { discardPreparedLinuxCapture("cancelled by the renderer"); + if (!linuxNativeCaptureSession && !appRecording) cameraPreview.setCaptureAllowed(true); return { success: true }; }); @@ -2852,11 +2865,13 @@ export function registerIpcHandlers( }); if (!prepared) { + cameraPreview.setCaptureAllowed(false); await session.start(); // Blocks until the user answers the portal picker, which has no // upper bound. On this path the countdown has already run. await session.waitUntilSourceSelected(); } + cameraPreview.setCaptureAllowed(session.grantedSourceKind === "window"); // Idempotent, and a no-op for a session that was not deferred. session.arm(); await session.waitUntilCapturing(); @@ -2879,6 +2894,7 @@ export function registerIpcHandlers( } catch (error) { console.error("Failed to start native Linux recording:", error); resetLinuxCaptureBackendProbe(); + cameraPreview.setCaptureAllowed(true); linuxNativeCaptureSession = null; linuxNativeCaptureRecordingId = null; linuxNativeCaptureCursorMode = "editable-overlay"; @@ -2995,6 +3011,7 @@ export function registerIpcHandlers( return { success: false, error: WINDOWS_UPDATE_PENDING_MESSAGE }; } let startReserved = false; + let mailbox: Awaited> | undefined; try { if (!isWindowsGraphicsCaptureOsSupported()) { return { @@ -3054,7 +3071,10 @@ export function registerIpcHandlers( request.preferSoftwareEncoder === true || envPreferSoftwareEncoder === "true" || envPreferSoftwareEncoder === "1"; + if (request.webcam.enabled) + mailbox = await createWindowsCameraPreviewMailbox(app.getPath("temp")); const config = { + webcamPreviewPath: mailbox?.file, schemaVersion: 2, recordingId, preferSoftwareEncoder, @@ -3160,6 +3180,19 @@ export function registerIpcHandlers( stdio: ["pipe", "pipe", "pipe"], windowsHide: true, }); + if (mailbox) { + const activeMailbox = mailbox; + const hud = getMainWindow(); + const stopPreview = watchWindowsCameraPreview(activeMailbox.file, (jpeg, ratio) => { + if (hud && !hud.isDestroyed()) cameraPreview.frame(hud.webContents, jpeg, ratio); + }); + proc.once("close", () => { + stopPreview(); + void activeMailbox + .remove() + .catch((error: unknown) => console.warn("Could not remove camera preview:", error)); + }); + } nativeWindowsCaptureProcess = proc; nativeWindowsCaptureDrainCleanup = attachNativeWindowsCaptureOutputDrain(proc); console.info("[native-wgc] helper spawned", { pid: proc.pid }); @@ -3227,6 +3260,7 @@ export function registerIpcHandlers( microphoneDefaulted, }; } catch (error) { + if (mailbox && !nativeWindowsCaptureProcess) await mailbox.remove().catch(() => undefined); console.error("Failed to start native Windows recording:", error); if (!startReserved) return { success: false, error: String(error) }; const failedProc = nativeWindowsCaptureProcess; diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt index 87eaf2af97..0ad32e9225 100644 --- a/electron/native/wgc-capture/CMakeLists.txt +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -65,6 +65,8 @@ add_executable(wgc-capture src/webcam_format.cpp src/webcam_format.h src/webcam_capture.h + src/webcam_preview.h + src/webcam_preview_bitmap.h src/wgc_session.cpp src/wgc_session.h ) @@ -216,3 +218,9 @@ target_link_libraries(mf_encoder_color_test PRIVATE ole32 propsys ) + +add_executable(webcam_preview_bitmap_test + src/webcam_preview_bitmap.h + src/webcam_preview_bitmap_test.cpp +) +target_compile_options(webcam_preview_bitmap_test PRIVATE /EHsc /W4 /utf-8) diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index 56987ac0f4..45c08c42d0 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -10,6 +10,7 @@ #include "wasapi_render_keepalive.h" #include "frame_visibility.h" #include "webcam_capture.h" +#include "webcam_preview.h" #include "capture_session.h" #include @@ -39,6 +40,7 @@ struct CaptureConfig { std::string windowHandle; std::string outputPath; std::string webcamOutputPath; + std::string webcamPreviewPath; int fps = 60; int width = 0; int height = 0; @@ -587,6 +589,7 @@ bool parseConfig(const std::string& json, CaptureConfig& config) { config.webcamDeviceName = findString(json, "webcamDeviceName"); config.webcamDirectShowClsid = findString(json, "webcamDirectShowClsid"); config.webcamOutputPath = findString(json, "webcamPath"); + config.webcamPreviewPath = findString(json, "webcamPreviewPath"); config.webcamWidth = findInt(json, "webcamWidth", 0); config.webcamHeight = findInt(json, "webcamHeight", 0); config.webcamFps = findInt(json, "webcamFps", 0); @@ -1498,6 +1501,8 @@ int wmain(int argc, wchar_t* argv[]) { } } + WebcamPreview webcamPreview(webcamCapture, config.webcamEnabled ? utf8ToWide(config.webcamPreviewPath) : L""); + if (!session.start()) { webcamCapture.stop(); microphoneCapture.stop(); diff --git a/electron/native/wgc-capture/src/webcam_preview.h b/electron/native/wgc-capture/src/webcam_preview.h new file mode 100644 index 0000000000..360324b3ab --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_preview.h @@ -0,0 +1,54 @@ +#pragma once + +#include "webcam_capture.h" +#include "webcam_preview_bitmap.h" +#include +#include +#include + +// Latest-frame mailbox, replaced atomically. A slow UI never blocks the camera +// or encoder, and no frame queue grows during a long take. +class WebcamPreview { +public: + WebcamPreview(WebcamCapture& capture, const std::wstring& path) { + if (path.empty()) return; + thread_ = std::thread([this, &capture, path] { + const std::filesystem::path target(path); + const std::filesystem::path pending(path + L".tmp"); + uint64_t sequence = 0; + std::unique_lock lock(mutex_); + while (!stopped_) { + lock.unlock(); + WebcamFrameSnapshot frame; + if (capture.copyLatestFrame(frame, sequence)) { + sequence = frame.sequence; + const auto bmp = webcamPreviewBitmap(frame.data, frame.width, frame.height, capture.deliversNv12()); + if (!bmp.empty()) { + std::ofstream out(pending, std::ios::binary | std::ios::trunc); + out.write(reinterpret_cast(bmp.data()), static_cast(bmp.size())); + out.close(); + if (out) MoveFileExW(pending.c_str(), target.c_str(), MOVEFILE_REPLACE_EXISTING); + } + } + lock.lock(); + cv_.wait_for(lock, std::chrono::milliseconds(200), [this] { return stopped_; }); + } + lock.unlock(); + std::error_code ignored; + std::filesystem::remove(pending, ignored); + std::filesystem::remove(target, ignored); + }); + } + ~WebcamPreview() { + { std::lock_guard lock(mutex_); stopped_ = true; } + cv_.notify_all(); + if (thread_.joinable()) thread_.join(); + } + WebcamPreview(const WebcamPreview&) = delete; + WebcamPreview& operator=(const WebcamPreview&) = delete; +private: + std::mutex mutex_; + std::condition_variable cv_; + bool stopped_ = false; + std::thread thread_; +}; diff --git a/electron/native/wgc-capture/src/webcam_preview_bitmap.h b/electron/native/wgc-capture/src/webcam_preview_bitmap.h new file mode 100644 index 0000000000..940128aebe --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_preview_bitmap.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + +// A bounded, top-down BMP for the presenter preview. Only these small pixels cross +// the process boundary; the native camera and recording clocks remain untouched. +inline std::vector webcamPreviewBitmap( + std::span frame, int width, int height, bool nv12) { + if (width < 2 || height < 2 || width > 16384 || height > 16384 || + (nv12 && (width % 2 || height % 2))) return {}; + const size_t required = static_cast(width) * height * (nv12 ? 3 : 8) / 2; + if (frame.size() < required) return {}; + const int outWidth = std::min(320, width); + const int outHeight = std::clamp(height * outWidth / width, 1, 2560); + const uint32_t bytes = outWidth * outHeight * 4; + std::vector bmp(54 + bytes, 0); + const auto put = [&](size_t at, uint32_t value, int count = 4) { + for (int i = 0; i < count; ++i) bmp[at + i] = static_cast(value >> (8 * i)); + }; + bmp[0] = 'B'; bmp[1] = 'M'; + put(2, static_cast(bmp.size())); put(10, 54); put(14, 40); + put(18, outWidth); put(22, static_cast(-outHeight)); + put(26, 1, 2); put(28, 32, 2); put(34, bytes); + for (int y = 0; y < outHeight; ++y) { + const int sy = y * height / outHeight; + for (int x = 0; x < outWidth; ++x) { + const int sx = x * width / outWidth; + const size_t to = 54 + (static_cast(y) * outWidth + x) * 4; + if (nv12) { + const int yy = std::max(0, static_cast(frame[sy * width + sx]) - 16); + const size_t uv = static_cast(width) * height + (sy / 2) * width + (sx & ~1); + const int u = static_cast(frame[uv]) - 128; + const int v = static_cast(frame[uv + 1]) - 128; + bmp[to] = static_cast(std::clamp((298 * yy + 516 * u + 128) >> 8, 0, 255)); + bmp[to + 1] = static_cast(std::clamp((298 * yy - 100 * u - 208 * v + 128) >> 8, 0, 255)); + bmp[to + 2] = static_cast(std::clamp((298 * yy + 409 * v + 128) >> 8, 0, 255)); + } else { + const size_t from = (static_cast(sy) * width + sx) * 4; + std::copy_n(frame.begin() + from, 3, bmp.begin() + to); + } + bmp[to + 3] = 255; + } + } + return bmp; +} diff --git a/electron/native/wgc-capture/src/webcam_preview_bitmap_test.cpp b/electron/native/wgc-capture/src/webcam_preview_bitmap_test.cpp new file mode 100644 index 0000000000..db9841beb9 --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_preview_bitmap_test.cpp @@ -0,0 +1,30 @@ +#include "webcam_preview_bitmap.h" +#include + +int main() { + int failures = 0; + const auto check = [&](bool pass) { if (!pass) ++failures; }; + std::vector bgra = {1, 2, 3, 0, 4, 5, 6, 0, 7, 8, 9, 0, 10, 11, 12, 0}; + const auto bmp = webcamPreviewBitmap(bgra, 2, 2, false); + check(bmp.size() == 70 && bmp[0] == 'B' && bmp[1] == 'M'); + check(bmp[18] == 2 && bmp[22] == 254 && bmp[23] == 255); // top-down + check(bmp[54] == 1 && bmp[56] == 3 && bmp[57] == 255 && bmp[62] == 7); + const std::vector black = {16, 16, 16, 16, 128, 128}; + const auto dark = webcamPreviewBitmap(black, 2, 2, true); + check(dark[54] == 0 && dark[55] == 0 && dark[56] == 0 && dark[57] == 255); + const std::vector white = {235, 235, 235, 235, 128, 128}; + const auto light = webcamPreviewBitmap(white, 2, 2, true); + check(light[54] == 255 && light[55] == 255 && light[56] == 255); + const std::vector red = {81, 81, 81, 81, 90, 240}; + const auto color = webcamPreviewBitmap(red, 2, 2, true); + check(color[54] < 3 && color[55] < 3 && color[56] > 250); + check(webcamPreviewBitmap({}, 2, 2, false).empty()); + check(webcamPreviewBitmap(bgra, 3, 2, true).empty()); + check(webcamPreviewBitmap(bgra, -1, 2, false).empty()); + check(webcamPreviewBitmap(bgra, 16385, 2, false).empty()); + const std::vector large(640 * 480 * 4, 32); + const auto small = webcamPreviewBitmap(large, 640, 480, false); + check(small.size() == 54 + 320 * 240 * 4 && small[54] == 32 && small[57] == 255); + std::printf("webcam preview bitmap: %d failures\n", failures); + return failures ? 1 : 0; +} diff --git a/electron/preload.ts b/electron/preload.ts index dc6eb1ea51..c049128194 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -170,6 +170,13 @@ contextBridge.exposeInMainWorld("electronAPI", { getLastPickedSource: (): Promise => { return ipcRenderer.invoke("get-last-picked-source"); }, + prepareCameraPreviewCapture: () => + ipcRenderer.invoke("camera-preview:prepare-capture") as Promise, + onCameraPreviewAvailability: (callback: (available: boolean) => void) => { + const listener = (_event: Electron.IpcRendererEvent, available: boolean) => callback(available); + ipcRenderer.on("camera-preview:availability", listener); + return () => ipcRenderer.removeListener("camera-preview:availability", listener); + }, configureCameraPreview: (enabled: boolean, visible: boolean): Promise => ipcRenderer.invoke("camera-preview:configure", enabled, visible), sendCameraPreviewFrame: (jpeg: string, aspectRatio: number): Promise => diff --git a/electron/windows.ts b/electron/windows.ts index a1df99bfe9..cb687166e5 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -921,6 +921,7 @@ export function createCameraPreviewWindow(): BrowserWindow { }); win.once("ready-to-show", () => win.setBounds(initialBounds())); applyContentProtection(win, "Camera Preview"); + keepAlwaysOnTop(win); followAcrossSpaces(win); cameraPreviewWindow = win; win.on("closed", () => { diff --git a/electron/windowsCameraPreview.test.ts b/electron/windowsCameraPreview.test.ts new file mode 100644 index 0000000000..519cc2361c --- /dev/null +++ b/electron/windowsCameraPreview.test.ts @@ -0,0 +1,97 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import { nativeImage } from "electron"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + createWindowsCameraPreviewMailbox, + watchWindowsCameraPreview, +} from "./windowsCameraPreview"; + +vi.mock("electron", () => ({ + nativeImage: { + createFromBitmap: vi.fn(() => ({ isEmpty: () => false, toJPEG: () => Buffer.from("jpeg") })), + }, +})); + +function frame() { + const data = Buffer.alloc(54 + 2 * 2 * 4); + data.write("BM"); + data.writeInt32LE(2, 18); + data.writeInt32LE(-2, 22); + return data; +} + +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + vi.clearAllMocks(); +}); + +describe("Windows native camera preview", () => { + it("delivers bounded top-down pixels without opening a camera", async () => { + vi.useFakeTimers(); + vi.spyOn(fs, "readFile").mockResolvedValue(frame()); + const onFrame = vi.fn(); + const stop = watchWindowsCameraPreview("latest.bmp", onFrame); + await vi.advanceTimersByTimeAsync(0); + expect(nativeImage.createFromBitmap).toHaveBeenCalledWith(frame().subarray(54), { + width: 2, + height: 2, + }); + expect(onFrame).toHaveBeenCalledWith("data:image/jpeg;base64,anBlZw==", 1); + stop(); + await vi.advanceTimersByTimeAsync(1000); + expect(onFrame).toHaveBeenCalledOnce(); + }); + + it("does not queue reads or deliver an in-flight frame after cleanup", async () => { + vi.useFakeTimers(); + let resolve: ((value: Buffer) => void) | undefined; + const read = vi.spyOn(fs, "readFile").mockImplementationOnce( + () => + new Promise>((done) => { + resolve = done; + }), + ); + const onFrame = vi.fn(); + const stop = watchWindowsCameraPreview("latest.bmp", onFrame); + await vi.advanceTimersByTimeAsync(1000); + expect(read).toHaveBeenCalledOnce(); + stop(); + resolve?.(frame()); + await vi.advanceTimersByTimeAsync(1000); + expect(onFrame).not.toHaveBeenCalled(); + expect(read).toHaveBeenCalledOnce(); + }); + + it("retries a missing frame and rejects oversized dimensions and truncated pixels", async () => { + vi.useFakeTimers(); + const huge = frame(); + huge.writeInt32LE(10000, 18); + vi.spyOn(fs, "readFile") + .mockRejectedValueOnce(new Error("ENOENT")) + .mockResolvedValueOnce(huge) + .mockResolvedValueOnce(frame().subarray(0, 54)) + .mockResolvedValue(frame()); + const onFrame = vi.fn(); + const stop = watchWindowsCameraPreview("latest.bmp", onFrame); + await vi.advanceTimersByTimeAsync(500); + expect(onFrame).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(100); + expect(onFrame).toHaveBeenCalledOnce(); + stop(); + }); + + it("removes the private directory and any half-written camera frame", async () => { + const mailbox = await createWindowsCameraPreviewMailbox(os.tmpdir()); + try { + await fs.writeFile(mailbox.file, frame()); + await fs.writeFile(`${mailbox.file}.tmp`, frame()); + await mailbox.remove(); + await expect(fs.stat(mailbox.file)).rejects.toThrow(); + await expect(fs.stat(`${mailbox.file}.tmp`)).rejects.toThrow(); + } finally { + await mailbox.remove(); + } + }); +}); diff --git a/electron/windowsCameraPreview.ts b/electron/windowsCameraPreview.ts new file mode 100644 index 0000000000..da054a39a1 --- /dev/null +++ b/electron/windowsCameraPreview.ts @@ -0,0 +1,52 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { nativeImage } from "electron"; + +/** Read the native camera's atomic latest-frame mailbox, with no queued reads. */ +export function watchWindowsCameraPreview( + file: string, + onFrame: (jpeg: string, aspectRatio: number) => void, +) { + let cancelled = false; + let timer: ReturnType | undefined; + const read = async () => { + try { + const data = await fs.readFile(file); + if (cancelled || data.length > 3_300_000) return; + if (data.length < 54 || data.toString("ascii", 0, 2) !== "BM") return; + const width = data.readInt32LE(18); + const height = -data.readInt32LE(22); + if ( + width < 1 || + width > 320 || + height < 1 || + height > 2560 || + data.length !== 54 + width * height * 4 + ) + return; + const image = nativeImage.createFromBitmap(data.subarray(54), { width, height }); + if (!image.isEmpty() && width > 0 && height > 0) { + onFrame(`data:image/jpeg;base64,${image.toJPEG(70).toString("base64")}`, width / height); + } + } catch { + // No first frame yet, or Windows still has the replaced file open. + } finally { + if (!cancelled) timer = setTimeout(() => void read(), 200); + } + }; + void read(); + return () => { + cancelled = true; + clearTimeout(timer); + }; +} + +/** A private per-take directory also lets cleanup remove a half-written frame. */ +export async function createWindowsCameraPreviewMailbox(tempDir: string) { + const directory = await fs.mkdtemp(path.join(tempDir, "openscreen-camera-preview-")); + return { + file: path.join(directory, "latest.bmp"), + remove: () => + fs.rm(directory, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }), + }; +} diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjs index 90b961ee79..9315aade79 100644 --- a/scripts/build-windows-wgc-helper.mjs +++ b/scripts/build-windows-wgc-helper.mjs @@ -164,6 +164,13 @@ if (!fs.existsSync(frameSlotClockTestPath)) { await run(frameSlotClockTestPath, [], { cwd: BUILD_DIR }); console.log(`Passed ${frameSlotClockTestPath}`); +const webcamPreviewTestPath = path.join(BUILD_DIR, "webcam_preview_bitmap_test.exe"); +if (!fs.existsSync(webcamPreviewTestPath)) { + throw new Error(`WGC helper build completed but ${webcamPreviewTestPath} was not found.`); +} +await run(webcamPreviewTestPath, [], { cwd: BUILD_DIR }); +console.log(`Passed ${webcamPreviewTestPath}`); + const webcamSnapshotTestPath = path.join(BUILD_DIR, "webcam_snapshot_test.exe"); if (!fs.existsSync(webcamSnapshotTestPath)) { throw new Error(`WGC helper build completed but ${webcamSnapshotTestPath} was not found.`); diff --git a/src/components/launch/LaunchWindow.test.tsx b/src/components/launch/LaunchWindow.test.tsx index 5c065f74ee..803b6439c2 100644 --- a/src/components/launch/LaunchWindow.test.tsx +++ b/src/components/launch/LaunchWindow.test.tsx @@ -2548,7 +2548,7 @@ describe("LaunchWindow toasts", () => { }); }); -describe("macOS floating camera preview control", () => { +describe("floating camera preview control", () => { beforeEach(() => { platformState.value = "darwin"; resetLaunchMocks(); @@ -2589,10 +2589,12 @@ describe("macOS floating camera preview control", () => { expect(screen.queryByTestId("launch-camera-preview-button")).not.toBeInTheDocument(); }); - it("does not offer the preview on Windows", () => { - platformState.value = "win32"; + it.each(["win32", "linux"] as const)("offers the preview on %s", async (platform) => { + platformState.value = platform; recorderState.value.webcamEnabled = true; renderLaunchWindow(); - expect(screen.queryByTestId("launch-camera-preview-button")).not.toBeInTheDocument(); + await waitFor(() => + expect(screen.getByTestId("launch-camera-preview-button")).toBeInTheDocument(), + ); }); }); diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index dcd36a7f48..5be6dec5df 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -149,8 +149,13 @@ export function LaunchWindow() { } = useScreenRecorder(); // Choosing a device and switching one on are deliberately separate concerns. - const cameraPreview = useFloatingCameraPreview(webcamPreviewStream, webcamEnabled && !saving); - const cameraPreviewShown = webcamEnabled && !saving && cameraPreview.visible; + const cameraPreview = useFloatingCameraPreview( + webcamPreviewStream, + webcamEnabled && !saving, + recording && getPlatform() === "win32", + ); + const cameraPreviewShown = + webcamEnabled && !saving && cameraPreview.available && cameraPreview.visible; // The mic and camera buttons are plain on/off toggles that use whatever device // is currently selected (the system default until the user says otherwise); @@ -1319,8 +1324,12 @@ export function LaunchWindow() { {cameraPreview.supported && webcamEnabled && ( )} diff --git a/src/hooks/useFloatingCameraPreview.test.tsx b/src/hooks/useFloatingCameraPreview.test.tsx index 0c0279c0db..e423f73163 100644 --- a/src/hooks/useFloatingCameraPreview.test.tsx +++ b/src/hooks/useFloatingCameraPreview.test.tsx @@ -6,6 +6,7 @@ import { useFloatingCameraPreview } from "./useFloatingCameraPreview"; vi.mock("../utils/platformUtils", () => ({ getPlatform: () => platform })); let platform = "darwin"; let hidden: (() => void) | undefined; +let availability: ((available: boolean) => void) | undefined; const configure = vi.fn().mockResolvedValue(undefined); const send = vi.fn().mockResolvedValue(undefined); const unsubscribe = vi.fn(); @@ -21,6 +22,10 @@ beforeEach(() => { value: { configureCameraPreview: configure, sendCameraPreviewFrame: send, + onCameraPreviewAvailability: (callback: (available: boolean) => void) => { + availability = callback; + return () => undefined; + }, onCameraPreviewHidden: (callback: () => void) => { hidden = callback; return unsubscribe; @@ -90,7 +95,7 @@ describe("floating camera publisher", () => { expect(send).toHaveBeenCalledTimes(2); }); - it("waits for the recorder stream and never opens a preview on other platforms", () => { + it("waits for the recorder stream and ignores unsupported platforms", () => { const { rerender, unmount } = renderHook( ({ enabled }) => useFloatingCameraPreview(null, enabled), { initialProps: { enabled: true } }, @@ -101,9 +106,40 @@ describe("floating camera publisher", () => { expect(configure).toHaveBeenLastCalledWith(false, false); unmount(); configure.mockClear(); - platform = "win32"; + platform = "unknown"; renderHook(() => useFloatingCameraPreview(stream, true)); expect(configure).not.toHaveBeenCalled(); expect(send).not.toHaveBeenCalled(); }); + it.each(["darwin", "win32", "linux"])("publishes preview frames on %s", (value) => { + platform = value; + const { result } = renderHook(() => useFloatingCameraPreview(stream, true)); + expect(result.current.supported).toBe(true); + expect(configure).toHaveBeenLastCalledWith(true, true); + expect(send).toHaveBeenCalledOnce(); + }); + + it("keeps the window visible after Windows hands the camera to the native recorder", () => { + platform = "win32"; + const { rerender } = renderHook( + ({ camera, native }) => useFloatingCameraPreview(camera, true, native), + { initialProps: { camera: stream as MediaStream | null, native: false } }, + ); + rerender({ camera: null, native: true }); + expect(configure).toHaveBeenLastCalledWith(true, true); + expect(stop).not.toHaveBeenCalled(); + }); + + it("suspends publishing for Linux monitor capture and restores the visibility preference", async () => { + platform = "linux"; + const { result } = renderHook(() => useFloatingCameraPreview(stream, true)); + act(() => availability?.(false)); + expect(result.current.available).toBe(false); + expect(result.current.visible).toBe(true); + await act(() => vi.advanceTimersByTimeAsync(500)); + expect(send).toHaveBeenCalledOnce(); + act(() => availability?.(true)); + expect(result.current.available).toBe(true); + expect(send).toHaveBeenCalledTimes(2); + }); }); diff --git a/src/hooks/useFloatingCameraPreview.ts b/src/hooks/useFloatingCameraPreview.ts index 3d1f1108ba..5c0c9d7188 100644 --- a/src/hooks/useFloatingCameraPreview.ts +++ b/src/hooks/useFloatingCameraPreview.ts @@ -2,21 +2,27 @@ import { useEffect, useState } from "react"; import { getPlatform } from "../utils/platformUtils"; /** A small self-view from the recorder's stream; never acquires or stops the camera. */ -export function useFloatingCameraPreview(stream: MediaStream | null, enabled: boolean) { - const supported = getPlatform() === "darwin"; +export function useFloatingCameraPreview( + stream: MediaStream | null, + enabled: boolean, + nativeCameraActive = false, +) { + const supported = ["darwin", "win32", "linux"].includes(getPlatform()); const [visible, setVisible] = useState(true); + const [available, setAvailable] = useState(true); + useEffect(() => window.electronAPI?.onCameraPreviewAvailability?.(setAvailable), []); useEffect(() => window.electronAPI?.onCameraPreviewHidden?.(() => setVisible(false)), []); useEffect(() => { if (!supported) return; void window.electronAPI - ?.configureCameraPreview?.(enabled, enabled && visible && !!stream) + ?.configureCameraPreview?.(enabled, enabled && visible && (!!stream || nativeCameraActive)) .catch((error: unknown) => console.warn("Could not configure camera preview:", error)); - }, [enabled, visible, stream, supported]); + }, [enabled, visible, stream, nativeCameraActive, supported]); useEffect(() => { - if (!supported || !enabled || !visible || !stream) return; + if (!supported || !enabled || !visible || !available || !stream) return; const video = document.createElement("video"); video.muted = true; video.playsInline = true; @@ -52,7 +58,7 @@ export function useFloatingCameraPreview(stream: MediaStream | null, enabled: bo video.pause(); video.srcObject = null; }; - }, [enabled, visible, stream, supported]); + }, [enabled, visible, available, stream, supported]); useEffect( () => () => { @@ -62,5 +68,5 @@ export function useFloatingCameraPreview(stream: MediaStream | null, enabled: bo [supported], ); - return { supported, visible, toggle: () => setVisible((value) => !value) }; + return { supported, available, visible, toggle: () => setVisible((value) => !value) }; } diff --git a/src/hooks/useScreenRecorder.cameraPrefs.test.tsx b/src/hooks/useScreenRecorder.cameraPrefs.test.tsx index d60d1a56d2..331ca7d313 100644 --- a/src/hooks/useScreenRecorder.cameraPrefs.test.tsx +++ b/src/hooks/useScreenRecorder.cameraPrefs.test.tsx @@ -38,7 +38,7 @@ afterEach(() => { vi.restoreAllMocks(); }); -function setup() { +function setup(platform = "darwin") { window.history.replaceState({}, "", "/?windowType=hud-overlay"); let publish: ((prefs: Prefs) => void) | undefined; const stops: Array> = []; @@ -50,7 +50,7 @@ function setup() { }); Object.defineProperty(navigator, "mediaDevices", { configurable: true, value: { getUserMedia } }); window.electronAPI = { - getPlatform: () => "darwin", + getPlatform: () => platform, getRecordingPrefs: async () => initial, onRecordingPrefsChanged: (callback: (prefs: Prefs) => void) => { publish = callback; @@ -67,8 +67,12 @@ function setup() { } describe("camera preview across recording preference broadcasts", () => { - it("keeps the resolved default camera stream alive when audio toggles publish unchanged camera preferences", async () => { - const { view, getUserMedia, stops, publish } = setup(); + it.each([ + "darwin", + "win32", + "linux", + ])("keeps the resolved camera alive across audio preferences on %s", async (platform) => { + const { view, getUserMedia, stops, publish } = setup(platform); await waitFor(() => expect(view.result.current.webcamPreviewStream).not.toBeNull()); const stream = view.result.current.webcamPreviewStream; const calls = getUserMedia.mock.calls.length; diff --git a/src/hooks/useScreenRecorder.nativeLinuxWebcamCursor.test.tsx b/src/hooks/useScreenRecorder.nativeLinuxWebcamCursor.test.tsx index 7711b35526..24ffa5951e 100644 --- a/src/hooks/useScreenRecorder.nativeLinuxWebcamCursor.test.tsx +++ b/src/hooks/useScreenRecorder.nativeLinuxWebcamCursor.test.tsx @@ -1,10 +1,11 @@ // @vitest-environment jsdom -import { act, renderHook } from "@testing-library/react"; +import { act, cleanup, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, @@ -51,6 +52,8 @@ function stubElectronAPI() { getPlatform: vi.fn(() => "linux"), getSelectedSource: vi.fn(async () => null), isNativeLinuxCaptureAvailable: vi.fn(async () => ({ success: true, available: true })), + prepareNativeLinuxRecording: vi.fn(async () => ({ success: true, recordingId: 7 })), + cancelNativeLinuxPrepare: vi.fn(async () => ({ success: true })), startNativeLinuxRecording: vi.fn(async () => ({ success: true, recordingId: 7 })), stopNativeLinuxRecording: vi.fn(), attachNativeLinuxWebcamRecording: vi.fn(async () => ({ success: true, session: {} })), @@ -90,6 +93,7 @@ beforeEach(() => { }); afterEach(() => { + cleanup(); vi.useRealTimers(); vi.unstubAllGlobals(); vi.restoreAllMocks(); diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 2f0ecedb45..fa8a0e6bee 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -317,11 +317,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // The HUD resolves a stored default (null) to a concrete device locally. // An audio-only save republishes that same null: applying it again would // reset the resolved device, stop its stream, and visibly blink the preview. - const isMacHud = - window.electronAPI?.getPlatform?.() === "darwin" && + const isHud = new URLSearchParams(window.location.search).get("windowType") === "hud-overlay"; if ( - !isMacHud || + !isHud || !appliedCameraSelection || appliedCameraSelection.deviceId !== deviceId || appliedCameraSelection.deviceName !== deviceName @@ -1901,6 +1900,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } + // The browser fallback cannot exclude a Linux preview from a monitor stream. + if (platform === "linux") await window.electronAPI.prepareCameraPreviewCapture?.(); + // Capture screen + microphone in parallel: the gap between the two // `getUserMedia` calls is the dominant source of the mic-vs-video lag at the // start of the recording (issue #57). @@ -2176,6 +2178,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } } catch (error) { console.error("Failed to start recording:", error); + window.electronAPI?.setRecordingState?.(false); const errorMsg = error instanceof Error ? error.message : "Failed to start recording"; if (errorMsg.includes("Permission denied") || errorMsg.includes("NotAllowedError")) { toast.error(t("recording.permissionDenied")); diff --git a/src/i18n/locales/ar/launch.json b/src/i18n/locales/ar/launch.json index 538b34e262..eef8e0bf04 100644 --- a/src/i18n/locales/ar/launch.json +++ b/src/i18n/locales/ar/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "في لينكس، تتوفر معاينة الكاميرا أثناء تسجيل نافذة فقط", "title": "معاينة الكاميرا", "show": "إظهار معاينة الكاميرا", "hide": "إخفاء معاينة الكاميرا", diff --git a/src/i18n/locales/cs/launch.json b/src/i18n/locales/cs/launch.json index a5a63939a2..c90e4d3807 100644 --- a/src/i18n/locales/cs/launch.json +++ b/src/i18n/locales/cs/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "V Linuxu je náhled kamery dostupný pouze při nahrávání okna", "title": "Náhled kamery", "show": "Zobrazit náhled kamery", "hide": "Skrýt náhled kamery", diff --git a/src/i18n/locales/de/launch.json b/src/i18n/locales/de/launch.json index 204b6e8589..50f2a5705a 100644 --- a/src/i18n/locales/de/launch.json +++ b/src/i18n/locales/de/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "Unter Linux ist die Kameravorschau nur bei Fensteraufnahmen verfügbar", "title": "Kameravorschau", "show": "Kameravorschau anzeigen", "hide": "Kameravorschau ausblenden", diff --git a/src/i18n/locales/en/launch.json b/src/i18n/locales/en/launch.json index 4987f2396e..1259dd7534 100644 --- a/src/i18n/locales/en/launch.json +++ b/src/i18n/locales/en/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "Camera preview is available during window recording only on Linux", "title": "Camera preview", "show": "Show camera preview", "hide": "Hide camera preview", diff --git a/src/i18n/locales/es/launch.json b/src/i18n/locales/es/launch.json index fc230bba7f..664e48002e 100644 --- a/src/i18n/locales/es/launch.json +++ b/src/i18n/locales/es/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "En Linux, la vista previa de la cámara solo está disponible al grabar una ventana", "title": "Vista previa de cámara", "show": "Mostrar vista previa de cámara", "hide": "Ocultar vista previa de cámara", diff --git a/src/i18n/locales/fr/launch.json b/src/i18n/locales/fr/launch.json index b768831edf..5497729bb5 100644 --- a/src/i18n/locales/fr/launch.json +++ b/src/i18n/locales/fr/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "Sous Linux, l’aperçu caméra est disponible uniquement lors de l’enregistrement d’une fenêtre", "title": "Aperçu de la caméra", "show": "Afficher l’aperçu de la caméra", "hide": "Masquer l’aperçu de la caméra", diff --git a/src/i18n/locales/it/launch.json b/src/i18n/locales/it/launch.json index cad6d15516..ad49301d94 100644 --- a/src/i18n/locales/it/launch.json +++ b/src/i18n/locales/it/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "Su Linux, l’anteprima della videocamera è disponibile solo durante la registrazione di una finestra", "title": "Anteprima della videocamera", "show": "Mostra anteprima della videocamera", "hide": "Nascondi anteprima della videocamera", diff --git a/src/i18n/locales/ja-JP/launch.json b/src/i18n/locales/ja-JP/launch.json index aa6d4b5677..824276a5f7 100644 --- a/src/i18n/locales/ja-JP/launch.json +++ b/src/i18n/locales/ja-JP/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "Linuxではカメラプレビューはウィンドウ録画中のみ利用できます", "title": "カメラプレビュー", "show": "カメラプレビューを表示", "hide": "カメラプレビューを非表示", diff --git a/src/i18n/locales/ko-KR/launch.json b/src/i18n/locales/ko-KR/launch.json index 455a84146a..c315b12ce2 100644 --- a/src/i18n/locales/ko-KR/launch.json +++ b/src/i18n/locales/ko-KR/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "Linux에서는 창 녹화 중에만 카메라 미리보기를 사용할 수 있습니다", "title": "카메라 미리보기", "show": "카메라 미리보기 표시", "hide": "카메라 미리보기 숨기기", diff --git a/src/i18n/locales/pt-BR/launch.json b/src/i18n/locales/pt-BR/launch.json index 893576b96f..d0aecac1fd 100644 --- a/src/i18n/locales/pt-BR/launch.json +++ b/src/i18n/locales/pt-BR/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "No Linux, a prévia da câmera só está disponível durante a gravação de uma janela", "title": "Prévia da câmera", "show": "Mostrar prévia da câmera", "hide": "Ocultar prévia da câmera", diff --git a/src/i18n/locales/ru/launch.json b/src/i18n/locales/ru/launch.json index a34023430a..059243e028 100644 --- a/src/i18n/locales/ru/launch.json +++ b/src/i18n/locales/ru/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "В Linux предпросмотр камеры доступен только при записи окна", "title": "Предпросмотр камеры", "show": "Показать предпросмотр камеры", "hide": "Скрыть предпросмотр камеры", diff --git a/src/i18n/locales/tr/launch.json b/src/i18n/locales/tr/launch.json index 929709e362..013406d450 100644 --- a/src/i18n/locales/tr/launch.json +++ b/src/i18n/locales/tr/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "Linux’ta kamera önizlemesi yalnızca pencere kaydı sırasında kullanılabilir", "title": "Kamera önizlemesi", "show": "Kamera önizlemesini göster", "hide": "Kamera önizlemesini gizle", diff --git a/src/i18n/locales/vi/launch.json b/src/i18n/locales/vi/launch.json index fe3ce29fa3..89093813a9 100644 --- a/src/i18n/locales/vi/launch.json +++ b/src/i18n/locales/vi/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "Trên Linux, bản xem trước camera chỉ khả dụng khi ghi một cửa sổ", "title": "Xem trước camera", "show": "Hiện bản xem trước camera", "hide": "Ẩn bản xem trước camera", diff --git a/src/i18n/locales/zh-CN/launch.json b/src/i18n/locales/zh-CN/launch.json index b53025f9c4..9e6bddb708 100644 --- a/src/i18n/locales/zh-CN/launch.json +++ b/src/i18n/locales/zh-CN/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "在 Linux 上，摄像头预览仅在录制窗口时可用", "title": "摄像头预览", "show": "显示摄像头预览", "hide": "隐藏摄像头预览", diff --git a/src/i18n/locales/zh-TW/launch.json b/src/i18n/locales/zh-TW/launch.json index 03d134c201..4184130810 100644 --- a/src/i18n/locales/zh-TW/launch.json +++ b/src/i18n/locales/zh-TW/launch.json @@ -1,5 +1,6 @@ { "floatingCamera": { + "windowCaptureOnly": "在 Linux 上，相機預覽僅在錄製視窗時可用", "title": "攝影機預覽", "show": "顯示攝影機預覽", "hide": "隱藏攝影機預覽", diff --git a/technical-documentation/architecture/recording.md b/technical-documentation/architecture/recording.md index 1396dd6ea7..13b9489977 100644 --- a/technical-documentation/architecture/recording.md +++ b/technical-documentation/architecture/recording.md @@ -22,19 +22,37 @@ The HUD starts and controls a recording. The source selector chooses a display o Electron applies `setContentProtection(true)` to the HUD window (`applyContentProtection` in `electron/windows.ts`). This keeps the controller out of captures and also makes it invisible in screenshots. It is skipped on macOS 26 and later, which never displays a protected window, and on Windows before 11 22H2 (build 22621), where it leaves the click-through HUD unable to take a click (#1105). On those Windows versions the HUD can appear in recordings; on macOS the ScreenCaptureKit helper still excludes it by window id. For a testing session only, `OPENSCREEN_DISABLE_CONTENT_PROTECTION=1` disables the protection; the code warns that the HUD then appears in captures. The tray icon is the reliable way to refocus OpenScreen or stop a recording when the click-through HUD is not convenient or is not visible. -## Presenter camera preview (macOS) +## Presenter camera preview The HUD camera has a draggable rounded rectangular self-view with show/hide and three width choices. It shows the complete camera frame at the shared recording stream’s aspect ratio, including when resized. Its preview button is hidden when the camera is off. The preview shares `useScreenRecorder`'s stream and stops with its owning HUD; it does not acquire another camera. `useFloatingCameraPreview` publishes small JPEG frames to the auxiliary window, with bounded -payloads and one IPC request in flight. Audio preference broadcasts in the macOS HUD preserve +payloads and one IPC request in flight. Audio preference broadcasts in the HUD preserve its resolved camera selection so the preview does not blink when audio is toggled. +On Windows, the renderer releases its validation stream before the native helper opens the +camera. A separate helper worker samples that same native camera at up to 5 fps into a bounded, +320px-wide top-down BMP mailbox, atomically replacing the latest frame in a private temporary +directory. Electron reads one frame at a time and relays a JPEG through the same preview +controller. Closing the helper stops polling and removes the directory, including incomplete +writes. The worker does not change the recording encoder or its clock. Rebuild the Windows +helper to enable preview during native takes; an older helper cannot publish those frames. + Hiding or resizing retains the window id. macOS capture excludes that id alongside the HUD and Notes, including windows hidden when capture starts. This requires rebuilding the native -helper. Windows and Linux keep their existing recording controls. +helper. Windows uses content protection for the preview (which does not use click-through). +`OPENSCREEN_DISABLE_CONTENT_PROTECTION=1` also disables Windows preview exclusion: use it +only for UI inspection, never for capture-exclusion evidence. + +Linux has no portal API to exclude an app window from a monitor capture. Preview is available +while idle and during native **window** recordings. Before opening the portal, the controller +hides it; only a confirmed `window` source allows it to return. Monitor, virtual, unknown, and +browser-fallback takes keep it hidden, including after show/hide requests. The HUD explains +this limitation and disables the preview toggle until recording ends. The camera sidecar +continues recording normally. Supporting a visible preview during full-monitor Linux capture +would require compositor-specific exclusion support; this implementation does not claim it. ## Capture backends diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 987dda5323..c55784bcd5 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -1080,3 +1080,4 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-19 | dev `codex/macos-floating-camera-preview`, rebased on `main` @ `3620cf8`, plus #1288 review changes; Electron 41.2.1; native helper rebuilt directly with `swiftc` + macOS 26.5 SDK | macOS 26.7, Apple M5 Pro, MacBook Pro Camera | **Partial — preview review slice passed** | Computer-use verified the complete rounded rectangular camera frame, aspect-preserving resize, camera-off hiding the HUD preview control, and show/hide during a finalized camera take. In the same take, after 5 s settling per condition, 30 s process-tree CPU deltas were **17.5% hidden / 22.3% shown** (100%=one core; includes Electron children and capture helper). Native `recording-stopped` reported **0 droppedVideoFrames over the entire take**, covering both intervals; this counts screen-writer backpressure/append failures, not upstream capture or webcam drops. Screen 1 at 3024×1964, saved webcam verified 1920×1080 (30 fps requested); mic/system audio off, system cursor. One sequential pair, no statistical claim. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras/platforms, restart/cancel, export, exhaustive exclusion and remaining checklist. | | 2026-10-19 | dev #1288 on `8c2608d9` + preview lifetime/quit fixes; Electron 41.2.1 | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — lifecycle slice passed** | Computer-use camera off/on removed/restored the HUD preview control. Normal Cmd+Q exited the OpenScreen Electron process with the preview enabled. Unit tests verify the same preview window survives initial camera-off and repeated toggles, with one owner-destruction listener and cleanup on HUD destruction. **Skipped:** recording, export, capture-exclusion measurements, other platforms and remaining checklist; no native capture changes in this slice. | | 2026-10-21 | dev #1288 on `20975d30` + synchronous preview reservation before capture exclusions; Electron 41.2.1; existing helper rebuilt from this branch on Oct 19 | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — capture-exclusion ordering slice passed** | Computer-use started native display recording with the preview hidden, showed it during the take and finalized into the editor. Helper diagnostics resolved both requested app window IDs and reported zero screen-writer drops; a decoded late screen frame contained no floating preview. Unit regression covers reserving the exclusion window before any renderer configuration and reusing it when enabled later without altering an existing preview’s visibility. **Skipped:** Apple’s picker (ordering path checked in code), forced renderer-effect race on desktop (covered by unit regression), other platforms, export, exhaustive frame-by-frame exclusion and remaining checklist. | +| 2026-10-29 | Dev `codex/macos-floating-camera-preview`, rebased on `main` @ `5d82a0cb`, Windows/Linux preview port working tree; Electron 41.2.1; macOS ScreenCaptureKit helper rebuilt directly with `swiftc` and the macOS 26.5 SDK (SwiftPM unavailable because of a local BuildServerProtocol loader error) | macOS 26.7, Apple Silicon; built-in camera | **Partial — HUD controls only; capture/export evidence outstanding** | Real coordinate clicks through `mcp__cua_repl`: opened device settings, selected the built-in camera, toggled camera off/on, and verified the preview control disappears/reappears. Restored the original camera-on state and quit the dev app. The tool bound only the HUD window; it did not expose the separate camera window, so live preview pixels were not visually verified. Native preview bitmap conversion tests passed locally; publisher/controller tests cover camera handoff, blocked monitor captures, cleanup and bounded frame delivery. **Skipped:** recording, export, capture-exclusion proof and the requested demonstration clips, because the preview could not be inspected in this desktop session; real Windows/Linux runs and the complete Windows helper build, because those operating systems are unavailable here; all remaining checklist sections. Linux monitor/virtual/unknown/browser-fallback capture deliberately hides the preview because the portal has no window-exclusion API; only native window capture permits it during a take. This row is not release-promotion QA. | From 3849d804d7002fa053338504c25fe4bdd00cdfc6 Mon Sep 17 00:00:00 2001 From: Satyam Raj Date: Tue, 3 Nov 2026 15:08:12 +0530 Subject: [PATCH 8/8] fix: address floating camera preview review findings --- electron/cameraPreview.test.ts | 43 +++++++- electron/cameraPreview.ts | 23 ++++- electron/electron-env.d.ts | 3 +- electron/ipc/handlers.ts | 14 ++- electron/preload.ts | 4 +- .../launch/FloatingCameraPreview.module.css | 14 --- src/components/launch/HudControls.tsx | 10 +- .../useScreenRecorder.restartFailure.test.tsx | 97 ++++++++++++++++++- src/hooks/useScreenRecorder.ts | 22 ++++- .../testing/manual-e2e-checklist.md | 1 + 10 files changed, 205 insertions(+), 26 deletions(-) diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts index bd25330edd..5e4660d0a8 100644 --- a/electron/cameraPreview.test.ts +++ b/electron/cameraPreview.test.ts @@ -126,7 +126,7 @@ describe("floating camera preview lifecycle", () => { }); it("recreates an unexpectedly closed preview and waits for the replacement to be ready", () => { - const { preview, sender, overlay, create } = setup(); + const { preview, sender, overlay, create, owner } = setup(); const replacement = setup().overlay; create .mockReturnValueOnce(overlay as unknown as BrowserWindow) @@ -134,6 +134,7 @@ describe("floating camera preview lifecycle", () => { preview.configure(sender, true, true); overlay.emit("ready-to-show"); overlay.emit("closed"); + expect(owner.send).toHaveBeenLastCalledWith("camera-preview:hidden"); preview.frame(sender, jpeg); expect(overlay.webContents.send).not.toHaveBeenCalled(); @@ -144,6 +145,27 @@ describe("floating camera preview lifecycle", () => { expect(replacement.showInactive).toHaveBeenCalledOnce(); }); + it("treats system-menu Close as hide, preserving size and capture exclusion", () => { + const { preview, sender, overlay, overlaySender, owner, create, resize } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + preview.resize(overlaySender, 320); + const event = { preventDefault: vi.fn() }; + overlay.emit("close", event); + expect(event.preventDefault).toHaveBeenCalledOnce(); + expect(overlay.hide).toHaveBeenCalledOnce(); + expect(owner.send).toHaveBeenLastCalledWith("camera-preview:hidden"); + preview.configure(sender, true, true); + expect(preview.ensure(sender)).toBe(overlay); + expect(create).toHaveBeenCalledOnce(); + expect(resize).toHaveBeenLastCalledWith(overlay, 320, 16 / 9); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + expect(overlay.destroy).not.toHaveBeenCalled(); + // Shutdown uses destroy(), which bypasses the cancellable close event. + preview.dispose(); + expect(overlay.destroy).toHaveBeenCalledOnce(); + }); + it("ignores a stale closed event from a previously disposed preview", () => { const { preview, sender, overlay, create } = setup(); // Delay closure to exercise a late event after the replacement already exists. @@ -168,8 +190,10 @@ describe("floating camera preview lifecycle", () => { it("destroys the auxiliary window when the HUD is destroyed", () => { const { preview, sender, owner, overlay } = setup(); preview.configure(sender, true, true); + owner.send.mockClear(); owner.emit("destroyed"); expect(overlay.destroy).toHaveBeenCalledOnce(); + expect(owner.send).not.toHaveBeenCalled(); preview.frame(sender, jpeg); expect(overlay.webContents.send).not.toHaveBeenCalled(); }); @@ -222,6 +246,23 @@ describe("floating camera preview lifecycle", () => { expect(overlay.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); }); + it("restores a cancelled start without restoring an older or active capture", () => { + const { preview, sender, overlay, owner } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + const oldToken = preview.prepareCapture(); + const newToken = preview.prepareCapture(); + preview.restoreCapture(oldToken); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + preview.restoreCapture(newToken); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + expect(owner.send).toHaveBeenLastCalledWith("camera-preview:availability", true); + const token = preview.prepareCapture(); + preview.setCaptureAllowed(false); // A newer native capture owns the suspension. + preview.restoreCapture(token); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + }); + it("does not flash a blocked preview on ready or restore one the user hid", () => { const { preview, sender, overlay, overlaySender } = setup(); preview.setCaptureAllowed(false); diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts index f91474d8ee..8b477072dd 100644 --- a/electron/cameraPreview.ts +++ b/electron/cameraPreview.ts @@ -11,6 +11,7 @@ export class CameraPreviewController { private owner: WebContents | null = null; private visible = false; private captureAllowed = true; + private captureGeneration = 0; private width: number = CAMERA_PREVIEW_DEFAULT_SIZE; private aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO; private ready = false; @@ -39,11 +40,18 @@ export class CameraPreviewController { if (!this.window) { this.window = this.createWindow(); const previewWindow = this.window; + // A system-menu close is the same action as the preview's hide button. + // Retain the native ID for capture exclusions and the renderer's size state. + previewWindow.on("close", (event) => { + event.preventDefault(); + this.hide(previewWindow.webContents); + }); previewWindow.once("closed", () => { if (this.window !== previewWindow) return; this.window = null; this.ready = false; this.visible = false; + this.owner?.send("camera-preview:hidden"); }); this.window.once("ready-to-show", () => { if (this.window !== previewWindow) return; @@ -65,12 +73,23 @@ export class CameraPreviewController { /** Linux portal monitor captures cannot exclude app windows. */ setCaptureAllowed(allowed: boolean): void { + this.captureGeneration++; this.captureAllowed = allowed; if (!allowed) this.window?.hide(); else if (this.visible && this.ready) this.window?.showInactive(); this.owner?.send("camera-preview:availability", allowed); } + /** A cancelled browser start may restore only its own preview suspension. */ + prepareCapture(): number { + this.setCaptureAllowed(false); + return this.captureGeneration; + } + + restoreCapture(token: unknown): void { + if (token === this.captureGeneration) this.setCaptureAllowed(true); + } + hide(sender: WebContents): void { if (sender !== this.window?.webContents) return; this.visible = false; @@ -120,11 +139,13 @@ export class CameraPreviewController { } this.onOwnerDestroyed = null; this.ready = false; - this.window?.destroy(); + const previewWindow = this.window; this.window = null; this.owner = null; + previewWindow?.destroy(); this.visible = false; this.captureAllowed = true; + this.captureGeneration++; this.width = CAMERA_PREVIEW_DEFAULT_SIZE; this.aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO; } diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 151eb90f04..f0994c063e 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -58,7 +58,8 @@ interface Window { usesSystemSourcePicker?: () => Promise; /** Name of the last source picked in Apple's picker, for display only: it is not selected. */ getLastPickedSource?: () => Promise; - prepareCameraPreviewCapture?: () => Promise; + prepareCameraPreviewCapture?: () => Promise; + restoreCameraPreviewCapture?: (token: number) => Promise; onCameraPreviewAvailability?: (callback: (available: boolean) => void) => () => void; configureCameraPreview?: (enabled: boolean, visible: boolean) => Promise; sendCameraPreviewFrame?: (jpeg: string, aspectRatio: number) => Promise; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index c7076e3435..7838799d93 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -2541,6 +2541,8 @@ export function registerIpcHandlers( createCameraPreviewWindow, resizeCameraPreviewWindow, ); + // destroy() bypasses the preview close-to-hide handler during application shutdown. + app.on("before-quit", () => cameraPreview.dispose()); function ensureMacCameraPreviewWindow() { const hud = getMainWindow(); if ( @@ -2562,7 +2564,17 @@ export function registerIpcHandlers( }); ipcMain.handle("camera-preview:prepare-capture", (event) => { if (process.platform === "linux" && event.sender === getMainWindow()?.webContents) { - cameraPreview.setCaptureAllowed(false); + return cameraPreview.prepareCapture(); + } + }); + ipcMain.handle("camera-preview:restore-capture", (event, token: unknown) => { + if ( + process.platform === "linux" && + event.sender === getMainWindow()?.webContents && + !appRecording && + !linuxNativeCaptureSession + ) { + cameraPreview.restoreCapture(token); } }); ipcMain.handle("camera-preview:frame", (event, jpeg: unknown, aspectRatio: unknown) => diff --git a/electron/preload.ts b/electron/preload.ts index c049128194..adaf464d6b 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -171,7 +171,9 @@ contextBridge.exposeInMainWorld("electronAPI", { return ipcRenderer.invoke("get-last-picked-source"); }, prepareCameraPreviewCapture: () => - ipcRenderer.invoke("camera-preview:prepare-capture") as Promise, + ipcRenderer.invoke("camera-preview:prepare-capture") as Promise, + restoreCameraPreviewCapture: (token: number) => + ipcRenderer.invoke("camera-preview:restore-capture", token) as Promise, onCameraPreviewAvailability: (callback: (available: boolean) => void) => { const listener = (_event: Electron.IpcRendererEvent, available: boolean) => callback(available); ipcRenderer.on("camera-preview:availability", listener); diff --git a/src/components/launch/FloatingCameraPreview.module.css b/src/components/launch/FloatingCameraPreview.module.css index ee65ec55e7..1d07938e74 100644 --- a/src/components/launch/FloatingCameraPreview.module.css +++ b/src/components/launch/FloatingCameraPreview.module.css @@ -36,13 +36,6 @@ background: #09090bbf; color: #f4f4f5; cursor: pointer; - opacity: 0; - transition: opacity 120ms ease; -} - -.preview:hover .hide, -.preview:focus-within .hide { - opacity: 1; } .hide:focus-visible { @@ -57,8 +50,6 @@ display: flex; gap: 4px; transform: translateX(-50%); - opacity: 0; - transition: opacity 120ms ease; } .sizeControls button { @@ -77,11 +68,6 @@ cursor: default; } -.preview:hover .sizeControls, -.preview:focus-within .sizeControls { - opacity: 1; -} - .sizeControls button:focus-visible { outline: 2px solid #10b981; } diff --git a/src/components/launch/HudControls.tsx b/src/components/launch/HudControls.tsx index 873c2db809..a48863c7e7 100644 --- a/src/components/launch/HudControls.tsx +++ b/src/components/launch/HudControls.tsx @@ -1,4 +1,12 @@ -import { Check, CircleArrowDown, Eye, EyeOff, Languages, NotepadText, Settings } from "lucide-react"; +import { + Check, + CircleArrowDown, + Eye, + EyeOff, + Languages, + NotepadText, + Settings, +} from "lucide-react"; import { createContext, memo, diff --git a/src/hooks/useScreenRecorder.restartFailure.test.tsx b/src/hooks/useScreenRecorder.restartFailure.test.tsx index 564936f41b..56d191cc98 100644 --- a/src/hooks/useScreenRecorder.restartFailure.test.tsx +++ b/src/hooks/useScreenRecorder.restartFailure.test.tsx @@ -2,9 +2,10 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, @@ -312,3 +313,93 @@ describe("useScreenRecorder Restart that does not start a new take", () => { expect(toast.error).not.toHaveBeenCalledWith("recording.restartFailed"); }); }); + +function stubWindowsCamera() { + stubElectronAPI("win32"); + api.getRecordingPrefs.mockResolvedValue({ + camEnabled: true, + camQuality: "1080p", + micEnabled: false, + systemAudioEnabled: false, + cursorCaptureMode: "editable-overlay", + }); + const track = { stop: vi.fn(), onended: null, getSettings: () => ({ deviceId: "camera-1" }) }; + const getUserMedia = vi.fn(async () => ({ + getTracks: () => [track], + getVideoTracks: () => [track], + })); + Object.defineProperty(navigator, "mediaDevices", { + configurable: true, + value: { getUserMedia }, + }); + return { getUserMedia, track }; +} + +describe("Windows camera handoff back to the idle preview", () => { + it("waits for Cancel to release the helper before reacquiring the enabled camera", async () => { + const { getUserMedia, track } = stubWindowsCamera(); + const view = renderHook(() => useScreenRecorder()); + await startTake(view); + expect(track.stop).toHaveBeenCalled(); + expect(view.result.current.webcamPreviewStream).toBeNull(); + const calls = getUserMedia.mock.calls.length; + let finishStop: ((result: { success: boolean; discarded: boolean }) => void) | undefined; + api.stopNativeWindowsRecording.mockImplementationOnce( + () => + new Promise((resolve) => { + finishStop = resolve; + }), + ); + await act(async () => view.result.current.cancelRecording()); + expect(getUserMedia).toHaveBeenCalledTimes(calls); + await act(async () => finishStop?.({ success: true, discarded: true })); + await settle(); + expect(view.result.current.recording).toBe(false); + expect(view.result.current.webcamEnabled).toBe(true); + expect(view.result.current.webcamPreviewStream).not.toBeNull(); + expect(getUserMedia).toHaveBeenCalledTimes(calls + 1); + }); + + it("does not open a competing renderer camera on Restart", async () => { + const { getUserMedia } = stubWindowsCamera(); + const view = renderHook(() => useScreenRecorder()); + await startTake(view); + const calls = getUserMedia.mock.calls.length; + await restart(view); + expect(view.result.current.recording).toBe(true); + expect(api.startNativeWindowsRecording).toHaveBeenCalledTimes(2); + expect(getUserMedia).toHaveBeenCalledTimes(calls); + expect(view.result.current.webcamPreviewStream).toBeNull(); + }); +}); + +describe("Linux fallback preview suspension", () => { + it("restores availability when capture resolves after its countdown was cancelled", async () => { + const getUserMedia = stubBrowserCapture(); + api.prepareCameraPreviewCapture = vi.fn(async () => 17); + api.restoreCameraPreviewCapture = vi.fn(async () => undefined); + const captured = await getUserMedia(); + let finishCapture: ((stream: typeof captured) => void) | undefined; + getUserMedia.mockImplementationOnce( + () => + new Promise((resolve) => { + finishCapture = resolve; + }), + ); + const view = renderHook(() => useScreenRecorder()); + await settle(); + await act(async () => view.result.current.toggleRecording()); + await settle(3500); + expect(api.prepareCameraPreviewCapture).toHaveBeenCalledOnce(); + expect(api.restoreCameraPreviewCapture).not.toHaveBeenCalled(); + // Replace the pending start with a new countdown, then cancel that countdown. + await act(async () => view.result.current.toggleRecording()); + await settle(); + await act(async () => view.result.current.cancelRecording()); + await act(async () => finishCapture?.(captured)); + await settle(); + expect(view.result.current.recording).toBe(false); + expect(api.restoreCameraPreviewCapture).toHaveBeenCalledWith(17); + expect(captured.getTracks()[0].stop).toHaveBeenCalled(); + }); +}); diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index fa8a0e6bee..6a69fb47bf 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -371,6 +371,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const screenStream = useRef(null); const microphoneStream = useRef(null); const webcamStream = useRef(null); + const [webcamPreviewGeneration, setWebcamPreviewGeneration] = useState(0); const [webcamPreviewStream, setWebcamPreviewStream] = useState(null); const mixingContext = useRef(null); const recordingId = useRef(0); @@ -514,6 +515,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { [t], ); + // biome-ignore lint/correctness/useExhaustiveDependencies: Cancel must reacquire the released Windows camera even when its settings are unchanged. useEffect(() => { if (!webcamEnabled) return; @@ -592,7 +594,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setWebcamPreviewStream(null); } }; - }, [webcamEnabled, webcamDeviceId, webcamDeviceName, webcamQuality, t]); + }, [webcamEnabled, webcamDeviceId, webcamDeviceName, webcamQuality, webcamPreviewGeneration, t]); const finalizeRecording = useCallback( ( @@ -1840,6 +1842,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const browserCursorCaptureMode = effectiveBrowserCursorMode(platform, cursorCaptureMode); let browserStartClaimed = false; let browserRecorderStarted = false; + let browserPreviewCaptureToken: number | undefined; try { if (!isCountdownRunActive(countdownRunToken)) { @@ -1901,7 +1904,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } // The browser fallback cannot exclude a Linux preview from a monitor stream. - if (platform === "linux") await window.electronAPI.prepareCameraPreviewCapture?.(); + if (platform === "linux") { + browserPreviewCaptureToken = await window.electronAPI.prepareCameraPreviewCapture?.(); + } // Capture screen + microphone in parallel: the gap between the two // `getUserMedia` calls is the dominant source of the mic-vs-video lag at the @@ -2215,6 +2220,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { console.warn("Failed to release the recording start:", error); }); } + if (browserPreviewCaptureToken !== undefined && !browserRecorderStarted) { + await window.electronAPI + .restoreCameraPreviewCapture?.(browserPreviewCaptureToken) + .catch((error) => { + console.warn("Failed to restore the idle camera preview:", error); + }); + } } }; @@ -2504,7 +2516,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const activeRecordingId = recordingId.current; discardRecordingId.current = activeRecordingId; allowAutoFinalize.current = false; - void finalizeNativeWindowsRecording(true); + void finalizeNativeWindowsRecording(true).then((stopped) => { + // Only Cancel returns to an idle camera. Restart hands it straight back + // to the helper, so it must not open a competing renderer session. + if (stopped) setWebcamPreviewGeneration((generation) => generation + 1); + }); return; } if (nativeMacRecording.current) { diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index c55784bcd5..06de5d3815 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -1081,3 +1081,4 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-19 | dev #1288 on `8c2608d9` + preview lifetime/quit fixes; Electron 41.2.1 | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — lifecycle slice passed** | Computer-use camera off/on removed/restored the HUD preview control. Normal Cmd+Q exited the OpenScreen Electron process with the preview enabled. Unit tests verify the same preview window survives initial camera-off and repeated toggles, with one owner-destruction listener and cleanup on HUD destruction. **Skipped:** recording, export, capture-exclusion measurements, other platforms and remaining checklist; no native capture changes in this slice. | | 2026-10-21 | dev #1288 on `20975d30` + synchronous preview reservation before capture exclusions; Electron 41.2.1; existing helper rebuilt from this branch on Oct 19 | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — capture-exclusion ordering slice passed** | Computer-use started native display recording with the preview hidden, showed it during the take and finalized into the editor. Helper diagnostics resolved both requested app window IDs and reported zero screen-writer drops; a decoded late screen frame contained no floating preview. Unit regression covers reserving the exclusion window before any renderer configuration and reusing it when enabled later without altering an existing preview’s visibility. **Skipped:** Apple’s picker (ordering path checked in code), forced renderer-effect race on desktop (covered by unit regression), other platforms, export, exhaustive frame-by-frame exclusion and remaining checklist. | | 2026-10-29 | Dev `codex/macos-floating-camera-preview`, rebased on `main` @ `5d82a0cb`, Windows/Linux preview port working tree; Electron 41.2.1; macOS ScreenCaptureKit helper rebuilt directly with `swiftc` and the macOS 26.5 SDK (SwiftPM unavailable because of a local BuildServerProtocol loader error) | macOS 26.7, Apple Silicon; built-in camera | **Partial — HUD controls only; capture/export evidence outstanding** | Real coordinate clicks through `mcp__cua_repl`: opened device settings, selected the built-in camera, toggled camera off/on, and verified the preview control disappears/reappears. Restored the original camera-on state and quit the dev app. The tool bound only the HUD window; it did not expose the separate camera window, so live preview pixels were not visually verified. Native preview bitmap conversion tests passed locally; publisher/controller tests cover camera handoff, blocked monitor captures, cleanup and bounded frame delivery. **Skipped:** recording, export, capture-exclusion proof and the requested demonstration clips, because the preview could not be inspected in this desktop session; real Windows/Linux runs and the complete Windows helper build, because those operating systems are unavailable here; all remaining checklist sections. Linux monitor/virtual/unknown/browser-fallback capture deliberately hides the preview because the portal has no window-exclusion API; only native window capture permits it during a take. This row is not release-promotion QA. | +| 2026-11-03 | PR #1288 dev branch rebased onto `main` @ `5e87ff39`, head `fe1122d3` plus review fixes; Electron dev launch with `OPENSCREEN_DISABLE_CONTENT_PROTECTION=1`, legacy source picker | macOS, Apple Silicon | **Partial — HUD toggle and quit passed; preview-window validation incomplete** | Real OS clicks through `mcp__cua_repl`: camera already enabled; Hide camera preview changed the HUD to Show camera preview; one click restored Hide camera preview. Cmd+Q exited the app with the preview enabled, without the close-to-hide handler blocking shutdown. **Skipped:** independent inspection of the separate preview window, resize controls and drag behavior (the computer-use app surface exposed only the HUD); real recording/exclusion/export and demo clips (not exercised in this slice); Windows/Linux desktop runs (no machines available). Unit regressions cover Windows cancel waiting for helper release, Restart avoiding renderer camera acquisition, system-menu close retaining the window/size and notifying the HUD, and Linux cancelled-start suspension restoration with stale-token protection. This row does not claim those platform paths were desktop-tested. |