diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts new file mode 100644 index 000000000..5e4660d0a --- /dev/null +++ b/electron/cameraPreview.test.ts @@ -0,0 +1,276 @@ +import { EventEmitter } from "node:events"; +import type { BrowserWindow, WebContents } from "electron"; +import { describe, expect, it, vi } from "vitest"; +import { collectMacCaptureExcludedWindowIds } from "../src/lib/nativeMacRecording"; +import { CameraPreviewController } from "./cameraPreview"; + +function setup() { + const owner = Object.assign(new EventEmitter(), { send: vi.fn() }); + const overlay = Object.assign(new EventEmitter(), { + getMediaSourceId: () => "window:123:0", + webContents: { send: vi.fn() }, + showInactive: vi.fn(), + hide: vi.fn(), + destroy: vi.fn(() => { + overlay.emit("closed"); + }), + }); + const create = vi.fn(() => overlay as unknown as BrowserWindow); + const resize = vi.fn(); + const preview = new CameraPreviewController(create, resize); + const sender = owner as unknown as WebContents; + const overlaySender = overlay.webContents as unknown as WebContents; + return { owner, overlay, create, resize, preview, sender, overlaySender }; +} + +const jpeg = "data:image/jpeg;base64,/9j/"; + +describe("floating camera preview lifecycle", () => { + it("reserves the same exclusion ID before the renderer enables the preview", () => { + const { preview, sender, overlay, create } = setup(); + const reserved = preview.ensure(sender); + const excludedIds = collectMacCaptureExcludedWindowIds([reserved!.getMediaSourceId()]); + expect(excludedIds).toContain(123); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).not.toHaveBeenCalled(); + preview.configure(sender, true, true); + expect(preview.ensure(sender)).toBe(reserved); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + expect(excludedIds).toContain(123); + }); + + it("ensuring an existing preview preserves visibility and refuses a different owner", () => { + const { preview, sender, overlay, create } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + expect(preview.ensure(sender)).toBe(overlay); + expect(preview.ensure({} as WebContents)).toBeNull(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + expect(overlay.hide).not.toHaveBeenCalled(); + expect(create).toHaveBeenCalledOnce(); + }); + + it("waits for the window to paint, then shows without taking focus", () => { + const { preview, sender, overlay } = setup(); + preview.configure(sender, true, true); + expect(overlay.showInactive).not.toHaveBeenCalled(); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + }); + + it("hides and restores the same window, retaining its capture exclusion id", () => { + const { preview, sender, overlaySender, overlay, owner, create } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + preview.hide(overlaySender); + expect(owner.send).toHaveBeenCalledWith("camera-preview:hidden"); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.destroy).not.toHaveBeenCalled(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + }); + + it("does not flash a preview disabled before its renderer is ready", () => { + const { preview, sender, overlay } = setup(); + preview.configure(sender, true, true); + preview.configure(sender, true, false); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).not.toHaveBeenCalled(); + }); + + it("ignores other renderers and rejects oversized or non-JPEG frames", () => { + const { preview, sender, overlay, create } = setup(); + const stranger = {} as WebContents; + preview.configure(sender, true, true); + preview.configure(stranger, false, false); + preview.hide(stranger); + for (const invalid of [undefined, "https://example.com/camera", jpeg + "x".repeat(128_000)]) { + preview.frame(sender, invalid); + } + preview.frame(stranger, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + expect(overlay.destroy).not.toHaveBeenCalled(); + expect(create).toHaveBeenCalledOnce(); + }); + + it("allows only the preview to resize to a supported width without replacing its window", () => { + const { preview, sender, overlaySender, resize, create, overlay } = setup(); + preview.configure(sender, true, true); + preview.resize(sender, 320); + preview.resize(overlaySender, Number.NaN); + preview.resize(overlaySender, 10000); + expect(resize).not.toHaveBeenCalled(); + preview.resize(overlaySender, 320); + expect(resize).toHaveBeenCalledWith(overlay, 320, 16 / 9); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.destroy).not.toHaveBeenCalled(); + }); + + it("sizes the same window to the delivered frame and preserves that ratio on resize", () => { + const { preview, sender, overlaySender, resize, overlay } = setup(); + preview.configure(sender, true, true); + preview.frame(sender, jpeg, 9 / 16); + expect(resize).toHaveBeenLastCalledWith(overlay, 224, 9 / 16); + preview.resize(overlaySender, 320); + expect(resize).toHaveBeenLastCalledWith(overlay, 320, 9 / 16); + const calls = resize.mock.calls.length; + for (const ratio of [0, -1, Number.NaN, Number.POSITIVE_INFINITY, 100, "16:9"]) { + preview.frame(sender, jpeg, ratio); + } + expect(resize).toHaveBeenCalledTimes(calls); + }); + + it("recreates an unexpectedly closed preview and waits for the replacement to be ready", () => { + const { preview, sender, overlay, create, owner } = setup(); + const replacement = setup().overlay; + create + .mockReturnValueOnce(overlay as unknown as BrowserWindow) + .mockReturnValueOnce(replacement as unknown as BrowserWindow); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + overlay.emit("closed"); + expect(owner.send).toHaveBeenLastCalledWith("camera-preview:hidden"); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledTimes(2); + expect(replacement.showInactive).not.toHaveBeenCalled(); + replacement.emit("ready-to-show"); + expect(replacement.showInactive).toHaveBeenCalledOnce(); + }); + + it("treats system-menu Close as hide, preserving size and capture exclusion", () => { + const { preview, sender, overlay, overlaySender, owner, create, resize } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + preview.resize(overlaySender, 320); + const event = { preventDefault: vi.fn() }; + overlay.emit("close", event); + expect(event.preventDefault).toHaveBeenCalledOnce(); + expect(overlay.hide).toHaveBeenCalledOnce(); + expect(owner.send).toHaveBeenLastCalledWith("camera-preview:hidden"); + preview.configure(sender, true, true); + expect(preview.ensure(sender)).toBe(overlay); + expect(create).toHaveBeenCalledOnce(); + expect(resize).toHaveBeenLastCalledWith(overlay, 320, 16 / 9); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + expect(overlay.destroy).not.toHaveBeenCalled(); + // Shutdown uses destroy(), which bypasses the cancellable close event. + preview.dispose(); + expect(overlay.destroy).toHaveBeenCalledOnce(); + }); + + it("ignores a stale closed event from a previously disposed preview", () => { + const { preview, sender, overlay, create } = setup(); + // Delay closure to exercise a late event after the replacement already exists. + overlay.destroy.mockImplementationOnce(() => undefined); + const replacement = setup().overlay; + create + .mockReturnValueOnce(overlay as unknown as BrowserWindow) + .mockReturnValueOnce(replacement as unknown as BrowserWindow); + preview.configure(sender, true, true); + preview.dispose(); + preview.configure(sender, true, true); + replacement.emit("ready-to-show"); + + overlay.emit("closed"); + preview.frame(sender, jpeg); + expect(replacement.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledTimes(2); + expect(replacement.showInactive).toHaveBeenCalledTimes(2); + }); + + it("destroys the auxiliary window when the HUD is destroyed", () => { + const { preview, sender, owner, overlay } = setup(); + preview.configure(sender, true, true); + owner.send.mockClear(); + owner.emit("destroyed"); + expect(overlay.destroy).toHaveBeenCalledOnce(); + expect(owner.send).not.toHaveBeenCalled(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + }); + + it("creates a hidden exclusion window even when the camera starts off", () => { + const { preview, sender, owner, overlay, create } = setup(); + preview.configure(sender, false, true); + overlay.emit("ready-to-show"); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.hide).toHaveBeenCalledOnce(); + expect(overlay.showInactive).not.toHaveBeenCalled(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + owner.emit("destroyed"); + expect(overlay.destroy).toHaveBeenCalledOnce(); + }); + + it("retains the exclusion window and one owner listener across camera toggles", () => { + const { preview, sender, owner, overlay, create } = setup(); + for (let i = 0; i < 15; i++) { + preview.configure(sender, true, true); + if (i === 0) overlay.emit("ready-to-show"); + preview.configure(sender, false, true); + } + expect(create).toHaveBeenCalledOnce(); + expect(overlay.hide).toHaveBeenCalledTimes(15); + expect(overlay.showInactive).toHaveBeenCalledTimes(15); + expect(overlay.destroy).not.toHaveBeenCalled(); + expect(owner.listenerCount("destroyed")).toBe(1); + owner.emit("destroyed"); + expect(owner.listenerCount("destroyed")).toBe(0); + expect(overlay.destroy).toHaveBeenCalledOnce(); + }); + it("blocks preview display and frames during a monitor take even across HUD toggles", () => { + const { preview, sender, overlay, owner } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + preview.setCaptureAllowed(false); + expect(owner.send).toHaveBeenLastCalledWith("camera-preview:availability", false); + preview.configure(sender, true, true); + preview.frame(sender, jpeg); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + preview.setCaptureAllowed(true); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + }); + + it("restores a cancelled start without restoring an older or active capture", () => { + const { preview, sender, overlay, owner } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + const oldToken = preview.prepareCapture(); + const newToken = preview.prepareCapture(); + preview.restoreCapture(oldToken); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + preview.restoreCapture(newToken); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + expect(owner.send).toHaveBeenLastCalledWith("camera-preview:availability", true); + const token = preview.prepareCapture(); + preview.setCaptureAllowed(false); // A newer native capture owns the suspension. + preview.restoreCapture(token); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + }); + + it("does not flash a blocked preview on ready or restore one the user hid", () => { + const { preview, sender, overlay, overlaySender } = setup(); + preview.setCaptureAllowed(false); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).not.toHaveBeenCalled(); + preview.hide(overlaySender); + preview.setCaptureAllowed(true); + expect(overlay.showInactive).not.toHaveBeenCalled(); + }); +}); diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts new file mode 100644 index 000000000..8b477072d --- /dev/null +++ b/electron/cameraPreview.ts @@ -0,0 +1,152 @@ +import type { BrowserWindow, WebContents } from "electron"; +import { + CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO, + CAMERA_PREVIEW_DEFAULT_SIZE, + CAMERA_PREVIEW_SIZES, +} from "../src/lib/cameraPreview"; + +/** One stable window per HUD: hiding it must not change the native capture exclusion id. */ +export class CameraPreviewController { + private window: BrowserWindow | null = null; + private owner: WebContents | null = null; + private visible = false; + private captureAllowed = true; + private captureGeneration = 0; + private width: number = CAMERA_PREVIEW_DEFAULT_SIZE; + private aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO; + private ready = false; + private onOwnerDestroyed: (() => void) | null = null; + + constructor( + private readonly createWindow: () => BrowserWindow, + private readonly resizeWindow: ( + window: BrowserWindow, + width: number, + aspectRatio: number, + ) => void, + ) {} + + /** Reserve the capture-exclusion window before renderer configuration arrives. */ + ensure(owner: WebContents): BrowserWindow | null { + if (this.owner && this.owner !== owner) return null; + if (!this.owner) { + this.owner = owner; + owner.send("camera-preview:availability", this.captureAllowed); + this.onOwnerDestroyed = () => { + if (this.owner === owner) this.dispose(); + }; + owner.once("destroyed", this.onOwnerDestroyed); + } + if (!this.window) { + this.window = this.createWindow(); + const previewWindow = this.window; + // A system-menu close is the same action as the preview's hide button. + // Retain the native ID for capture exclusions and the renderer's size state. + previewWindow.on("close", (event) => { + event.preventDefault(); + this.hide(previewWindow.webContents); + }); + previewWindow.once("closed", () => { + if (this.window !== previewWindow) return; + this.window = null; + this.ready = false; + this.visible = false; + this.owner?.send("camera-preview:hidden"); + }); + this.window.once("ready-to-show", () => { + if (this.window !== previewWindow) return; + this.ready = true; + this.resizeWindow(previewWindow, this.width, this.aspectRatio); + if (this.visible && this.captureAllowed) this.window?.showInactive(); + }); + } + return this.window; + } + + configure(owner: WebContents, enabled: boolean, visible: boolean): void { + const previewWindow = this.ensure(owner); + if (!previewWindow) return; + this.visible = enabled && visible; + if (this.visible && this.captureAllowed && this.ready) previewWindow.showInactive(); + else if (!this.visible || !this.captureAllowed) previewWindow.hide(); + } + + /** Linux portal monitor captures cannot exclude app windows. */ + setCaptureAllowed(allowed: boolean): void { + this.captureGeneration++; + this.captureAllowed = allowed; + if (!allowed) this.window?.hide(); + else if (this.visible && this.ready) this.window?.showInactive(); + this.owner?.send("camera-preview:availability", allowed); + } + + /** A cancelled browser start may restore only its own preview suspension. */ + prepareCapture(): number { + this.setCaptureAllowed(false); + return this.captureGeneration; + } + + restoreCapture(token: unknown): void { + if (token === this.captureGeneration) this.setCaptureAllowed(true); + } + + hide(sender: WebContents): void { + if (sender !== this.window?.webContents) return; + this.visible = false; + this.window.hide(); + this.owner?.send("camera-preview:hidden"); + } + + resize(sender: WebContents, width: unknown): void { + if ( + sender !== this.window?.webContents || + typeof width !== "number" || + !CAMERA_PREVIEW_SIZES.some((size) => size === width) + ) + return; + this.width = width; + this.resizeWindow(this.window, this.width, this.aspectRatio); + } + + frame(sender: WebContents, jpeg: unknown, aspectRatio?: unknown): void { + if (sender !== this.owner || !this.visible || !this.captureAllowed || !this.window) return; + // A 320px JPEG is usually under 20KB. Never relay arbitrary URLs or unbounded IPC data. + if ( + typeof jpeg !== "string" || + jpeg.length > 128_000 || + !jpeg.startsWith("data:image/jpeg;base64,") + ) + return; + if (aspectRatio !== undefined) { + if ( + typeof aspectRatio !== "number" || + !Number.isFinite(aspectRatio) || + aspectRatio < 1 / 8 || + aspectRatio > 8 + ) + return; + if (this.aspectRatio !== aspectRatio) { + this.aspectRatio = aspectRatio; + this.resizeWindow(this.window, this.width, aspectRatio); + } + } + this.window.webContents.send("camera-preview:frame", jpeg); + } + + dispose(): void { + if (this.owner && this.onOwnerDestroyed) { + this.owner.removeListener("destroyed", this.onOwnerDestroyed); + } + this.onOwnerDestroyed = null; + this.ready = false; + const previewWindow = this.window; + this.window = null; + this.owner = null; + previewWindow?.destroy(); + this.visible = false; + this.captureAllowed = true; + this.captureGeneration++; + this.width = CAMERA_PREVIEW_DEFAULT_SIZE; + this.aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO; + } +} diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 9a4095c19..f0994c063 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -58,6 +58,15 @@ interface Window { usesSystemSourcePicker?: () => Promise; /** Name of the last source picked in Apple's picker, for display only: it is not selected. */ getLastPickedSource?: () => Promise; + prepareCameraPreviewCapture?: () => Promise; + restoreCameraPreviewCapture?: (token: number) => Promise; + onCameraPreviewAvailability?: (callback: (available: boolean) => void) => () => void; + configureCameraPreview?: (enabled: boolean, visible: boolean) => Promise; + sendCameraPreviewFrame?: (jpeg: string, aspectRatio: number) => Promise; + resizeCameraPreview?: (width: number) => Promise; + hideCameraPreview?: () => Promise; + onCameraPreviewFrame?: (callback: (jpeg: string) => void) => () => void; + onCameraPreviewHidden?: (callback: () => void) => () => void; openNotes: () => Promise<{ opened: boolean; reason?: string; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index b54ffe87a..7838799d9 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -70,6 +70,7 @@ import { DocumentService } from "../ai-edition/document-service"; import { LlmConfigStore } from "../ai-edition/llm-config-store"; import { StylePresetService } from "../ai-edition/style-preset-service"; import { AppSettingsStore } from "../app-settings"; +import { CameraPreviewController } from "../cameraPreview"; import { createRunnerHost } from "../cli/runnerHost"; import { hideCountdownOverlay } from "../countdownOverlay"; import { isDiagnosticModeEnabled, mainLogFile } from "../diagnostics/main-log-file"; @@ -169,12 +170,21 @@ import { shouldPersistSelectedSource, } from "../recording-source-settings"; import { getSttManager } from "../stt"; -import { createRunnerWindow } from "../windows"; +import { + createCameraPreviewWindow, + createRunnerWindow, + getCameraPreviewWindow, + resizeCameraPreviewWindow, +} from "../windows"; import { assertWindowsUpdateIdle, isWindowsUpdatePending, WINDOWS_UPDATE_PENDING_MESSAGE, } from "../windows-update"; +import { + createWindowsCameraPreviewMailbox, + watchWindowsCameraPreview, +} from "../windowsCameraPreview"; import { getSourcesByType, shouldEnumerateSourceTypesSeparately } from "./desktopSourceTypes"; import { registerNativeBridgeHandlers } from "./nativeBridge"; import { createNativeMacMidCaptureErrorWatch } from "./nativeMacMidCaptureErrorWatch"; @@ -2013,6 +2023,7 @@ export function registerIpcHandlers( */ const setAppRecording = (recording: boolean, sourceName: string, preparing = false) => { appRecording = recording; + if (!recording && !preparing) cameraPreview.setCaptureAllowed(true); if (!preparing) mcpAppRecorder.recordingChanged(recording); onRecordingStateChange?.(recording, sourceName, preparing); }; @@ -2186,7 +2197,8 @@ export function registerIpcHandlers( * pick would record them unless the picker is told to leave them out. */ function macPickerExcludedWindowIds() { - const appWindowSourceIds = [getMainWindow(), getNotesWindow()] + ensureMacCameraPreviewWindow(); + const appWindowSourceIds = [getMainWindow(), getNotesWindow(), getCameraPreviewWindow()] .filter((window): window is BrowserWindow => !!window && !window.isDestroyed()) .map((window) => window.getMediaSourceId()); return collectMacCaptureExcludedWindowIds(appWindowSourceIds); @@ -2525,6 +2537,54 @@ export function registerIpcHandlers( return { opened: true }; }); + const cameraPreview = new CameraPreviewController( + createCameraPreviewWindow, + resizeCameraPreviewWindow, + ); + // destroy() bypasses the preview close-to-hide handler during application shutdown. + app.on("before-quit", () => cameraPreview.dispose()); + function ensureMacCameraPreviewWindow() { + const hud = getMainWindow(); + if ( + process.platform === "darwin" && + hud && + !hud.isDestroyed() && + new URL(hud.webContents.getURL()).searchParams.get("windowType") === "hud-overlay" + ) { + cameraPreview.ensure(hud.webContents); + } + } + ipcMain.handle("camera-preview:configure", (event, enabled: unknown, visible: unknown) => { + if ( + event.sender !== getMainWindow()?.webContents || + new URL(event.sender.getURL()).searchParams.get("windowType") !== "hud-overlay" + ) + return; + cameraPreview.configure(event.sender, enabled === true, visible === true); + }); + ipcMain.handle("camera-preview:prepare-capture", (event) => { + if (process.platform === "linux" && event.sender === getMainWindow()?.webContents) { + return cameraPreview.prepareCapture(); + } + }); + ipcMain.handle("camera-preview:restore-capture", (event, token: unknown) => { + if ( + process.platform === "linux" && + event.sender === getMainWindow()?.webContents && + !appRecording && + !linuxNativeCaptureSession + ) { + cameraPreview.restoreCapture(token); + } + }); + ipcMain.handle("camera-preview:frame", (event, jpeg: unknown, aspectRatio: unknown) => + cameraPreview.frame(event.sender, jpeg, aspectRatio), + ); + ipcMain.handle("camera-preview:hide", (event) => cameraPreview.hide(event.sender)); + ipcMain.handle("camera-preview:resize", (event, diameter: unknown) => + cameraPreview.resize(event.sender, diameter), + ); + ipcMain.handle("open-notes", async () => { const notesSelectorWin = getNotesWindow(); if (notesSelectorWin) { @@ -2716,6 +2776,7 @@ export function registerIpcHandlers( }); lastLinuxCaptureSession = session; + cameraPreview.setCaptureAllowed(false); await session.start(); // The picker is up now. No timeout: a human is reading a dialog. await session.waitUntilSourceSelected(); @@ -2729,6 +2790,7 @@ export function registerIpcHandlers( } preparingLinuxCaptureToken = null; + cameraPreview.setCaptureAllowed(session.grantedSourceKind === "window"); preparedLinuxCapture = { session, outputPath, request }; return { success: true, @@ -2740,6 +2802,7 @@ export function registerIpcHandlers( // The portal may have come or gone since the probe. resetLinuxCaptureBackendProbe(); discardPreparedLinuxCapture("prepare failed"); + cameraPreview.setCaptureAllowed(true); return { success: false, error: String(error) }; } }, @@ -2748,6 +2811,7 @@ export function registerIpcHandlers( /** Drops a prepared session, e.g. when the countdown was cancelled. */ ipcMain.handle("cancel-native-linux-prepare", async () => { discardPreparedLinuxCapture("cancelled by the renderer"); + if (!linuxNativeCaptureSession && !appRecording) cameraPreview.setCaptureAllowed(true); return { success: true }; }); @@ -2813,11 +2877,13 @@ export function registerIpcHandlers( }); if (!prepared) { + cameraPreview.setCaptureAllowed(false); await session.start(); // Blocks until the user answers the portal picker, which has no // upper bound. On this path the countdown has already run. await session.waitUntilSourceSelected(); } + cameraPreview.setCaptureAllowed(session.grantedSourceKind === "window"); // Idempotent, and a no-op for a session that was not deferred. session.arm(); await session.waitUntilCapturing(); @@ -2840,6 +2906,7 @@ export function registerIpcHandlers( } catch (error) { console.error("Failed to start native Linux recording:", error); resetLinuxCaptureBackendProbe(); + cameraPreview.setCaptureAllowed(true); linuxNativeCaptureSession = null; linuxNativeCaptureRecordingId = null; linuxNativeCaptureCursorMode = "editable-overlay"; @@ -2956,6 +3023,7 @@ export function registerIpcHandlers( return { success: false, error: WINDOWS_UPDATE_PENDING_MESSAGE }; } let startReserved = false; + let mailbox: Awaited> | undefined; try { if (!isWindowsGraphicsCaptureOsSupported()) { return { @@ -3015,7 +3083,10 @@ export function registerIpcHandlers( request.preferSoftwareEncoder === true || envPreferSoftwareEncoder === "true" || envPreferSoftwareEncoder === "1"; + if (request.webcam.enabled) + mailbox = await createWindowsCameraPreviewMailbox(app.getPath("temp")); const config = { + webcamPreviewPath: mailbox?.file, schemaVersion: 2, recordingId, preferSoftwareEncoder, @@ -3121,6 +3192,19 @@ export function registerIpcHandlers( stdio: ["pipe", "pipe", "pipe"], windowsHide: true, }); + if (mailbox) { + const activeMailbox = mailbox; + const hud = getMainWindow(); + const stopPreview = watchWindowsCameraPreview(activeMailbox.file, (jpeg, ratio) => { + if (hud && !hud.isDestroyed()) cameraPreview.frame(hud.webContents, jpeg, ratio); + }); + proc.once("close", () => { + stopPreview(); + void activeMailbox + .remove() + .catch((error: unknown) => console.warn("Could not remove camera preview:", error)); + }); + } nativeWindowsCaptureProcess = proc; nativeWindowsCaptureDrainCleanup = attachNativeWindowsCaptureOutputDrain(proc); console.info("[native-wgc] helper spawned", { pid: proc.pid }); @@ -3188,6 +3272,7 @@ export function registerIpcHandlers( microphoneDefaulted, }; } catch (error) { + if (mailbox && !nativeWindowsCaptureProcess) await mailbox.remove().catch(() => undefined); console.error("Failed to start native Windows recording:", error); if (!startReserved) return { success: false, error: String(error) }; const failedProc = nativeWindowsCaptureProcess; @@ -3241,6 +3326,7 @@ export function registerIpcHandlers( normalizeCursorCaptureMode(request.cursor?.mode) ?? "editable-overlay"; // A source from Apple's picker records through the session that holds the pick, // and needs no Screen Recording grant -- so nothing here may go near one. + ensureMacCameraPreviewWindow(); const pickerSession = isMacPickerSourceId(request.source.sourceId) ? macPickerSession : null; // Normally caught by `get-selected-source` first; this is the last word (#965). pickerSession?.forgetSelectionUnlessExcluding(macPickerExcludedWindowIds()); @@ -3288,7 +3374,7 @@ export function registerIpcHandlers( pick?.bounds ?? request.source.bounds ?? sourceDisplay?.bounds ?? getSelectedSourceBounds(); const captureExcludedWindowSourceIds: string[] = []; if (request.source.type === "display") { - for (const window of [getMainWindow(), getNotesWindow()]) { + for (const window of [getMainWindow(), getNotesWindow(), getCameraPreviewWindow()]) { if (window && !window.isDestroyed()) { captureExcludedWindowSourceIds.push(window.getMediaSourceId()); } diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift index 4d47d274d..039b2484d 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift @@ -217,9 +217,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { if let picked { target = makeCaptureTarget(picked: picked) } else { + // Keep hidden recording overlays in the filter, so showing a camera preview + // during the take never exposes it while the exclusion-refresh poll catches up. let content = try await SCShareableContent.excludingDesktopWindows( false, - onScreenWindowsOnly: true + onScreenWindowsOnly: false ) target = try makeCaptureTarget(from: content) if recordsSystemAudioFromDisplay, let display = content.displays.first { @@ -700,7 +702,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { guard let self, let stream = self.stream, let content = try? await SCShareableContent.excludingDesktopWindows( false, - onScreenWindowsOnly: true + onScreenWindowsOnly: false ), let display = content.displays.first(where: { $0.displayID == displayID }) else { diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt index 87eaf2af9..0ad32e922 100644 --- a/electron/native/wgc-capture/CMakeLists.txt +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -65,6 +65,8 @@ add_executable(wgc-capture src/webcam_format.cpp src/webcam_format.h src/webcam_capture.h + src/webcam_preview.h + src/webcam_preview_bitmap.h src/wgc_session.cpp src/wgc_session.h ) @@ -216,3 +218,9 @@ target_link_libraries(mf_encoder_color_test PRIVATE ole32 propsys ) + +add_executable(webcam_preview_bitmap_test + src/webcam_preview_bitmap.h + src/webcam_preview_bitmap_test.cpp +) +target_compile_options(webcam_preview_bitmap_test PRIVATE /EHsc /W4 /utf-8) diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index 56987ac0f..45c08c42d 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -10,6 +10,7 @@ #include "wasapi_render_keepalive.h" #include "frame_visibility.h" #include "webcam_capture.h" +#include "webcam_preview.h" #include "capture_session.h" #include @@ -39,6 +40,7 @@ struct CaptureConfig { std::string windowHandle; std::string outputPath; std::string webcamOutputPath; + std::string webcamPreviewPath; int fps = 60; int width = 0; int height = 0; @@ -587,6 +589,7 @@ bool parseConfig(const std::string& json, CaptureConfig& config) { config.webcamDeviceName = findString(json, "webcamDeviceName"); config.webcamDirectShowClsid = findString(json, "webcamDirectShowClsid"); config.webcamOutputPath = findString(json, "webcamPath"); + config.webcamPreviewPath = findString(json, "webcamPreviewPath"); config.webcamWidth = findInt(json, "webcamWidth", 0); config.webcamHeight = findInt(json, "webcamHeight", 0); config.webcamFps = findInt(json, "webcamFps", 0); @@ -1498,6 +1501,8 @@ int wmain(int argc, wchar_t* argv[]) { } } + WebcamPreview webcamPreview(webcamCapture, config.webcamEnabled ? utf8ToWide(config.webcamPreviewPath) : L""); + if (!session.start()) { webcamCapture.stop(); microphoneCapture.stop(); diff --git a/electron/native/wgc-capture/src/webcam_preview.h b/electron/native/wgc-capture/src/webcam_preview.h new file mode 100644 index 000000000..360324b3a --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_preview.h @@ -0,0 +1,54 @@ +#pragma once + +#include "webcam_capture.h" +#include "webcam_preview_bitmap.h" +#include +#include +#include + +// Latest-frame mailbox, replaced atomically. A slow UI never blocks the camera +// or encoder, and no frame queue grows during a long take. +class WebcamPreview { +public: + WebcamPreview(WebcamCapture& capture, const std::wstring& path) { + if (path.empty()) return; + thread_ = std::thread([this, &capture, path] { + const std::filesystem::path target(path); + const std::filesystem::path pending(path + L".tmp"); + uint64_t sequence = 0; + std::unique_lock lock(mutex_); + while (!stopped_) { + lock.unlock(); + WebcamFrameSnapshot frame; + if (capture.copyLatestFrame(frame, sequence)) { + sequence = frame.sequence; + const auto bmp = webcamPreviewBitmap(frame.data, frame.width, frame.height, capture.deliversNv12()); + if (!bmp.empty()) { + std::ofstream out(pending, std::ios::binary | std::ios::trunc); + out.write(reinterpret_cast(bmp.data()), static_cast(bmp.size())); + out.close(); + if (out) MoveFileExW(pending.c_str(), target.c_str(), MOVEFILE_REPLACE_EXISTING); + } + } + lock.lock(); + cv_.wait_for(lock, std::chrono::milliseconds(200), [this] { return stopped_; }); + } + lock.unlock(); + std::error_code ignored; + std::filesystem::remove(pending, ignored); + std::filesystem::remove(target, ignored); + }); + } + ~WebcamPreview() { + { std::lock_guard lock(mutex_); stopped_ = true; } + cv_.notify_all(); + if (thread_.joinable()) thread_.join(); + } + WebcamPreview(const WebcamPreview&) = delete; + WebcamPreview& operator=(const WebcamPreview&) = delete; +private: + std::mutex mutex_; + std::condition_variable cv_; + bool stopped_ = false; + std::thread thread_; +}; diff --git a/electron/native/wgc-capture/src/webcam_preview_bitmap.h b/electron/native/wgc-capture/src/webcam_preview_bitmap.h new file mode 100644 index 000000000..940128aeb --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_preview_bitmap.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include + +// A bounded, top-down BMP for the presenter preview. Only these small pixels cross +// the process boundary; the native camera and recording clocks remain untouched. +inline std::vector webcamPreviewBitmap( + std::span frame, int width, int height, bool nv12) { + if (width < 2 || height < 2 || width > 16384 || height > 16384 || + (nv12 && (width % 2 || height % 2))) return {}; + const size_t required = static_cast(width) * height * (nv12 ? 3 : 8) / 2; + if (frame.size() < required) return {}; + const int outWidth = std::min(320, width); + const int outHeight = std::clamp(height * outWidth / width, 1, 2560); + const uint32_t bytes = outWidth * outHeight * 4; + std::vector bmp(54 + bytes, 0); + const auto put = [&](size_t at, uint32_t value, int count = 4) { + for (int i = 0; i < count; ++i) bmp[at + i] = static_cast(value >> (8 * i)); + }; + bmp[0] = 'B'; bmp[1] = 'M'; + put(2, static_cast(bmp.size())); put(10, 54); put(14, 40); + put(18, outWidth); put(22, static_cast(-outHeight)); + put(26, 1, 2); put(28, 32, 2); put(34, bytes); + for (int y = 0; y < outHeight; ++y) { + const int sy = y * height / outHeight; + for (int x = 0; x < outWidth; ++x) { + const int sx = x * width / outWidth; + const size_t to = 54 + (static_cast(y) * outWidth + x) * 4; + if (nv12) { + const int yy = std::max(0, static_cast(frame[sy * width + sx]) - 16); + const size_t uv = static_cast(width) * height + (sy / 2) * width + (sx & ~1); + const int u = static_cast(frame[uv]) - 128; + const int v = static_cast(frame[uv + 1]) - 128; + bmp[to] = static_cast(std::clamp((298 * yy + 516 * u + 128) >> 8, 0, 255)); + bmp[to + 1] = static_cast(std::clamp((298 * yy - 100 * u - 208 * v + 128) >> 8, 0, 255)); + bmp[to + 2] = static_cast(std::clamp((298 * yy + 409 * v + 128) >> 8, 0, 255)); + } else { + const size_t from = (static_cast(sy) * width + sx) * 4; + std::copy_n(frame.begin() + from, 3, bmp.begin() + to); + } + bmp[to + 3] = 255; + } + } + return bmp; +} diff --git a/electron/native/wgc-capture/src/webcam_preview_bitmap_test.cpp b/electron/native/wgc-capture/src/webcam_preview_bitmap_test.cpp new file mode 100644 index 000000000..db9841beb --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_preview_bitmap_test.cpp @@ -0,0 +1,30 @@ +#include "webcam_preview_bitmap.h" +#include + +int main() { + int failures = 0; + const auto check = [&](bool pass) { if (!pass) ++failures; }; + std::vector bgra = {1, 2, 3, 0, 4, 5, 6, 0, 7, 8, 9, 0, 10, 11, 12, 0}; + const auto bmp = webcamPreviewBitmap(bgra, 2, 2, false); + check(bmp.size() == 70 && bmp[0] == 'B' && bmp[1] == 'M'); + check(bmp[18] == 2 && bmp[22] == 254 && bmp[23] == 255); // top-down + check(bmp[54] == 1 && bmp[56] == 3 && bmp[57] == 255 && bmp[62] == 7); + const std::vector black = {16, 16, 16, 16, 128, 128}; + const auto dark = webcamPreviewBitmap(black, 2, 2, true); + check(dark[54] == 0 && dark[55] == 0 && dark[56] == 0 && dark[57] == 255); + const std::vector white = {235, 235, 235, 235, 128, 128}; + const auto light = webcamPreviewBitmap(white, 2, 2, true); + check(light[54] == 255 && light[55] == 255 && light[56] == 255); + const std::vector red = {81, 81, 81, 81, 90, 240}; + const auto color = webcamPreviewBitmap(red, 2, 2, true); + check(color[54] < 3 && color[55] < 3 && color[56] > 250); + check(webcamPreviewBitmap({}, 2, 2, false).empty()); + check(webcamPreviewBitmap(bgra, 3, 2, true).empty()); + check(webcamPreviewBitmap(bgra, -1, 2, false).empty()); + check(webcamPreviewBitmap(bgra, 16385, 2, false).empty()); + const std::vector large(640 * 480 * 4, 32); + const auto small = webcamPreviewBitmap(large, 640, 480, false); + check(small.size() == 54 + 320 * 240 * 4 && small[54] == 32 && small[57] == 255); + std::printf("webcam preview bitmap: %d failures\n", failures); + return failures ? 1 : 0; +} diff --git a/electron/preload.ts b/electron/preload.ts index 021932a33..adaf464d6 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -170,6 +170,32 @@ contextBridge.exposeInMainWorld("electronAPI", { getLastPickedSource: (): Promise => { return ipcRenderer.invoke("get-last-picked-source"); }, + prepareCameraPreviewCapture: () => + ipcRenderer.invoke("camera-preview:prepare-capture") as Promise, + restoreCameraPreviewCapture: (token: number) => + ipcRenderer.invoke("camera-preview:restore-capture", token) as Promise, + onCameraPreviewAvailability: (callback: (available: boolean) => void) => { + const listener = (_event: Electron.IpcRendererEvent, available: boolean) => callback(available); + ipcRenderer.on("camera-preview:availability", listener); + return () => ipcRenderer.removeListener("camera-preview:availability", listener); + }, + configureCameraPreview: (enabled: boolean, visible: boolean): Promise => + ipcRenderer.invoke("camera-preview:configure", enabled, visible), + sendCameraPreviewFrame: (jpeg: string, aspectRatio: number): Promise => + ipcRenderer.invoke("camera-preview:frame", jpeg, aspectRatio), + resizeCameraPreview: (width: number): Promise => + ipcRenderer.invoke("camera-preview:resize", width), + hideCameraPreview: (): Promise => ipcRenderer.invoke("camera-preview:hide"), + onCameraPreviewFrame: (callback: (jpeg: string) => void) => { + const listener = (_event: Electron.IpcRendererEvent, jpeg: string) => callback(jpeg); + ipcRenderer.on("camera-preview:frame", listener); + return () => ipcRenderer.removeListener("camera-preview:frame", listener); + }, + onCameraPreviewHidden: (callback: () => void) => { + const listener = () => callback(); + ipcRenderer.on("camera-preview:hidden", listener); + return () => ipcRenderer.removeListener("camera-preview:hidden", listener); + }, openNotes: () => { return ipcRenderer.invoke("open-notes"); }, diff --git a/electron/windows.ts b/electron/windows.ts index f0ff7c89d..cb687166e 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { app, BrowserWindow, ipcMain, screen } from "electron"; +import { cameraPreviewBounds, resizedCameraPreviewBounds } from "../src/lib/cameraPreview"; import { fitEditorRect, loadEditorWindowState, @@ -879,6 +880,79 @@ export function createPermissionsWindow(): BrowserWindow { return win; } +let cameraPreviewWindow: BrowserWindow | null = null; + +export function getCameraPreviewWindow(): BrowserWindow | null { + return cameraPreviewWindow && !cameraPreviewWindow.isDestroyed() ? cameraPreviewWindow : null; +} + +/** The camera stream belongs to the HUD; this window only displays preview frames. */ +export function createCameraPreviewWindow(): BrowserWindow { + const initialBounds = () => { + const bar = + hudOverlayWindow && !hudOverlayWindow.isDestroyed() && hudContentRect + ? hudContentScreenRect(hudOverlayWindow.getBounds(), hudContentRect) + : undefined; + const workArea = bar + ? screen.getDisplayMatching(bar).workArea + : screen.getPrimaryDisplay().workArea; + return cameraPreviewBounds(workArea, bar); + }; + const win = new BrowserWindow({ + ...initialBounds(), + title: "OpenScreen - Camera Preview", + hasShadow: false, + frame: false, + transparent: true, + backgroundColor: "#00000000", + resizable: false, + maximizable: false, + minimizable: false, + alwaysOnTop: true, + skipTaskbar: true, + show: false, + webPreferences: { + preload: path.join(__dirname, "preload.mjs"), + additionalArguments: [ASSET_BASE_URL_ARG], + nodeIntegration: false, + contextIsolation: true, + backgroundThrottling: false, + }, + }); + win.once("ready-to-show", () => win.setBounds(initialBounds())); + applyContentProtection(win, "Camera Preview"); + keepAlwaysOnTop(win); + followAcrossSpaces(win); + cameraPreviewWindow = win; + win.on("closed", () => { + if (cameraPreviewWindow === win) cameraPreviewWindow = null; + }); + if (VITE_DEV_SERVER_URL) { + win.loadURL(VITE_DEV_SERVER_URL + "?windowType=camera-preview"); + } else { + win.loadFile(path.join(RENDERER_DIST, "index.html"), { + query: { windowType: "camera-preview" }, + }); + } + return win; +} + +export function resizeCameraPreviewWindow( + win: BrowserWindow, + width: number, + aspectRatio: number, +): void { + const bounds = win.getBounds(); + win.setBounds( + resizedCameraPreviewBounds( + bounds, + screen.getDisplayMatching(bounds).workArea, + width, + aspectRatio, + ), + ); +} + // Frameless Notes Window for taking notes during a recording. export function createNotesWindow(): BrowserWindow { const win = new BrowserWindow({ diff --git a/electron/windowsCameraPreview.test.ts b/electron/windowsCameraPreview.test.ts new file mode 100644 index 000000000..519cc2361 --- /dev/null +++ b/electron/windowsCameraPreview.test.ts @@ -0,0 +1,97 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import { nativeImage } from "electron"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + createWindowsCameraPreviewMailbox, + watchWindowsCameraPreview, +} from "./windowsCameraPreview"; + +vi.mock("electron", () => ({ + nativeImage: { + createFromBitmap: vi.fn(() => ({ isEmpty: () => false, toJPEG: () => Buffer.from("jpeg") })), + }, +})); + +function frame() { + const data = Buffer.alloc(54 + 2 * 2 * 4); + data.write("BM"); + data.writeInt32LE(2, 18); + data.writeInt32LE(-2, 22); + return data; +} + +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + vi.clearAllMocks(); +}); + +describe("Windows native camera preview", () => { + it("delivers bounded top-down pixels without opening a camera", async () => { + vi.useFakeTimers(); + vi.spyOn(fs, "readFile").mockResolvedValue(frame()); + const onFrame = vi.fn(); + const stop = watchWindowsCameraPreview("latest.bmp", onFrame); + await vi.advanceTimersByTimeAsync(0); + expect(nativeImage.createFromBitmap).toHaveBeenCalledWith(frame().subarray(54), { + width: 2, + height: 2, + }); + expect(onFrame).toHaveBeenCalledWith("data:image/jpeg;base64,anBlZw==", 1); + stop(); + await vi.advanceTimersByTimeAsync(1000); + expect(onFrame).toHaveBeenCalledOnce(); + }); + + it("does not queue reads or deliver an in-flight frame after cleanup", async () => { + vi.useFakeTimers(); + let resolve: ((value: Buffer) => void) | undefined; + const read = vi.spyOn(fs, "readFile").mockImplementationOnce( + () => + new Promise>((done) => { + resolve = done; + }), + ); + const onFrame = vi.fn(); + const stop = watchWindowsCameraPreview("latest.bmp", onFrame); + await vi.advanceTimersByTimeAsync(1000); + expect(read).toHaveBeenCalledOnce(); + stop(); + resolve?.(frame()); + await vi.advanceTimersByTimeAsync(1000); + expect(onFrame).not.toHaveBeenCalled(); + expect(read).toHaveBeenCalledOnce(); + }); + + it("retries a missing frame and rejects oversized dimensions and truncated pixels", async () => { + vi.useFakeTimers(); + const huge = frame(); + huge.writeInt32LE(10000, 18); + vi.spyOn(fs, "readFile") + .mockRejectedValueOnce(new Error("ENOENT")) + .mockResolvedValueOnce(huge) + .mockResolvedValueOnce(frame().subarray(0, 54)) + .mockResolvedValue(frame()); + const onFrame = vi.fn(); + const stop = watchWindowsCameraPreview("latest.bmp", onFrame); + await vi.advanceTimersByTimeAsync(500); + expect(onFrame).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(100); + expect(onFrame).toHaveBeenCalledOnce(); + stop(); + }); + + it("removes the private directory and any half-written camera frame", async () => { + const mailbox = await createWindowsCameraPreviewMailbox(os.tmpdir()); + try { + await fs.writeFile(mailbox.file, frame()); + await fs.writeFile(`${mailbox.file}.tmp`, frame()); + await mailbox.remove(); + await expect(fs.stat(mailbox.file)).rejects.toThrow(); + await expect(fs.stat(`${mailbox.file}.tmp`)).rejects.toThrow(); + } finally { + await mailbox.remove(); + } + }); +}); diff --git a/electron/windowsCameraPreview.ts b/electron/windowsCameraPreview.ts new file mode 100644 index 000000000..da054a39a --- /dev/null +++ b/electron/windowsCameraPreview.ts @@ -0,0 +1,52 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { nativeImage } from "electron"; + +/** Read the native camera's atomic latest-frame mailbox, with no queued reads. */ +export function watchWindowsCameraPreview( + file: string, + onFrame: (jpeg: string, aspectRatio: number) => void, +) { + let cancelled = false; + let timer: ReturnType | undefined; + const read = async () => { + try { + const data = await fs.readFile(file); + if (cancelled || data.length > 3_300_000) return; + if (data.length < 54 || data.toString("ascii", 0, 2) !== "BM") return; + const width = data.readInt32LE(18); + const height = -data.readInt32LE(22); + if ( + width < 1 || + width > 320 || + height < 1 || + height > 2560 || + data.length !== 54 + width * height * 4 + ) + return; + const image = nativeImage.createFromBitmap(data.subarray(54), { width, height }); + if (!image.isEmpty() && width > 0 && height > 0) { + onFrame(`data:image/jpeg;base64,${image.toJPEG(70).toString("base64")}`, width / height); + } + } catch { + // No first frame yet, or Windows still has the replaced file open. + } finally { + if (!cancelled) timer = setTimeout(() => void read(), 200); + } + }; + void read(); + return () => { + cancelled = true; + clearTimeout(timer); + }; +} + +/** A private per-take directory also lets cleanup remove a half-written frame. */ +export async function createWindowsCameraPreviewMailbox(tempDir: string) { + const directory = await fs.mkdtemp(path.join(tempDir, "openscreen-camera-preview-")); + return { + file: path.join(directory, "latest.bmp"), + remove: () => + fs.rm(directory, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }), + }; +} diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjs index 90b961ee7..9315aade7 100644 --- a/scripts/build-windows-wgc-helper.mjs +++ b/scripts/build-windows-wgc-helper.mjs @@ -164,6 +164,13 @@ if (!fs.existsSync(frameSlotClockTestPath)) { await run(frameSlotClockTestPath, [], { cwd: BUILD_DIR }); console.log(`Passed ${frameSlotClockTestPath}`); +const webcamPreviewTestPath = path.join(BUILD_DIR, "webcam_preview_bitmap_test.exe"); +if (!fs.existsSync(webcamPreviewTestPath)) { + throw new Error(`WGC helper build completed but ${webcamPreviewTestPath} was not found.`); +} +await run(webcamPreviewTestPath, [], { cwd: BUILD_DIR }); +console.log(`Passed ${webcamPreviewTestPath}`); + const webcamSnapshotTestPath = path.join(BUILD_DIR, "webcam_snapshot_test.exe"); if (!fs.existsSync(webcamSnapshotTestPath)) { throw new Error(`WGC helper build completed but ${webcamSnapshotTestPath} was not found.`); diff --git a/src/App.tsx b/src/App.tsx index 681bb5993..c326d3b83 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -6,6 +6,7 @@ import { installBrowserShims } from "./native/browserShim"; installBrowserShims(); import { CountdownOverlay } from "./components/launch/CountdownOverlay.tsx"; +import { FloatingCameraPreview } from "./components/launch/FloatingCameraPreview"; import { LaunchWindow } from "./components/launch/LaunchWindow"; import { NotesWindow } from "./components/launch/NotesWindow.tsx"; import { SourceSelector } from "./components/launch/SourceSelector"; @@ -52,7 +53,7 @@ export default function App() { setWindowType(type); } - if (type === "hud-overlay" || type === "source-selector" || type === "countdown-overlay") { + if (["hud-overlay", "source-selector", "countdown-overlay", "camera-preview"].includes(type)) { document.body.style.background = "transparent"; document.documentElement.style.background = "transparent"; document.getElementById("root")?.style.setProperty("background", "transparent"); @@ -60,7 +61,7 @@ export default function App() { // HUD is a fixed-size BrowserWindow; pin the document shell and hide overflow // so the renderer can't introduce scrollbars (see issue #305). - if (type === "hud-overlay") { + if (type === "hud-overlay" || type === "camera-preview") { document.documentElement.style.height = "100%"; document.documentElement.style.overflow = "hidden"; document.body.style.height = "100%"; @@ -85,6 +86,8 @@ export default function App() { switch (windowType) { case "hud-overlay": return ; + case "camera-preview": + return ; case "source-selector": return ; case "countdown-overlay": diff --git a/src/components/launch/FloatingCameraPreview.module.css b/src/components/launch/FloatingCameraPreview.module.css new file mode 100644 index 000000000..1d07938e7 --- /dev/null +++ b/src/components/launch/FloatingCameraPreview.module.css @@ -0,0 +1,73 @@ +.preview { + -webkit-app-region: drag; + position: relative; + display: grid; + place-items: center; + width: calc(100% - 16px); + height: calc(100% - 16px); + margin: 8px; + overflow: hidden; + border-radius: 12px; + background: #09090b; + color: #a1a1aa; + font-size: 12px; + cursor: grab; +} + +.preview img { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + object-fit: contain; + object-position: center; +} + +.hide { + -webkit-app-region: no-drag; + position: absolute; + top: 8px; + right: 8px; + display: grid; + place-items: center; + width: 28px; + height: 28px; + border-radius: 50%; + background: #09090bbf; + color: #f4f4f5; + cursor: pointer; +} + +.hide:focus-visible { + outline: 2px solid #10b981; +} + +.sizeControls { + -webkit-app-region: no-drag; + position: absolute; + bottom: 8px; + left: 50%; + display: flex; + gap: 4px; + transform: translateX(-50%); +} + +.sizeControls button { + display: grid; + place-items: center; + width: 28px; + height: 28px; + border-radius: 50%; + background: #09090bbf; + color: #f4f4f5; + cursor: pointer; +} + +.sizeControls button:disabled { + opacity: 0.4; + cursor: default; +} + +.sizeControls button:focus-visible { + outline: 2px solid #10b981; +} diff --git a/src/components/launch/FloatingCameraPreview.test.tsx b/src/components/launch/FloatingCameraPreview.test.tsx new file mode 100644 index 000000000..08194f994 --- /dev/null +++ b/src/components/launch/FloatingCameraPreview.test.tsx @@ -0,0 +1,37 @@ +// @vitest-environment jsdom +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, expect, it, vi } from "vitest"; +import { FloatingCameraPreview } from "./FloatingCameraPreview"; + +vi.mock("../../contexts/I18nContext", () => ({ useScopedT: () => (key: string) => key })); +afterEach(cleanup); + +it("displays incoming frames and offers three bounded preview sizes", async () => { + let frame: ((jpeg: string) => void) | undefined; + const resize = vi.fn().mockResolvedValue(undefined); + const unsubscribe = vi.fn(); + window.electronAPI = { + onCameraPreviewFrame: (callback: (jpeg: string) => void) => { + frame = callback; + return unsubscribe; + }, + resizeCameraPreview: resize, + } as unknown as Window["electronAPI"]; + const view = render(); + act(() => frame?.("data:image/jpeg;base64,frame")); + expect(screen.getByAltText("floatingCamera.title").getAttribute("src")).toBe( + "data:image/jpeg;base64,frame", + ); + const larger = screen.getByRole("button", { name: "floatingCamera.larger" }); + const smaller = screen.getByRole("button", { name: "floatingCamera.smaller" }); + fireEvent.click(larger); + await waitFor(() => expect((larger as HTMLButtonElement).disabled).toBe(true)); + expect(resize).toHaveBeenLastCalledWith(320); + fireEvent.click(smaller); + await waitFor(() => expect((larger as HTMLButtonElement).disabled).toBe(false)); + fireEvent.click(smaller); + await waitFor(() => expect((smaller as HTMLButtonElement).disabled).toBe(true)); + expect(resize).toHaveBeenLastCalledWith(160); + view.unmount(); + expect(unsubscribe).toHaveBeenCalledOnce(); +}); diff --git a/src/components/launch/FloatingCameraPreview.tsx b/src/components/launch/FloatingCameraPreview.tsx new file mode 100644 index 000000000..cdd21f351 --- /dev/null +++ b/src/components/launch/FloatingCameraPreview.tsx @@ -0,0 +1,59 @@ +import { Minus, Plus, X } from "lucide-react"; +import { useEffect, useRef, useState } from "react"; +import { useScopedT } from "../../contexts/I18nContext"; +import { CAMERA_PREVIEW_SIZES } from "../../lib/cameraPreview"; +import styles from "./FloatingCameraPreview.module.css"; + +export function FloatingCameraPreview() { + const t = useScopedT("launch"); + const image = useRef(null); + const [ready, setReady] = useState(false); + const [sizeIndex, setSizeIndex] = useState(1); + const resize = (next: number) => { + void window.electronAPI + ?.resizeCameraPreview?.(CAMERA_PREVIEW_SIZES[next]) + .then(() => setSizeIndex(next)) + .catch((error: unknown) => console.warn("Could not resize camera preview:", error)); + }; + useEffect( + () => + window.electronAPI?.onCameraPreviewFrame?.((jpeg) => { + if (image.current) image.current.src = jpeg; + setReady(true); + }), + [], + ); + + return ( +
+ + {!ready && {t("deviceSettings.previewUnavailable")}} + +
+ + +
+
+ ); +} diff --git a/src/components/launch/HudControls.tsx b/src/components/launch/HudControls.tsx index c417b754c..a48863c7e 100644 --- a/src/components/launch/HudControls.tsx +++ b/src/components/launch/HudControls.tsx @@ -1,4 +1,12 @@ -import { Check, CircleArrowDown, Languages, NotepadText, Settings } from "lucide-react"; +import { + Check, + CircleArrowDown, + Eye, + EyeOff, + Languages, + NotepadText, + Settings, +} from "lucide-react"; import { createContext, memo, @@ -301,6 +309,38 @@ export const HudMicButton = memo(function HudMicButton({ ); }); +export function HudCameraPreviewButton({ + visible, + disabled, + label, + onClick, +}: { + visible: boolean; + disabled: boolean; + label: string; + onClick: () => void; +}) { + return ( + + + + ); +} + export const HudCameraButton = memo(function HudCameraButton({ enabled, locked, diff --git a/src/components/launch/LaunchWindow.test.tsx b/src/components/launch/LaunchWindow.test.tsx index 5da604fa7..803b6439c 100644 --- a/src/components/launch/LaunchWindow.test.tsx +++ b/src/components/launch/LaunchWindow.test.tsx @@ -2547,3 +2547,54 @@ describe("LaunchWindow toasts", () => { }); }); }); + +describe("floating camera preview control", () => { + beforeEach(() => { + platformState.value = "darwin"; + resetLaunchMocks(); + }); + + it("keeps hide/show available during recording without changing camera capture", () => { + recorderState.value.webcamEnabled = true; + recorderState.value.recording = true; + renderLaunchWindow(); + const button = screen.getByTestId("launch-camera-preview-button"); + expect(button).toHaveAttribute("aria-pressed", "true"); + fireEvent.click(button); + expect(button).toHaveAttribute("aria-pressed", "false"); + fireEvent.click(button); + expect(button).toHaveAttribute("aria-pressed", "true"); + expect(recorderState.value.setWebcamEnabled).not.toHaveBeenCalled(); + }); + + it("hides the preview control when the camera is off", () => { + const view = renderLaunchWindow(); + expect(screen.queryByTestId("launch-camera-preview-button")).not.toBeInTheDocument(); + recorderState.value.webcamEnabled = true; + view.rerender( + + + , + ); + expect(screen.getByTestId("launch-camera-preview-button")).toHaveAttribute( + "aria-pressed", + "true", + ); + recorderState.value.webcamEnabled = false; + view.rerender( + + + , + ); + expect(screen.queryByTestId("launch-camera-preview-button")).not.toBeInTheDocument(); + }); + + it.each(["win32", "linux"] as const)("offers the preview on %s", async (platform) => { + platformState.value = platform; + recorderState.value.webcamEnabled = true; + renderLaunchWindow(); + await waitFor(() => + expect(screen.getByTestId("launch-camera-preview-button")).toBeInTheDocument(), + ); + }); +}); diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index c094d6230..5be6dec5d 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -8,6 +8,7 @@ import { nativeBridgeClient } from "@/native"; import { canRecordMicrophone, getPlatform } from "@/utils/platformUtils"; import { type CameraDevice } from "../../hooks/useCameraDevices"; import { useCameraHudSync } from "../../hooks/useCameraHudSync"; +import { useFloatingCameraPreview } from "../../hooks/useFloatingCameraPreview"; import { isPlaceholderMicrophoneLabel, type MicrophoneDevice, @@ -22,6 +23,7 @@ import { requestCameraAccess } from "../../lib/requestCameraAccess"; import { Toaster } from "../ui/sonner"; import { HudCameraButton, + HudCameraPreviewButton, HudCursorButton, HudDivider, HudDragHandle, @@ -113,6 +115,7 @@ export function LaunchWindow() { const languageCode = locale.split("-")[0].toUpperCase(); const { + webcamPreviewStream, recording, paused, saving, @@ -146,6 +149,14 @@ export function LaunchWindow() { } = useScreenRecorder(); // Choosing a device and switching one on are deliberately separate concerns. + const cameraPreview = useFloatingCameraPreview( + webcamPreviewStream, + webcamEnabled && !saving, + recording && getPlatform() === "win32", + ); + const cameraPreviewShown = + webcamEnabled && !saving && cameraPreview.available && cameraPreview.visible; + // The mic and camera buttons are plain on/off toggles that use whatever device // is currently selected (the system default until the user says otherwise); // picking a different device — and checking it actually works — happens in the @@ -1310,6 +1321,18 @@ export function LaunchWindow() { tooltip={t("tooltips.camera")} onClick={toggleWebcam} /> + {cameraPreview.supported && webcamEnabled && ( + + )} ({ getPlatform: () => platform })); +let platform = "darwin"; +let hidden: (() => void) | undefined; +let availability: ((available: boolean) => void) | undefined; +const configure = vi.fn().mockResolvedValue(undefined); +const send = vi.fn().mockResolvedValue(undefined); +const unsubscribe = vi.fn(); +const stop = vi.fn(); +const stream = { getTracks: () => [{ stop }] } as unknown as MediaStream; + +beforeEach(() => { + vi.useFakeTimers(); + platform = "darwin"; + vi.clearAllMocks(); + Object.defineProperty(window, "electronAPI", { + configurable: true, + value: { + configureCameraPreview: configure, + sendCameraPreviewFrame: send, + onCameraPreviewAvailability: (callback: (available: boolean) => void) => { + availability = callback; + return () => undefined; + }, + onCameraPreviewHidden: (callback: () => void) => { + hidden = callback; + return unsubscribe; + }, + }, + }); + vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(undefined); + vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => undefined); + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({ + drawImage: vi.fn(), + } as unknown as CanvasRenderingContext2D); + vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue( + "data:image/jpeg;base64,frame", + ); + vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockReturnValue(2); + vi.spyOn(HTMLVideoElement.prototype, "videoWidth", "get").mockReturnValue(1280); + vi.spyOn(HTMLVideoElement.prototype, "videoHeight", "get").mockReturnValue(720); +}); + +afterEach(() => { + cleanup(); + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +describe("floating camera publisher", () => { + it("shares frames without acquiring or stopping the recording camera", async () => { + const { unmount } = renderHook(() => useFloatingCameraPreview(stream, true)); + expect(configure).toHaveBeenLastCalledWith(true, true); + expect(send).toHaveBeenCalledOnce(); + expect(send).toHaveBeenCalledWith("data:image/jpeg;base64,frame", 1280 / 720); + await act(() => vi.advanceTimersByTimeAsync(300)); + expect(send).toHaveBeenCalledTimes(4); + unmount(); + await vi.advanceTimersByTimeAsync(500); + expect(send).toHaveBeenCalledTimes(4); + expect(stop).not.toHaveBeenCalled(); + expect(configure).toHaveBeenLastCalledWith(false, false); + expect(unsubscribe).toHaveBeenCalledOnce(); + }); + + it("hides without disabling the camera, then resumes from the HUD control", () => { + const { result } = renderHook(() => useFloatingCameraPreview(stream, true)); + act(() => hidden?.()); + expect(configure).toHaveBeenLastCalledWith(true, false); + expect(result.current.visible).toBe(false); + act(() => result.current.toggle()); + expect(configure).toHaveBeenLastCalledWith(true, true); + expect(stop).not.toHaveBeenCalled(); + }); + + it("applies backpressure while a previous frame is still being delivered", async () => { + let delivered: (() => void) | undefined; + send.mockImplementationOnce( + () => + new Promise((resolve) => { + delivered = resolve; + }), + ); + renderHook(() => useFloatingCameraPreview(stream, true)); + await act(() => vi.advanceTimersByTimeAsync(1000)); + expect(send).toHaveBeenCalledOnce(); + await act(async () => { + delivered?.(); + }); + await act(() => vi.advanceTimersByTimeAsync(100)); + expect(send).toHaveBeenCalledTimes(2); + }); + + it("waits for the recorder stream and ignores unsupported platforms", () => { + const { rerender, unmount } = renderHook( + ({ enabled }) => useFloatingCameraPreview(null, enabled), + { initialProps: { enabled: true } }, + ); + expect(configure).toHaveBeenLastCalledWith(true, false); + expect(send).not.toHaveBeenCalled(); + rerender({ enabled: false }); + expect(configure).toHaveBeenLastCalledWith(false, false); + unmount(); + configure.mockClear(); + platform = "unknown"; + renderHook(() => useFloatingCameraPreview(stream, true)); + expect(configure).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + }); + it.each(["darwin", "win32", "linux"])("publishes preview frames on %s", (value) => { + platform = value; + const { result } = renderHook(() => useFloatingCameraPreview(stream, true)); + expect(result.current.supported).toBe(true); + expect(configure).toHaveBeenLastCalledWith(true, true); + expect(send).toHaveBeenCalledOnce(); + }); + + it("keeps the window visible after Windows hands the camera to the native recorder", () => { + platform = "win32"; + const { rerender } = renderHook( + ({ camera, native }) => useFloatingCameraPreview(camera, true, native), + { initialProps: { camera: stream as MediaStream | null, native: false } }, + ); + rerender({ camera: null, native: true }); + expect(configure).toHaveBeenLastCalledWith(true, true); + expect(stop).not.toHaveBeenCalled(); + }); + + it("suspends publishing for Linux monitor capture and restores the visibility preference", async () => { + platform = "linux"; + const { result } = renderHook(() => useFloatingCameraPreview(stream, true)); + act(() => availability?.(false)); + expect(result.current.available).toBe(false); + expect(result.current.visible).toBe(true); + await act(() => vi.advanceTimersByTimeAsync(500)); + expect(send).toHaveBeenCalledOnce(); + act(() => availability?.(true)); + expect(result.current.available).toBe(true); + expect(send).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/hooks/useFloatingCameraPreview.ts b/src/hooks/useFloatingCameraPreview.ts new file mode 100644 index 000000000..5c0c9d718 --- /dev/null +++ b/src/hooks/useFloatingCameraPreview.ts @@ -0,0 +1,72 @@ +import { useEffect, useState } from "react"; +import { getPlatform } from "../utils/platformUtils"; + +/** A small self-view from the recorder's stream; never acquires or stops the camera. */ +export function useFloatingCameraPreview( + stream: MediaStream | null, + enabled: boolean, + nativeCameraActive = false, +) { + const supported = ["darwin", "win32", "linux"].includes(getPlatform()); + const [visible, setVisible] = useState(true); + const [available, setAvailable] = useState(true); + useEffect(() => window.electronAPI?.onCameraPreviewAvailability?.(setAvailable), []); + + useEffect(() => window.electronAPI?.onCameraPreviewHidden?.(() => setVisible(false)), []); + + useEffect(() => { + if (!supported) return; + void window.electronAPI + ?.configureCameraPreview?.(enabled, enabled && visible && (!!stream || nativeCameraActive)) + .catch((error: unknown) => console.warn("Could not configure camera preview:", error)); + }, [enabled, visible, stream, nativeCameraActive, supported]); + + useEffect(() => { + if (!supported || !enabled || !visible || !available || !stream) return; + const video = document.createElement("video"); + video.muted = true; + video.playsInline = true; + video.srcObject = stream; + void video.play().catch(() => undefined); + const canvas = document.createElement("canvas"); + canvas.width = 320; + canvas.height = 180; + const context = canvas.getContext("2d"); + let cancelled = false; + let timer: ReturnType; + const draw = async () => { + try { + if (context && video.readyState >= 2 && video.videoWidth > 0 && video.videoHeight > 0) { + canvas.height = Math.max(1, Math.round((320 * video.videoHeight) / video.videoWidth)); + context.drawImage(video, 0, 0, canvas.width, canvas.height); + // Await delivery: a slow renderer must not build a queue of stale frames. + await window.electronAPI?.sendCameraPreviewFrame?.( + canvas.toDataURL("image/jpeg", 0.7), + video.videoWidth / video.videoHeight, + ); + } + } catch (error) { + console.warn("Could not update camera preview:", error); + } finally { + if (!cancelled) timer = setTimeout(() => void draw(), 100); + } + }; + void draw(); + return () => { + cancelled = true; + clearTimeout(timer); + video.pause(); + video.srcObject = null; + }; + }, [enabled, visible, available, stream, supported]); + + useEffect( + () => () => { + if (supported) + void window.electronAPI?.configureCameraPreview?.(false, false).catch(() => undefined); + }, + [supported], + ); + + return { supported, available, visible, toggle: () => setVisible((value) => !value) }; +} diff --git a/src/hooks/useScreenRecorder.cameraPrefs.test.tsx b/src/hooks/useScreenRecorder.cameraPrefs.test.tsx new file mode 100644 index 000000000..331ca7d31 --- /dev/null +++ b/src/hooks/useScreenRecorder.cameraPrefs.test.tsx @@ -0,0 +1,103 @@ +// @vitest-environment jsdom +import { act, cleanup, renderHook, waitFor } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useCameraHudSync } from "./useCameraHudSync"; +import { useScreenRecorder } from "./useScreenRecorder"; + +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); +vi.mock("sonner", () => ({ toast: { error: vi.fn(), warning: vi.fn(), info: vi.fn() } })); +vi.mock("./useCameraDevices", () => ({ + useCameraDevices: (_enabled: boolean, preferred?: string) => ({ + selectedDeviceId: preferred ?? "cam-1", + selectedDevice: { label: preferred === "cam-2" ? "Camera 2" : "Camera 1" }, + isReady: true, + }), +})); + +type Prefs = Awaited>; +const initial: Prefs = { + micEnabled: false, + micDeviceId: null, + micDeviceName: null, + camEnabled: true, + camDeviceId: null, + camDeviceName: null, + camQuality: "2160p", + systemAudioEnabled: false, + cursorCaptureMode: "editable-overlay", + hideDesktopIcons: false, + autoZoomEnabled: true, +}; + +afterEach(() => { + cleanup(); + window.history.replaceState({}, "", "/"); + vi.restoreAllMocks(); +}); + +function setup(platform = "darwin") { + window.history.replaceState({}, "", "/?windowType=hud-overlay"); + let publish: ((prefs: Prefs) => void) | undefined; + const stops: Array> = []; + const getUserMedia = vi.fn(async () => { + const stop = vi.fn(); + stops.push(stop); + const track = { stop, onended: null }; + return { getTracks: () => [track], getVideoTracks: () => [track] } as unknown as MediaStream; + }); + Object.defineProperty(navigator, "mediaDevices", { configurable: true, value: { getUserMedia } }); + window.electronAPI = { + getPlatform: () => platform, + getRecordingPrefs: async () => initial, + onRecordingPrefsChanged: (callback: (prefs: Prefs) => void) => { + publish = callback; + return () => undefined; + }, + hideCountdownOverlay: vi.fn().mockResolvedValue(undefined), + } as unknown as Window["electronAPI"]; + const view = renderHook(() => { + const recorder = useScreenRecorder(); + useCameraHudSync(recorder); + return recorder; + }); + return { view, getUserMedia, stops, publish: (prefs: Prefs) => publish?.(prefs) }; +} + +describe("camera preview across recording preference broadcasts", () => { + it.each([ + "darwin", + "win32", + "linux", + ])("keeps the resolved camera alive across audio preferences on %s", async (platform) => { + const { view, getUserMedia, stops, publish } = setup(platform); + await waitFor(() => expect(view.result.current.webcamPreviewStream).not.toBeNull()); + const stream = view.result.current.webcamPreviewStream; + const calls = getUserMedia.mock.calls.length; + const stop = stops.at(-1); + await act(async () => { + view.result.current.setSystemAudioEnabled(true); + publish({ ...initial, systemAudioEnabled: true }); + }); + await act(async () => { + view.result.current.setMicrophoneEnabled(true); + publish({ ...initial, micEnabled: true, systemAudioEnabled: true }); + }); + expect(view.result.current.webcamPreviewStream).toBe(stream); + expect(getUserMedia).toHaveBeenCalledTimes(calls); + expect(stop).not.toHaveBeenCalled(); + }); + + it("still applies a changed camera selection and an explicit reset to the default", async () => { + const { view, publish } = setup(); + await waitFor(() => expect(view.result.current.webcamPreviewStream).not.toBeNull()); + const first = view.result.current.webcamPreviewStream; + await act(async () => publish({ ...initial, camDeviceId: "cam-2", camDeviceName: "Camera 2" })); + expect(view.result.current.webcamDeviceId).toBe("cam-2"); + expect(view.result.current.webcamPreviewStream).not.toBe(first); + await act(async () => publish(initial)); + expect(view.result.current.webcamDeviceId).toBe("cam-1"); + }); +}); diff --git a/src/hooks/useScreenRecorder.nativeLinuxWebcamCursor.test.tsx b/src/hooks/useScreenRecorder.nativeLinuxWebcamCursor.test.tsx index 7711b3552..24ffa5951 100644 --- a/src/hooks/useScreenRecorder.nativeLinuxWebcamCursor.test.tsx +++ b/src/hooks/useScreenRecorder.nativeLinuxWebcamCursor.test.tsx @@ -1,10 +1,11 @@ // @vitest-environment jsdom -import { act, renderHook } from "@testing-library/react"; +import { act, cleanup, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, @@ -51,6 +52,8 @@ function stubElectronAPI() { getPlatform: vi.fn(() => "linux"), getSelectedSource: vi.fn(async () => null), isNativeLinuxCaptureAvailable: vi.fn(async () => ({ success: true, available: true })), + prepareNativeLinuxRecording: vi.fn(async () => ({ success: true, recordingId: 7 })), + cancelNativeLinuxPrepare: vi.fn(async () => ({ success: true })), startNativeLinuxRecording: vi.fn(async () => ({ success: true, recordingId: 7 })), stopNativeLinuxRecording: vi.fn(), attachNativeLinuxWebcamRecording: vi.fn(async () => ({ success: true, session: {} })), @@ -90,6 +93,7 @@ beforeEach(() => { }); afterEach(() => { + cleanup(); vi.useRealTimers(); vi.unstubAllGlobals(); vi.restoreAllMocks(); diff --git a/src/hooks/useScreenRecorder.noCamera.test.tsx b/src/hooks/useScreenRecorder.noCamera.test.tsx index cd75f16b8..4958a8fbf 100644 --- a/src/hooks/useScreenRecorder.noCamera.test.tsx +++ b/src/hooks/useScreenRecorder.noCamera.test.tsx @@ -2,9 +2,10 @@ import { act, renderHook, waitFor } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, @@ -101,7 +102,9 @@ describe("useScreenRecorder on a machine without a camera (#967)", () => { expect(ok).toBe(true); expect(view.result.current.webcamEnabled).toBe(true); - await waitFor(() => expect(getUserMedia).toHaveBeenCalled()); + await waitFor(() => expect(view.result.current.webcamPreviewStream).not.toBeNull()); + expect(view.result.current.webcamPreviewStream).toBe(await getUserMedia.mock.results[0].value); + expect(getUserMedia).toHaveBeenCalledTimes(1); expect(setRecordingPrefs).not.toHaveBeenCalled(); }); diff --git a/src/hooks/useScreenRecorder.restartFailure.test.tsx b/src/hooks/useScreenRecorder.restartFailure.test.tsx index 564936f41..56d191cc9 100644 --- a/src/hooks/useScreenRecorder.restartFailure.test.tsx +++ b/src/hooks/useScreenRecorder.restartFailure.test.tsx @@ -2,9 +2,10 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("@/contexts/I18nContext", () => ({ - useScopedT: () => (key: string) => key, -})); +vi.mock("@/contexts/I18nContext", () => { + const translate = (key: string) => key; + return { useScopedT: () => translate }; +}); vi.mock("sonner", () => ({ toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, @@ -312,3 +313,93 @@ describe("useScreenRecorder Restart that does not start a new take", () => { expect(toast.error).not.toHaveBeenCalledWith("recording.restartFailed"); }); }); + +function stubWindowsCamera() { + stubElectronAPI("win32"); + api.getRecordingPrefs.mockResolvedValue({ + camEnabled: true, + camQuality: "1080p", + micEnabled: false, + systemAudioEnabled: false, + cursorCaptureMode: "editable-overlay", + }); + const track = { stop: vi.fn(), onended: null, getSettings: () => ({ deviceId: "camera-1" }) }; + const getUserMedia = vi.fn(async () => ({ + getTracks: () => [track], + getVideoTracks: () => [track], + })); + Object.defineProperty(navigator, "mediaDevices", { + configurable: true, + value: { getUserMedia }, + }); + return { getUserMedia, track }; +} + +describe("Windows camera handoff back to the idle preview", () => { + it("waits for Cancel to release the helper before reacquiring the enabled camera", async () => { + const { getUserMedia, track } = stubWindowsCamera(); + const view = renderHook(() => useScreenRecorder()); + await startTake(view); + expect(track.stop).toHaveBeenCalled(); + expect(view.result.current.webcamPreviewStream).toBeNull(); + const calls = getUserMedia.mock.calls.length; + let finishStop: ((result: { success: boolean; discarded: boolean }) => void) | undefined; + api.stopNativeWindowsRecording.mockImplementationOnce( + () => + new Promise((resolve) => { + finishStop = resolve; + }), + ); + await act(async () => view.result.current.cancelRecording()); + expect(getUserMedia).toHaveBeenCalledTimes(calls); + await act(async () => finishStop?.({ success: true, discarded: true })); + await settle(); + expect(view.result.current.recording).toBe(false); + expect(view.result.current.webcamEnabled).toBe(true); + expect(view.result.current.webcamPreviewStream).not.toBeNull(); + expect(getUserMedia).toHaveBeenCalledTimes(calls + 1); + }); + + it("does not open a competing renderer camera on Restart", async () => { + const { getUserMedia } = stubWindowsCamera(); + const view = renderHook(() => useScreenRecorder()); + await startTake(view); + const calls = getUserMedia.mock.calls.length; + await restart(view); + expect(view.result.current.recording).toBe(true); + expect(api.startNativeWindowsRecording).toHaveBeenCalledTimes(2); + expect(getUserMedia).toHaveBeenCalledTimes(calls); + expect(view.result.current.webcamPreviewStream).toBeNull(); + }); +}); + +describe("Linux fallback preview suspension", () => { + it("restores availability when capture resolves after its countdown was cancelled", async () => { + const getUserMedia = stubBrowserCapture(); + api.prepareCameraPreviewCapture = vi.fn(async () => 17); + api.restoreCameraPreviewCapture = vi.fn(async () => undefined); + const captured = await getUserMedia(); + let finishCapture: ((stream: typeof captured) => void) | undefined; + getUserMedia.mockImplementationOnce( + () => + new Promise((resolve) => { + finishCapture = resolve; + }), + ); + const view = renderHook(() => useScreenRecorder()); + await settle(); + await act(async () => view.result.current.toggleRecording()); + await settle(3500); + expect(api.prepareCameraPreviewCapture).toHaveBeenCalledOnce(); + expect(api.restoreCameraPreviewCapture).not.toHaveBeenCalled(); + // Replace the pending start with a new countdown, then cancel that countdown. + await act(async () => view.result.current.toggleRecording()); + await settle(); + await act(async () => view.result.current.cancelRecording()); + await act(async () => finishCapture?.(captured)); + await settle(); + expect(view.result.current.recording).toBe(false); + expect(api.restoreCameraPreviewCapture).toHaveBeenCalledWith(17); + expect(captured.getTracks()[0].stop).toHaveBeenCalled(); + }); +}); diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 8ed63698c..6a69fb47b 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -88,6 +88,7 @@ const AUDIO_BITRATE_VOICE = 128_000; const AUDIO_BITRATE_SYSTEM = 192_000; type UseScreenRecorderReturn = { + webcamPreviewStream: MediaStream | null; recording: boolean; paused: boolean; saving: boolean; @@ -285,6 +286,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { useEffect(() => { let cancelled = false; let receivedNewerSnapshot = false; + let appliedCameraSelection: { + deviceId: string | undefined; + deviceName: string | undefined; + } | null = null; const applyPrefs = (prefs: { micEnabled: boolean; micDeviceId?: string | null; @@ -307,8 +312,23 @@ export function useScreenRecorder(): UseScreenRecorderReturn { new URLSearchParams(window.location.search).get("windowType") === "cli-record"; if (!isCliRecord) { setWebcamEnabledState(prefs.camEnabled); - setWebcamDeviceId(prefs.camDeviceId ?? undefined); - setWebcamDeviceName(prefs.camDeviceName ?? undefined); + const deviceId = prefs.camDeviceId ?? undefined; + const deviceName = prefs.camDeviceName ?? undefined; + // The HUD resolves a stored default (null) to a concrete device locally. + // An audio-only save republishes that same null: applying it again would + // reset the resolved device, stop its stream, and visibly blink the preview. + const isHud = + new URLSearchParams(window.location.search).get("windowType") === "hud-overlay"; + if ( + !isHud || + !appliedCameraSelection || + appliedCameraSelection.deviceId !== deviceId || + appliedCameraSelection.deviceName !== deviceName + ) { + setWebcamDeviceId(deviceId); + setWebcamDeviceName(deviceName); + appliedCameraSelection = { deviceId, deviceName }; + } } setWebcamQuality(webcamQualityFrom(prefs.camQuality)); setSystemAudioEnabled(prefs.systemAudioEnabled); @@ -351,6 +371,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const screenStream = useRef(null); const microphoneStream = useRef(null); const webcamStream = useRef(null); + const [webcamPreviewGeneration, setWebcamPreviewGeneration] = useState(0); + const [webcamPreviewStream, setWebcamPreviewStream] = useState(null); const mixingContext = useRef(null); const recordingId = useRef(0); const accumulatedDurationMs = useRef(0); @@ -472,6 +494,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { track.stop(); }); webcamStream.current = null; + setWebcamPreviewStream(null); webcamReady.current = true; }, []); @@ -492,6 +515,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { [t], ); + // biome-ignore lint/correctness/useExhaustiveDependencies: Cancel must reacquire the released Windows camera even when its settings are unchanged. useEffect(() => { if (!webcamEnabled) return; @@ -519,6 +543,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { stream.getVideoTracks().forEach((track) => { track.onended = () => { webcamStream.current = null; + setWebcamPreviewStream(null); if (!restarting.current) { setWebcamEnabledState(false); toast.error(t("recording.cameraDisconnected")); @@ -526,6 +551,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; }); webcamStream.current = stream; + setWebcamPreviewStream(stream); webcamReady.current = true; } catch (cameraError) { if (isRecoverableWebcamConstraintError(cameraError, webcamDeviceName)) { @@ -565,9 +591,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { track.stop(); }); webcamStream.current = null; + setWebcamPreviewStream(null); } }; - }, [webcamEnabled, webcamDeviceId, webcamDeviceName, webcamQuality, t]); + }, [webcamEnabled, webcamDeviceId, webcamDeviceName, webcamQuality, webcamPreviewGeneration, t]); const finalizeRecording = useCallback( ( @@ -1815,6 +1842,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const browserCursorCaptureMode = effectiveBrowserCursorMode(platform, cursorCaptureMode); let browserStartClaimed = false; let browserRecorderStarted = false; + let browserPreviewCaptureToken: number | undefined; try { if (!isCountdownRunActive(countdownRunToken)) { @@ -1875,6 +1903,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } + // The browser fallback cannot exclude a Linux preview from a monitor stream. + if (platform === "linux") { + browserPreviewCaptureToken = await window.electronAPI.prepareCameraPreviewCapture?.(); + } + // Capture screen + microphone in parallel: the gap between the two // `getUserMedia` calls is the dominant source of the mic-vs-video lag at the // start of the recording (issue #57). @@ -2150,6 +2183,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } } catch (error) { console.error("Failed to start recording:", error); + window.electronAPI?.setRecordingState?.(false); const errorMsg = error instanceof Error ? error.message : "Failed to start recording"; if (errorMsg.includes("Permission denied") || errorMsg.includes("NotAllowedError")) { toast.error(t("recording.permissionDenied")); @@ -2186,6 +2220,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { console.warn("Failed to release the recording start:", error); }); } + if (browserPreviewCaptureToken !== undefined && !browserRecorderStarted) { + await window.electronAPI + .restoreCameraPreviewCapture?.(browserPreviewCaptureToken) + .catch((error) => { + console.warn("Failed to restore the idle camera preview:", error); + }); + } } }; @@ -2475,7 +2516,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const activeRecordingId = recordingId.current; discardRecordingId.current = activeRecordingId; allowAutoFinalize.current = false; - void finalizeNativeWindowsRecording(true); + void finalizeNativeWindowsRecording(true).then((stopped) => { + // Only Cancel returns to an idle camera. Restart hands it straight back + // to the helper, so it must not open a competing renderer session. + if (stopped) setWebcamPreviewGeneration((generation) => generation + 1); + }); return; } if (nativeMacRecording.current) { @@ -2520,6 +2565,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; return { + webcamPreviewStream, recording, paused, saving, diff --git a/src/i18n/locales/ar/launch.json b/src/i18n/locales/ar/launch.json index ef613d15d..eef8e0bf0 100644 --- a/src/i18n/locales/ar/launch.json +++ b/src/i18n/locales/ar/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "في لينكس، تتوفر معاينة الكاميرا أثناء تسجيل نافذة فقط", + "title": "معاينة الكاميرا", + "show": "إظهار معاينة الكاميرا", + "hide": "إخفاء معاينة الكاميرا", + "smaller": "تصغير المعاينة", + "larger": "تكبير المعاينة" + }, "tooltips": { "hideHUD": "إخفاء واجهة العرض", "hideHUDTip": "إخفاء الشريط. يمكنك إظهاره مجددًا من أيقونة علبة النظام.", diff --git a/src/i18n/locales/cs/launch.json b/src/i18n/locales/cs/launch.json index d9912da51..c90e4d380 100644 --- a/src/i18n/locales/cs/launch.json +++ b/src/i18n/locales/cs/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "V Linuxu je náhled kamery dostupný pouze při nahrávání okna", + "title": "Náhled kamery", + "show": "Zobrazit náhled kamery", + "hide": "Skrýt náhled kamery", + "smaller": "Zmenšit náhled", + "larger": "Zvětšit náhled" + }, "tooltips": { "hideHUD": "Skrýt HUD", "hideHUDTip": "Skrýt panel. Znovu ho zobrazíte z ikony v oznamovací oblasti.", diff --git a/src/i18n/locales/de/launch.json b/src/i18n/locales/de/launch.json index b4dab82a9..50f2a5705 100644 --- a/src/i18n/locales/de/launch.json +++ b/src/i18n/locales/de/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "Unter Linux ist die Kameravorschau nur bei Fensteraufnahmen verfügbar", + "title": "Kameravorschau", + "show": "Kameravorschau anzeigen", + "hide": "Kameravorschau ausblenden", + "smaller": "Vorschau verkleinern", + "larger": "Vorschau vergrößern" + }, "tooltips": { "hideHUD": "HUD ausblenden", "hideHUDTip": "Aufnahmeleiste ausblenden. Über das Symbol im Infobereich wieder einblenden.", diff --git a/src/i18n/locales/en/launch.json b/src/i18n/locales/en/launch.json index 97263f184..1259dd753 100644 --- a/src/i18n/locales/en/launch.json +++ b/src/i18n/locales/en/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "Camera preview is available during window recording only on Linux", + "title": "Camera preview", + "show": "Show camera preview", + "hide": "Hide camera preview", + "smaller": "Decrease preview size", + "larger": "Increase preview size" + }, "tooltips": { "hideHUD": "Hide recording bar", "hideHUDTip": "Hide the recording bar. Show it again from the tray icon.", diff --git a/src/i18n/locales/es/launch.json b/src/i18n/locales/es/launch.json index 3114921ec..664e48002 100644 --- a/src/i18n/locales/es/launch.json +++ b/src/i18n/locales/es/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "En Linux, la vista previa de la cámara solo está disponible al grabar una ventana", + "title": "Vista previa de cámara", + "show": "Mostrar vista previa de cámara", + "hide": "Ocultar vista previa de cámara", + "smaller": "Reducir vista previa", + "larger": "Ampliar vista previa" + }, "tooltips": { "hideHUD": "Ocultar HUD", "hideHUDTip": "Ocultar la barra. Vuelve a mostrarla desde el icono de la bandeja del sistema.", diff --git a/src/i18n/locales/fr/launch.json b/src/i18n/locales/fr/launch.json index c956fb37a..5497729bb 100644 --- a/src/i18n/locales/fr/launch.json +++ b/src/i18n/locales/fr/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "Sous Linux, l’aperçu caméra est disponible uniquement lors de l’enregistrement d’une fenêtre", + "title": "Aperçu de la caméra", + "show": "Afficher l’aperçu de la caméra", + "hide": "Masquer l’aperçu de la caméra", + "smaller": "Réduire l’aperçu", + "larger": "Agrandir l’aperçu" + }, "tooltips": { "hideHUD": "Masquer le HUD", "hideHUDTip": "Masquer la barre. Réaffichez-la depuis l'icône de la zone de notification.", diff --git a/src/i18n/locales/it/launch.json b/src/i18n/locales/it/launch.json index 9897b92eb..ad49301d9 100644 --- a/src/i18n/locales/it/launch.json +++ b/src/i18n/locales/it/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "Su Linux, l’anteprima della videocamera è disponibile solo durante la registrazione di una finestra", + "title": "Anteprima della videocamera", + "show": "Mostra anteprima della videocamera", + "hide": "Nascondi anteprima della videocamera", + "smaller": "Riduci anteprima", + "larger": "Ingrandisci anteprima" + }, "tooltips": { "hideHUD": "Nascondi", "hideHUDTip": "Nascondi la barra. Puoi mostrarla di nuovo dall'icona nell'area di notifica.", diff --git a/src/i18n/locales/ja-JP/launch.json b/src/i18n/locales/ja-JP/launch.json index 345da320d..824276a5f 100644 --- a/src/i18n/locales/ja-JP/launch.json +++ b/src/i18n/locales/ja-JP/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "Linuxではカメラプレビューはウィンドウ録画中のみ利用できます", + "title": "カメラプレビュー", + "show": "カメラプレビューを表示", + "hide": "カメラプレビューを非表示", + "smaller": "プレビューを縮小", + "larger": "プレビューを拡大" + }, "tooltips": { "hideHUD": "HUDを隠す", "hideHUDTip": "録画バーを隠します。通知領域のアイコンから再表示できます。", diff --git a/src/i18n/locales/ko-KR/launch.json b/src/i18n/locales/ko-KR/launch.json index 5ce2b8e72..c315b12ce 100644 --- a/src/i18n/locales/ko-KR/launch.json +++ b/src/i18n/locales/ko-KR/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "Linux에서는 창 녹화 중에만 카메라 미리보기를 사용할 수 있습니다", + "title": "카메라 미리보기", + "show": "카메라 미리보기 표시", + "hide": "카메라 미리보기 숨기기", + "smaller": "미리보기 축소", + "larger": "미리보기 확대" + }, "tooltips": { "hideHUD": "HUD 숨기기", "hideHUDTip": "녹화 바를 숨깁니다. 알림 영역 아이콘에서 다시 표시할 수 있습니다.", diff --git a/src/i18n/locales/pt-BR/launch.json b/src/i18n/locales/pt-BR/launch.json index 886615088..d0aecac1f 100644 --- a/src/i18n/locales/pt-BR/launch.json +++ b/src/i18n/locales/pt-BR/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "No Linux, a prévia da câmera só está disponível durante a gravação de uma janela", + "title": "Prévia da câmera", + "show": "Mostrar prévia da câmera", + "hide": "Ocultar prévia da câmera", + "smaller": "Diminuir prévia", + "larger": "Aumentar prévia" + }, "tooltips": { "hideHUD": "Ocultar HUD", "hideHUDTip": "Ocultar a barra. Mostre-a de novo pelo ícone da bandeja do sistema.", diff --git a/src/i18n/locales/ru/launch.json b/src/i18n/locales/ru/launch.json index cdc103c0b..059243e02 100644 --- a/src/i18n/locales/ru/launch.json +++ b/src/i18n/locales/ru/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "В Linux предпросмотр камеры доступен только при записи окна", + "title": "Предпросмотр камеры", + "show": "Показать предпросмотр камеры", + "hide": "Скрыть предпросмотр камеры", + "smaller": "Уменьшить предпросмотр", + "larger": "Увеличить предпросмотр" + }, "tooltips": { "hideHUD": "Скрыть HUD", "hideHUDTip": "Скрыть панель. Вернуть её можно через значок в области уведомлений.", diff --git a/src/i18n/locales/tr/launch.json b/src/i18n/locales/tr/launch.json index 530157dd2..013406d45 100644 --- a/src/i18n/locales/tr/launch.json +++ b/src/i18n/locales/tr/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "Linux’ta kamera önizlemesi yalnızca pencere kaydı sırasında kullanılabilir", + "title": "Kamera önizlemesi", + "show": "Kamera önizlemesini göster", + "hide": "Kamera önizlemesini gizle", + "smaller": "Önizlemeyi küçült", + "larger": "Önizlemeyi büyüt" + }, "tooltips": { "hideHUD": "Kontrol panelini gizle", "hideHUDTip": "Araç çubuğunu gizle. Bildirim alanındaki simgeden yeniden gösterebilirsiniz.", diff --git a/src/i18n/locales/vi/launch.json b/src/i18n/locales/vi/launch.json index 53f452e27..89093813a 100644 --- a/src/i18n/locales/vi/launch.json +++ b/src/i18n/locales/vi/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "Trên Linux, bản xem trước camera chỉ khả dụng khi ghi một cửa sổ", + "title": "Xem trước camera", + "show": "Hiện bản xem trước camera", + "hide": "Ẩn bản xem trước camera", + "smaller": "Thu nhỏ bản xem trước", + "larger": "Phóng to bản xem trước" + }, "tooltips": { "hideHUD": "Ẩn HUD", "hideHUDTip": "Ẩn thanh ghi hình. Hiện lại từ biểu tượng ở khay hệ thống.", diff --git a/src/i18n/locales/zh-CN/launch.json b/src/i18n/locales/zh-CN/launch.json index a4bbf2c57..9e6bddb70 100644 --- a/src/i18n/locales/zh-CN/launch.json +++ b/src/i18n/locales/zh-CN/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "在 Linux 上,摄像头预览仅在录制窗口时可用", + "title": "摄像头预览", + "show": "显示摄像头预览", + "hide": "隐藏摄像头预览", + "smaller": "缩小预览", + "larger": "放大预览" + }, "tooltips": { "hideHUD": "隐藏录制栏", "hideHUDTip": "隐藏录制栏,可通过系统托盘图标重新显示", diff --git a/src/i18n/locales/zh-TW/launch.json b/src/i18n/locales/zh-TW/launch.json index 2e460592e..418413081 100644 --- a/src/i18n/locales/zh-TW/launch.json +++ b/src/i18n/locales/zh-TW/launch.json @@ -1,4 +1,12 @@ { + "floatingCamera": { + "windowCaptureOnly": "在 Linux 上,相機預覽僅在錄製視窗時可用", + "title": "攝影機預覽", + "show": "顯示攝影機預覽", + "hide": "隱藏攝影機預覽", + "smaller": "縮小預覽", + "larger": "放大預覽" + }, "tooltips": { "hideHUD": "隱藏控制面板", "hideHUDTip": "隱藏控制面板。可從系統匣圖示重新顯示。", diff --git a/src/lib/cameraPreview.test.ts b/src/lib/cameraPreview.test.ts new file mode 100644 index 000000000..0a2ebd6b3 --- /dev/null +++ b/src/lib/cameraPreview.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { cameraPreviewBounds, resizedCameraPreviewBounds } from "./cameraPreview"; + +const workArea = { x: 0, y: 25, width: 1512, height: 900 }; +const bar = { x: 300, y: 840, width: 900, height: 64 }; + +describe("camera preview placement", () => { + it("starts just above the recording bar's right end", () => { + const preview = cameraPreviewBounds(workArea, bar); + expect(preview.x + preview.width).toBe(bar.x + bar.width); + expect(preview.y + preview.height).toBe(bar.y - 8); + }); + + it("keeps the bottom centre fixed when resized near the bar", () => { + const before = cameraPreviewBounds(workArea, bar); + const after = resizedCameraPreviewBounds(before, workArea, 320); + expect(after.width).toBe(336); + expect(after.height).toBe(196); + expect(after.y + after.height).toBe(before.y + before.height); + expect(after.x + after.width / 2).toBe(before.x + before.width / 2); + }); + + it("preserves the full portrait, landscape and square frame at every preview size", () => { + for (const ratio of [9 / 16, 16 / 9, 1]) { + for (const width of [160, 224, 320]) { + const bounds = resizedCameraPreviewBounds(bar, workArea, width, ratio); + expect((bounds.width - 16) / (bounds.height - 16)).toBeCloseTo(ratio, 2); + } + } + }); + + it("keeps a preview on-screen on negative-origin displays and near the screen edge", () => { + const display = { x: -1280, y: -100, width: 1280, height: 720 }; + const preview = resizedCameraPreviewBounds( + { x: -1280, y: -100, width: 176, height: 176 }, + display, + 320, + ); + expect(preview.x).toBeGreaterThanOrEqual(display.x); + expect(preview.y).toBeGreaterThanOrEqual(display.y); + expect(preview.x + preview.width).toBeLessThanOrEqual(0); + expect(preview.y + preview.height).toBeLessThanOrEqual(620); + }); +}); diff --git a/src/lib/cameraPreview.ts b/src/lib/cameraPreview.ts new file mode 100644 index 000000000..8525b3247 --- /dev/null +++ b/src/lib/cameraPreview.ts @@ -0,0 +1,62 @@ +export const CAMERA_PREVIEW_SIZES = [160, 224, 320] as const; +export const CAMERA_PREVIEW_DEFAULT_SIZE = 224; +export const CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO = 16 / 9; +const WINDOW_PADDING = 16; + +type Rect = { x: number; y: number; width: number; height: number }; + +function previewSize(width: number, aspectRatio: number, workArea: Rect) { + const contentWidth = Math.min( + width, + workArea.width - WINDOW_PADDING, + (workArea.height - WINDOW_PADDING) * aspectRatio, + ); + return { + width: Math.round(contentWidth) + WINDOW_PADDING, + height: Math.round(contentWidth / aspectRatio) + WINDOW_PADDING, + }; +} + +function clampBounds( + x: number, + y: number, + size: { width: number; height: number }, + workArea: Rect, +): Rect { + return { + x: Math.round(Math.max(workArea.x, Math.min(x, workArea.x + workArea.width - size.width))), + y: Math.round(Math.max(workArea.y, Math.min(y, workArea.y + workArea.height - size.height))), + ...size, + }; +} + +/** Place the complete camera frame just above the bar's right end. */ +export function cameraPreviewBounds(workArea: Rect, bar?: Rect): Rect { + const size = previewSize( + CAMERA_PREVIEW_DEFAULT_SIZE, + CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO, + workArea, + ); + return clampBounds( + bar ? bar.x + bar.width - size.width : workArea.x + workArea.width / 2 + 160, + (bar?.y ?? workArea.y + workArea.height - 84) - size.height - 8, + size, + workArea, + ); +} + +/** Resize without cropping, retaining the bottom centre and distance from the bar. */ +export function resizedCameraPreviewBounds( + bounds: Rect, + workArea: Rect, + width: number, + aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO, +): Rect { + const size = previewSize(width, aspectRatio, workArea); + return clampBounds( + bounds.x + (bounds.width - size.width) / 2, + bounds.y + bounds.height - size.height, + size, + workArea, + ); +} diff --git a/technical-documentation/architecture/recording.md b/technical-documentation/architecture/recording.md index 2f643ec22..13b948997 100644 --- a/technical-documentation/architecture/recording.md +++ b/technical-documentation/architecture/recording.md @@ -22,6 +22,38 @@ The HUD starts and controls a recording. The source selector chooses a display o Electron applies `setContentProtection(true)` to the HUD window (`applyContentProtection` in `electron/windows.ts`). This keeps the controller out of captures and also makes it invisible in screenshots. It is skipped on macOS 26 and later, which never displays a protected window, and on Windows before 11 22H2 (build 22621), where it leaves the click-through HUD unable to take a click (#1105). On those Windows versions the HUD can appear in recordings; on macOS the ScreenCaptureKit helper still excludes it by window id. For a testing session only, `OPENSCREEN_DISABLE_CONTENT_PROTECTION=1` disables the protection; the code warns that the HUD then appears in captures. The tray icon is the reliable way to refocus OpenScreen or stop a recording when the click-through HUD is not convenient or is not visible. +## Presenter camera preview + +The HUD camera has a draggable rounded rectangular self-view with show/hide and three width choices. +It shows the complete camera frame at the shared recording stream’s aspect ratio, including when resized. +Its preview button is hidden when the camera is off. The preview shares +`useScreenRecorder`'s stream and stops with its owning HUD; it does not acquire another camera. +`useFloatingCameraPreview` publishes small JPEG frames to the auxiliary window, with bounded +payloads and one IPC request in flight. Audio preference broadcasts in the HUD preserve +its resolved camera selection so the preview does not blink when audio is toggled. + +On Windows, the renderer releases its validation stream before the native helper opens the +camera. A separate helper worker samples that same native camera at up to 5 fps into a bounded, +320px-wide top-down BMP mailbox, atomically replacing the latest frame in a private temporary +directory. Electron reads one frame at a time and relays a JPEG through the same preview +controller. Closing the helper stops polling and removes the directory, including incomplete +writes. The worker does not change the recording encoder or its clock. Rebuild the Windows +helper to enable preview during native takes; an older helper cannot publish those frames. + +Hiding or resizing retains the window id. macOS capture excludes that id alongside the HUD +and Notes, including windows hidden when capture starts. This requires rebuilding the native +helper. Windows uses content protection for the preview (which does not use click-through). +`OPENSCREEN_DISABLE_CONTENT_PROTECTION=1` also disables Windows preview exclusion: use it +only for UI inspection, never for capture-exclusion evidence. + +Linux has no portal API to exclude an app window from a monitor capture. Preview is available +while idle and during native **window** recordings. Before opening the portal, the controller +hides it; only a confirmed `window` source allows it to return. Monitor, virtual, unknown, and +browser-fallback takes keep it hidden, including after show/hide requests. The HUD explains +this limitation and disables the preview toggle until recording ends. The camera sidecar +continues recording normally. Supporting a visible preview during full-monitor Linux capture +would require compositor-specific exclusion support; this implementation does not claim it. + ## Capture backends | Platform | Backend | Code | Produces | diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 22b5601ee..06de5d381 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -1076,3 +1076,9 @@ These checks are the human slice: an agent cannot click Apple's picker (see abov | 2026-10-22 | PR #1177, maintainer presentation follow-up on `54ee4e15` (dev build, Electron 41.2.1) | macOS 15.8, Apple Silicon | **Partial: Agent usage presentation** | Real Electron driven through OS computer-use with an isolated `/tmp` profile and a local scripted Anthropic-compatible provider (synthetic counts, no paid provider). A two-call turn showed muted `≥10.3k tokens` beside the spinner, one expandable Thinking label while reasoning streamed, then `12.3k tokens · 45s` beneath the reply. Captured [running](https://raw.githubusercontent.com/auberginewly/openscreen-community/2cf6e8435c9214981243088aebc470380e6f8611/technical-documentation/testing/screenshots/pr-1177/running.png) and [completed](https://raw.githubusercontent.com/auberginewly/openscreen-community/2cf6e8435c9214981243088aebc470380e6f8611/technical-documentation/testing/screenshots/pr-1177/completed.png) screenshots. Component tests also cover partial totals with details confined to the tooltip, unavailable usage showing duration only, and unavailable usage without duration rendering nothing. Skipped: capture/preview/export, native binaries, paid-provider billing, restart/session switching (covered in the preceding pass), Windows/Linux and the rest of this checklist; chat presentation only. | | 2026-10-27 | Dev `feat/agent-execution-steps-1169` on `main` @ `5bb1b3fb`, Electron 41.2.1; Vite 5191, private `/tmp/openscreen-execution-ui-*` profile and local Anthropic-shaped fixture provider | macOS 15.8, Apple Silicon | **Partial — execution steps and Markdown** | Real Electron driven through OS computer-use. Sent a multiline Markdown request with a heading, list and indented code fence; observed two separate successful `getCurrentDocument` rows and one failed `getTranscript` row, with two provider-exposed reasoning blocks in order. Expanded reasoning and failure details. Completed answer rendered heading, list, code and GFM table. Corrected code-block contrast to use theme colors. New conversations remained empty; switching during the second turn cancelled it and saved a stopped assistant with five steps in its original session. Fully quit/relaunched, reopened Studio and verified both completed/stopped traces, failure labels and Markdown restored without usable old rewind controls. Final observer implementation was exercised in a third real SDK/HTTP turn with no duplicate rows. Automated tests additionally cover schema-invalid/unknown calls, out-of-order completions, failed-turn persistence, source copying and unsafe Markdown. Screenshots retained in the task evidence. Skipped: paid/remote providers, capture/audio/webcam, native preview/export, Windows/Linux runtime and the rest of this checklist; chat-only slice, not release-promotion QA. | | 2026-10-17 | Local source build and packaged 2.1.1 AppImage, `integration/linux-x11-capture`, based on `main` 2.1.1; rebuilt Linux capture helper, complete unchanged compositor/runtime directory from official 2.1.1 AppImage | Ubuntu 24.04.5, Xfce/X11, ScreenCast portal interface absent | **Partial — native X11 CLI capture/export passed** | Recorded an isolated GTK window through `record --window` for 12 s, with real OS mouse movement/press/release events through XTest. Saved 301 cursor samples, 7 click events and 4 cursor sprites; positions outside the moved window remained hidden. `export --auto-zoom` generated one region and produced a fully decodable H.264/AAC MP4, 1624×1080, 30 fps, 12.033 s. 158 of 360 decoded frames differed from the same export without zoom; inspected the rendered hold. Packaged AppImage also recorded the GTK window on an isolated X11 desktop with XTest: 286 cursor samples, 3 clicks and 1 sprite; CLI auto-zoom export decoded as 1624×1080 H.264 at 30 fps. Final AppImage also passed explicit `--display 0` monitor capture (309 samples, 3 clicks) and window capture (325 samples, 3 clicks). The source-index `--display` switch is removed from Chromium before GTK initializes, avoiding an X-server argument collision. A window-unmap test exposed finalized partial files being reported as success; nonzero helper exits now reject both pending and subsequent stops, with unit regression coverage and a CLI failure check. No ScreenCast portal was installed or activated. **Not covered:** the full HUD/editor checklist, Wayland, Windows, macOS, mixed-DPI monitor capture (explicitly rejected; window capture remains available), or changing window size during a take (stops with an error). Computer-use MCP was unavailable; the native CLI slice used real X11 events rather than injected DOM events. | +| 2026-10-16 | dev `main` @ `b149c55` + #1278 macOS camera preview; Electron 41.2.1; capture helper built from the working tree with `swiftc` + macOS 26.5 SDK (SHA256 `06f142f075e4dd776b653f174379b2795a60d750557655f9bca6e2d2596c8f99`) | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — preview slice passed** | `mcp__cua_repl`: live circular self-view, show/hide during recording, three sizes and limit buttons, disabled preview button when camera is off, and recording finalization into screen MP4 + webcam WebM. Exercised OS dragging without independently measuring the final position. A take started with the preview hidden and showed it during capture; a decoded late screen frame contained no floating circle. Default placement and resize anchoring are covered by geometry tests. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras, restart/cancel, export, exhaustive frame-by-frame exclusion, Windows/Linux and the remaining checklist. SwiftPM/XCTest unavailable locally; direct helper compilation passed. | +| 2026-10-19 | dev `codex/macos-floating-camera-preview`, rebased on `main` @ `3620cf8`, plus #1288 review changes; Electron 41.2.1; native helper rebuilt directly with `swiftc` + macOS 26.5 SDK | macOS 26.7, Apple M5 Pro, MacBook Pro Camera | **Partial — preview review slice passed** | Computer-use verified the complete rounded rectangular camera frame, aspect-preserving resize, camera-off hiding the HUD preview control, and show/hide during a finalized camera take. In the same take, after 5 s settling per condition, 30 s process-tree CPU deltas were **17.5% hidden / 22.3% shown** (100%=one core; includes Electron children and capture helper). Native `recording-stopped` reported **0 droppedVideoFrames over the entire take**, covering both intervals; this counts screen-writer backpressure/append failures, not upstream capture or webcam drops. Screen 1 at 3024×1964, saved webcam verified 1920×1080 (30 fps requested); mic/system audio off, system cursor. One sequential pair, no statistical claim. **Skipped:** Apple’s picker, fullscreen Spaces, other cameras/platforms, restart/cancel, export, exhaustive exclusion and remaining checklist. | +| 2026-10-19 | dev #1288 on `8c2608d9` + preview lifetime/quit fixes; Electron 41.2.1 | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — lifecycle slice passed** | Computer-use camera off/on removed/restored the HUD preview control. Normal Cmd+Q exited the OpenScreen Electron process with the preview enabled. Unit tests verify the same preview window survives initial camera-off and repeated toggles, with one owner-destruction listener and cleanup on HUD destruction. **Skipped:** recording, export, capture-exclusion measurements, other platforms and remaining checklist; no native capture changes in this slice. | +| 2026-10-21 | dev #1288 on `20975d30` + synchronous preview reservation before capture exclusions; Electron 41.2.1; existing helper rebuilt from this branch on Oct 19 | macOS 26.7, Apple Silicon, MacBook Pro Camera | **Partial — capture-exclusion ordering slice passed** | Computer-use started native display recording with the preview hidden, showed it during the take and finalized into the editor. Helper diagnostics resolved both requested app window IDs and reported zero screen-writer drops; a decoded late screen frame contained no floating preview. Unit regression covers reserving the exclusion window before any renderer configuration and reusing it when enabled later without altering an existing preview’s visibility. **Skipped:** Apple’s picker (ordering path checked in code), forced renderer-effect race on desktop (covered by unit regression), other platforms, export, exhaustive frame-by-frame exclusion and remaining checklist. | +| 2026-10-29 | Dev `codex/macos-floating-camera-preview`, rebased on `main` @ `5d82a0cb`, Windows/Linux preview port working tree; Electron 41.2.1; macOS ScreenCaptureKit helper rebuilt directly with `swiftc` and the macOS 26.5 SDK (SwiftPM unavailable because of a local BuildServerProtocol loader error) | macOS 26.7, Apple Silicon; built-in camera | **Partial — HUD controls only; capture/export evidence outstanding** | Real coordinate clicks through `mcp__cua_repl`: opened device settings, selected the built-in camera, toggled camera off/on, and verified the preview control disappears/reappears. Restored the original camera-on state and quit the dev app. The tool bound only the HUD window; it did not expose the separate camera window, so live preview pixels were not visually verified. Native preview bitmap conversion tests passed locally; publisher/controller tests cover camera handoff, blocked monitor captures, cleanup and bounded frame delivery. **Skipped:** recording, export, capture-exclusion proof and the requested demonstration clips, because the preview could not be inspected in this desktop session; real Windows/Linux runs and the complete Windows helper build, because those operating systems are unavailable here; all remaining checklist sections. Linux monitor/virtual/unknown/browser-fallback capture deliberately hides the preview because the portal has no window-exclusion API; only native window capture permits it during a take. This row is not release-promotion QA. | +| 2026-11-03 | PR #1288 dev branch rebased onto `main` @ `5e87ff39`, head `fe1122d3` plus review fixes; Electron dev launch with `OPENSCREEN_DISABLE_CONTENT_PROTECTION=1`, legacy source picker | macOS, Apple Silicon | **Partial — HUD toggle and quit passed; preview-window validation incomplete** | Real OS clicks through `mcp__cua_repl`: camera already enabled; Hide camera preview changed the HUD to Show camera preview; one click restored Hide camera preview. Cmd+Q exited the app with the preview enabled, without the close-to-hide handler blocking shutdown. **Skipped:** independent inspection of the separate preview window, resize controls and drag behavior (the computer-use app surface exposed only the HUD); real recording/exclusion/export and demo clips (not exercised in this slice); Windows/Linux desktop runs (no machines available). Unit regressions cover Windows cancel waiting for helper release, Restart avoiding renderer camera acquisition, system-menu close retaining the window/size and notifying the HUD, and Linux cancelled-start suspension restoration with stale-token protection. This row does not claim those platform paths were desktop-tested. |