Skip to content

Commit f77bc81

Browse files
committed
feat(camera): show the camera background on the crop thumbnail
The crop thumbnail segments its frame once, through a new `segmentFrame` addon call that runs the compositor's own model on one image, and lays the mask over the video in CSS. The blurred frame or the custom background sits under it. Nothing runs while the background is Original, and changing the mode, the blur or the wallpaper afterwards is CSS only.
1 parent 6dbab8d commit f77bc81

11 files changed

Lines changed: 306 additions & 38 deletions

File tree

‎crates/compositor-view-napi/src/lib.rs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,56 @@ pub fn segmentation_runtime_available() -> bool {
8787
openscreen_compositor::segmentation::runtime_available()
8888
}
8989

90+
pub struct SegmentFrameTask {
91+
model_path: String,
92+
rgba: Vec<u8>,
93+
}
94+
95+
impl Task for SegmentFrameTask {
96+
type Output = Vec<u8>;
97+
type JsValue = Buffer;
98+
99+
fn compute(&mut self) -> Result<Self::Output> {
100+
let rgb: Vec<u8> = self
101+
.rgba
102+
.chunks_exact(4)
103+
.flat_map(|p| [p[0], p[1], p[2]])
104+
.collect();
105+
// ponytail: une session ONNX chargée par appel (quelques dizaines de ms), parce qu'il
106+
// n'y a qu'un appel par caméra et par ouverture du panneau. À garder en cache si un
107+
// appelant se met à segmenter en continu.
108+
let mut segmenter = openscreen_compositor::segmentation::Segmenter::load(
109+
std::path::Path::new(&self.model_path),
110+
)
111+
.map_err(|e| Error::from_reason(format!("{e:#}")))?;
112+
segmenter
113+
.run(&rgb)
114+
.map(<[u8]>::to_vec)
115+
.map_err(|e| Error::from_reason(format!("{e:#}")))
116+
}
117+
118+
fn resolve(&mut self, _env: Env, mask: Self::Output) -> Result<Self::JsValue> {
119+
Ok(mask.into())
120+
}
121+
}
122+
123+
/// Masque du sujet pour UNE image, sans vue : la vignette de recadrage du panneau caméra s'en
124+
/// sert pour montrer le fond choisi sur sa propre frame.
125+
///
126+
/// `rgba` est une frame RGBA8 déjà réduite à la taille du modèle (`segmentation::MODEL_WIDTH` x
127+
/// `MODEL_HEIGHT`), soit le `getImageData` d'un canvas tel quel. Le retour a le format de
128+
/// `Segmenter::run` : un octet par pixel, 0 = fond, 255 = sujet.
129+
///
130+
/// `AsyncTask`, comme `remux_seekable` : charger la session prend plus longtemps que l'inférence,
131+
/// et ce temps n'a rien à faire sur le thread principal de Node.
132+
#[napi]
133+
pub fn segment_frame(model_path: String, rgba: Buffer) -> AsyncTask<SegmentFrameTask> {
134+
AsyncTask::new(SegmentFrameTask {
135+
model_path,
136+
rgba: rgba.to_vec(),
137+
})
138+
}
139+
90140
#[napi]
91141
pub fn create_view(
92142
rect: CompositorViewRect,

‎electron/ipc/nativeBridge.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,10 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) {
372372
return createSuccessResponse(requestId, {
373373
support: compositorViewService.probeSegmentation(),
374374
});
375+
case "segmentFrame":
376+
return createSuccessResponse(requestId, {
377+
mask: await compositorViewService.segmentFrame(request.payload.rgba),
378+
});
375379
case "setRect":
376380
compositorViewService.setRect(request.payload.id, request.payload.rect);
377381
return createSuccessResponse(requestId, { ok: true });

‎electron/native-bridge/services/compositorViewService.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -556,6 +556,20 @@ export class CompositorViewService {
556556
return resolveSceneAssetPath(SEGMENTATION_MODEL_ASSET) ? "ready" : "no-model";
557557
}
558558

559+
/** Subject mask for one camera frame, no view needed — see `segmentFrame` in the addon.
560+
* `null` when the addon (or this version of it) or the model is missing. */
561+
async segmentFrame(rgba: Uint8Array): Promise<Uint8Array | null> {
562+
const addon = this.ensureAddon();
563+
const modelPath = resolveSceneAssetPath(SEGMENTATION_MODEL_ASSET);
564+
if (!addon?.segmentFrame || !modelPath) {
565+
return null;
566+
}
567+
return addon.segmentFrame(
568+
modelPath,
569+
Buffer.from(rgba.buffer, rgba.byteOffset, rgba.byteLength),
570+
);
571+
}
572+
559573
/** Allocates an offscreen compositor view sized to `rect.width`x`rect.height`.
560574
* `rect.x` / `rect.y` are vestigial (ignored native-side) — the renderer
561575
* keeps them on the wire so the existing `CompositorViewRect` shape stays

‎electron/native/compositor-view/addon.d.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,11 @@ export interface CompositorViewAddon {
123123
* `--dir` build has none either. */
124124
segmentationRuntimeAvailable(): boolean;
125125

126+
/** Subject mask for ONE frame, outside any view — the crop thumbnail shows the camera
127+
* background with it. `rgba` is already at the model's size (256x144 RGBA8); resolves one
128+
* byte per pixel, 0 = background, 255 = subject. Optional: an older `.node` predates it. */
129+
segmentFrame?(modelPath: string, rgba: Buffer): Promise<Buffer>;
130+
126131
/** Allocates an offscreen compositor view sized to `rect.width`x`rect.height` (the
127132
* target preview resolution; `rect.x` / `rect.y` are vestigial and ignored native-side).
128133
* No HWND/native-window-handle is passed: there's no OS window to parent to. The

‎src/components/ai-edition/NewEditorShell.module.css‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -350,7 +350,7 @@
350350
var(--scene-foliage-d) 100%);
351351
/* ponytail: the wallpaper card is a floating, rounded surface (per the v4
352352
design's preview frame) — rounded corners + a layered drop shadow lift it
353-
off the scene gradient. The custom background (buildFrameStyle) fills this
353+
off the scene gradient. The custom background (wallpaperStyle) fills this
354354
card; the video sits on top with its own padding so the wallpaper shows
355355
around it. */
356356
border-radius: 16px;

‎src/components/ai-edition/PreviewCanvas.tsx‎

Lines changed: 2 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ import {
5757
type WebcamCompositeLayout,
5858
} from "@/lib/compositeLayout";
5959
import { webcamAnchorAt } from "@/lib/projectDefaults";
60-
import { classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper";
60+
import { wallpaperStyle } from "@/lib/wallpaper";
6161
import { getCssClipPath } from "@/lib/webcamMaskShapes";
6262
import { computeCameraFullscreenProgress } from "@/lib/zoomMath/cameraFullscreenUtils";
6363
import { AnnotationLayer } from "./AnnotationLayer";
@@ -321,7 +321,7 @@ export function PreviewCanvas(props: PreviewCanvasProps) {
321321
// The stage hosting the interactive overlays is the CONTENT rect, and a frame never changes
322322
// it: the compositor keeps the footage the same size with or without a frame and grows the
323323
// frame outward, so every handle (annotations, privacy blur, zoom focus) sits on `layout`.
324-
const frameStyle = useMemo(() => buildFrameStyle(settings), [settings]);
324+
const frameStyle = useMemo(() => wallpaperStyle(settings.wallpaper), [settings.wallpaper]);
325325
const screenStyle = useMemo(
326326
() => buildScreenStyle(layout, settings, frameSize),
327327
[layout, settings, frameSize],
@@ -509,38 +509,6 @@ export function PreviewCanvas(props: PreviewCanvasProps) {
509509
);
510510
}
511511

512-
// ponytail: resolve `settings.wallpaper` to an actual CSS background. Image
513-
// wallpapers must go through resolveImageWallpaperUrl (→ getAssetPath): in the
514-
// packaged Electron app the renderer loads over file://, where a bare
515-
// `/wallpapers/foo.jpg` points at the filesystem root and 404s, so the custom
516-
// background silently failed to paint (worked in the http dev server only).
517-
// classifyWallpaper also handles color-function (rgb/hsl/…) and every gradient
518-
// variant, which the old ad-hoc startsWith checks missed.
519-
function resolveWallpaperImageUrl(imagePath: string): string | null {
520-
try {
521-
return resolveImageWallpaperUrl(imagePath);
522-
} catch {
523-
return null;
524-
}
525-
}
526-
527-
// Canvas: wallpaper only — no padding, no shadow.
528-
function buildFrameStyle(
529-
settings: ReturnType<typeof useEditorSettings>["settings"],
530-
): React.CSSProperties {
531-
const w = classifyWallpaper(settings.wallpaper);
532-
if (w.kind === "color") return { backgroundColor: w.value };
533-
if (w.kind === "gradient") return { backgroundImage: w.value, backgroundSize: "cover" };
534-
const url = resolveWallpaperImageUrl(w.path);
535-
if (!url) return {};
536-
return {
537-
backgroundImage: `url(${url})`,
538-
backgroundSize: "cover",
539-
backgroundPosition: "center",
540-
backgroundRepeat: "no-repeat",
541-
};
542-
}
543-
544512
// Screen stage: rectangle from the composite layout, converted to percentages
545513
// of the canvas. Positions/sizes the interactive overlays (ZoomFocusOverlay,
546514
// AnnotationLayer) hosted inside it and the CSS-hidden <video> (decode/clock

‎src/components/ai-edition/RightPanes.layout.test.tsx‎

Lines changed: 76 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
// the saved preference is left untouched on disk and the help popover says so.
66

77
import "@testing-library/jest-dom";
8-
import { cleanup, fireEvent, render, screen, within } from "@testing-library/react";
8+
import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
99
import userEvent from "@testing-library/user-event";
1010
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
1111
import { I18nProvider } from "@/contexts/I18nContext";
@@ -14,6 +14,15 @@ import { type AxcutDocument, createEmptyDocument } from "@/lib/ai-edition/schema
1414
import { useProjectStore } from "@/lib/ai-edition/store/projectStore";
1515
import { LayoutPane } from "./RightPanes";
1616

17+
const segmentation = vi.hoisted(() => ({ supported: false, segmentCameraFrame: vi.fn() }));
18+
vi.mock("@/native/hooks/useSegmentationSupport", () => ({
19+
useCanSegmentCamera: () => segmentation.supported,
20+
}));
21+
vi.mock("@/native/compositorViewClient", async (importOriginal) => ({
22+
...(await importOriginal<typeof import("@/native/compositorViewClient")>()),
23+
segmentCameraFrame: segmentation.segmentCameraFrame,
24+
}));
25+
1726
function seedProject(hasCamera: boolean): AxcutDocument {
1827
const base = createEmptyDocument({ projectId: "project_layout", title: "Layout" });
1928
return {
@@ -154,6 +163,72 @@ describe("LayoutPane webcam framing picture", () => {
154163
});
155164
});
156165

166+
// The thumbnail shows the camera background the preview draws. The promise that makes it
167+
// cheap: nothing runs while the background is the camera's own, and the frame is segmented
168+
// once, every later change of mode, blur or wallpaper being CSS over the same mask.
169+
describe("LayoutPane webcam framing background", () => {
170+
beforeEach(() => {
171+
segmentation.supported = true;
172+
segmentation.segmentCameraFrame.mockResolvedValue(new Uint8Array(256 * 144).fill(255));
173+
// jsdom has no 2D canvas; the thumbnail only draws into one and reads it back.
174+
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
175+
drawImage: vi.fn(),
176+
getImageData: (_x: number, _y: number, w: number, h: number) => ({
177+
data: new Uint8ClampedArray(w * h * 4),
178+
}),
179+
createImageData: (w: number, h: number) => ({ data: new Uint8ClampedArray(w * h * 4) }),
180+
putImageData: vi.fn(),
181+
} as unknown as CanvasRenderingContext2D);
182+
vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue("data:image/png;base64,AA");
183+
});
184+
afterEach(() => {
185+
segmentation.supported = false;
186+
segmentation.segmentCameraFrame.mockReset();
187+
vi.restoreAllMocks();
188+
});
189+
190+
const withBackground = (webcamBackgroundMode: string) => {
191+
const doc = seedProject(true);
192+
return { ...doc, legacyEditor: { ...doc.legacyEditor, webcamBackgroundMode } };
193+
};
194+
/** Loads the thumbnail's frame the way the browser would: metadata, then the seek. */
195+
const showFrame = () => {
196+
const video = document.querySelector('[class*="framingVideo"]') as HTMLVideoElement;
197+
Object.defineProperty(video, "videoWidth", { configurable: true, value: 640 });
198+
Object.defineProperty(video, "videoHeight", { configurable: true, value: 480 });
199+
fireEvent.loadedMetadata(video);
200+
fireEvent.seeked(video);
201+
return video;
202+
};
203+
const backdrop = () =>
204+
document.querySelector('[class*="framingBox"] > div[aria-hidden="true"]') as HTMLElement | null;
205+
206+
it("segments nothing while the camera keeps its own background", () => {
207+
renderLayout(withBackground("none"));
208+
const video = showFrame();
209+
210+
expect(segmentation.segmentCameraFrame).not.toHaveBeenCalled();
211+
expect(video.style.maskImage).toBe("");
212+
expect(backdrop()).toBeNull();
213+
});
214+
215+
it("cuts the subject out once, then restyles the background around it", async () => {
216+
renderLayout(withBackground("blur"));
217+
const video = showFrame();
218+
219+
await waitFor(() => expect(video.style.maskImage).toContain("data:image/png"));
220+
expect(backdrop()?.style.filter).toMatch(/^blur\(/);
221+
222+
fireEvent.click(screen.getByRole("button", { name: "Custom" }));
223+
expect(backdrop()?.style.filter).toBe("");
224+
fireEvent.click(screen.getByRole("button", { name: "Cutout" }));
225+
expect(backdrop()).toBeNull();
226+
expect(video.style.maskImage).toContain("data:image/png");
227+
228+
expect(segmentation.segmentCameraFrame).toHaveBeenCalledTimes(1);
229+
});
230+
});
231+
157232
describe("LayoutPane picture-in-picture camera", () => {
158233
const stored = () => useProjectStore.getState().document?.legacyEditor as Record<string, unknown>;
159234

0 commit comments

Comments
 (0)