fix(editor): record voiceovers against the uncut timeline (#1195)

Author: mikemikimike

src/components/ai-edition/NativeCompositorOverlay.test.tsx

@@ -18,14 +18,15 @@ import { publishNativePosition } from "@/native/nativeSync";
 const native = vi.hoisted(() => ({
 	setActiveClip: vi.fn(async () => ({ ok: true })),
 	setNativePlaying: vi.fn(),
+	setNativeScene: vi.fn(),
 }));
 
 vi.mock("@/native", () => ({
 	pushAllNativeParams: vi.fn(),
 	setActiveClip: native.setActiveClip,
 	setCurrentNativeViewId: vi.fn(),
 	setNativePlaying: native.setNativePlaying,
-	setNativeScene: vi.fn(),
+	setNativeScene: native.setNativeScene,
 	subscribeNativeCompositor: () => () => undefined,
 	useIsCpuCompositor: () => false,
 	useNativeCompositorView: () => ({ viewId: 7, error: null }),
@@ -94,6 +95,29 @@ function makeDocument(): AxcutDocument {
 	};
 }
 
+function makeTrimmedDocument(): AxcutDocument {
+	const document = makeDocument();
+	return {
+		...document,
+		timeline: {
+			...document.timeline,
+			clips: [{ ...FIRST, sourceEndSec: 10, timelineEndSec: 10 }],
+			trimRanges: [
+				{
+					id: "trim_a",
+					assetId: "asset_take",
+					clipId: "clip_a",
+					startSec: 4,
+					endSec: 6,
+					reason: "",
+					origin: "user",
+				},
+			],
+		},
+		legacyEditor: { speedRegions: [{ id: "speed_a", startMs: 0, endMs: 10_000, speed: 2 }] },
+	};
+}
+
 function setPlayhead(currentTimeSec: number, playing: boolean) {
 	act(() => {
 		useProjectStore.setState({ currentTimeSec, playing });
@@ -248,4 +272,34 @@ describe("NativeCompositorOverlay while playing", () => {
 		expect(native.setNativePlaying).toHaveBeenCalledWith(false);
 		expect(native.setActiveClip).toHaveBeenCalledTimes(1);
 	});
+
+	it("uses an untrimmed 1x scene while recording and restores the edited scene afterward", async () => {
+		useProjectStore.setState({ document: makeTrimmedDocument(), currentTimeSec: 8 });
+		const view = render(<NativeCompositorOverlay />);
+		await act(async () => {
+			await Promise.resolve();
+		});
+		native.setActiveClip.mockClear();
+		native.setNativeScene.mockClear();
+
+		view.rerender(<NativeCompositorOverlay playUneditedTimeline />);
+		await act(async () => {
+			await Promise.resolve();
+		});
+		const recordingScene = JSON.parse(native.setNativeScene.mock.lastCall?.[0] ?? "{}");
+		expect(recordingScene.clips).toHaveLength(1);
+		expect(recordingScene.speedRegions).toEqual([]);
+		expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 0, 8);
+
+		native.setActiveClip.mockClear();
+		native.setNativeScene.mockClear();
+		view.rerender(<NativeCompositorOverlay />);
+		await act(async () => {
+			await Promise.resolve();
+		});
+		const editedScene = JSON.parse(native.setNativeScene.mock.lastCall?.[0] ?? "{}");
+		expect(editedScene.clips).toHaveLength(2);
+		expect(editedScene.speedRegions.length).toBeGreaterThan(0);
+		expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 1, 8);
+	});
 });

src/components/ai-edition/NativeCompositorOverlay.tsx

@@ -55,7 +55,11 @@ import {
  * calques interactifs (zoom gimbal, annotations, drag webcam) qui restent des
  * éléments DOM cliquables au-dessus.
  */
-export function NativeCompositorOverlay() {
+export function NativeCompositorOverlay({
+	playUneditedTimeline = false,
+}: {
+	playUneditedTimeline?: boolean;
+}) {
 	const canvasRef = useRef<HTMLCanvasElement>(null);
 	const previousActiveClipIdRef = useRef<string | null>(null);
 	const document = useProjectStore((s) => s.document);
@@ -70,22 +74,25 @@ export function NativeCompositorOverlay() {
 		() => 0,
 	);
 
-	// `currentTimeSec` est en temps RAW/document (la règle où les trims occupent encore leur
-	// place — même référentiel que le playhead V4 et l'overlay webcam web). Le natif, lui, joue
-	// les segments COMPACTÉS (`resolveVisibleClips`, trims retirés) — le même flux que la SCÈNE
-	// envoyée via `setNativeScene`. `resolveNativePosition` fait le pont : il mappe le playhead
-	// RAW → temps source via le layout RAW (`document.timeline.clips`) PUIS localise le segment
-	// compacté correspondant, d'où sortent le `clipIndex` natif et la bonne paire écran/webcam.
-	// Sans ça (ancien `resolveNativePlaybackPosition(nativeClips, currentTimeSec)`), un playhead
-	// RAW lu contre des clips compactés désignait le mauvais clip après un trim → mauvaise caméra
-	// + décalage écran/cam.
-	const nativeClips = useMemo(() => {
-		if (!document) return [];
-		return resolveVisibleClips(document);
-	}, [document]);
+	// `currentTimeSec` stays on the raw document ruler. During normal playback the native scene
+	// uses trim-compressed segments; while recording, `playbackDocument` removes trims and speed
+	// regions so the compositor shows the same unedited timeline as the video clock.
+	const playbackDocument = useMemo(() => {
+		if (!document || !playUneditedTimeline) return document;
+		return {
+			...document,
+			timeline: { ...document.timeline, trimRanges: [] },
+			legacyEditor: document.legacyEditor ? { ...document.legacyEditor, speedRegions: [] } : null,
+		};
+	}, [document, playUneditedTimeline]);
+	const nativeClips = useMemo(
+		() => (playbackDocument ? resolveVisibleClips(playbackDocument) : []),
+		[playbackDocument],
+	);
 	const activePosition = useMemo(
-		() => resolveNativePosition(currentTimeSec, nativeClips, document?.timeline.clips ?? []),
-		[nativeClips, currentTimeSec, document],
+		() =>
+			resolveNativePosition(currentTimeSec, nativeClips, playbackDocument?.timeline.clips ?? []),
+		[nativeClips, currentTimeSec, playbackDocument],
 	);
 	const activeClip = activePosition?.clip ?? null;
 
@@ -149,12 +156,12 @@ export function NativeCompositorOverlay() {
 	// quand on pousse.
 	// biome-ignore lint/correctness/useExhaustiveDependencies: size revision
 	useEffect(() => {
-		if (viewId === null || !document) {
+		if (viewId === null || !playbackDocument) {
 			return;
 		}
 		try {
 			const webcamSourceSize = cameraPath ? getWebcamNativeSize(cameraPath) : null;
-			const scene = buildSceneDescription(document, webcamSourceSize);
+			const scene = buildSceneDescription(playbackDocument, webcamSourceSize);
 			setNativeScene(JSON.stringify(scene));
 		} catch (error) {
 			console.warn("[compositor-view] build/push scene failed:", error);
@@ -164,7 +171,7 @@ export function NativeCompositorOverlay() {
 		// re-read fresh via getWebcamNativeSize() above on every run (biome flags this as
 		// an "unnecessary" dependency, but removing it would mean a probed webcam size
 		// arriving after mount never gets pushed to native).
-	}, [viewId, document, sources, _webcamSizeRevision]);
+	}, [viewId, playbackDocument, sources, _webcamSizeRevision]);
 
 	// SYNCHRO COMPLETE DES PARAMS, en un seul endroit.
 	//
@@ -195,14 +202,17 @@ export function NativeCompositorOverlay() {
 	const activeSourceTimeSec = activePosition?.sourceTimeSec ?? null;
 	// Matched on the raw ruler, like `currentTimeSec` and the regions' pills.
 	const speedRegions = useMemo(
-		() => (document ? readSpeedRegions<SpeedRegion>(document) : []),
-		[document],
+		() => (playbackDocument ? readSpeedRegions<SpeedRegion>(playbackDocument) : []),
+		[playbackDocument],
 	);
-	const playbackSpeed = findActiveSpeedRegion(speedRegions, currentTimeSec * 1000)?.speed ?? 1;
+	const playbackSpeed = playUneditedTimeline
+		? 1
+		: (findActiveSpeedRegion(speedRegions, currentTimeSec * 1000)?.speed ?? 1);
 	const pendingTargetClipIdRef = useRef<string | null>(null);
 
 	const playing = useProjectStore((s) => s.playing);
 	const wasPlayingRef = useRef(playing);
+	const previousUneditedTimelineRef = useRef(playUneditedTimeline);
 
 	// Change les décodeurs screen/webcam quand le playhead entre dans un autre clip, à l'arrêt.
 	//
@@ -214,6 +224,8 @@ export function NativeCompositorOverlay() {
 	useEffect(() => {
 		const justPaused = wasPlayingRef.current && !playing;
 		wasPlayingRef.current = playing;
+		const playbackModeChanged = previousUneditedTimelineRef.current !== playUneditedTimeline;
+		previousUneditedTimelineRef.current = playUneditedTimeline;
 		if (
 			viewId === null ||
 			!document ||
@@ -230,7 +242,7 @@ export function NativeCompositorOverlay() {
 		// y retourne, sans quoi le recalage en temps (`setNativeTime`) chercherait dans le
 		// mauvais fichier.
 		const nativeElsewhere = justPaused && native !== null && native.clipIndex !== activeClipIndex;
-		if (!clipChanged && !nativeElsewhere) {
+		if (!clipChanged && !playbackModeChanged && !nativeElsewhere) {
 			return;
 		}
 		const asset = document.assets.find((candidate) => candidate.id === activeClip.assetId);
@@ -305,6 +317,7 @@ export function NativeCompositorOverlay() {
 		activeClipIndex,
 		activeSourceTimeSec,
 		playing,
+		playUneditedTimeline,
 		nativeClips,
 		playbackSpeed,
 	]);

src/components/ai-edition/NewEditorShell.tsx

@@ -106,15 +106,22 @@ const NO_AUDIO_TRACKS: AxcutAudioTrack[] = [];
 function NativePlaybackSync({
 	visibleClips,
 	clips,
+	playUneditedTimeline,
 }: {
 	visibleClips: AxcutClip[];
 	clips: AxcutClip[];
+	playUneditedTimeline: boolean;
 }) {
 	const playing = useProjectStore((s) => s.playing);
 	const currentTimeSec = useProjectStore((s) => s.currentTimeSec);
-	// visibleClips = trim-compressed native stream; `clips` = RAW layout currentTimeSec
-	// is measured against. resolveNativePosition needs both (see timelineMap).
-	useNativePlaybackSync(playing, currentTimeSec, visibleClips, clips);
+	// The native stream follows the edited timeline except while a voiceover records.
+	// `currentTimeSec` always stays on the raw ruler, including during playback.
+	useNativePlaybackSync(
+		playing,
+		currentTimeSec,
+		playUneditedTimeline ? clips : visibleClips,
+		clips,
+	);
 	return null;
 }
 
@@ -1012,12 +1019,20 @@ export function NewEditorShell() {
 	// where it is passed to the preview.
 	const [voiceoverRecording, setVoiceoverRecording] = useState(false);
 
-	const handleVoiceoverRecordingStart = useCallback(() => {
+	const handleVoiceoverRecordingStart = useCallback(async () => {
 		// Re-capture: the user may have scrubbed between opening the dialog and
 		// hitting Record, and playback starts from wherever the playhead is now.
-		voiceoverStartSecRef.current = useProjectStore.getState().currentTimeSec;
+		const doc = useProjectStore.getState().document;
+		const playhead = useProjectStore.getState().currentTimeSec;
+		voiceoverStartSecRef.current = playhead;
+		if (doc) {
+			const total = doc.timeline.clips.reduce((max, c) => Math.max(max, c.timelineEndSec), 0);
+			setVoiceoverFlow({ maxDurationSec: Math.max(0.5, total - playhead) });
+		}
 		setVoiceoverRecording(true);
-		if (videoElement?.paused) void videoElement.play().catch(() => undefined);
+		if (!videoElement) throw new Error("The preview is not ready.");
+		videoElement.playbackRate = 1;
+		if (videoElement.paused) await videoElement.play();
 	}, [videoElement]);
 
 	const handleVoiceoverRecordingStop = useCallback(() => {
@@ -1028,7 +1043,8 @@ export function NewEditorShell() {
 	const handleVoiceoverReady = useCallback(
 		async (assetId: string, durationSec: number) => {
 			setVoiceoverFlow(null);
-			await tl.addAudioTrack(assetId, voiceoverStartSecRef.current, {
+			const startSec = voiceoverStartSecRef.current;
+			await tl.addAudioTrack(assetId, startSec, {
 				kind: "voiceover",
 				durationSec,
 				spanSec: durationSec,
@@ -1558,7 +1574,11 @@ export function NewEditorShell() {
 			className={v4.app}
 			style={{ gridTemplateRows: `58px 1fr ${showTimeline ? timelineRow : "0px"}` }}
 		>
-			<NativePlaybackSync visibleClips={visibleClips} clips={clips} />
+			<NativePlaybackSync
+				visibleClips={visibleClips}
+				clips={clips}
+				playUneditedTimeline={voiceoverRecording}
+			/>
 			<EditorTopBar
 				mode={mode}
 				onModeChange={setMode}
@@ -1653,6 +1673,7 @@ export function NewEditorShell() {
 									clips={clips}
 									zoomRegions={tl.zoomRegions}
 									speedRegions={tl.speedRegions}
+									playUneditedTimeline={voiceoverRecording}
 									cameraFullscreenRegions={tl.cameraFullscreenRegions}
 									trimRanges={tl.trimRanges}
 									selectedZoomRegionId={tl.selection?.kind === "zoom" ? tl.selection.id : null}