fix: draw clip filters in the preview as the render applies them (#265)

Author: dazzatronus

src/components/canvas/players/color-filter.ts

@@ -0,0 +1,133 @@
+import type { ResolvedClip } from "@schemas";
+import { defaultFilterVert, Filter, GlProgram, GpuProgram, UniformGroup } from "pixi.js";
+
+/**
+ * Recolours a clip with the render's brightness, saturation and negate maths.
+ *
+ * Per unpremultiplied pixel: contrast `f · (c − mid) + mid`, then `+ brightness`, each clamped; then saturation
+ * around the HSP brightness `√(.299r² + .587g² + .114b²)`; then inversion.
+ */
+
+interface ColorParams {
+	/** −255…255 */
+	contrast?: number;
+	brightness?: number;
+	saturation?: number;
+	negate?: boolean;
+}
+
+export const COLOR_FILTERS: Partial<Record<NonNullable<ResolvedClip["filter"]>, ColorParams>> = {
+	boost: { saturation: 1.75 },
+	contrast: { contrast: 15 },
+	darken: { brightness: -0.15 },
+	greyscale: { saturation: 0 },
+	lighten: { brightness: 0.15 },
+	muted: { contrast: 85, brightness: -0.05, saturation: 0.6 },
+	negative: { negate: true }
+};
+
+const MID = "0.5019608"; // 128 / 255
+const HSP = "0.299, 0.587, 0.114";
+
+const glFragment = `
+in vec2 vTextureCoord;
+out vec4 finalColor;
+
+uniform sampler2D uTexture;
+uniform float uContrast;
+uniform float uBrightness;
+uniform float uSaturation;
+uniform float uNegate;
+
+void main(void)
+{
+    vec4 color = texture(uTexture, vTextureCoord);
+    if (color.a == 0.0) {
+        finalColor = color;
+        return;
+    }
+    vec3 rgb = clamp(uContrast * (color.rgb / color.a - ${MID}) + ${MID}, 0.0, 1.0);
+    rgb = clamp(rgb + uBrightness, 0.0, 1.0);
+    float p = sqrt(dot(rgb * rgb, vec3(${HSP})));
+    rgb = clamp(p + (rgb - p) * uSaturation, 0.0, 1.0);
+    rgb = mix(rgb, 1.0 - rgb, uNegate);
+    finalColor = vec4(rgb * color.a, color.a);
+}
+`;
+
+const gpuSource = `
+struct GlobalFilterUniforms {
+  uInputSize:vec4<f32>,
+  uInputPixel:vec4<f32>,
+  uInputClamp:vec4<f32>,
+  uOutputFrame:vec4<f32>,
+  uGlobalFrame:vec4<f32>,
+  uOutputTexture:vec4<f32>,
+};
+
+struct ColorUniforms {
+  uContrast:f32,
+  uBrightness:f32,
+  uSaturation:f32,
+  uNegate:f32,
+};
+
+@group(0) @binding(0) var<uniform> gfu: GlobalFilterUniforms;
+@group(0) @binding(1) var uTexture: texture_2d<f32>;
+@group(0) @binding(2) var uSampler : sampler;
+@group(1) @binding(0) var<uniform> colorUniforms : ColorUniforms;
+
+struct VSOutput {
+    @builtin(position) position: vec4<f32>,
+    @location(0) uv : vec2<f32>
+};
+
+fn filterVertexPosition(aPosition:vec2<f32>) -> vec4<f32>
+{
+    var position = aPosition * gfu.uOutputFrame.zw + gfu.uOutputFrame.xy;
+    position.x = position.x * (2.0 / gfu.uOutputTexture.x) - 1.0;
+    position.y = position.y * (2.0*gfu.uOutputTexture.z / gfu.uOutputTexture.y) - gfu.uOutputTexture.z;
+    return vec4(position, 0.0, 1.0);
+}
+
+fn filterTextureCoord( aPosition:vec2<f32> ) -> vec2<f32>
+{
+    return aPosition * (gfu.uOutputFrame.zw * gfu.uInputSize.zw);
+}
+
+@vertex
+fn mainVertex(@location(0) aPosition : vec2<f32>) -> VSOutput {
+  return VSOutput(filterVertexPosition(aPosition), filterTextureCoord(aPosition));
+}
+
+@fragment
+fn mainFragment(@location(0) uv: vec2<f32>, @builtin(position) position: vec4<f32>) -> @location(0) vec4<f32> {
+    let color = textureSample(uTexture, uSampler, uv);
+    if (color.a == 0.0) {
+        return color;
+    }
+    var rgb = saturate(colorUniforms.uContrast * (color.rgb / color.a - ${MID}) + ${MID});
+    rgb = saturate(rgb + colorUniforms.uBrightness);
+    let p = sqrt(dot(rgb * rgb, vec3<f32>(${HSP})));
+    rgb = saturate(p + (rgb - p) * colorUniforms.uSaturation);
+    rgb = select(rgb, 1.0 - rgb, colorUniforms.uNegate > 0.5);
+    return vec4<f32>(rgb * color.a, color.a);
+}
+`;
+
+export class ColorFilter extends Filter {
+	constructor({ contrast = 0, brightness = 0, saturation = 1, negate = false }: ColorParams) {
+		const glProgram = GlProgram.from({ vertex: defaultFilterVert, fragment: glFragment, name: "color-filter" });
+		const gpuProgram = GpuProgram.from({
+			vertex: { source: gpuSource, entryPoint: "mainVertex" },
+			fragment: { source: gpuSource, entryPoint: "mainFragment" }
+		});
+		const colorUniforms = new UniformGroup({
+			uContrast: { value: (259 * (contrast + 255)) / (255 * (259 - contrast)), type: "f32" },
+			uBrightness: { value: brightness, type: "f32" },
+			uSaturation: { value: saturation, type: "f32" },
+			uNegate: { value: negate ? 1 : 0, type: "f32" }
+		});
+		super({ glProgram, gpuProgram, resources: { colorUniforms }, padding: 0 });
+	}
+}

src/components/canvas/players/player.ts

@@ -23,6 +23,8 @@ import * as pixi from "pixi.js";
 
 import { Entity } from "../../../core/shared/entity";
 
+import { COLOR_FILTERS, ColorFilter } from "./color-filter";
+
 /**
  * Tracks a merge field binding for a specific property path.
  * Used to restore placeholders on export for properties that haven't changed.
@@ -52,6 +54,21 @@ export enum PlayerType {
 	RichCaption = "rich-caption"
 }
 
+// The render filters clips it draws as images and leaves those it plays as video untouched.
+const FILTERED_PLAYERS = new Set([
+	PlayerType.Image,
+	PlayerType.Text,
+	PlayerType.RichText,
+	PlayerType.Shape,
+	PlayerType.Caption,
+	PlayerType.Html,
+	PlayerType.Svg,
+	PlayerType.TextToImage
+]);
+
+// pixi's Gaussian at this strength spreads as far as the render's radius-10 box blur (σ ≈ 6.06 px).
+const CLIP_BLUR_STRENGTH = 4.69;
+
 /**
  * Base class for all visual content players in the canvas.
  *
@@ -101,6 +118,9 @@ export abstract class Player extends Entity {
 	private wipeFilter: WipeFilter | null = null;
 	private zoomBlurBuilder?: KeyframeBuilder;
 	private zoomBlurFilter: pixi.BlurFilter | null = null;
+	private zoomBlurRadius = 0;
+	private clipFilterName: ResolvedClip["filter"];
+	private clipFilter: pixi.Filter | null = null;
 	private rotationOrigin?: { x: KeyframeBuilder; y: KeyframeBuilder };
 
 	private wipeMask: pixi.Graphics | null;
@@ -126,6 +146,7 @@ export abstract class Player extends Entity {
 		this.contentContainer = new pixi.Container();
 		this.lumaWrapper.addChild(this.contentContainer);
 		this.getContainer().addChild(this.lumaWrapper);
+		this.getContainer().onRender = () => this.sizeBlurs();
 	}
 
 	public reconfigureAfterRestore(): void {
@@ -265,6 +286,8 @@ export abstract class Player extends Entity {
 			this.applyFixedDimensions();
 		}
 
+		this.updateClipFilter();
+
 		// Update wipe/reveal mask animation
 		this.updateWipeMask();
 		this.updateWipeFilter();
@@ -301,14 +324,43 @@ export abstract class Player extends Entity {
 		this.wipeMask.fill(0xffffff);
 	}
 
+	private updateClipFilter(): void {
+		const { asset } = this.clipConfiguration;
+		const rendersAsVideo = asset.type === "rich-text" && Boolean(asset.animation?.preset);
+		const name = FILTERED_PLAYERS.has(this.playerType) && !rendersAsVideo ? this.clipConfiguration.filter : undefined;
+
+		if (name !== this.clipFilterName) {
+			this.clipFilter?.destroy();
+			this.clipFilter = this.createClipFilter(name);
+			this.clipFilterName = name;
+			this.syncContentFilters();
+		}
+	}
+
+	private createClipFilter(name: ResolvedClip["filter"]): pixi.Filter | null {
+		if (name === "blur") {
+			const blur = new pixi.BlurFilter();
+			// The render's box blur clamps at the clip's edges, so they stay sharp.
+			blur.repeatEdgePixels = true;
+			return blur;
+		}
+		const color = name && COLOR_FILTERS[name];
+		return color ? new ColorFilter(color) : null;
+	}
+
+	private syncContentFilters(): void {
+		// The render filters a clip before its transitions.
+		this.contentContainer.filters = [this.clipFilter, this.wipeFilter].filter((filter): filter is pixi.Filter => filter !== null);
+	}
+
 	private updateWipeFilter(): void {
 		if (!this.wipeBrightnessBuilder) {
 			this.removeWipeFilter();
 			return;
 		}
 		if (!this.wipeFilter) {
 			this.wipeFilter = new WipeFilter();
-			this.contentContainer.filters = [this.wipeFilter];
+			this.syncContentFilters();
 		}
 		const time = this.getPlaybackTime();
 		this.wipeFilter.brightness = this.wipeBrightnessBuilder.getValue(time);
@@ -317,9 +369,9 @@ export abstract class Player extends Entity {
 
 	private removeWipeFilter(): void {
 		if (!this.wipeFilter) return;
-		this.contentContainer.filters = [];
 		this.wipeFilter.destroy();
 		this.wipeFilter = null;
+		this.syncContentFilters();
 	}
 
 	private updateZoomBlur(): void {
@@ -334,14 +386,19 @@ export abstract class Player extends Entity {
 			this.zoomBlurFilter = new pixi.BlurFilter();
 			this.getContainer().filters = [this.zoomBlurFilter];
 		}
-		// The render blurs by a whole-pixel radius before applying the clip's scale, so on screen the blur grows with scale
-		// and the canvas zoom. pixi's Gaussian approximates the render's three box-blur passes.
-		const time = this.getPlaybackTime();
-		const radius = Math.trunc(this.zoomBlurBuilder.getValue(time));
-		const canvas = this.getContainer().parent?.worldTransform;
-		const canvasZoom = canvas ? Math.hypot(canvas.a, canvas.b) : 1;
-		this.zoomBlurFilter.enabled = radius > 0;
-		this.zoomBlurFilter.strength = radius * (this.scaleKeyframeBuilder?.getValue(time) ?? 1) * canvasZoom;
+		this.zoomBlurRadius = Math.trunc(this.zoomBlurBuilder.getValue(this.getPlaybackTime()));
+		this.zoomBlurFilter.enabled = this.zoomBlurRadius > 0;
+	}
+
+	// The render blurs by whole pixels before applying the clip's scale, so a blur grows with scale and the zoom it is
+	// drawn at. Sizing it as each frame draws follows that zoom: the editor's on screen, 1:1 in a capture.
+	private sizeBlurs(): void {
+		if (!this.zoomBlurFilter && !(this.clipFilter instanceof pixi.BlurFilter)) return;
+		const canvas = this.getContainer().parent?.getGlobalTransform();
+		const scale = (this.scaleKeyframeBuilder?.getValue(this.getPlaybackTime()) ?? 1) * (canvas ? Math.hypot(canvas.a, canvas.b) : 1);
+		// pixi's Gaussian approximates the render's three box-blur passes.
+		if (this.zoomBlurFilter) this.zoomBlurFilter.strength = this.zoomBlurRadius * scale;
+		if (this.clipFilter instanceof pixi.BlurFilter) this.clipFilter.strength = CLIP_BLUR_STRENGTH * scale;
 	}
 
 	public override dispose(): void {
@@ -351,6 +408,9 @@ export abstract class Player extends Entity {
 		this.wipeFilter = null;
 		this.zoomBlurFilter?.destroy();
 		this.zoomBlurFilter = null;
+		this.clipFilter?.destroy();
+		this.clipFilter = null;
+		this.clipFilterName = undefined;
 
 		this.contentContainer?.destroy();
 		this.lumaWrapper?.destroy();

tests/helpers/pixi-mock-filters.ts

@@ -1,10 +1,17 @@
-/** pixi.js filter/shader stubs so WipeFilter (extends pixi.Filter at module load) survives a mocked pixi. */
+/** pixi.js filter/shader stubs so the player's filters (some extend pixi.Filter at module load) survive a mocked pixi. */
 /* eslint-disable max-classes-per-file, class-methods-use-this, import/prefer-default-export -- minimal pixi API stubs for jest mocks */
 
 export const pixiFilterStubs = {
 	Filter: class {
 		destroy(): void {}
 	},
+	BlurFilter: class {
+		strength = 0;
+
+		repeatEdgePixels = false;
+
+		destroy(): void {}
+	},
 	GlProgram: { from: (): Record<string, unknown> => ({}) },
 	GpuProgram: { from: (): Record<string, unknown> => ({}) },
 	UniformGroup: class {

tests/player-filters.test.ts

@@ -0,0 +1,75 @@
+import { WipeFilter } from "@animations/wipe-filter";
+import { ColorFilter } from "@canvas/players/color-filter";
+import { Player, PlayerType } from "@canvas/players/player";
+import type { Edit } from "@core/edit-session";
+import { sec } from "@core/timing/types";
+import type { Size } from "@layouts/geometry";
+import type { ResolvedClip } from "@schemas";
+
+jest.mock("pixi.js", () => {
+	const point = () => ({ set: jest.fn() });
+	return {
+		// eslint-disable-next-line global-require, @typescript-eslint/no-require-imports -- jest.mock factories can't use imports
+		...require("./helpers/pixi-mock-filters").pixiFilterStubs,
+		Container: jest.fn().mockImplementation(() => ({ addChild: jest.fn(), scale: point(), pivot: point(), position: point(), skew: point() }))
+	};
+});
+
+class StillPlayer extends Player {
+	public override getSize(): Size {
+		return this.edit.size;
+	}
+}
+
+const draw = (playerType: PlayerType, fields: Partial<ResolvedClip>): Player => {
+	const edit = { playbackTime: 1, size: { width: 1920, height: 1080 } };
+	const clip = { id: "clip-1", asset: { type: "image", src: "https://example.com/a.png" }, start: sec(0), length: sec(5), ...fields };
+	const player = new StillPlayer(edit as unknown as Edit, clip as ResolvedClip, playerType);
+	player.reconfigureAfterRestore();
+	player.applyPlayheadState();
+	return player;
+};
+
+const filtersOf = (player: Player) => player.getContentContainer().filters ?? [];
+
+describe("Player clip filters", () => {
+	it("draws an image through its filter", () => {
+		expect(filtersOf(draw(PlayerType.Image, { filter: "negative" }))).toEqual([expect.any(ColorFilter)]);
+	});
+
+	it("blurs with edges clamped like the render's box blur, sized by the clip's scale as it draws", () => {
+		const player = draw(PlayerType.Image, { filter: "blur", scale: 0.5 });
+		player.getContainer().onRender?.({} as never);
+		const [blur] = filtersOf(player) as unknown as { repeatEdgePixels: boolean; strength: number }[];
+
+		expect(blur.repeatEdgePixels).toBe(true);
+		expect(blur.strength).toBeCloseTo(4.69 * 0.5);
+	});
+
+	it("leaves clips the render draws as video unfiltered", () => {
+		expect(filtersOf(draw(PlayerType.Video, { filter: "greyscale" }))).toEqual([]);
+		expect(filtersOf(draw(PlayerType.Html5, { filter: "greyscale" }))).toEqual([]);
+	});
+
+	it("filters rich text only while it renders as an image", () => {
+		const text = { type: "rich-text", text: "Hi" } as ResolvedClip["asset"];
+		const animated = { ...text, animation: { preset: "typewriter" } } as ResolvedClip["asset"];
+
+		expect(filtersOf(draw(PlayerType.RichText, { asset: text, filter: "darken" }))).toEqual([expect.any(ColorFilter)]);
+		expect(filtersOf(draw(PlayerType.RichText, { asset: animated, filter: "darken" }))).toEqual([]);
+	});
+
+	it("applies the filter before a wipe transition", () => {
+		const player = draw(PlayerType.Image, { filter: "lighten", transition: { in: "wipeRight" } });
+
+		expect(filtersOf(player)).toEqual([expect.any(ColorFilter), expect.any(WipeFilter)]);
+	});
+
+	it("drops the filter when the clip clears it", () => {
+		const player = draw(PlayerType.Image, { filter: "muted" });
+		player.clipConfiguration.filter = "none";
+		player.applyPlayheadState();
+
+		expect(filtersOf(player)).toEqual([]);
+	});
+});