diff --git a/docs/public/guides/migrating-to-0.2.md b/docs/public/guides/migrating-to-0.2.md index 7103cfce..e71e1ce0 100644 --- a/docs/public/guides/migrating-to-0.2.md +++ b/docs/public/guides/migrating-to-0.2.md @@ -6,10 +6,10 @@ summary: Update playback gates, playback feedback, and interaction highlights fr # Migrating to 0.2 -Upgrade to `0.2.8` with: +Upgrade to `0.2.9` with: ```sh -npm install supervision@0.2.8 +npm install supervision@0.2.9 ``` The engine remains a subpath of `supervision`; do not add a separate engine or diff --git a/docs/public/guides/public-api.md b/docs/public/guides/public-api.md index e3203538..84dada02 100644 --- a/docs/public/guides/public-api.md +++ b/docs/public/guides/public-api.md @@ -569,6 +569,14 @@ still returns the presented source pixels. Brightness/contrast adjustments retai the configured renderer resolution, including on Retina displays. Omit the option for smooth sampling at every zoom. +`renderer.setDisplayAdjustments({ brightness, contrast, gamma, autoContrast })` +changes only media display. Brightness, contrast and gamma default to `1`; gamma +above `1` brightens midtones. For still images, `autoContrast: true` applies +Contrast Limited Adaptive Histogram Equalization (CLAHE) instead of manual +contrast. Brightness and gamma remain active. Auto contrast is ignored for video; +passing an empty object restores the unadjusted display. Source pixels and +annotation styles are unchanged. + `renderer.setViewportTransform({ scale, x, y })` applies an exact transform in natural media pixels and container-relative CSS pixels. Explicit updates work while the viewport is locked and can restore scales outside the ordinary gesture diff --git a/docs/public/typedoc-icons.js b/docs/public/typedoc-icons.js index 707611cb..c17299b2 100644 --- a/docs/public/typedoc-icons.js +++ b/docs/public/typedoc-icons.js @@ -2,7 +2,7 @@ (function () { const packageName = "supervision"; - const packageVersion = "0.2.8"; + const packageVersion = "0.2.9"; const packageReleaseStatus = ""; const kindIconMap = { Accessor: "A", diff --git a/package-lock.json b/package-lock.json index 1991df60..94eb5887 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10791,7 +10791,7 @@ }, "packages/web": { "name": "supervision", - "version": "0.2.8", + "version": "0.2.9", "license": "MIT", "dependencies": { "mediabunny": "^1.55.5", diff --git a/packages/core/src/types/media-rendering.ts b/packages/core/src/types/media-rendering.ts index 97f261ed..65c18984 100644 --- a/packages/core/src/types/media-rendering.ts +++ b/packages/core/src/types/media-rendering.ts @@ -322,6 +322,10 @@ export interface AnnotationVisibility { } export interface MediaDisplayAdjustments { + /** Gamma correction; 1 is unchanged and larger values brighten midtones. */ + readonly gamma?: number; + /** CLAHE for still images. Replaces manual contrast; ignored for video. */ + readonly autoContrast?: boolean; /** Normalized contrast where 1 is unchanged. */ readonly contrast?: number; /** Normalized brightness where 1 is unchanged. */ diff --git a/packages/web/package.json b/packages/web/package.json index 790511a4..93c42b24 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -1,6 +1,6 @@ { "name": "supervision", - "version": "0.2.8", + "version": "0.2.9", "description": "Browser-native computer vision media rendering and annotation engine.", "repository": { "type": "git", diff --git a/packages/web/src/renderers/media-clahe.test.ts b/packages/web/src/renderers/media-clahe.test.ts new file mode 100644 index 00000000..c25ef73c --- /dev/null +++ b/packages/web/src/renderers/media-clahe.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { computeMediaClahe } from "./media-clahe"; + +function grayscale( + width: number, + height: number, + pixel: (x: number, y: number) => number, +) { + const rgba = new Uint8ClampedArray(width * height * 4); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const offset = (y * width + x) * 4; + rgba.fill(pixel(x, y), offset, offset + 3); + rgba[offset + 3] = 255; + } + } + return rgba; +} + +describe("media Auto contrast", () => { + it("expands local low contrast without amplifying a flat image or changing another tile", () => { + const width = 512; + const height = 512; + const rgba = grayscale(width, height, (x, y) => 100 + ((x + y * 7) % 41)); + const original = computeMediaClahe(rgba, width, height); + expect(original.tilesX).toBe(8); + expect(original.tilesY).toBe(8); + for (let tile = 0; tile < 64; tile++) { + const lut = original.luts.subarray(tile * 256, (tile + 1) * 256); + expect(lut[140]! - lut[100]!).toBeGreaterThan(80); + expect(lut[255]).toBe(255); + expect( + Array.from(lut).every((value, i) => i === 0 || value >= lut[i - 1]!), + ).toBe(true); + } + const flat = computeMediaClahe( + grayscale(width, height, () => 100), + width, + height, + ); + expect(Math.abs(flat.luts[100]! - 100)).toBeLessThanOrEqual(3); + + const changed = computeMediaClahe( + grayscale(width, height, (x, y) => + x < 64 ? 200 : 100 + ((x + y * 7) % 41), + ), + width, + height, + ); + expect(changed.luts.subarray(0, 256)).not.toEqual( + original.luts.subarray(0, 256), + ); + expect(changed.luts.subarray(256, 512)).toEqual( + original.luts.subarray(256, 512), + ); + }); + + it("uses Rec.601 intensity and keeps tiny images' lookup tables valid", () => { + const red = computeMediaClahe( + new Uint8ClampedArray([255, 0, 0, 255]), + 1, + 1, + ); + expect(red.luts[75]).toBe(0); + expect(red.luts[76]).toBe(255); + const tiny = computeMediaClahe( + grayscale(3, 2, (x, y) => 10 + x * 10 + y * 30), + 3, + 2, + ); + expect([tiny.tilesX, tiny.tilesY]).toEqual([3, 2]); + for (let tile = 0; tile < 6; tile++) + expect(tiny.luts[tile * 256 + 255]).toBe(255); + }); +}); diff --git a/packages/web/src/renderers/media-clahe.ts b/packages/web/src/renderers/media-clahe.ts new file mode 100644 index 00000000..0f3031b8 --- /dev/null +++ b/packages/web/src/renderers/media-clahe.ts @@ -0,0 +1,64 @@ +const BINS = 256; +const TILE_COUNT = 8; +const CLIP_LIMIT = 2; + +export function computeMediaClahe( + rgba: Uint8ClampedArray, + width: number, + height: number, +) { + const tilesX = Math.min(TILE_COUNT, width); + const tilesY = Math.min(TILE_COUNT, height); + const luts = new Uint8Array(tilesX * tilesY * BINS); + const histogram = new Uint32Array(BINS); + + for (let ty = 0; ty < tilesY; ty++) { + const y0 = Math.floor((ty * height) / tilesY); + const y1 = Math.floor(((ty + 1) * height) / tilesY); + for (let tx = 0; tx < tilesX; tx++) { + const x0 = Math.floor((tx * width) / tilesX); + const x1 = Math.floor(((tx + 1) * width) / tilesX); + histogram.fill(0); + for (let y = y0; y < y1; y++) { + for (let x = x0; x < x1; x++) { + const pixel = (y * width + x) * 4; + const luma = Math.round( + 0.299 * rgba[pixel]! + + 0.587 * rgba[pixel + 1]! + + 0.114 * rgba[pixel + 2]!, + ); + histogram[luma]!++; + } + } + + const area = (x1 - x0) * (y1 - y0); + const limit = Math.max(1, Math.floor((CLIP_LIMIT * area) / BINS)); + let excess = 0; + for (let i = 0; i < BINS; i++) { + if (histogram[i]! > limit) { + excess += histogram[i]! - limit; + histogram[i] = limit; + } + } + const perBin = Math.floor(excess / BINS); + let residual = excess - perBin * BINS; + for (let i = 0; i < BINS; i++) histogram[i]! += perBin; + const residualStep = Math.max( + Math.floor(BINS / Math.max(residual, 1)), + 1, + ); + for (let i = 0; i < BINS && residual > 0; i += residualStep, residual--) { + histogram[i]!++; + } + + const offset = (ty * tilesX + tx) * BINS; + let cumulative = 0; + for (let i = 0; i < BINS; i++) { + cumulative += histogram[i]!; + luts[offset + i] = Math.round((cumulative * (BINS - 1)) / area); + } + } + } + + return { luts, tilesX, tilesY }; +} diff --git a/packages/web/src/renderers/pixi-media-inspection-filter.ts b/packages/web/src/renderers/pixi-media-inspection-filter.ts new file mode 100644 index 00000000..7a5a053b --- /dev/null +++ b/packages/web/src/renderers/pixi-media-inspection-filter.ts @@ -0,0 +1,199 @@ +import type { Sprite } from "pixi.js"; +import { computeMediaClahe } from "./media-clahe"; +import { destroyShaderKeepingProgram } from "./pixi-shader-lifecycle"; + +const HISTOGRAM_MAX_DIMENSION = 1024; + +const vertex = ` +in vec2 aPosition; +out vec2 vTextureCoord; +out vec2 vImageCoord; +uniform vec4 uInputSize; +uniform vec4 uOutputFrame; +uniform vec4 uOutputTexture; +uniform mat3 uImageMatrix; +void main() { + vec2 position = aPosition * uOutputFrame.zw + uOutputFrame.xy; + position.x = position.x * (2.0 / uOutputTexture.x) - 1.0; + position.y = position.y * (2.0 * uOutputTexture.z / uOutputTexture.y) - uOutputTexture.z; + gl_Position = vec4(position, 0.0, 1.0); + vTextureCoord = aPosition * (uOutputFrame.zw * uInputSize.zw); + vImageCoord = (uImageMatrix * vec3(vTextureCoord, 1.0)).xy; +} +`; + +const fragment = ` +in vec2 vTextureCoord; +in vec2 vImageCoord; +out vec4 finalColor; +uniform sampler2D uTexture; +uniform sampler2D uLut; +uniform vec2 uTiles; +uniform float uGamma; +uniform float uAuto; +float lookup(vec2 tile, float luma) { + float row = tile.y * uTiles.x + tile.x; + return texture(uLut, vec2((luma * 255.0 + 0.5) / 256.0, + (row + 0.5) / (uTiles.x * uTiles.y))).r; +} +void main() { + vec4 color = texture(uTexture, vTextureCoord); + if (color.a == 0.0) { finalColor = color; return; } + vec3 rgb = color.rgb / color.a; + if (uAuto > 0.5) { + float luma = dot(rgb, vec3(0.299, 0.587, 0.114)); + vec2 tilePosition = clamp(vImageCoord, 0.0, 1.0) * uTiles - 0.5; + vec2 t0 = floor(tilePosition); + vec2 f = tilePosition - t0; + vec2 t1 = clamp(t0 + 1.0, vec2(0.0), uTiles - 1.0); + t0 = clamp(t0, vec2(0.0), uTiles - 1.0); + float top = mix(lookup(t0, luma), lookup(vec2(t1.x, t0.y), luma), f.x); + float bottom = mix(lookup(vec2(t0.x, t1.y), luma), lookup(t1, luma), f.x); + rgb = clamp(rgb + (mix(top, bottom, f.y) - luma), 0.0, 1.0); + } + finalColor = vec4(pow(rgb, vec3(1.0 / uGamma)) * color.a, color.a); +} +`; + +const gpuSource = ` +struct GlobalFilterUniforms { + uInputSize: vec4, uInputPixel: vec4, uInputClamp: vec4, + uOutputFrame: vec4, uGlobalFrame: vec4, uOutputTexture: vec4, +}; +struct InspectionUniforms { + uImageMatrix: mat3x3, uTiles: vec2, uGamma: f32, uAuto: f32, +}; +@group(0) @binding(0) var gfu: GlobalFilterUniforms; +@group(0) @binding(1) var uTexture: texture_2d; +@group(0) @binding(2) var uSampler: sampler; +@group(1) @binding(0) var inspectionUniforms: InspectionUniforms; +@group(1) @binding(1) var uLut: texture_2d; +@group(1) @binding(2) var uLutSampler: sampler; +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, @location(1) image: vec2, +}; +@vertex fn mainVertex(@location(0) aPosition: vec2) -> VertexOutput { + var position = aPosition * gfu.uOutputFrame.zw + gfu.uOutputFrame.xy; + position.x = position.x * (2.0 / gfu.uOutputTexture.x) - 1.0; + position.y = position.y * (2.0 * gfu.uOutputTexture.z / gfu.uOutputTexture.y) - gfu.uOutputTexture.z; + let uv = aPosition * (gfu.uOutputFrame.zw * gfu.uInputSize.zw); + return VertexOutput(vec4(position, 0.0, 1.0), uv, + (inspectionUniforms.uImageMatrix * vec3(uv, 1.0)).xy); +} +fn lookup(tile: vec2, luma: f32) -> f32 { + let tiles = inspectionUniforms.uTiles; + let row = tile.y * tiles.x + tile.x; + return textureSampleLevel(uLut, uLutSampler, + vec2((luma * 255.0 + 0.5) / 256.0, (row + 0.5) / (tiles.x * tiles.y)), 0.0).r; +} +@fragment fn mainFragment(input: VertexOutput) -> @location(0) vec4 { + let color = textureSample(uTexture, uSampler, input.uv); + if (color.a == 0.0) { return color; } + var rgb = color.rgb / color.a; + if (inspectionUniforms.uAuto > 0.5) { + let luma = dot(rgb, vec3(0.299, 0.587, 0.114)); + let tiles = inspectionUniforms.uTiles; + let tilePosition = clamp(input.image, vec2(0.0), vec2(1.0)) * tiles - 0.5; + let unbounded = floor(tilePosition); + let f = tilePosition - unbounded; + let t0 = clamp(unbounded, vec2(0.0), tiles - 1.0); + let t1 = clamp(unbounded + 1.0, vec2(0.0), tiles - 1.0); + let top = mix(lookup(t0, luma), lookup(vec2(t1.x, t0.y), luma), f.x); + let bottom = mix(lookup(vec2(t0.x, t1.y), luma), lookup(t1, luma), f.x); + rgb = clamp(rgb + (mix(top, bottom, f.y) - luma), vec3(0.0), vec3(1.0)); + } + return vec4(pow(rgb, vec3(1.0 / inspectionUniforms.uGamma)) * color.a, color.a); +} +`; + +export function createPixiMediaInspectionFilter( + sprite: Sprite, + pixi: Pick< + typeof import("pixi.js"), + "BufferImageSource" | "Filter" | "Matrix" + >, +) { + const { BufferImageSource, Filter, Matrix } = pixi; + const pixels = new Uint8Array(256 * 64 * 4); + const lutSource = new BufferImageSource({ + resource: pixels, + width: 256, + height: 64, + scaleMode: "linear", + alphaMode: "no-premultiply-alpha", + }); + const filter = Filter.from({ + gl: { vertex: vertex, fragment: fragment }, + gpu: { + vertex: { source: gpuSource, entryPoint: "mainVertex" }, + fragment: { source: gpuSource, entryPoint: "mainFragment" }, + }, + resolution: "inherit", + resources: { + inspectionUniforms: { + uImageMatrix: { value: new Matrix(), type: "mat3x3" }, + uTiles: { value: new Float32Array([8, 8]), type: "vec2" }, + uGamma: { value: 1, type: "f32" }, + uAuto: { value: 0, type: "f32" }, + }, + uLut: lutSource, + uLutSampler: lutSource.style, + }, + }); + const uniforms = filter.resources.inspectionUniforms.uniforms; + let hasHistogram = false; + + filter.apply = (manager, input, output, clear) => { + // The filter input is clipped by the viewport; lookup tiles stay anchored + // to the full image when the user zooms or pans. + manager.calculateSpriteMatrix(uniforms.uImageMatrix, sprite); + manager.applyFilter(filter, input, output, clear); + }; + + return { + filter, + invalidateImage() { + hasHistogram = false; + }, + update( + gamma: number, + autoContrast: boolean, + image: HTMLCanvasElement | null, + ) { + uniforms.uGamma = Math.max(gamma, 0.0001); + if (autoContrast && image && !hasHistogram) { + const scale = Math.min( + 1, + HISTOGRAM_MAX_DIMENSION / Math.max(image.width, image.height), + ); + const sample = document.createElement("canvas"); + sample.width = Math.max(1, Math.round(image.width * scale)); + sample.height = Math.max(1, Math.round(image.height * scale)); + const context = sample.getContext("2d", { willReadFrequently: true })!; + context.drawImage(image, 0, 0, sample.width, sample.height); + const { luts, tilesX, tilesY } = computeMediaClahe( + context.getImageData(0, 0, sample.width, sample.height).data, + sample.width, + sample.height, + ); + for (let i = 0; i < luts.length; i++) { + pixels[i * 4] = luts[i]!; + pixels[i * 4 + 1] = luts[i]!; + pixels[i * 4 + 2] = luts[i]!; + pixels[i * 4 + 3] = 255; + } + uniforms.uTiles.set([tilesX, tilesY]); + lutSource.resource = pixels.subarray(0, luts.length * 4); + lutSource.resize(256, tilesX * tilesY); + lutSource.update(); + hasHistogram = true; + } + uniforms.uAuto = autoContrast && hasHistogram ? 1 : 0; + }, + destroy() { + destroyShaderKeepingProgram(filter); + lutSource.destroy(); + }, + }; +} diff --git a/packages/web/src/renderers/pixi-media-scene.push.test.ts b/packages/web/src/renderers/pixi-media-scene.push.test.ts index 9fa5bf97..d9ad6f5a 100644 --- a/packages/web/src/renderers/pixi-media-scene.push.test.ts +++ b/packages/web/src/renderers/pixi-media-scene.push.test.ts @@ -10,6 +10,7 @@ import { RegionRendererSourceKind, type DepthMap, type DetectionFrame, + type MediaDisplayAdjustments, } from "supervision-js-core"; import type { @@ -59,6 +60,7 @@ interface MockTextureInstance { */ interface PaintedSprite { readonly destroy: ReturnType; + readonly filters?: unknown[]; readonly quad: { height: number; width: number }; readonly scale: { x: number; y: number }; } @@ -66,7 +68,9 @@ interface PaintedSprite { // Push mode asks Pixi for WebGPU, so the mock renderer carries a device: the // scene then composites through the GPU path production runs, not the staging // canvas fallback. -vi.mock("pixi.js", () => { +vi.mock("pixi.js", async (importOriginal) => { + const { BufferImageSource } = + await importOriginal(); class Stub { destroy = vi.fn(); } @@ -342,6 +346,29 @@ vi.mock("pixi.js", () => { style = {}; } + class Filter extends Stub { + static from(options: { resources: Record }) { + const definitions = options.resources.inspectionUniforms as Record< + string, + { value: unknown } + >; + return { + destroy: vi.fn(), + resources: { + ...options.resources, + inspectionUniforms: { + uniforms: Object.fromEntries( + Object.entries(definitions).map(([name, definition]) => [ + name, + definition.value, + ]), + ), + }, + }, + }; + } + } + class UniformGroup { uniforms: Record = {}; update = vi.fn(); @@ -359,13 +386,14 @@ vi.mock("pixi.js", () => { Application, Assets: { load: vi.fn(), unload: vi.fn() }, BlurFilter: Stub, - BufferImageSource: Destroyable, + BufferImageSource, CanvasSource, ColorMatrixFilter, Container, defaultFilterVert: "default-filter-vertex", ExternalSource, - Filter: Stub, + Filter, + Matrix: class {}, Graphics, ImageSource: Destroyable, Mesh, @@ -384,7 +412,12 @@ vi.mock("pixi.js", () => { vi.mock("pixi.js/gif", () => ({ GifSprite: class {} })); /** Shared so a test can see what the staging fallback drew into it. */ -const stagingContext = { drawImage: vi.fn() }; +const stagingContext = { + drawImage: vi.fn(), + getImageData: (_x: number, _y: number, width: number, height: number) => ({ + data: new Uint8ClampedArray(width * height * 4).fill(100), + }), +}; function createBrushCanvas(): HTMLCanvasElement { const circles: Array<{ x: number; y: number; radius: number }> = []; @@ -1231,6 +1264,130 @@ describe("push-presented Pixi scene", () => { ]); }); + it("repaints Gamma and still-image Auto without filtering annotations, and Reset restores manual contrast", async () => { + const { createPixiMediaScene } = await import("./pixi-media-scene"); + const scene = await createPixiMediaScene({ + ...createSceneOptions(undefined), + stillFrame: true, + }); + scene.initializeMedia({ height: 240, width: 320 }); + const sample = { timestamp: 0, duration: 0, draw: vi.fn(), close: vi.fn() }; + scene.presentSample(sample); + const original = scene.getRenderCount?.() ?? 0; + await applyDisplayAdjustment(scene, { gamma: 2, contrast: 1.8 }); + expect(scene.getRenderCount?.()).toBe(original + 1); + const filtered = pixiMock.sprites.filter( + (sprite) => sprite.filters?.length, + ); + expect(filtered).toHaveLength(1); + expect(filtered[0]?.filters).toHaveLength(2); + const inspection = filtered[0]?.filters?.[0] as { + resources: { inspectionUniforms: { uniforms: Record } }; + }; + expect(inspection.resources.inspectionUniforms.uniforms).toMatchObject({ + uGamma: 2, + uAuto: 0, + }); + + await applyDisplayAdjustment(scene, { + gamma: 2, + contrast: 1.8, + autoContrast: true, + }); + expect(scene.getRenderCount?.()).toBe(original + 2); + expect(inspection.resources.inspectionUniforms.uniforms).toMatchObject({ + uGamma: 2, + uAuto: 1, + }); + expect(pixiMock.displayFilters.at(-1)).toMatchObject({ + contrast: expect.any(Function), + }); + const auto = pixiMock.displayFilters.at(-1) as { + contrast: ReturnType; + }; + expect(auto.contrast).toHaveBeenCalledWith(0, true); + await applyDisplayAdjustment(scene, { + gamma: 2, + contrast: 1.8, + autoContrast: true, + }); + expect(scene.getRenderCount?.()).toBe(original + 2); + + await applyDisplayAdjustment(scene, { + gamma: 1, + contrast: 1.8, + autoContrast: false, + }); + expect(scene.getRenderCount?.()).toBe(original + 3); + expect(filtered[0]?.filters).toHaveLength(1); + const manual = pixiMock.displayFilters.at(-1) as { + contrast: ReturnType; + }; + expect(manual.contrast).toHaveBeenCalledWith(0.8, true); + await applyDisplayAdjustment(scene, { gamma: 2, autoContrast: true }); + expect(scene.getRenderCount?.()).toBe(original + 4); + await applyDisplayAdjustment(scene, {}); + expect(scene.getRenderCount?.()).toBe(original + 5); + const identity = pixiMock.displayFilters.at(-1) as { + brightness: ReturnType; + contrast: ReturnType; + }; + expect(filtered[0]?.filters).toStrictEqual([identity]); + expect(identity.brightness).toHaveBeenCalledWith(1, false); + expect(identity.contrast).toHaveBeenCalledWith(0, true); + scene.destroy(); + }); + + it.each([ + { width: 3, height: 2, tiles: 6 }, + { width: 2048, height: 1, tiles: 8 }, + ])( + "uploads complete Auto lookup-table rows for $width×$height images", + async ({ width, height, tiles }) => { + const { createPixiMediaScene } = await import("./pixi-media-scene"); + const { gpuUploadBufferImageResource } = + await vi.importActual("pixi.js"); + const scene = await createPixiMediaScene({ + ...createSceneOptions(undefined), + stillFrame: true, + }); + scene.initializeMedia({ width, height }); + scene.presentSample({ + timestamp: 0, + duration: 0, + draw: vi.fn(), + close: vi.fn(), + }); + await applyDisplayAdjustment(scene, { gamma: 2, autoContrast: true }); + const sprite = pixiMock.sprites.find( + (candidate) => candidate.filters?.length, + ); + const inspection = sprite?.filters?.[0] as { + resources: { uLut: import("pixi.js").BufferImageSource }; + }; + const source = inspection.resources.uLut; + const writeTexture = vi.fn(); + const gpu = { + device: { queue: { writeTexture } }, + } as unknown as Parameters[2]; + gpuUploadBufferImageResource.upload(source, {} as GPUTexture, gpu); + const [, resource, layout, size] = writeTexture.mock.calls[0]!; + expect(resource).toBe(source.resource); + expect(layout).toEqual({ + offset: 0, + rowsPerImage: tiles, + bytesPerRow: 256 * 4, + }); + expect(size).toEqual({ + width: 256, + height: tiles, + depthOrArrayLayers: 1, + }); + expect(source.resource.byteLength).toBe(256 * tiles * 4); + scene.destroy(); + }, + ); + it("keeps the media on the staging canvas when the device refuses a decoded frame", async () => { // The refusal lands on the capability probe, which is the first copy the // scene asks for. Firefox refuses every one after it too, and the point of @@ -1478,10 +1635,13 @@ describe("push-presented Pixi scene", () => { * against the real Pixi rather than the mock, so they go one at a time, and the * filter each one builds is what says the write landed. */ -async function applyDisplayAdjustment(scene: MediaRendererScene) { +async function applyDisplayAdjustment( + scene: MediaRendererScene, + adjustments: MediaDisplayAdjustments = { brightness: 1.4, contrast: 0.8 }, +) { const built = pixiMock.displayFilters.length + 1; - scene.setDisplayAdjustments?.({ brightness: 1.4, contrast: 0.8 }); + scene.setDisplayAdjustments?.(adjustments); await vi.waitFor(() => expect(pixiMock.displayFilters).toHaveLength(built)); } diff --git a/packages/web/src/renderers/pixi-media-scene.ts b/packages/web/src/renderers/pixi-media-scene.ts index 6e6128ad..881e5b6e 100644 --- a/packages/web/src/renderers/pixi-media-scene.ts +++ b/packages/web/src/renderers/pixi-media-scene.ts @@ -59,6 +59,7 @@ import type { PresentedVideoFrame, } from "./presented-frame-channel"; import { createSceneRenderScheduler } from "./scene-render-scheduler"; +import { createPixiMediaInspectionFilter } from "./pixi-media-inspection-filter"; import type { SceneRenderSignature } from "./scene-render-scheduler"; import { createPreparedAnnotationWindow } from "./prepared-annotation-window"; import { @@ -342,6 +343,10 @@ export async function createPixiMediaScene( let isScrubbing = false; let displayBrightness = 1; let displayContrast = 1; + let displayGamma = 1; + let displayAutoContrast = false; + let inspectionFilter: + ReturnType | undefined; let viewportScale = 1; let hasPresentedSample = false; let mediaHeight = 0; @@ -1337,6 +1342,7 @@ export async function createPixiMediaScene( if (!collectFrameTimings) { sample.draw(stagingContext, 0, 0, mediaWidth, mediaHeight); presentedSampleTimestamp = sample.timestamp; + refreshInspectionImage(); stagingTextureSource?.update(); stagingTexture?.update(); @@ -1362,6 +1368,7 @@ export async function createPixiMediaScene( const mediaUploadMs = measure(() => { sample.draw(stagingContext, 0, 0, mediaWidth, mediaHeight); presentedSampleTimestamp = sample.timestamp; + refreshInspectionImage(); stagingTextureSource?.update(); stagingTexture?.update(); }); @@ -1445,14 +1452,32 @@ export async function createPixiMediaScene( setDisplayAdjustments(adjustments) { void import("pixi.js").then(({ ColorMatrixFilter }) => { + if (isDestroyed) return; const brightness = adjustments.brightness ?? 1; const contrast = adjustments.contrast ?? 1; const filter = new ColorMatrixFilter(); // Preserve the renderer's pixel grid through the adjustment pass. filter.resolution = "inherit"; filter.brightness(brightness, false); - filter.contrast(contrast - 1, true); - if (mediaSprite) mediaSprite.filters = [filter]; + displayGamma = adjustments.gamma ?? 1; + displayAutoContrast = stillFrame && adjustments.autoContrast === true; + filter.contrast(displayAutoContrast ? 0 : contrast - 1, true); + if (mediaSprite) { + if (displayGamma !== 1 || displayAutoContrast) { + inspectionFilter ??= createPixiMediaInspectionFilter( + mediaSprite, + pixi, + ); + inspectionFilter.update( + displayGamma, + displayAutoContrast, + hasPresentedSample ? stagingCanvas : null, + ); + mediaSprite.filters = [inspectionFilter.filter, filter]; + } else { + mediaSprite.filters = [filter]; + } + } displayBrightness = brightness; displayContrast = contrast; renderOnChange(); @@ -1827,6 +1852,7 @@ export async function createPixiMediaScene( // a decoder buffer in a producer that outlives the scene. frameChannel?.onPresentedFrame((presented) => presented.frame.close()); destroyBatchTextureBindings(); + inspectionFilter?.destroy(); mediaCompositor?.destroy(); interactionLayer?.destroy(); interactionPresentationLayer?.destroy(); @@ -2758,6 +2784,12 @@ export async function createPixiMediaScene( } } + function refreshInspectionImage() { + if (!stillFrame || !inspectionFilter) return; + inspectionFilter.invalidateImage(); + inspectionFilter.update(displayGamma, displayAutoContrast, stagingCanvas); + } + function hasLoadingDetections() { const loading = currentVisibility?.loadingDetectionIds; if (!loading) return false; @@ -2790,6 +2822,8 @@ export async function createPixiMediaScene( app.renderer.resolution, displayBrightness, displayContrast, + displayGamma, + displayAutoContrast, ...RENDERED_PRESENTATION_FIELDS.map( (field) => appliedPresentation?.[field], ),