From 52efd81f6941f59585e4504f372a2f30dfbea7ad Mon Sep 17 00:00:00 2001 From: borys3kk Date: Fri, 23 Oct 2026 16:20:57 +0200 Subject: [PATCH 01/17] feat(cv): add the image inpainting task Native crop, paste and blendMasked ops (windowed copies, and a masked composite with the mask eroded and Gaussian-feathered so the seam blends), and on top of them createInpainter / useInpainter: run a MI-GAN style model (RGB in [0, 1] plus a 1 = known / 0 = hole mask) on the square crop around the painted region, then blend the output back into the full-resolution frame, following upstream's MIGAN_Pipeline. The public mask marks the hole with any non-zero pixel; the inversion to the graph's convention happens inside the task. An empty mask returns the input unchanged. --- .cspell-wordlist.txt | 9 + .../api/__snapshots__/apiSurface.test.ts.snap | 7 + .../__tests__/support/fakeOps.ts | 123 ++++++ .../__tests__/tasks/inpainter.test.ts | 275 +++++++++++++ .../cpp/extensions/cv/image_ops.cpp | 158 ++++++++ .../cpp/extensions/cv/image_ops.h | 3 + .../cpp/extensions/cv/install.cpp | 3 + .../cpp/tests/extensions/CvOpsTest.cpp | 186 +++++++++ .../src/extensions/cv/ops/image.ts | 103 +++++ .../src/extensions/cv/tasks/inpainting.ts | 374 ++++++++++++++++++ .../src/hooks/useInpainter.ts | 34 ++ packages/react-native-executorch/src/index.ts | 2 + 12 files changed, 1277 insertions(+) create mode 100644 packages/react-native-executorch/__tests__/tasks/inpainter.test.ts create mode 100644 packages/react-native-executorch/src/extensions/cv/tasks/inpainting.ts create mode 100644 packages/react-native-executorch/src/hooks/useInpainter.ts diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 882f3cfd22..7c22c311a3 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -391,3 +391,12 @@ repoints basenames xcconfig toggleable +migan +inpaint +inpainted +inpainter +inpainting +Picsart +FFHQ +grayscale +Premul diff --git a/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap b/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap index 0998656867..acd091a551 100644 --- a/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap +++ b/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap @@ -4,6 +4,7 @@ exports[`public API surface exports one hook per user-facing task 1`] = ` [ "useClassifier", "useImageEmbedder", + "useInpainter", "useInstanceSegmenter", "useKeypointDetector", "useLLMChatSession", @@ -29,8 +30,10 @@ exports[`public API surface matches the recorded contents of each namespace 1`] "FORMAT_CHANNELS", "FORMAT_CONVERSION", "applyColormap", + "blendMasked", "boundingBoxOfPoints", "createImagePreprocessor", + "crop", "cvtColor", "decodeBox", "distance", @@ -39,6 +42,7 @@ exports[`public API surface matches the recorded contents of each namespace 1`] "nms", "normalize", "orderQuad", + "paste", "quadSize", "rectifyQuad", "resize", @@ -130,9 +134,11 @@ exports[`public API surface matches the recorded export list 1`] = ` "VALID_ERROR_CODES", "WHISPER_LANGUAGES", "WHISPER_SAMPLE_RATE_HZ", + "computeInpaintingCrop", "createClassifier", "createFsmnVoiceActivityDetector", "createImageEmbedder", + "createInpainter", "createInstanceSegmenter", "createKeypointDetector", "createKokoroTextToSpeech", @@ -165,6 +171,7 @@ exports[`public API surface matches the recorded export list 1`] = ` "tensor", "useClassifier", "useImageEmbedder", + "useInpainter", "useInstanceSegmenter", "useKeypointDetector", "useLLMChatSession", diff --git a/packages/react-native-executorch/__tests__/support/fakeOps.ts b/packages/react-native-executorch/__tests__/support/fakeOps.ts index 95cd7ef521..4847aac2b9 100644 --- a/packages/react-native-executorch/__tests__/support/fakeOps.ts +++ b/packages/react-native-executorch/__tests__/support/fakeOps.ts @@ -556,6 +556,129 @@ export const cv = { } return dst; }, + + /** Copies the `dst`-sized window of `src` starting at `origin`. */ + crop(src: FakeTensor, dst: FakeTensor, origin: { x: number; y: number }): FakeTensor { + const [srcH, srcW, channels] = src.shape as [number, number, number]; + const [dstH, dstW, dstChannels] = dst.shape as [number, number, number]; + if (channels !== dstChannels) { + throw new Error(`crop: channel mismatch (${channels} vs ${dstChannels})`); + } + if (origin.x < 0 || origin.y < 0 || origin.x + dstW > srcW || origin.y + dstH > srcH) { + throw new Error( + `crop: a ${dstW}x${dstH} window at (${origin.x}, ${origin.y}) does not fit in a ${srcW}x${srcH} image` + ); + } + for (let y = 0; y < dstH; y++) { + for (let x = 0; x < dstW; x++) { + for (let c = 0; c < channels; c++) { + dst.setElement( + (y * dstW + x) * channels + c, + src.getElement(((origin.y + y) * srcW + origin.x + x) * channels + c) + ); + } + } + } + return dst; + }, + + /** Writes the whole of `src` into the window of `dst` starting at `origin`. */ + paste(src: FakeTensor, dst: FakeTensor, origin: { x: number; y: number }): FakeTensor { + const [srcH, srcW, channels] = src.shape as [number, number, number]; + const [dstH, dstW, dstChannels] = dst.shape as [number, number, number]; + if (channels !== dstChannels) { + throw new Error(`paste: channel mismatch (${channels} vs ${dstChannels})`); + } + if (origin.x < 0 || origin.y < 0 || origin.x + srcW > dstW || origin.y + srcH > dstH) { + throw new Error( + `paste: a ${srcW}x${srcH} window at (${origin.x}, ${origin.y}) does not fit in a ${dstW}x${dstH} image` + ); + } + for (let y = 0; y < srcH; y++) { + for (let x = 0; x < srcW; x++) { + for (let c = 0; c < channels; c++) { + dst.setElement( + ((origin.y + y) * dstW + origin.x + x) * channels + c, + src.getElement((y * srcW + x) * channels + c) + ); + } + } + } + return dst; + }, + + /** + * Composites `overlay` over `src` where `mask` is set. The native op erodes + * the mask with a square kernel and feathers it with a Gaussian; this erodes + * the same way and feathers with a box mean of the same width, which keeps + * both options load-bearing (a seam pixel moves when they change) without + * reproducing OpenCV's kernel weights. + */ + blendMasked( + src: FakeTensor, + overlay: FakeTensor, + mask: FakeTensor, + dst: FakeTensor, + opts: { shrink: number; feather: number } + ): FakeTensor { + const [height, width, channels] = src.shape as [number, number, number]; + expectShape(overlay, src.shape, 'blendMasked: overlay'); + expectShape(mask, [height, width, 1], 'blendMasked: mask'); + expectShape(dst, src.shape, 'blendMasked: dst'); + if (opts.feather > 0 && opts.feather % 2 === 0) { + throw new Error(`blendMasked: feather must be 0 or an odd kernel size (got ${opts.feather})`); + } + + const inRange = (y: number, x: number) => y >= 0 && y < height && x >= 0 && x < width; + let weight = Float32Array.from({ length: height * width }, (_, i) => + mask.getElement(i) !== 0 ? 1 : 0 + ); + + if (opts.shrink > 0) { + const eroded = new Float32Array(weight.length); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + let min = 1; + for (let dy = -opts.shrink; dy <= opts.shrink; dy++) { + for (let dx = -opts.shrink; dx <= opts.shrink; dx++) { + if (inRange(y + dy, x + dx)) min = Math.min(min, weight[(y + dy) * width + x + dx]!); + } + } + eroded[y * width + x] = min; + } + } + weight = eroded; + } + + if (opts.feather > 0) { + const radius = (opts.feather - 1) / 2; + const blurred = new Float32Array(weight.length); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + let sum = 0; + let count = 0; + for (let dy = -radius; dy <= radius; dy++) { + for (let dx = -radius; dx <= radius; dx++) { + if (!inRange(y + dy, x + dx)) continue; + sum += weight[(y + dy) * width + x + dx]!; + count++; + } + } + blurred[y * width + x] = sum / count; + } + } + weight = blurred; + } + + for (let p = 0; p < height * width; p++) { + const w = weight[p]!; + for (let c = 0; c < channels; c++) { + const at = p * channels + c; + dst.setElement(at, Math.round(src.getElement(at) * (1 - w) + overlay.getElement(at) * w)); + } + } + return dst; + }, }; // ============================================================================ diff --git a/packages/react-native-executorch/__tests__/tasks/inpainter.test.ts b/packages/react-native-executorch/__tests__/tasks/inpainter.test.ts new file mode 100644 index 0000000000..53749a2294 --- /dev/null +++ b/packages/react-native-executorch/__tests__/tasks/inpainter.test.ts @@ -0,0 +1,275 @@ +import { f32, method } from '../../src/core/schema'; +import { createInpainter, computeInpaintingCrop } from '../../src/extensions/cv/tasks/inpainting'; +import type { ImageBuffer } from '../../src/extensions/cv/image'; +import { fakeJsi } from '../support/fakeJsi'; +import type { FakeTensor } from '../support/fakeTensor'; +import { tracked } from '../support/lifetime'; +import { exported, imageBuffer, writesOutputs } from '../support/fixtures'; + +const MODEL_PATH = '/models/inpaint.pte'; +const SIZE = 4; + +/** The value the fake model paints, and the pixel it comes out as (0.2 * 255). */ +const FILL = 0.2; +const FILL_PIXEL = 51; + +const options = { + interpolation: 'nearest', + outInterpolation: 'nearest', + cropPadding: 0, + blend: { shrink: 0, feather: 0 }, +} as const; + +const config = { modelPath: MODEL_PATH, modelOpts: options }; + +const BATCHED = method( + 'forward', + [f32(1, 3, SIZE, SIZE), f32(1, 1, SIZE, SIZE)], + [f32(1, 3, SIZE, SIZE)] +); + +/** A model that paints every output pixel with `FILL`. */ +const registerPainter = () => { + fakeJsi.registerModel(MODEL_PATH, { + schema: exported(BATCHED), + execute: writesOutputs(new Array(3 * SIZE * SIZE).fill(FILL)), + }); +}; + +/** + * A grayscale mask over a `width` x `height` image marking the `hole` rectangle + * (inclusive corners) with 255. + */ +const maskBuffer = ( + width: number, + height: number, + hole: { x0: number; y0: number; x1: number; y1: number } | null +): ImageBuffer => { + const data = new Uint8Array(width * height); + if (hole) { + for (let y = hole.y0; y <= hole.y1; y++) { + for (let x = hole.x0; x <= hole.x1; x++) data[y * width + x] = 255; + } + } + return { data, width, height, format: 'gray', layout: 'hwc' }; +}; + +const rgbaAt = (output: ImageBuffer, x: number, y: number) => { + const at = (y * output.width + x) * 4; + return [...output.data.slice(at, at + 4)]; +}; + +describe('computeInpaintingCrop', () => { + it('returns null for a mask that marks nothing', () => { + expect(computeInpaintingCrop(maskBuffer(10, 10, null), { padding: 0, minSize: 1 })).toBeNull(); + }); + + it('frames the hole in a square padded on each side', () => { + // A 2x4 hole at (4, 3): the longer side is 4, plus 1 px of padding on each + // side makes a 6x6 window centred on the hole. + const crop = computeInpaintingCrop(maskBuffer(20, 20, { x0: 4, y0: 3, x1: 5, y1: 6 }), { + padding: 1, + minSize: 1, + }); + expect(crop).toEqual({ x: 1, y: 1, width: 6, height: 6 }); + }); + + it('never returns a window smaller than minSize', () => { + const crop = computeInpaintingCrop(maskBuffer(20, 20, { x0: 10, y0: 10, x1: 10, y1: 10 }), { + padding: 0, + minSize: 8, + }); + expect(crop).toMatchObject({ width: 8, height: 8 }); + }); + + it('pushes a window that overhangs the image back inside it', () => { + // A hole in the top-left corner would centre a 6x6 window at (0, 0); the + // overhang is pushed to the right and bottom so the window keeps its size. + const crop = computeInpaintingCrop(maskBuffer(20, 20, { x0: 0, y0: 0, x1: 1, y1: 1 }), { + padding: 2, + minSize: 1, + }); + expect(crop).toEqual({ x: 0, y: 0, width: 6, height: 6 }); + }); + + it('clamps a window larger than the image to the image', () => { + const crop = computeInpaintingCrop(maskBuffer(5, 7, { x0: 2, y0: 3, x1: 2, y1: 3 }), { + padding: 0, + minSize: 64, + }); + expect(crop).toEqual({ x: 0, y: 0, width: 5, height: 7 }); + }); + + it('treats any non-zero color channel of a color mask as the hole', () => { + const mask = imageBuffer(6, 6, 'rgba'); + mask.data.fill(0); + mask.data[(2 * 6 + 3) * 4 + 2] = 255; // B of pixel (3, 2) + expect(computeInpaintingCrop(mask, { padding: 0, minSize: 1 })).toEqual({ + x: 3, + y: 2, + width: 1, + height: 1, + }); + }); +}); + +describe('createInpainter', () => { + beforeEach(registerPainter); + + it('returns an opaque RGBA buffer at the input resolution', async () => { + const inpainter = tracked(await createInpainter(config)); + + const output = await inpainter.inpaint( + imageBuffer(10, 6), + maskBuffer(10, 6, { x0: 3, y0: 2, x1: 4, y1: 3 }) + ); + + expect(output).toMatchObject({ width: 10, height: 6, format: 'rgba', layout: 'hwc' }); + expect(output.data).toHaveLength(10 * 6 * 4); + expect([...output.data].filter((_, i) => i % 4 === 3).every((a) => a === 255)).toBe(true); + }); + + it('fills the hole with the model output and leaves the rest untouched', async () => { + // The hole's 4x4 window matches the model size, so with nearest resizing + // and a hard blend the hole takes the painted value exactly and every + // other pixel is the input. + const inpainter = tracked(await createInpainter(config)); + const input = imageBuffer(8, 8); + const hole = { x0: 2, y0: 2, x1: 3, y1: 3 }; + + const output = await inpainter.inpaint(input, maskBuffer(8, 8, hole)); + + for (let y = 0; y < 8; y++) { + for (let x = 0; x < 8; x++) { + const inHole = x >= hole.x0 && x <= hole.x1 && y >= hole.y0 && y <= hole.y1; + const at = (y * 8 + x) * 3; + const original = [...input.data.slice(at, at + 3), 255]; + expect(rgbaAt(output, x, y)).toEqual( + inHole ? [FILL_PIXEL, FILL_PIXEL, FILL_PIXEL, 255] : original + ); + } + } + }); + + it('feeds the model 1 for known pixels and 0 for the hole', async () => { + let maskSeen: number[] = []; + fakeJsi.registerModel(MODEL_PATH, { + schema: exported(BATCHED), + execute: (_method, inputs) => { + const mask = inputs[1] as FakeTensor; + maskSeen = Array.from({ length: mask.numel }, (_, i) => mask.getElement(i)); + }, + }); + const inpainter = tracked(await createInpainter(config)); + + // Any non-zero mask value is the hole, however faint. + const mask = maskBuffer(SIZE, SIZE, null); + mask.data[1 * SIZE + 2] = 1; + await inpainter.inpaint(imageBuffer(SIZE, SIZE), mask); + + const expected = new Array(SIZE * SIZE).fill(1); + expected[1 * SIZE + 2] = 0; + expect(maskSeen).toEqual(expected); + }); + + it('returns the input unchanged, without running the model, for an empty mask', async () => { + const inpainter = tracked(await createInpainter(config)); + const input = imageBuffer(6, 5); + + const output = await inpainter.inpaint(input, maskBuffer(6, 5, null)); + + expect(fakeJsi.executions()).toEqual([]); + for (let p = 0; p < 6 * 5; p++) { + expect([...output.data.slice(p * 4, p * 4 + 3)]).toEqual([ + ...input.data.slice(p * 3, p * 3 + 3), + ]); + } + }); + + it('accepts a color mask, taking any non-zero color channel as the hole', async () => { + const inpainter = tracked(await createInpainter(config)); + const mask = imageBuffer(SIZE, SIZE, 'rgba'); + mask.data.fill(0); + mask.data[(1 * SIZE + 1) * 4] = 255; + + const output = await inpainter.inpaint(imageBuffer(SIZE, SIZE), mask); + + expect(rgbaAt(output, 1, 1)).toEqual([FILL_PIXEL, FILL_PIXEL, FILL_PIXEL, 255]); + expect(rgbaAt(output, 0, 0).slice(0, 3)).not.toEqual([FILL_PIXEL, FILL_PIXEL, FILL_PIXEL]); + }); + + it('accepts the unbatched [3, H, W] variant', async () => { + fakeJsi.registerModel(MODEL_PATH, { + schema: exported( + method('forward', [f32(3, SIZE, SIZE), f32(1, SIZE, SIZE)], [f32(3, SIZE, SIZE)]) + ), + execute: writesOutputs(new Array(3 * SIZE * SIZE).fill(FILL)), + }); + const inpainter = tracked(await createInpainter(config)); + + const output = await inpainter.inpaint( + imageBuffer(SIZE, SIZE), + maskBuffer(SIZE, SIZE, { x0: 0, y0: 0, x1: 0, y1: 0 }) + ); + + expect(rgbaAt(output, 0, 0)).toEqual([FILL_PIXEL, FILL_PIXEL, FILL_PIXEL, 255]); + }); + + it('rejects a model whose mask input does not match the image input', async () => { + fakeJsi.registerModel(MODEL_PATH, { + schema: exported( + method( + 'forward', + [f32(1, 3, SIZE, SIZE), f32(1, 1, SIZE * 2, SIZE)], + [f32(1, 3, SIZE, SIZE)] + ) + ), + }); + + await expect(createInpainter(config)).rejects.toThrow(/inconsistent bindings/); + }); + + it('rejects a mask whose dimensions differ from the image', async () => { + const inpainter = tracked(await createInpainter(config)); + + await expect( + inpainter.inpaint(imageBuffer(6, 6), maskBuffer(3, 3, { x0: 0, y0: 0, x1: 0, y1: 0 })) + ).rejects.toThrow(/mask is 3x3 but image is 6x6/); + }); + + it('produces the same result synchronously and asynchronously', async () => { + const inpainter = tracked(await createInpainter(config)); + const input = imageBuffer(8, 8); + const mask = maskBuffer(8, 8, { x0: 2, y0: 2, x1: 3, y1: 3 }); + + expect(inpainter.inpaintWorklet(input, mask)).toEqual(await inpainter.inpaint(input, mask)); + }); + + it('releases every native resource on dispose', async () => { + const inpainter = await createInpainter(config); + await inpainter.inpaint( + imageBuffer(20, 15), + maskBuffer(20, 15, { x0: 5, y0: 5, x1: 8, y1: 9 }) + ); + + inpainter.dispose(); + + expect(fakeJsi.liveTensors()).toBe(0); + expect(fakeJsi.liveModels()).toEqual([]); + }); + + it('frees the per-call tensors across repeated calls', async () => { + const inpainter = tracked(await createInpainter(config)); + const afterConstruction = fakeJsi.liveTensors(); + + for (const size of [8, 16, 32]) { + await inpainter.inpaint( + imageBuffer(size, size), + maskBuffer(size, size, { x0: 1, y0: 1, x1: 2, y1: 2 }) + ); + } + await inpainter.inpaint(imageBuffer(8, 8), maskBuffer(8, 8, null)); + + expect(fakeJsi.liveTensors()).toBe(afterConstruction); + }); +}); diff --git a/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp b/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp index f620ef1dff..1ca5d83ef0 100644 --- a/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp +++ b/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp @@ -510,4 +510,162 @@ void install_rectifyQuad(jsi::Runtime &rt, jsi::Object &module) { module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 4, error::guarded(fnBody))); } +// --- Windowed copies --------------------------------------------------------- + +namespace { +struct Window { + int32_t x, y; +}; + +// Reads the `{x, y}` origin of a `w` x `h` window and checks that it lies inside +// a `frameW` x `frameH` frame. +Window windowInside(jsi::Runtime &rt, const std::string &ctx, const jsi::Value &optsVal, + int32_t w, int32_t h, int32_t frameW, int32_t frameH) { + const auto opts = conversions::asType(rt, ctx + ": options", optsVal); + const auto x = conversions::getRequiredProperty(rt, ctx + ": options", opts, "x"); + const auto y = conversions::getRequiredProperty(rt, ctx + ": options", opts, "y"); + if (x < 0 || y < 0 || x + w > frameW || y + h > frameH) { + throw error::InvalidArgument(std::format("{}: a {}x{} window at ({}, {}) does not fit in a {}x{} image", + ctx, w, h, x, y, frameW, frameH)); + } + return {.x = x, .y = y}; +} +} // namespace + +// Copy the window of `src` that `dst` is sized for, starting at options.{x, y}. +void install_crop(jsi::Runtime &rt, jsi::Object &module) { + const auto *name = "crop"; + auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count != 3) { + throw error::InvalidArgument("crop: Usage: crop(src, dst, options)"); + } + + auto src = tensor::fromJs(rt, "crop: src", args[0], std::nullopt, {"H", "W", "C"}); + auto dst = tensor::fromJs(rt, "crop: dst", args[1], src->dtype_, {"H'", "W'", src->shape_[2]}); + + tensor::checkNotSameTensor(rt, "crop: src", src, "crop: dst", dst); + auto srcLock = tensor::tryLockShared(rt, "crop: src", src); + auto dstLock = tensor::tryLockUnique(rt, "crop: dst", dst); + + const int32_t dstH = dst->shape_[0]; + const int32_t dstW = dst->shape_[1]; + const Window win = windowInside(rt, "crop", args[2], dstW, dstH, src->shape_[1], src->shape_[0]); + + const int cvType = CV_MAKETYPE(dtypeToCvDepth(src->dtype_), src->shape_[2]); + try { + const ::cv::Mat srcMat(src->shape_[0], src->shape_[1], cvType, src->data_.get()); + ::cv::Mat dstMat(dstH, dstW, cvType, dst->data_.get()); + srcMat(::cv::Rect(win.x, win.y, dstW, dstH)).copyTo(dstMat); + } catch (const std::exception &e) { + throw error::Unknown("crop: " + std::string(e.what())); + } + + return jsi::Value(rt, args[1]); + }; + + module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 3, error::guarded(fnBody))); +} + +// Paste the whole of `src` into the window of `dst` starting at options.{x, y}; +// the rest of `dst` is left untouched. +void install_paste(jsi::Runtime &rt, jsi::Object &module) { + const auto *name = "paste"; + auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count != 3) { + throw error::InvalidArgument("paste: Usage: paste(src, dst, options)"); + } + + auto src = tensor::fromJs(rt, "paste: src", args[0], std::nullopt, {"H", "W", "C"}); + auto dst = tensor::fromJs(rt, "paste: dst", args[1], src->dtype_, {"H'", "W'", src->shape_[2]}); + + tensor::checkNotSameTensor(rt, "paste: src", src, "paste: dst", dst); + auto srcLock = tensor::tryLockShared(rt, "paste: src", src); + auto dstLock = tensor::tryLockUnique(rt, "paste: dst", dst); + + const int32_t srcH = src->shape_[0]; + const int32_t srcW = src->shape_[1]; + const Window win = windowInside(rt, "paste", args[2], srcW, srcH, dst->shape_[1], dst->shape_[0]); + + const int cvType = CV_MAKETYPE(dtypeToCvDepth(src->dtype_), src->shape_[2]); + try { + const ::cv::Mat srcMat(srcH, srcW, cvType, src->data_.get()); + ::cv::Mat dstMat(dst->shape_[0], dst->shape_[1], cvType, dst->data_.get()); + srcMat.copyTo(dstMat(::cv::Rect(win.x, win.y, srcW, srcH))); + } catch (const std::exception &e) { + throw error::Unknown("paste: " + std::string(e.what())); + } + + return jsi::Value(rt, args[1]); + }; + + module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 3, error::guarded(fnBody))); +} + +// Composite `overlay` over `src` where `mask` is non-zero, with the masked +// region eroded by `shrink` px and its edge feathered by a `feather`-wide +// Gaussian, so the seam blends instead of cutting: dst = src*(1-w) + overlay*w. +void install_blendMasked(jsi::Runtime &rt, jsi::Object &module) { + const auto *name = "blendMasked"; + auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count != 5) { + throw error::InvalidArgument("blendMasked: Usage: blendMasked(src, overlay, mask, dst, options)"); + } + + auto src = tensor::fromJs(rt, "blendMasked: src", args[0], DType::uint8, {"H", "W", "C"}); + auto overlay = tensor::fromJs(rt, "blendMasked: overlay", args[1], DType::uint8, src->shape_); + auto mask = tensor::fromJs(rt, "blendMasked: mask", args[2], DType::uint8, {src->shape_[0], src->shape_[1], 1}); + auto dst = tensor::fromJs(rt, "blendMasked: dst", args[3], DType::uint8, src->shape_); + + tensor::checkNotSameTensor(rt, "blendMasked: src", src, "blendMasked: dst", dst); + tensor::checkNotSameTensor(rt, "blendMasked: overlay", overlay, "blendMasked: dst", dst); + auto srcLock = tensor::tryLockShared(rt, "blendMasked: src", src); + auto overlayLock = tensor::tryLockShared(rt, "blendMasked: overlay", overlay); + auto maskLock = tensor::tryLockShared(rt, "blendMasked: mask", mask); + auto dstLock = tensor::tryLockUnique(rt, "blendMasked: dst", dst); + + const auto opts = conversions::asType(rt, "blendMasked: options", args[4]); + const auto shrink = conversions::getRequiredProperty(rt, "blendMasked: options", opts, "shrink"); + const auto feather = conversions::getRequiredProperty(rt, "blendMasked: options", opts, "feather"); + if (shrink < 0) { + throw error::InvalidArgument(std::format("blendMasked: shrink must be >= 0 (got {})", shrink)); + } + if (feather < 0 || (feather > 0 && feather % 2 == 0)) { + throw error::InvalidArgument(std::format("blendMasked: feather must be 0 or an odd kernel size (got {})", feather)); + } + + const int32_t h = src->shape_[0]; + const int32_t w = src->shape_[1]; + const int cvType = CV_MAKETYPE(CV_8U, src->shape_[2]); + + try { + const ::cv::Mat srcMat(h, w, cvType, src->data_.get()); + const ::cv::Mat overlayMat(h, w, cvType, overlay->data_.get()); + const ::cv::Mat maskMat(h, w, CV_8UC1, mask->data_.get()); + ::cv::Mat dstMat(h, w, cvType, dst->data_.get()); + + // Weight of the overlay: 1 inside the (eroded, feathered) mask, 0 outside. + ::cv::Mat region; + ::cv::threshold(maskMat, region, 0, 255, ::cv::THRESH_BINARY); + if (shrink > 0) { + const int k = 2 * shrink + 1; + ::cv::erode(region, region, ::cv::getStructuringElement(::cv::MORPH_RECT, ::cv::Size(k, k))); + } + ::cv::Mat weight; + region.convertTo(weight, CV_32F, 1.0 / 255.0); + if (feather > 0) { + ::cv::GaussianBlur(weight, weight, ::cv::Size(feather, feather), 0); + } + + ::cv::Mat ones = ::cv::Mat::ones(h, w, CV_32F); + ::cv::blendLinear(srcMat, overlayMat, ones - weight, weight, dstMat); + } catch (const std::exception &e) { + throw error::Unknown("blendMasked: " + std::string(e.what())); + } + + return jsi::Value(rt, args[3]); + }; + + module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 5, error::guarded(fnBody))); +} + } // namespace rnexecutorch::extensions::cv::image_ops diff --git a/packages/react-native-executorch/cpp/extensions/cv/image_ops.h b/packages/react-native-executorch/cpp/extensions/cv/image_ops.h index 09b2417fe9..afbffad404 100644 --- a/packages/react-native-executorch/cpp/extensions/cv/image_ops.h +++ b/packages/react-native-executorch/cpp/extensions/cv/image_ops.h @@ -10,4 +10,7 @@ void install_toChannelsLast(facebook::jsi::Runtime &rt, facebook::jsi::Object &m void install_normalize(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); void install_applyColormap(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); void install_rectifyQuad(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +void install_crop(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +void install_paste(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +void install_blendMasked(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); } // namespace rnexecutorch::extensions::cv::image_ops diff --git a/packages/react-native-executorch/cpp/extensions/cv/install.cpp b/packages/react-native-executorch/cpp/extensions/cv/install.cpp index 9a95cc8d6b..32b5066abe 100644 --- a/packages/react-native-executorch/cpp/extensions/cv/install.cpp +++ b/packages/react-native-executorch/cpp/extensions/cv/install.cpp @@ -16,6 +16,9 @@ void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module) { image_ops::install_normalize(rt, cvModule); image_ops::install_applyColormap(rt, cvModule); image_ops::install_rectifyQuad(rt, cvModule); + image_ops::install_crop(rt, cvModule); + image_ops::install_paste(rt, cvModule); + image_ops::install_blendMasked(rt, cvModule); box_ops::install_nms(rt, cvModule); box_ops::install_restrictToBox(rt, cvModule); diff --git a/packages/react-native-executorch/cpp/tests/extensions/CvOpsTest.cpp b/packages/react-native-executorch/cpp/tests/extensions/CvOpsTest.cpp index 658389c5b9..0cdb98fe18 100644 --- a/packages/react-native-executorch/cpp/tests/extensions/CvOpsTest.cpp +++ b/packages/react-native-executorch/cpp/tests/extensions/CvOpsTest.cpp @@ -415,5 +415,191 @@ TEST_F(CvOpsTest, RectifyQuadRejectsWrongArgumentCounts) { "INVALID_ARGUMENT", "Usage: rectifyQuad(src, dst, quad, options)")); } +// --- Windowed copies --------------------------------------------------------- + +TEST_F(CvOpsTest, CropCopiesTheWindowDstIsSizedFor) { + // A 3x3 single-channel image numbered row-major; the 2x2 window at (1, 1) + // is its bottom-right corner. + auto result = evalNumberArray(std::format(R"( + {} + const src = fillU8(createTensor([3, 3, 1], 'uint8'), [1, 2, 3, 4, 5, 6, 7, 8, 9]); + const dst = createTensor([2, 2, 1], 'uint8'); + cv.crop(src, dst, {{ x: 1, y: 1 }}); + return readU8(dst); + )", + kNs)); + EXPECT_TRUE(almostEqual(result, {5, 6, 8, 9})); +} + +TEST_F(CvOpsTest, CropKeepsChannelsInterleaved) { + auto result = evalNumberArray(std::format(R"( + {} + const src = fillU8(createTensor([1, 3, 2], 'uint8'), [1, 2, 3, 4, 5, 6]); + const dst = createTensor([1, 2, 2], 'uint8'); + cv.crop(src, dst, {{ x: 1, y: 0 }}); + return readU8(dst); + )", + kNs)); + EXPECT_TRUE(almostEqual(result, {3, 4, 5, 6})); +} + +TEST_F(CvOpsTest, CropRejectsAWindowOutsideTheSource) { + EXPECT_TRUE(isCodedError(evalThrowing(std::format(R"( + {} + cv.crop(createTensor([3, 3, 1], 'uint8'), createTensor([2, 2, 1], 'uint8'), {{ x: 2, y: 0 }}); + )", + kNs)), + "INVALID_ARGUMENT", "does not fit in a 3x3 image")); +} + +TEST_F(CvOpsTest, CropRejectsAChannelMismatch) { + EXPECT_TRUE(isCodedError(evalThrowing(std::format(R"( + {} + cv.crop(createTensor([3, 3, 3], 'uint8'), createTensor([2, 2, 1], 'uint8'), {{ x: 0, y: 0 }}); + )", + kNs)), + "INVALID_ARGUMENT", "crop: dst")); +} + +TEST_F(CvOpsTest, CropRejectsWrongArgumentCounts) { + EXPECT_TRUE(isCodedError(evalThrowing(std::format("{} cv.crop();", kNs)), + "INVALID_ARGUMENT", "Usage: crop(src, dst, options)")); +} + +TEST_F(CvOpsTest, PasteWritesOnlyTheWindow) { + auto result = evalNumberArray(std::format(R"( + {} + const src = fillU8(createTensor([2, 2, 1], 'uint8'), [5, 6, 8, 9]); + const dst = fillU8(createTensor([3, 3, 1], 'uint8'), [0, 0, 0, 0, 0, 0, 0, 0, 0]); + cv.paste(src, dst, {{ x: 1, y: 1 }}); + return readU8(dst); + )", + kNs)); + EXPECT_TRUE(almostEqual(result, {0, 0, 0, 0, 5, 6, 0, 8, 9})); +} + +TEST_F(CvOpsTest, PasteRoundTripsWithCrop) { + auto result = evalNumberArray(std::format(R"( + {} + const image = fillU8(createTensor([3, 3, 1], 'uint8'), [1, 2, 3, 4, 5, 6, 7, 8, 9]); + const window = createTensor([2, 2, 1], 'uint8'); + cv.crop(image, window, {{ x: 1, y: 1 }}); + const copy = fillU8(createTensor([3, 3, 1], 'uint8'), [1, 2, 3, 4, 0, 0, 7, 0, 0]); + cv.paste(window, copy, {{ x: 1, y: 1 }}); + return readU8(copy); + )", + kNs)); + EXPECT_TRUE(almostEqual(result, {1, 2, 3, 4, 5, 6, 7, 8, 9})); +} + +TEST_F(CvOpsTest, PasteRejectsASourceThatOverhangsTheDestination) { + EXPECT_TRUE(isCodedError(evalThrowing(std::format(R"( + {} + cv.paste(createTensor([2, 2, 1], 'uint8'), createTensor([3, 3, 1], 'uint8'), {{ x: 2, y: 2 }}); + )", + kNs)), + "INVALID_ARGUMENT", "does not fit in a 3x3 image")); +} + +TEST_F(CvOpsTest, PasteRejectsWrongArgumentCounts) { + EXPECT_TRUE(isCodedError(evalThrowing(std::format("{} cv.paste();", kNs)), + "INVALID_ARGUMENT", "Usage: paste(src, dst, options)")); +} + +// --- Masked blending --------------------------------------------------------- + +TEST_F(CvOpsTest, BlendMaskedTakesTheOverlayWhereTheMaskIsSet) { + // Hard edge: masked pixels come from the overlay, the rest from src. + auto result = evalNumberArray(std::format(R"( + {} + const src = fillU8(createTensor([1, 4, 1], 'uint8'), [10, 10, 10, 10]); + const overlay = fillU8(createTensor([1, 4, 1], 'uint8'), [200, 200, 200, 200]); + const mask = fillU8(createTensor([1, 4, 1], 'uint8'), [0, 255, 7, 0]); + const dst = createTensor([1, 4, 1], 'uint8'); + cv.blendMasked(src, overlay, mask, dst, {{ shrink: 0, feather: 0 }}); + return readU8(dst); + )", + kNs)); + EXPECT_TRUE(almostEqual(result, {10, 200, 200, 10})); +} + +TEST_F(CvOpsTest, BlendMaskedBlendsEveryChannel) { + auto result = evalNumberArray(std::format(R"( + {} + const src = fillU8(createTensor([1, 2, 3], 'uint8'), [1, 2, 3, 4, 5, 6]); + const overlay = fillU8(createTensor([1, 2, 3], 'uint8'), [9, 9, 9, 9, 9, 9]); + const mask = fillU8(createTensor([1, 2, 1], 'uint8'), [0, 1]); + const dst = createTensor([1, 2, 3], 'uint8'); + cv.blendMasked(src, overlay, mask, dst, {{ shrink: 0, feather: 0 }}); + return readU8(dst); + )", + kNs)); + EXPECT_TRUE(almostEqual(result, {1, 2, 3, 9, 9, 9})); +} + +TEST_F(CvOpsTest, BlendMaskedShrinkErodesTheMaskedRegion) { + // A 3-wide masked run eroded by 1 px keeps only its centre pixel. + auto result = evalNumberArray(std::format(R"( + {} + const src = fillU8(createTensor([1, 5, 1], 'uint8'), [0, 0, 0, 0, 0]); + const overlay = fillU8(createTensor([1, 5, 1], 'uint8'), [255, 255, 255, 255, 255]); + const mask = fillU8(createTensor([1, 5, 1], 'uint8'), [0, 255, 255, 255, 0]); + const dst = createTensor([1, 5, 1], 'uint8'); + cv.blendMasked(src, overlay, mask, dst, {{ shrink: 1, feather: 0 }}); + return readU8(dst); + )", + kNs)); + EXPECT_TRUE(almostEqual(result, {0, 0, 255, 0, 0})); +} + +TEST_F(CvOpsTest, BlendMaskedFeatherSoftensTheSeam) { + // With a 3-wide Gaussian the pixels either side of the mask edge end up + // strictly between the two sources rather than on one of them. + auto result = evalNumberArray(std::format(R"( + {} + const src = fillU8(createTensor([1, 6, 1], 'uint8'), [0, 0, 0, 0, 0, 0]); + const overlay = fillU8(createTensor([1, 6, 1], 'uint8'), [200, 200, 200, 200, 200, 200]); + const mask = fillU8(createTensor([1, 6, 1], 'uint8'), [0, 0, 0, 255, 255, 255]); + const dst = createTensor([1, 6, 1], 'uint8'); + cv.blendMasked(src, overlay, mask, dst, {{ shrink: 0, feather: 3 }}); + return readU8(dst); + )", + kNs)); + ASSERT_EQ(result.size(), 6u); + EXPECT_EQ(result[0], 0); + EXPECT_GT(result[2], 0); + EXPECT_LT(result[2], 200); + EXPECT_GT(result[3], 0); + EXPECT_LT(result[3], 200); + EXPECT_EQ(result[5], 200); +} + +TEST_F(CvOpsTest, BlendMaskedRejectsAnEvenFeather) { + EXPECT_TRUE(isCodedError(evalThrowing(std::format(R"( + {} + cv.blendMasked(createTensor([1, 2, 1], 'uint8'), createTensor([1, 2, 1], 'uint8'), + createTensor([1, 2, 1], 'uint8'), createTensor([1, 2, 1], 'uint8'), + {{ shrink: 0, feather: 4 }}); + )", + kNs)), + "INVALID_ARGUMENT", "feather must be 0 or an odd kernel size")); +} + +TEST_F(CvOpsTest, BlendMaskedRejectsAMaskOfTheWrongShape) { + EXPECT_TRUE(isCodedError(evalThrowing(std::format(R"( + {} + cv.blendMasked(createTensor([2, 2, 3], 'uint8'), createTensor([2, 2, 3], 'uint8'), + createTensor([2, 2, 3], 'uint8'), createTensor([2, 2, 3], 'uint8'), + {{ shrink: 0, feather: 0 }}); + )", + kNs)), + "INVALID_ARGUMENT", "blendMasked: mask")); +} + +TEST_F(CvOpsTest, BlendMaskedRejectsWrongArgumentCounts) { + EXPECT_TRUE(isCodedError(evalThrowing(std::format("{} cv.blendMasked();", kNs)), + "INVALID_ARGUMENT", "Usage: blendMasked(src, overlay, mask, dst, options)")); +} + } // namespace } // namespace rnexecutorch::tests diff --git a/packages/react-native-executorch/src/extensions/cv/ops/image.ts b/packages/react-native-executorch/src/extensions/cv/ops/image.ts index e15421db1b..f86807bb1e 100644 --- a/packages/react-native-executorch/src/extensions/cv/ops/image.ts +++ b/packages/react-native-executorch/src/extensions/cv/ops/image.ts @@ -265,3 +265,106 @@ export function applyColormap( 'worklet'; return rnexecutorchJsi.cv.applyColormap(src, dst, colormap); } + +/** + * Position of a window inside an image, in pixels from its top-left corner. + * @category CV / Types + */ +export type WindowOrigin = { + /** Column of the window's left edge. */ + readonly x: number; + /** Row of the window's top edge. */ + readonly y: number; +}; + +/** + * Copies a window of `src` into `dst`. The window is the size of `dst` and its + * top-left corner sits at `origin`, so the caller sizes the destination rather + * than passing the extent twice. + * @category CV / Functions + * @param src The source image tensor in HWC layout. Expected shape `[H, W, C]` + * (channels-last). Supports any numeric data type. + * @param dst The pre-allocated destination image tensor, `[H', W', C]` in HWC + * layout with the same channel count and data type as `src`. Its spatial + * dimensions are the window size. + * @param origin Where the window starts in `src`. See {@link WindowOrigin}. + * @returns The destination image tensor `dst`. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if the window does + * not fit inside `src` or if tensor shapes, layouts, or data types are invalid, + * `RESOURCE_BUSY` if a tensor is in use, or `RESOURCE_DISPOSED` if either + * tensor was disposed. + */ +export function crop(src: Tensor, dst: Tensor, origin: WindowOrigin): Tensor { + 'worklet'; + return rnexecutorchJsi.cv.crop(src, dst, { x: origin.x, y: origin.y }); +} + +/** + * Pastes the whole of `src` into the window of `dst` whose top-left corner + * sits at `origin`, leaving the rest of `dst` untouched. The inverse of + * {@link crop}. + * @category CV / Functions + * @param src The source image tensor in HWC layout. Expected shape `[H, W, C]` + * (channels-last). Supports any numeric data type. + * @param dst The pre-allocated destination image tensor, `[H', W', C]` in HWC + * layout with the same channel count and data type as `src`, at least as large + * as `src` in both spatial dimensions. + * @param origin Where the window starts in `dst`. See {@link WindowOrigin}. + * @returns The destination image tensor `dst`. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if `src` does not fit + * inside `dst` at `origin` or if tensor shapes, layouts, or data types are + * invalid, `RESOURCE_BUSY` if a tensor is in use, or `RESOURCE_DISPOSED` if + * either tensor was disposed. + */ +export function paste(src: Tensor, dst: Tensor, origin: WindowOrigin): Tensor { + 'worklet'; + return rnexecutorchJsi.cv.paste(src, dst, { x: origin.x, y: origin.y }); +} + +/** + * Options softening the mask edge in {@link blendMasked}. + * @category CV / Types + */ +export type BlendMaskedOptions = { + /** + * Pixels to erode the masked region by before feathering, so the seam sits + * inside the region rather than on its edge. Default `0`. + */ + readonly shrink?: number; + /** + * Size in pixels of the Gaussian kernel that feathers the mask edge. Must be + * odd; `0` keeps the edge hard. Default `0`. + */ + readonly feather?: number; +}; + +/** + * Composites `overlay` over `src` where `mask` is non-zero, writing + * `src * (1 - w) + overlay * w` into `dst`, where `w` is the mask eroded and + * feathered per `options`. Pixels outside the mask come from `src` unchanged. + * @category CV / Functions + * @param src The base image tensor in HWC layout, `[H, W, C]`, data type `uint8`. + * @param overlay The image tensor to composite, `[H, W, C]`, data type `uint8`. + * @param mask The region to take from `overlay`, `[H, W, 1]`, data type `uint8`; + * any non-zero value marks a masked pixel. + * @param dst The pre-allocated destination image tensor, `[H, W, C]`, data type + * `uint8`. Must not alias `src` or `overlay`. + * @param options Edge erosion and feathering. See {@link BlendMaskedOptions}. + * @returns The destination image tensor `dst`. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes, + * layouts, or data types are invalid, or `feather` is even, `RESOURCE_BUSY` if + * a tensor is in use, or `RESOURCE_DISPOSED` if any tensor was disposed. + */ +export function blendMasked( + src: Tensor, + overlay: Tensor, + mask: Tensor, + dst: Tensor, + options?: BlendMaskedOptions +): Tensor { + 'worklet'; + return rnexecutorchJsi.cv.blendMasked(src, overlay, mask, dst, { + shrink: options?.shrink ?? 0, + feather: options?.feather ?? 0, + }); +} diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/inpainting.ts b/packages/react-native-executorch/src/extensions/cv/tasks/inpainting.ts new file mode 100644 index 0000000000..b6793566c2 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/cv/tasks/inpainting.ts @@ -0,0 +1,374 @@ +/** + * Image inpainting task pipeline: fills a masked region of a photo with + * plausible content (object removal, blemish cleanup), with the crop around the + * hole, the model I/O normalization, and the feathered blend back into the + * frame handled here. + */ + +import type { WorkletRuntime } from 'react-native-worklets'; + +import { tensor, type Tensor } from '../../../core/tensor'; +import { loadModel } from '../../../core/model'; +import { validateSpec, method, f32 } from '../../../core/schema'; +import { wrapAsync } from '../../../core/runtime'; +import { createResourceScope } from '../../../core/lifetime'; +import { RnExecuTorchError } from '../../../core/error'; +import { threshold } from '../../math'; + +import type { ImageBuffer } from '../image'; +import { + FORMAT_CHANNELS, + FORMAT_CONVERSION, + resize, + cvtColor, + toChannelsFirst, + toChannelsLast, + normalize, + crop, + paste, + blendMasked, + type InterpolationMethod, + type BlendMaskedOptions, +} from '../ops/image'; + +/** + * Options for configuring the inpainting preprocessor and postprocessor. + * @category CV / Types + */ +export type InpainterOptions = { + /** + * Interpolation used to bring the crop around the hole to the model's input + * size. The mask is always resized nearest-neighbor so it stays binary. + */ + readonly interpolation: InterpolationMethod; + /** Interpolation used to bring the model output back to the crop's size. */ + readonly outInterpolation: InterpolationMethod; + /** + * Context kept around the hole, in pixels on each side of its bounding box, + * when cropping the region the model sees. The crop is square, never smaller + * than the model input, and clamped to the image. + */ + readonly cropPadding: number; + /** + * How the filled region is blended back over the original pixels. See + * {@link BlendMaskedOptions}. + */ + readonly blend: BlendMaskedOptions; +}; + +/** + * Model configuration required to instantiate an inpainting task runner. + * @category CV / Types + */ +export type InpainterModel = { + /** Local path or remote URL of the `.pte` model file. */ + readonly modelPath: string; + /** + * Crop, interpolation and blending options. See {@link InpainterOptions}. + */ + readonly modelOpts: InpainterOptions; +}; + +/** + * The window of an image an inpainting model is run on, in pixels of that + * image. + * @category CV / Types + */ +export type InpaintingCrop = { + /** Column of the window's left edge. */ + readonly x: number; + /** Row of the window's top edge. */ + readonly y: number; + /** Window width. */ + readonly width: number; + /** Window height. */ + readonly height: number; +}; + +/** + * Image inpainting task runner. + * @category CV / Types + */ +export type Inpainter = { + /** + * Releases all allocated native resources. + */ + readonly dispose: () => void; + + /** + * Fills the masked region of `image` and returns the whole frame with the + * hole filled. + * @param image The input image buffer. + * @param mask A buffer of the same dimensions as `image` marking the region + * to fill: any non-zero pixel is part of the hole. A `'gray'` buffer is used + * as is; other formats are converted to grayscale first. An all-zero mask + * returns the input unchanged without running the model. + * @returns A promise resolving to the inpainted RGBA image buffer at the + * input dimensions. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if `mask` does not + * match the dimensions of `image`, `RESOURCE_BUSY` if the model is in use, + * or `RESOURCE_DISPOSED` if disposed. + */ + readonly inpaint: (image: ImageBuffer, mask: ImageBuffer) => Promise; + + /** + * Synchronous version of {@link inpaint} to be executed directly on the + * caller or worklet thread. + */ + readonly inpaintWorklet: (image: ImageBuffer, mask: ImageBuffer) => ImageBuffer; +}; + +/** + * Finds the window of an image an inpainting model should be run on: a square + * around the hole's bounding box, padded by `padding` on each side, never + * smaller than `minSize`, and clamped to the image, with any overhang pushed + * to the opposite side so the window stays as large as it can. + * + * A pixel is masked when any of its color channels is non-zero (alpha is + * ignored), which is also what the grayscale conversion `inpaint` applies to + * a color mask yields. + * @category CV / Functions + * @param mask The hole mask, any format, in HWC layout. + * @param options Context to keep around the hole and the smallest window to + * return, both in pixels. + * @returns The window, or `null` when the mask marks no pixel. + */ +export function computeInpaintingCrop( + mask: ImageBuffer, + options: { readonly padding: number; readonly minSize: number } +): InpaintingCrop | null { + 'worklet'; + const { data, width, height, format } = mask; + const channels = FORMAT_CHANNELS[format]; + const colorChannels = Math.min(channels, 3); + + let left = width; + let right = -1; + let top = height; + let bottom = -1; + for (let y = 0; y < height; y++) { + const row = y * width * channels; + for (let x = 0; x < width; x++) { + const at = row + x * channels; + let masked = false; + for (let c = 0; c < colorChannels; c++) { + if (data[at + c] !== 0) { + masked = true; + break; + } + } + if (!masked) continue; + if (x < left) left = x; + if (x > right) right = x; + if (y < top) top = y; + if (y > bottom) bottom = y; + } + } + if (right < 0) return null; + + const centerX = Math.floor((left + right) / 2); + const centerY = Math.floor((top + bottom) / 2); + const extent = Math.max(right - left + 1, bottom - top + 1) + 2 * options.padding; + const size = Math.max(extent, options.minSize); + const half = Math.floor(size / 2); + + let x0 = Math.max(centerX - half, 0); + let x1 = Math.min(centerX - half + size, width); + let y0 = Math.max(centerY - half, 0); + let y1 = Math.min(centerY - half + size, height); + + const overhangX = Math.max(size - (x1 - x0), 0); + const overhangY = Math.max(size - (y1 - y0), 0); + x0 = Math.max(x0 - overhangX, 0); + x1 = Math.min(x1 + overhangX, width); + y0 = Math.max(y0 - overhangY, 0); + y1 = Math.min(y1 + overhangY, height); + + return { x: x0, y: y0, width: x1 - x0, height: y1 - y0 }; +} + +/** + * Creates an image inpainting runner for executing local inpainting models. + * + * It validates the model inputs and outputs requirements, pre-allocates the + * necessary static execution tensors, and registers clean disposal hooks to + * clear all native memory. The model is expected to take an RGB image in + * `[0, 1]` and a mask with `1` for known pixels and `0` for the hole, and to + * return the filled RGB image in `[0, 1]`, all at a fixed resolution. + * @category CV / Tasks + * @param config Inpainting task configuration containing path and options. + * See {@link InpainterModel}. + * @param runtime Optional worklet runtime thread on which to run the model execution. + * @returns A promise resolving to the instantiated {@link Inpainter} runner. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if model fails to load, + * or `SCHEMA_MISMATCH` if model schema does not match the inpainting spec. + */ +export async function createInpainter( + config: InpainterModel, + runtime?: WorkletRuntime +): Promise { + const scope = createResourceScope(); + const dispose = scope.dispose; + + try { + const { modelPath, modelOpts } = config; + const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + + const { variant, dims } = validateSpec(model.schema, { + batched: method( + 'forward', // prettier-ignore + [f32(1, 3, 'H', 'W'), f32(1, 1, 'H', 'W')], + [f32(1, 3, 'H', 'W')] + ), + unbatched: method( + 'forward', // prettier-ignore + [f32(3, 'H', 'W'), f32(1, 'H', 'W')], + [f32(3, 'H', 'W')] + ), + }); + + const [H, W] = dims.constant('H', 'W'); + const imageShape = { batched: [1, 3, H, W], unbatched: [3, H, W] }[variant]; + const maskShape = { batched: [1, 1, H, W], unbatched: [1, H, W] }[variant]; + + const tensors = [ + // Model I/O. + tensor('float32', imageShape), + tensor('float32', maskShape), + tensor('float32', imageShape), + // Image: resized RGB crop -> CHW -> normalized. + tensor('uint8', [H, W, 3]), + tensor('uint8', [3, H, W]), + tensor('float32', [3, H, W]), + // Mask: resized -> CHW -> float -> binarized (1 = hole) -> inverted (1 = known). + tensor('uint8', [H, W, 1]), + tensor('uint8', [1, H, W]), + tensor('float32', [1, H, W]), + tensor('float32', [1, H, W]), + tensor('float32', [1, H, W]), + // Output: CHW float -> uint8 -> HWC -> RGBA. + tensor('float32', [3, H, W]), + tensor('uint8', [3, H, W]), + tensor('uint8', [H, W, 3]), + tensor('uint8', [H, W, 4]), + ] as const; + + tensors.forEach(scope.track); + + const [ + tModelImage, + tModelMask, + tModelOutput, + tInRgb, + tInChw, + tInNorm, + tMaskResized, + tMaskChw, + tMaskFloat, + tMaskHole, + tMaskKnown, + tOutChw, + tOutU8, + tOutHwc, + tOutRgba, + ] = tensors; + + const { interpolation, outInterpolation, cropPadding, blend } = modelOpts; + const minCropSize = Math.max(H, W); + + const inpaintWorklet = (image: ImageBuffer, mask: ImageBuffer): ImageBuffer => { + 'worklet'; + if (mask.width !== image.width || mask.height !== image.height) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `inpaint: mask is ${mask.width}x${mask.height} but image is ${image.width}x${image.height}` + ); + } + + const { width, height } = image; + const imageChannels = FORMAT_CHANNELS[image.format]; + const toRgba = FORMAT_CONVERSION[image.format].rgba; + const maskChannels = FORMAT_CHANNELS[mask.format]; + const toGray = FORMAT_CONVERSION[mask.format].gray; + + const perCall: Tensor[] = []; + const scratch = (dtype: 'uint8', shape: number[]): Tensor => { + const t = tensor(dtype, shape); + perCall.push(t); + return t; + }; + + try { + // The full-resolution RGBA frame the result is assembled in. + const tImage = scratch('uint8', [height, width, imageChannels]).setData(image.data); + const tFrame = + toRgba === null + ? tImage + : tImage.through(cvtColor, scratch('uint8', [height, width, 4]), toRgba); + + const window = computeInpaintingCrop(mask, { padding: cropPadding, minSize: minCropSize }); + if (window !== null) { + const tMaskIn = scratch('uint8', [height, width, maskChannels]).setData(mask.data); + const tMaskGray = + toGray === null + ? tMaskIn + : tMaskIn.through(cvtColor, scratch('uint8', [height, width, 1]), toGray); + + const { x, y, width: cw, height: ch } = window; + + // Image crop -> model input. + const tCrop = tFrame.through(crop, scratch('uint8', [ch, cw, 4]), { x, y }); + tCrop + .through(cvtColor, scratch('uint8', [ch, cw, 3]), 'RGBA2RGB') + .through(resize, tInRgb, { mode: 'stretch', interpolation }) + .through(toChannelsFirst, tInChw) + .through(normalize, tInNorm, { alpha: 1 / 255, beta: 0 }) + .copyTo(tModelImage); + + // Mask crop -> model input, binarized so any non-zero value is the + // hole, then inverted to the model's 1 = known convention. + const tMaskCrop = tMaskGray.through(crop, scratch('uint8', [ch, cw, 1]), { x, y }); + tMaskCrop + .through(resize, tMaskResized, { mode: 'stretch', interpolation: 'nearest' }) + .through(toChannelsFirst, tMaskChw) + .through(normalize, tMaskFloat, { alpha: 1, beta: 0 }) + .through(threshold, tMaskHole, 0.5) + .through(normalize, tMaskKnown, { alpha: -1, beta: 1 }) + .copyTo(tModelMask); + + model.execute('forward', [tModelImage, tModelMask], [tModelOutput]); + + // Model output -> RGBA at crop size, blended into the crop over the + // original pixels, then pasted back into the frame. + const tFilled = tModelOutput + .copyTo(tOutChw) + .through(normalize, tOutU8, { alpha: 255, beta: 0 }) + .through(toChannelsLast, tOutHwc) + .through(cvtColor, tOutRgba, 'RGB2RGBA') + .through(resize, scratch('uint8', [ch, cw, 4]), { + mode: 'stretch', + interpolation: outInterpolation, + }); + + blendMasked(tCrop, tFilled, tMaskCrop, scratch('uint8', [ch, cw, 4]), blend).through( + paste, + tFrame, + { x, y } + ); + } + + const data = tFrame.getData(new Uint8Array(width * height * 4)); + return { data, width, height, format: 'rgba', layout: 'hwc' }; + } finally { + perCall.forEach((t) => t.dispose()); + } + }; + + const inpaint = wrapAsync(inpaintWorklet, runtime); + + return { inpaint, inpaintWorklet, dispose }; + } catch (error) { + dispose(); + throw error; + } +} diff --git a/packages/react-native-executorch/src/hooks/useInpainter.ts b/packages/react-native-executorch/src/hooks/useInpainter.ts new file mode 100644 index 0000000000..5e24080e1f --- /dev/null +++ b/packages/react-native-executorch/src/hooks/useInpainter.ts @@ -0,0 +1,34 @@ +import { useModel } from './useModel'; +import { useResourceDownload, type ResourceOptions } from './useResourceDownload'; +import { createInpainter, type InpainterModel } from '../extensions/cv/tasks/inpainting'; + +/** + * React hook to load and run an image inpainting model. + * + * This hook manages downloading (if remote URLs are provided) and loading the + * model assets, compiling them, tracking download progress and load errors, and + * releasing native memory when the component unmounts or the configuration + * changes. + * + * For imperative usage, see {@link createInpainter}. + * @category Hooks + * @param config The inpainting model configuration. + * See {@link InpainterModel}. + * @param options Load and caching options. See {@link ResourceOptions}. + * @returns The same object as {@link Inpainter} (without `dispose`), + * combined with loading state and download progress. + * @see {@link Inpainter} + */ +export function useInpainter(config: InpainterModel, options?: ResourceOptions) { + const { resource, downloadProgress, downloadError } = useResourceDownload(config, options); + const { model, error } = useModel(createInpainter, resource); + + return { + isReady: !!model, + error: downloadError || error, + downloadProgress, + resource, + inpaint: model?.inpaint, + inpaintWorklet: model?.inpaintWorklet, + }; +} diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts index cedcbaa5ba..0558dfbdb6 100644 --- a/packages/react-native-executorch/src/index.ts +++ b/packages/react-native-executorch/src/index.ts @@ -46,6 +46,7 @@ // Hooks — primary API for app developers export * from './hooks/useClassifier'; export * from './hooks/useStyleTransfer'; +export * from './hooks/useInpainter'; export * from './hooks/useSemanticSegmenter'; export * from './hooks/useInstanceSegmenter'; export * from './hooks/useKeypointDetector'; @@ -73,6 +74,7 @@ export * from './constants'; // Task APIs — for developers needing manual lifetime/disposal control export * from './extensions/cv/tasks/classification'; export * from './extensions/cv/tasks/styleTransfer'; +export * from './extensions/cv/tasks/inpainting'; export * from './extensions/cv/tasks/semanticSegmentation'; export * from './extensions/cv/tasks/instanceSegmentation'; export * from './extensions/cv/tasks/keypointDetection'; From a2d30790d87d8f48b27d761e6fa54369182322a5 Mon Sep 17 00:00:00 2001 From: borys3kk Date: Fri, 23 Oct 2026 16:21:06 +0200 Subject: [PATCH 02/17] feat(models): register the MI-GAN inpainting exports models.inpainting.MIGAN with the Places2 512 and 256 checkpoints and the FFHQ faces checkpoint, fp16 only, on XNNPACK, Core ML and Vulkan, plus the matching inpainting feature in the native libs map. --- .../08-native-libraries.md | 11 +-- .../__tests__/api/modelVariants.test.ts | 1 + .../scripts/download-libs.js | 4 +- .../react-native-executorch/src/models.ts | 87 +++++++++++++++++++ 4 files changed, 97 insertions(+), 6 deletions(-) diff --git a/docs/docs/03-core-and-advanced/08-native-libraries.md b/docs/docs/03-core-and-advanced/08-native-libraries.md index 39d255429e..b8400184d0 100644 --- a/docs/docs/03-core-and-advanced/08-native-libraries.md +++ b/docs/docs/03-core-and-advanced/08-native-libraries.md @@ -39,11 +39,11 @@ Add a `react-native-executorch` block to your `package.json`: ### Options -| Option | Purpose | Accepted values | -| ---------- | ------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `features` | High-level tasks — each automatically expands to the backends and libraries it needs | `"classification"`, `"imageEmbeddings"`, `"instanceSegmentation"`, `"keypointDetection"`, `"llm"`, `"multimodalLLM"`, `"objectDetection"`, `"ocr"`, `"privacyFilter"`, `"segmentAnything"`, `"semanticSegmentation"`, `"speechToText"`, `"styleTransfer"`, `"textEmbeddings"`, `"textToImage"`, `"textToSpeech"`, `"tokenizer"`, `"vad"`, `"verticalOCR"` | -| `backends` | Hardware acceleration backends directly | `"xnnpack"`, `"coreml"`, `"mlx"`, `"vulkan"` | -| `libs` | Extra native C++ libraries | `"opencv"`, `"phonemis"` | +| Option | Purpose | Accepted values | +| ---------- | ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `features` | High-level tasks — each automatically expands to the backends and libraries it needs | `"classification"`, `"imageEmbeddings"`, `"inpainting"`, `"instanceSegmentation"`, `"keypointDetection"`, `"llm"`, `"multimodalLLM"`, `"objectDetection"`, `"ocr"`, `"privacyFilter"`, `"segmentAnything"`, `"semanticSegmentation"`, `"speechToText"`, `"styleTransfer"`, `"textEmbeddings"`, `"textToImage"`, `"textToSpeech"`, `"tokenizer"`, `"vad"`, `"verticalOCR"` | +| `backends` | Hardware acceleration backends directly | `"xnnpack"`, `"coreml"`, `"mlx"`, `"vulkan"` | +| `libs` | Extra native C++ libraries | `"opencv"`, `"phonemis"` | The three lists are merged, so you can pair high-level `features` with specific `backends` or `libs`. Re-run your package manager install after editing. @@ -100,6 +100,7 @@ Specifying a task under `features` is shorthand: it automatically expands to the | `verticalOCR` | xnnpack | opencv | | `styleTransfer` | xnnpack, coreml, vulkan | opencv | | `textToImage` | xnnpack, coreml, vulkan | opencv | +| `inpainting` | xnnpack, coreml, vulkan | opencv | | `segmentAnything` | xnnpack, coreml, vulkan | opencv | | `tokenizer` | — | — | diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts index 8c82b22af3..f0defb0dda 100644 --- a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts +++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts @@ -349,6 +349,7 @@ describe('feature map', () => { privacyFilter: 'privacyFilter', imageEmbeddings: 'imageEmbeddings', textToImage: 'textToImage', + inpainting: 'inpainting', textToSpeech: 'textToSpeech', ocr: 'ocr', }; diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js index 10e305bdee..a252241c7c 100644 --- a/packages/react-native-executorch/scripts/download-libs.js +++ b/packages/react-native-executorch/scripts/download-libs.js @@ -46,7 +46,7 @@ * textEmbeddings, imageEmbeddings, * classification, objectDetection, semanticSegmentation, instanceSegmentation, * ocr, verticalOCR, keypointDetection, styleTransfer, textToImage, segmentAnything, - * tokenizer + * inpainting, tokenizer * * Platform applicability: * opencv Android + iOS — downloaded artifact @@ -176,6 +176,8 @@ const FEATURE_MAP = { styleTransfer: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] }, // SDXS ships xnnpack, coreml and a vulkan Android export. textToImage: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] }, + // MI-GAN ships xnnpack, coreml and a vulkan Android export. + inpainting: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] }, // FastSAM ships xnnpack, coreml and a vulkan Android export. segmentAnything: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] }, // Tokenizer is pure-CPU string ops resolved from libexecutorch; needs no diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 1ab72f13a6..0fc7a8c648 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -19,6 +19,7 @@ import { getRegisteredBackends } from './utils'; import type { ClassifierModel } from './extensions/cv/tasks/classification'; import type { ObjectDetectorModel } from './extensions/cv/tasks/objectDetection'; import type { StyleTransferModel } from './extensions/cv/tasks/styleTransfer'; +import type { InpainterModel } from './extensions/cv/tasks/inpainting'; import type { SemanticSegmenterModel } from './extensions/cv/tasks/semanticSegmentation'; import type { KeypointDetectorModel } from './extensions/cv/tasks/keypointDetection'; import type { InstanceSegmenterModel } from './extensions/cv/tasks/instanceSegmentation'; @@ -277,6 +278,56 @@ const STYLE_TRANSFER_UDNIE_VULKAN_FP16: StyleTransferModel = { modelOpts: STYLE_TRANSFER_OPTS, }; +// ============================================================================= +// Inpainting +// ============================================================================= +// MI-GAN takes the image in [0, 1] and a 1 = known / 0 = hole mask and folds +// its own normalization into the graph; the crop around the hole and the +// feathered blend follow upstream's reference pipeline (128 px of context, the +// hole eroded by 1 px, a 5-tap Gaussian seam). +const MIGAN_OPTS = { + interpolation: 'linear' as const, + outInterpolation: 'linear' as const, + cropPadding: 128, + blend: { shrink: 1, feather: 5 }, +}; +const MIGAN_512_XNNPACK_FP16: InpainterModel = { + modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/xnnpack/512/migan_512_xnnpack_fp16.pte`, + modelOpts: MIGAN_OPTS, +}; +const MIGAN_512_COREML_FP16: InpainterModel = { + modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/coreml/512/migan_512_coreml_fp16.pte`, + modelOpts: MIGAN_OPTS, +}; +const MIGAN_512_VULKAN_FP16: InpainterModel = { + modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/vulkan/512/migan_512_vulkan_fp16.pte`, + modelOpts: MIGAN_OPTS, +}; +const MIGAN_256_XNNPACK_FP16: InpainterModel = { + modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/xnnpack/256/migan_256_xnnpack_fp16.pte`, + modelOpts: MIGAN_OPTS, +}; +const MIGAN_256_COREML_FP16: InpainterModel = { + modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/coreml/256/migan_256_coreml_fp16.pte`, + modelOpts: MIGAN_OPTS, +}; +const MIGAN_256_VULKAN_FP16: InpainterModel = { + modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/vulkan/256/migan_256_vulkan_fp16.pte`, + modelOpts: MIGAN_OPTS, +}; +const MIGAN_FACES_256_XNNPACK_FP16: InpainterModel = { + modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/xnnpack/faces_256/migan_faces_256_xnnpack_fp16.pte`, + modelOpts: MIGAN_OPTS, +}; +const MIGAN_FACES_256_COREML_FP16: InpainterModel = { + modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/coreml/faces_256/migan_faces_256_coreml_fp16.pte`, + modelOpts: MIGAN_OPTS, +}; +const MIGAN_FACES_256_VULKAN_FP16: InpainterModel = { + modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/vulkan/faces_256/migan_faces_256_vulkan_fp16.pte`, + modelOpts: MIGAN_OPTS, +}; + // ============================================================================= // Semantic Segmentation // ============================================================================= @@ -2133,6 +2184,42 @@ export const models = { }), }, + /** + * Image inpainting models that fill a masked region of a photo with + * plausible content, for object removal and photo cleanup. + */ + inpainting: { + /** + * MI-GAN, a mobile inpainting GAN trained on Places2 (scenes) and FFHQ + * (faces). The size names the resolution the model runs at; the region + * around the hole is cropped and resized to it, so larger photos work at + * either size, with `SIZE_512` keeping more detail at a higher cost. + */ + MIGAN: family({ + /** Places2 checkpoint at 512x512, the default for general photos. */ + SIZE_512: variants({ + XNNPACK_FP16: MIGAN_512_XNNPACK_FP16, + COREML_FP16: MIGAN_512_COREML_FP16, + VULKAN_FP16: MIGAN_512_VULKAN_FP16, + }), + /** Places2 checkpoint at 256x256, for lower latency. */ + SIZE_256: variants({ + XNNPACK_FP16: MIGAN_256_XNNPACK_FP16, + COREML_FP16: MIGAN_256_COREML_FP16, + VULKAN_FP16: MIGAN_256_VULKAN_FP16, + }), + /** + * FFHQ checkpoint at 256x256, trained on faces. Reconstructs eyes and + * mouths where the scene checkpoints smear them. + */ + FACES_256: variants({ + XNNPACK_FP16: MIGAN_FACES_256_XNNPACK_FP16, + COREML_FP16: MIGAN_FACES_256_COREML_FP16, + VULKAN_FP16: MIGAN_FACES_256_VULKAN_FP16, + }), + }), + }, + /** * Semantic segmentation models that classify each pixel into target object or * background classes. From e752ea2a43e17cc0da63465c86678370e2111d79 Mon Sep 17 00:00:00 2001 From: borys3kk Date: Fri, 23 Oct 2026 16:21:15 +0200 Subject: [PATCH 03/17] docs: add the image inpainting guide Camera integration moves to 12 so the task pages stay together. --- .../computer-vision/11-image-inpainting.md | 187 ++++++++++++++++++ ...ntegration.md => 12-camera-integration.md} | 0 2 files changed, 187 insertions(+) create mode 100644 docs/docs/02-extensions/computer-vision/11-image-inpainting.md rename docs/docs/02-extensions/computer-vision/{11-camera-integration.md => 12-camera-integration.md} (100%) diff --git a/docs/docs/02-extensions/computer-vision/11-image-inpainting.md b/docs/docs/02-extensions/computer-vision/11-image-inpainting.md new file mode 100644 index 0000000000..2023820fb1 --- /dev/null +++ b/docs/docs/02-extensions/computer-vision/11-image-inpainting.md @@ -0,0 +1,187 @@ +--- +title: Image Inpainting +slug: /extensions/image-inpainting +description: 'Remove objects and fill masked regions of photos on-device with MI-GAN, with the crop, mask handling and seam blending built in.' +keywords: + [ + react native, + inpainting, + object removal, + photo cleanup, + image editing, + mi-gan, + migan, + mask, + mobile ml, + on-device ai, + ] +--- + +# Image Inpainting + +Image inpainting fills a region of a photo with content that matches its surroundings: paint over an unwanted object, a blemish, or a passer-by, and the model reconstructs what would plausibly be behind it. + +The pipeline runs entirely on-device. It crops the area around the painted region, runs the model on that crop, and blends the result back into the full-resolution photo with a feathered seam, so the rest of the image is returned pixel for pixel as it came in. + +## Quick Start + +The [`useInpainter`](../../06-api-reference/functions/useInpainter.md) hook manages model downloading, initialization, and lifecycle: + +```tsx +import { models, useInpainter } from 'react-native-executorch'; +import type { ImageBuffer } from 'react-native-executorch/cv'; + +function MyComponent() { + const inpainter = useInpainter(models.inpainting.MIGAN.DEFAULT); + + // Hook state: + // inpainter.isReady — true once model is downloaded and loaded in memory + // inpainter.downloadProgress — 0 to 100 download progress + // inpainter.error — Error instance if download or load failed + // inpainter.resource — resolved config with all URLs replaced by local file paths + + const handleInpaint = async (image: ImageBuffer, mask: ImageBuffer) => { + if (!inpainter.isReady || !inpainter.inpaint) return; + + // Run inference on background thread + const result = await inpainter.inpaint(image, mask); + console.log('Inpainted image dimensions:', result.width, result.height); + }; + + // Build `mask` from what the user painted, then call handleInpaint +} +``` + +:::tip Full Interactive Example +See [`apps/computer-vision/app/inpainting`](https://github.com/software-mansion/react-native-executorch/tree/main/apps/computer-vision/app/inpainting) in this repository for a complete screen with a photo picker, a brush to paint the mask over the photo, and before/after comparison. +::: + +## The Mask + +[`inpaint()`](../../06-api-reference/type-aliases/Inpainter.md#inpaint) takes two [`ImageBuffer`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/ImageBuffer.md)s of the same dimensions: the photo and the mask. In the mask, **any non-zero pixel marks the region to fill**, so the natural way to build one is to paint white strokes on a black canvas. A `'gray'` buffer is used as is; a color buffer is converted to grayscale first, so the RGBA pixels read back from a drawing surface work directly. + +```typescript +// A mask with a 100x100 hole at (200, 150) over an 800x600 photo +const mask = new Uint8Array(800 * 600); +for (let y = 150; y < 250; y++) { + for (let x = 200; x < 300; x++) mask[y * 800 + x] = 255; +} +const maskBuffer = { data: mask, width: 800, height: 600, format: 'gray', layout: 'hwc' } as const; +``` + +An all-zero mask returns the input unchanged without running the model. + +:::tip Cover the whole object +Paint over the entire object, with a little margin. MI-GAN continues whatever touches the edge of the hole, so a mask that cuts through an object makes the model extend that object into the hole instead of removing it. +::: + +## Output Format + +The result is an [`ImageBuffer`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/ImageBuffer.md) containing the whole frame as RGBA at the input dimensions, with the masked region filled: + +```typescript +type ImageBuffer = { + readonly width: number; + readonly height: number; + readonly format: 'rgba'; + readonly data: Uint8Array; +}; +``` + +Pixels outside the mask are copied from the input; the model only ever touches the masked region and a thin feathered seam around it. + +## How the Crop Works + +The models run at a fixed resolution (256 or 512 px), far below a phone photo. Rather than shrinking the whole photo, the pipeline follows the reference MI-GAN implementation: + +1. Find the bounding box of the mask and frame it in a square with [`cropPadding`](../../06-api-reference/type-aliases/InpainterOptions.md#croppadding) pixels of context on each side, never smaller than the model input. The square is clamped to the photo, with any overhang pushed to the opposite side. +2. Resize that crop to the model input (the mask with nearest-neighbor, so it stays binary), run the model, and resize the output back to the crop. +3. Blend the output over the original crop through the mask, eroded and feathered per [`blend`](../../06-api-reference/type-aliases/InpainterOptions.md#blend), and paste the crop back into the photo. + +A small hole in a large photo therefore keeps its surroundings at native resolution, while a large hole is filled at the model's resolution and upsampled. The square the model saw is available through [`computeInpaintingCrop()`](../../06-api-reference/functions/computeInpaintingCrop.md) if you want to show it or size the mask to it. + +## Imperative API + +For background photo processing, headless workflows, or manual lifecycle management outside React components, create the pipeline using [`createInpainter`](../../06-api-reference/functions/createInpainter.md): + +```typescript +import { createInpainter, download, models } from 'react-native-executorch'; + +// Download and cache model assets before creating the pipeline +const model = await download(models.inpainting.MIGAN.DEFAULT); +const inpainter = await createInpainter(model); + +try { + const result = await inpainter.inpaint(imageBuffer, maskBuffer); + console.log('Inpainted output byte length:', result.data.byteLength); +} finally { + // Always release native resources when finished + inpainter.dispose(); +} +``` + +## Synchronous Execution + +[`createInpainter`](../../06-api-reference/functions/createInpainter.md) also exposes a synchronous [`inpaintWorklet`](../../06-api-reference/type-aliases/Inpainter.md#inpaintworklet) function that runs directly on the worklet thread with zero Promise scheduling overhead: + +```typescript +// Called synchronously on the UI worklet thread +const result = inpainter.inpaintWorklet(imageBuffer, maskBuffer); +``` + +See [Worklets & Threading](../../03-core-and-advanced/06-worklets-and-threading.md) for details on worklet execution contexts and zero-copy host objects. + +## Available Models + +The library provides ready-to-use inpainting models from the [Software Mansion HuggingFace MI-GAN repository](https://huggingface.co/software-mansion/react-native-executorch-migan), available in [`models.inpainting`](../../06-api-reference/variables/models.md#inpainting): + +| Model Family | Variants | Size | Supported Backends | Notes | +| :------------------- | :------------------------------------------------------------------------- | :------- | :------------------------------------------- | :--------------------------------------------------------------------------------- | +| **MI-GAN 512** | [See](../../06-api-reference/variables/models.md#inpaintingmigansize_512) | 12–26 MB | XNNPACK (CPU), Core ML (Apple), Vulkan (GPU) | Places2 (scenes) checkpoint at 512x512. The default; keeps the most detail. | +| **MI-GAN 256** | [See](../../06-api-reference/variables/models.md#inpaintingmigansize_256) | 12–24 MB | XNNPACK (CPU), Core ML (Apple), Vulkan (GPU) | Places2 checkpoint at 256x256, for lower latency. | +| **MI-GAN Faces 256** | [See](../../06-api-reference/variables/models.md#inpaintingmiganfaces_256) | 12–24 MB | XNNPACK (CPU), Core ML (Apple), Vulkan (GPU) | FFHQ (faces) checkpoint. Reconstructs eyes and mouths the scene checkpoints smear. | + +Every variant ships as FP16. MI-GAN is published by [Picsart AI Research](https://github.com/Picsart-AI-Research/MI-GAN) under the MIT license. + +:::tip Using Custom Models +To use your own inpainting `.pte` model, pass an [`InpainterModel`](../../06-api-reference/type-aliases/InpainterModel.md) configuration object to [`useInpainter`](../../06-api-reference/functions/useInpainter.md) or [`createInpainter`](../../06-api-reference/functions/createInpainter.md). The model must take an RGB image in `[0, 1]` and a mask with `1` for known pixels and `0` for the hole, both `[1, 3|1, H, W]` or `[3|1, H, W]` `float32`, and return the filled RGB image in `[0, 1]` at the same shape as the image: + +```typescript +const customInpainter = await createInpainter({ + modelPath: 'https://example.com/my-inpainter.pte', + modelOpts: { + interpolation: 'linear', + outInterpolation: 'linear', + cropPadding: 128, + blend: { shrink: 1, feather: 5 }, + }, +}); +``` + +The pipeline automatically verifies that the model's exported input and output shapes match its requirements. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline). +::: + +## API Reference + +### Hooks & Pipelines + +- [`useInpainter()`](../../06-api-reference/functions/useInpainter.md) — React hook for inpainting model downloading, state, and lifecycle. +- [`createInpainter()`](../../06-api-reference/functions/createInpainter.md) — Imperative factory for inpainting pipelines. +- [`computeInpaintingCrop()`](../../06-api-reference/functions/computeInpaintingCrop.md) — The square around the mask the model is run on. + +### Types & Options + +- [`Inpainter`](../../06-api-reference/type-aliases/Inpainter.md) — Inpainting runner interface (`inpaint`, [`inpaintWorklet`](../../06-api-reference/type-aliases/Inpainter.md#inpaintworklet)). +- [`InpainterModel`](../../06-api-reference/type-aliases/InpainterModel.md) — Model configuration spec for inpainting models. +- [`InpainterOptions`](../../06-api-reference/type-aliases/InpainterOptions.md) — Crop, interpolation and blend options. +- [`ImageBuffer`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/ImageBuffer.md) — Input, mask and output image buffer structure. + +### Model Presets + +- [`models.inpainting`](../../06-api-reference/variables/models.md#inpainting) — Pre-configured inpainting models registry. + +:::info Source Code +View the implementation on GitHub: + +- [`src/extensions/cv/tasks/inpainting.ts` ↗](https://github.com/software-mansion/react-native-executorch/blob/main/packages/react-native-executorch/src/extensions/cv/tasks/inpainting.ts) + ::: diff --git a/docs/docs/02-extensions/computer-vision/11-camera-integration.md b/docs/docs/02-extensions/computer-vision/12-camera-integration.md similarity index 100% rename from docs/docs/02-extensions/computer-vision/11-camera-integration.md rename to docs/docs/02-extensions/computer-vision/12-camera-integration.md From 720c9f9502a2fe1c01e8465ab74c4df335b4e5a2 Mon Sep 17 00:00:00 2001 From: borys3kk Date: Fri, 23 Oct 2026 16:21:16 +0200 Subject: [PATCH 04/17] feat(example): add the image inpainting screen Pick a photo, paint the mask with a brush over the viewport, run the model and compare with the original. The gesture-handler ScrollView lets the brush win over scrolling only above the photo. app.json declares NSLocalNetworkUsageDescription so a dev build on a physical iPhone can reach Metro and local model files. --- apps/computer-vision/app.json | 4 +- apps/computer-vision/app/_layout.tsx | 7 + apps/computer-vision/app/index.tsx | 3 + apps/computer-vision/app/inpainting/index.tsx | 285 ++++++++++++++++++ .../computer-vision/components/MaskCanvas.tsx | 193 ++++++++++++ 5 files changed, 491 insertions(+), 1 deletion(-) create mode 100644 apps/computer-vision/app/inpainting/index.tsx create mode 100644 apps/computer-vision/components/MaskCanvas.tsx diff --git a/apps/computer-vision/app.json b/apps/computer-vision/app.json index 3a47ccac25..6e56cdea40 100644 --- a/apps/computer-vision/app.json +++ b/apps/computer-vision/app.json @@ -17,7 +17,9 @@ "supportsTablet": true, "bundleIdentifier": "com.anonymous.computervision", "infoPlist": { - "NSCameraUsageDescription": "Process photo from camera" + "NSCameraUsageDescription": "Process photo from camera", + "NSLocalNetworkUsageDescription": "Connect to the Metro dev server and local model files on your computer", + "NSBonjourServices": ["_http._tcp"] } }, "android": { diff --git a/apps/computer-vision/app/_layout.tsx b/apps/computer-vision/app/_layout.tsx index 2ee132c0fb..c714dff29d 100644 --- a/apps/computer-vision/app/_layout.tsx +++ b/apps/computer-vision/app/_layout.tsx @@ -48,6 +48,13 @@ export default function Layout() { title: 'Style Transfer', }} /> + router.navigate('styleTransfer/')}> Style Transfer + router.navigate('inpainting/')}> + Image Inpainting + router.navigate('textToImage/')}> Text to Image diff --git a/apps/computer-vision/app/inpainting/index.tsx b/apps/computer-vision/app/inpainting/index.tsx new file mode 100644 index 0000000000..afbe5cbe28 --- /dev/null +++ b/apps/computer-vision/app/inpainting/index.tsx @@ -0,0 +1,285 @@ +import React, { useState } from 'react'; +import { View, Text, Platform, TouchableOpacity, StyleSheet } from 'react-native'; +// The gesture-handler ScrollView yields to the brush pan over the photo, so +// painting never scrolls the page and scrolling still works everywhere else. +import { ScrollView } from 'react-native-gesture-handler'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; +import { commonStyles, theme } from '../../theme'; +import { + useImage, + Skia, + ColorType, + AlphaType, + type SkImage as SkiaImageType, +} from '@shopify/react-native-skia'; +import { useInpainter, models } from 'react-native-executorch'; +import ScreenWrapper from '../../components/ScreenWrapper'; +import { getImage, skImageToBuffer } from '../../utils'; +import { ModelPicker, type ModelOption } from '../../components/ModelPicker'; +import { ImageViewport } from '../../components/ImageViewport'; +import { ModelStatus } from '../../components/ModelStatus'; +import { LatencyIndicator } from '../../components/LatencyIndicator'; +import { Button } from '../../components/Button'; +import { MaskCanvas, rasterizeMask, type MaskStroke } from '../../components/MaskCanvas'; + +const MODEL_OPTIONS: ModelOption[] = [ + { label: 'MI-GAN 512 (XNNPACK FP16)', value: models.inpainting.MIGAN.SIZE_512.XNNPACK_FP16 }, + { + label: 'MI-GAN 512 (CoreML FP16)', + value: models.inpainting.MIGAN.SIZE_512.COREML_FP16, + disabled: Platform.OS !== 'ios', + }, + { + label: 'MI-GAN 512 (Vulkan FP16)', + value: models.inpainting.MIGAN.SIZE_512.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { label: 'MI-GAN 256 (XNNPACK FP16)', value: models.inpainting.MIGAN.SIZE_256.XNNPACK_FP16 }, + { + label: 'MI-GAN 256 (CoreML FP16)', + value: models.inpainting.MIGAN.SIZE_256.COREML_FP16, + disabled: Platform.OS !== 'ios', + }, + { + label: 'MI-GAN 256 (Vulkan FP16)', + value: models.inpainting.MIGAN.SIZE_256.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + label: 'MI-GAN Faces 256 (XNNPACK FP16)', + value: models.inpainting.MIGAN.FACES_256.XNNPACK_FP16, + }, + { + label: 'MI-GAN Faces 256 (CoreML FP16)', + value: models.inpainting.MIGAN.FACES_256.COREML_FP16, + disabled: Platform.OS !== 'ios', + }, + { + label: 'MI-GAN Faces 256 (Vulkan FP16)', + value: models.inpainting.MIGAN.FACES_256.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, +]; + +/** Brush diameters in image pixels. */ +const BRUSH_SIZES = [16, 32, 64] as const; + +function InpaintingContent() { + const insets = useSafeAreaInsets(); + const [selectedModel, setSelectedModel] = useState(MODEL_OPTIONS[0].value); + const [imageUri, setImageUri] = useState(null); + const [strokes, setStrokes] = useState([]); + const [brushWidth, setBrushWidth] = useState(BRUSH_SIZES[1]); + const [isProcessing, setIsProcessing] = useState(false); + const [resultImage, setResultImage] = useState(null); + const [showOriginal, setShowOriginal] = useState(false); + const [latency, setLatency] = useState(null); + const [error, setError] = useState(null); + + const skiaImage = useImage(imageUri, (err) => setError(err.message || String(err))); + + const { + isReady, + downloadProgress, + error: loadError, + inpaint, + inpaintWorklet, + } = useInpainter(selectedModel); + + const resetResult = () => { + setResultImage(null); + setShowOriginal(false); + setLatency(null); + }; + + const handlePickImage = async (useCamera: boolean) => { + setError(null); + try { + const uri = await getImage(useCamera); + if (uri) { + setImageUri(uri); + setStrokes([]); + resetResult(); + } + } catch (e: any) { + setError(e.message || String(e)); + } + }; + + const runInpainting = async (sync: boolean) => { + if (!skiaImage || !inpaint || !inpaintWorklet) return; + if (!sync) setIsProcessing(true); + setError(null); + try { + const image = skImageToBuffer(skiaImage); + const mask = rasterizeMask(strokes, image.width, image.height); + if (!mask) throw new Error('Could not rasterize the mask'); + + const start = Date.now(); + const output = sync ? inpaintWorklet(image, mask) : await inpaint(image, mask); + setLatency(Date.now() - start); + + const info = { + width: output.width, + height: output.height, + colorType: ColorType.RGBA_8888, + alphaType: AlphaType.Premul, + }; + const result = Skia.Image.MakeImage(info, Skia.Data.fromBytes(output.data), output.width * 4); + if (!result) throw new Error('Failed to decode the inpainted image in Skia'); + setResultImage(result); + setShowOriginal(false); + } catch (e: any) { + setError(e.message || String(e)); + } finally { + if (!sync) setIsProcessing(false); + } + }; + + const activeError = loadError ? String(loadError) : error; + const showingResult = resultImage !== null && !showOriginal; + const activeImage = showingResult ? resultImage : skiaImage; + const canRun = !!skiaImage && isReady && !isProcessing && strokes.length > 0; + + return ( + + + Pick a photo, paint over what should disappear, and run the model to fill it in. + + + { + setSelectedModel(model); + resetResult(); + setError(null); + }} + /> + + + + handlePickImage(false)}> + {skiaImage && !showingResult && ( + + )} + + + {skiaImage && !showingResult && ( + + Brush + {BRUSH_SIZES.map((size) => ( + setBrushWidth(size)} + > + + {size}px + + + ))} + { + setStrokes([]); + resetResult(); + }} + disabled={strokes.length === 0} + > + Clear + + + )} + + +