diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt
index 411a9c8d64..f841ffbafd 100644
--- a/.cspell-wordlist.txt
+++ b/.cspell-wordlist.txt
@@ -350,6 +350,7 @@ dontMock
BIES
binarize
binarized
+binarizing
unclip
phonemizations
phonemizes
@@ -428,7 +429,17 @@ zipdepth
xiaobei
xiaoxiao
xiaoyi
+xiaoni
yunjian
yunxi
yunxia
yunyang
+migan
+inpaint
+inpainted
+inpainter
+inpainting
+Picsart
+FFHQ
+grayscale
+Premul
diff --git a/apps/computer-vision/app.json b/apps/computer-vision/app.json
index 3a47ccac25..6e56cdea40 100644
--- a/apps/computer-vision/app.json
+++ b/apps/computer-vision/app.json
@@ -17,7 +17,9 @@
"supportsTablet": true,
"bundleIdentifier": "com.anonymous.computervision",
"infoPlist": {
- "NSCameraUsageDescription": "Process photo from camera"
+ "NSCameraUsageDescription": "Process photo from camera",
+ "NSLocalNetworkUsageDescription": "Connect to the Metro dev server and local model files on your computer",
+ "NSBonjourServices": ["_http._tcp"]
}
},
"android": {
diff --git a/apps/computer-vision/app/_layout.tsx b/apps/computer-vision/app/_layout.tsx
index db3837fe98..f40943ea8b 100644
--- a/apps/computer-vision/app/_layout.tsx
+++ b/apps/computer-vision/app/_layout.tsx
@@ -48,6 +48,13 @@ export default function Layout() {
title: 'Style Transfer',
}}
/>
+
router.navigate('styleTransfer/')}>
Style Transfer
+ router.navigate('inpainting/')}>
+ Image Inpainting
+
router.navigate('textToImage/')}>
Text to Image
diff --git a/apps/computer-vision/app/inpainting/index.tsx b/apps/computer-vision/app/inpainting/index.tsx
new file mode 100644
index 0000000000..afbe5cbe28
--- /dev/null
+++ b/apps/computer-vision/app/inpainting/index.tsx
@@ -0,0 +1,285 @@
+import React, { useState } from 'react';
+import { View, Text, Platform, TouchableOpacity, StyleSheet } from 'react-native';
+// The gesture-handler ScrollView yields to the brush pan over the photo, so
+// painting never scrolls the page and scrolling still works everywhere else.
+import { ScrollView } from 'react-native-gesture-handler';
+import { useSafeAreaInsets } from 'react-native-safe-area-context';
+import { commonStyles, theme } from '../../theme';
+import {
+ useImage,
+ Skia,
+ ColorType,
+ AlphaType,
+ type SkImage as SkiaImageType,
+} from '@shopify/react-native-skia';
+import { useInpainter, models } from 'react-native-executorch';
+import ScreenWrapper from '../../components/ScreenWrapper';
+import { getImage, skImageToBuffer } from '../../utils';
+import { ModelPicker, type ModelOption } from '../../components/ModelPicker';
+import { ImageViewport } from '../../components/ImageViewport';
+import { ModelStatus } from '../../components/ModelStatus';
+import { LatencyIndicator } from '../../components/LatencyIndicator';
+import { Button } from '../../components/Button';
+import { MaskCanvas, rasterizeMask, type MaskStroke } from '../../components/MaskCanvas';
+
+const MODEL_OPTIONS: ModelOption[] = [
+ { label: 'MI-GAN 512 (XNNPACK FP16)', value: models.inpainting.MIGAN.SIZE_512.XNNPACK_FP16 },
+ {
+ label: 'MI-GAN 512 (CoreML FP16)',
+ value: models.inpainting.MIGAN.SIZE_512.COREML_FP16,
+ disabled: Platform.OS !== 'ios',
+ },
+ {
+ label: 'MI-GAN 512 (Vulkan FP16)',
+ value: models.inpainting.MIGAN.SIZE_512.VULKAN_FP16,
+ disabled: Platform.OS !== 'android',
+ },
+ { label: 'MI-GAN 256 (XNNPACK FP16)', value: models.inpainting.MIGAN.SIZE_256.XNNPACK_FP16 },
+ {
+ label: 'MI-GAN 256 (CoreML FP16)',
+ value: models.inpainting.MIGAN.SIZE_256.COREML_FP16,
+ disabled: Platform.OS !== 'ios',
+ },
+ {
+ label: 'MI-GAN 256 (Vulkan FP16)',
+ value: models.inpainting.MIGAN.SIZE_256.VULKAN_FP16,
+ disabled: Platform.OS !== 'android',
+ },
+ {
+ label: 'MI-GAN Faces 256 (XNNPACK FP16)',
+ value: models.inpainting.MIGAN.FACES_256.XNNPACK_FP16,
+ },
+ {
+ label: 'MI-GAN Faces 256 (CoreML FP16)',
+ value: models.inpainting.MIGAN.FACES_256.COREML_FP16,
+ disabled: Platform.OS !== 'ios',
+ },
+ {
+ label: 'MI-GAN Faces 256 (Vulkan FP16)',
+ value: models.inpainting.MIGAN.FACES_256.VULKAN_FP16,
+ disabled: Platform.OS !== 'android',
+ },
+];
+
+/** Brush diameters in image pixels. */
+const BRUSH_SIZES = [16, 32, 64] as const;
+
+function InpaintingContent() {
+ const insets = useSafeAreaInsets();
+ const [selectedModel, setSelectedModel] = useState(MODEL_OPTIONS[0].value);
+ const [imageUri, setImageUri] = useState(null);
+ const [strokes, setStrokes] = useState([]);
+ const [brushWidth, setBrushWidth] = useState(BRUSH_SIZES[1]);
+ const [isProcessing, setIsProcessing] = useState(false);
+ const [resultImage, setResultImage] = useState(null);
+ const [showOriginal, setShowOriginal] = useState(false);
+ const [latency, setLatency] = useState(null);
+ const [error, setError] = useState(null);
+
+ const skiaImage = useImage(imageUri, (err) => setError(err.message || String(err)));
+
+ const {
+ isReady,
+ downloadProgress,
+ error: loadError,
+ inpaint,
+ inpaintWorklet,
+ } = useInpainter(selectedModel);
+
+ const resetResult = () => {
+ setResultImage(null);
+ setShowOriginal(false);
+ setLatency(null);
+ };
+
+ const handlePickImage = async (useCamera: boolean) => {
+ setError(null);
+ try {
+ const uri = await getImage(useCamera);
+ if (uri) {
+ setImageUri(uri);
+ setStrokes([]);
+ resetResult();
+ }
+ } catch (e: any) {
+ setError(e.message || String(e));
+ }
+ };
+
+ const runInpainting = async (sync: boolean) => {
+ if (!skiaImage || !inpaint || !inpaintWorklet) return;
+ if (!sync) setIsProcessing(true);
+ setError(null);
+ try {
+ const image = skImageToBuffer(skiaImage);
+ const mask = rasterizeMask(strokes, image.width, image.height);
+ if (!mask) throw new Error('Could not rasterize the mask');
+
+ const start = Date.now();
+ const output = sync ? inpaintWorklet(image, mask) : await inpaint(image, mask);
+ setLatency(Date.now() - start);
+
+ const info = {
+ width: output.width,
+ height: output.height,
+ colorType: ColorType.RGBA_8888,
+ alphaType: AlphaType.Premul,
+ };
+ const result = Skia.Image.MakeImage(info, Skia.Data.fromBytes(output.data), output.width * 4);
+ if (!result) throw new Error('Failed to decode the inpainted image in Skia');
+ setResultImage(result);
+ setShowOriginal(false);
+ } catch (e: any) {
+ setError(e.message || String(e));
+ } finally {
+ if (!sync) setIsProcessing(false);
+ }
+ };
+
+ const activeError = loadError ? String(loadError) : error;
+ const showingResult = resultImage !== null && !showOriginal;
+ const activeImage = showingResult ? resultImage : skiaImage;
+ const canRun = !!skiaImage && isReady && !isProcessing && strokes.length > 0;
+
+ return (
+
+
+ Pick a photo, paint over what should disappear, and run the model to fill it in.
+
+
+ {
+ setSelectedModel(model);
+ resetResult();
+ setError(null);
+ }}
+ />
+
+
+
+ handlePickImage(false)}>
+ {skiaImage && !showingResult && (
+
+ )}
+
+
+ {skiaImage && !showingResult && (
+
+ Brush
+ {BRUSH_SIZES.map((size) => (
+ setBrushWidth(size)}
+ >
+
+ {size}px
+
+
+ ))}
+ {
+ setStrokes([]);
+ resetResult();
+ }}
+ disabled={strokes.length === 0}
+ >
+ Clear
+
+
+ )}
+
+
+
+
+
+
+
+ {resultImage && (
+
+
+ )}
+
+
+
+ );
+}
+
+export default function InpaintingScreen() {
+ return (
+
+
+
+ );
+}
+
+const styles = StyleSheet.create({
+ brushRow: {
+ flexDirection: 'row',
+ alignItems: 'center',
+ gap: 8,
+ marginBottom: 20,
+ },
+ brushLabel: {
+ color: theme.colors.textMuted,
+ marginRight: 4,
+ },
+ brushChip: {
+ borderWidth: 1,
+ borderColor: theme.colors.border,
+ borderRadius: 16,
+ paddingVertical: 6,
+ paddingHorizontal: 12,
+ },
+ brushChipActive: {
+ backgroundColor: theme.colors.primary,
+ borderColor: theme.colors.primary,
+ },
+ brushText: {
+ color: theme.colors.textMuted,
+ fontSize: 13,
+ },
+ brushTextActive: {
+ color: '#fff',
+ },
+});
diff --git a/apps/computer-vision/components/MaskCanvas.tsx b/apps/computer-vision/components/MaskCanvas.tsx
new file mode 100644
index 0000000000..f7c546623a
--- /dev/null
+++ b/apps/computer-vision/components/MaskCanvas.tsx
@@ -0,0 +1,210 @@
+import React, { useCallback, useMemo, useRef, useState } from 'react';
+import { StyleSheet, View, type LayoutChangeEvent } from 'react-native';
+import { Gesture, GestureDetector } from 'react-native-gesture-handler';
+import {
+ AlphaType,
+ Canvas,
+ ColorType,
+ Path,
+ Skia,
+ PaintStyle,
+ StrokeCap,
+ StrokeJoin,
+ type SkImage,
+ type SkPath,
+} from '@shopify/react-native-skia';
+
+import { skImageToBuffer } from '../utils';
+
+/** A 2D point in the image's pixel coordinates. */
+export type MaskPoint = { readonly x: number; readonly y: number };
+
+/** One brush stroke over the image, in the image's pixel coordinates. */
+export type MaskStroke = {
+ readonly points: readonly MaskPoint[];
+ /** Brush diameter in image pixels. */
+ readonly width: number;
+};
+
+export interface MaskCanvasProps {
+ /** The image the strokes are painted over; sets the coordinate space. */
+ image: SkImage;
+ strokes: readonly MaskStroke[];
+ onStrokesChange: (strokes: readonly MaskStroke[]) => void;
+ /** Brush diameter in image pixels for new strokes. */
+ brushWidth: number;
+ /** When false the strokes are shown but cannot be edited. */
+ enabled?: boolean;
+ color?: string;
+}
+
+/**
+ * The contain-fit transform `ImageViewport` draws the image with, so strokes
+ * land where the finger is.
+ * @param view - The canvas size in view points.
+ * @param image - The image size in pixels.
+ * @returns The scale and offset mapping image pixels to view points.
+ */
+const containFit = (
+ view: { width: number; height: number },
+ image: { width: number; height: number }
+) => {
+ const scale = Math.min(view.width / image.width, view.height / image.height);
+ const dx = (view.width - image.width * scale) / 2;
+ const dy = (view.height - image.height * scale) / 2;
+ return { scale, dx, dy };
+};
+
+const strokePath = (stroke: MaskStroke, transform: (p: MaskPoint) => MaskPoint): SkPath => {
+ const builder = Skia.PathBuilder.Make();
+ const first = transform(stroke.points[0]!);
+ builder.moveTo(first.x, first.y);
+ // A single tap still leaves a round dot thanks to the round cap.
+ if (stroke.points.length === 1) builder.lineTo(first.x, first.y);
+ for (let i = 1; i < stroke.points.length; i++) {
+ const p = transform(stroke.points[i]!);
+ builder.lineTo(p.x, p.y);
+ }
+ return builder.build();
+};
+
+// A paint-over layer for `ImageViewport`: fills its parent, converts pan
+// gestures into brush strokes in image pixels, and shows them as a translucent
+// overlay. Render the strokes into a mask with `rasterizeMask`.
+export function MaskCanvas({
+ image,
+ strokes,
+ onStrokesChange,
+ brushWidth,
+ enabled = true,
+ color = 'rgba(255, 64, 64, 0.55)',
+}: MaskCanvasProps) {
+ const [view, setView] = useState({ width: 0, height: 0 });
+ // The stroke being drawn lives in a ref so each move appends in place and
+ // only the committed list goes through props.
+ const current = useRef(null);
+
+ const onLayout = useCallback((e: LayoutChangeEvent) => {
+ const { width, height } = e.nativeEvent.layout;
+ setView({ width, height });
+ }, []);
+
+ const fit = useMemo(
+ () => containFit(view, { width: image.width(), height: image.height() }),
+ [view, image]
+ );
+
+ const toImage = useCallback(
+ (x: number, y: number): MaskPoint => ({
+ x: (x - fit.dx) / fit.scale,
+ y: (y - fit.dy) / fit.scale,
+ }),
+ [fit]
+ );
+ const toView = useCallback(
+ (p: MaskPoint): MaskPoint => ({ x: fit.dx + p.x * fit.scale, y: fit.dy + p.y * fit.scale }),
+ [fit]
+ );
+
+ const pan = useMemo(
+ () =>
+ Gesture.Pan()
+ .enabled(enabled)
+ .minDistance(0)
+ .runOnJS(true)
+ .onBegin((e) => {
+ current.current = [toImage(e.x, e.y)];
+ onStrokesChange([...strokes, { points: current.current, width: brushWidth }]);
+ })
+ .onUpdate((e) => {
+ if (!current.current) return;
+ current.current = [...current.current, toImage(e.x, e.y)];
+ onStrokesChange([...strokes, { points: current.current, width: brushWidth }]);
+ })
+ .onFinalize(() => {
+ current.current = null;
+ }),
+ [enabled, strokes, brushWidth, toImage, onStrokesChange]
+ );
+
+ const paths = useMemo(
+ () =>
+ view.width === 0
+ ? []
+ : strokes.map((stroke) => ({
+ path: strokePath(stroke, toView),
+ width: stroke.width * fit.scale,
+ })),
+ [strokes, toView, fit.scale, view.width]
+ );
+
+ return (
+
+
+
+
+
+ );
+}
+
+/**
+ * Renders brush strokes into a grayscale mask buffer at the image's
+ * resolution: 255 where painted, 0 elsewhere, which is what `inpaint` reads as
+ * the hole.
+ * @param strokes - The strokes to render, in image pixels.
+ * @param width - The image width in pixels.
+ * @param height - The image height in pixels.
+ * @returns The mask buffer, or `null` if Skia cannot allocate an offscreen surface.
+ */
+export function rasterizeMask(strokes: readonly MaskStroke[], width: number, height: number) {
+ const surface = Skia.Surface.MakeOffscreen(width, height);
+ if (!surface) return null;
+
+ const canvas = surface.getCanvas();
+ canvas.clear(Skia.Color('black'));
+
+ const paint = Skia.Paint();
+ paint.setColor(Skia.Color('white'));
+ paint.setStyle(PaintStyle.Stroke);
+ paint.setStrokeCap(StrokeCap.Round);
+ paint.setStrokeJoin(StrokeJoin.Round);
+ paint.setAntiAlias(false);
+
+ for (const stroke of strokes) {
+ paint.setStrokeWidth(stroke.width);
+ canvas.drawPath(
+ strokePath(stroke, (p) => p),
+ paint
+ );
+ }
+
+ surface.flush();
+ const snapshot = surface.makeImageSnapshot();
+ const gray = snapshot.readPixels(0, 0, {
+ width,
+ height,
+ colorType: ColorType.Gray_8,
+ alphaType: AlphaType.Opaque,
+ });
+ if (gray instanceof Uint8Array && gray.length === width * height) {
+ return { data: gray, width, height, format: 'gray' as const, layout: 'hwc' as const };
+ }
+ // Skia could not read back as grayscale; the strokes are white on black, so
+ // the red channel of the RGBA read-back is the same mask.
+ const rgba = skImageToBuffer(snapshot);
+ const data = new Uint8Array(width * height);
+ for (let i = 0; i < data.length; i++) data[i] = rgba.data[i * 4]!;
+ return { data, width, height, format: 'gray' as const, layout: 'hwc' as const };
+}
diff --git a/docs/docs/02-extensions/computer-vision/11-image-inpainting.md b/docs/docs/02-extensions/computer-vision/11-image-inpainting.md
new file mode 100644
index 0000000000..b9480f59d0
--- /dev/null
+++ b/docs/docs/02-extensions/computer-vision/11-image-inpainting.md
@@ -0,0 +1,188 @@
+---
+title: Image Inpainting
+slug: /extensions/image-inpainting
+description: 'Remove objects and fill masked regions of photos on-device with MI-GAN, with the resizing, mask handling and seam blending built in.'
+keywords:
+ [
+ react native,
+ inpainting,
+ object removal,
+ photo cleanup,
+ image editing,
+ mi-gan,
+ migan,
+ mask,
+ mobile ml,
+ on-device ai,
+ ]
+---
+
+# Image Inpainting
+
+Image inpainting fills a region of a photo with content that matches its surroundings: paint over an unwanted object, a blemish, or a passer-by, and the model reconstructs what would plausibly be behind it.
+
+The pipeline runs entirely on-device. It runs the model on the whole photo resized to the model's resolution and blends the result back into the full-resolution photo through the mask, so the rest of the image is returned pixel for pixel as it came in.
+
+## Quick Start
+
+The [`useInpainter`](../../06-api-reference/functions/useInpainter.md) hook manages model downloading, initialization, and lifecycle:
+
+```tsx
+import { models, useInpainter } from 'react-native-executorch';
+import type { ImageBuffer } from 'react-native-executorch/cv';
+
+function MyComponent() {
+ const inpainter = useInpainter(models.inpainting.MIGAN.DEFAULT);
+
+ // Hook state:
+ // inpainter.isReady — true once model is downloaded and loaded in memory
+ // inpainter.downloadProgress — 0 to 100 download progress
+ // inpainter.error — Error instance if download or load failed
+ // inpainter.resource — resolved config with all URLs replaced by local file paths
+
+ const handleInpaint = async (image: ImageBuffer, mask: ImageBuffer) => {
+ if (!inpainter.isReady || !inpainter.inpaint) return;
+
+ // Run inference on background thread
+ const result = await inpainter.inpaint(image, mask);
+ console.log('Inpainted image dimensions:', result.width, result.height);
+ };
+
+ // Build `mask` from what the user painted, then call handleInpaint
+}
+```
+
+:::tip Full Interactive Example
+See [`apps/computer-vision/app/inpainting`](https://github.com/software-mansion/react-native-executorch/tree/main/apps/computer-vision/app/inpainting) in this repository for a complete screen with a photo picker, a brush to paint the mask over the photo, and before/after comparison.
+:::
+
+## The Mask
+
+[`inpaint()`](../../06-api-reference/type-aliases/Inpainter.md#inpaint) takes two [`ImageBuffer`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/ImageBuffer.md)s of the same dimensions: the photo and the mask. The mask is a `'gray'` buffer, one byte per pixel, in which **any non-zero pixel marks the region to fill**, so the natural way to build one is to paint white strokes on a black canvas and read the pixels back as grayscale (Skia's `readPixels` with `ColorType.Gray_8`, for example). Other formats are rejected.
+
+```typescript
+// A mask with a 100x100 hole at (200, 150) over an 800x600 photo
+const mask = new Uint8Array(800 * 600);
+for (let y = 150; y < 250; y++) {
+ for (let x = 200; x < 300; x++) mask[y * 800 + x] = 255;
+}
+const maskBuffer = { data: mask, width: 800, height: 600, format: 'gray', layout: 'hwc' } as const;
+```
+
+An all-zero mask returns the input unchanged, but the model still runs, so skip the call when there is nothing to fill.
+
+:::tip Cover the whole object
+Paint over the entire object, with a little margin. MI-GAN continues whatever touches the edge of the hole, so a mask that cuts through an object makes the model extend that object into the hole instead of removing it.
+:::
+
+## Output Format
+
+The result is an [`ImageBuffer`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/ImageBuffer.md) containing the whole frame as RGBA at the input dimensions, with the masked region filled:
+
+```typescript
+type ImageBuffer = {
+ readonly width: number;
+ readonly height: number;
+ readonly format: 'rgba';
+ readonly data: Uint8Array;
+};
+```
+
+Pixels outside the mask are copied from the input; only the masked region changes.
+
+## How It Works
+
+The models run at a fixed resolution (256 or 512 px):
+
+1. Resize the whole photo and the mask to the model input and run the model. A model pixel is part of the hole when any photo pixel it covers is, so thin strokes survive the downscale.
+2. Resize the output back to the photo's resolution.
+3. Blend it over the original photo through the full-resolution mask. [`blendOpts`](../../06-api-reference/type-aliases/InpainterOptions.md#blendopts) can erode the mask and feather its edge inward; the built-in models use neither, since the upsampled fill is already soft.
+
+The fill is therefore generated at the model's resolution and upsampled. To fill a small hole in a large photo at higher detail, crop the region around it yourself, inpaint the crop, and paste the result back.
+
+## Imperative API
+
+For background photo processing, headless workflows, or manual lifecycle management outside React components, create the pipeline using [`createInpainter`](../../06-api-reference/functions/createInpainter.md):
+
+```typescript
+import { createInpainter, download, models } from 'react-native-executorch';
+
+// Download and cache model assets before creating the pipeline
+const model = await download(models.inpainting.MIGAN.DEFAULT);
+const inpainter = await createInpainter(model);
+
+try {
+ const result = await inpainter.inpaint(imageBuffer, maskBuffer);
+ console.log('Inpainted output byte length:', result.data.byteLength);
+} finally {
+ // Always release native resources when finished
+ inpainter.dispose();
+}
+```
+
+## Synchronous Execution
+
+[`createInpainter`](../../06-api-reference/functions/createInpainter.md) also exposes a synchronous [`inpaintWorklet`](../../06-api-reference/type-aliases/Inpainter.md#inpaintworklet) function that runs directly on the worklet thread with zero Promise scheduling overhead:
+
+```typescript
+// Called synchronously on the UI worklet thread
+const result = inpainter.inpaintWorklet(imageBuffer, maskBuffer);
+```
+
+See [Worklets & Threading](../../03-core-and-advanced/06-worklets-and-threading.md) for details on worklet execution contexts and zero-copy host objects.
+
+## Available Models
+
+The library provides ready-to-use inpainting models from the [Software Mansion HuggingFace MI-GAN repository](https://huggingface.co/software-mansion/react-native-executorch-migan), available in [`models.inpainting`](../../06-api-reference/variables/models.md#inpainting):
+
+| Model Family | Variants | Size | Supported Backends | Notes |
+| :------------------- | :---------------------------------------------------------------- | :------- | :------------------------------------------- | :--------------------------------------------------------------------------------- |
+| **MI-GAN 512** | [See](../../06-api-reference/variables/models.md#inpaintingmigan) | 14–25 MB | XNNPACK (CPU), Core ML (Apple), Vulkan (GPU) | Places2 (scenes) checkpoint at 512x512. The default; keeps the most detail. |
+| **MI-GAN 256** | [See](../../06-api-reference/variables/models.md#inpaintingmigan) | 12–24 MB | XNNPACK (CPU), Core ML (Apple), Vulkan (GPU) | Places2 checkpoint at 256x256, for lower latency. |
+| **MI-GAN Faces 256** | [See](../../06-api-reference/variables/models.md#inpaintingmigan) | 12–24 MB | XNNPACK (CPU), Core ML (Apple), Vulkan (GPU) | FFHQ (faces) checkpoint. Reconstructs eyes and mouths the scene checkpoints smear. |
+
+Every variant ships as FP16. MI-GAN is published by [Picsart AI Research](https://github.com/Picsart-AI-Research/MI-GAN) under the MIT license.
+
+:::tip Using Custom Models
+To use your own inpainting `.pte` model, pass an [`InpainterModel`](../../06-api-reference/type-aliases/InpainterModel.md) configuration object to [`useInpainter`](../../06-api-reference/functions/useInpainter.md) or [`createInpainter`](../../06-api-reference/functions/createInpainter.md). The model must take an RGB image in `[0, 1]` and a mask with `1` for known pixels and `0` for the hole, both `[1, 3|1, H, W]` or `[3|1, H, W]` `float32`, and return the filled RGB image in `[0, 1]` at the same shape as the image:
+
+```typescript
+const customInpainter = await createInpainter({
+ modelPath: 'https://example.com/my-inpainter.pte',
+ modelOpts: {
+ resizeMode: 'stretch',
+ interpolation: 'linear',
+ normalizeOpts: { alpha: 1 / 255, beta: 0 },
+ outNormalizeOpts: { alpha: 255, beta: 0 },
+ outInterpolation: 'linear',
+ blendOpts: { shrink: 0, feather: 0 },
+ },
+});
+```
+
+The pipeline automatically verifies that the model's exported input and output shapes match its requirements. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline).
+:::
+
+## API Reference
+
+### Hooks & Pipelines
+
+- [`useInpainter()`](../../06-api-reference/functions/useInpainter.md) — React hook for inpainting model downloading, state, and lifecycle.
+- [`createInpainter()`](../../06-api-reference/functions/createInpainter.md) — Imperative factory for inpainting pipelines.
+
+### Types & Options
+
+- [`Inpainter`](../../06-api-reference/type-aliases/Inpainter.md) — Inpainting runner interface (`inpaint`, [`inpaintWorklet`](../../06-api-reference/type-aliases/Inpainter.md#inpaintworklet)).
+- [`InpainterModel`](../../06-api-reference/type-aliases/InpainterModel.md) — Model configuration spec for inpainting models.
+- [`InpainterOptions`](../../06-api-reference/type-aliases/InpainterOptions.md) — Input preprocessing, output postprocessing and blend options.
+- [`ImageBuffer`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/ImageBuffer.md) — Input, mask and output image buffer structure.
+
+### Model Presets
+
+- [`models.inpainting`](../../06-api-reference/variables/models.md#inpainting) — Pre-configured inpainting models registry.
+
+:::info Source Code
+View the implementation on GitHub:
+
+- [`src/extensions/cv/tasks/inpainting.ts` ↗](https://github.com/software-mansion/react-native-executorch/blob/main/packages/react-native-executorch/src/extensions/cv/tasks/inpainting.ts)
+ :::
diff --git a/docs/docs/02-extensions/computer-vision/11-camera-integration.md b/docs/docs/02-extensions/computer-vision/12-camera-integration.md
similarity index 100%
rename from docs/docs/02-extensions/computer-vision/11-camera-integration.md
rename to docs/docs/02-extensions/computer-vision/12-camera-integration.md
diff --git a/docs/docs/03-core-and-advanced/08-native-libraries.md b/docs/docs/03-core-and-advanced/08-native-libraries.md
index 258e0e38a3..74fc42bb54 100644
--- a/docs/docs/03-core-and-advanced/08-native-libraries.md
+++ b/docs/docs/03-core-and-advanced/08-native-libraries.md
@@ -39,11 +39,11 @@ Add a `react-native-executorch` block to your `package.json`:
### Options
-| Option | Purpose | Accepted values |
-| ---------- | ------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `features` | High-level tasks — each automatically expands to the backends and libraries it needs | `"classification"`, `"imageEmbeddings"`, `"instanceSegmentation"`, `"keypointDetection"`, `"llm"`, `"multimodalLLM"`, `"objectDetection"`, `"ocr"`, `"privacyFilter"`, `"segmentAnything"`, `"semanticSegmentation"`, `"speechToText"`, `"styleTransfer"`, `"textEmbeddings"`, `"textToImage"`, `"textToSpeech"`, `"tokenizer"`, `"vad"`, `"verticalOCR"` |
-| `backends` | Hardware acceleration backends directly | `"xnnpack"`, `"coreml"`, `"mlx"`, `"vulkan"` |
-| `libs` | Extra native C++ libraries | `"opencv"`, `"phonemis"` |
+| Option | Purpose | Accepted values |
+| ---------- | ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `features` | High-level tasks — each automatically expands to the backends and libraries it needs | `"classification"`, `"imageEmbeddings"`, `"inpainting"`, `"instanceSegmentation"`, `"keypointDetection"`, `"llm"`, `"multimodalLLM"`, `"objectDetection"`, `"ocr"`, `"privacyFilter"`, `"segmentAnything"`, `"semanticSegmentation"`, `"speechToText"`, `"styleTransfer"`, `"textEmbeddings"`, `"textToImage"`, `"textToSpeech"`, `"tokenizer"`, `"vad"`, `"verticalOCR"` |
+| `backends` | Hardware acceleration backends directly | `"xnnpack"`, `"coreml"`, `"mlx"`, `"vulkan"` |
+| `libs` | Extra native C++ libraries | `"opencv"`, `"phonemis"` |
The three lists are merged, so you can pair high-level `features` with specific `backends` or `libs`. Re-run your package manager install after editing.
@@ -102,6 +102,7 @@ Specifying a task under `features` is shorthand: it automatically expands to the
| `verticalOCR` | xnnpack | opencv |
| `styleTransfer` | xnnpack, coreml, vulkan | opencv |
| `textToImage` | xnnpack, coreml, vulkan | opencv |
+| `inpainting` | xnnpack, coreml, vulkan | opencv |
| `segmentAnything` | xnnpack, coreml, vulkan | opencv |
| `tokenizer` | — | — |
diff --git a/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap b/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap
index 7b39ced3d9..90884cd0ba 100644
--- a/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap
+++ b/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap
@@ -6,6 +6,7 @@ exports[`public API surface exports one hook per user-facing task 1`] = `
"useColbertEmbedder",
"useDepthEstimator",
"useImageEmbedder",
+ "useInpainter",
"useInstanceSegmenter",
"useKeypointDetector",
"useLLMChatSession",
@@ -31,6 +32,7 @@ exports[`public API surface matches the recorded contents of each namespace 1`]
"FORMAT_CHANNELS",
"FORMAT_CONVERSION",
"applyColormap",
+ "blendMasked",
"boundingBoxOfPoints",
"createImagePreprocessor",
"cvtColor",
@@ -138,6 +140,7 @@ exports[`public API surface matches the recorded export list 1`] = `
"createDepthEstimator",
"createFsmnVoiceActivityDetector",
"createImageEmbedder",
+ "createInpainter",
"createInstanceSegmenter",
"createKeypointDetector",
"createKokoroTextToSpeech",
@@ -174,6 +177,7 @@ exports[`public API surface matches the recorded export list 1`] = `
"useColbertEmbedder",
"useDepthEstimator",
"useImageEmbedder",
+ "useInpainter",
"useInstanceSegmenter",
"useKeypointDetector",
"useLLMChatSession",
diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts
index 42976f98f6..6c0ff0a581 100644
--- a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts
+++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts
@@ -429,6 +429,7 @@ describe('feature map', () => {
privacyFilter: 'privacyFilter',
imageEmbeddings: 'imageEmbeddings',
textToImage: 'textToImage',
+ inpainting: 'inpainting',
textToSpeech: 'textToSpeech',
ocr: 'ocr',
};
diff --git a/packages/react-native-executorch/__tests__/support/fakeOps.ts b/packages/react-native-executorch/__tests__/support/fakeOps.ts
index 2ef9d5d8fa..4f69fd5ab7 100644
--- a/packages/react-native-executorch/__tests__/support/fakeOps.ts
+++ b/packages/react-native-executorch/__tests__/support/fakeOps.ts
@@ -556,6 +556,81 @@ export const cv = {
}
return dst;
},
+
+ /**
+ * Composites `overlay` over `src` where `mask` is set. The native op erodes
+ * the mask with a square kernel and feathers it with a Gaussian; this erodes
+ * the same way and feathers with a box mean of the same width, which keeps
+ * both options load-bearing (a seam pixel moves when they change) without
+ * reproducing OpenCV's kernel weights.
+ */
+ blendMasked(
+ src: FakeTensor,
+ overlay: FakeTensor,
+ mask: FakeTensor,
+ dst: FakeTensor,
+ opts: { shrink: number; feather: number }
+ ): FakeTensor {
+ const [height, width, channels] = src.shape as [number, number, number];
+ expectShape(overlay, src.shape, 'blendMasked: overlay');
+ expectShape(mask, [height, width, 1], 'blendMasked: mask');
+ expectShape(dst, src.shape, 'blendMasked: dst');
+ if (opts.feather > 0 && opts.feather % 2 === 0) {
+ throw new Error(`blendMasked: feather must be 0 or an odd kernel size (got ${opts.feather})`);
+ }
+
+ const inRange = (y: number, x: number) => y >= 0 && y < height && x >= 0 && x < width;
+ let weight = Float32Array.from({ length: height * width }, (_, i) =>
+ mask.getElement(i) !== 0 ? 1 : 0
+ );
+
+ if (opts.shrink > 0) {
+ const eroded = new Float32Array(weight.length);
+ for (let y = 0; y < height; y++) {
+ for (let x = 0; x < width; x++) {
+ let min = 1;
+ for (let dy = -opts.shrink; dy <= opts.shrink; dy++) {
+ for (let dx = -opts.shrink; dx <= opts.shrink; dx++) {
+ if (inRange(y + dy, x + dx)) min = Math.min(min, weight[(y + dy) * width + x + dx]!);
+ }
+ }
+ eroded[y * width + x] = min;
+ }
+ }
+ // Like the native op: a region that erodes to nothing is kept as is.
+ if (eroded.some((w) => w > 0)) weight = eroded;
+ }
+
+ if (opts.feather > 0) {
+ const radius = (opts.feather - 1) / 2;
+ const blurred = new Float32Array(weight.length);
+ for (let y = 0; y < height; y++) {
+ for (let x = 0; x < width; x++) {
+ let sum = 0;
+ let count = 0;
+ for (let dy = -radius; dy <= radius; dy++) {
+ for (let dx = -radius; dx <= radius; dx++) {
+ if (!inRange(y + dy, x + dx)) continue;
+ sum += weight[(y + dy) * width + x + dx]!;
+ count++;
+ }
+ }
+ blurred[y * width + x] = sum / count;
+ }
+ }
+ // Like the native op: the ramp stays inside the mask.
+ weight = blurred.map((w, p) => (mask.getElement(p) !== 0 ? w : 0));
+ }
+
+ for (let p = 0; p < height * width; p++) {
+ const w = weight[p]!;
+ for (let c = 0; c < channels; c++) {
+ const at = p * channels + c;
+ dst.setElement(at, Math.round(src.getElement(at) * (1 - w) + overlay.getElement(at) * w));
+ }
+ }
+ return dst;
+ },
};
// ============================================================================
diff --git a/packages/react-native-executorch/__tests__/tasks/inpainter.test.ts b/packages/react-native-executorch/__tests__/tasks/inpainter.test.ts
new file mode 100644
index 0000000000..7098ba64cc
--- /dev/null
+++ b/packages/react-native-executorch/__tests__/tasks/inpainter.test.ts
@@ -0,0 +1,252 @@
+import { f32, method } from '../../src/core/schema';
+import { createInpainter, type MaskBuffer } from '../../src/extensions/cv/tasks/inpainting';
+import type { ImageBuffer } from '../../src/extensions/cv/image';
+import { fakeJsi } from '../support/fakeJsi';
+import type { FakeTensor } from '../support/fakeTensor';
+import { tracked } from '../support/lifetime';
+import { exported, imageBuffer, writesOutputs } from '../support/fixtures';
+
+const MODEL_PATH = '/models/inpaint.pte';
+const SIZE = 4;
+
+/** The value the fake model paints, and the pixel it comes out as (0.2 * 255). */
+const FILL = 0.2;
+const FILL_PIXEL = 51;
+
+const options = {
+ resizeMode: 'stretch',
+ interpolation: 'nearest',
+ normalizeOpts: { alpha: 1 / 255, beta: 0 },
+ outNormalizeOpts: { alpha: 255, beta: 0 },
+ outInterpolation: 'nearest',
+ blendOpts: { shrink: 0, feather: 0 },
+} as const;
+
+const config = { modelPath: MODEL_PATH, modelOpts: options };
+
+const BATCHED = method(
+ 'forward',
+ [f32(1, 3, SIZE, SIZE), f32(1, 1, SIZE, SIZE)],
+ [f32(1, 3, SIZE, SIZE)]
+);
+
+/** A model that paints every output pixel with `FILL`. */
+const registerPainter = () => {
+ fakeJsi.registerModel(MODEL_PATH, {
+ schema: exported(BATCHED),
+ execute: writesOutputs(new Array(3 * SIZE * SIZE).fill(FILL)),
+ });
+};
+
+/**
+ * A grayscale mask over a `width` x `height` image marking the `hole` rectangle
+ * (inclusive corners) with 255.
+ */
+const maskBuffer = (
+ width: number,
+ height: number,
+ hole: { x0: number; y0: number; x1: number; y1: number } | null
+): MaskBuffer => {
+ const data = new Uint8Array(width * height);
+ if (hole) {
+ for (let y = hole.y0; y <= hole.y1; y++) {
+ for (let x = hole.x0; x <= hole.x1; x++) data[y * width + x] = 255;
+ }
+ }
+ return { data, width, height, format: 'gray', layout: 'hwc' };
+};
+
+const rgbaAt = (output: ImageBuffer, x: number, y: number) => {
+ const at = (y * output.width + x) * 4;
+ return [...output.data.slice(at, at + 4)];
+};
+
+describe('createInpainter', () => {
+ beforeEach(registerPainter);
+
+ it('returns an opaque RGBA buffer at the input resolution', async () => {
+ const inpainter = tracked(await createInpainter(config));
+
+ const output = await inpainter.inpaint(
+ imageBuffer(10, 6),
+ maskBuffer(10, 6, { x0: 3, y0: 2, x1: 4, y1: 3 })
+ );
+
+ expect(output).toMatchObject({ width: 10, height: 6, format: 'rgba', layout: 'hwc' });
+ expect(output.data).toHaveLength(10 * 6 * 4);
+ expect([...output.data].filter((_, i) => i % 4 === 3).every((a) => a === 255)).toBe(true);
+ });
+
+ it('fills the hole with the model output and leaves the rest untouched', async () => {
+ // The model paints the whole (downscaled) frame, so with nearest resizing
+ // and a hard blend the hole takes the painted value exactly and every
+ // other pixel is the input.
+ const inpainter = tracked(await createInpainter(config));
+ const input = imageBuffer(8, 8);
+ const hole = { x0: 2, y0: 2, x1: 3, y1: 3 };
+
+ const output = await inpainter.inpaint(input, maskBuffer(8, 8, hole));
+
+ for (let y = 0; y < 8; y++) {
+ for (let x = 0; x < 8; x++) {
+ const inHole = x >= hole.x0 && x <= hole.x1 && y >= hole.y0 && y <= hole.y1;
+ const at = (y * 8 + x) * 3;
+ const original = [...input.data.slice(at, at + 3), 255];
+ expect(rgbaAt(output, x, y)).toEqual(
+ inHole ? [FILL_PIXEL, FILL_PIXEL, FILL_PIXEL, 255] : original
+ );
+ }
+ }
+ });
+
+ it('feeds the model the whole frame resized to its input', async () => {
+ let imageSeen: number[] = [];
+ fakeJsi.registerModel(MODEL_PATH, {
+ schema: exported(BATCHED),
+ execute: (_method, inputs) => {
+ const image = inputs[0] as FakeTensor;
+ imageSeen = Array.from({ length: image.numel }, (_, i) => image.getElement(i));
+ },
+ });
+ const inpainter = tracked(await createInpainter(config));
+ const input = imageBuffer(2 * SIZE, 2 * SIZE);
+
+ // A hole in one corner still sends the model every part of the frame.
+ await inpainter.inpaint(input, maskBuffer(2 * SIZE, 2 * SIZE, { x0: 0, y0: 0, x1: 0, y1: 0 }));
+
+ // Nearest downscaling by 2 keeps every other pixel, laid out CHW in [0, 1].
+ const expected: number[] = [];
+ for (let c = 0; c < 3; c++) {
+ for (let y = 0; y < SIZE; y++) {
+ for (let x = 0; x < SIZE; x++) {
+ expected.push(input.data[(2 * y * 2 * SIZE + 2 * x) * 3 + c]! / 255);
+ }
+ }
+ }
+ expect(imageSeen).toHaveLength(expected.length);
+ imageSeen.forEach((v, i) => expect(v).toBeCloseTo(expected[i]!, 5));
+ });
+
+ it('feeds the model 1 for known pixels and 0 for the hole', async () => {
+ let maskSeen: number[] = [];
+ fakeJsi.registerModel(MODEL_PATH, {
+ schema: exported(BATCHED),
+ execute: (_method, inputs) => {
+ const mask = inputs[1] as FakeTensor;
+ maskSeen = Array.from({ length: mask.numel }, (_, i) => mask.getElement(i));
+ },
+ });
+ const inpainter = tracked(await createInpainter(config));
+
+ // Any non-zero mask value is the hole, however faint.
+ const mask = maskBuffer(SIZE, SIZE, null);
+ mask.data[1 * SIZE + 2] = 1;
+ await inpainter.inpaint(imageBuffer(SIZE, SIZE), mask);
+
+ const expected = new Array(SIZE * SIZE).fill(1);
+ expected[1 * SIZE + 2] = 0;
+ expect(maskSeen).toEqual(expected);
+ });
+
+ it('returns the input unchanged for an empty mask', async () => {
+ const inpainter = tracked(await createInpainter(config));
+ const input = imageBuffer(6, 5);
+
+ const output = await inpainter.inpaint(input, maskBuffer(6, 5, null));
+
+ for (let p = 0; p < 6 * 5; p++) {
+ expect([...output.data.slice(p * 4, p * 4 + 3)]).toEqual([
+ ...input.data.slice(p * 3, p * 3 + 3),
+ ]);
+ }
+ });
+
+ it('rejects a mask that is not a gray buffer', async () => {
+ const inpainter = tracked(await createInpainter(config));
+ const mask = imageBuffer(SIZE, SIZE, 'rgba') as unknown as MaskBuffer;
+
+ await expect(inpainter.inpaint(imageBuffer(SIZE, SIZE), mask)).rejects.toMatchObject({
+ code: 'INVALID_ARGUMENT',
+ });
+ expect(fakeJsi.executions()).toEqual([]);
+ });
+
+ it('accepts the unbatched [3, H, W] variant', async () => {
+ fakeJsi.registerModel(MODEL_PATH, {
+ schema: exported(
+ method('forward', [f32(3, SIZE, SIZE), f32(1, SIZE, SIZE)], [f32(3, SIZE, SIZE)])
+ ),
+ execute: writesOutputs(new Array(3 * SIZE * SIZE).fill(FILL)),
+ });
+ const inpainter = tracked(await createInpainter(config));
+
+ const output = await inpainter.inpaint(
+ imageBuffer(SIZE, SIZE),
+ maskBuffer(SIZE, SIZE, { x0: 0, y0: 0, x1: 0, y1: 0 })
+ );
+
+ expect(rgbaAt(output, 0, 0)).toEqual([FILL_PIXEL, FILL_PIXEL, FILL_PIXEL, 255]);
+ });
+
+ it('rejects a model whose mask input does not match the image input', async () => {
+ fakeJsi.registerModel(MODEL_PATH, {
+ schema: exported(
+ method(
+ 'forward',
+ [f32(1, 3, SIZE, SIZE), f32(1, 1, SIZE * 2, SIZE)],
+ [f32(1, 3, SIZE, SIZE)]
+ )
+ ),
+ });
+
+ await expect(createInpainter(config)).rejects.toThrow(/inconsistent bindings/);
+ });
+
+ it('rejects a mask whose dimensions differ from the image', async () => {
+ const inpainter = tracked(await createInpainter(config));
+
+ await expect(
+ inpainter.inpaint(imageBuffer(6, 6), maskBuffer(3, 3, { x0: 0, y0: 0, x1: 0, y1: 0 }))
+ ).rejects.toThrow(/mask is 3x3 but image is 6x6/);
+ });
+
+ it('produces the same result synchronously and asynchronously', async () => {
+ const inpainter = tracked(await createInpainter(config));
+ const input = imageBuffer(8, 8);
+ const mask = maskBuffer(8, 8, { x0: 2, y0: 2, x1: 3, y1: 3 });
+
+ expect(inpainter.inpaintWorklet(input, mask)).toEqual(await inpainter.inpaint(input, mask));
+ });
+
+ it('releases every native resource on dispose', async () => {
+ const inpainter = await createInpainter(config);
+ await inpainter.inpaint(
+ imageBuffer(20, 15),
+ maskBuffer(20, 15, { x0: 5, y0: 5, x1: 8, y1: 9 })
+ );
+
+ inpainter.dispose();
+
+ expect(fakeJsi.liveTensors()).toBe(0);
+ expect(fakeJsi.liveModels()).toEqual([]);
+ });
+
+ it('frees the per-call tensors across repeated calls', async () => {
+ const inpainter = tracked(await createInpainter(config));
+ const afterConstruction = fakeJsi.liveTensors();
+
+ for (const size of [8, 16, 32]) {
+ await inpainter.inpaint(
+ imageBuffer(size, size),
+ maskBuffer(size, size, { x0: 1, y0: 1, x1: 2, y1: 2 })
+ );
+ }
+ await inpainter.inpaint(imageBuffer(8, 8), maskBuffer(8, 8, null));
+ await inpainter.inpaint(
+ imageBuffer(8, 8, 'rgba'),
+ maskBuffer(8, 8, { x0: 1, y0: 1, x1: 2, y1: 2 })
+ );
+
+ expect(fakeJsi.liveTensors()).toBe(afterConstruction);
+ });
+});
diff --git a/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp b/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp
index f620ef1dff..cbac64dc6f 100644
--- a/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp
+++ b/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp
@@ -510,4 +510,81 @@ void install_rectifyQuad(jsi::Runtime &rt, jsi::Object &module) {
module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 4, error::guarded(fnBody)));
}
+// Composite `overlay` over `src` where `mask` is non-zero, with the masked
+// region eroded by `shrink` px and its edge feathered by a `feather`-wide
+// Gaussian, so the seam blends instead of cutting: dst = src*(1-w) + overlay*w.
+void install_blendMasked(jsi::Runtime &rt, jsi::Object &module) {
+ const auto *name = "blendMasked";
+ auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value {
+ if (count != 5) {
+ throw error::InvalidArgument("blendMasked: Usage: blendMasked(src, overlay, mask, dst, options)");
+ }
+
+ auto src = tensor::fromJs(rt, "blendMasked: src", args[0], DType::uint8, {"H", "W", "C"});
+ auto overlay = tensor::fromJs(rt, "blendMasked: overlay", args[1], DType::uint8, src->shape_);
+ auto mask = tensor::fromJs(rt, "blendMasked: mask", args[2], DType::uint8, {src->shape_[0], src->shape_[1], 1});
+ auto dst = tensor::fromJs(rt, "blendMasked: dst", args[3], DType::uint8, src->shape_);
+
+ tensor::checkNotSameTensor(rt, "blendMasked: src", src, "blendMasked: dst", dst);
+ tensor::checkNotSameTensor(rt, "blendMasked: overlay", overlay, "blendMasked: dst", dst);
+ auto srcLock = tensor::tryLockShared(rt, "blendMasked: src", src);
+ auto overlayLock = tensor::tryLockShared(rt, "blendMasked: overlay", overlay);
+ auto maskLock = tensor::tryLockShared(rt, "blendMasked: mask", mask);
+ auto dstLock = tensor::tryLockUnique(rt, "blendMasked: dst", dst);
+
+ const auto opts = conversions::asType(rt, "blendMasked: options", args[4]);
+ const auto shrink = conversions::getRequiredProperty(rt, "blendMasked: options", opts, "shrink");
+ const auto feather = conversions::getRequiredProperty(rt, "blendMasked: options", opts, "feather");
+ if (shrink < 0) {
+ throw error::InvalidArgument(std::format("blendMasked: shrink must be >= 0 (got {})", shrink));
+ }
+ if (feather < 0 || (feather > 0 && feather % 2 == 0)) {
+ throw error::InvalidArgument(std::format("blendMasked: feather must be 0 or an odd kernel size (got {})", feather));
+ }
+
+ const int32_t h = src->shape_[0];
+ const int32_t w = src->shape_[1];
+ const int cvType = CV_MAKETYPE(CV_8U, src->shape_[2]);
+
+ try {
+ const ::cv::Mat srcMat(h, w, cvType, src->data_.get());
+ const ::cv::Mat overlayMat(h, w, cvType, overlay->data_.get());
+ const ::cv::Mat maskMat(h, w, CV_8UC1, mask->data_.get());
+ ::cv::Mat dstMat(h, w, cvType, dst->data_.get());
+
+ // Weight of the overlay: 1 inside the (eroded, feathered) mask, 0 outside.
+ ::cv::Mat hole;
+ ::cv::threshold(maskMat, hole, 0, 255, ::cv::THRESH_BINARY);
+ ::cv::Mat region = hole;
+ if (shrink > 0) {
+ const int k = 2 * shrink + 1;
+ ::cv::Mat eroded;
+ ::cv::erode(region, eroded, ::cv::getStructuringElement(::cv::MORPH_RECT, ::cv::Size(k, k)));
+ // A region thinner than the kernel erodes to nothing, which would
+ // silently leave `src` untouched; keep the full region instead.
+ if (::cv::countNonZero(eroded) > 0) {
+ region = eroded;
+ }
+ }
+ ::cv::Mat weight;
+ region.convertTo(weight, CV_32F, 1.0 / 255.0);
+ if (feather > 0) {
+ ::cv::GaussianBlur(weight, weight, ::cv::Size(feather, feather), 0);
+ // The blur spreads both ways; keep the ramp inside the mask so
+ // pixels outside it are returned exactly as in `src`.
+ weight.setTo(::cv::Scalar::all(0), hole == 0);
+ }
+
+ ::cv::Mat ones = ::cv::Mat::ones(h, w, CV_32F);
+ ::cv::blendLinear(srcMat, overlayMat, ones - weight, weight, dstMat);
+ } catch (const std::exception &e) {
+ throw error::Unknown("blendMasked: " + std::string(e.what()));
+ }
+
+ return jsi::Value(rt, args[3]);
+ };
+
+ module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 5, error::guarded(fnBody)));
+}
+
} // namespace rnexecutorch::extensions::cv::image_ops
diff --git a/packages/react-native-executorch/cpp/extensions/cv/image_ops.h b/packages/react-native-executorch/cpp/extensions/cv/image_ops.h
index 09b2417fe9..c4cb5b5af5 100644
--- a/packages/react-native-executorch/cpp/extensions/cv/image_ops.h
+++ b/packages/react-native-executorch/cpp/extensions/cv/image_ops.h
@@ -10,4 +10,5 @@ void install_toChannelsLast(facebook::jsi::Runtime &rt, facebook::jsi::Object &m
void install_normalize(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
void install_applyColormap(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
void install_rectifyQuad(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
+void install_blendMasked(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
} // namespace rnexecutorch::extensions::cv::image_ops
diff --git a/packages/react-native-executorch/cpp/extensions/cv/install.cpp b/packages/react-native-executorch/cpp/extensions/cv/install.cpp
index 9a95cc8d6b..c9da8df8fe 100644
--- a/packages/react-native-executorch/cpp/extensions/cv/install.cpp
+++ b/packages/react-native-executorch/cpp/extensions/cv/install.cpp
@@ -16,6 +16,7 @@ void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module) {
image_ops::install_normalize(rt, cvModule);
image_ops::install_applyColormap(rt, cvModule);
image_ops::install_rectifyQuad(rt, cvModule);
+ image_ops::install_blendMasked(rt, cvModule);
box_ops::install_nms(rt, cvModule);
box_ops::install_restrictToBox(rt, cvModule);
diff --git a/packages/react-native-executorch/cpp/tests/extensions/CvOpsTest.cpp b/packages/react-native-executorch/cpp/tests/extensions/CvOpsTest.cpp
index 658389c5b9..52b9f10e76 100644
--- a/packages/react-native-executorch/cpp/tests/extensions/CvOpsTest.cpp
+++ b/packages/react-native-executorch/cpp/tests/extensions/CvOpsTest.cpp
@@ -415,5 +415,116 @@ TEST_F(CvOpsTest, RectifyQuadRejectsWrongArgumentCounts) {
"INVALID_ARGUMENT", "Usage: rectifyQuad(src, dst, quad, options)"));
}
+// --- blendMasked ------------------------------------------------------------
+
+TEST_F(CvOpsTest, BlendMaskedTakesTheOverlayWhereTheMaskIsSet) {
+ // Hard edge: masked pixels come from the overlay, the rest from src.
+ auto result = evalNumberArray(std::format(R"(
+ {}
+ const src = fillU8(createTensor([1, 4, 1], 'uint8'), [10, 10, 10, 10]);
+ const overlay = fillU8(createTensor([1, 4, 1], 'uint8'), [200, 200, 200, 200]);
+ const mask = fillU8(createTensor([1, 4, 1], 'uint8'), [0, 255, 7, 0]);
+ const dst = createTensor([1, 4, 1], 'uint8');
+ cv.blendMasked(src, overlay, mask, dst, {{ shrink: 0, feather: 0 }});
+ return readU8(dst);
+ )",
+ kNs));
+ EXPECT_TRUE(almostEqual(result, {10, 200, 200, 10}));
+}
+
+TEST_F(CvOpsTest, BlendMaskedBlendsEveryChannel) {
+ auto result = evalNumberArray(std::format(R"(
+ {}
+ const src = fillU8(createTensor([1, 2, 3], 'uint8'), [1, 2, 3, 4, 5, 6]);
+ const overlay = fillU8(createTensor([1, 2, 3], 'uint8'), [9, 9, 9, 9, 9, 9]);
+ const mask = fillU8(createTensor([1, 2, 1], 'uint8'), [0, 1]);
+ const dst = createTensor([1, 2, 3], 'uint8');
+ cv.blendMasked(src, overlay, mask, dst, {{ shrink: 0, feather: 0 }});
+ return readU8(dst);
+ )",
+ kNs));
+ EXPECT_TRUE(almostEqual(result, {1, 2, 3, 9, 9, 9}));
+}
+
+TEST_F(CvOpsTest, BlendMaskedShrinkErodesTheMaskedRegion) {
+ // A 3-wide masked run eroded by 1 px keeps only its centre pixel.
+ auto result = evalNumberArray(std::format(R"(
+ {}
+ const src = fillU8(createTensor([1, 5, 1], 'uint8'), [0, 0, 0, 0, 0]);
+ const overlay = fillU8(createTensor([1, 5, 1], 'uint8'), [255, 255, 255, 255, 255]);
+ const mask = fillU8(createTensor([1, 5, 1], 'uint8'), [0, 255, 255, 255, 0]);
+ const dst = createTensor([1, 5, 1], 'uint8');
+ cv.blendMasked(src, overlay, mask, dst, {{ shrink: 1, feather: 0 }});
+ return readU8(dst);
+ )",
+ kNs));
+ EXPECT_TRUE(almostEqual(result, {0, 0, 255, 0, 0}));
+}
+
+TEST_F(CvOpsTest, BlendMaskedKeepsARegionThinnerThanShrink) {
+ // A 1-wide masked run would erode to nothing with shrink 1; it is blended
+ // un-eroded instead of silently leaving src untouched.
+ auto result = evalNumberArray(std::format(R"(
+ {}
+ const src = fillU8(createTensor([1, 5, 1], 'uint8'), [0, 0, 0, 0, 0]);
+ const overlay = fillU8(createTensor([1, 5, 1], 'uint8'), [255, 255, 255, 255, 255]);
+ const mask = fillU8(createTensor([1, 5, 1], 'uint8'), [0, 0, 255, 0, 0]);
+ const dst = createTensor([1, 5, 1], 'uint8');
+ cv.blendMasked(src, overlay, mask, dst, {{ shrink: 1, feather: 0 }});
+ return readU8(dst);
+ )",
+ kNs));
+ EXPECT_TRUE(almostEqual(result, {0, 0, 255, 0, 0}));
+}
+
+TEST_F(CvOpsTest, BlendMaskedFeatherSoftensTheSeamInsideTheMask) {
+ // With a 3-wide Gaussian the first masked pixel ends up strictly between
+ // the two sources, while the pixels outside the mask stay on src.
+ auto result = evalNumberArray(std::format(R"(
+ {}
+ const src = fillU8(createTensor([1, 6, 1], 'uint8'), [0, 0, 0, 0, 0, 0]);
+ const overlay = fillU8(createTensor([1, 6, 1], 'uint8'), [200, 200, 200, 200, 200, 200]);
+ const mask = fillU8(createTensor([1, 6, 1], 'uint8'), [0, 0, 0, 255, 255, 255]);
+ const dst = createTensor([1, 6, 1], 'uint8');
+ cv.blendMasked(src, overlay, mask, dst, {{ shrink: 0, feather: 3 }});
+ return readU8(dst);
+ )",
+ kNs));
+ ASSERT_EQ(result.size(), 6u);
+ EXPECT_EQ(result[0], 0);
+ EXPECT_EQ(result[1], 0);
+ EXPECT_EQ(result[2], 0);
+ EXPECT_GT(result[3], 0);
+ EXPECT_LT(result[3], 200);
+ EXPECT_EQ(result[5], 200);
+}
+
+TEST_F(CvOpsTest, BlendMaskedRejectsAnEvenFeather) {
+ EXPECT_TRUE(isCodedError(evalThrowing(std::format(R"(
+ {}
+ cv.blendMasked(createTensor([1, 2, 1], 'uint8'), createTensor([1, 2, 1], 'uint8'),
+ createTensor([1, 2, 1], 'uint8'), createTensor([1, 2, 1], 'uint8'),
+ {{ shrink: 0, feather: 4 }});
+ )",
+ kNs)),
+ "INVALID_ARGUMENT", "feather must be 0 or an odd kernel size"));
+}
+
+TEST_F(CvOpsTest, BlendMaskedRejectsAMaskOfTheWrongShape) {
+ EXPECT_TRUE(isCodedError(evalThrowing(std::format(R"(
+ {}
+ cv.blendMasked(createTensor([2, 2, 3], 'uint8'), createTensor([2, 2, 3], 'uint8'),
+ createTensor([2, 2, 3], 'uint8'), createTensor([2, 2, 3], 'uint8'),
+ {{ shrink: 0, feather: 0 }});
+ )",
+ kNs)),
+ "INVALID_ARGUMENT", "blendMasked: mask"));
+}
+
+TEST_F(CvOpsTest, BlendMaskedRejectsWrongArgumentCounts) {
+ EXPECT_TRUE(isCodedError(evalThrowing(std::format("{} cv.blendMasked();", kNs)),
+ "INVALID_ARGUMENT", "Usage: blendMasked(src, overlay, mask, dst, options)"));
+}
+
} // namespace
} // namespace rnexecutorch::tests
diff --git a/packages/react-native-executorch/package.json b/packages/react-native-executorch/package.json
index 881fe1e5cd..b20d6c0925 100644
--- a/packages/react-native-executorch/package.json
+++ b/packages/react-native-executorch/package.json
@@ -1,7 +1,7 @@
{
"name": "react-native-executorch",
"version": "0.11.0",
- "nativeLibsVersion": "0.10.6",
+ "nativeLibsVersion": "0.10.7",
"description": "An easy way to run AI models in React Native with ExecuTorch",
"main": "./lib/module/index.js",
"module": "./lib/module/index.js",
diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js
index 0b0b215f04..38b990f29c 100644
--- a/packages/react-native-executorch/scripts/download-libs.js
+++ b/packages/react-native-executorch/scripts/download-libs.js
@@ -46,7 +46,7 @@
* textEmbeddings, imageEmbeddings,
* classification, objectDetection, depthEstimation, semanticSegmentation, instanceSegmentation,
* ocr, verticalOCR, keypointDetection, styleTransfer, textToImage, segmentAnything,
- * tokenizer
+ * inpainting, tokenizer
*
* Platform applicability:
* opencv Android + iOS — downloaded artifact
@@ -178,6 +178,8 @@ const FEATURE_MAP = {
styleTransfer: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] },
// SDXS ships xnnpack, coreml and a vulkan Android export.
textToImage: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] },
+ // MI-GAN ships xnnpack, coreml and a vulkan Android export.
+ inpainting: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] },
// FastSAM ships xnnpack, coreml and a vulkan Android export.
segmentAnything: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] },
// Tokenizer is pure-CPU string ops resolved from libexecutorch; needs no
diff --git a/packages/react-native-executorch/src/extensions/cv/ops/image.ts b/packages/react-native-executorch/src/extensions/cv/ops/image.ts
index e15421db1b..c8baa2873b 100644
--- a/packages/react-native-executorch/src/extensions/cv/ops/image.ts
+++ b/packages/react-native-executorch/src/extensions/cv/ops/image.ts
@@ -265,3 +265,54 @@ export function applyColormap(
'worklet';
return rnexecutorchJsi.cv.applyColormap(src, dst, colormap);
}
+
+/**
+ * Options softening the mask edge in {@link blendMasked}.
+ * @category CV / Types
+ */
+export type BlendMaskedOptions = {
+ /**
+ * Pixels to erode the masked region by before feathering, so the seam sits
+ * inside the region rather than on its edge. A region thinner than
+ * `2 * shrink + 1` px would erode to nothing, so such a mask is used as is.
+ * Default `0`.
+ */
+ readonly shrink?: number;
+ /**
+ * Size in pixels of the Gaussian kernel that feathers the mask edge. The
+ * ramp runs inward from the edge, so pixels outside the mask are never
+ * touched. Must be odd; `0` keeps the edge hard. Default `0`.
+ */
+ readonly feather?: number;
+};
+
+/**
+ * Composites `overlay` over `src` where `mask` is non-zero, writing
+ * `src * (1 - w) + overlay * w` into `dst`, where `w` is the mask eroded and
+ * feathered per `options`. Pixels outside the mask come from `src` unchanged.
+ * @category CV / Functions
+ * @param src The base image tensor in HWC layout, `[H, W, C]`, data type `uint8`.
+ * @param overlay The image tensor to composite, `[H, W, C]`, data type `uint8`.
+ * @param mask The region to take from `overlay`, `[H, W, 1]`, data type `uint8`;
+ * any non-zero value marks a masked pixel.
+ * @param dst The pre-allocated destination image tensor, `[H, W, C]`, data type
+ * `uint8`. Must not alias `src` or `overlay`.
+ * @param options Edge erosion and feathering. See {@link BlendMaskedOptions}.
+ * @returns The destination image tensor `dst`.
+ * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes,
+ * layouts, or data types are invalid, or `feather` is even, `RESOURCE_BUSY` if
+ * a tensor is in use, or `RESOURCE_DISPOSED` if any tensor was disposed.
+ */
+export function blendMasked(
+ src: Tensor,
+ overlay: Tensor,
+ mask: Tensor,
+ dst: Tensor,
+ options?: BlendMaskedOptions
+): Tensor {
+ 'worklet';
+ return rnexecutorchJsi.cv.blendMasked(src, overlay, mask, dst, {
+ shrink: options?.shrink ?? 0,
+ feather: options?.feather ?? 0,
+ });
+}
diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/inpainting.ts b/packages/react-native-executorch/src/extensions/cv/tasks/inpainting.ts
new file mode 100644
index 0000000000..56f35b4a0a
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/cv/tasks/inpainting.ts
@@ -0,0 +1,269 @@
+/**
+ * Image inpainting task pipeline: fills a masked region of a photo with
+ * plausible content (object removal, blemish cleanup). The whole frame is
+ * resized to the model input, and the output is resized back and blended into
+ * the original pixels through the mask, so only the hole changes.
+ */
+
+import type { WorkletRuntime } from 'react-native-worklets';
+
+import { tensor } from '../../../core/tensor';
+import { loadModel } from '../../../core/model';
+import { validateSpec, method, f32 } from '../../../core/schema';
+import { wrapAsync } from '../../../core/runtime';
+import { createResourceScope } from '../../../core/lifetime';
+import { RnExecuTorchError } from '../../../core/error';
+import { threshold } from '../../math';
+
+import type { ImageBuffer } from '../image';
+import {
+ FORMAT_CHANNELS,
+ FORMAT_CONVERSION,
+ resize,
+ cvtColor,
+ toChannelsFirst,
+ toChannelsLast,
+ normalize,
+ blendMasked,
+ type InterpolationMethod,
+ type NormalizeOptions,
+ type BlendMaskedOptions,
+} from '../ops/image';
+import { createImagePreprocessor, type ImagePreprocessorOptions } from '../utils/imagePreprocessor';
+
+/**
+ * Options for configuring the inpainting preprocessor and postprocessor. The
+ * image goes through the shared image preprocessor; the mask is always
+ * area-resized and re-binarized, so a model pixel is part of the hole when any
+ * of the photo pixels it covers is.
+ * @category CV / Types
+ */
+export type InpainterOptions = Omit & {
+ /** Resize mode for input images. Must be `'stretch'`. */
+ readonly resizeMode: 'stretch';
+ /** Normalization options for postprocessing output tensors back to uint8 pixel values. */
+ readonly outNormalizeOpts: NormalizeOptions;
+ /** Interpolation method used when resizing the filled frame back to input dimensions. */
+ readonly outInterpolation: InterpolationMethod;
+ /**
+ * How the filled region is blended back over the original pixels. See
+ * {@link BlendMaskedOptions}.
+ */
+ readonly blendOpts: BlendMaskedOptions;
+};
+
+/**
+ * Model configuration required to instantiate an inpainting task runner.
+ * @category CV / Types
+ */
+export type InpainterModel = {
+ /** Local path or remote URL of the `.pte` model file. */
+ readonly modelPath: string;
+ /**
+ * Input preprocessing, output postprocessing and blending options.
+ * `resizeMode` is fixed to `'stretch'`. See {@link InpainterOptions}.
+ */
+ readonly modelOpts: InpainterOptions;
+};
+
+/**
+ * The hole to fill, as a single-channel buffer: any non-zero pixel is part of
+ * the hole.
+ * @category CV / Types
+ */
+export type MaskBuffer = ImageBuffer & { readonly format: 'gray' };
+
+/**
+ * Image inpainting task runner.
+ * @category CV / Types
+ */
+export type Inpainter = {
+ /**
+ * Releases all allocated native resources.
+ */
+ readonly dispose: () => void;
+
+ /**
+ * Fills the masked region of `image` and returns the whole frame with the
+ * hole filled. The model sees the whole frame stretched to its input size;
+ * to fill a small hole in a large photo at higher detail, crop the region
+ * around it first and inpaint the crop.
+ * @param image The input image buffer.
+ * @param mask A `'gray'` buffer of the same dimensions as `image` marking the
+ * region to fill: any non-zero pixel is part of the hole. Pixels outside the
+ * hole are returned unchanged.
+ * @returns A promise resolving to the inpainted RGBA image buffer at the
+ * input dimensions.
+ * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if `mask` is not
+ * a `'gray'` buffer or does not match the dimensions of `image`,
+ * `RESOURCE_BUSY` if the model is in use, or `RESOURCE_DISPOSED` if disposed.
+ */
+ readonly inpaint: (image: ImageBuffer, mask: MaskBuffer) => Promise;
+
+ /**
+ * Synchronous version of {@link inpaint} to be executed directly on the
+ * caller or worklet thread.
+ */
+ readonly inpaintWorklet: (image: ImageBuffer, mask: MaskBuffer) => ImageBuffer;
+};
+
+/**
+ * Creates an image inpainting runner for executing local inpainting models.
+ *
+ * It validates the model inputs and outputs requirements, pre-allocates the
+ * necessary static execution tensors, and registers clean disposal hooks to
+ * clear all native memory. The model is expected to take an RGB image in
+ * `[0, 1]` and a mask with `1` for known pixels and `0` for the hole, and to
+ * return the filled RGB image in `[0, 1]`, all at a fixed resolution.
+ * @category CV / Tasks
+ * @param config Inpainting task configuration containing path and options.
+ * See {@link InpainterModel}.
+ * @param runtime Optional worklet runtime thread on which to run the model execution.
+ * @returns A promise resolving to the instantiated {@link Inpainter} runner.
+ * @throws {RnExecuTorchError} With code `LOAD_FAILED` if model fails to load,
+ * or `SCHEMA_MISMATCH` if model schema does not match the inpainting spec.
+ */
+export async function createInpainter(
+ config: InpainterModel,
+ runtime?: WorkletRuntime
+): Promise {
+ const scope = createResourceScope();
+ const dispose = scope.dispose;
+
+ try {
+ const { modelPath, modelOpts } = config;
+ const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath));
+
+ const { variant, dims } = validateSpec(model.schema, {
+ batched: method(
+ 'forward', // prettier-ignore
+ [f32(1, 3, 'H', 'W'), f32(1, 1, 'H', 'W')],
+ [f32(1, 3, 'H', 'W')]
+ ),
+ unbatched: method(
+ 'forward', // prettier-ignore
+ [f32(3, 'H', 'W'), f32(1, 'H', 'W')],
+ [f32(3, 'H', 'W')]
+ ),
+ });
+
+ const [H, W] = dims.constant('H', 'W');
+ const imageShape = { batched: [1, 3, H, W], unbatched: [3, H, W] }[variant];
+ const maskShape = { batched: [1, 1, H, W], unbatched: [1, H, W] }[variant];
+
+ const preprocessor = scope.track(createImagePreprocessor(modelOpts, imageShape));
+
+ const tensors = [
+ // Model I/O; the image input is owned by the preprocessor.
+ tensor('float32', maskShape),
+ tensor('float32', imageShape),
+ // Mask: resized -> CHW -> float (1 = known, < 1 = hole) -> binarized.
+ tensor('uint8', [H, W, 1]),
+ tensor('uint8', [1, H, W]),
+ tensor('float32', [1, H, W]),
+ tensor('float32', [1, H, W]),
+ // Output: CHW float -> uint8 -> HWC -> RGBA.
+ tensor('float32', [3, H, W]),
+ tensor('uint8', [3, H, W]),
+ tensor('uint8', [H, W, 3]),
+ tensor('uint8', [H, W, 4]),
+ ] as const;
+
+ tensors.forEach(scope.track);
+
+ const [
+ tModelMask,
+ tModelOutput,
+ tMaskResized,
+ tMaskChw,
+ tMaskScaled,
+ tMaskKnown,
+ tOutChw,
+ tOutU8,
+ tOutHwc,
+ tOutRgba,
+ ] = tensors;
+
+ const { outNormalizeOpts, outInterpolation, blendOpts } = modelOpts;
+
+ const inpaintWorklet = (image: ImageBuffer, mask: MaskBuffer): ImageBuffer => {
+ 'worklet';
+ if (mask.format !== 'gray') {
+ throw RnExecuTorchError(
+ 'INVALID_ARGUMENT',
+ `inpaint: mask must be a 'gray' buffer (got '${mask.format}')`
+ );
+ }
+ if (mask.width !== image.width || mask.height !== image.height) {
+ throw RnExecuTorchError(
+ 'INVALID_ARGUMENT',
+ `inpaint: mask is ${mask.width}x${mask.height} but image is ${image.width}x${image.height}`
+ );
+ }
+
+ const { width, height } = image;
+ const imageChannels = FORMAT_CHANNELS[image.format];
+ const toRgba = FORMAT_CONVERSION[image.format].rgba;
+
+ // Per-call buffers at the frame's resolution: the input as given, the
+ // mask, the upsampled fill, the blended result and, only when the input
+ // needs converting, the frame as RGBA.
+ const auxTensors = [
+ tensor('uint8', [height, width, imageChannels]),
+ tensor('uint8', [height, width, 1]),
+ tensor('uint8', [height, width, 4]),
+ tensor('uint8', [height, width, 4]),
+ ...(toRgba !== null ? [tensor('uint8', [height, width, 4])] : []),
+ ] as const;
+ const [tImage, tMask, tFilled, tResult, tRgba] = auxTensors;
+
+ try {
+ // The full-resolution RGBA frame the filled region is blended into.
+ const tFrame = tImage
+ .setData(image.data)
+ .throughIf(toRgba !== null, cvtColor, tRgba!, toRgba!);
+ tMask.setData(mask.data);
+
+ // Whole frame -> model input.
+ const tModelImage = preprocessor.process(image);
+
+ // Mask -> model input. Area resizing keeps strokes thinner than the
+ // downscale factor as non-zero values, so binarizing at any non-zero
+ // value never drops part of the hole. The scaling maps 0 (known) to 1
+ // and every non-zero value below 1, so a threshold just under 1 yields
+ // the model's 1 = known / 0 = hole mask in one step; the copy reshapes
+ // it to the model's rank.
+ tMask
+ .through(resize, tMaskResized, { mode: 'stretch', interpolation: 'area' })
+ .through(toChannelsFirst, tMaskChw)
+ .through(normalize, tMaskScaled, { alpha: -1 / 255, beta: 1 })
+ .through(threshold, tMaskKnown, 1 - 0.5 / 255)
+ .copyTo(tModelMask);
+
+ model.execute('forward', [tModelImage, tModelMask], [tModelOutput]);
+
+ // Model output -> RGBA at the frame's size, blended over the original
+ // pixels through the full-resolution mask.
+ tModelOutput
+ .copyTo(tOutChw)
+ .through(normalize, tOutU8, outNormalizeOpts)
+ .through(toChannelsLast, tOutHwc)
+ .through(cvtColor, tOutRgba, 'RGB2RGBA')
+ .through(resize, tFilled, { mode: 'stretch', interpolation: outInterpolation });
+ blendMasked(tFrame, tFilled, tMask, tResult, blendOpts);
+
+ const data = tResult.getData(new Uint8Array(width * height * 4));
+ return { data, width, height, format: 'rgba', layout: 'hwc' };
+ } finally {
+ auxTensors.forEach((t) => t.dispose());
+ }
+ };
+
+ const inpaint = wrapAsync(inpaintWorklet, runtime);
+
+ return { inpaint, inpaintWorklet, dispose };
+ } catch (error) {
+ dispose();
+ throw error;
+ }
+}
diff --git a/packages/react-native-executorch/src/hooks/useInpainter.ts b/packages/react-native-executorch/src/hooks/useInpainter.ts
new file mode 100644
index 0000000000..5e24080e1f
--- /dev/null
+++ b/packages/react-native-executorch/src/hooks/useInpainter.ts
@@ -0,0 +1,34 @@
+import { useModel } from './useModel';
+import { useResourceDownload, type ResourceOptions } from './useResourceDownload';
+import { createInpainter, type InpainterModel } from '../extensions/cv/tasks/inpainting';
+
+/**
+ * React hook to load and run an image inpainting model.
+ *
+ * This hook manages downloading (if remote URLs are provided) and loading the
+ * model assets, compiling them, tracking download progress and load errors, and
+ * releasing native memory when the component unmounts or the configuration
+ * changes.
+ *
+ * For imperative usage, see {@link createInpainter}.
+ * @category Hooks
+ * @param config The inpainting model configuration.
+ * See {@link InpainterModel}.
+ * @param options Load and caching options. See {@link ResourceOptions}.
+ * @returns The same object as {@link Inpainter} (without `dispose`),
+ * combined with loading state and download progress.
+ * @see {@link Inpainter}
+ */
+export function useInpainter(config: InpainterModel, options?: ResourceOptions) {
+ const { resource, downloadProgress, downloadError } = useResourceDownload(config, options);
+ const { model, error } = useModel(createInpainter, resource);
+
+ return {
+ isReady: !!model,
+ error: downloadError || error,
+ downloadProgress,
+ resource,
+ inpaint: model?.inpaint,
+ inpaintWorklet: model?.inpaintWorklet,
+ };
+}
diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts
index 9a1040aada..876061b944 100644
--- a/packages/react-native-executorch/src/index.ts
+++ b/packages/react-native-executorch/src/index.ts
@@ -47,6 +47,7 @@
export * from './hooks/useClassifier';
export * from './hooks/useStyleTransfer';
export * from './hooks/useDepthEstimator';
+export * from './hooks/useInpainter';
export * from './hooks/useSemanticSegmenter';
export * from './hooks/useInstanceSegmenter';
export * from './hooks/useKeypointDetector';
@@ -76,6 +77,7 @@ export * from './constants';
export * from './extensions/cv/tasks/classification';
export * from './extensions/cv/tasks/styleTransfer';
export * from './extensions/cv/tasks/depthEstimation';
+export * from './extensions/cv/tasks/inpainting';
export * from './extensions/cv/tasks/semanticSegmentation';
export * from './extensions/cv/tasks/instanceSegmentation';
export * from './extensions/cv/tasks/keypointDetection';
diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts
index a5e1fe322d..aadb5d4071 100644
--- a/packages/react-native-executorch/src/models.ts
+++ b/packages/react-native-executorch/src/models.ts
@@ -20,6 +20,7 @@ import { registerDeprecation } from './deprecation';
import type { ClassifierModel } from './extensions/cv/tasks/classification';
import type { ObjectDetectorModel } from './extensions/cv/tasks/objectDetection';
import type { StyleTransferModel } from './extensions/cv/tasks/styleTransfer';
+import type { InpainterModel } from './extensions/cv/tasks/inpainting';
import type { SemanticSegmenterModel } from './extensions/cv/tasks/semanticSegmentation';
import type { KeypointDetectorModel } from './extensions/cv/tasks/keypointDetection';
import type { InstanceSegmenterModel } from './extensions/cv/tasks/instanceSegmentation';
@@ -400,6 +401,61 @@ const ZIPDEPTH_BASE_VULKAN_FP16: DepthEstimatorModel = {
modelOpts: ZIPDEPTH_OPTS,
};
+// =============================================================================
+// Inpainting
+// =============================================================================
+// MI-GAN takes the image in [0, 1] and a 1 = known / 0 = hole mask and folds
+// its own normalization into the graph. Upstream's reference pipeline erodes
+// the hole by 1 px and feathers the seam with a 5-tap Gaussian, but it blends
+// at the model's resolution; here the fill is upsampled to the photo and
+// blended there, so its edge is already soft, the erosion would only drop
+// strokes thinner than 3 px and a feather that runs inward only would leave a
+// 1 px stroke 60% original. The mask is therefore applied as is.
+const MIGAN_OPTS = {
+ resizeMode: 'stretch' as const,
+ interpolation: 'linear' as const,
+ normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 },
+ outNormalizeOpts: { alpha: 255.0, beta: 0.0 },
+ outInterpolation: 'linear' as const,
+ blendOpts: { shrink: 0, feather: 0 },
+};
+const MIGAN_512_XNNPACK_FP16: InpainterModel = {
+ modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/xnnpack/512/migan_512_xnnpack_fp16.pte`,
+ modelOpts: MIGAN_OPTS,
+};
+const MIGAN_512_COREML_FP16: InpainterModel = {
+ modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/coreml/512/migan_512_coreml_fp16.pte`,
+ modelOpts: MIGAN_OPTS,
+};
+const MIGAN_512_VULKAN_FP16: InpainterModel = {
+ modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/vulkan/512/migan_512_vulkan_fp16.pte`,
+ modelOpts: MIGAN_OPTS,
+};
+const MIGAN_256_XNNPACK_FP16: InpainterModel = {
+ modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/xnnpack/256/migan_256_xnnpack_fp16.pte`,
+ modelOpts: MIGAN_OPTS,
+};
+const MIGAN_256_COREML_FP16: InpainterModel = {
+ modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/coreml/256/migan_256_coreml_fp16.pte`,
+ modelOpts: MIGAN_OPTS,
+};
+const MIGAN_256_VULKAN_FP16: InpainterModel = {
+ modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/vulkan/256/migan_256_vulkan_fp16.pte`,
+ modelOpts: MIGAN_OPTS,
+};
+const MIGAN_FACES_256_XNNPACK_FP16: InpainterModel = {
+ modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/xnnpack/faces_256/migan_faces_256_xnnpack_fp16.pte`,
+ modelOpts: MIGAN_OPTS,
+};
+const MIGAN_FACES_256_COREML_FP16: InpainterModel = {
+ modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/coreml/faces_256/migan_faces_256_coreml_fp16.pte`,
+ modelOpts: MIGAN_OPTS,
+};
+const MIGAN_FACES_256_VULKAN_FP16: InpainterModel = {
+ modelPath: `${BASE_URL}-migan/${NEXT_VERSION_TAG}/vulkan/faces_256/migan_faces_256_vulkan_fp16.pte`,
+ modelOpts: MIGAN_OPTS,
+};
+
// =============================================================================
// Semantic Segmentation
// =============================================================================
@@ -2547,6 +2603,42 @@ export const models = {
}),
},
+ /**
+ * Image inpainting models that fill a masked region of a photo with
+ * plausible content, for object removal and photo cleanup.
+ */
+ inpainting: {
+ /**
+ * MI-GAN, a mobile inpainting GAN trained on Places2 (scenes) and FFHQ
+ * (faces). The size names the resolution the model runs at; the photo is
+ * resized to it, so larger photos work at either size, with `SIZE_512`
+ * keeping more detail at a higher cost.
+ */
+ MIGAN: family({
+ /** Places2 checkpoint at 512x512, the default for general photos. */
+ SIZE_512: variants({
+ XNNPACK_FP16: MIGAN_512_XNNPACK_FP16,
+ COREML_FP16: MIGAN_512_COREML_FP16,
+ VULKAN_FP16: MIGAN_512_VULKAN_FP16,
+ }),
+ /** Places2 checkpoint at 256x256, for lower latency. */
+ SIZE_256: variants({
+ XNNPACK_FP16: MIGAN_256_XNNPACK_FP16,
+ COREML_FP16: MIGAN_256_COREML_FP16,
+ VULKAN_FP16: MIGAN_256_VULKAN_FP16,
+ }),
+ /**
+ * FFHQ checkpoint at 256x256, trained on faces. Reconstructs eyes and
+ * mouths where the scene checkpoints smear them.
+ */
+ FACES_256: variants({
+ XNNPACK_FP16: MIGAN_FACES_256_XNNPACK_FP16,
+ COREML_FP16: MIGAN_FACES_256_COREML_FP16,
+ VULKAN_FP16: MIGAN_FACES_256_VULKAN_FP16,
+ }),
+ }),
+ },
+
/**
* Semantic segmentation models that classify each pixel into target object or
* background classes.