Skip to content

Commit 378dea7

Browse files
barhancbarhanc
committed
Add MIGAN inpainting benchmarks across Android and iOS
1 parent 3c16e3f commit 378dea7

12 files changed

Lines changed: 357 additions & 5 deletions

‎apps/benchmarks/BENCHMARK_SPEC.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ report and the comparator refuses to diff across a change to it.
152152
| -------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- |
153153
| Vision (classification, depth estimation, detection, segmentation, style transfer, OCR, keypoints, image embeddings) | A fixed 512x512 or 640x640 scene: a vertical gradient, three solid ellipses, and a seeded dither |
154154
| ColBERT embeddings | The fixed text-embedding paragraph, encoded as a document; query encoding and MaxSim scoring are not timed |
155+
| Image inpainting | The fixed 512x512 RGBA scene with a central 256x256 hole (25% of the image), marked by a gray mask; includes resizing, inference, and blending |
155156
| Text embeddings, privacy filter | Fixed prose, with the PII text seeded with the entity types a detector should find |
156157
| VAD, speech-to-text | 10 s of a voice-shaped waveform: a 130 Hz pulse train through three formant resonators, 0.7 s bursts alternating with 0.3 s of near-silence |
157158
| Text-to-speech | Fixed text; the voice is the alphabetically first the config publishes, so the choice is a property of the voice set rather than of declaration order |

‎apps/benchmarks/scripts/variant-sizes.json‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -491,5 +491,14 @@
491491
"https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/x/xnnpack/yolo26x_640_xnnpack_int8.pte": 66161216,
492492
"https://huggingface.co/software-mansion/react-native-executorch-zipdepth/resolve/v0.11.0/base/coreml/zipdepth_base_384_coreml_fp32.pte": 24858521,
493493
"https://huggingface.co/software-mansion/react-native-executorch-zipdepth/resolve/v0.11.0/base/vulkan/zipdepth_base_384_vulkan_fp16.pte": 12794626,
494-
"https://huggingface.co/software-mansion/react-native-executorch-zipdepth/resolve/v0.11.0/base/xnnpack/zipdepth_base_384_xnnpack_fp32.pte": 24609344
494+
"https://huggingface.co/software-mansion/react-native-executorch-zipdepth/resolve/v0.11.0/base/xnnpack/zipdepth_base_384_xnnpack_fp32.pte": 24609344,
495+
"https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/xnnpack/faces_256/migan_faces_256_xnnpack_fp16.pte": 24002700,
496+
"https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/xnnpack/512/migan_512_xnnpack_fp16.pte": 25181068,
497+
"https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/xnnpack/256/migan_256_xnnpack_fp16.pte": 24002700,
498+
"https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/vulkan/faces_256/migan_faces_256_vulkan_fp16.pte": 12411650,
499+
"https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/vulkan/512/migan_512_vulkan_fp16.pte": 14058114,
500+
"https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/vulkan/256/migan_256_vulkan_fp16.pte": 12411650,
501+
"https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/coreml/256/migan_256_coreml_fp16.pte": 12769586,
502+
"https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/coreml/512/migan_512_coreml_fp16.pte": 13968400,
503+
"https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/coreml/faces_256/migan_faces_256_coreml_fp16.pte": 12769586
495504
}

‎apps/benchmarks/src/drivers.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import {
1919
createDepthEstimator,
2020
createFsmnVoiceActivityDetector,
2121
createImageEmbedder,
22+
createInpainter,
2223
createInstanceSegmenter,
2324
createKeypointDetector,
2425
createKokoroTextToSpeech,
@@ -45,6 +46,7 @@ import RNBlobUtil from 'react-native-blob-util';
4546
import {
4647
IMAGE_512,
4748
IMAGE_640,
49+
INPAINTING_MASK,
4850
LLM_MAX_NEW_TOKENS,
4951
LLM_PROMPT,
5052
SAMPLE_PII_TEXT,
@@ -161,6 +163,14 @@ const depthEstimation: Driver = {
161163
},
162164
};
163165

166+
const inpainting: Driver = {
167+
create: createInpainter,
168+
run: (instance) => () => {
169+
'worklet';
170+
return instance.inpaintWorklet(IMAGE_512, INPAINTING_MASK).data.length;
171+
},
172+
};
173+
164174
const colbertEmbeddings: Driver = {
165175
create: createColbertEmbedder,
166176
run: (instance) => () => {
@@ -378,6 +388,7 @@ const BY_TASK: Readonly<Record<string, Driver>> = {
378388
imageEmbeddings,
379389
textEmbeddings,
380390
depthEstimation,
391+
inpainting,
381392
colbertEmbeddings,
382393
privacyFilter,
383394
voiceActivityDetection,

‎apps/benchmarks/src/inputs.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,13 @@ export const INPUT_SPEC_VERSION = 1;
201201
* construction into the numbers.
202202
*/
203203
export const IMAGE_512 = syntheticImage(512, 512);
204+
205+
/** Fixed central 256x256 hole, covering one quarter of the 512x512 frame. */
206+
export const INPAINTING_MASK: ImageBuffer & { readonly format: 'gray' } = (() => {
207+
const data = new Uint8Array(512 * 512);
208+
for (let y = 128; y < 384; y++) data.fill(255, y * 512 + 128, y * 512 + 384);
209+
return { data, width: 512, height: 512, format: 'gray', layout: 'hwc' };
210+
})();
204211
export const IMAGE_640 = syntheticImage(640, 640);
205212

206213
/** Fixed text for the text-to-speech pipelines. Long enough to span chunks. */

‎docs/docs/04-benchmarks/01-benchmarks.mdx‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,15 +39,15 @@ Explore the charts below to compare backends and find the optimal balance for yo
3939

4040
## Devices
4141

42-
Measurements were collected on physical iOS and Android devices spanning flagship, modern mid-tier, and legacy hardware tiers. All numbers come from release builds of React Native ExecuTorch 0.11.0-dev (`main` shortly after v0.10.0), measured in September 2026, with supplemental quantized LLM, depth estimation, and ColBERT runs in October 2026.
42+
Measurements were collected on physical iOS and Android devices spanning flagship, modern mid-tier, and legacy hardware tiers. All numbers come from release builds of React Native ExecuTorch 0.11.0-dev (`main` shortly after v0.10.0), measured in September 2026, with supplemental quantized LLM, depth estimation, ColBERT, and inpainting runs in October 2026.
4343

4444
| Device | Backends tested | Variants | Notes |
4545
|---|---|---|---|
4646
| Samsung Galaxy S26 Ultra | XNNPACK (121), Vulkan (24) | 145 | Snapdragon SM8850; historical standard-task results and 21 quantized LLM variants |
4747
| Google Pixel 10 | XNNPACK (100), Vulkan (24) | 124 | Google Tensor G5; historical standard-task measurements |
48-
| Samsung Galaxy S20+ | XNNPACK (129), Vulkan (31) | 160 | Exynos 990; historical results plus supplemental quantized LLM, depth, and ColBERT measurements |
49-
| Samsung Galaxy S24 | XNNPACK (29), Vulkan (6) | 35 | Supplemental quantized LLM, depth estimation, and ColBERT measurements |
50-
| iPhone 17 | XNNPACK (127), Core ML (78), MLX (38) | 243 | A19; historical results plus supplemental quantized LLM, depth, and ColBERT measurements |
48+
| Samsung Galaxy S20+ | XNNPACK (132), Vulkan (34) | 166 | Exynos 990; historical results plus supplemental quantized LLM, depth, ColBERT, and inpainting measurements |
49+
| Samsung Galaxy S24 | XNNPACK (32), Vulkan (9) | 41 | Supplemental quantized LLM, depth estimation, ColBERT, and inpainting measurements |
50+
| iPhone 17 | XNNPACK (130), Core ML (81), MLX (38) | 249 | A19; historical results plus supplemental quantized LLM, depth, ColBERT, and inpainting measurements |
5151
| iPhone SE (3rd gen) | XNNPACK (99), Core ML (73), MLX (15) | 187 | A15 Bionic with 4 GB RAM; historical standard-task measurements |
5252

5353
Memory values report peak process footprint during a separate inference pass, including the app baseline: total PSS on Android and `phys_footprint` on iOS. Some supplemental iPhone LLM measurements use a fresh process for each model.
@@ -126,6 +126,15 @@ These benchmarks measure execution latency, memory utilization, and initializati
126126

127127
<BenchmarkChart task="depth-estimation" data={benchmarks} />
128128

129+
### [Image Inpainting](../02-extensions/computer-vision/11-image-inpainting.md)
130+
131+
:::info Input & Pipeline Details
132+
- **Input:** Fixed 512x512 RGBA synthetic frame and a single-channel mask marking the central 256x256 square as the region to fill (25% of the image).
133+
- **Pipeline Structure:** MI-GAN resizes the image and mask to the model's 256x256 or 512x512 input, normalizes the image, and reconstructs the masked region. The output is resized back to 512x512 and blended with the original image through the mask. Timings include preprocessing, inference, and blending for the Places2 and Faces checkpoints.
134+
:::
135+
136+
<BenchmarkChart task="inpainting" data={benchmarks} />
137+
129138
### [Image Embeddings](../02-extensions/computer-vision/09-image-embeddings.md)
130139

131140
:::info Input & Pipeline Details

‎docs/docs/04-benchmarks/measured.json‎

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,88 @@
11
[
2+
{
3+
"task": "inpainting",
4+
"model": "migan-size-256",
5+
"backend": "coreml",
6+
"precision": "fp16",
7+
"device": "iphone17",
8+
"medianMs": 13.42,
9+
"meanMs": 13.46,
10+
"etMs": 12.43,
11+
"share": 92,
12+
"peakMb": 89,
13+
"loadMs": 41,
14+
"sizeMb": 12
15+
},
16+
{
17+
"task": "inpainting",
18+
"model": "migan-faces-256",
19+
"backend": "coreml",
20+
"precision": "fp16",
21+
"device": "iphone17",
22+
"medianMs": 13.34,
23+
"meanMs": 13.35,
24+
"etMs": 12.39,
25+
"share": 93,
26+
"peakMb": 81,
27+
"loadMs": 42,
28+
"sizeMb": 12
29+
},
30+
{
31+
"task": "inpainting",
32+
"model": "migan-size-512",
33+
"backend": "coreml",
34+
"precision": "fp16",
35+
"device": "iphone17",
36+
"medianMs": 31.95,
37+
"meanMs": 31.99,
38+
"etMs": 30.07,
39+
"share": 94,
40+
"peakMb": 96,
41+
"loadMs": 61,
42+
"sizeMb": 13
43+
},
44+
{
45+
"task": "inpainting",
46+
"model": "migan-size-256",
47+
"backend": "xnnpack",
48+
"precision": "fp16",
49+
"device": "iphone17",
50+
"medianMs": 45.79,
51+
"meanMs": 45.87,
52+
"etMs": 44.95,
53+
"share": 98,
54+
"peakMb": 138,
55+
"loadMs": 22,
56+
"sizeMb": 23
57+
},
58+
{
59+
"task": "inpainting",
60+
"model": "migan-faces-256",
61+
"backend": "xnnpack",
62+
"precision": "fp16",
63+
"device": "iphone17",
64+
"medianMs": 46.52,
65+
"meanMs": 46.5,
66+
"etMs": 45.58,
67+
"share": 98,
68+
"peakMb": 137,
69+
"loadMs": 21,
70+
"sizeMb": 23
71+
},
72+
{
73+
"task": "inpainting",
74+
"model": "migan-size-512",
75+
"backend": "xnnpack",
76+
"precision": "fp16",
77+
"device": "iphone17",
78+
"medianMs": 83.5,
79+
"meanMs": 83.13,
80+
"etMs": 81.68,
81+
"share": 98,
82+
"peakMb": 262,
83+
"loadMs": 24,
84+
"sizeMb": 24
85+
},
286
{
387
"task": "depth-estimation",
488
"model": "zipdepth-base",
@@ -741,6 +825,90 @@
741825
"loadMs": 2636,
742826
"sizeMb": 3482
743827
},
828+
{
829+
"task": "inpainting",
830+
"model": "migan-size-256",
831+
"backend": "vulkan",
832+
"precision": "fp16",
833+
"device": "s20plus",
834+
"medianMs": 250,
835+
"meanMs": 251.52,
836+
"etMs": 240.33,
837+
"share": 96,
838+
"peakMb": 210,
839+
"loadMs": 48,
840+
"sizeMb": 12
841+
},
842+
{
843+
"task": "inpainting",
844+
"model": "migan-faces-256",
845+
"backend": "vulkan",
846+
"precision": "fp16",
847+
"device": "s20plus",
848+
"medianMs": 248.69,
849+
"meanMs": 249.05,
850+
"etMs": 238.94,
851+
"share": 96,
852+
"peakMb": 190,
853+
"loadMs": 32,
854+
"sizeMb": 12
855+
},
856+
{
857+
"task": "inpainting",
858+
"model": "migan-size-512",
859+
"backend": "vulkan",
860+
"precision": "fp16",
861+
"device": "s20plus",
862+
"medianMs": 399.33,
863+
"meanMs": 400.49,
864+
"etMs": 380.53,
865+
"share": 95,
866+
"peakMb": 209,
867+
"loadMs": 40,
868+
"sizeMb": 13
869+
},
870+
{
871+
"task": "inpainting",
872+
"model": "migan-size-256",
873+
"backend": "xnnpack",
874+
"precision": "fp16",
875+
"device": "s20plus",
876+
"medianMs": 257.6,
877+
"meanMs": 261.41,
878+
"etMs": 253.74,
879+
"share": 97,
880+
"peakMb": 277,
881+
"loadMs": 49,
882+
"sizeMb": 23
883+
},
884+
{
885+
"task": "inpainting",
886+
"model": "migan-faces-256",
887+
"backend": "xnnpack",
888+
"precision": "fp16",
889+
"device": "s20plus",
890+
"medianMs": 262.43,
891+
"meanMs": 263.2,
892+
"etMs": 254.68,
893+
"share": 97,
894+
"peakMb": 283,
895+
"loadMs": 49,
896+
"sizeMb": 23
897+
},
898+
{
899+
"task": "inpainting",
900+
"model": "migan-size-512",
901+
"backend": "xnnpack",
902+
"precision": "fp16",
903+
"device": "s20plus",
904+
"medianMs": 364.97,
905+
"meanMs": 364.18,
906+
"etMs": 356.45,
907+
"share": 98,
908+
"peakMb": 416,
909+
"loadMs": 49,
910+
"sizeMb": 24
911+
},
744912
{
745913
"task": "depth-estimation",
746914
"model": "zipdepth-base",
@@ -1245,6 +1413,90 @@
12451413
"loadMs": 17602,
12461414
"sizeMb": 3661
12471415
},
1416+
{
1417+
"task": "inpainting",
1418+
"model": "migan-size-256",
1419+
"backend": "vulkan",
1420+
"precision": "fp16",
1421+
"device": "s24",
1422+
"medianMs": 90.57,
1423+
"meanMs": 91.49,
1424+
"etMs": 84.26,
1425+
"share": 92,
1426+
"peakMb": 349,
1427+
"loadMs": 26,
1428+
"sizeMb": 12
1429+
},
1430+
{
1431+
"task": "inpainting",
1432+
"model": "migan-faces-256",
1433+
"backend": "vulkan",
1434+
"precision": "fp16",
1435+
"device": "s24",
1436+
"medianMs": 90.9,
1437+
"meanMs": 91.6,
1438+
"etMs": 84.59,
1439+
"share": 92,
1440+
"peakMb": 329,
1441+
"loadMs": 37,
1442+
"sizeMb": 12
1443+
},
1444+
{
1445+
"task": "inpainting",
1446+
"model": "migan-size-512",
1447+
"backend": "vulkan",
1448+
"precision": "fp16",
1449+
"device": "s24",
1450+
"medianMs": 175.22,
1451+
"meanMs": 174.78,
1452+
"etMs": 165.84,
1453+
"share": 95,
1454+
"peakMb": 438,
1455+
"loadMs": 45,
1456+
"sizeMb": 13
1457+
},
1458+
{
1459+
"task": "inpainting",
1460+
"model": "migan-size-256",
1461+
"backend": "xnnpack",
1462+
"precision": "fp16",
1463+
"device": "s24",
1464+
"medianMs": 176.79,
1465+
"meanMs": 185.28,
1466+
"etMs": 181.29,
1467+
"share": 98,
1468+
"peakMb": 287,
1469+
"loadMs": 46,
1470+
"sizeMb": 23
1471+
},
1472+
{
1473+
"task": "inpainting",
1474+
"model": "migan-faces-256",
1475+
"backend": "xnnpack",
1476+
"precision": "fp16",
1477+
"device": "s24",
1478+
"medianMs": 188.48,
1479+
"meanMs": 188.45,
1480+
"etMs": 184.98,
1481+
"share": 98,
1482+
"peakMb": 296,
1483+
"loadMs": 48,
1484+
"sizeMb": 23
1485+
},
1486+
{
1487+
"task": "inpainting",
1488+
"model": "migan-size-512",
1489+
"backend": "xnnpack",
1490+
"precision": "fp16",
1491+
"device": "s24",
1492+
"medianMs": 235.15,
1493+
"meanMs": 234.74,
1494+
"etMs": 230.79,
1495+
"share": 98,
1496+
"peakMb": 436,
1497+
"loadMs": 29,
1498+
"sizeMb": 24
1499+
},
12481500
{
12491501
"task": "depth-estimation",
12501502
"model": "zipdepth-base",

0 commit comments

Comments
 (0)