Skip to content

Commit b098b90

Browse files
authored
feat: add depth estimation task implementation (#1552)
## Description This PR introduces new depth estimation task implementation along with a test suite and a simple example screen in computer vision app to test the new feature end-to-end. Alongside it we upload 3 depth estimation models to SWM HuggingFace repo to use with this new task with different performance/quality tradeoffs; all accessible in the new collection https://huggingface.co/collections/software-mansion/depth-estimation. ### Introduces a breaking change? - [ ] Yes - [x] No ### Type of change - [ ] Bug fix (change which fixes an issue) - [x] New feature (change which adds functionality) - [ ] Documentation update (improves or adds clarity to existing documentation) - [ ] Other (chores, tests, code style improvements etc.) ### Tested on - [x] iOS - [x] Android ### Testing instructions - [ ] Run new example screen on device (both Android and iOS) and test newly added models. - [ ] Check the HF repos from the linked collection https://huggingface.co/collections/software-mansion/depth-estimation, ### Screenshots <!-- Add screenshots here, if applicable --> ### Related issues Closes #1372 ### Checklist - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have updated the documentation accordingly - [x] My changes generate no new warnings ### Additional notes <!-- Include any additional information, assumptions, or context that reviewers might need to understand this PR. -->
1 parent 64677dd commit b098b90

18 files changed

Lines changed: 1309 additions & 1 deletion

File tree

‎.cspell-wordlist.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -423,3 +423,4 @@ xcrun
423423
denoiser
424424
imageembeddings
425425
unparseable
426+
zipdepth

‎apps/computer-vision/app/_layout.tsx‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,13 @@ export default function Layout() {
6969
title: 'Keypoint Detection',
7070
}}
7171
/>
72+
<Drawer.Screen
73+
name="depthEstimation/index"
74+
options={{
75+
drawerLabel: 'Depth Estimation',
76+
title: 'Depth Estimation',
77+
}}
78+
/>
7279
<Drawer.Screen
7380
name="instanceSegmentation/index"
7481
options={{
Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
import React, { useEffect, useRef, useState } from 'react';
2+
import { View, Text, ScrollView, Platform } from 'react-native';
3+
import { useSafeAreaInsets } from 'react-native-safe-area-context';
4+
import { AlphaType, ColorType, Skia, useImage, type SkImage } from '@shopify/react-native-skia';
5+
import {
6+
useDepthEstimator,
7+
models,
8+
type DepthMap,
9+
type DepthEstimatorModel,
10+
} from 'react-native-executorch';
11+
import ScreenWrapper from '../../components/ScreenWrapper';
12+
import { ImageViewport } from '../../components/ImageViewport';
13+
import { DepthPhoto } from '../../components/DepthPhoto';
14+
import { ModelPicker, type ModelOption } from '../../components/ModelPicker';
15+
import { ModelStatus } from '../../components/ModelStatus';
16+
import { LatencyIndicator } from '../../components/LatencyIndicator';
17+
import { Button } from '../../components/Button';
18+
import { commonStyles, theme } from '../../theme';
19+
import { getImage, skImageToBuffer } from '../../utils';
20+
21+
const DEPTH_MODELS = models.depthEstimation.DEPTH_ANYTHING_V2_SMALL;
22+
const DEPTH_V3_MODELS = models.depthEstimation.DEPTH_ANYTHING_V3_SMALL;
23+
const ZIPDEPTH_MODELS = models.depthEstimation.ZIPDEPTH_BASE;
24+
type Backend = 'xnnpack' | 'coreml' | 'vulkan';
25+
type DepthModelOption = ModelOption & { family: 'v2' | 'v3' | 'zipdepth' };
26+
const BACKEND_OPTIONS: ModelOption[] = [
27+
{ label: 'XNNPACK', value: 'xnnpack' },
28+
{ label: 'Core ML', value: 'coreml', disabled: Platform.OS !== 'ios' },
29+
{ label: 'Vulkan', value: 'vulkan', disabled: Platform.OS !== 'android' },
30+
];
31+
const MODEL_OPTIONS: Record<Backend, DepthModelOption[]> = {
32+
xnnpack: [
33+
{ label: 'Depth Anything V2 Small (INT8)', family: 'v2', value: DEPTH_MODELS.XNNPACK_INT8 },
34+
{ label: 'Depth Anything V2 Small (FP32)', family: 'v2', value: DEPTH_MODELS.XNNPACK_FP32 },
35+
{ label: 'Depth Anything V3 Small (INT8)', family: 'v3', value: DEPTH_V3_MODELS.XNNPACK_INT8 },
36+
{ label: 'Depth Anything V3 Small (FP32)', family: 'v3', value: DEPTH_V3_MODELS.XNNPACK_FP32 },
37+
{ label: 'ZipDepth Base (FP32)', family: 'zipdepth', value: ZIPDEPTH_MODELS.XNNPACK_FP32 },
38+
],
39+
coreml: [
40+
{ label: 'Depth Anything V2 Small (FP16)', family: 'v2', value: DEPTH_MODELS.COREML_FP16 },
41+
{ label: 'Depth Anything V3 Small (FP16)', family: 'v3', value: DEPTH_V3_MODELS.COREML_FP16 },
42+
{ label: 'ZipDepth Base (FP32)', family: 'zipdepth', value: ZIPDEPTH_MODELS.COREML_FP32 },
43+
],
44+
vulkan: [
45+
{ label: 'Depth Anything V2 Small (FP16)', family: 'v2', value: DEPTH_MODELS.VULKAN_FP16 },
46+
{ label: 'Depth Anything V3 Small (FP16)', family: 'v3', value: DEPTH_V3_MODELS.VULKAN_FP16 },
47+
{ label: 'ZipDepth Base (FP16)', family: 'zipdepth', value: ZIPDEPTH_MODELS.VULKAN_FP16 },
48+
],
49+
};
50+
51+
type DepthPreview = { image: SkImage; min: number; max: number };
52+
53+
// Display conversion belongs to the demo; the library retains the raw float map.
54+
function createDepthPreview(depth: DepthMap, isDepth: boolean): DepthPreview {
55+
// Effects use inverse depth so near surfaces retain contrast across model families.
56+
const values = isDepth
57+
? depth.data.map((value) => (Number.isFinite(value) && value > 0 ? 1 / value : NaN))
58+
: depth.data;
59+
let rawMin = Infinity;
60+
let rawMax = -Infinity;
61+
let min = Infinity;
62+
let max = -Infinity;
63+
for (let i = 0; i < values.length; i++) {
64+
const raw = depth.data[i]!;
65+
if (Number.isFinite(raw)) {
66+
rawMin = Math.min(rawMin, raw);
67+
rawMax = Math.max(rawMax, raw);
68+
}
69+
const value = values[i]!;
70+
if (!Number.isFinite(value)) continue;
71+
min = Math.min(min, value);
72+
max = Math.max(max, value);
73+
}
74+
if (!Number.isFinite(min))
75+
throw new Error('The model returned no valid depth values for the preview.');
76+
const range = max - min;
77+
const rgba = new Uint8Array(depth.width * depth.height * 4);
78+
for (let i = 0; i < depth.data.length; i++) {
79+
const value = values[i]!;
80+
const normalized = range > 0 && Number.isFinite(value) ? (value - min) / range : 0;
81+
const gray = Math.round(normalized * 255);
82+
rgba[i * 4] = gray;
83+
rgba[i * 4 + 1] = gray;
84+
rgba[i * 4 + 2] = gray;
85+
rgba[i * 4 + 3] = 255;
86+
}
87+
const image = Skia.Image.MakeImage(
88+
{
89+
width: depth.width,
90+
height: depth.height,
91+
colorType: ColorType.RGBA_8888,
92+
alphaType: AlphaType.Opaque,
93+
},
94+
Skia.Data.fromBytes(rgba),
95+
depth.width * 4
96+
);
97+
if (!image) throw new Error('Failed to render the depth preview.');
98+
return { image, min: rawMin, max: rawMax };
99+
}
100+
101+
function DepthEstimationContent() {
102+
const insets = useSafeAreaInsets();
103+
const [selectedBackend, setSelectedBackend] = useState<Backend>('xnnpack');
104+
const [selectedModel, setSelectedModel] = useState<DepthEstimatorModel>(
105+
MODEL_OPTIONS.xnnpack[0]!.value
106+
);
107+
const [busy, setBusy] = useState(false);
108+
const [imageUri, setImageUri] = useState<string | null>(null);
109+
const [preview, setPreview] = useState<DepthPreview | null>(null);
110+
const [latency, setLatency] = useState<number | null>(null);
111+
const [error, setError] = useState<string | null>(null);
112+
const mounted = useRef(true);
113+
useEffect(() => {
114+
mounted.current = true;
115+
return () => {
116+
mounted.current = false;
117+
};
118+
}, []);
119+
120+
const selectedOption = MODEL_OPTIONS[selectedBackend].find(
121+
(option) => option.value === selectedModel
122+
)!;
123+
const image = useImage(imageUri, (failure) => setError(failure.message));
124+
const estimator = useDepthEstimator(selectedModel);
125+
126+
const selectModel = (model: DepthEstimatorModel) => {
127+
if (busy || model === selectedModel) return;
128+
setSelectedModel(model);
129+
setPreview(null);
130+
setLatency(null);
131+
setError(null);
132+
};
133+
134+
const selectBackend = (backend: Backend) => {
135+
if (busy || backend === selectedBackend) return;
136+
const nextModel =
137+
MODEL_OPTIONS[backend].find((option) => option.family === selectedOption.family) ??
138+
MODEL_OPTIONS[backend][0]!;
139+
setSelectedBackend(backend);
140+
selectModel(nextModel.value);
141+
};
142+
143+
const pickImage = async (camera: boolean) => {
144+
setBusy(true);
145+
setError(null);
146+
try {
147+
const uri = await getImage(camera);
148+
if (uri && mounted.current) {
149+
setImageUri(uri);
150+
setPreview(null);
151+
setLatency(null);
152+
}
153+
} catch (failure) {
154+
if (mounted.current) setError(String(failure));
155+
} finally {
156+
if (mounted.current) setBusy(false);
157+
}
158+
};
159+
160+
const runDepthEstimation = async (sync: boolean) => {
161+
if (!image || !estimator.estimateDepth || !estimator.estimateDepthWorklet || busy) return;
162+
setBusy(true);
163+
setError(null);
164+
try {
165+
const input = skImageToBuffer(image);
166+
const start = Date.now();
167+
const depth = sync
168+
? estimator.estimateDepthWorklet(input)
169+
: await estimator.estimateDepth(input);
170+
const elapsed = Date.now() - start;
171+
if (!mounted.current) return;
172+
setPreview(createDepthPreview(depth, selectedOption.family === 'v3'));
173+
setLatency(elapsed);
174+
} catch (failure) {
175+
if (mounted.current) setError(String(failure));
176+
} finally {
177+
if (mounted.current) setBusy(false);
178+
}
179+
};
180+
181+
return (
182+
<ScrollView
183+
style={commonStyles.container}
184+
contentContainerStyle={[
185+
commonStyles.contentContainer,
186+
{ paddingBottom: insets.bottom + theme.spacing.large },
187+
]}
188+
>
189+
<Text style={commonStyles.description}>
190+
Turn a photo into a moving scene. Estimate depth once, then explore perspective, focus, and
191+
the space between subjects.
192+
</Text>
193+
<ModelPicker
194+
label="Backend"
195+
options={BACKEND_OPTIONS}
196+
selectedValue={selectedBackend}
197+
onValueChange={selectBackend}
198+
/>
199+
<ModelPicker
200+
label="Model"
201+
options={MODEL_OPTIONS[selectedBackend]}
202+
selectedValue={selectedModel}
203+
onValueChange={selectModel}
204+
/>
205+
<ModelStatus
206+
isReady={estimator.isReady}
207+
downloadProgress={estimator.downloadProgress}
208+
error={estimator.error ? String(estimator.error) : error}
209+
modelTypeLabel="depth model"
210+
/>
211+
{preview && image ? (
212+
<DepthPhoto key={imageUri} image={image} depthTexture={preview.image} />
213+
) : (
214+
<ImageViewport
215+
skiaImage={image}
216+
onPressPlaceholder={() => {
217+
if (!busy) pickImage(false);
218+
}}
219+
/>
220+
)}
221+
<View style={commonStyles.buttonRow}>
222+
<Button
223+
title="Gallery"
224+
onPress={() => pickImage(false)}
225+
disabled={busy}
226+
variant="secondary"
227+
/>
228+
<Button
229+
title="Camera"
230+
onPress={() => pickImage(true)}
231+
disabled={busy}
232+
variant="secondary"
233+
/>
234+
</View>
235+
<View style={commonStyles.buttonRow}>
236+
<Button
237+
title="Run Async"
238+
onPress={() => runDepthEstimation(false)}
239+
disabled={!image || !estimator.isReady || busy}
240+
/>
241+
<Button
242+
title="Run Sync"
243+
onPress={() => runDepthEstimation(true)}
244+
disabled={!image || !estimator.isReady || busy}
245+
variant="accent"
246+
/>
247+
</View>
248+
{preview && (
249+
<Text style={commonStyles.description}>
250+
Raw relative depth: {preview.min.toFixed(3)}–{preview.max.toFixed(3)}. Values are not
251+
meters.
252+
</Text>
253+
)}
254+
<LatencyIndicator latency={latency} />
255+
</ScrollView>
256+
);
257+
}
258+
259+
export default function DepthEstimationScreen() {
260+
return (
261+
<ScreenWrapper>
262+
<DepthEstimationContent />
263+
</ScreenWrapper>
264+
);
265+
}

‎apps/computer-vision/app/index.tsx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ export default function Home() {
3636
<TouchableOpacity style={styles.button} onPress={() => router.navigate('segmentation/')}>
3737
<Text style={styles.buttonText}>Semantic Segmentation</Text>
3838
</TouchableOpacity>
39+
<TouchableOpacity style={styles.button} onPress={() => router.navigate('depthEstimation/')}>
40+
<Text style={styles.buttonText}>Depth Estimation</Text>
41+
</TouchableOpacity>
3942
<TouchableOpacity
4043
style={styles.button}
4144
onPress={() => router.navigate('instanceSegmentation/')}

0 commit comments

Comments
 (0)