From efdb33f682ac4be844f0a84de433c4fce0abd170 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 26 Oct 2026 20:52:57 +0100 Subject: [PATCH 1/4] Complete quantized LLM, depth and ColBERT device benchmarks --- apps/benchmarks/BENCHMARK_SPEC.md | 25 +- apps/benchmarks/README.md | 21 +- apps/benchmarks/app.json | 9 +- apps/benchmarks/package.json | 20 +- .../plugins/withBenchmarkBundleInputs.js | 18 + apps/benchmarks/scripts/docs-data.mjs | 24 +- apps/benchmarks/scripts/docs-data.test.mjs | 86 + .../scripts/fetch-variant-sizes.mjs | 12 + apps/benchmarks/scripts/llm-driver.test.mjs | 136 ++ apps/benchmarks/scripts/variant-sizes.json | 382 ++-- apps/benchmarks/src/drivers.ts | 139 +- docs/docs/04-benchmarks/01-benchmarks.mdx | 43 +- docs/docs/04-benchmarks/imported.json | 28 - docs/docs/04-benchmarks/measured.json | 1739 ++++++++++++++++- .../components/Benchmarks/BenchmarkChart.tsx | 2 +- docs/src/data/benchmarks.ts | 17 +- yarn.lock | 245 +-- 17 files changed, 2430 insertions(+), 516 deletions(-) create mode 100644 apps/benchmarks/plugins/withBenchmarkBundleInputs.js create mode 100644 apps/benchmarks/scripts/docs-data.test.mjs create mode 100644 apps/benchmarks/scripts/llm-driver.test.mjs diff --git a/apps/benchmarks/BENCHMARK_SPEC.md b/apps/benchmarks/BENCHMARK_SPEC.md index d32e18a969..1621627ca8 100644 --- a/apps/benchmarks/BENCHMARK_SPEC.md +++ b/apps/benchmarks/BENCHMARK_SPEC.md @@ -39,9 +39,9 @@ Per variant, per repeat: | `memory.disposed` | Footprint after `dispose` — a leak is a case that never returns to baseline | `memory.*` is process footprint, not native heap: `phys_footprint` on iOS (what -jetsam measures an app against) and total PSS on Android. Both count the -resident pages of a memory-mapped `.pte`, which the heap counters miss entirely -and which is most of a model's cost. +jetsam measures an app against) and total PSS on Android. Raw results record the +process baseline separately, so process memory can be distinguished from a +model's incremental footprint. Memory is sampled in a separate pass from the timings. Reading total PSS on Android walks `/proc/self/smaps` and costs milliseconds; polling that during a @@ -148,14 +148,15 @@ would move for reasons unrelated to the device. `src/inputs.ts` is the definition; `INPUT_SPEC_VERSION` is recorded in every report and the comparator refuses to diff across a change to it. -| Task | Input | -| -------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -| Vision (classification, detection, segmentation, style transfer, OCR, keypoints, image embeddings) | A fixed 512x512 or 640x640 scene: a vertical gradient, three solid ellipses, and a seeded dither | -| Text embeddings, privacy filter | Fixed prose, with the PII text seeded with the entity types a detector should find | -| VAD, speech-to-text | 10 s of a voice-shaped waveform: a 130 Hz pulse train through three formant resonators, 0.7 s bursts alternating with 0.3 s of near-silence | -| Text-to-speech | Fixed text; the voice is the alphabetically first the config publishes, so the choice is a property of the voice set rather than of declaration order | -| Text-to-image | Fixed prompt, fixed seed | -| LLM | Fixed short prompt, **64 tokens decoded with EOS ignored and temperature 0** | +| Task | Input | +| -------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| Vision (classification, depth estimation, detection, segmentation, style transfer, OCR, keypoints, image embeddings) | A fixed 512x512 or 640x640 scene: a vertical gradient, three solid ellipses, and a seeded dither | +| ColBERT embeddings | The fixed text-embedding paragraph, encoded as a document; query encoding and MaxSim scoring are not timed | +| Text embeddings, privacy filter | Fixed prose, with the PII text seeded with the entity types a detector should find | +| VAD, speech-to-text | 10 s of a voice-shaped waveform: a 130 Hz pulse train through three formant resonators, 0.7 s bursts alternating with 0.3 s of near-silence | +| Text-to-speech | Fixed text; the voice is the alphabetically first the config publishes, so the choice is a property of the voice set rather than of declaration order | +| Text-to-image | Fixed prompt, fixed seed | +| LLM | Fixed short prompt, **64 tokens decoded with EOS ignored and temperature 0** | Two caveats worth knowing before reading the output: @@ -166,7 +167,7 @@ Two caveats worth knowing before reading the output: - **LLMs.** Decode is pinned to a fixed token count because generation length is otherwise a property of the model and its quantisation, and a wall-clock figure over a varying token count compares nothing. Divide `pipeline.median` - by 64 for milliseconds per token; `detail.timeToFirstTokenMs` and + by the recorded generated-token count (`units`) for milliseconds per token; `detail.timeToFirstTokenMs` and `detail.prefillMs` are reported separately. Multimodal models are driven text-only. diff --git a/apps/benchmarks/README.md b/apps/benchmarks/README.md index 0edf2a7dab..574852c982 100644 --- a/apps/benchmarks/README.md +++ b/apps/benchmarks/README.md @@ -173,8 +173,8 @@ post-processing in it. ## Publishing to the docs -The docs benchmarks page is built from raw runs stored beside it, so each docs -version keeps the runs its numbers came from: +The docs benchmarks page reads `measured.json`; raw runs are only needed to +regenerate it. Temporarily collect the runs under the page directory: ```bash mkdir -p ../../docs/docs/04-benchmarks/runs/s26ultra @@ -183,12 +183,15 @@ yarn bench:docs # regenerates docs/docs/04-benchmarks/measured.json ``` The directory name is the device id the docs list it under (`DEVICES` in -`docs/src/data/benchmarks.ts`). A device with runs replaces its rows in -`imported.json`, which holds published numbers that have no raw runs behind -them. To check a build against a release, compare against its archived runs: +`docs/src/data/benchmarks.ts`). Generation replaces `measured.json`, so include +all runs whose measurements it should retain. Archive the raw runs outside the +repository and remove the temporary `runs/` directory before committing; the +docs build only needs the generated JSON. Measured variants replace matching +rows in `imported.json`, preserving other published results. To check a build +against a release, compare against its archived runs: ```bash -yarn bench:compare ../../docs/versioned_docs/version-0.11.0/04-benchmarks/runs/s26ultra/.jsonl results/.jsonl +yarn bench:compare /path/to/archived/.jsonl results/.jsonl ``` ## What gets measured @@ -322,6 +325,6 @@ a **model or variant** means nothing here beyond `yarn bench:sizes`. `modules/bench-probe` is a local Expo module reading the process footprint: `task_vm_info.phys_footprint` on iOS (what jetsam measures an app against) and -total PSS on Android. Both count the resident pages of a memory-mapped `.pte`, -which the native-heap counters miss entirely — and that is most of a model's -footprint. +total PSS on Android. These are process-footprint counters, rather than +native-heap totals. The raw results also record the baseline before loading and +the footprint after disposal. diff --git a/apps/benchmarks/app.json b/apps/benchmarks/app.json index f9a3ad3315..6b698c0b96 100644 --- a/apps/benchmarks/app.json +++ b/apps/benchmarks/app.json @@ -29,8 +29,7 @@ "foregroundImage": "./assets/icons/adaptive-icon.png", "backgroundColor": "#ffffff" }, - "package": "com.anonymous.benchmarks", - "usesCleartextTraffic": true + "package": "com.anonymous.benchmarks" }, "web": { "favicon": "./assets/icons/favicon.png" @@ -40,13 +39,15 @@ "expo-build-properties", { "android": { - "minSdkVersion": 26 + "minSdkVersion": 26, + "usesCleartextTraffic": true }, "ios": { "deploymentTarget": "17.0" } } - ] + ], + "./plugins/withBenchmarkBundleInputs" ] } } diff --git a/apps/benchmarks/package.json b/apps/benchmarks/package.json index c5718beaec..c8bb0e960a 100644 --- a/apps/benchmarks/package.json +++ b/apps/benchmarks/package.json @@ -24,25 +24,27 @@ "bench:compare": "node scripts/compare.mjs", "bench:summary": "node scripts/summarize.mjs", "bench:sizes": "node scripts/fetch-variant-sizes.mjs", - "bench:docs": "node scripts/docs-data.mjs" + "bench:docs": "node scripts/docs-data.mjs", + "test": "node --test scripts/*.test.mjs" }, "dependencies": { - "expo": "~56.0.9", - "expo-build-properties": "~56.0.17", - "expo-keep-awake": "~56.0.3", + "expo": "~57.0.28", + "expo-build-properties": "~57.0.22", + "expo-keep-awake": "~57.0.2", "react": "19.2.3", - "react-native": "0.85.3", + "react-native": "0.86.3", "react-native-blob-util": "^0.25.0", "react-native-executorch": "workspace:*", "react-native-safe-area-context": "~5.7.0", - "react-native-worklets": "0.10.3" + "react-native-worklets": "0.13.0" }, "devDependencies": { - "@babel/core": "^7.29.0", + "@babel/core": "^7.29.7", "@react-native/metro-config": "^0.86.0", "@types/react": "~19.2.0", - "babel-preset-expo": "~56.0.14", - "react-refresh": "^0.14.0" + "babel-preset-expo": "~57.0.15", + "react-refresh": "^0.14.0", + "typescript": "~5.9.2" }, "private": true } diff --git a/apps/benchmarks/plugins/withBenchmarkBundleInputs.js b/apps/benchmarks/plugins/withBenchmarkBundleInputs.js new file mode 100644 index 0000000000..29fb63d4cc --- /dev/null +++ b/apps/benchmarks/plugins/withBenchmarkBundleInputs.js @@ -0,0 +1,18 @@ +const { withAppBuildGradle } = require('expo/config-plugins'); + +// React Native's bundle task tracks source files, but Expo's inlined benchmark +// settings come from the environment. Give Gradle those inputs as well so a +// release APK cannot reuse another device's collector or validation settings. +module.exports = (config) => + withAppBuildGradle(config, (modConfig) => { + const marker = '// Benchmark environment inputs'; + if (!modConfig.modResults.contents.includes(marker)) { + modConfig.modResults.contents += ` +${marker} +tasks.withType(com.facebook.react.tasks.BundleHermesCTask).configureEach { + inputs.property("benchmarkEnvironment", providers.environmentVariablesPrefixedBy("EXPO_PUBLIC_BENCH_")) +} +`; + } + return modConfig; + }); diff --git a/apps/benchmarks/scripts/docs-data.mjs b/apps/benchmarks/scripts/docs-data.mjs index 6707e104c6..a42c015a04 100644 --- a/apps/benchmarks/scripts/docs-data.mjs +++ b/apps/benchmarks/scripts/docs-data.mjs @@ -2,15 +2,14 @@ /** * Builds the docs benchmark data from raw runs. * - * The raw runs are the source of truth and live beside the docs page, one + * To generate data, collect raw runs beside the docs page, one * directory per device: `runs//*.jsonl`, where `` is the - * id the docs list the device under. A docs version cut copies them along with - * the page, so every release keeps the runs its numbers came from, and - * `bench:compare` can diff a new run against any of them. + * id the docs list the device under. Only the generated JSON is needed by the + * docs build; archive raw runs outside the repository after generation. * * This writes `measured.json` beside them. The page merges it with * `imported.json`, the published rows that have no raw runs behind them: a - * device with runs here replaces its imported rows entirely. + * measured variant replaces its matching imported row. * * Usage: * yarn bench:docs # ../../docs/docs/04-benchmarks @@ -67,8 +66,19 @@ function toRow(deviceId, entries) { }; if (task === 'llm') { - // `units` is the runner's generated-token count over a pinned-length - // generation: 63 for 64 new tokens, the first coming from prefill. + if ( + entries.some( + (entry) => + !Number.isFinite(entry.units) || + entry.units <= 0 || + !Number.isFinite(entry.detail?.timeToFirstTokenMs) || + entry.detail.timeToFirstTokenMs < 0 + ) + ) { + throw new Error(`LLM measurement ${first.id} is missing valid token or TTFT stats`); + } + // Use the runner's actual generated-token count for the fixed-length + // generation; older runtime exports counted the first token separately. row.tokensPerSecond = round(first.units / (medianMs / 1000), 1); row.ttftMs = round(pick((entry) => entry.detail?.timeToFirstTokenMs)); } else { diff --git a/apps/benchmarks/scripts/docs-data.test.mjs b/apps/benchmarks/scripts/docs-data.test.mjs new file mode 100644 index 0000000000..e5cade575e --- /dev/null +++ b/apps/benchmarks/scripts/docs-data.test.mjs @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { runInNewContext } from 'node:vm'; +import { ModuleKind, ScriptTarget, transpileModule } from 'typescript'; + +test('partial device runs replace matching variants and retain other published results', () => { + const exports = {}; + const source = readFileSync( + new URL('../../../docs/src/data/benchmarks.ts', import.meta.url), + 'utf8' + ); + const code = transpileModule(source, { + compilerOptions: { module: ModuleKind.CommonJS, target: ScriptTarget.ES2022 }, + }).outputText; + runInNewContext(code, { exports }); + const old = { + device: 's20plus', + task: 'llm', + model: 'test', + backend: 'xnnpack', + precision: '8da4w', + medianMs: 10, + }; + const fresh = { ...old, medianMs: 5 }; + const otherDevice = { ...old, device: 's26ultra' }; + const otherTask = { ...old, task: 'classification' }; + const otherBackend = { ...old, backend: 'vulkan' }; + const otherPrecision = { ...old, precision: '8da8w' }; + const rows = exports.mergeBenchmarkRows( + [fresh], + [old, otherDevice, otherTask, otherBackend, otherPrecision] + ); + assert.equal(rows.length, 5); + assert.equal(rows[0], fresh); + assert.equal(rows[1], otherDevice); + assert.equal(rows[2], otherTask); + assert.equal(rows[3], otherBackend); + assert.equal(rows[4], otherPrecision); +}); + +test('docs generation publishes valid LLM throughput and rejects missing token stats', () => { + const page = mkdtempSync(join(tmpdir(), 'rne-docs-test-')); + try { + const dir = join(page, 'runs', 'iphone17'); + mkdirSync(dir, { recursive: true }); + const file = join(dir, 'test.jsonl'); + const header = { type: 'run', label: 'test', device: { model: 'test' } }; + const entry = { + id: 'llm/test-xnnpack-8-da4-w', + status: 'ok', + backend: 'xnnpack', + precision: '8da4w', + pipeline: { mean: 100, median: 100 }, + units: 64, + detail: { timeToFirstTokenMs: 5 }, + taskLoadMs: 10, + bytes: 1024 ** 2, + memory: { peakMb: 20 }, + }; + const convert = () => + spawnSync( + process.execPath, + [fileURLToPath(new URL('./docs-data.mjs', import.meta.url)), page], + { encoding: 'utf8' } + ); + const write = (measurement) => + writeFileSync(file, `${JSON.stringify(header)}\n${JSON.stringify(measurement)}\n`); + write(entry); + const valid = convert(); + assert.equal(valid.status, 0, valid.stderr); + const [row] = JSON.parse(readFileSync(join(page, 'measured.json'), 'utf8')); + assert.equal(row.tokensPerSecond, 640); + assert.equal(row.ttftMs, 5); + write({ ...entry, units: undefined }); + const invalid = convert(); + assert.equal(invalid.status, 2); + assert.match(invalid.stderr, /missing valid token or TTFT stats/); + } finally { + rmSync(page, { recursive: true, force: true }); + } +}); diff --git a/apps/benchmarks/scripts/fetch-variant-sizes.mjs b/apps/benchmarks/scripts/fetch-variant-sizes.mjs index 149205e42b..dd00333625 100644 --- a/apps/benchmarks/scripts/fetch-variant-sizes.mjs +++ b/apps/benchmarks/scripts/fetch-variant-sizes.mjs @@ -51,6 +51,10 @@ function loadRegistry() { ) .replace("} from './extensions/speech/tasks/whisperSpeechToText';", "} from './_stub.ts';") .replace("} from './constants';", "} from './constants.ts';") + .replace( + "import { registerDeprecation } from './deprecation';", + "import { registerDeprecation } from './_stub.ts';" + ) // Type-only imports name files that are not copied into the scratch dir. // Node strips the annotations but still resolves the specifier, so they // have to go. @@ -70,6 +74,7 @@ function loadRegistry() { "export const getRegisteredBackends = () => ['XnnpackBackend', 'CoreMLBackend', 'MLXBackend', 'VulkanBackend'];", "export const WHISPER_LANGUAGES = ['en'];", 'export const WhisperSttModel = undefined;', + 'export const registerDeprecation = () => {};', '', ].join('\n') ); @@ -119,12 +124,19 @@ async function measure(urls, cached) { for (let url = queue.pop(); url !== undefined; url = queue.pop()) { try { const response = await fetch(url, { method: 'HEAD', redirect: 'follow' }); + if (!response.ok) throw new Error(`HTTP ${response.status} for ${url}`); // Hugging Face serves LFS objects through a redirect and reports the // real object size in this header; Content-Length on the redirect hop // describes the pointer, not the file. const linked = response.headers.get('x-linked-size'); const length = response.headers.get('content-length'); sizes[url] = Number(linked ?? length ?? 0) || 0; + // Small JSON resources can be served without Content-Length. + if (sizes[url] === 0 && new URL(url).pathname.endsWith('.json')) { + const body = await fetch(url); + if (!body.ok) throw new Error(`HTTP ${body.status} for ${url}`); + sizes[url] = (await body.arrayBuffer()).byteLength; + } } catch { sizes[url] = 0; } diff --git a/apps/benchmarks/scripts/llm-driver.test.mjs b/apps/benchmarks/scripts/llm-driver.test.mjs new file mode 100644 index 0000000000..b670392aa5 --- /dev/null +++ b/apps/benchmarks/scripts/llm-driver.test.mjs @@ -0,0 +1,136 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { test } from 'node:test'; +import { runInNewContext } from 'node:vm'; +import { ModuleKind, ScriptTarget, transpileModule } from 'typescript'; + +// Load the real driver without booting React Native. The doubles deliberately +// expose only the current runner stats, so obsolete stats fields fail the test. +function fixture() { + const state = { resets: 0, clears: 0, disposed: 0, pos: 0, tokens: 64, promptTokens: 12 }; + const preprocessor = { + process(history, lastKMessages) { + assert.equal(history.length, 1); + assert.equal(lastKMessages, 1); + assert.equal(history[0].role, 'user'); + assert.equal(history[0].content, 'List three uses for a paperclip.'); + return 'formatted independent prompt'; + }, + clear() { + state.clears++; + }, + dispose() { + state.disposed++; + }, + }; + const runner = { + reset() { + state.resets++; + state.pos = 0; + }, + generate(prompt, options, onToken) { + assert.equal(state.pos, 0, 'each generation must start with an empty KV cache'); + assert.equal(prompt, 'formatted independent prompt'); + assert.equal(options.maxNewTokens, 64); + assert.equal(options.temperature, 0); + assert.equal(options.ignoreEos, true); + state.pos += state.tokens; + onToken('first'); + return { + numTokens: state.tokens, + durationMs: 100, + tokensPerSecond: 640, + prefill: { numTokens: state.promptTokens, durationMs: 3, tokensPerSecond: 4000 }, + }; + }, + dispose() { + state.disposed++; + }, + }; + const library = { + createResourceScope() { + const owned = []; + return { + track(value) { + owned.push(value); + return value; + }, + dispose() { + owned + .splice(0) + .reverse() + .forEach((value) => value.dispose()); + }, + }; + }, + defaultWorkletRuntime: {}, + wrapAsync: + (fn) => + async (...args) => + fn(...args), + llm: { + parseTokenizerConfig: () => ({ chatTemplate: 'test' }), + createChatPreprocessor: () => preprocessor, + createLLMRunner: async () => runner, + }, + }; + const inputs = { + syntheticWaveform: () => [], + LLM_PROMPT: 'List three uses for a paperclip.', + LLM_MAX_NEW_TOKENS: 64, + }; + const exports = {}; + const code = transpileModule( + readFileSync(new URL('../src/drivers.ts', import.meta.url), 'utf8'), + { + compilerOptions: { module: ModuleKind.CommonJS, target: ScriptTarget.ES2022 }, + } + ).outputText; + runInNewContext(code, { + exports, + performance, + require: (name) => + name === 'react-native-executorch' + ? library + : name === 'react-native-blob-util' + ? { default: { fs: { readFile: async () => '{}' } } } + : inputs, + }); + return { state, driver: exports.driverFor({ task: 'llm', model: 'test' }) }; +} + +test('LLM iterations retain a fixed prompt, reset KV state, and report current stats', async () => { + const { state, driver } = fixture(); + const instance = await driver.create({ tokenizerConfigPath: 'test' }); + try { + const run = driver.runAsync(instance); + for (let i = 0; i < 30; i++) assert.equal(await run(), 64); + assert.equal(state.resets, 30); + assert.equal(state.clears, 30); + const detail = driver.detail(instance); + assert.equal(detail.promptTokens, 12); + assert.equal(detail.prefillMs, 3); + assert.ok(Number.isFinite(detail.timeToFirstTokenMs)); + assert.ok(detail.timeToFirstTokenMs >= 0); + } finally { + instance.dispose(); + } + assert.equal(state.disposed, 2); +}); + +test('LLM iterations reject a changed token count or growing prompt', async () => { + const { state, driver } = fixture(); + const instance = await driver.create({ tokenizerConfigPath: 'test' }); + try { + const run = driver.runAsync(instance); + await run(); + state.tokens = 63; + await assert.rejects(run(), /workload changed/); + state.tokens = 64; + state.promptTokens = 13; + await assert.rejects(run(), /workload changed/); + assert.equal(state.clears, 3, 'prompt resources must be cleared after rejected iterations'); + } finally { + instance.dispose(); + } +}); diff --git a/apps/benchmarks/scripts/variant-sizes.json b/apps/benchmarks/scripts/variant-sizes.json index ab7ff4b88f..9d495eb984 100644 --- a/apps/benchmarks/scripts/variant-sizes.json +++ b/apps/benchmarks/scripts/variant-sizes.json @@ -3,15 +3,20 @@ "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.10.0/tokenizer.json": 711544, "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.10.0/vulkan/all_minilm_l6_v2_vulkan_fp16.pte": 45208066, "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.10.0/xnnpack/all_minilm_l6_v2_xnnpack_fp32.pte": 90391296, + "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/coreml/all_minilm_l6_v2_coreml_int8.pte": 22817740, + "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/tokenizer.json": 711544, + "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/xnnpack/all_minilm_l6_v2_xnnpack_int8.pte": 23010688, "https://huggingface.co/software-mansion/react-native-executorch-all-mpnet-base-v2/resolve/v0.10.0/tokenizer.json": 710906, "https://huggingface.co/software-mansion/react-native-executorch-all-mpnet-base-v2/resolve/v0.10.0/vulkan/all_mpnet_base_v2_vulkan_fp16.pte": 217937154, "https://huggingface.co/software-mansion/react-native-executorch-all-mpnet-base-v2/resolve/v0.10.0/vulkan/all_mpnet_base_v2_vulkan_int8.pte": 133198978, "https://huggingface.co/software-mansion/react-native-executorch-all-mpnet-base-v2/resolve/v0.10.0/xnnpack/all_mpnet_base_v2_xnnpack_fp32.pte": 435801088, - "https://huggingface.co/software-mansion/react-native-executorch-bielik-v3.0/resolve/v0.10.0/tokenizer.json": 1976579, - "https://huggingface.co/software-mansion/react-native-executorch-bielik-v3.0/resolve/v0.10.0/tokenizer_config.json": 2034, - "https://huggingface.co/software-mansion/react-native-executorch-bielik-v3.0/resolve/v0.10.0/xnnpack/bielik_v3_0_1_5b_xnnpack_8da4w.pte": 923218176, - "https://huggingface.co/software-mansion/react-native-executorch-bielik-v3.0/resolve/v0.10.0/xnnpack/bielik_v3_0_1_5b_xnnpack_fp16.pte": 3193760512, + "https://huggingface.co/software-mansion/react-native-executorch-all-mpnet-base-v2/resolve/v0.11.0/xnnpack/all_mpnet_base_v2_xnnpack_8da4w.pte": 72474880, + "https://huggingface.co/software-mansion/react-native-executorch-all-mpnet-base-v2/resolve/v0.11.0/xnnpack/all_mpnet_base_v2_xnnpack_int8.pte": 109965568, + "https://huggingface.co/software-mansion/react-native-executorch-bielik-v3.0/resolve/v0.11.0/tokenizer.json": 1976579, + "https://huggingface.co/software-mansion/react-native-executorch-bielik-v3.0/resolve/v0.11.0/tokenizer_config.json": 2034, + "https://huggingface.co/software-mansion/react-native-executorch-bielik-v3.0/resolve/v0.11.0/xnnpack/bielik_v3_0_1_5b_xnnpack_8da4w.pte": 937321344, "https://huggingface.co/software-mansion/react-native-executorch-blazeface/resolve/v0.10.0/xnnpack/blazeface_xnnpack_fp32.pte": 649752, + "https://huggingface.co/software-mansion/react-native-executorch-blazeface/resolve/v0.11.0/vulkan/blazeface_vulkan_fp16.pte": 372866, "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.10.0/coreml/clip_vit_base_patch32_image_coreml_fp16.pte": 176209305, "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.10.0/coreml/clip_vit_base_patch32_text_coreml_fp16.pte": 127415056, "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.10.0/mlx/clip_vit_base_patch32_image_mlx_int8.pte": 98200964, @@ -20,43 +25,71 @@ "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.10.0/vulkan/clip_vit_base_patch32_text_vulkan_fp16.pte": 126991872, "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.10.0/xnnpack/clip_vit_base_patch32_image_xnnpack_fp32.pte": 351606400, "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.10.0/xnnpack/clip_vit_base_patch32_text_xnnpack_fp32.pte": 253943040, + "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.11.0/coreml/clip_vit_base_patch32_text_coreml_int8.pte": 63921048, + "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.11.0/tokenizer.json": 2224112, + "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.11.0/xnnpack/clip_vit_base_patch32_image_xnnpack_int8.pte": 95947264, + "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.11.0/xnnpack/clip_vit_base_patch32_text_xnnpack_int8.pte": 64353408, "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.10.0/xnnpack/deeplab_v3_mobilenet_v3_large_xnnpack_fp32.pte": 44128340, - "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.10.0/xnnpack/deeplab_v3_mobilenet_v3_large_xnnpack_int8.pte": 11359316, "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.10.0/xnnpack/deeplab_v3_resnet101_xnnpack_fp32.pte": 234419156, "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.10.0/xnnpack/deeplab_v3_resnet101_xnnpack_int8.pte": 61708884, "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.10.0/xnnpack/deeplab_v3_resnet50_xnnpack_fp32.pte": 158505428, "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.10.0/xnnpack/deeplab_v3_resnet50_xnnpack_int8.pte": 42379092, + "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.11.0/coreml/deeplab_v3_mobilenet_v3_large_coreml_fp16.pte": 22547278, + "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.11.0/coreml/deeplab_v3_resnet101_coreml_fp16.pte": 117854661, + "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.11.0/coreml/deeplab_v3_resnet50_coreml_fp16.pte": 79613357, + "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.11.0/xnnpack/deeplab_v3_mobilenet_v3_large_xnnpack_int8.pte": 11361620, + "https://huggingface.co/software-mansion/react-native-executorch-depth-anything-v2/resolve/v0.11.0/small/coreml/depth_anything_v2_small_coreml_fp16.pte": 50181365, + "https://huggingface.co/software-mansion/react-native-executorch-depth-anything-v2/resolve/v0.11.0/small/vulkan/depth_anything_v2_small_vulkan_fp16.pte": 49595524, + "https://huggingface.co/software-mansion/react-native-executorch-depth-anything-v2/resolve/v0.11.0/small/xnnpack/depth_anything_v2_small_xnnpack_fp32.pte": 98979332, + "https://huggingface.co/software-mansion/react-native-executorch-depth-anything-v2/resolve/v0.11.0/small/xnnpack/depth_anything_v2_small_xnnpack_int8.pte": 35481732, + "https://huggingface.co/software-mansion/react-native-executorch-depth-anything-v3/resolve/v0.11.0/small/coreml/depth_anything_v3_small_coreml_fp16.pte": 70865495, + "https://huggingface.co/software-mansion/react-native-executorch-depth-anything-v3/resolve/v0.11.0/small/vulkan/depth_anything_v3_small_vulkan_fp16.pte": 56407810, + "https://huggingface.co/software-mansion/react-native-executorch-depth-anything-v3/resolve/v0.11.0/small/xnnpack/depth_anything_v3_small_xnnpack_fp32.pte": 106480776, + "https://huggingface.co/software-mansion/react-native-executorch-depth-anything-v3/resolve/v0.11.0/small/xnnpack/depth_anything_v3_small_xnnpack_int8.pte": 40603272, "https://huggingface.co/software-mansion/react-native-executorch-distiluse-base-multilingual-cased-v2/resolve/v0.10.0/coreml/distiluse_base_multilingual_cased_v2_coreml_fp16.pte": 270553163, "https://huggingface.co/software-mansion/react-native-executorch-distiluse-base-multilingual-cased-v2/resolve/v0.10.0/mlx/distiluse_base_multilingual_cased_v2_mlx_int8.pte": 139603588, "https://huggingface.co/software-mansion/react-native-executorch-distiluse-base-multilingual-cased-v2/resolve/v0.10.0/tokenizer.json": 2919615, "https://huggingface.co/software-mansion/react-native-executorch-distiluse-base-multilingual-cased-v2/resolve/v0.10.0/vulkan/distiluse_base_multilingual_cased_v2_vulkan_fp16.pte": 270327296, - "https://huggingface.co/software-mansion/react-native-executorch-distiluse-base-multilingual-cased-v2/resolve/v0.10.0/xnnpack/distiluse_base_multilingual_cased_v2_xnnpack_8da4w.pte": 393317248, "https://huggingface.co/software-mansion/react-native-executorch-distiluse-base-multilingual-cased-v2/resolve/v0.10.0/xnnpack/distiluse_base_multilingual_cased_v2_xnnpack_fp32.pte": 540634368, + "https://huggingface.co/software-mansion/react-native-executorch-distiluse-base-multilingual-cased-v2/resolve/v0.11.0/coreml/distiluse_base_multilingual_cased_v2_coreml_int8.pte": 135622796, + "https://huggingface.co/software-mansion/react-native-executorch-distiluse-base-multilingual-cased-v2/resolve/v0.11.0/tokenizer.json": 2919615, + "https://huggingface.co/software-mansion/react-native-executorch-distiluse-base-multilingual-cased-v2/resolve/v0.11.0/xnnpack/distiluse_base_multilingual_cased_v2_xnnpack_8da4w.pte": 117181824, "https://huggingface.co/software-mansion/react-native-executorch-efficientnet-v2-s/resolve/v0.10.0/coreml/efficientnet_v2_s_coreml_fp16.pte": 43864953, "https://huggingface.co/software-mansion/react-native-executorch-efficientnet-v2-s/resolve/v0.10.0/xnnpack/efficientnet_v2_s_xnnpack_fp32.pte": 85714464, - "https://huggingface.co/software-mansion/react-native-executorch-efficientnet-v2-s/resolve/v0.10.0/xnnpack/efficientnet_v2_s_xnnpack_int8.pte": 22943904, + "https://huggingface.co/software-mansion/react-native-executorch-efficientnet-v2-s/resolve/v0.11.0/xnnpack/efficientnet_v2_s_xnnpack_int8.pte": 33572128, + "https://huggingface.co/software-mansion/react-native-executorch-facemesh/resolve/v0.11.0/coreml/facemesh_coreml_fp16.pte": 1559427, + "https://huggingface.co/software-mansion/react-native-executorch-facemesh/resolve/v0.11.0/xnnpack/facemesh_xnnpack_fp32.pte": 2461956, "https://huggingface.co/software-mansion/react-native-executorch-fast-sam/resolve/v0.10.0/s/coreml/fast_sam_s_coreml_fp16.pte": 24219702, "https://huggingface.co/software-mansion/react-native-executorch-fast-sam/resolve/v0.10.0/s/xnnpack/fast_sam_s_xnnpack_fp32.pte": 47346688, "https://huggingface.co/software-mansion/react-native-executorch-fast-sam/resolve/v0.10.0/x/coreml/fast_sam_x_coreml_fp16.pte": 145311073, "https://huggingface.co/software-mansion/react-native-executorch-fast-sam/resolve/v0.10.0/x/xnnpack/fast_sam_x_xnnpack_fp32.pte": 289089280, + "https://huggingface.co/software-mansion/react-native-executorch-fast-sam/resolve/v0.11.0/s/vulkan/fast_sam_s_vulkan_fp16.pte": 23721536, + "https://huggingface.co/software-mansion/react-native-executorch-fast-sam/resolve/v0.11.0/s/xnnpack/fast_sam_s_xnnpack_int8.pte": 19346816, + "https://huggingface.co/software-mansion/react-native-executorch-fast-sam/resolve/v0.11.0/x/vulkan/fast_sam_x_vulkan_fp16.pte": 144618176, + "https://huggingface.co/software-mansion/react-native-executorch-fast-sam/resolve/v0.11.0/x/xnnpack/fast_sam_x_xnnpack_int8.pte": 104428800, "https://huggingface.co/software-mansion/react-native-executorch-fcn/resolve/v0.10.0/xnnpack/fcn_resnet101_xnnpack_fp32.pte": 207690836, "https://huggingface.co/software-mansion/react-native-executorch-fcn/resolve/v0.10.0/xnnpack/fcn_resnet101_xnnpack_int8.pte": 55004756, "https://huggingface.co/software-mansion/react-native-executorch-fcn/resolve/v0.10.0/xnnpack/fcn_resnet50_xnnpack_fp32.pte": 131777108, "https://huggingface.co/software-mansion/react-native-executorch-fcn/resolve/v0.10.0/xnnpack/fcn_resnet50_xnnpack_int8.pte": 35675220, + "https://huggingface.co/software-mansion/react-native-executorch-fcn/resolve/v0.11.0/coreml/fcn_resnet101_coreml_fp16.pte": 104451281, + "https://huggingface.co/software-mansion/react-native-executorch-fcn/resolve/v0.11.0/coreml/fcn_resnet50_coreml_fp16.pte": 66210276, "https://huggingface.co/software-mansion/react-native-executorch-fsmn-vad/resolve/v0.10.0/xnnpack/fsmn_vad_xnnpack_fp32.pte": 1827808, - "https://huggingface.co/software-mansion/react-native-executorch-gemma-4/resolve/v0.10.0/e2b/mlx/gemma4_e2b_mlx_int4.pte": 2894305792, - "https://huggingface.co/software-mansion/react-native-executorch-gemma-4/resolve/v0.10.0/e2b/tokenizer.json": 32169868, - "https://huggingface.co/software-mansion/react-native-executorch-gemma-4/resolve/v0.10.0/e2b/tokenizer_config.json": 22312, - "https://huggingface.co/software-mansion/react-native-executorch-gemma-4/resolve/v0.10.0/e2b/vulkan/gemma_4_e2b_vulkan_8da4w.pte": 2570073732, - "https://huggingface.co/software-mansion/react-native-executorch-gemma-4/resolve/v0.10.0/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte": 2635017344, - "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.10.0/0_5b/xnnpack/hammer_2_1_0_5b_xnnpack_8da4w.pte": 417114808, - "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.10.0/0_5b/xnnpack/hammer_2_1_0_5b_xnnpack_bf16.pte": 988976720, - "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.10.0/1_5b/xnnpack/hammer_2_1_1_5b_xnnpack_8da4w.pte": 1135475424, - "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.10.0/1_5b/xnnpack/hammer_2_1_1_5b_xnnpack_bf16.pte": 3089098064, - "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.10.0/3b/xnnpack/hammer_2_1_3b_xnnpack_8da4w.pte": 2052539520, - "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.10.0/3b/xnnpack/hammer_2_1_3b_xnnpack_bf16.pte": 6173382608, - "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.10.0/tokenizer.json": 11421896, - "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.10.0/tokenizer_config.json": 7188, + "https://huggingface.co/software-mansion/react-native-executorch-gemma-4-multimodal/resolve/v0.11.0/e2b/mlx/gemma_4_e2b_mlx_int4.pte": 2728363136, + "https://huggingface.co/software-mansion/react-native-executorch-gemma-4-multimodal/resolve/v0.11.0/e2b/vulkan/gemma_4_e2b_vulkan_8da4w.pte": 3806518400, + "https://huggingface.co/software-mansion/react-native-executorch-gemma-4-multimodal/resolve/v0.11.0/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte": 3619175296, + "https://huggingface.co/software-mansion/react-native-executorch-gemma-4/resolve/v0.11.0/e2b/mlx/gemma_4_e2b_mlx_int4.pte": 2526966656, + "https://huggingface.co/software-mansion/react-native-executorch-gemma-4/resolve/v0.11.0/e2b/tokenizer.json": 32169868, + "https://huggingface.co/software-mansion/react-native-executorch-gemma-4/resolve/v0.11.0/e2b/tokenizer_config.json": 22312, + "https://huggingface.co/software-mansion/react-native-executorch-gemma-4/resolve/v0.11.0/e2b/vulkan/gemma_4_e2b_vulkan_8da4w.pte": 2956364036, + "https://huggingface.co/software-mansion/react-native-executorch-gemma-4/resolve/v0.11.0/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte": 3215277312, + "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.11.0/0_5b/mlx/hammer_2_1_0_5b_mlx_int4.pte": 534436288, + "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.11.0/0_5b/xnnpack/hammer_2_1_0_5b_xnnpack_8da4w.pte": 431795512, + "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.11.0/1_5b/mlx/hammer_2_1_1_5b_mlx_int4.pte": 1286262400, + "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.11.0/1_5b/xnnpack/hammer_2_1_1_5b_xnnpack_8da4w.pte": 1164836192, + "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.11.0/3b/mlx/hammer_2_1_3b_mlx_int4.pte": 2261014272, + "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.11.0/3b/xnnpack/hammer_2_1_3b_xnnpack_8da4w.pte": 2065123200, + "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.11.0/tokenizer.json": 11421896, + "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.11.0/tokenizer_config.json": 7188, "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.10.0/coreml/german/duration_predictor_de_coreml_fp32.pte": 68064056, "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.10.0/coreml/german/synthesizer_de_coreml_fp32.pte": 296217572, "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.10.0/coreml/polish/duration_predictor_pl_coreml_fp32.pte": 68064056, @@ -102,55 +135,77 @@ "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.10.0/xnnpack/polish/synthesizer_pl_xnnpack_fp32.pte": 272463344, "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.10.0/xnnpack/standard/duration_predictor_std_xnnpack_fp32.pte": 59307208, "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.10.0/xnnpack/standard/synthesizer_std_xnnpack_fp32.pte": 272463344, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/1_2b/mlx/lfm_2_5_1_2b_mlx_int4.pte": 890504064, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/1_2b/tokenizer.json": 4733389, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/1_2b/tokenizer_config.json": 94609, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/1_2b/xnnpack/lfm_2_5_1_2b_xnnpack_8da4w.pte": 795679104, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/1_2b/xnnpack/lfm_2_5_1_2b_xnnpack_fp16.pte": 2610648192, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/350m/mlx/lfm_2_5_350m_mlx_int4.pte": 322767744, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/350m/tokenizer.json": 4733383, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/350m/tokenizer_config.json": 2979, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/350m/xnnpack/lfm_2_5_350m_xnnpack_8da4w.pte": 277555584, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/350m/xnnpack/lfm_2_5_350m_xnnpack_fp16.pte": 844660224, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int4.pte": 1518688384, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int8.pte": 2103792896, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/vl_1_6b/tokenizer.json": 4733040, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/vl_1_6b/tokenizer_config.json": 3227, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/vl_1_6b/vulkan/lfm_2_5_vl_1_6b_vulkan_8da4w.pte": 2533361152, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/vl_1_6b/xnnpack/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte": 2427656704, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/vl_450m/mlx/lfm_2_5_vl_450m_mlx_int4.pte": 394106112, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/vl_450m/tokenizer.json": 4733040, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/vl_450m/tokenizer_config.json": 3227, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/vl_450m/vulkan/lfm_2_5_vl_450m_vulkan_8da4w.pte": 655954432, - "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.10.0/vl_450m/xnnpack/lfm_2_5_vl_450m_xnnpack_8da4w.pte": 648917376, - "https://huggingface.co/software-mansion/react-native-executorch-lfm2.5-embedding-350m/resolve/v0.10.0/mlx/lfm_2_5_embedding_350m_mlx_int4.pte": 188544770, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/phonemizer/zh/lexicon.json": 2334963, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/phonemizer/zh/segmenter.json": 8288194, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaobei.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoxiao.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoyi.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunjian.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunxi.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunxia.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunyang.bin": 522240, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/1_2b/mlx/lfm_2_5_1_2b_mlx_int4.pte": 890503424, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/1_2b/tokenizer.json": 4733389, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/1_2b/tokenizer_config.json": 94609, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/1_2b/xnnpack/lfm_2_5_1_2b_xnnpack_8da4w.pte": 810359808, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/2_6b/mlx/lfm_2_5_2_6b_mlx_int4.pte": 1665769984, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/2_6b/tokenizer.json": 17905598, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/2_6b/tokenizer_config.json": 6065, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/2_6b/xnnpack/lfm_2_5_2_6b_xnnpack_8da4w.pte": 1789728256, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/350m/mlx/lfm_2_5_350m_mlx_int4.pte": 322767104, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/350m/tokenizer.json": 4733383, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/350m/tokenizer_config.json": 2979, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/350m/xnnpack/lfm_2_5_350m_xnnpack_8da4w.pte": 292236288, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int4.pte": 2370669056, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/vl_1_6b/mlx/lfm_2_5_vl_1_6b_mlx_int8.pte": 2955774464, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/vl_1_6b/tokenizer.json": 4733040, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/vl_1_6b/tokenizer_config.json": 3227, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/vl_1_6b/vulkan/lfm_2_5_vl_1_6b_vulkan_8da4w.pte": 2466256128, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/vl_1_6b/xnnpack/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte": 2474863104, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/vl_450m/mlx/lfm_2_5_vl_450m_mlx_int4.pte": 582183552, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/vl_450m/tokenizer.json": 4733040, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/vl_450m/tokenizer_config.json": 3227, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/vl_450m/vulkan/lfm_2_5_vl_450m_vulkan_8da4w.pte": 622403840, + "https://huggingface.co/software-mansion/react-native-executorch-lfm-2.5/resolve/v0.11.0/vl_450m/xnnpack/lfm_2_5_vl_450m_xnnpack_8da4w.pte": 648867584, + "https://huggingface.co/software-mansion/react-native-executorch-lfm2.5-colbert-350m/resolve/v0.11.0/mlx/lfm_2_5_colbert_350m_mlx_int4.pte": 187992834, + "https://huggingface.co/software-mansion/react-native-executorch-lfm2.5-colbert-350m/resolve/v0.11.0/tokenizer.json": 4733638, + "https://huggingface.co/software-mansion/react-native-executorch-lfm2.5-colbert-350m/resolve/v0.11.0/xnnpack/lfm_2_5_colbert_350m_xnnpack_8da4w.pte": 372627840, + "https://huggingface.co/software-mansion/react-native-executorch-lfm2.5-embedding-350m/resolve/v0.10.0/mlx/lfm_2_5_embedding_350m_mlx_int4.pte": 188543746, "https://huggingface.co/software-mansion/react-native-executorch-lfm2.5-embedding-350m/resolve/v0.10.0/tokenizer.json": 4733275, "https://huggingface.co/software-mansion/react-native-executorch-lfm2.5-embedding-350m/resolve/v0.10.0/xnnpack/lfm_2_5_embedding_350m_xnnpack_8da4w.pte": 574784896, - "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.10.0/1b/xnnpack/llama_3_2_1b_xnnpack_bf16.pte": 2472937860, - "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.10.0/1b/xnnpack/llama_3_2_1b_xnnpack_spinquant.pte": 1135951488, - "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.10.0/3b/xnnpack/llama_3_2_3b_xnnpack_bf16.pte": 6428047108, - "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.10.0/3b/xnnpack/llama_3_2_3b_xnnpack_spinquant.pte": 2553367552, - "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.10.0/tokenizer.json": 9906781, - "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.10.0/tokenizer_config.json": 54528, + "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.11.0/1b/mlx/llama_3_2_1b_mlx_int4.pte": 1182101632, + "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.11.0/1b/xnnpack/llama_3_2_1b_xnnpack_spinquant.pte": 1082483584, + "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.11.0/3b/mlx/llama_3_2_3b_mlx_int4.pte": 2495274496, + "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.11.0/3b/xnnpack/llama_3_2_3b_xnnpack_spinquant.pte": 2379319808, + "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.11.0/tokenizer.json": 9906781, + "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.11.0/tokenizer_config.json": 54528, "https://huggingface.co/software-mansion/react-native-executorch-lraspp/resolve/v0.10.0/xnnpack/lraspp_mobilenet_v3_large_xnnpack_fp32.pte": 12913492, "https://huggingface.co/software-mansion/react-native-executorch-lraspp/resolve/v0.10.0/xnnpack/lraspp_mobilenet_v3_large_xnnpack_int8.pte": 3529044, + "https://huggingface.co/software-mansion/react-native-executorch-lraspp/resolve/v0.11.0/coreml/lraspp_mobilenet_v3_large_coreml_fp16.pte": 6917692, "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-MiniLM-L6-cos-v1/resolve/v0.10.0/coreml/multi_qa_minilm_l6_cos_v1_coreml_fp16.pte": 45460244, "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-MiniLM-L6-cos-v1/resolve/v0.10.0/tokenizer.json": 711544, "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-MiniLM-L6-cos-v1/resolve/v0.10.0/vulkan/multi_qa_minilm_l6_cos_v1_vulkan_fp16.pte": 45208066, "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-MiniLM-L6-cos-v1/resolve/v0.10.0/xnnpack/multi_qa_minilm_l6_cos_v1_xnnpack_fp32.pte": 90391424, + "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-MiniLM-L6-cos-v1/resolve/v0.11.0/coreml/multi_qa_minilm_l6_cos_v1_coreml_int8.pte": 22817740, + "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-MiniLM-L6-cos-v1/resolve/v0.11.0/tokenizer.json": 711544, + "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-MiniLM-L6-cos-v1/resolve/v0.11.0/xnnpack/multi_qa_minilm_l6_cos_v1_xnnpack_int8.pte": 23010688, "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-mpnet-base-dot-v1/resolve/v0.10.0/tokenizer.json": 710871, "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-mpnet-base-dot-v1/resolve/v0.10.0/vulkan/multi_qa_mpnet_base_dot_v1_vulkan_fp16.pte": 217937154, "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-mpnet-base-dot-v1/resolve/v0.10.0/vulkan/multi_qa_mpnet_base_dot_v1_vulkan_int8.pte": 133198978, "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-mpnet-base-dot-v1/resolve/v0.10.0/xnnpack/multi_qa_mpnet_base_dot_v1_xnnpack_fp32.pte": 435801088, + "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-mpnet-base-dot-v1/resolve/v0.11.0/xnnpack/multi_qa_mpnet_base_dot_v1_xnnpack_8da4w.pte": 72474880, + "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-mpnet-base-dot-v1/resolve/v0.11.0/xnnpack/multi_qa_mpnet_base_dot_v1_xnnpack_int8.pte": 109965568, "https://huggingface.co/software-mansion/react-native-executorch-paraphrase-multilingual-MiniLM-L12-v2/resolve/v0.10.0/coreml/paraphrase_multilingual_minilm_l12_v2_coreml_fp16.pte": 235565340, "https://huggingface.co/software-mansion/react-native-executorch-paraphrase-multilingual-MiniLM-L12-v2/resolve/v0.10.0/tokenizer.json": 17082987, "https://huggingface.co/software-mansion/react-native-executorch-paraphrase-multilingual-MiniLM-L12-v2/resolve/v0.10.0/vulkan/paraphrase_multilingual_minilm_l12_v2_vulkan_fp16.pte": 235144578, - "https://huggingface.co/software-mansion/react-native-executorch-paraphrase-multilingual-MiniLM-L12-v2/resolve/v0.10.0/xnnpack/paraphrase_multilingual_minilm_l12_v2_xnnpack_8da4w.pte": 397299584, "https://huggingface.co/software-mansion/react-native-executorch-paraphrase-multilingual-MiniLM-L12-v2/resolve/v0.10.0/xnnpack/paraphrase_multilingual_minilm_l12_v2_xnnpack_fp32.pte": 470258816, - "https://huggingface.co/software-mansion/react-native-executorch-phi-4-mini/resolve/v0.10.0/tokenizer.json": 15524095, - "https://huggingface.co/software-mansion/react-native-executorch-phi-4-mini/resolve/v0.10.0/tokenizer_config.json": 2938, - "https://huggingface.co/software-mansion/react-native-executorch-phi-4-mini/resolve/v0.10.0/xnnpack/phi_4_mini_xnnpack_8da4w.pte": 2815289472, - "https://huggingface.co/software-mansion/react-native-executorch-phi-4-mini/resolve/v0.10.0/xnnpack/phi_4_mini_xnnpack_bf16.pte": 7672525956, + "https://huggingface.co/software-mansion/react-native-executorch-paraphrase-multilingual-MiniLM-L12-v2/resolve/v0.11.0/coreml/paraphrase_multilingual_minilm_l12_v2_coreml_int8.pte": 118351833, + "https://huggingface.co/software-mansion/react-native-executorch-paraphrase-multilingual-MiniLM-L12-v2/resolve/v0.11.0/tokenizer.json": 17082987, + "https://huggingface.co/software-mansion/react-native-executorch-paraphrase-multilingual-MiniLM-L12-v2/resolve/v0.11.0/xnnpack/paraphrase_multilingual_minilm_l12_v2_xnnpack_8da4w.pte": 109667328, + "https://huggingface.co/software-mansion/react-native-executorch-phi-4-mini/resolve/v0.11.0/mlx/phi_4_mini_mlx_int4.pte": 3267645440, + "https://huggingface.co/software-mansion/react-native-executorch-phi-4-mini/resolve/v0.11.0/tokenizer.json": 15524095, + "https://huggingface.co/software-mansion/react-native-executorch-phi-4-mini/resolve/v0.11.0/tokenizer_config.json": 2938, + "https://huggingface.co/software-mansion/react-native-executorch-phi-4-mini/resolve/v0.11.0/xnnpack/phi_4_mini_xnnpack_8da4w.pte": 2818435968, "https://huggingface.co/software-mansion/react-native-executorch-pp-ocrv6/resolve/v0.10.0/charset.json": 131078, "https://huggingface.co/software-mansion/react-native-executorch-pp-ocrv6/resolve/v0.10.0/coreml/pp_ocrv6_coreml_int8.pte": 8269910, "https://huggingface.co/software-mansion/react-native-executorch-pp-ocrv6/resolve/v0.10.0/vulkan/pp_ocrv6_vulkan_fp16.pte": 26176856, @@ -162,59 +217,74 @@ "https://huggingface.co/software-mansion/react-native-executorch-privacy-filter-openai/resolve/v0.10.0/mlx/privacy_filter_openai_mlx_int4.pte": 875085888, "https://huggingface.co/software-mansion/react-native-executorch-privacy-filter-openai/resolve/v0.10.0/tokenizer.json": 27868174, "https://huggingface.co/software-mansion/react-native-executorch-privacy-filter-openai/resolve/v0.10.0/xnnpack/privacy_filter_openai_xnnpack_8da4w.pte": 1241699460, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.10.0/0_5b/xnnpack/qwen_2_5_0_5b_xnnpack_8da4w.pte": 417495168, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.10.0/0_5b/xnnpack/qwen_2_5_0_5b_xnnpack_bf16.pte": 989451488, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.10.0/1_5b/xnnpack/qwen_2_5_1_5b_xnnpack_8da4w.pte": 1136177792, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.10.0/1_5b/xnnpack/qwen_2_5_1_5b_xnnpack_bf16.pte": 3089917920, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.10.0/3b/xnnpack/qwen_2_5_3b_xnnpack_8da4w.pte": 2091695744, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.10.0/3b/xnnpack/qwen_2_5_3b_xnnpack_bf16.pte": 6174476384, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.10.0/tokenizer.json": 7031645, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.10.0/tokenizer_config.json": 7305, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.10.0/0_6b/xnnpack/qwen_3_0_6b_xnnpack_8da4w.pte": 505686400, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.10.0/0_6b/xnnpack/qwen_3_0_6b_xnnpack_bf16.pte": 1194587396, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.10.0/1_7b/xnnpack/qwen_3_1_7b_xnnpack_8da4w.pte": 1303738496, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.10.0/1_7b/xnnpack/qwen_3_1_7b_xnnpack_bf16.pte": 3443637508, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.10.0/4b/xnnpack/qwen_3_4b_xnnpack_8da4w.pte": 2681646976, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.10.0/4b/xnnpack/qwen_3_4b_xnnpack_bf16.pte": 8047533060, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.10.0/tokenizer.json": 11422654, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.10.0/tokenizer_config.json": 8023, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.11.0/0_5b/mlx/qwen_2_5_0_5b_mlx_int4.pte": 535050752, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.11.0/0_5b/xnnpack/qwen_2_5_0_5b_xnnpack_8da4w.pte": 432176000, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.11.0/1_5b/mlx/qwen_2_5_1_5b_mlx_int4.pte": 1287315840, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.11.0/1_5b/xnnpack/qwen_2_5_1_5b_xnnpack_8da4w.pte": 1165538560, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.11.0/3b/mlx/qwen_2_5_3b_mlx_int4.pte": 2262418944, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.11.0/3b/xnnpack/qwen_2_5_3b_xnnpack_8da4w.pte": 2104279296, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.11.0/tokenizer.json": 7031645, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.11.0/tokenizer_config.json": 7305, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.11.0/0_6b/mlx/qwen_3_0_6b_mlx_int8.pte": 926157696, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.11.0/0_6b/xnnpack/qwen_3_0_6b_xnnpack_8da4w.pte": 518271616, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.11.0/1_7b/mlx/qwen_3_1_7b_mlx_int4.pte": 1536819840, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.11.0/1_7b/xnnpack/qwen_3_1_7b_xnnpack_8da4w.pte": 1307933568, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.11.0/4b/mlx/qwen_3_4b_mlx_int4.pte": 2915459584, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.11.0/4b/xnnpack/qwen_3_4b_xnnpack_8da4w.pte": 2685842048, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.11.0/tokenizer.json": 11422654, + "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.11.0/tokenizer_config.json": 8426, "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-keypoint/resolve/v0.10.0/coreml/rfdetr_keypoint_preview_coreml_fp16.pte": 75024502, "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-keypoint/resolve/v0.10.0/xnnpack/rfdetr_keypoint_preview_xnnpack_fp32.pte": 145598480, + "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-keypoint/resolve/v0.11.0/vulkan/rfdetr_keypoint_preview_vulkan_fp16.pte": 73445632, + "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-keypoint/resolve/v0.11.0/xnnpack/rfdetr_keypoint_preview_xnnpack_int8.pte": 59381136, "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-nano-detector/resolve/v0.10.0/coreml/rfdetr_nano_coreml_fp16.pte": 54747821, "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-nano-detector/resolve/v0.10.0/xnnpack/rfdetr_nano_xnnpack_fp32.pte": 111571820, + "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-nano-detector/resolve/v0.11.0/vulkan/rfdetr_nano_vulkan_fp16.pte": 54006916, + "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-nano-detector/resolve/v0.11.0/xnnpack/rfdetr_nano_xnnpack_int8.pte": 39082384, "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-nano-segmentation/resolve/v0.10.0/coreml/rfdetr_nano_coreml_fp16.pte": 62353090, "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-nano-segmentation/resolve/v0.10.0/xnnpack/rfdetr_nano_xnnpack_fp32.pte": 124001408, + "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-nano-segmentation/resolve/v0.11.0/vulkan/rfdetr_nano_vulkan_fp16.pte": 61434756, + "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-nano-segmentation/resolve/v0.11.0/xnnpack/rfdetr_nano_xnnpack_int8.pte": 41852416, "https://huggingface.co/software-mansion/react-native-executorch-sdxs-512-dreamshaper/resolve/v0.10.0/coreml/sdxs_512_dreamshaper_coreml_fp16.pte": 880687279, "https://huggingface.co/software-mansion/react-native-executorch-sdxs-512-dreamshaper/resolve/v0.10.0/tokenizer.json": 3642073, "https://huggingface.co/software-mansion/react-native-executorch-sdxs-512-dreamshaper/resolve/v0.10.0/xnnpack/sdxs_512_dreamshaper_xnnpack_fp32.pte": 1760575244, + "https://huggingface.co/software-mansion/react-native-executorch-sdxs-512-dreamshaper/resolve/v0.11.0/vulkan/sdxs_512_dreamshaper_vulkan_fp16.pte": 880465922, + "https://huggingface.co/software-mansion/react-native-executorch-sdxs-512-dreamshaper/resolve/v0.11.0/xnnpack/sdxs_512_dreamshaper_xnnpack_int8.pte": 813206924, "https://huggingface.co/software-mansion/react-native-executorch-selfie-segmentation/resolve/v0.10.0/coreml/selfie_segmentation_coreml_fp16.pte": 614993, "https://huggingface.co/software-mansion/react-native-executorch-selfie-segmentation/resolve/v0.10.0/coreml/selfie_segmentation_landscape_coreml_fp16.pte": 614976, "https://huggingface.co/software-mansion/react-native-executorch-selfie-segmentation/resolve/v0.10.0/xnnpack/selfie_segmentation_landscape_xnnpack_fp32.pte": 473732, "https://huggingface.co/software-mansion/react-native-executorch-selfie-segmentation/resolve/v0.10.0/xnnpack/selfie_segmentation_xnnpack_fp32.pte": 497924, - "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.10.0/135m/xnnpack/smollm2_135m_xnnpack_8da8w.pte": 165664000, - "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.10.0/1_7b/xnnpack/smollm2_1_7b_xnnpack_8da8w.pte": 1816874880, - "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.10.0/360m/xnnpack/smollm2_360m_xnnpack_8da8w.pte": 412454016, - "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.10.0/tokenizer.json": 2104556, - "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.10.0/tokenizer_config.json": 3779, + "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.11.0/135m/mlx/smollm2_135m_mlx_int8.pte": 195631616, + "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.11.0/135m/xnnpack/smollm2_135m_xnnpack_8da8w.pte": 173005952, + "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.11.0/1_7b/mlx/smollm2_1_7b_mlx_int8.pte": 1966486656, + "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.11.0/1_7b/xnnpack/smollm2_1_7b_xnnpack_8da8w.pte": 1816875520, + "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.11.0/360m/mlx/smollm2_360m_mlx_int8.pte": 467832832, + "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.11.0/360m/xnnpack/smollm2_360m_xnnpack_8da8w.pte": 419795968, + "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.11.0/tokenizer.json": 2104556, + "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.11.0/tokenizer_config.json": 3779, "https://huggingface.co/software-mansion/react-native-executorch-ssdlite320-mobilenet-v3-large/resolve/v0.10.0/coreml/ssdlite320_mobilenet_v3_large_coreml_fp16.pte": 8457606, "https://huggingface.co/software-mansion/react-native-executorch-ssdlite320-mobilenet-v3-large/resolve/v0.10.0/xnnpack/ssdlite320_mobilenet_v3_large_xnnpack_fp32.pte": 13896712, "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-candy/resolve/v0.10.0/coreml/style_transfer_candy_coreml_fp16.pte": 3791869, "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-candy/resolve/v0.10.0/xnnpack/style_transfer_candy_xnnpack_fp32.pte": 6816524, "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-candy/resolve/v0.10.0/xnnpack/style_transfer_candy_xnnpack_int8.pte": 1835532, + "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-candy/resolve/v0.11.0/vulkan/style_transfer_candy_vulkan_fp16.pte": 3418114, "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-mosaic/resolve/v0.10.0/coreml/style_transfer_mosaic_coreml_fp16.pte": 3791869, "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-mosaic/resolve/v0.10.0/xnnpack/style_transfer_mosaic_xnnpack_fp32.pte": 6816524, "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-mosaic/resolve/v0.10.0/xnnpack/style_transfer_mosaic_xnnpack_int8.pte": 1835532, + "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-mosaic/resolve/v0.11.0/vulkan/style_transfer_mosaic_vulkan_fp16.pte": 3418114, "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-rain-princess/resolve/v0.10.0/coreml/style_transfer_rain_princess_coreml_fp16.pte": 3791869, "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-rain-princess/resolve/v0.10.0/xnnpack/style_transfer_rain_princess_xnnpack_fp32.pte": 6816524, "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-rain-princess/resolve/v0.10.0/xnnpack/style_transfer_rain_princess_xnnpack_int8.pte": 1835532, + "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-rain-princess/resolve/v0.11.0/vulkan/style_transfer_rain_princess_vulkan_fp16.pte": 3418114, "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-udnie/resolve/v0.10.0/coreml/style_transfer_udnie_coreml_fp16.pte": 3791869, "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-udnie/resolve/v0.10.0/xnnpack/style_transfer_udnie_xnnpack_fp32.pte": 6816524, "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-udnie/resolve/v0.10.0/xnnpack/style_transfer_udnie_xnnpack_int8.pte": 1835532, + "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-udnie/resolve/v0.11.0/vulkan/style_transfer_udnie_vulkan_fp16.pte": 3418114, "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.10.0/mlx/duration_predictor_mlx_fp32.pte": 3517056, "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.10.0/mlx/text_encoder_mlx_fp32.pte": 36091520, "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.10.0/mlx/vector_estimator_mlx_fp32.pte": 256232192, "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.10.0/mlx/vocoder_mlx_fp32.pte": 101400064, - "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.10.0/unicode_indexer.json": 0, + "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.10.0/unicode_indexer.json": 277676, "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.10.0/voice_styles/F1.json": 292046, "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.10.0/voice_styles/F2.json": 292423, "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.10.0/voice_styles/F3.json": 290794, @@ -233,6 +303,11 @@ "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.10.0/xnnpack/text_encoder_xnnpack_fp32.pte": 36196736, "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.10.0/xnnpack/vector_estimator_xnnpack_fp32.pte": 256435072, "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.10.0/xnnpack/vocoder_xnnpack_fp32.pte": 101429248, + "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.11.0/unicode_indexer.json": 277676, + "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.11.0/xnnpack/duration_predictor_xnnpack_fp32.pte": 3571204, + "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.11.0/xnnpack/text_encoder_xnnpack_fp32.pte": 36196736, + "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.11.0/xnnpack/vector_estimator_xnnpack_int8.pte": 65983360, + "https://huggingface.co/software-mansion/react-native-executorch-supertonic/resolve/v0.11.0/xnnpack/vocoder_xnnpack_int8.pte": 38614400, "https://huggingface.co/software-mansion/react-native-executorch-whisper-base.en/resolve/v0.10.0/coreml/whisper_base_en_coreml_fp16.pte": 146287992, "https://huggingface.co/software-mansion/react-native-executorch-whisper-base.en/resolve/v0.10.0/mlx/whisper_base_en_mlx_bf16.pte": 145863936, "https://huggingface.co/software-mansion/react-native-executorch-whisper-base.en/resolve/v0.10.0/mlx/whisper_base_en_mlx_int8.pte": 102572160, @@ -240,7 +315,7 @@ "https://huggingface.co/software-mansion/react-native-executorch-whisper-base.en/resolve/v0.10.0/vulkan/whisper_base_en_vulkan_fp16.pte": 351067904, "https://huggingface.co/software-mansion/react-native-executorch-whisper-base.en/resolve/v0.10.0/vulkan/whisper_base_en_vulkan_int8.pte": 205707104, "https://huggingface.co/software-mansion/react-native-executorch-whisper-base.en/resolve/v0.10.0/xnnpack/whisper_base_en_xnnpack_fp32.pte": 398646144, - "https://huggingface.co/software-mansion/react-native-executorch-whisper-base.en/resolve/v0.10.0/xnnpack/whisper_base_en_xnnpack_int8.pte": 247307520, + "https://huggingface.co/software-mansion/react-native-executorch-whisper-base.en/resolve/v0.11.0/xnnpack/whisper_base_en_xnnpack_int8.pte": 266793216, "https://huggingface.co/software-mansion/react-native-executorch-whisper-base/resolve/v0.10.0/coreml/whisper_base_coreml_fp16.pte": 146289018, "https://huggingface.co/software-mansion/react-native-executorch-whisper-base/resolve/v0.10.0/mlx/whisper_base_mlx_bf16.pte": 145864960, "https://huggingface.co/software-mansion/react-native-executorch-whisper-base/resolve/v0.10.0/mlx/whisper_base_mlx_int8.pte": 102573184, @@ -248,19 +323,23 @@ "https://huggingface.co/software-mansion/react-native-executorch-whisper-base/resolve/v0.10.0/vulkan/whisper_base_vulkan_fp16.pte": 351072000, "https://huggingface.co/software-mansion/react-native-executorch-whisper-base/resolve/v0.10.0/vulkan/whisper_base_vulkan_int8.pte": 205709668, "https://huggingface.co/software-mansion/react-native-executorch-whisper-base/resolve/v0.10.0/xnnpack/whisper_base_xnnpack_fp32.pte": 398650240, + "https://huggingface.co/software-mansion/react-native-executorch-whisper-base/resolve/v0.11.0/tokenizer.json": 2480466, + "https://huggingface.co/software-mansion/react-native-executorch-whisper-base/resolve/v0.11.0/xnnpack/whisper_base_xnnpack_int8.pte": 266797312, "https://huggingface.co/software-mansion/react-native-executorch-whisper-small.en/resolve/v0.10.0/coreml/whisper_small_en_coreml_fp16.pte": 484938675, "https://huggingface.co/software-mansion/react-native-executorch-whisper-small.en/resolve/v0.10.0/mlx/whisper_small_en_mlx_int8.pte": 289454208, "https://huggingface.co/software-mansion/react-native-executorch-whisper-small.en/resolve/v0.10.0/tokenizer.json": 2405679, "https://huggingface.co/software-mansion/react-native-executorch-whisper-small.en/resolve/v0.10.0/vulkan/whisper_small_en_vulkan_fp16.pte": 923903872, "https://huggingface.co/software-mansion/react-native-executorch-whisper-small.en/resolve/v0.10.0/vulkan/whisper_small_en_vulkan_int8.pte": 507790432, "https://huggingface.co/software-mansion/react-native-executorch-whisper-small.en/resolve/v0.10.0/xnnpack/whisper_small_en_xnnpack_fp32.pte": 1128711680, - "https://huggingface.co/software-mansion/react-native-executorch-whisper-small.en/resolve/v0.10.0/xnnpack/whisper_small_en_xnnpack_int8.pte": 447563136, + "https://huggingface.co/software-mansion/react-native-executorch-whisper-small.en/resolve/v0.11.0/xnnpack/whisper_small_en_xnnpack_int8.pte": 534974080, "https://huggingface.co/software-mansion/react-native-executorch-whisper-small/resolve/v0.10.0/coreml/whisper_small_coreml_fp16.pte": 484940213, "https://huggingface.co/software-mansion/react-native-executorch-whisper-small/resolve/v0.10.0/mlx/whisper_small_mlx_int8.pte": 289455744, "https://huggingface.co/software-mansion/react-native-executorch-whisper-small/resolve/v0.10.0/tokenizer.json": 2480466, "https://huggingface.co/software-mansion/react-native-executorch-whisper-small/resolve/v0.10.0/vulkan/whisper_small_vulkan_fp16.pte": 923910016, "https://huggingface.co/software-mansion/react-native-executorch-whisper-small/resolve/v0.10.0/vulkan/whisper_small_vulkan_int8.pte": 507794276, "https://huggingface.co/software-mansion/react-native-executorch-whisper-small/resolve/v0.10.0/xnnpack/whisper_small_xnnpack_fp32.pte": 1128717824, + "https://huggingface.co/software-mansion/react-native-executorch-whisper-small/resolve/v0.11.0/tokenizer.json": 2480466, + "https://huggingface.co/software-mansion/react-native-executorch-whisper-small/resolve/v0.11.0/xnnpack/whisper_small_xnnpack_int8.pte": 534980224, "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny.en/resolve/v0.10.0/coreml/whisper_tiny_en_coreml_fp16.pte": 76477024, "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny.en/resolve/v0.10.0/mlx/whisper_tiny_en_mlx_bf16.pte": 76105728, "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny.en/resolve/v0.10.0/mlx/whisper_tiny_en_mlx_int8.pte": 59889024, @@ -268,7 +347,7 @@ "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny.en/resolve/v0.10.0/vulkan/whisper_tiny_en_vulkan_fp16.pte": 213750912, "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny.en/resolve/v0.10.0/vulkan/whisper_tiny_en_vulkan_int8.pte": 129519456, "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny.en/resolve/v0.10.0/xnnpack/whisper_tiny_en_xnnpack_fp32.pte": 232606208, - "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny.en/resolve/v0.10.0/xnnpack/whisper_tiny_en_xnnpack_int8.pte": 175868672, + "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny.en/resolve/v0.11.0/xnnpack/whisper_tiny_en_xnnpack_int8.pte": 183194496, "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.10.0/coreml/whisper_tiny_coreml_fp16.pte": 76477794, "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.10.0/mlx/whisper_tiny_mlx_bf16.pte": 76106496, "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.10.0/mlx/whisper_tiny_mlx_int8.pte": 59889792, @@ -276,12 +355,20 @@ "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.10.0/vulkan/whisper_tiny_vulkan_fp16.pte": 213753984, "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.10.0/vulkan/whisper_tiny_vulkan_int8.pte": 129521380, "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.10.0/xnnpack/whisper_tiny_xnnpack_fp32.pte": 232609280, + "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.11.0/tokenizer.json": 2480466, + "https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.11.0/xnnpack/whisper_tiny_xnnpack_int8.pte": 183197568, "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.10.0/coreml/yolo26n_pose_384_coreml_fp16.pte": 6769863, "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.10.0/coreml/yolo26n_pose_512_coreml_fp16.pte": 6793319, "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.10.0/coreml/yolo26n_pose_640_coreml_fp16.pte": 6823655, "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.10.0/xnnpack/yolo26n_pose_384_xnnpack_fp32.pte": 11917828, "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.10.0/xnnpack/yolo26n_pose_512_xnnpack_fp32.pte": 11946116, "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.10.0/xnnpack/yolo26n_pose_640_xnnpack_fp32.pte": 11982340, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.11.0/384/vulkan/yolo26n_pose_384_vulkan_fp16.pte": 6048644, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.11.0/512/vulkan/yolo26n_pose_512_vulkan_fp16.pte": 6062596, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.11.0/640/vulkan/yolo26n_pose_640_vulkan_fp16.pte": 6080900, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.11.0/xnnpack/yolo26n_pose_384_xnnpack_int8.pte": 7005316, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.11.0/xnnpack/yolo26n_pose_512_xnnpack_int8.pte": 7033604, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.11.0/xnnpack/yolo26n_pose_640_xnnpack_int8.pte": 7069828, "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.10.0/l/coreml/yolo26_seg_l_384_coreml_fp16.pte": 57319863, "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.10.0/l/coreml/yolo26_seg_l_512_coreml_fp16.pte": 57334058, "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.10.0/l/coreml/yolo26_seg_l_640_coreml_fp16.pte": 57352234, @@ -312,6 +399,36 @@ "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.10.0/x/xnnpack/yolo26_seg_x_384_xnnpack_fp32.pte": 251609344, "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.10.0/x/xnnpack/yolo26_seg_x_512_xnnpack_fp32.pte": 251637632, "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.10.0/x/xnnpack/yolo26_seg_x_640_xnnpack_fp32.pte": 251673856, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/l/vulkan/yolo26_seg_l_384_vulkan_fp16.pte": 56236416, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/l/vulkan/yolo26_seg_l_512_vulkan_fp16.pte": 56250368, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/l/vulkan/yolo26_seg_l_640_vulkan_fp16.pte": 56268672, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/l/xnnpack/yolo26_seg_l_384_xnnpack_int8.pte": 35605248, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/l/xnnpack/yolo26_seg_l_512_xnnpack_int8.pte": 35633664, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/l/xnnpack/yolo26_seg_l_640_xnnpack_int8.pte": 35669760, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/m/vulkan/yolo26_seg_m_384_vulkan_fp16.pte": 47377024, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/m/vulkan/yolo26_seg_m_512_vulkan_fp16.pte": 47390976, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/m/vulkan/yolo26_seg_m_640_vulkan_fp16.pte": 47409280, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/m/xnnpack/yolo26_seg_m_384_xnnpack_int8.pte": 31032320, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/m/xnnpack/yolo26_seg_m_512_xnnpack_int8.pte": 31060608, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/m/xnnpack/yolo26_seg_m_640_xnnpack_int8.pte": 31096960, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/n/vulkan/yolo26_seg_n_384_vulkan_fp16.pte": 5651072, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/n/vulkan/yolo26_seg_n_512_vulkan_fp16.pte": 5665024, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/n/vulkan/yolo26_seg_n_640_vulkan_fp16.pte": 5683328, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/n/xnnpack/yolo26_seg_n_384_xnnpack_int8.pte": 4020480, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/n/xnnpack/yolo26_seg_n_512_xnnpack_int8.pte": 4048640, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/n/xnnpack/yolo26_seg_n_640_xnnpack_int8.pte": 4084864, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/s/vulkan/yolo26_seg_s_384_vulkan_fp16.pte": 21003264, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/s/vulkan/yolo26_seg_s_512_vulkan_fp16.pte": 21017216, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/s/vulkan/yolo26_seg_s_640_vulkan_fp16.pte": 21035520, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/s/xnnpack/yolo26_seg_s_384_xnnpack_int8.pte": 13186304, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/s/xnnpack/yolo26_seg_s_512_xnnpack_int8.pte": 13214592, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/s/xnnpack/yolo26_seg_s_640_xnnpack_int8.pte": 13250816, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/x/vulkan/yolo26_seg_x_384_vulkan_fp16.pte": 125965440, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/x/vulkan/yolo26_seg_x_512_vulkan_fp16.pte": 125979392, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/x/vulkan/yolo26_seg_x_640_vulkan_fp16.pte": 125997696, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/x/xnnpack/yolo26_seg_x_384_xnnpack_int8.pte": 79084544, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/x/xnnpack/yolo26_seg_x_512_xnnpack_int8.pte": 79112704, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/x/xnnpack/yolo26_seg_x_640_xnnpack_int8.pte": 79148928, "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.10.0/l/coreml/yolo26l_384_coreml_fp16.pte": 50874277, "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.10.0/l/coreml/yolo26l_512_coreml_fp16.pte": 50888447, "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.10.0/l/coreml/yolo26l_640_coreml_fp16.pte": 50906623, @@ -342,90 +459,37 @@ "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.10.0/x/xnnpack/yolo26x_384_xnnpack_fp32.pte": 223205568, "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.10.0/x/xnnpack/yolo26x_512_xnnpack_fp32.pte": 223233856, "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.10.0/x/xnnpack/yolo26x_640_xnnpack_fp32.pte": 223270080, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/l/vulkan/yolo26l_384_vulkan_fp16.pte": 49880068, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/l/vulkan/yolo26l_512_vulkan_fp16.pte": 49880068, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/l/vulkan/yolo26l_640_vulkan_fp16.pte": 49880068, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/l/xnnpack/yolo26l_384_xnnpack_int8.pte": 29814976, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/l/xnnpack/yolo26l_512_xnnpack_int8.pte": 29815232, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/l/xnnpack/yolo26l_640_xnnpack_int8.pte": 29815104, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/m/vulkan/yolo26m_384_vulkan_fp16.pte": 41020932, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/m/vulkan/yolo26m_512_vulkan_fp16.pte": 41020932, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/m/vulkan/yolo26m_640_vulkan_fp16.pte": 41020932, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/m/xnnpack/yolo26m_384_xnnpack_int8.pte": 25242304, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/m/xnnpack/yolo26m_512_xnnpack_int8.pte": 25242048, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/m/xnnpack/yolo26m_640_xnnpack_int8.pte": 25242048, "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/n/vulkan/yolo26n_384_vulkan_fp16.pte": 4986756, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/s/vulkan/yolo26s_640_vulkan_fp16.pte": 19165956, "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/n/vulkan/yolo26n_512_vulkan_fp16.pte": 4986756, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/m/vulkan/yolo26m_512_vulkan_fp16.pte": 41020932, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/x/vulkan/yolo26x_384_vulkan_fp16.pte": 111734788, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/n/vulkan/yolo26n_640_vulkan_fp16.pte": 4986756, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/n/xnnpack/yolo26n_384_xnnpack_int8.pte": 3170112, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/n/xnnpack/yolo26n_512_xnnpack_int8.pte": 3170112, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/n/xnnpack/yolo26n_640_xnnpack_int8.pte": 3170240, "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/s/vulkan/yolo26s_384_vulkan_fp16.pte": 19165956, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/s/vulkan/yolo26s_512_vulkan_fp16.pte": 19165956, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/s/vulkan/yolo26s_640_vulkan_fp16.pte": 19165956, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/s/xnnpack/yolo26s_384_xnnpack_int8.pte": 11358656, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/s/xnnpack/yolo26s_512_xnnpack_int8.pte": 11358912, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/s/xnnpack/yolo26s_640_xnnpack_int8.pte": 11358784, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/x/vulkan/yolo26x_384_vulkan_fp16.pte": 111734788, "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/x/vulkan/yolo26x_512_vulkan_fp16.pte": 111734788, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/m/vulkan/yolo26m_384_vulkan_fp16.pte": 41020932, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/n/vulkan/yolo26n_640_vulkan_fp16.pte": 4986756, "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/x/vulkan/yolo26x_640_vulkan_fp16.pte": 111734788, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/m/vulkan/yolo26m_640_vulkan_fp16.pte": 41020932, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/s/vulkan/yolo26s_512_vulkan_fp16.pte": 19165956, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/l/vulkan/yolo26l_640_vulkan_fp16.pte": 49880068, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/l/vulkan/yolo26l_512_vulkan_fp16.pte": 49880068, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/l/vulkan/yolo26l_384_vulkan_fp16.pte": 49880068, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/s/vulkan/yolo26_seg_s_384_vulkan_fp16.pte": 21003264, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/x/vulkan/yolo26_seg_x_512_vulkan_fp16.pte": 125979392, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/n/vulkan/yolo26_seg_n_640_vulkan_fp16.pte": 5683328, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/n/vulkan/yolo26_seg_n_512_vulkan_fp16.pte": 5665024, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/x/vulkan/yolo26_seg_x_384_vulkan_fp16.pte": 125965440, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/s/vulkan/yolo26_seg_s_512_vulkan_fp16.pte": 21017216, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/s/vulkan/yolo26_seg_s_640_vulkan_fp16.pte": 21035520, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/x/vulkan/yolo26_seg_x_640_vulkan_fp16.pte": 125997696, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/n/vulkan/yolo26_seg_n_384_vulkan_fp16.pte": 5651072, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/l/vulkan/yolo26_seg_l_384_vulkan_fp16.pte": 56236416, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/l/vulkan/yolo26_seg_l_512_vulkan_fp16.pte": 56250368, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/m/vulkan/yolo26_seg_m_512_vulkan_fp16.pte": 47390976, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.11.0/384/vulkan/yolo26n_pose_384_vulkan_fp16.pte": 6048644, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/l/vulkan/yolo26_seg_l_640_vulkan_fp16.pte": 56268672, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/m/vulkan/yolo26_seg_m_384_vulkan_fp16.pte": 47377024, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.11.0/512/vulkan/yolo26n_pose_512_vulkan_fp16.pte": 6062596, - "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-udnie/resolve/v0.11.0/vulkan/style_transfer_udnie_vulkan_fp16.pte": 3418114, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-seg/resolve/v0.11.0/m/vulkan/yolo26_seg_m_640_vulkan_fp16.pte": 47409280, - "https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose/resolve/v0.11.0/640/vulkan/yolo26n_pose_640_vulkan_fp16.pte": 6080900, - "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-rain-princess/resolve/v0.11.0/vulkan/style_transfer_rain_princess_vulkan_fp16.pte": 3418114, - "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.10.0/135m/mlx/smollm2_135m_mlx_int8.pte": 195630848, - "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.10.0/1_7b/mlx/smollm2_1_7b_mlx_int8.pte": 1966486016, - "https://huggingface.co/software-mansion/react-native-executorch-sdxs-512-dreamshaper/resolve/v0.11.0/vulkan/sdxs_512_dreamshaper_vulkan_fp16.pte": 880465922, - "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-nano-detector/resolve/v0.11.0/vulkan/rfdetr_nano_vulkan_fp16.pte": 54006916, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.10.0/1_7b/mlx/qwen_3_1_7b_mlx_int4.pte": 1536819200, - "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-nano-segmentation/resolve/v0.11.0/vulkan/rfdetr_nano_vulkan_fp16.pte": 61434756, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.10.0/4b/mlx/qwen_3_4b_mlx_int4.pte": 2915458688, - "https://huggingface.co/software-mansion/react-native-executorch-rfdetr-keypoint/resolve/v0.11.0/vulkan/rfdetr_keypoint_preview_vulkan_fp16.pte": 73445632, - "https://huggingface.co/software-mansion/react-native-executorch-smolLm-2/resolve/v0.10.0/360m/mlx/smollm2_360m_mlx_int8.pte": 467832064, - "https://huggingface.co/software-mansion/react-native-executorch-paraphrase-multilingual-MiniLM-L12-v2/resolve/v0.11.0/tokenizer.json": 17082987, - "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-mosaic/resolve/v0.11.0/vulkan/style_transfer_mosaic_vulkan_fp16.pte": 3418114, - "https://huggingface.co/software-mansion/react-native-executorch-style-transfer-candy/resolve/v0.11.0/vulkan/style_transfer_candy_vulkan_fp16.pte": 3418114, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.10.0/0_5b/mlx/qwen_2_5_0_5b_mlx_int4.pte": 535050112, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.10.0/1_5b/mlx/qwen_2_5_1_5b_mlx_int4.pte": 1287315200, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-2.5/resolve/v0.10.0/3b/mlx/qwen_2_5_3b_mlx_int4.pte": 2262418304, - "https://huggingface.co/software-mansion/react-native-executorch-paraphrase-multilingual-MiniLM-L12-v2/resolve/v0.11.0/coreml/paraphrase_multilingual_minilm_l12_v2_coreml_int8.pte": 118351833, - "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-MiniLM-L6-cos-v1/resolve/v0.11.0/coreml/multi_qa_minilm_l6_cos_v1_coreml_int8.pte": 22817740, - "https://huggingface.co/software-mansion/react-native-executorch-phi-4-mini/resolve/v0.10.0/mlx/phi_4_mini_mlx_int4.pte": 3267644544, - "https://huggingface.co/software-mansion/react-native-executorch-qwen-3/resolve/v0.10.0/0_6b/mlx/qwen_3_0_6b_mlx_int4.pte": 628164608, - "https://huggingface.co/software-mansion/react-native-executorch-multi-qa-MiniLM-L6-cos-v1/resolve/v0.11.0/tokenizer.json": 711544, - "https://huggingface.co/software-mansion/react-native-executorch-fcn/resolve/v0.11.0/coreml/fcn_resnet101_coreml_fp16.pte": 104451281, - "https://huggingface.co/software-mansion/react-native-executorch-lraspp/resolve/v0.11.0/coreml/lraspp_mobilenet_v3_large_coreml_fp16.pte": 6917692, - "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.10.0/1_5b/mlx/hammer_2_1_1_5b_mlx_int4.pte": 1286261760, - "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.10.0/3b/mlx/hammer_2_1_3b_mlx_int4.pte": 2261013504, - "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.10.0/3b/mlx/llama_3_2_3b_mlx_int4.pte": 2495273728, - "https://huggingface.co/software-mansion/react-native-executorch-fcn/resolve/v0.11.0/coreml/fcn_resnet50_coreml_fp16.pte": 66210276, - "https://huggingface.co/software-mansion/react-native-executorch-hammer-2.1/resolve/v0.10.0/0_5b/mlx/hammer_2_1_0_5b_mlx_int4.pte": 534435648, - "https://huggingface.co/software-mansion/react-native-executorch-fast-sam/resolve/v0.11.0/s/vulkan/fast_sam_s_vulkan_fp16.pte": 23721536, - "https://huggingface.co/software-mansion/react-native-executorch-llama-3.2/resolve/v0.10.0/1b/mlx/llama_3_2_1b_mlx_int4.pte": 1182100864, - "https://huggingface.co/software-mansion/react-native-executorch-fast-sam/resolve/v0.11.0/x/vulkan/fast_sam_x_vulkan_fp16.pte": 144618176, - "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.11.0/tokenizer.json": 2224112, - "https://huggingface.co/software-mansion/react-native-executorch-facemesh/resolve/v0.11.0/xnnpack/facemesh_xnnpack_fp32.pte": 2461956, - "https://huggingface.co/software-mansion/react-native-executorch-distiluse-base-multilingual-cased-v2/resolve/v0.11.0/tokenizer.json": 2919615, - "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.11.0/coreml/deeplab_v3_resnet50_coreml_fp16.pte": 79613357, - "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.11.0/coreml/deeplab_v3_mobilenet_v3_large_coreml_fp16.pte": 22547278, - "https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.11.0/coreml/deeplab_v3_resnet101_coreml_fp16.pte": 117854661, - "https://huggingface.co/software-mansion/react-native-executorch-facemesh/resolve/v0.11.0/coreml/facemesh_coreml_fp16.pte": 1559427, - "https://huggingface.co/software-mansion/react-native-executorch-distiluse-base-multilingual-cased-v2/resolve/v0.11.0/coreml/distiluse_base_multilingual_cased_v2_coreml_int8.pte": 135622796, - "https://huggingface.co/software-mansion/react-native-executorch-clip-vit-base-patch32/resolve/v0.11.0/coreml/clip_vit_base_patch32_text_coreml_int8.pte": 63921048, - "https://huggingface.co/software-mansion/react-native-executorch-blazeface/resolve/v0.11.0/vulkan/blazeface_vulkan_fp16.pte": 372866, - "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/tokenizer.json": 711544, - "https://huggingface.co/software-mansion/react-native-executorch-all-MiniLM-L6-v2/resolve/v0.11.0/coreml/all_minilm_l6_v2_coreml_int8.pte": 22817740, - "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/phonemizer/zh/lexicon.json": 2334963, - "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/phonemizer/zh/segmenter.json": 8288194, - "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaobei.bin": 522240, - "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoxiao.bin": 522240, - "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zf_xiaoyi.bin": 522240, - "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunjian.bin": 522240, - "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunxi.bin": 522240, - "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunxia.bin": 522240, - "https://huggingface.co/software-mansion/react-native-executorch-kokoro/resolve/v0.11.0/voices/zm_yunyang.bin": 522240 + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/x/xnnpack/yolo26x_384_xnnpack_int8.pte": 66161088, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/x/xnnpack/yolo26x_512_xnnpack_int8.pte": 66161088, + "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/x/xnnpack/yolo26x_640_xnnpack_int8.pte": 66161216, + "https://huggingface.co/software-mansion/react-native-executorch-zipdepth/resolve/v0.11.0/base/coreml/zipdepth_base_384_coreml_fp32.pte": 24858521, + "https://huggingface.co/software-mansion/react-native-executorch-zipdepth/resolve/v0.11.0/base/vulkan/zipdepth_base_384_vulkan_fp16.pte": 12794626, + "https://huggingface.co/software-mansion/react-native-executorch-zipdepth/resolve/v0.11.0/base/xnnpack/zipdepth_base_384_xnnpack_fp32.pte": 24609344 } diff --git a/apps/benchmarks/src/drivers.ts b/apps/benchmarks/src/drivers.ts index 5beafe17d6..f3da5e109d 100644 --- a/apps/benchmarks/src/drivers.ts +++ b/apps/benchmarks/src/drivers.ts @@ -15,12 +15,13 @@ import { createClassifier, + createColbertEmbedder, + createDepthEstimator, createFsmnVoiceActivityDetector, createImageEmbedder, createInstanceSegmenter, createKeypointDetector, createKokoroTextToSpeech, - createLLMChatSession, createObjectDetector, createPaddleOcr, createPrivacyFilter, @@ -30,10 +31,17 @@ import { createSupertonicTextToSpeech, createTextEmbedder, createWhisperSpeechToText, + createResourceScope, + defaultWorkletRuntime, + llm as llmApi, + wrapAsync, + type LLMModel, FSMN_VAD_SAMPLE_RATE_HZ, WHISPER_SAMPLE_RATE_HZ, } from 'react-native-executorch'; +import RNBlobUtil from 'react-native-blob-util'; + import { IMAGE_512, IMAGE_640, @@ -145,6 +153,22 @@ const textEmbeddings: Driver = { }, }; +const depthEstimation: Driver = { + create: createDepthEstimator, + run: (instance) => () => { + 'worklet'; + return instance.estimateDepthWorklet(IMAGE_512).data.length; + }, +}; + +const colbertEmbeddings: Driver = { + create: createColbertEmbedder, + run: (instance) => () => { + 'worklet'; + return instance.embedWorklet(SAMPLE_TEXT, 'document').numTokens; + }, +}; + const privacyFilter: Driver = { create: createPrivacyFilter, run: (instance) => () => { @@ -248,55 +272,96 @@ const kokoroTts: Driver = { note: 'timed on the RN thread; includes per-chunk thread hops', }; +type LLMBenchmarkInstance = Disposable & { + readonly runner: llmApi.LLMRunner; + readonly preprocessor: llmApi.ChatPreprocessor; +}; + /** - * The LLM driver. + * Benchmarks independent, fixed-length generations from the same chat prompt. * - * Generation is pinned to a fixed token count with EOS ignored and temperature - * zero, so every iteration on every device does exactly the same amount of - * decode work. Without that pin the measurement would depend on how long an - * answer the model felt like giving, which differs per model, per quantisation, - * and between two runs of the same model — and a wall-clock figure over a - * varying token count measures nothing. - * - * `resetOnTurn` keeps each iteration a cold prompt rather than an append onto a - * KV cache the previous iteration grew, which would make iteration N cheaper - * than iteration 1 for reasons unrelated to the build under test. - * @returns The driver, holding the last turn's stats for `detail` to read. + * A chat session retains conversation history even with `resetOnTurn`, which + * resets only its KV cache. Use the runner directly so every iteration starts + * with an empty cache and exactly one user message. + * @returns The driver with the latest generation's prefill and decode stats. */ -function llmDriver(): Driver { - // Written by each timed call, read once by `detail` after the timed pass. +function llmDriver(): Driver { let lastTtftMs = 0; let lastPrefillMs = 0; let lastPromptTokens = 0; return { - create: (config) => - createLLMChatSession(config, { - generationConfig: { - maxNewTokens: LLM_MAX_NEW_TOKENS, - temperature: 0, - ignoreEos: true, - }, - resetOnTurn: true, - }), - mode: 'async', - runAsync: (session) => async () => { - const turn = await session.sendMessage(LLM_PROMPT); - const stats = turn.stats[turn.stats.length - 1]; - if (stats) { - lastTtftMs = Math.max(0, stats.firstTokenMs - stats.inferenceStartMs); - lastPrefillMs = stats.prefillDurationMs ?? 0; - lastPromptTokens = stats.numPromptTokens; - return stats.numGeneratedTokens; + create: async (config: LLMModel) => { + const scope = createResourceScope(); + try { + const text = await RNBlobUtil.fs.readFile(config.tokenizerConfigPath, 'utf8'); + const { chatTemplate, bosToken, eosToken } = llmApi.parseTokenizerConfig(JSON.parse(text)); + const preprocessor = scope.track( + llmApi.createChatPreprocessor({ + chatTemplate, + bosToken, + eosToken, + modalities: config.modalities, + preprocessorConfig: config.preprocessorConfig, + }) + ); + const runner = scope.track( + await llmApi.createLLMRunner(config.modelPath, config.tokenizerPath, config.modalities) + ); + return { runner, preprocessor, dispose: scope.dispose }; + } catch (error) { + scope.dispose(); + throw error; } - return 0; + }, + mode: 'async', + runAsync: ({ runner, preprocessor }) => { + let expectedPromptTokens: number | undefined; + return async () => { + const prompt = preprocessor.process([{ role: 'user', content: LLM_PROMPT }], 1); + try { + const stats = await wrapAsync((input: llmApi.Prompt) => { + 'worklet'; + runner.reset(); + const started = performance.now(); + let timeToFirstTokenMs = -1; + const generation = runner.generate( + input, + { + maxNewTokens: LLM_MAX_NEW_TOKENS, + temperature: 0, + ignoreEos: true, + }, + () => { + if (timeToFirstTokenMs < 0) timeToFirstTokenMs = performance.now() - started; + } + ); + return { ...generation, timeToFirstTokenMs }; + }, defaultWorkletRuntime)(prompt); + if ( + stats.numTokens !== LLM_MAX_NEW_TOKENS || + (expectedPromptTokens !== undefined && stats.prefill.numTokens !== expectedPromptTokens) + ) { + throw new Error( + 'LLM benchmark workload changed: expected a fixed prompt and 64 generated tokens' + ); + } + expectedPromptTokens = stats.prefill.numTokens; + lastTtftMs = stats.timeToFirstTokenMs; + lastPrefillMs = stats.prefill.durationMs; + lastPromptTokens = stats.prefill.numTokens; + return stats.numTokens; + } finally { + preprocessor.clear(); + } + }; }, detail: () => ({ timeToFirstTokenMs: Math.round(lastTtftMs * 1000) / 1000, - prefillMs: Math.round(lastPrefillMs * 1000) / 1000, + prefillMs: lastPrefillMs, promptTokens: lastPromptTokens, }), - note: `decode pinned to ${LLM_MAX_NEW_TOKENS} tokens with EOS ignored; multimodal models are driven text-only`, + note: `independent single-user chat prompt; KV cache reset each iteration; ${LLM_MAX_NEW_TOKENS} generated tokens with EOS ignored; multimodal models are driven text-only`, }; } @@ -312,6 +377,8 @@ const BY_TASK: Readonly> = { instanceSegmentation, imageEmbeddings, textEmbeddings, + depthEstimation, + colbertEmbeddings, privacyFilter, voiceActivityDetection, speechToText, diff --git a/docs/docs/04-benchmarks/01-benchmarks.mdx b/docs/docs/04-benchmarks/01-benchmarks.mdx index 313f6e6fe5..53f72a0543 100644 --- a/docs/docs/04-benchmarks/01-benchmarks.mdx +++ b/docs/docs/04-benchmarks/01-benchmarks.mdx @@ -26,28 +26,31 @@ export const benchmarks = mergeBenchmarkRows(measured, imported); # Benchmarks -Every model variant in the React Native ExecuTorch registry is measured directly on physical iOS and Android hardware. We profile the complete end-to-end pipeline—from camera frames, audio, and text through pre-processing, forward inference on mobile silicon, and post-processing—alongside isolated ExecuTorch runtime latency, peak RAM, cold-start load times, and binary download sizes. +These benchmarks measure model variants directly on physical iOS and Android hardware. Coverage varies by task, model variant, and device; each chart shows the measurements available for its selected model. We profile the complete end-to-end pipeline—from camera frames, audio, and text through pre-processing, forward inference on mobile silicon, and post-processing—alongside isolated ExecuTorch runtime latency, peak RAM, cold-start load times, and binary download sizes. ### Backends and Precision Each variant is defined by two choices: - **Backend (Where it runs):** The hardware execution engine. **XNNPACK** runs on mobile CPUs for universal compatibility across iOS and Android; **Core ML** targets the Apple Neural Engine (NPU) and GPU on iOS; **Vulkan** runs on Android GPUs; and **MLX** uses Apple Silicon unified memory. To configure which backends are compiled into your app, see [Native Libraries](../03-core-and-advanced/08-native-libraries.md). -- **Precision (How weights are stored):** The numerical format of model weights. Full precision (`fp32`) preserves original fidelity at the cost of larger downloads and RAM. Quantized formats (`fp16`, `int8`, `4-bit`) compress weights, drastically cutting file size and memory while speeding up compute with minimal impact on real-world accuracy. See [Exporting Custom Models](../03-core-and-advanced/07-exporting-custom-models.md) for lowering details. +- **Precision (How weights are stored):** The numerical format of model weights. Full precision (`fp32`) preserves original fidelity at the cost of larger downloads and RAM. Reduced precision (`fp16`) and integer quantization (`int8`, `4-bit`) reduce weight storage and memory use and can speed up computation; evaluate accuracy for your task. See [Exporting Custom Models](../03-core-and-advanced/07-exporting-custom-models.md) for lowering details. Explore the charts below to compare backends and find the optimal balance for your app. ## Devices -Measurements were collected on physical iOS and Android devices spanning flagship, modern mid-tier, and legacy hardware tiers. All numbers come from release builds of React Native ExecuTorch 0.11.0-dev (`main` shortly after v0.10.0), measured in September 2026. +Measurements were collected on physical iOS and Android devices spanning flagship, modern mid-tier, and legacy hardware tiers. All numbers come from release builds of React Native ExecuTorch 0.11.0-dev (`main` shortly after v0.10.0), measured in September 2026, with supplemental quantized LLM, depth estimation, and ColBERT runs in October 2026. | Device | Backends tested | Variants | Notes | |---|---|---|---| -| Samsung Galaxy S26 Ultra | XNNPACK (123), Vulkan (24) | 147 | Flagship Android test device (Snapdragon SM8850); covers all standard tasks and 23 LLM variants | -| Google Pixel 10 | XNNPACK (100), Vulkan (24) | 124 | Modern Android device featuring Google Tensor G5 | -| Samsung Galaxy S20+ | XNNPACK (100), Vulkan (24) | 124 | Baseline Android device (Exynos 990) representing older generation hardware | -| iPhone 17 | XNNPACK (100), Core ML (75), MLX (15) | 190 | Flagship iOS reference device (A19) testing XNNPACK, Core ML, and MLX execution | -| iPhone SE (3rd gen) | XNNPACK (99), Core ML (73), MLX (15) | 187 | Compact iOS hardware (A15 Bionic) with 4 GB RAM to test performance and memory bounds | +| Samsung Galaxy S26 Ultra | XNNPACK (121), Vulkan (24) | 145 | Snapdragon SM8850; historical standard-task results and 21 quantized LLM variants | +| Google Pixel 10 | XNNPACK (100), Vulkan (24) | 124 | Google Tensor G5; historical standard-task measurements | +| Samsung Galaxy S20+ | XNNPACK (129), Vulkan (31) | 160 | Exynos 990; historical results plus supplemental quantized LLM, depth, and ColBERT measurements | +| Samsung Galaxy S24 | XNNPACK (29), Vulkan (6) | 35 | Supplemental quantized LLM, depth estimation, and ColBERT measurements | +| iPhone 17 | XNNPACK (127), Core ML (78), MLX (38) | 243 | A19; historical results plus supplemental quantized LLM, depth, and ColBERT measurements | +| iPhone SE (3rd gen) | XNNPACK (99), Core ML (73), MLX (15) | 187 | A15 Bionic with 4 GB RAM; historical standard-task measurements | + +Memory values report peak process footprint during a separate inference pass, including the app baseline: total PSS on Android and `phys_footprint` on iOS. Some supplemental iPhone LLM measurements use a fresh process for each model. ## Results @@ -110,6 +113,15 @@ These benchmarks measure execution latency, memory utilization, and initializati +### [Depth Estimation](../02-extensions/computer-vision/12-depth-estimation.md) + +:::info Input & Pipeline Details +- **Input:** Fixed 512x512 RGBA synthetic frame. +- **Pipeline Structure:** The depth estimator resizes and normalizes the frame, executes the model, and resizes the raw depth map back to the input dimensions. Timings cover the full pipeline; output values retain the model's depth interpretation. +::: + + + ### [Image Embeddings](../02-extensions/computer-vision/09-image-embeddings.md) :::info Input & Pipeline Details @@ -147,6 +159,15 @@ These benchmarks measure execution latency, memory utilization, and initializati +### [ColBERT Embeddings](../02-extensions/natural-language/06-colbert-retrieval.md) + +:::info Input & Pipeline Details +- **Input:** The same fixed English paragraph used for text embeddings, encoded as a document. +- **Pipeline Structure:** Tokenization, document marker insertion and padding, model execution, and removal of masked or skiplisted token vectors. The result is a matrix of normalized per-token embeddings. Query encoding and MaxSim retrieval scoring are outside this measurement. +::: + + + ### [Privacy Filter](../02-extensions/natural-language/04-privacy-filter.md) :::info Input & Pipeline Details @@ -159,10 +180,14 @@ These benchmarks measure execution latency, memory utilization, and initializati ### [LLM Chat & Text Generation](../02-extensions/natural-language/02-llm-chat-and-generation.md) :::info Input & Pipeline Details -- **Input:** Prompt (`List three uses for a paperclip.`), 64 tokens decoded with EOS ignored, temperature 0 (greedy decoding). +- **Variants:** Quantized LLMs only. +- **Collection:** The S26 Ultra LLM results retain the September measurements. Supplemental S20+, S24, and iPhone 17 LLM runs were collected in October using the fixed-prompt harness described below. +- **Input:** Prompt (`List three uses for a paperclip.`), 64 tokens generated with EOS ignored, temperature 0 (greedy decoding), and the KV cache reset before each independent prompt. - **Prefill vs. Decode & Memory:** The short prompt isolates autoregressive token generation throughput (tokens/second) and time to first token (TTFT). Autoregressive decoding executes one forward pass per generated token. Peak memory is driven by model weights and allocated key-value (KV) cache buffers, with models larger than 2 GB peaking between 2.4 and 3.5 GB of resident RAM. ::: +On iPhone 17, Phi-4 Mini was omitted. Qwen3 4B (XNNPACK) terminated after critical memory-pressure warnings, including in a fresh-process retry. On Galaxy S24, Android's low-memory killer terminated Gemma4 Omni E2B (Vulkan) during loading in a fresh-process retry. Only successful measurements are charted. + ### [Speech to Text](../02-extensions/speech/03-speech-to-text.md) diff --git a/docs/docs/04-benchmarks/imported.json b/docs/docs/04-benchmarks/imported.json index 621b2c012c..115469c858 100644 --- a/docs/docs/04-benchmarks/imported.json +++ b/docs/docs/04-benchmarks/imported.json @@ -10597,20 +10597,6 @@ "loadMs": 1070, "sizeMb": 1093 }, - { - "task": "llm", - "model": "lfm2-5-350-m", - "backend": "xnnpack", - "precision": "fp16", - "device": "s26ultra", - "medianMs": 2664, - "meanMs": 2664, - "tokensPerSecond": 23.6, - "ttftMs": 21, - "peakMb": 1212, - "loadMs": 585, - "sizeMb": 810 - }, { "task": "llm", "model": "qwen2-5-1-5-b", @@ -10792,19 +10778,5 @@ "peakMb": 3428, "loadMs": 3089, "sizeMb": 2543 - }, - { - "task": "llm", - "model": "qwen2-5-0-5-b", - "backend": "xnnpack", - "precision": "bf16", - "device": "s26ultra", - "medianMs": 135431, - "meanMs": 135431, - "tokensPerSecond": 0.5, - "ttftMs": 321, - "peakMb": 1536, - "loadMs": 500, - "sizeMb": 950 } ] diff --git a/docs/docs/04-benchmarks/measured.json b/docs/docs/04-benchmarks/measured.json index fe51488c70..069ffdb931 100644 --- a/docs/docs/04-benchmarks/measured.json +++ b/docs/docs/04-benchmarks/measured.json @@ -1 +1,1738 @@ -[] +[ + { + "task": "depth-estimation", + "model": "zipdepth-base", + "backend": "xnnpack", + "precision": "fp32", + "device": "iphone17", + "medianMs": 22, + "meanMs": 22.05, + "etMs": 21.51, + "share": 98, + "peakMb": 163, + "loadMs": 24, + "sizeMb": 23 + }, + { + "task": "depth-estimation", + "model": "zipdepth-base", + "backend": "coreml", + "precision": "fp32", + "device": "iphone17", + "medianMs": 5.98, + "meanMs": 6.01, + "etMs": 5.51, + "share": 92, + "peakMb": 163, + "loadMs": 41, + "sizeMb": 24 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v2-small", + "backend": "xnnpack", + "precision": "int8", + "device": "iphone17", + "medianMs": 269.08, + "meanMs": 269.01, + "etMs": 267.65, + "share": 99, + "peakMb": 220, + "loadMs": 25, + "sizeMb": 34 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v3-small", + "backend": "xnnpack", + "precision": "int8", + "device": "iphone17", + "medianMs": 340.06, + "meanMs": 339.54, + "etMs": 337.93, + "share": 100, + "peakMb": 334, + "loadMs": 26, + "sizeMb": 39 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v2-small", + "backend": "coreml", + "precision": "fp16", + "device": "iphone17", + "medianMs": 52.15, + "meanMs": 52.84, + "etMs": 50.88, + "share": 96, + "peakMb": 72, + "loadMs": 113, + "sizeMb": 48 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v3-small", + "backend": "coreml", + "precision": "fp16", + "device": "iphone17", + "medianMs": 58.4, + "meanMs": 59.05, + "etMs": 56.88, + "share": 96, + "peakMb": 75, + "loadMs": 130, + "sizeMb": 68 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v2-small", + "backend": "xnnpack", + "precision": "fp32", + "device": "iphone17", + "medianMs": 432.13, + "meanMs": 430.27, + "etMs": 428.8, + "share": 100, + "peakMb": 272, + "loadMs": 27, + "sizeMb": 94 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v3-small", + "backend": "xnnpack", + "precision": "fp32", + "device": "iphone17", + "medianMs": 512.84, + "meanMs": 511.74, + "etMs": 510, + "share": 100, + "peakMb": 396, + "loadMs": 26, + "sizeMb": 102 + }, + { + "task": "colbert-embeddings", + "model": "lfm2-5-colbert-350-m", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 19.78, + "meanMs": 19.79, + "etMs": 19.61, + "share": 99, + "peakMb": 163, + "loadMs": 77, + "sizeMb": 184 + }, + { + "task": "colbert-embeddings", + "model": "lfm2-5-colbert-350-m", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 72.09, + "meanMs": 71.79, + "etMs": 71.46, + "share": 100, + "peakMb": 733, + "loadMs": 206, + "sizeMb": 360 + }, + { + "task": "llm", + "model": "smollm2-135-m", + "backend": "xnnpack", + "precision": "8da8w", + "device": "iphone17", + "medianMs": 453.34, + "meanMs": 452.16, + "tokensPerSecond": 141.2, + "ttftMs": 32, + "peakMb": 956, + "loadMs": 125, + "sizeMb": 167 + }, + { + "task": "llm", + "model": "smollm2-135-m", + "backend": "mlx", + "precision": "int8", + "device": "iphone17", + "medianMs": 247.97, + "meanMs": 249.15, + "tokensPerSecond": 258.1, + "ttftMs": 13, + "peakMb": 467, + "loadMs": 76, + "sizeMb": 189 + }, + { + "task": "llm", + "model": "lfm2-5-350-m", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 480.43, + "meanMs": 479.76, + "tokensPerSecond": 133.2, + "ttftMs": 28, + "peakMb": 1047, + "loadMs": 239, + "sizeMb": 283 + }, + { + "task": "llm", + "model": "lfm2-5-350-m", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 268.32, + "meanMs": 270.69, + "tokensPerSecond": 238.5, + "ttftMs": 15, + "peakMb": 523, + "loadMs": 91, + "sizeMb": 312 + }, + { + "task": "llm", + "model": "smollm2-360-m", + "backend": "xnnpack", + "precision": "8da8w", + "device": "iphone17", + "medianMs": 902.93, + "meanMs": 903.6, + "tokensPerSecond": 70.9, + "ttftMs": 66, + "peakMb": 2035, + "loadMs": 256, + "sizeMb": 402 + }, + { + "task": "llm", + "model": "qwen2-5-0-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 716.7, + "meanMs": 716.93, + "tokensPerSecond": 89.3, + "ttftMs": 64, + "peakMb": 1163, + "loadMs": 397, + "sizeMb": 419 + }, + { + "task": "llm", + "model": "hammer2-1-0-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 665.02, + "meanMs": 663.43, + "tokensPerSecond": 96.2, + "ttftMs": 49, + "peakMb": 1162, + "loadMs": 394, + "sizeMb": 423 + }, + { + "task": "llm", + "model": "smollm2-360-m", + "backend": "mlx", + "precision": "int8", + "device": "iphone17", + "medianMs": 550.98, + "meanMs": 552.96, + "tokensPerSecond": 116.2, + "ttftMs": 36, + "peakMb": 990, + "loadMs": 90, + "sizeMb": 448 + }, + { + "task": "llm", + "model": "qwen3-0-6-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 817.25, + "meanMs": 814.83, + "tokensPerSecond": 78.3, + "ttftMs": 46, + "peakMb": 2639, + "loadMs": 538, + "sizeMb": 505 + }, + { + "task": "llm", + "model": "qwen2-5-0-5-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 406.46, + "meanMs": 409.86, + "tokensPerSecond": 157.5, + "ttftMs": 28, + "peakMb": 544, + "loadMs": 179, + "sizeMb": 517 + }, + { + "task": "llm", + "model": "hammer2-1-0-5-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 402.41, + "meanMs": 405.6, + "tokensPerSecond": 159, + "ttftMs": 22, + "peakMb": 559, + "loadMs": 225, + "sizeMb": 521 + }, + { + "task": "llm", + "model": "lfm2-5-vl-450-m", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 334.18, + "meanMs": 335.71, + "tokensPerSecond": 191.5, + "ttftMs": 17, + "peakMb": 393, + "loadMs": 88, + "sizeMb": 560 + }, + { + "task": "llm", + "model": "lfm2-5-vl-450-m", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 840.3, + "meanMs": 843.82, + "tokensPerSecond": 76.2, + "ttftMs": 34, + "peakMb": 1287, + "loadMs": 346, + "sizeMb": 623 + }, + { + "task": "llm", + "model": "lfm2-5-1-2-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 1393.2, + "meanMs": 1363.53, + "tokensPerSecond": 45.9, + "ttftMs": 72, + "peakMb": 1619, + "loadMs": 565, + "sizeMb": 777 + }, + { + "task": "llm", + "model": "lfm2-5-1-2-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 776.57, + "meanMs": 778.33, + "tokensPerSecond": 82.4, + "ttftMs": 54, + "peakMb": 551, + "loadMs": 94, + "sizeMb": 854 + }, + { + "task": "llm", + "model": "qwen3-0-6-b", + "backend": "mlx", + "precision": "int8", + "device": "iphone17", + "medianMs": 763.12, + "meanMs": 766.06, + "tokensPerSecond": 83.9, + "ttftMs": 27, + "peakMb": 1220, + "loadMs": 236, + "sizeMb": 894 + }, + { + "task": "llm", + "model": "bielik-v3-1-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 2002.22, + "meanMs": 1918.2, + "tokensPerSecond": 32, + "ttftMs": 122, + "peakMb": 2055, + "loadMs": 701, + "sizeMb": 896 + }, + { + "task": "llm", + "model": "llama3-2-1-b", + "backend": "xnnpack", + "precision": "spinquant", + "device": "iphone17", + "medianMs": 1645.41, + "meanMs": 1614.32, + "tokensPerSecond": 38.9, + "ttftMs": 142, + "peakMb": 2445, + "loadMs": 797, + "sizeMb": 1042 + }, + { + "task": "llm", + "model": "qwen2-5-1-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 1893.7, + "meanMs": 1812.4, + "tokensPerSecond": 33.8, + "ttftMs": 173, + "peakMb": 2444, + "loadMs": 866, + "sizeMb": 1118 + }, + { + "task": "llm", + "model": "hammer2-1-1-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 1915.97, + "meanMs": 1848.99, + "tokensPerSecond": 33.4, + "ttftMs": 110, + "peakMb": 2444, + "loadMs": 850, + "sizeMb": 1122 + }, + { + "task": "llm", + "model": "llama3-2-1-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 828.66, + "meanMs": 834.16, + "tokensPerSecond": 77.2, + "ttftMs": 83, + "peakMb": 898, + "loadMs": 239, + "sizeMb": 1137 + }, + { + "task": "llm", + "model": "qwen2-5-1-5-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 1066.01, + "meanMs": 1071.65, + "tokensPerSecond": 60, + "ttftMs": 87, + "peakMb": 868, + "loadMs": 188, + "sizeMb": 1234 + }, + { + "task": "llm", + "model": "hammer2-1-1-5-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 1037.77, + "meanMs": 1049.2, + "tokensPerSecond": 61.7, + "ttftMs": 76, + "peakMb": 865, + "loadMs": 221, + "sizeMb": 1238 + }, + { + "task": "llm", + "model": "qwen3-1-7-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 1983.52, + "meanMs": 1972.73, + "tokensPerSecond": 32.3, + "ttftMs": 95, + "peakMb": 2459, + "loadMs": 931, + "sizeMb": 1258 + }, + { + "task": "llm", + "model": "qwen3-1-7-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 1150.58, + "meanMs": 1157.8, + "tokensPerSecond": 55.6, + "ttftMs": 66, + "peakMb": 878, + "loadMs": 221, + "sizeMb": 1477 + }, + { + "task": "llm", + "model": "lfm2-5-2-6-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 1940.24, + "meanMs": 1956.91, + "tokensPerSecond": 33, + "ttftMs": 116, + "peakMb": 799, + "loadMs": 304, + "sizeMb": 1606 + }, + { + "task": "llm", + "model": "lfm2-5-2-6-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 3285.95, + "meanMs": 3173.94, + "tokensPerSecond": 19.5, + "ttftMs": 159, + "peakMb": 2119, + "loadMs": 1445, + "sizeMb": 1724 + }, + { + "task": "llm", + "model": "smollm2-1-7-b", + "backend": "xnnpack", + "precision": "8da8w", + "device": "iphone17", + "medianMs": 3608.63, + "meanMs": 3366.15, + "tokensPerSecond": 17.7, + "ttftMs": 167, + "peakMb": 3068, + "loadMs": 1850, + "sizeMb": 1735 + }, + { + "task": "llm", + "model": "smollm2-1-7-b", + "backend": "mlx", + "precision": "int8", + "device": "iphone17", + "medianMs": 2078.59, + "meanMs": 2082.61, + "tokensPerSecond": 30.8, + "ttftMs": 98, + "peakMb": 810, + "loadMs": 82, + "sizeMb": 1877 + }, + { + "task": "llm", + "model": "hammer2-1-3-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 3678.86, + "meanMs": 3546.37, + "tokensPerSecond": 17.4, + "ttftMs": 231, + "peakMb": 3050, + "loadMs": 1755, + "sizeMb": 1980 + }, + { + "task": "llm", + "model": "qwen2-5-3-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 3847.98, + "meanMs": 3672.98, + "tokensPerSecond": 16.6, + "ttftMs": 304, + "peakMb": 3043, + "loadMs": 1566, + "sizeMb": 2014 + }, + { + "task": "llm", + "model": "qwen2-5-3-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 2054.34, + "meanMs": 2069.73, + "tokensPerSecond": 31.2, + "ttftMs": 174, + "peakMb": 774, + "loadMs": 193, + "sizeMb": 2164 + }, + { + "task": "llm", + "model": "hammer2-1-3-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 2026.17, + "meanMs": 2038.97, + "tokensPerSecond": 31.6, + "ttftMs": 123, + "peakMb": 790, + "loadMs": 222, + "sizeMb": 2167 + }, + { + "task": "llm", + "model": "lfm2-5-vl-1-6-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 888.95, + "meanMs": 891.52, + "tokensPerSecond": 72, + "ttftMs": 60, + "peakMb": 497, + "loadMs": 94, + "sizeMb": 2265 + }, + { + "task": "llm", + "model": "llama3-2-3-b", + "backend": "xnnpack", + "precision": "spinquant", + "device": "iphone17", + "medianMs": 4297.54, + "meanMs": 4125.63, + "tokensPerSecond": 14.9, + "ttftMs": 372, + "peakMb": 3150, + "loadMs": 2333, + "sizeMb": 2279 + }, + { + "task": "llm", + "model": "lfm2-5-vl-1-6-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 1653.66, + "meanMs": 1598.27, + "tokensPerSecond": 38.7, + "ttftMs": 60, + "peakMb": 2974, + "loadMs": 1204, + "sizeMb": 2365 + }, + { + "task": "llm", + "model": "llama3-2-3-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 2221.09, + "meanMs": 2240.28, + "tokensPerSecond": 28.8, + "ttftMs": 206, + "peakMb": 709, + "loadMs": 246, + "sizeMb": 2389 + }, + { + "task": "llm", + "model": "gemma4-e2-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 1834.95, + "meanMs": 1851.91, + "tokensPerSecond": 34.9, + "ttftMs": 111, + "peakMb": 534, + "loadMs": 614, + "sizeMb": 2441 + }, + { + "task": "llm", + "model": "gemma4-omni-e2-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 1891.81, + "meanMs": 1927.59, + "tokensPerSecond": 33.8, + "ttftMs": 109, + "peakMb": 612, + "loadMs": 781, + "sizeMb": 2633 + }, + { + "task": "llm", + "model": "qwen3-4-b", + "backend": "mlx", + "precision": "int4", + "device": "iphone17", + "medianMs": 2828.86, + "meanMs": 2871.85, + "tokensPerSecond": 22.6, + "ttftMs": 181, + "peakMb": 816, + "loadMs": 236, + "sizeMb": 2791 + }, + { + "task": "llm", + "model": "lfm2-5-vl-1-6-b", + "backend": "mlx", + "precision": "int8", + "device": "iphone17", + "medianMs": 1504.48, + "meanMs": 1514.6, + "tokensPerSecond": 42.5, + "ttftMs": 70, + "peakMb": 216, + "loadMs": 93, + "sizeMb": 2823 + }, + { + "task": "llm", + "model": "gemma4-e2-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 6377.98, + "meanMs": 6280.15, + "tokensPerSecond": 10, + "ttftMs": 256, + "peakMb": 1934, + "loadMs": 2150, + "sizeMb": 3097 + }, + { + "task": "llm", + "model": "gemma4-omni-e2-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "iphone17", + "medianMs": 6249, + "meanMs": 6195.53, + "tokensPerSecond": 10.2, + "ttftMs": 281, + "peakMb": 2394, + "loadMs": 2636, + "sizeMb": 3482 + }, + { + "task": "depth-estimation", + "model": "zipdepth-base", + "backend": "vulkan", + "precision": "fp16", + "device": "s20plus", + "medianMs": 66.53, + "meanMs": 68.75, + "etMs": 59.33, + "share": 86, + "peakMb": 198, + "loadMs": 63, + "sizeMb": 12 + }, + { + "task": "depth-estimation", + "model": "zipdepth-base", + "backend": "xnnpack", + "precision": "fp32", + "device": "s20plus", + "medianMs": 146.82, + "meanMs": 161.69, + "etMs": 156.38, + "share": 97, + "peakMb": 271, + "loadMs": 46, + "sizeMb": 23 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v2-small", + "backend": "xnnpack", + "precision": "int8", + "device": "s20plus", + "medianMs": 926.98, + "meanMs": 938.87, + "etMs": 930.2, + "share": 99, + "peakMb": 359, + "loadMs": 73, + "sizeMb": 34 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v3-small", + "backend": "xnnpack", + "precision": "int8", + "device": "s20plus", + "medianMs": 1083.15, + "meanMs": 1083.94, + "etMs": 1074.56, + "share": 99, + "peakMb": 479, + "loadMs": 108, + "sizeMb": 39 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v2-small", + "backend": "vulkan", + "precision": "fp16", + "device": "s20plus", + "medianMs": 886.57, + "meanMs": 887.16, + "etMs": 870.82, + "share": 98, + "peakMb": 214, + "loadMs": 82, + "sizeMb": 47 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v3-small", + "backend": "vulkan", + "precision": "fp16", + "device": "s20plus", + "medianMs": 1131.9, + "meanMs": 1131.27, + "etMs": 1113.97, + "share": 98, + "peakMb": 219, + "loadMs": 84, + "sizeMb": 54 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v2-small", + "backend": "xnnpack", + "precision": "fp32", + "device": "s20plus", + "medianMs": 1332.5, + "meanMs": 1322.95, + "etMs": 1313.43, + "share": 99, + "peakMb": 438, + "loadMs": 98, + "sizeMb": 94 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v3-small", + "backend": "xnnpack", + "precision": "fp32", + "device": "s20plus", + "medianMs": 1472.76, + "meanMs": 1467.81, + "etMs": 1458.13, + "share": 99, + "peakMb": 572, + "loadMs": 112, + "sizeMb": 102 + }, + { + "task": "colbert-embeddings", + "model": "lfm2-5-colbert-350-m", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 214.96, + "meanMs": 260.77, + "etMs": 259.45, + "share": 99, + "peakMb": 674, + "loadMs": 1506, + "sizeMb": 360 + }, + { + "task": "llm", + "model": "smollm2-135-m", + "backend": "xnnpack", + "precision": "8da8w", + "device": "s20plus", + "medianMs": 645.75, + "meanMs": 647.45, + "tokensPerSecond": 99.1, + "ttftMs": 53, + "peakMb": 1104, + "loadMs": 750, + "sizeMb": 167 + }, + { + "task": "llm", + "model": "lfm2-5-350-m", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 747.28, + "meanMs": 747.55, + "tokensPerSecond": 85.6, + "ttftMs": 52, + "peakMb": 993, + "loadMs": 973, + "sizeMb": 283 + }, + { + "task": "llm", + "model": "smollm2-360-m", + "backend": "xnnpack", + "precision": "8da8w", + "device": "s20plus", + "medianMs": 1367.59, + "meanMs": 1370.19, + "tokensPerSecond": 46.8, + "ttftMs": 108, + "peakMb": 1948, + "loadMs": 1536, + "sizeMb": 402 + }, + { + "task": "llm", + "model": "qwen2-5-0-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 1224.25, + "meanMs": 1241.43, + "tokensPerSecond": 52.3, + "ttftMs": 160, + "peakMb": 1159, + "loadMs": 2373, + "sizeMb": 419 + }, + { + "task": "llm", + "model": "hammer2-1-0-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 1260.94, + "meanMs": 1266.13, + "tokensPerSecond": 50.8, + "ttftMs": 124, + "peakMb": 1150, + "loadMs": 3596, + "sizeMb": 423 + }, + { + "task": "llm", + "model": "qwen3-0-6-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 1580.73, + "meanMs": 1581.17, + "tokensPerSecond": 40.5, + "ttftMs": 104, + "peakMb": 2638, + "loadMs": 4865, + "sizeMb": 505 + }, + { + "task": "llm", + "model": "lfm2-5-vl-450-m", + "backend": "vulkan", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 3735.85, + "meanMs": 3734.64, + "tokensPerSecond": 17.1, + "ttftMs": 279, + "peakMb": 652, + "loadMs": 2005, + "sizeMb": 598 + }, + { + "task": "llm", + "model": "lfm2-5-vl-450-m", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 1691.66, + "meanMs": 1687.42, + "tokensPerSecond": 37.8, + "ttftMs": 80, + "peakMb": 1247, + "loadMs": 2493, + "sizeMb": 623 + }, + { + "task": "llm", + "model": "lfm2-5-1-2-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 2237.59, + "meanMs": 2246.54, + "tokensPerSecond": 28.6, + "ttftMs": 187, + "peakMb": 1607, + "loadMs": 4405, + "sizeMb": 777 + }, + { + "task": "llm", + "model": "bielik-v3-1-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 3214.33, + "meanMs": 3242.84, + "tokensPerSecond": 19.9, + "ttftMs": 388, + "peakMb": 1972, + "loadMs": 5136, + "sizeMb": 896 + }, + { + "task": "llm", + "model": "llama3-2-1-b", + "backend": "xnnpack", + "precision": "spinquant", + "device": "s20plus", + "medianMs": 3009.68, + "meanMs": 3010.25, + "tokensPerSecond": 21.3, + "ttftMs": 435, + "peakMb": 2425, + "loadMs": 6865, + "sizeMb": 1042 + }, + { + "task": "llm", + "model": "qwen2-5-1-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 3331.43, + "meanMs": 3349.98, + "tokensPerSecond": 19.2, + "ttftMs": 467, + "peakMb": 2402, + "loadMs": 6700, + "sizeMb": 1118 + }, + { + "task": "llm", + "model": "hammer2-1-1-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 3187.46, + "meanMs": 3200.34, + "tokensPerSecond": 20.1, + "ttftMs": 344, + "peakMb": 2398, + "loadMs": 7026, + "sizeMb": 1122 + }, + { + "task": "llm", + "model": "qwen3-1-7-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 3406.91, + "meanMs": 3412.99, + "tokensPerSecond": 18.8, + "ttftMs": 236, + "peakMb": 2465, + "loadMs": 7410, + "sizeMb": 1258 + }, + { + "task": "llm", + "model": "lfm2-5-2-6-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 4935.98, + "meanMs": 4952.27, + "tokensPerSecond": 13, + "ttftMs": 428, + "peakMb": 2103, + "loadMs": 10317, + "sizeMb": 1724 + }, + { + "task": "llm", + "model": "smollm2-1-7-b", + "backend": "xnnpack", + "precision": "8da8w", + "device": "s20plus", + "medianMs": 5481.17, + "meanMs": 5478.19, + "tokensPerSecond": 11.7, + "ttftMs": 380, + "peakMb": 2991, + "loadMs": 8496, + "sizeMb": 1735 + }, + { + "task": "llm", + "model": "hammer2-1-3-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 5937.64, + "meanMs": 5977.03, + "tokensPerSecond": 10.8, + "ttftMs": 701, + "peakMb": 3043, + "loadMs": 11143, + "sizeMb": 1980 + }, + { + "task": "llm", + "model": "qwen2-5-3-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 6247.28, + "meanMs": 6377.18, + "tokensPerSecond": 10.2, + "ttftMs": 951, + "peakMb": 3043, + "loadMs": 10801, + "sizeMb": 2014 + }, + { + "task": "llm", + "model": "llama3-2-3-b", + "backend": "xnnpack", + "precision": "spinquant", + "device": "s20plus", + "medianMs": 7174.2, + "meanMs": 7191.57, + "tokensPerSecond": 8.9, + "ttftMs": 1118, + "peakMb": 3461, + "loadMs": 12027, + "sizeMb": 2279 + }, + { + "task": "llm", + "model": "lfm2-5-vl-1-6-b", + "backend": "vulkan", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 7708.82, + "meanMs": 7702.3, + "tokensPerSecond": 8.3, + "ttftMs": 715, + "peakMb": 2000, + "loadMs": 5950, + "sizeMb": 2357 + }, + { + "task": "llm", + "model": "lfm2-5-vl-1-6-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 2967.07, + "meanMs": 2970.59, + "tokensPerSecond": 21.6, + "ttftMs": 187, + "peakMb": 3210, + "loadMs": 7150, + "sizeMb": 2365 + }, + { + "task": "llm", + "model": "qwen3-4-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 7443.8, + "meanMs": 7455.25, + "tokensPerSecond": 8.6, + "ttftMs": 529, + "peakMb": 4122, + "loadMs": 14155, + "sizeMb": 2572 + }, + { + "task": "llm", + "model": "phi4-mini", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 6812.34, + "meanMs": 6867.26, + "tokensPerSecond": 9.4, + "ttftMs": 376, + "peakMb": 3780, + "loadMs": 14275, + "sizeMb": 2703 + }, + { + "task": "llm", + "model": "gemma4-e2-b", + "backend": "vulkan", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 22629.21, + "meanMs": 22627.66, + "tokensPerSecond": 2.8, + "ttftMs": 1479, + "peakMb": 327, + "loadMs": 14580, + "sizeMb": 2850 + }, + { + "task": "llm", + "model": "gemma4-e2-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 22126.72, + "meanMs": 21977.8, + "tokensPerSecond": 2.9, + "ttftMs": 1202, + "peakMb": 2124, + "loadMs": 15037, + "sizeMb": 3097 + }, + { + "task": "llm", + "model": "gemma4-omni-e2-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 22170.49, + "meanMs": 22159.63, + "tokensPerSecond": 2.9, + "ttftMs": 1261, + "peakMb": 2602, + "loadMs": 17011, + "sizeMb": 3482 + }, + { + "task": "llm", + "model": "gemma4-omni-e2-b", + "backend": "vulkan", + "precision": "8da4w", + "device": "s20plus", + "medianMs": 21888.72, + "meanMs": 21882.11, + "tokensPerSecond": 2.9, + "ttftMs": 1336, + "peakMb": 623, + "loadMs": 17602, + "sizeMb": 3661 + }, + { + "task": "depth-estimation", + "model": "zipdepth-base", + "backend": "vulkan", + "precision": "fp16", + "device": "s24", + "medianMs": 19.08, + "meanMs": 20.16, + "etMs": 16.67, + "share": 83, + "peakMb": 264, + "loadMs": 18, + "sizeMb": 12 + }, + { + "task": "depth-estimation", + "model": "zipdepth-base", + "backend": "xnnpack", + "precision": "fp32", + "device": "s24", + "medianMs": 76.59, + "meanMs": 74.74, + "etMs": 72.64, + "share": 97, + "peakMb": 299, + "loadMs": 36, + "sizeMb": 23 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v2-small", + "backend": "xnnpack", + "precision": "int8", + "device": "s24", + "medianMs": 493.13, + "meanMs": 493, + "etMs": 489.05, + "share": 99, + "peakMb": 392, + "loadMs": 31, + "sizeMb": 34 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v3-small", + "backend": "xnnpack", + "precision": "int8", + "device": "s24", + "medianMs": 591.37, + "meanMs": 595.11, + "etMs": 591.66, + "share": 99, + "peakMb": 498, + "loadMs": 53, + "sizeMb": 39 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v2-small", + "backend": "vulkan", + "precision": "fp16", + "device": "s24", + "medianMs": 261.12, + "meanMs": 261.34, + "etMs": 256.54, + "share": 98, + "peakMb": 370, + "loadMs": 59, + "sizeMb": 47 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v3-small", + "backend": "vulkan", + "precision": "fp16", + "device": "s24", + "medianMs": 324.28, + "meanMs": 328.2, + "etMs": 317.63, + "share": 97, + "peakMb": 476, + "loadMs": 81, + "sizeMb": 54 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v2-small", + "backend": "xnnpack", + "precision": "fp32", + "device": "s24", + "medianMs": 751.06, + "meanMs": 753.33, + "etMs": 750.35, + "share": 100, + "peakMb": 460, + "loadMs": 32, + "sizeMb": 94 + }, + { + "task": "depth-estimation", + "model": "depth-anything-v3-small", + "backend": "xnnpack", + "precision": "fp32", + "device": "s24", + "medianMs": 839.16, + "meanMs": 831.49, + "etMs": 827.92, + "share": 100, + "peakMb": 581, + "loadMs": 54, + "sizeMb": 102 + }, + { + "task": "colbert-embeddings", + "model": "lfm2-5-colbert-350-m", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 82.69, + "meanMs": 84.58, + "etMs": 83.86, + "share": 99, + "peakMb": 680, + "loadMs": 503, + "sizeMb": 360 + }, + { + "task": "llm", + "model": "smollm2-135-m", + "backend": "xnnpack", + "precision": "8da8w", + "device": "s24", + "medianMs": 359, + "meanMs": 357.87, + "tokensPerSecond": 178.3, + "ttftMs": 27, + "peakMb": 1102, + "loadMs": 387, + "sizeMb": 167 + }, + { + "task": "llm", + "model": "lfm2-5-350-m", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 431.9, + "meanMs": 433.23, + "tokensPerSecond": 148.2, + "ttftMs": 20, + "peakMb": 1007, + "loadMs": 554, + "sizeMb": 283 + }, + { + "task": "llm", + "model": "smollm2-360-m", + "backend": "xnnpack", + "precision": "8da8w", + "device": "s24", + "medianMs": 841.62, + "meanMs": 884.88, + "tokensPerSecond": 76, + "ttftMs": 55, + "peakMb": 1923, + "loadMs": 1038, + "sizeMb": 402 + }, + { + "task": "llm", + "model": "qwen2-5-0-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 638.09, + "meanMs": 671.51, + "tokensPerSecond": 100.3, + "ttftMs": 70, + "peakMb": 1129, + "loadMs": 876, + "sizeMb": 419 + }, + { + "task": "llm", + "model": "hammer2-1-0-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 705.12, + "meanMs": 731.01, + "tokensPerSecond": 90.8, + "ttftMs": 54, + "peakMb": 1138, + "loadMs": 1246, + "sizeMb": 423 + }, + { + "task": "llm", + "model": "qwen3-0-6-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 790.38, + "meanMs": 825.03, + "tokensPerSecond": 81, + "ttftMs": 38, + "peakMb": 2621, + "loadMs": 1532, + "sizeMb": 505 + }, + { + "task": "llm", + "model": "lfm2-5-vl-450-m", + "backend": "vulkan", + "precision": "8da4w", + "device": "s24", + "medianMs": 864.7, + "meanMs": 867.22, + "tokensPerSecond": 74, + "ttftMs": 41, + "peakMb": 1060, + "loadMs": 430, + "sizeMb": 598 + }, + { + "task": "llm", + "model": "lfm2-5-vl-450-m", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 677.91, + "meanMs": 677.88, + "tokensPerSecond": 94.4, + "ttftMs": 21, + "peakMb": 1277, + "loadMs": 585, + "sizeMb": 623 + }, + { + "task": "llm", + "model": "lfm2-5-1-2-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 1193.68, + "meanMs": 1226.77, + "tokensPerSecond": 53.6, + "ttftMs": 74, + "peakMb": 1610, + "loadMs": 1127, + "sizeMb": 777 + }, + { + "task": "llm", + "model": "bielik-v3-1-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 1673.15, + "meanMs": 1699.8, + "tokensPerSecond": 38.3, + "ttftMs": 186, + "peakMb": 1925, + "loadMs": 1262, + "sizeMb": 896 + }, + { + "task": "llm", + "model": "llama3-2-1-b", + "backend": "xnnpack", + "precision": "spinquant", + "device": "s24", + "medianMs": 1507.22, + "meanMs": 1506.59, + "tokensPerSecond": 42.5, + "ttftMs": 128, + "peakMb": 2422, + "loadMs": 1877, + "sizeMb": 1042 + }, + { + "task": "llm", + "model": "qwen2-5-1-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 1685.75, + "meanMs": 1723.34, + "tokensPerSecond": 38, + "ttftMs": 152, + "peakMb": 2291, + "loadMs": 1880, + "sizeMb": 1118 + }, + { + "task": "llm", + "model": "hammer2-1-1-5-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 1641.22, + "meanMs": 1689.9, + "tokensPerSecond": 39, + "ttftMs": 179, + "peakMb": 2286, + "loadMs": 1820, + "sizeMb": 1122 + }, + { + "task": "llm", + "model": "qwen3-1-7-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 1800.62, + "meanMs": 1820.76, + "tokensPerSecond": 35.5, + "ttftMs": 90, + "peakMb": 2439, + "loadMs": 2068, + "sizeMb": 1258 + }, + { + "task": "llm", + "model": "lfm2-5-2-6-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 2669.93, + "meanMs": 2739.3, + "tokensPerSecond": 24, + "ttftMs": 152, + "peakMb": 2068, + "loadMs": 2802, + "sizeMb": 1724 + }, + { + "task": "llm", + "model": "smollm2-1-7-b", + "backend": "xnnpack", + "precision": "8da8w", + "device": "s24", + "medianMs": 2899.66, + "meanMs": 2903.2, + "tokensPerSecond": 22.1, + "ttftMs": 118, + "peakMb": 2943, + "loadMs": 2431, + "sizeMb": 1735 + }, + { + "task": "llm", + "model": "hammer2-1-3-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 3132.28, + "meanMs": 3125.52, + "tokensPerSecond": 20.4, + "ttftMs": 224, + "peakMb": 2901, + "loadMs": 3367, + "sizeMb": 1980 + }, + { + "task": "llm", + "model": "qwen2-5-3-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 3217.7, + "meanMs": 3293.93, + "tokensPerSecond": 19.9, + "ttftMs": 288, + "peakMb": 2903, + "loadMs": 3078, + "sizeMb": 2014 + }, + { + "task": "llm", + "model": "llama3-2-3-b", + "backend": "xnnpack", + "precision": "spinquant", + "device": "s24", + "medianMs": 3683.29, + "meanMs": 3763.78, + "tokensPerSecond": 17.4, + "ttftMs": 340, + "peakMb": 3307, + "loadMs": 3943, + "sizeMb": 2279 + }, + { + "task": "llm", + "model": "lfm2-5-vl-1-6-b", + "backend": "vulkan", + "precision": "8da4w", + "device": "s24", + "medianMs": 1808.85, + "meanMs": 1954.08, + "tokensPerSecond": 35.4, + "ttftMs": 77, + "peakMb": 2962, + "loadMs": 2173, + "sizeMb": 2357 + }, + { + "task": "llm", + "model": "lfm2-5-vl-1-6-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 1422.02, + "meanMs": 1431.45, + "tokensPerSecond": 45, + "ttftMs": 54, + "peakMb": 3146, + "loadMs": 2451, + "sizeMb": 2365 + }, + { + "task": "llm", + "model": "qwen3-4-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 4044.97, + "meanMs": 4177.73, + "tokensPerSecond": 15.8, + "ttftMs": 160, + "peakMb": 3980, + "loadMs": 5897, + "sizeMb": 2572 + }, + { + "task": "llm", + "model": "phi4-mini", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 3720.36, + "meanMs": 3732.14, + "tokensPerSecond": 17.2, + "ttftMs": 146, + "peakMb": 3677, + "loadMs": 5327, + "sizeMb": 2703 + }, + { + "task": "llm", + "model": "gemma4-e2-b", + "backend": "vulkan", + "precision": "8da4w", + "device": "s24", + "medianMs": 4323.73, + "meanMs": 4312.48, + "tokensPerSecond": 14.8, + "ttftMs": 186, + "peakMb": 3667, + "loadMs": 5769, + "sizeMb": 2850 + }, + { + "task": "llm", + "model": "gemma4-e2-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 9060.46, + "meanMs": 9642.7, + "tokensPerSecond": 7.1, + "ttftMs": 442, + "peakMb": 2162, + "loadMs": 3831, + "sizeMb": 3097 + }, + { + "task": "llm", + "model": "gemma4-omni-e2-b", + "backend": "xnnpack", + "precision": "8da4w", + "device": "s24", + "medianMs": 9029.08, + "meanMs": 9477.6, + "tokensPerSecond": 7.1, + "ttftMs": 479, + "peakMb": 2609, + "loadMs": 4726, + "sizeMb": 3482 + } +] diff --git a/docs/src/components/Benchmarks/BenchmarkChart.tsx b/docs/src/components/Benchmarks/BenchmarkChart.tsx index bc8e39b727..33d436ca83 100644 --- a/docs/src/components/Benchmarks/BenchmarkChart.tsx +++ b/docs/src/components/Benchmarks/BenchmarkChart.tsx @@ -209,7 +209,7 @@ export default function BenchmarkChart({ task, data }: Props) { return rankPrecision(a.precision) - rankPrecision(b.precision); }); return { id: device.id, label: device.shortName, bars }; - }); + }).filter((group) => group.bars.length > 0); }, [rows, model, isSizeMetric]); const maxValue = Math.max( diff --git a/docs/src/data/benchmarks.ts b/docs/src/data/benchmarks.ts index 6b28b318f1..2e1fe17f89 100644 --- a/docs/src/data/benchmarks.ts +++ b/docs/src/data/benchmarks.ts @@ -17,8 +17,9 @@ export type BenchmarkRow = { /** * Joins rows generated from raw runs (`measured.json`) with published rows - * that have no raw runs behind them (`imported.json`). A device present in - * the measured rows replaces all of its imported ones. + * that have no raw runs behind them (`imported.json`). Measured variants + * replace matching imported variants; other published results remain available + * when a device has only a partial new run. * @param measured Rows built by `yarn bench:docs` in `apps/benchmarks`. * @param imported Rows with no raw runs. * @returns The rows to chart. @@ -27,8 +28,10 @@ export function mergeBenchmarkRows( measured: readonly BenchmarkRow[], imported: readonly BenchmarkRow[] ): BenchmarkRow[] { - const measuredDevices = new Set(measured.map((row) => row.device)); - return [...measured, ...imported.filter((row) => !measuredDevices.has(row.device))]; + const key = (row: BenchmarkRow) => + JSON.stringify([row.device, row.task, row.model, row.backend, row.precision]); + const measuredVariants = new Set(measured.map(key)); + return [...measured, ...imported.filter((row) => !measuredVariants.has(key(row)))]; } export type MetricId = @@ -97,9 +100,10 @@ export const METRICS: Metric[] = [ }, { id: 'peakMb', - label: 'Peak Memory', + label: 'Peak Process Memory', unit: 'MB', - description: 'Peak resident memory (RAM) allocated during model execution.', + description: + 'Peak process footprint during inference, including the app baseline: total PSS on Android and phys_footprint on iOS.', lowerIsBetter: true, }, { @@ -122,6 +126,7 @@ export const DEVICES: Device[] = [ { id: 's26ultra', shortName: 'S26 Ultra' }, { id: 'pixel10', shortName: 'Pixel 10' }, { id: 's20plus', shortName: 'S20+' }, + { id: 's24', shortName: 'S24' }, { id: 'iphone17', shortName: 'iPhone 17' }, { id: 'iphonese', shortName: 'iPhone SE 3' }, ]; diff --git a/yarn.lock b/yarn.lock index 03703242c6..9c39ad9f8f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,7 +80,7 @@ __metadata: languageName: node linkType: hard -"@babel/core@npm:^7.21.3, @babel/core@npm:^7.23.9, @babel/core@npm:^7.29.0, @babel/core@npm:^7.29.7": +"@babel/core@npm:^7.21.3, @babel/core@npm:^7.23.9, @babel/core@npm:^7.29.7": version: 7.29.7 resolution: "@babel/core@npm:7.29.7" dependencies: @@ -3519,84 +3519,6 @@ __metadata: languageName: node linkType: hard -"@expo/cli@npm:^56.1.26": - version: 56.1.26 - resolution: "@expo/cli@npm:56.1.26" - dependencies: - "@expo/code-signing-certificates": "npm:^0.0.6" - "@expo/config": "npm:~56.0.14" - "@expo/config-plugins": "npm:~56.0.16" - "@expo/devcert": "npm:^1.2.1" - "@expo/env": "npm:~2.3.1" - "@expo/image-utils": "npm:^0.10.5" - "@expo/inline-modules": "npm:^0.0.16" - "@expo/json-file": "npm:^10.2.0" - "@expo/log-box": "npm:^56.0.14" - "@expo/metro": "npm:~56.0.2" - "@expo/metro-config": "npm:~56.0.19" - "@expo/metro-file-map": "npm:^56.0.4" - "@expo/osascript": "npm:^2.6.0" - "@expo/package-manager": "npm:^1.12.1" - "@expo/plist": "npm:^0.7.0" - "@expo/prebuild-config": "npm:^56.0.23" - "@expo/require-utils": "npm:^56.1.7" - "@expo/router-server": "npm:^56.0.19" - "@expo/schema-utils": "npm:^56.0.2" - "@expo/spawn-async": "npm:^1.8.0" - "@expo/ws-tunnel": "npm:^2.0.0" - "@expo/xcpretty": "npm:^4.4.4" - "@react-native/dev-middleware": "npm:0.85.3" - accepts: "npm:^1.3.8" - agent-cli-detector: "npm:0.1.7" - arg: "npm:^5.0.2" - bplist-creator: "npm:0.1.0" - bplist-parser: "npm:^0.3.1" - chalk: "npm:^4.0.0" - ci-info: "npm:^3.3.0" - compression: "npm:^1.7.4" - connect: "npm:^3.7.0" - debug: "npm:^4.3.4" - dnssd-advertise: "npm:^1.1.6" - expo-server: "npm:^56.0.6" - fetch-nodeshim: "npm:^0.4.10" - getenv: "npm:^2.0.0" - glob: "npm:^13.0.0" - lan-network: "npm:^0.2.1" - multitars: "npm:^1.0.2" - node-forge: "npm:^1.3.3" - npm-package-arg: "npm:^11.0.0" - ora: "npm:^3.4.0" - picomatch: "npm:^4.0.4" - pretty-format: "npm:^29.7.0" - progress: "npm:^2.0.3" - prompts: "npm:^2.3.2" - resolve-from: "npm:^5.0.0" - sandbox-cli-detector: "npm:^0.2.0" - semver: "npm:^7.6.0" - send: "npm:^0.19.0" - slugify: "npm:^1.3.4" - stacktrace-parser: "npm:^0.1.10" - structured-headers: "npm:^0.4.1" - terminal-link: "npm:^2.1.1" - toqr: "npm:^0.1.1" - wrap-ansi: "npm:^7.0.0" - ws: "npm:^8.12.1" - zod: "npm:^3.25.76" - peerDependencies: - expo: "*" - expo-router: "*" - react-native: "*" - peerDependenciesMeta: - expo-router: - optional: true - react-native: - optional: true - bin: - expo-internal: main.js - checksum: 10/8ada1f6893769733c45bfb49f58523e3ad09dc53abc8e053cde9b7c47c2c19c3411633983160af0d882d598f0a9f9a6becf1531d53ee91283380e492ab0bddb0 - languageName: node - linkType: hard - "@expo/cli@npm:^57.0.29": version: 57.0.29 resolution: "@expo/cli@npm:57.0.29" @@ -4388,34 +4310,6 @@ __metadata: languageName: node linkType: hard -"@expo/router-server@npm:^56.0.19": - version: 56.0.19 - resolution: "@expo/router-server@npm:56.0.19" - dependencies: - debug: "npm:^4.3.4" - peerDependencies: - "@expo/metro-runtime": ^56.0.21 - expo: "*" - expo-constants: ^56.0.26 - expo-font: ^56.0.7 - expo-router: "*" - expo-server: ^56.0.6 - react: "*" - react-dom: "*" - react-server-dom-webpack: ~19.0.1 || ~19.1.2 || ~19.2.1 - peerDependenciesMeta: - "@expo/metro-runtime": - optional: true - expo-router: - optional: true - react-dom: - optional: true - react-server-dom-webpack: - optional: true - checksum: 10/ad9af42720b828c9c98849b9ded16cd4628a3f6e721b91f47021c2a425ad5c84523ed3e41b07fc948bb0beb522732824731e854c62e9e18bbc568b8650964048 - languageName: node - linkType: hard - "@expo/router-server@npm:^57.0.12": version: 57.0.12 resolution: "@expo/router-server@npm:57.0.12" @@ -7994,7 +7888,7 @@ __metadata: languageName: node linkType: hard -"babel-preset-expo@npm:~56.0.14, babel-preset-expo@npm:~56.0.20": +"babel-preset-expo@npm:~56.0.20": version: 56.0.20 resolution: "babel-preset-expo@npm:56.0.20" dependencies: @@ -8171,20 +8065,21 @@ __metadata: version: 0.0.0-use.local resolution: "benchmarks@workspace:apps/benchmarks" dependencies: - "@babel/core": "npm:^7.29.0" + "@babel/core": "npm:^7.29.7" "@react-native/metro-config": "npm:^0.86.0" "@types/react": "npm:~19.2.0" - babel-preset-expo: "npm:~56.0.14" - expo: "npm:~56.0.9" - expo-build-properties: "npm:~56.0.17" - expo-keep-awake: "npm:~56.0.3" + babel-preset-expo: "npm:~57.0.15" + expo: "npm:~57.0.28" + expo-build-properties: "npm:~57.0.22" + expo-keep-awake: "npm:~57.0.2" react: "npm:19.2.3" - react-native: "npm:0.85.3" + react-native: "npm:0.86.3" react-native-blob-util: "npm:^0.25.0" react-native-executorch: "workspace:*" react-native-safe-area-context: "npm:~5.7.0" - react-native-worklets: "npm:0.10.3" + react-native-worklets: "npm:0.13.0" react-refresh: "npm:^0.14.0" + typescript: "npm:~5.9.2" languageName: unknown linkType: soft @@ -10281,20 +10176,6 @@ __metadata: languageName: node linkType: hard -"expo-asset@npm:~56.0.25": - version: 56.0.25 - resolution: "expo-asset@npm:56.0.25" - dependencies: - "@expo/image-utils": "npm:^0.10.5" - expo-constants: "npm:~56.0.27" - peerDependencies: - expo: "*" - react: "*" - react-native: "*" - checksum: 10/f24bedde7e224e7d7d5f8e212da28b371e98a418e27e13367f0a63764635814669f95d2696d2aabaf725b155121770e7ab8660899647c1b92bb79c416e0bdef4 - languageName: node - linkType: hard - "expo-asset@npm:~57.0.19": version: 57.0.19 resolution: "expo-asset@npm:57.0.19" @@ -10319,19 +10200,6 @@ __metadata: languageName: node linkType: hard -"expo-build-properties@npm:~56.0.17": - version: 56.0.27 - resolution: "expo-build-properties@npm:56.0.27" - dependencies: - "@expo/schema-utils": "npm:^56.0.2" - resolve-from: "npm:^5.0.0" - semver: "npm:^7.6.0" - peerDependencies: - expo: "*" - checksum: 10/a59e4d4a6391ec662ea81c1f1cb0341e0d2c48903a160fbb30c82574971d0ce14c6b265e3df9cd83f7d5f3c41400afe31040226c5b5c42f3f07cb8b183bca965 - languageName: node - linkType: hard - "expo-build-properties@npm:~57.0.22": version: 57.0.22 resolution: "expo-build-properties@npm:57.0.22" @@ -10367,18 +10235,6 @@ __metadata: languageName: node linkType: hard -"expo-constants@npm:~56.0.27": - version: 56.0.27 - resolution: "expo-constants@npm:56.0.27" - dependencies: - "@expo/env": "npm:~2.3.1" - peerDependencies: - expo: "*" - react-native: "*" - checksum: 10/95e2480db7db36d3d416594101fbefbc82ed85d9751df2b5e6f378ffdce62395feb3d192cf25c1481a792d2d1c40e0fd6a0494e7fec12c0095d2e4126592d9ec - languageName: node - linkType: hard - "expo-constants@npm:~57.0.21": version: 57.0.21 resolution: "expo-constants@npm:57.0.21" @@ -10578,24 +10434,6 @@ __metadata: languageName: node linkType: hard -"expo-modules-core@npm:~56.0.27": - version: 56.0.27 - resolution: "expo-modules-core@npm:56.0.27" - dependencies: - "@expo/expo-modules-macros-plugin": "npm:0.2.2" - expo-modules-jsi: "npm:~56.0.14" - invariant: "npm:^2.2.4" - peerDependencies: - react: "*" - react-native: "*" - react-native-worklets: ^0.7.4 || ^0.8.0 - peerDependenciesMeta: - react-native-worklets: - optional: true - checksum: 10/3f4b4d18265aa737d512b0d55048077e6c98436d047b25a71a968d76774ee27d1ed6c3e94979c1263c7d7a8773cd4a260b87c0f98257d11bbdc4920e5bac4105 - languageName: node - linkType: hard - "expo-modules-core@npm:~57.0.22": version: 57.0.22 resolution: "expo-modules-core@npm:57.0.22" @@ -10623,15 +10461,6 @@ __metadata: languageName: node linkType: hard -"expo-modules-jsi@npm:~56.0.14": - version: 56.0.14 - resolution: "expo-modules-jsi@npm:56.0.14" - peerDependencies: - react-native: "*" - checksum: 10/693837158130298b2824aae792f1e47f275fad0bf2ad48863cd82100e0e4c8ed06bf8eaa54952d7a688126ccad539e82fd14d7c4f768e784762f615f85b01e40 - languageName: node - linkType: hard - "expo-modules-jsi@npm:~57.1.2": version: 57.1.2 resolution: "expo-modules-jsi@npm:57.1.2" @@ -10868,60 +10697,6 @@ __metadata: languageName: node linkType: hard -"expo@npm:~56.0.9": - version: 56.0.23 - resolution: "expo@npm:56.0.23" - dependencies: - "@babel/runtime": "npm:^7.20.0" - "@expo/cli": "npm:^56.1.26" - "@expo/config": "npm:~56.0.14" - "@expo/config-plugins": "npm:~56.0.16" - "@expo/devtools": "npm:~56.0.2" - "@expo/dom-webview": "npm:~56.0.6" - "@expo/fingerprint": "npm:^0.19.10" - "@expo/local-build-cache-provider": "npm:^56.0.12" - "@expo/log-box": "npm:^56.0.14" - "@expo/metro": "npm:~56.0.2" - "@expo/metro-config": "npm:~56.0.19" - "@ungap/structured-clone": "npm:^1.3.0" - babel-preset-expo: "npm:~56.0.20" - expo-asset: "npm:~56.0.25" - expo-constants: "npm:~56.0.27" - expo-file-system: "npm:~56.0.11" - expo-font: "npm:~56.0.7" - expo-keep-awake: "npm:~56.0.3" - expo-modules-autolinking: "npm:~56.0.23" - expo-modules-core: "npm:~56.0.27" - pretty-format: "npm:^29.7.0" - react-refresh: "npm:^0.14.2" - whatwg-url-minimum: "npm:^0.1.2" - peerDependencies: - "@expo/dom-webview": "*" - "@expo/metro-runtime": "*" - react: "*" - react-dom: "*" - react-native: "*" - react-native-web: "*" - react-native-webview: "*" - peerDependenciesMeta: - "@expo/dom-webview": - optional: true - "@expo/metro-runtime": - optional: true - react-dom: - optional: true - react-native-web: - optional: true - react-native-webview: - optional: true - bin: - expo: bin/cli - expo-modules-autolinking: bin/autolinking - fingerprint: bin/fingerprint - checksum: 10/925c5534c109364d439142dda9e8bc3e41d9dd857c69626df4be578a747cc85efb876e916a316484f09c1e0b2f429f315a72421d98ad4ec52d04b8f3457fe1d8 - languageName: node - linkType: hard - "expo@npm:~57.0.28": version: 57.0.28 resolution: "expo@npm:57.0.28" From 8ef949b83dcf883694166b999228733e64de623d Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 27 Oct 2026 13:49:58 +0100 Subject: [PATCH 2/4] Clarify benchmark memory readings and preserve source runs --- apps/benchmarks/README.md | 12 ++--- apps/benchmarks/scripts/docs-data.mjs | 4 +- docs/docs/04-benchmarks/01-benchmarks.mdx | 6 ++- ...11.0-missing-20261026-ios-iPhone18-3.jsonl | 51 +++++++++++++++++++ ...missing-20261026-ios-iPhone18-3.notes.json | 30 +++++++++++ ...ing-20261026-isolated-ios-iPhone18-3.jsonl | 4 ++ ...0261026-isolated-ios-iPhone18-3.notes.json | 15 ++++++ ...1026-isolated-no-phi4-ios-iPhone18-3.jsonl | 5 ++ ...isolated-no-phi4-ios-iPhone18-3.notes.json | 25 +++++++++ ....0-missing-20261026-android-SM-G986B.jsonl | 38 ++++++++++++++ ....0-missing-20261026-android-SM-S921B.jsonl | 38 ++++++++++++++ ...261026-retry-vulkan-android-SM-S921B.jsonl | 2 + ...6-retry-vulkan-android-SM-S921B.notes.json | 5 ++ docs/sidebars.js | 18 +++---- docs/src/data/benchmarks.ts | 2 +- 15 files changed, 236 insertions(+), 19 deletions(-) create mode 100644 docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-ios-iPhone18-3.jsonl create mode 100644 docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-ios-iPhone18-3.notes.json create mode 100644 docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-ios-iPhone18-3.jsonl create mode 100644 docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-ios-iPhone18-3.notes.json create mode 100644 docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-no-phi4-ios-iPhone18-3.jsonl create mode 100644 docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-no-phi4-ios-iPhone18-3.notes.json create mode 100644 docs/docs/04-benchmarks/runs/s20plus/v0.11.0-missing-20261026-android-SM-G986B.jsonl create mode 100644 docs/docs/04-benchmarks/runs/s24/v0.11.0-missing-20261026-android-SM-S921B.jsonl create mode 100644 docs/docs/04-benchmarks/runs/s24/v0.11.0-missing-20261026-retry-vulkan-android-SM-S921B.jsonl create mode 100644 docs/docs/04-benchmarks/runs/s24/v0.11.0-missing-20261026-retry-vulkan-android-SM-S921B.notes.json diff --git a/apps/benchmarks/README.md b/apps/benchmarks/README.md index 574852c982..5e631affd6 100644 --- a/apps/benchmarks/README.md +++ b/apps/benchmarks/README.md @@ -173,8 +173,9 @@ post-processing in it. ## Publishing to the docs -The docs benchmarks page reads `measured.json`; raw runs are only needed to -regenerate it. Temporarily collect the runs under the page directory: +The docs benchmarks page reads `measured.json`. Its source runs are committed +under `docs/docs/04-benchmarks/runs/`, grouped by device. Add new runs there +and regenerate the published data: ```bash mkdir -p ../../docs/docs/04-benchmarks/runs/s26ultra @@ -184,14 +185,13 @@ yarn bench:docs # regenerates docs/docs/04-benchmarks/measured.json The directory name is the device id the docs list it under (`DEVICES` in `docs/src/data/benchmarks.ts`). Generation replaces `measured.json`, so include -all runs whose measurements it should retain. Archive the raw runs outside the -repository and remove the temporary `runs/` directory before committing; the -docs build only needs the generated JSON. Measured variants replace matching +all runs whose measurements it should retain. Commit the raw runs alongside +the generated JSON so others can reproduce it. Measured variants replace matching rows in `imported.json`, preserving other published results. To check a build against a release, compare against its archived runs: ```bash -yarn bench:compare /path/to/archived/.jsonl results/.jsonl +yarn bench:compare ../../docs/docs/04-benchmarks/runs/s26ultra/.jsonl results/.jsonl ``` ## What gets measured diff --git a/apps/benchmarks/scripts/docs-data.mjs b/apps/benchmarks/scripts/docs-data.mjs index a42c015a04..2ae7128438 100644 --- a/apps/benchmarks/scripts/docs-data.mjs +++ b/apps/benchmarks/scripts/docs-data.mjs @@ -4,8 +4,8 @@ * * To generate data, collect raw runs beside the docs page, one * directory per device: `runs//*.jsonl`, where `` is the - * id the docs list the device under. Only the generated JSON is needed by the - * docs build; archive raw runs outside the repository after generation. + * id the docs list the device under. Commit the source runs alongside the + * generated JSON so the published data can be regenerated. * * This writes `measured.json` beside them. The page merges it with * `imported.json`, the published rows that have no raw runs behind them: a diff --git a/docs/docs/04-benchmarks/01-benchmarks.mdx b/docs/docs/04-benchmarks/01-benchmarks.mdx index 53f72a0543..60a43c3871 100644 --- a/docs/docs/04-benchmarks/01-benchmarks.mdx +++ b/docs/docs/04-benchmarks/01-benchmarks.mdx @@ -52,6 +52,10 @@ Measurements were collected on physical iOS and Android devices spanning flagshi Memory values report peak process footprint during a separate inference pass, including the app baseline: total PSS on Android and `phys_footprint` on iOS. Some supplemental iPhone LLM measurements use a fresh process for each model. +:::caution GPU memory accounting +These sampled process counters do not measure total GPU memory or the peak during model loading. GPU backends can report unexpectedly low values: for example, MLX LFM2.5-VL-1.6B (int8) reports 216 MB despite a roughly 2.8 GB download, and Gemma4 E2B on Vulkan reports 327 MB on S20+ versus 3667 MB on S24. These discrepancies suggest incomplete or device-dependent memory accounting; the exact cause has not been established. Treat the figures as observed process-counter readings, not total model memory requirements or a reliable comparison of GPU memory across devices and backends. Sampling can also miss brief peaks. +::: + ## Results :::note Performance and Precision Trade-offs @@ -183,7 +187,7 @@ These benchmarks measure execution latency, memory utilization, and initializati - **Variants:** Quantized LLMs only. - **Collection:** The S26 Ultra LLM results retain the September measurements. Supplemental S20+, S24, and iPhone 17 LLM runs were collected in October using the fixed-prompt harness described below. - **Input:** Prompt (`List three uses for a paperclip.`), 64 tokens generated with EOS ignored, temperature 0 (greedy decoding), and the KV cache reset before each independent prompt. -- **Prefill vs. Decode & Memory:** The short prompt isolates autoregressive token generation throughput (tokens/second) and time to first token (TTFT). Autoregressive decoding executes one forward pass per generated token. Peak memory is driven by model weights and allocated key-value (KV) cache buffers, with models larger than 2 GB peaking between 2.4 and 3.5 GB of resident RAM. +- **Prefill vs. Decode & Memory:** The short prompt isolates autoregressive token generation throughput (tokens/second) and time to first token (TTFT). Autoregressive decoding executes one forward pass per generated token. Memory requirements include model weights, allocated key-value (KV) cache buffers, and intermediate tensors. Observed process memory reaches 4122 MB for Qwen3 4B on S20+; GPU readings have the accounting limitations described above. ::: On iPhone 17, Phi-4 Mini was omitted. Qwen3 4B (XNNPACK) terminated after critical memory-pressure warnings, including in a fresh-process retry. On Galaxy S24, Android's low-memory killer terminated Gemma4 Omni E2B (Vulkan) during loading in a fresh-process retry. Only successful measurements are charted. diff --git a/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-ios-iPhone18-3.jsonl b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-ios-iPhone18-3.jsonl new file mode 100644 index 0000000000..aaaf1491eb --- /dev/null +++ b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-ios-iPhone18-3.jsonl @@ -0,0 +1,51 @@ +{"type":"run","label":"v0.11.0-missing-20261026","platform":"ios","device":{"model":"iPhone18,3","cpuCores":6,"soc":"iPhone18,3","totalMemoryBytes":8044216320,"isEmulator":false,"osVersion":"26.6"},"inputSpecVersion":1,"buildType":"release","clocksPinned":false,"settings":{"suite":"depth-estimation/depth-anything-v2-small-xnnpack-int8,depth-estimation/depth-anything-v2-small-xnnpack-fp32,depth-estimation/depth-anything-v2-small-coreml-fp16,depth-estimation/depth-anything-v3-small-xnnpack-int8,depth-estimation/depth-anything-v3-small-xnnpack-fp32,depth-estimation/depth-anything-v3-small-coreml-fp16,depth-estimation/zipdepth-base-xnnpack-fp32,depth-estimation/zipdepth-base-coreml-fp32,llm/lfm2-5-1-2-b-xnnpack-8-da4-w,llm/lfm2-5-1-2-b-mlx-int4,llm/lfm2-5-2-6-b-xnnpack-8-da4-w,llm/lfm2-5-2-6-b-mlx-int4,llm/lfm2-5-350-m-xnnpack-8-da4-w,llm/lfm2-5-350-m-mlx-int4,llm/lfm2-5-vl-450-m-xnnpack-8-da4-w,llm/lfm2-5-vl-450-m-mlx-int4,llm/lfm2-5-vl-1-6-b-xnnpack-8-da4-w,llm/lfm2-5-vl-1-6-b-mlx-int4,llm/lfm2-5-vl-1-6-b-mlx-int8,llm/bielik-v3-1-5-b-xnnpack-8-da4-w,llm/llama3-2-1-b-xnnpack-spinquant,llm/llama3-2-1-b-mlx-int4,llm/llama3-2-3-b-xnnpack-spinquant,llm/llama3-2-3-b-mlx-int4,llm/smollm2-135-m-xnnpack-8-da8-w,llm/smollm2-135-m-mlx-int8,llm/smollm2-360-m-xnnpack-8-da8-w,llm/smollm2-360-m-mlx-int8,llm/smollm2-1-7-b-xnnpack-8-da8-w,llm/smollm2-1-7-b-mlx-int8,llm/hammer2-1-0-5-b-xnnpack-8-da4-w,llm/hammer2-1-0-5-b-mlx-int4,llm/hammer2-1-1-5-b-xnnpack-8-da4-w,llm/hammer2-1-1-5-b-mlx-int4,llm/hammer2-1-3-b-xnnpack-8-da4-w,llm/hammer2-1-3-b-mlx-int4,llm/phi4-mini-xnnpack-8-da4-w,llm/phi4-mini-mlx-int4,llm/qwen2-5-0-5-b-xnnpack-8-da4-w,llm/qwen2-5-0-5-b-mlx-int4,llm/qwen2-5-1-5-b-xnnpack-8-da4-w,llm/qwen2-5-1-5-b-mlx-int4,llm/qwen2-5-3-b-xnnpack-8-da4-w,llm/qwen2-5-3-b-mlx-int4,llm/qwen3-0-6-b-xnnpack-8-da4-w,llm/qwen3-0-6-b-mlx-int8,llm/qwen3-1-7-b-xnnpack-8-da4-w,llm/qwen3-1-7-b-mlx-int4,llm/qwen3-4-b-xnnpack-8-da4-w,llm/qwen3-4-b-mlx-int4,llm/gemma4-e2-b-xnnpack-8-da4-w,llm/gemma4-e2-b-mlx-int4,llm/gemma4-omni-e2-b-xnnpack-8-da4-w,llm/gemma4-omni-e2-b-mlx-int4,colbert-embeddings/lfm2-5-colbert-350-m-xnnpack-8-da4-w,colbert-embeddings/lfm2-5-colbert-350-m-mlx-int4","iterations":20,"warmup":3,"memoryIterations":5,"loadIterations":3,"repeats":1,"maxTempC":37,"maxBytes":6000000000,"urlMap":{}},"startedAt":"2026-10-26T15:42:04.058Z"} +{"id":"depth-estimation/zipdepth-base-xnnpack-fp32","task":"depthEstimation","model":"depthEstimation.ZIPDEPTH_BASE.XNNPACK_FP32","backend":"xnnpack","precision":"fp32","bytes":24609344,"mode":"worklet","progress":{"caseIndex":1,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":1801.347,"taskLoadMs":23.755,"taskLoad":{"count":3,"min":16.807,"max":57.596,"mean":32.719,"median":23.755,"p90":57.596,"iqr":40.788,"stdev":17.817},"pipeline":{"count":20,"min":21.57,"max":22.744,"mean":22.051,"median":22.003,"p90":22.455,"iqr":0.544,"stdev":0.338},"execution":{"perIteration":{"forward":{"count":1,"ms":21.513860450000003}},"totalMs":21.513860450000003},"units":262144,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"statusName":"none","status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":6},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":49.92,"loadedMb":87.45,"peakMb":163.38,"disposedMb":156.27}} +{"id":"depth-estimation/zipdepth-base-coreml-fp32","task":"depthEstimation","model":"depthEstimation.ZIPDEPTH_BASE.COREML_FP32","backend":"coreml","precision":"fp32","bytes":24858521,"mode":"worklet","progress":{"caseIndex":2,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":1897.634,"taskLoadMs":41.097,"taskLoad":{"count":3,"min":30.079,"max":2719.148,"mean":930.108,"median":41.097,"p90":2719.148,"iqr":2689.069,"stdev":1265.05},"pipeline":{"count":20,"min":5.862,"max":6.315,"mean":6.005,"median":5.981,"p90":6.123,"iqr":0.068,"stdev":0.098},"execution":{"perIteration":{"forward":{"count":1,"ms":5.51052505}},"totalMs":5.51052505},"units":262144,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","batteryTemperatureC":-1,"status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":3},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":116.03,"loadedMb":155.78,"peakMb":163.11,"disposedMb":141.55}} +{"id":"depth-estimation/depth-anything-v2-small-xnnpack-int8","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V2_SMALL.XNNPACK_INT8","backend":"xnnpack","precision":"int8","bytes":35481732,"mode":"worklet","progress":{"caseIndex":3,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":2338.916,"taskLoadMs":25.095,"taskLoad":{"count":3,"min":18.901,"max":60.483,"mean":34.827,"median":25.095,"p90":60.483,"iqr":41.582,"stdev":18.317},"pipeline":{"count":20,"min":257.429,"max":278.569,"mean":269.012,"median":269.075,"p90":276.096,"iqr":10.316,"stdev":6.173},"execution":{"perIteration":{"forward":{"count":1,"ms":267.6505646}},"totalMs":267.6505646},"units":262144,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":-1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":60},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":82.36,"loadedMb":128.92,"peakMb":219.64,"disposedMb":195.64}} +{"id":"depth-estimation/depth-anything-v3-small-xnnpack-int8","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V3_SMALL.XNNPACK_INT8","backend":"xnnpack","precision":"int8","bytes":40603272,"mode":"worklet","progress":{"caseIndex":4,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":2205.48,"taskLoadMs":25.592,"taskLoad":{"count":3,"min":19.727,"max":63.318,"mean":36.212,"median":25.592,"p90":63.318,"iqr":43.591,"stdev":19.315},"pipeline":{"count":20,"min":325.527,"max":349.56,"mean":339.538,"median":340.057,"p90":347.106,"iqr":11.061,"stdev":6.534},"execution":{"perIteration":{"forward":{"count":1,"ms":337.93052094999996}},"totalMs":337.93052094999996},"units":262144,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"batteryTemperatureC":-1,"statusName":"none"},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":76},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":168.05,"loadedMb":290.2,"peakMb":333.67,"disposedMb":308.97}} +{"id":"depth-estimation/depth-anything-v2-small-coreml-fp16","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V2_SMALL.COREML_FP16","backend":"coreml","precision":"fp16","bytes":50181365,"mode":"worklet","progress":{"caseIndex":5,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":3293.01,"taskLoadMs":112.921,"taskLoad":{"count":3,"min":111.952,"max":6481.543,"mean":2235.472,"median":112.921,"p90":6481.543,"iqr":6369.591,"stdev":3002.426},"pipeline":{"count":20,"min":51.869,"max":55.962,"mean":52.843,"median":52.15,"p90":55.363,"iqr":0.973,"stdev":1.299},"execution":{"perIteration":{"forward":{"count":1,"ms":50.8773354}},"totalMs":50.8773354},"units":262144,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","batteryTemperatureC":-1,"status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":13},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":148.67,"loadedMb":77.44,"peakMb":72,"disposedMb":68.24}} +{"id":"depth-estimation/depth-anything-v3-small-coreml-fp16","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V3_SMALL.COREML_FP16","backend":"coreml","precision":"fp16","bytes":70865495,"mode":"worklet","progress":{"caseIndex":6,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":3906.216,"taskLoadMs":129.89,"taskLoad":{"count":3,"min":125.717,"max":7722.986,"mean":2659.531,"median":129.89,"p90":7722.986,"iqr":7597.268,"stdev":3580.403},"pipeline":{"count":20,"min":57.579,"max":61.992,"mean":59.054,"median":58.401,"p90":61.304,"iqr":1.346,"stdev":1.286},"execution":{"perIteration":{"forward":{"count":1,"ms":56.8787188}},"totalMs":56.8787188},"units":262144,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","batteryTemperatureC":-1,"status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":15},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":55.08,"loadedMb":76.02,"peakMb":75.2,"disposedMb":69.42}} +{"id":"depth-estimation/depth-anything-v2-small-xnnpack-fp32","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V2_SMALL.XNNPACK_FP32","backend":"xnnpack","precision":"fp32","bytes":98979332,"mode":"worklet","progress":{"caseIndex":7,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":4006.209,"taskLoadMs":27.323,"taskLoad":{"count":3,"min":21.538,"max":65.008,"mean":37.956,"median":27.323,"p90":65.008,"iqr":43.47,"stdev":19.274},"pipeline":{"count":20,"min":406.056,"max":446.841,"mean":430.269,"median":432.126,"p90":442.315,"iqr":18.453,"stdev":11.543},"execution":{"perIteration":{"forward":{"count":1,"ms":428.80220425}},"totalMs":428.80220425},"units":262144,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":-1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":96},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":56.02,"loadedMb":165.17,"peakMb":271.85,"disposedMb":194.81}} +{"id":"depth-estimation/depth-anything-v3-small-xnnpack-fp32","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V3_SMALL.XNNPACK_FP32","backend":"xnnpack","precision":"fp32","bytes":106480776,"mode":"worklet","progress":{"caseIndex":8,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":3840.068,"taskLoadMs":25.895,"taskLoad":{"count":3,"min":21.996,"max":74.322,"mean":40.738,"median":25.895,"p90":74.322,"iqr":52.326,"stdev":23.801},"pipeline":{"count":20,"min":488.802,"max":526.505,"mean":511.741,"median":512.836,"p90":523.438,"iqr":16.566,"stdev":11.237},"execution":{"perIteration":{"forward":{"count":1,"ms":509.99547924999996}},"totalMs":509.99547924999996},"units":262144,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","status":0,"batteryTemperatureC":-1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":113},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":57.16,"loadedMb":243.08,"peakMb":396.11,"disposedMb":318.49}} +{"id":"colbert-embeddings/lfm2-5-colbert-350-m-mlx-int4","task":"colbertEmbeddings","model":"colbertEmbeddings.LFM2_5_COLBERT_350M.MLX_INT4","backend":"mlx","precision":"int4","bytes":192726472,"mode":"worklet","progress":{"caseIndex":10,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":6835.742,"taskLoadMs":77.275,"taskLoad":{"count":3,"min":77.062,"max":451.399,"mean":201.912,"median":77.275,"p90":451.399,"iqr":374.337,"stdev":176.414},"pipeline":{"count":20,"min":19.681,"max":20.082,"mean":19.787,"median":19.784,"p90":19.854,"iqr":0.082,"stdev":0.087},"execution":{"perIteration":{"forward":{"count":1,"ms":19.6062062}},"totalMs":19.6062062},"units":56,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","batteryTemperatureC":-1,"status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":26},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":65.38,"loadedMb":113.64,"peakMb":163.06,"disposedMb":163.06}} +{"id":"colbert-embeddings/lfm2-5-colbert-350-m-xnnpack-8-da4-w","task":"colbertEmbeddings","model":"colbertEmbeddings.LFM2_5_COLBERT_350M.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":377361478,"mode":"worklet","progress":{"caseIndex":14,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":12280.677,"taskLoadMs":206.077,"taskLoad":{"count":3,"min":200.296,"max":246.292,"mean":217.555,"median":206.077,"p90":246.292,"iqr":45.996,"stdev":20.457},"pipeline":{"count":20,"min":69.133,"max":73.547,"mean":71.789,"median":72.087,"p90":73.369,"iqr":1.854,"stdev":1.271},"execution":{"perIteration":{"forward":{"count":1,"ms":71.46421875}},"totalMs":71.46421875},"units":56,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"statusName":"none","status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":18},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":356.39,"loadedMb":712.25,"peakMb":732.7,"disposedMb":477.72}} +{"type":"run","label":"v0.11.0-missing-20261026","platform":"ios","device":{"totalMemoryBytes":8044216320,"osVersion":"26.6","soc":"iPhone18,3","isEmulator":false,"model":"iPhone18,3","cpuCores":6},"inputSpecVersion":1,"buildType":"release","clocksPinned":false,"settings":{"suite":"depth-estimation/depth-anything-v2-small-xnnpack-int8,depth-estimation/depth-anything-v2-small-xnnpack-fp32,depth-estimation/depth-anything-v2-small-coreml-fp16,depth-estimation/depth-anything-v3-small-xnnpack-int8,depth-estimation/depth-anything-v3-small-xnnpack-fp32,depth-estimation/depth-anything-v3-small-coreml-fp16,depth-estimation/zipdepth-base-xnnpack-fp32,depth-estimation/zipdepth-base-coreml-fp32,llm/lfm2-5-1-2-b-xnnpack-8-da4-w,llm/lfm2-5-1-2-b-mlx-int4,llm/lfm2-5-2-6-b-xnnpack-8-da4-w,llm/lfm2-5-2-6-b-mlx-int4,llm/lfm2-5-350-m-xnnpack-8-da4-w,llm/lfm2-5-350-m-mlx-int4,llm/lfm2-5-vl-450-m-xnnpack-8-da4-w,llm/lfm2-5-vl-450-m-mlx-int4,llm/lfm2-5-vl-1-6-b-xnnpack-8-da4-w,llm/lfm2-5-vl-1-6-b-mlx-int4,llm/lfm2-5-vl-1-6-b-mlx-int8,llm/bielik-v3-1-5-b-xnnpack-8-da4-w,llm/llama3-2-1-b-xnnpack-spinquant,llm/llama3-2-1-b-mlx-int4,llm/llama3-2-3-b-xnnpack-spinquant,llm/llama3-2-3-b-mlx-int4,llm/smollm2-135-m-xnnpack-8-da8-w,llm/smollm2-135-m-mlx-int8,llm/smollm2-360-m-xnnpack-8-da8-w,llm/smollm2-360-m-mlx-int8,llm/smollm2-1-7-b-xnnpack-8-da8-w,llm/smollm2-1-7-b-mlx-int8,llm/hammer2-1-0-5-b-xnnpack-8-da4-w,llm/hammer2-1-0-5-b-mlx-int4,llm/hammer2-1-1-5-b-xnnpack-8-da4-w,llm/hammer2-1-1-5-b-mlx-int4,llm/hammer2-1-3-b-xnnpack-8-da4-w,llm/hammer2-1-3-b-mlx-int4,llm/phi4-mini-xnnpack-8-da4-w,llm/phi4-mini-mlx-int4,llm/qwen2-5-0-5-b-xnnpack-8-da4-w,llm/qwen2-5-0-5-b-mlx-int4,llm/qwen2-5-1-5-b-xnnpack-8-da4-w,llm/qwen2-5-1-5-b-mlx-int4,llm/qwen2-5-3-b-xnnpack-8-da4-w,llm/qwen2-5-3-b-mlx-int4,llm/qwen3-0-6-b-xnnpack-8-da4-w,llm/qwen3-0-6-b-mlx-int8,llm/qwen3-1-7-b-xnnpack-8-da4-w,llm/qwen3-1-7-b-mlx-int4,llm/qwen3-4-b-xnnpack-8-da4-w,llm/qwen3-4-b-mlx-int4,llm/gemma4-e2-b-xnnpack-8-da4-w,llm/gemma4-e2-b-mlx-int4,llm/gemma4-omni-e2-b-xnnpack-8-da4-w,llm/gemma4-omni-e2-b-mlx-int4,colbert-embeddings/lfm2-5-colbert-350-m-xnnpack-8-da4-w,colbert-embeddings/lfm2-5-colbert-350-m-mlx-int4","iterations":20,"warmup":3,"memoryIterations":5,"loadIterations":3,"repeats":1,"maxTempC":37,"maxBytes":6000000000,"urlMap":{}},"startedAt":"2026-10-26T16:20:55.075Z"} +{"id":"llm/smollm2-135-m-xnnpack-8-da8-w","task":"llm","model":"llm.SMOLLM2_135M.XNNPACK_8DA8W","backend":"xnnpack","precision":"8da8w","bytes":175114287,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":9,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":3775.645,"taskLoadMs":125.348,"taskLoad":{"count":3,"min":116.033,"max":200.002,"mean":147.128,"median":125.348,"p90":200.002,"iqr":83.969,"stdev":37.581},"pipeline":{"count":20,"min":436.111,"max":470.857,"mean":452.164,"median":453.335,"p90":463.66,"iqr":15.77,"stdev":9.465},"execution":{"perIteration":{"forward":{"count":65,"ms":445.97240395}},"totalMs":445.97240395},"units":64,"detail":{"timeToFirstTokenMs":31.524,"prefillMs":32,"promptTokens":38},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":-1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":105},"thermalValid":true,"holdsTimedOut":0,"coolingMs":239,"memory":{"baselineMb":47.8,"loadedMb":973.49,"peakMb":956.42,"disposedMb":855.35}} +{"id":"llm/smollm2-135-m-mlx-int8","task":"llm","model":"llm.SMOLLM2_135M.MLX_INT8","backend":"mlx","precision":"int8","bytes":197739951,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":11,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":4361.364,"taskLoadMs":75.984,"taskLoad":{"count":3,"min":74.396,"max":189.565,"mean":113.315,"median":75.984,"p90":189.565,"iqr":115.168,"stdev":53.921},"pipeline":{"count":20,"min":246.51,"max":273.568,"mean":249.153,"median":247.973,"p90":248.824,"iqr":1.44,"stdev":5.655},"execution":{"perIteration":{"forward":{"count":65,"ms":245.17912835}},"totalMs":245.17912835},"units":64,"detail":{"timeToFirstTokenMs":13.074,"prefillMs":13,"promptTokens":38},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","batteryTemperatureC":-1,"status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":60},"thermalValid":true,"holdsTimedOut":0,"coolingMs":315,"memory":{"baselineMb":60.25,"loadedMb":450.55,"peakMb":467.47,"disposedMb":331.81}} +{"id":"llm/lfm2-5-350-m-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_350M.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":296972650,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":12,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":6085.363,"taskLoadMs":239.159,"taskLoad":{"count":3,"min":238.957,"max":299.142,"mean":259.086,"median":239.159,"p90":299.142,"iqr":60.185,"stdev":28.324},"pipeline":{"count":20,"min":466.635,"max":486.52,"mean":479.758,"median":480.428,"p90":486.297,"iqr":6.004,"stdev":4.923},"execution":{"perIteration":{"forward":{"count":65,"ms":471.74434775}},"totalMs":471.74434775},"units":64,"detail":{"timeToFirstTokenMs":27.652,"prefillMs":28,"promptTokens":17},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"batteryTemperatureC":-1,"statusName":"none"},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":108},"thermalValid":true,"holdsTimedOut":0,"coolingMs":263,"memory":{"baselineMb":316.27,"loadedMb":1028.83,"peakMb":1047.08,"disposedMb":927.28}} +{"id":"llm/lfm2-5-350-m-mlx-int4","task":"llm","model":"llm.LFM2_5_350M.MLX_INT4","backend":"mlx","precision":"int4","bytes":327503466,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":13,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":8219.616,"taskLoadMs":91.3,"taskLoad":{"count":3,"min":89.887,"max":174.215,"mean":118.467,"median":91.3,"p90":174.215,"iqr":84.328,"stdev":39.424},"pipeline":{"count":20,"min":267.217,"max":293.115,"mean":270.691,"median":268.323,"p90":269.959,"iqr":0.908,"stdev":7.345},"execution":{"perIteration":{"forward":{"count":65,"ms":264.730194}},"totalMs":264.730194},"units":64,"detail":{"timeToFirstTokenMs":15.275,"prefillMs":15,"promptTokens":17},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","batteryTemperatureC":-1,"status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":65},"thermalValid":true,"holdsTimedOut":0,"coolingMs":402,"memory":{"baselineMb":315.36,"loadedMb":510.99,"peakMb":522.97,"disposedMb":336.39}} +{"id":"llm/smollm2-360-m-xnnpack-8-da8-w","task":"llm","model":"llm.SMOLLM2_360M.XNNPACK_8DA8W","backend":"xnnpack","precision":"8da8w","bytes":421904303,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":15,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":9908.884,"taskLoadMs":255.587,"taskLoad":{"count":3,"min":241.546,"max":358.07,"mean":285.068,"median":255.587,"p90":358.07,"iqr":116.525,"stdev":51.938},"pipeline":{"count":20,"min":844.607,"max":979.152,"mean":903.6,"median":902.933,"p90":972.587,"iqr":83.105,"stdev":46.761},"execution":{"perIteration":{"forward":{"count":65,"ms":895.6594251500001}},"totalMs":895.6594251500001},"units":64,"detail":{"timeToFirstTokenMs":66.263,"prefillMs":66,"promptTokens":38},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","status":0,"batteryTemperatureC":-1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":196},"thermalValid":true,"holdsTimedOut":0,"coolingMs":357,"memory":{"baselineMb":321.41,"loadedMb":2022.75,"peakMb":2034.92,"disposedMb":1765.63}} +{"id":"llm/qwen2-5-0-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN2_5_0_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":439214950,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":16,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":9846.484,"taskLoadMs":396.956,"taskLoad":{"count":3,"min":392.363,"max":441.557,"mean":410.292,"median":396.956,"p90":441.557,"iqr":49.194,"stdev":22.187},"pipeline":{"count":20,"min":694.615,"max":754.697,"mean":716.931,"median":716.697,"p90":731.166,"iqr":16.298,"stdev":14.385},"execution":{"perIteration":{"forward":{"count":65,"ms":703.14264435}},"totalMs":703.14264435},"units":64,"detail":{"timeToFirstTokenMs":64.393,"prefillMs":65,"promptTokens":37},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","status":0,"batteryTemperatureC":-1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":160},"thermalValid":true,"holdsTimedOut":0,"coolingMs":450,"memory":{"baselineMb":330.72,"loadedMb":1181.92,"peakMb":1162.72,"disposedMb":993.58}} +{"id":"llm/hammer2-1-0-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.HAMMER2_1_0_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":443224596,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":17,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":13926.227,"taskLoadMs":394.075,"taskLoad":{"count":3,"min":390.32,"max":464.778,"mean":416.391,"median":394.075,"p90":464.778,"iqr":74.458,"stdev":34.249},"pipeline":{"count":20,"min":628.765,"max":688.84,"mean":663.428,"median":665.017,"p90":684.41,"iqr":28.034,"stdev":17.07},"execution":{"perIteration":{"forward":{"count":65,"ms":650.86715845}},"totalMs":650.86715845},"units":64,"detail":{"timeToFirstTokenMs":48.935,"prefillMs":49,"promptTokens":27},"gate":{"kind":"device","waitedS":60,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","batteryTemperatureC":-1,"status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":150},"thermalValid":true,"holdsTimedOut":0,"coolingMs":684,"memory":{"baselineMb":326.22,"loadedMb":1200.58,"peakMb":1162.49,"disposedMb":994.8}} +{"id":"llm/smollm2-360-m-mlx-int8","task":"llm","model":"llm.SMOLLM2_360M.MLX_INT8","backend":"mlx","precision":"int8","bytes":469941167,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":18,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":8794.8,"taskLoadMs":89.823,"taskLoad":{"count":3,"min":87.848,"max":200.69,"mean":126.12,"median":89.823,"p90":200.69,"iqr":112.842,"stdev":52.735},"pipeline":{"count":20,"min":547.049,"max":564.58,"mean":552.964,"median":550.982,"p90":559.488,"iqr":7.513,"stdev":4.895},"execution":{"perIteration":{"forward":{"count":65,"ms":547.29715055}},"totalMs":547.29715055},"units":64,"detail":{"timeToFirstTokenMs":36.103,"prefillMs":36,"promptTokens":38},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":-1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":128},"thermalValid":true,"holdsTimedOut":0,"coolingMs":285,"memory":{"baselineMb":330.11,"loadedMb":973.02,"peakMb":989.97,"disposedMb":348.81}} +{"id":"llm/qwen3-0-6-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN3_0_6B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":529702696,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":19,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":9753.455,"taskLoadMs":538.175,"taskLoad":{"count":3,"min":515.105,"max":781.477,"mean":611.586,"median":538.175,"p90":781.477,"iqr":266.372,"stdev":120.5},"pipeline":{"count":20,"min":712.032,"max":908.401,"mean":814.834,"median":817.249,"p90":882.397,"iqr":79.257,"stdev":53.503},"execution":{"perIteration":{"forward":{"count":65,"ms":802.1096468}},"totalMs":802.1096468},"units":64,"detail":{"timeToFirstTokenMs":45.608,"prefillMs":46,"promptTokens":16},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"batteryTemperatureC":-1,"statusName":"none"},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":905},"thermalValid":true,"holdsTimedOut":0,"coolingMs":80350,"memory":{"baselineMb":347.33,"loadedMb":2659.86,"peakMb":2638.99,"disposedMb":2401.66}} +{"id":"llm/qwen2-5-0-5-b-mlx-int4","task":"llm","model":"llm.QWEN2_5_0_5B.MLX_INT4","backend":"mlx","precision":"int4","bytes":542089702,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":20,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":10174.619,"taskLoadMs":178.608,"taskLoad":{"count":3,"min":177.81,"max":262.821,"mean":206.413,"median":178.608,"p90":262.821,"iqr":85.011,"stdev":39.888},"pipeline":{"count":20,"min":404.014,"max":421.837,"mean":409.864,"median":406.463,"p90":418.747,"iqr":10.405,"stdev":6.139},"execution":{"perIteration":{"forward":{"count":65,"ms":401.71537025}},"totalMs":401.71537025},"units":64,"detail":{"timeToFirstTokenMs":28.16,"prefillMs":28,"promptTokens":37},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"statusName":"none","status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":93},"thermalValid":true,"holdsTimedOut":0,"coolingMs":226,"memory":{"baselineMb":332.28,"loadedMb":588.44,"peakMb":544.49,"disposedMb":344.8}} +{"id":"llm/hammer2-1-0-5-b-mlx-int4","task":"llm","model":"llm.HAMMER2_1_0_5B.MLX_INT4","backend":"mlx","precision":"int4","bytes":545865372,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":21,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":9994.574,"taskLoadMs":224.55,"taskLoad":{"count":3,"min":209.905,"max":293.941,"mean":242.799,"median":224.55,"p90":293.941,"iqr":84.035,"stdev":36.654},"pipeline":{"count":20,"min":399.953,"max":445.377,"mean":405.602,"median":402.414,"p90":409.61,"iqr":2.961,"stdev":9.718},"execution":{"perIteration":{"forward":{"count":65,"ms":397.30038479999996}},"totalMs":397.30038479999996},"units":64,"detail":{"timeToFirstTokenMs":21.911,"prefillMs":22,"promptTokens":27},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"status":0,"statusName":"none"},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":907},"thermalValid":true,"holdsTimedOut":0,"coolingMs":90261,"memory":{"baselineMb":335.11,"loadedMb":592.2,"peakMb":559.2,"disposedMb":355.52}} +{"id":"llm/lfm2-5-vl-450-m-mlx-int4","task":"llm","model":"llm.LFM2_5_VL_450M.MLX_INT4","backend":"mlx","precision":"int4","bytes":586919819,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":22,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":9769.839,"taskLoadMs":87.935,"taskLoad":{"count":3,"min":87.361,"max":161.156,"mean":112.151,"median":87.935,"p90":161.156,"iqr":73.795,"stdev":34.653},"pipeline":{"count":20,"min":332.352,"max":345.946,"mean":335.707,"median":334.182,"p90":344.851,"iqr":1.985,"stdev":4.211},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":11.01423805},"text_decoder":{"count":65,"ms":316.61380005}},"totalMs":327.6280381},"units":64,"detail":{"timeToFirstTokenMs":16.54,"prefillMs":17,"promptTokens":17},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"statusName":"none","status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":79},"thermalValid":true,"holdsTimedOut":0,"coolingMs":406,"memory":{"baselineMb":338.74,"loadedMb":400.24,"peakMb":392.63,"disposedMb":357.3}} +{"id":"llm/lfm2-5-vl-450-m-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_VL_450M.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":653603851,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":23,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":407.821,"taskLoadMs":345.706,"taskLoad":{"count":3,"min":339.353,"max":900.631,"mean":528.563,"median":345.706,"p90":900.631,"iqr":561.278,"stdev":263.104},"pipeline":{"count":20,"min":792.569,"max":894.589,"mean":843.821,"median":840.295,"p90":882.272,"iqr":45.898,"stdev":27.966},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":1.6624744},"text_decoder":{"count":65,"ms":831.6608096}},"totalMs":833.3232840000001},"units":64,"detail":{"timeToFirstTokenMs":33.938,"prefillMs":34,"promptTokens":17},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"light","status":1,"batteryTemperatureC":-1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":183},"thermalValid":true,"holdsTimedOut":0,"coolingMs":571,"memory":{"baselineMb":357.78,"loadedMb":1262.22,"peakMb":1287.27,"disposedMb":659.53}} +{"id":"llm/lfm2-5-1-2-b-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_1_2B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":815187806,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":24,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":20325.397,"taskLoadMs":565.479,"taskLoad":{"count":3,"min":565.434,"max":642.224,"mean":591.045,"median":565.479,"p90":642.224,"iqr":76.79,"stdev":36.188},"pipeline":{"count":20,"min":1087.325,"max":1550.825,"mean":1363.533,"median":1393.196,"p90":1520.617,"iqr":218.868,"stdev":135.119},"execution":{"perIteration":{"forward":{"count":65,"ms":1353.89382565}},"totalMs":1353.89382565},"units":64,"detail":{"timeToFirstTokenMs":72.154,"prefillMs":72,"promptTokens":17},"gate":{"kind":"device","waitedS":80,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","batteryTemperatureC":-1,"status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":282},"thermalValid":true,"holdsTimedOut":0,"coolingMs":543,"memory":{"baselineMb":328.2,"loadedMb":1567.69,"peakMb":1618.91,"disposedMb":1138.19}} +{"id":"llm/lfm2-5-1-2-b-mlx-int4","task":"llm","model":"llm.LFM2_5_1_2B.MLX_INT4","backend":"mlx","precision":"int4","bytes":895331422,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":25,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":30316.947,"taskLoadMs":94.342,"taskLoad":{"count":3,"min":93.963,"max":172.091,"mean":120.132,"median":94.342,"p90":172.091,"iqr":78.128,"stdev":36.741},"pipeline":{"count":20,"min":774.146,"max":789.978,"mean":778.331,"median":776.573,"p90":783.263,"iqr":3.492,"stdev":4.258},"execution":{"perIteration":{"forward":{"count":65,"ms":772.50459435}},"totalMs":772.50459435},"units":64,"detail":{"timeToFirstTokenMs":54.133,"prefillMs":53,"promptTokens":17},"gate":{"kind":"device","waitedS":70,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","status":0,"batteryTemperatureC":-1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":173},"thermalValid":true,"holdsTimedOut":0,"coolingMs":319,"memory":{"baselineMb":330.08,"loadedMb":532.81,"peakMb":550.77,"disposedMb":355.67}} +{"id":"llm/qwen3-0-6-b-mlx-int8","task":"llm","model":"llm.QWEN3_0_6B.MLX_INT8","backend":"mlx","precision":"int8","bytes":937588776,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":26,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":28345.343,"taskLoadMs":235.569,"taskLoad":{"count":3,"min":233.743,"max":355.589,"mean":274.967,"median":235.569,"p90":355.589,"iqr":121.846,"stdev":57.013},"pipeline":{"count":20,"min":758.68,"max":800.46,"mean":766.056,"median":763.12,"p90":768.104,"iqr":1.374,"stdev":9.575},"execution":{"perIteration":{"forward":{"count":65,"ms":757.88237125}},"totalMs":757.88237125},"units":64,"detail":{"timeToFirstTokenMs":26.578,"prefillMs":26,"promptTokens":16},"gate":{"kind":"device","waitedS":60,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"status":0,"statusName":"none"},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":175},"thermalValid":true,"holdsTimedOut":0,"coolingMs":416,"memory":{"baselineMb":334.72,"loadedMb":1258.56,"peakMb":1220.47,"disposedMb":370.55}} +{"id":"llm/bielik-v3-1-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.BIELIK_V3_1_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":939299957,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":27,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":15751.293,"taskLoadMs":701.458,"taskLoad":{"count":3,"min":695.806,"max":774.544,"mean":723.936,"median":701.458,"p90":774.544,"iqr":78.738,"stdev":35.859},"pipeline":{"count":20,"min":1410.476,"max":2202.569,"mean":1918.203,"median":2002.217,"p90":2168.908,"iqr":389.869,"stdev":242.371},"execution":{"perIteration":{"forward":{"count":65,"ms":1910.5696901}},"totalMs":1910.5696901},"units":64,"detail":{"timeToFirstTokenMs":121.833,"prefillMs":122,"promptTokens":26},"gate":{"kind":"device","waitedS":70,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"batteryTemperatureC":-1,"statusName":"none"},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1297},"thermalValid":true,"holdsTimedOut":0,"coolingMs":100448,"memory":{"baselineMb":346.33,"loadedMb":1974.39,"peakMb":2055.13,"disposedMb":1332.88}} +{"id":"llm/llama3-2-1-b-xnnpack-spinquant","task":"llm","model":"llm.LLAMA3_2_1B.XNNPACK_SPINQUANT","backend":"xnnpack","precision":"spinquant","bytes":1092444893,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":28,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":17963.451,"taskLoadMs":796.89,"taskLoad":{"count":3,"min":796.271,"max":936.267,"mean":843.142,"median":796.89,"p90":936.267,"iqr":139.996,"stdev":65.849},"pipeline":{"count":20,"min":1190.497,"max":1901.707,"mean":1614.319,"median":1645.407,"p90":1861.253,"iqr":372.613,"stdev":221.705},"execution":{"perIteration":{"forward":{"count":65,"ms":1601.1245603}},"totalMs":1601.1245603},"units":64,"detail":{"timeToFirstTokenMs":141.647,"prefillMs":142,"promptTokens":43},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"status":0,"statusName":"none"},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1288},"thermalValid":true,"holdsTimedOut":0,"coolingMs":100351,"memory":{"baselineMb":353.47,"loadedMb":2502.71,"peakMb":2445.19,"disposedMb":2012.39}} +{"id":"llm/qwen2-5-1-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN2_5_1_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":1172577510,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":29,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":19787.787,"taskLoadMs":866.338,"taskLoad":{"count":3,"min":856.742,"max":940.087,"mean":887.722,"median":866.338,"p90":940.087,"iqr":83.344,"stdev":37.234},"pipeline":{"count":20,"min":1394.249,"max":2059.833,"mean":1812.397,"median":1893.696,"p90":2022.224,"iqr":334.375,"stdev":204.318},"execution":{"perIteration":{"forward":{"count":65,"ms":1798.0228309499998}},"totalMs":1798.0228309499998},"units":64,"detail":{"timeToFirstTokenMs":172.79,"prefillMs":173,"promptTokens":37},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","status":0,"batteryTemperatureC":-1},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1291},"thermalValid":true,"holdsTimedOut":0,"coolingMs":100499,"memory":{"baselineMb":368.97,"loadedMb":2434.39,"peakMb":2443.74,"disposedMb":1820.07}} +{"id":"llm/hammer2-1-1-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.HAMMER2_1_1_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":1176265276,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":30,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":34255.507,"taskLoadMs":850.426,"taskLoad":{"count":3,"min":842.973,"max":959.736,"mean":884.378,"median":850.426,"p90":959.736,"iqr":116.763,"stdev":53.373},"pipeline":{"count":20,"min":1358.945,"max":2137.238,"mean":1848.991,"median":1915.965,"p90":2100.859,"iqr":375.64,"stdev":237.842},"execution":{"perIteration":{"forward":{"count":65,"ms":1834.2279881499999}},"totalMs":1834.2279881499999},"units":64,"detail":{"timeToFirstTokenMs":109.751,"prefillMs":110,"promptTokens":27},"gate":{"kind":"device","waitedS":70,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":-1},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1289},"thermalValid":true,"holdsTimedOut":0,"coolingMs":100340,"memory":{"baselineMb":366.89,"loadedMb":2449.28,"peakMb":2444.35,"disposedMb":1820.77}} +{"id":"llm/llama3-2-1-b-mlx-int4","task":"llm","model":"llm.LLAMA3_2_1B.MLX_INT4","backend":"mlx","precision":"int4","bytes":1192062941,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":31,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":32619.255,"taskLoadMs":238.795,"taskLoad":{"count":3,"min":238.438,"max":345.87,"mean":274.368,"median":238.795,"p90":345.87,"iqr":107.431,"stdev":50.56},"pipeline":{"count":20,"min":824.327,"max":859.983,"mean":834.156,"median":828.662,"p90":846.097,"iqr":16.522,"stdev":10.505},"execution":{"perIteration":{"forward":{"count":65,"ms":826.2932995499999}},"totalMs":826.2932995499999},"units":64,"detail":{"timeToFirstTokenMs":83.376,"prefillMs":83,"promptTokens":43},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"statusName":"light","status":1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":184},"thermalValid":true,"holdsTimedOut":0,"coolingMs":280,"memory":{"baselineMb":377.81,"loadedMb":894.41,"peakMb":898.05,"disposedMb":397.22}} +{"id":"llm/qwen2-5-1-5-b-mlx-int4","task":"llm","model":"llm.QWEN2_5_1_5B.MLX_INT4","backend":"mlx","precision":"int4","bytes":1294354790,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":32,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":29850.962,"taskLoadMs":188.029,"taskLoad":{"count":3,"min":187.945,"max":312.775,"mean":229.583,"median":188.029,"p90":312.775,"iqr":124.83,"stdev":58.826},"pipeline":{"count":20,"min":1053.196,"max":1135.361,"mean":1071.651,"median":1066.008,"p90":1087.687,"iqr":26.44,"stdev":20.715},"execution":{"perIteration":{"forward":{"count":65,"ms":1062.85407115}},"totalMs":1062.85407115},"units":64,"detail":{"timeToFirstTokenMs":87,"prefillMs":87,"promptTokens":37},"gate":{"kind":"device","waitedS":80,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"light","status":1,"batteryTemperatureC":-1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":238},"thermalValid":true,"holdsTimedOut":0,"coolingMs":325,"memory":{"baselineMb":367.22,"loadedMb":905.77,"peakMb":868.16,"disposedMb":388.86}} +{"id":"llm/hammer2-1-1-5-b-mlx-int4","task":"llm","model":"llm.HAMMER2_1_1_5B.MLX_INT4","backend":"mlx","precision":"int4","bytes":1297691484,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":33,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":28511.597,"taskLoadMs":220.923,"taskLoad":{"count":3,"min":220.169,"max":351.675,"mean":264.256,"median":220.923,"p90":351.675,"iqr":131.506,"stdev":61.816},"pipeline":{"count":20,"min":1028.755,"max":1094.545,"mean":1049.198,"median":1037.768,"p90":1075.283,"iqr":28.742,"stdev":20.696},"execution":{"perIteration":{"forward":{"count":65,"ms":1040.60637615}},"totalMs":1040.60637615},"units":64,"detail":{"timeToFirstTokenMs":75.851,"prefillMs":75,"promptTokens":27},"gate":{"kind":"device","waitedS":90,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"status":1,"statusName":"light"},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":236},"thermalValid":true,"holdsTimedOut":0,"coolingMs":425,"memory":{"baselineMb":365.56,"loadedMb":896.24,"peakMb":865.1,"disposedMb":386.19}} +{"id":"llm/qwen3-1-7-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN3_1_7B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":1319364648,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":34,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":23257.441,"taskLoadMs":930.66,"taskLoad":{"count":3,"min":929.361,"max":1217.73,"mean":1025.917,"median":930.66,"p90":1217.73,"iqr":288.369,"stdev":135.633},"pipeline":{"count":20,"min":1462.147,"max":2354.966,"mean":1972.725,"median":1983.517,"p90":2298.996,"iqr":411.221,"stdev":267.331},"execution":{"perIteration":{"forward":{"count":65,"ms":1958.39808155}},"totalMs":1958.39808155},"units":64,"detail":{"timeToFirstTokenMs":95.061,"prefillMs":95,"promptTokens":16},"gate":{"kind":"device","waitedS":90,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"batteryTemperatureC":-1,"statusName":"none"},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1312},"thermalValid":true,"holdsTimedOut":0,"coolingMs":100337,"memory":{"baselineMb":366.99,"loadedMb":2467.75,"peakMb":2459.44,"disposedMb":1755.17}} +{"id":"llm/qwen3-1-7-b-mlx-int4","task":"llm","model":"llm.QWEN3_1_7B.MLX_INT4","backend":"mlx","precision":"int4","bytes":1548250920,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":35,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":45416.945,"taskLoadMs":221.417,"taskLoad":{"count":3,"min":219.568,"max":346.631,"mean":262.539,"median":221.417,"p90":346.631,"iqr":127.064,"stdev":59.467},"pipeline":{"count":20,"min":1140.191,"max":1198.917,"mean":1157.801,"median":1150.583,"p90":1174.737,"iqr":21.905,"stdev":14.693},"execution":{"perIteration":{"forward":{"count":65,"ms":1148.79435635}},"totalMs":1148.79435635},"units":64,"detail":{"timeToFirstTokenMs":66.325,"prefillMs":65,"promptTokens":16},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":-1},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1267},"thermalValid":true,"holdsTimedOut":0,"coolingMs":110275,"memory":{"baselineMb":370.25,"loadedMb":903.38,"peakMb":878.39,"disposedMb":403.1}} +{"id":"llm/lfm2-5-2-6-b-mlx-int4","task":"llm","model":"llm.LFM2_5_2_6B.MLX_INT4","backend":"mlx","precision":"int4","bytes":1683681647,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":36,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":29694.484,"taskLoadMs":303.792,"taskLoad":{"count":3,"min":303.084,"max":396.524,"mean":334.467,"median":303.792,"p90":396.524,"iqr":93.441,"stdev":43.882},"pipeline":{"count":20,"min":1910.242,"max":2079.218,"mean":1956.913,"median":1940.244,"p90":1995.675,"iqr":36.84,"stdev":42.514},"execution":{"perIteration":{"forward":{"count":65,"ms":1946.3072826}},"totalMs":1946.3072826},"units":64,"detail":{"timeToFirstTokenMs":115.965,"prefillMs":116,"promptTokens":18},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"statusName":"light","status":1},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1434},"thermalValid":true,"holdsTimedOut":0,"coolingMs":110526,"memory":{"baselineMb":368.06,"loadedMb":798.7,"peakMb":799.06,"disposedMb":397.53}} +{"id":"llm/lfm2-5-2-6-b-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_2_6B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":1807639919,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":37,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":28956.414,"taskLoadMs":1444.923,"taskLoad":{"count":3,"min":1438.412,"max":1752.91,"mean":1545.415,"median":1444.923,"p90":1752.91,"iqr":314.498,"stdev":146.745},"pipeline":{"count":20,"min":2160.745,"max":3585.83,"mean":3173.937,"median":3285.953,"p90":3563.947,"iqr":491.725,"stdev":397.194},"execution":{"perIteration":{"forward":{"count":65,"ms":3160.18452995}},"totalMs":3160.18452995},"units":64,"detail":{"timeToFirstTokenMs":158.705,"prefillMs":158,"promptTokens":18},"gate":{"kind":"device","waitedS":100,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"light","status":1,"batteryTemperatureC":-1},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1648},"thermalValid":true,"holdsTimedOut":0,"coolingMs":110745,"memory":{"baselineMb":390.2,"loadedMb":2210.53,"peakMb":2119.05,"disposedMb":892.33}} +{"id":"llm/smollm2-1-7-b-xnnpack-8-da8-w","task":"llm","model":"llm.SMOLLM2_1_7B.XNNPACK_8DA8W","backend":"xnnpack","precision":"8da8w","bytes":1818983855,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":38,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":31142.314,"taskLoadMs":1850.296,"taskLoad":{"count":3,"min":1377.175,"max":2235.009,"mean":1820.827,"median":1850.296,"p90":2235.009,"iqr":857.834,"stdev":350.829},"pipeline":{"count":20,"min":2319.849,"max":3819.492,"mean":3366.152,"median":3608.627,"p90":3813.196,"iqr":743.891,"stdev":484.891},"execution":{"perIteration":{"forward":{"count":65,"ms":3357.3558153}},"totalMs":3357.3558153},"units":64,"detail":{"timeToFirstTokenMs":167.068,"prefillMs":167,"promptTokens":38},"gate":{"kind":"device","waitedS":100,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","status":0,"batteryTemperatureC":-1},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":2587},"thermalValid":true,"holdsTimedOut":0,"coolingMs":210539,"memory":{"baselineMb":383.14,"loadedMb":2996.36,"peakMb":3068.11,"disposedMb":1630.82}} +{"id":"llm/smollm2-1-7-b-mlx-int8","task":"llm","model":"llm.SMOLLM2_1_7B.MLX_INT8","backend":"mlx","precision":"int8","bytes":1968594991,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":39,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":36634.787,"taskLoadMs":82.458,"taskLoad":{"count":3,"min":80.947,"max":198.767,"mean":120.724,"median":82.458,"p90":198.767,"iqr":117.821,"stdev":55.188},"pipeline":{"count":20,"min":2064.996,"max":2107.731,"mean":2082.613,"median":2078.595,"p90":2097.649,"iqr":9.437,"stdev":10.743},"execution":{"perIteration":{"forward":{"count":65,"ms":2070.84891255}},"totalMs":2070.84891255},"units":64,"detail":{"timeToFirstTokenMs":97.528,"prefillMs":98,"promptTokens":38},"gate":{"kind":"device","waitedS":60,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":-1},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1451},"thermalValid":true,"holdsTimedOut":0,"coolingMs":110429,"memory":{"baselineMb":383.52,"loadedMb":773.02,"peakMb":809.99,"disposedMb":382.97}} +{"id":"llm/hammer2-1-3-b-xnnpack-8-da4-w","task":"llm","model":"llm.HAMMER2_1_3B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2076552284,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":40,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":33034.026,"taskLoadMs":1755.267,"taskLoad":{"count":3,"min":1655.711,"max":2342.059,"mean":1917.679,"median":1755.267,"p90":2342.059,"iqr":686.348,"stdev":302.822},"pipeline":{"count":20,"min":2522.138,"max":4145.214,"mean":3546.365,"median":3678.861,"p90":4109.093,"iqr":821.469,"stdev":520.512},"execution":{"perIteration":{"forward":{"count":65,"ms":3531.5280045}},"totalMs":3531.5280045},"units":64,"detail":{"timeToFirstTokenMs":230.848,"prefillMs":230,"promptTokens":27},"gate":{"kind":"device","waitedS":80,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"status":0,"statusName":"none"},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":2550},"thermalValid":true,"holdsTimedOut":0,"coolingMs":200458,"memory":{"baselineMb":372.81,"loadedMb":3073.14,"peakMb":3049.69,"disposedMb":1646.32}} +{"id":"llm/qwen2-5-3-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN2_5_3B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2111318246,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":41,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":35253.157,"taskLoadMs":1565.858,"taskLoad":{"count":3,"min":1553.587,"max":2003.632,"mean":1707.692,"median":1565.858,"p90":2003.632,"iqr":450.044,"stdev":209.321},"pipeline":{"count":20,"min":2596.694,"max":4276.525,"mean":3672.975,"median":3847.979,"p90":4218.361,"iqr":840.785,"stdev":541.499},"execution":{"perIteration":{"forward":{"count":65,"ms":3657.80083845}},"totalMs":3657.80083845},"units":64,"detail":{"timeToFirstTokenMs":304.084,"prefillMs":304,"promptTokens":37},"gate":{"kind":"device","waitedS":80,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"batteryTemperatureC":-1,"statusName":"none"},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":2757},"thermalValid":true,"holdsTimedOut":0,"coolingMs":220499,"memory":{"baselineMb":382.17,"loadedMb":3028.72,"peakMb":3042.74,"disposedMb":1639.24}} +{"id":"llm/qwen2-5-3-b-mlx-int4","task":"llm","model":"llm.QWEN2_5_3B.MLX_INT4","backend":"mlx","precision":"int4","bytes":2269457894,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":42,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":51173.74,"taskLoadMs":192.529,"taskLoad":{"count":3,"min":190.121,"max":290.565,"mean":224.405,"median":192.529,"p90":290.565,"iqr":100.444,"stdev":46.793},"pipeline":{"count":20,"min":2032.263,"max":2194.545,"mean":2069.728,"median":2054.336,"p90":2090.706,"iqr":44.627,"stdev":43.282},"execution":{"perIteration":{"forward":{"count":65,"ms":2060.4170523000003}},"totalMs":2060.4170523000003},"units":64,"detail":{"timeToFirstTokenMs":173.608,"prefillMs":173,"promptTokens":37},"gate":{"kind":"device","waitedS":50,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":-1},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1449},"thermalValid":true,"holdsTimedOut":0,"coolingMs":110297,"memory":{"baselineMb":379.74,"loadedMb":824,"peakMb":773.7,"disposedMb":397.67}} +{"id":"llm/hammer2-1-3-b-mlx-int4","task":"llm","model":"llm.HAMMER2_1_3B.MLX_INT4","backend":"mlx","precision":"int4","bytes":2272443356,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":43,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":45306.367,"taskLoadMs":221.58,"taskLoad":{"count":3,"min":221.4,"max":332.433,"mean":258.471,"median":221.58,"p90":332.433,"iqr":111.033,"stdev":52.299},"pipeline":{"count":20,"min":2003.537,"max":2236.131,"mean":2038.972,"median":2026.174,"p90":2061.99,"iqr":37.908,"stdev":50.02},"execution":{"perIteration":{"forward":{"count":65,"ms":2029.58649345}},"totalMs":2029.58649345},"units":64,"detail":{"timeToFirstTokenMs":123.155,"prefillMs":122,"promptTokens":27},"gate":{"kind":"device","waitedS":80,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":-1},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1538},"thermalValid":true,"holdsTimedOut":0,"coolingMs":120473,"memory":{"baselineMb":368.16,"loadedMb":828.7,"peakMb":790.22,"disposedMb":414.27}} +{"id":"llm/lfm2-5-vl-1-6-b-mlx-int4","task":"llm","model":"llm.LFM2_5_VL_1_6B.MLX_INT4","backend":"mlx","precision":"int4","bytes":2375405323,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":44,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":57301.897,"taskLoadMs":94.496,"taskLoad":{"count":3,"min":92.43,"max":154.634,"mean":113.853,"median":94.496,"p90":154.634,"iqr":62.203,"stdev":28.848},"pipeline":{"count":20,"min":884.533,"max":909.062,"mean":891.52,"median":888.946,"p90":900.974,"iqr":3.961,"stdev":6.467},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":11.6181876},"text_decoder":{"count":65,"ms":871.25826025}},"totalMs":882.8764478500001},"units":64,"detail":{"timeToFirstTokenMs":60.281,"prefillMs":60,"promptTokens":17},"gate":{"kind":"device","waitedS":80,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"statusName":"light","status":1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":197},"thermalValid":true,"holdsTimedOut":0,"coolingMs":266,"memory":{"baselineMb":364.63,"loadedMb":500.2,"peakMb":496.94,"disposedMb":397.06}} +{"id":"llm/llama3-2-3-b-xnnpack-spinquant","task":"llm","model":"llm.LLAMA3_2_3B.XNNPACK_SPINQUANT","backend":"xnnpack","precision":"spinquant","bytes":2389281117,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":45,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":17.423,"taskLoadMs":2333.22,"taskLoad":{"count":3,"min":2285.863,"max":2885.033,"mean":2501.372,"median":2333.22,"p90":2885.033,"iqr":599.17,"stdev":271.977},"pipeline":{"count":20,"min":2882.99,"max":4697.985,"mean":4125.632,"median":4297.54,"p90":4652.011,"iqr":796.922,"stdev":558.245},"execution":{"perIteration":{"forward":{"count":65,"ms":4111.8548866500005}},"totalMs":4111.8548866500005},"units":64,"detail":{"timeToFirstTokenMs":372.317,"prefillMs":373,"promptTokens":43},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"light","status":1,"batteryTemperatureC":-1},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":2487},"thermalValid":true,"holdsTimedOut":0,"coolingMs":180721,"memory":{"baselineMb":37.67,"loadedMb":3128.88,"peakMb":3150.39,"disposedMb":1760.02}} +{"id":"llm/lfm2-5-vl-1-6-b-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_VL_1_6B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2479599371,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":46,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":39865.332,"taskLoadMs":1203.994,"taskLoad":{"count":3,"min":1034.691,"max":1877.22,"mean":1371.969,"median":1203.994,"p90":1877.22,"iqr":842.529,"stdev":363.891},"pipeline":{"count":20,"min":1156.387,"max":1875.48,"mean":1598.268,"median":1653.655,"p90":1824.601,"iqr":334.336,"stdev":212.584},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":1.9833646999999999},"text_decoder":{"count":65,"ms":1586.2710897000002}},"totalMs":1588.2544544},"units":64,"detail":{"timeToFirstTokenMs":59.909,"prefillMs":60,"promptTokens":17},"gate":{"kind":"device","waitedS":80,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","batteryTemperatureC":-1,"status":0},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1236},"thermalValid":true,"holdsTimedOut":0,"coolingMs":100379,"memory":{"baselineMb":59.3,"loadedMb":2945.94,"peakMb":2973.75,"disposedMb":629.52}} +{"id":"llm/llama3-2-3-b-mlx-int4","task":"llm","model":"llm.LLAMA3_2_3B.MLX_INT4","backend":"mlx","precision":"int4","bytes":2505235805,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":47,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":52104.566,"taskLoadMs":246.338,"taskLoad":{"count":3,"min":241.895,"max":374.702,"mean":287.645,"median":246.338,"p90":374.702,"iqr":132.808,"stdev":61.586},"pipeline":{"count":20,"min":2199.657,"max":2435.992,"mean":2240.28,"median":2221.092,"p90":2280.419,"iqr":36.704,"stdev":51.705},"execution":{"perIteration":{"forward":{"count":65,"ms":2223.64065365}},"totalMs":2223.64065365},"units":64,"detail":{"timeToFirstTokenMs":205.725,"prefillMs":205,"promptTokens":43},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"batteryTemperatureC":-1,"statusName":"light"},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1483},"thermalValid":true,"holdsTimedOut":0,"coolingMs":110353,"memory":{"baselineMb":72.28,"loadedMb":717.49,"peakMb":709.02,"disposedMb":357.22}} +{"id":"llm/gemma4-e2-b-mlx-int4","task":"llm","model":"llm.GEMMA4_E2B.MLX_INT4","backend":"mlx","precision":"int4","bytes":2559158836,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":48,"caseCount":56,"repeat":1,"repeats":1},"status":"ok","downloadMs":63551.099,"taskLoadMs":614.445,"taskLoad":{"count":3,"min":611.922,"max":685.038,"mean":637.135,"median":614.445,"p90":685.038,"iqr":73.116,"stdev":33.888},"pipeline":{"count":20,"min":1800.205,"max":2036.393,"mean":1851.912,"median":1834.947,"p90":1895.752,"iqr":66.142,"stdev":58.741},"execution":{"perIteration":{"forward":{"count":65,"ms":1836.87946755}},"totalMs":1836.87946755},"units":64,"detail":{"timeToFirstTokenMs":110.686,"prefillMs":110,"promptTokens":17},"gate":{"kind":"device","waitedS":80,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":-1},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1523},"thermalValid":true,"holdsTimedOut":0,"coolingMs":120658,"memory":{"baselineMb":341.64,"loadedMb":678.8,"peakMb":534.06,"disposedMb":353.05}} +{"id":"llm/qwen3-4-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN3_4B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2697273128,"mode":"async","status":"error","error":"iOS terminated the app with signal 9 after a critical memory-pressure warning; the same benchmark also terminated in a fresh-process retry before a measurement completed.","progress":{"caseIndex":49,"caseCount":56,"repeat":1,"repeats":1},"downloadMs":0,"taskLoadMs":0} diff --git a/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-ios-iPhone18-3.notes.json b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-ios-iPhone18-3.notes.json new file mode 100644 index 0000000000..44af87b9c2 --- /dev/null +++ b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-ios-iPhone18-3.notes.json @@ -0,0 +1,30 @@ +{ + "events": [ + { + "caseId": "llm/llama3-2-3-b-xnnpack-spinquant", + "event": "process-terminated", + "signal": 9, + "precedingConsoleWarning": "TRIM_MEMORY_RUNNING_CRITICAL", + "consoleWarningTime": "2026-10-26 18:53:21", + "assessment": "Memory pressure suspected; no matching Jetsam report was available through devicectl. A memory leak was not established.", + "recovery": "Relaunched the same release app in a fresh process; resumed from saved measurements. The retried case completed successfully." + }, + { + "caseId": "llm/qwen3-4-b-xnnpack-8-da4-w", + "event": "process-terminated", + "signal": 9, + "precedingConsoleWarning": "TRIM_MEMORY_RUNNING_CRITICAL", + "consoleWarningTime": "2026-10-26 19:16:17", + "assessment": "Memory pressure suspected. A repeated-load leak has not been established.", + "recovery": "Fresh-process retry also terminated with signal 9 immediately after a critical memory warning. Recorded as a failed attempt; no timing published." + } + ], + "remainingCases": "Fresh process per model after each saved measurement.", + "skippedByUserRequest": ["llm/phi4-mini-xnnpack-8-da4-w", "llm/phi4-mini-mlx-int4"], + "completion": { + "successfulVariants": 53, + "failedVariants": ["llm/qwen3-4-b-xnnpack-8-da4-w"], + "skippedByUserRequest": ["llm/phi4-mini-xnnpack-8-da4-w", "llm/phi4-mini-mlx-int4"], + "allRequestedCasesAccountedFor": true + } +} diff --git a/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-ios-iPhone18-3.jsonl b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-ios-iPhone18-3.jsonl new file mode 100644 index 0000000000..4e5f948544 --- /dev/null +++ b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-ios-iPhone18-3.jsonl @@ -0,0 +1,4 @@ +{"type":"run","label":"v0.11.0-missing-20261026-isolated","platform":"ios","device":{"model":"iPhone18,3","totalMemoryBytes":8044216320,"isEmulator":false,"soc":"iPhone18,3","cpuCores":6,"osVersion":"26.6"},"inputSpecVersion":1,"buildType":"release","clocksPinned":false,"settings":{"suite":"llm/gemma4-omni-e2-b-mlx-int4,llm/phi4-mini-xnnpack-8-da4-w,llm/qwen3-4-b-mlx-int4,llm/lfm2-5-vl-1-6-b-mlx-int8,llm/gemma4-e2-b-xnnpack-8-da4-w,llm/phi4-mini-mlx-int4,llm/gemma4-omni-e2-b-xnnpack-8-da4-w","iterations":20,"warmup":3,"memoryIterations":5,"loadIterations":3,"repeats":1,"maxTempC":37,"maxBytes":6000000000,"urlMap":{}},"startedAt":"2026-10-26T18:28:38.113Z"} +{"id":"llm/gemma4-omni-e2-b-mlx-int4","task":"llm","model":"llm.GEMMA4_OMNI_E2B.MLX_INT4","backend":"mlx","precision":"int4","bytes":2760555316,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":1,"caseCount":7,"repeat":1,"repeats":1},"status":"ok","downloadMs":40170.667,"taskLoadMs":780.654,"taskLoad":{"count":3,"min":755.83,"max":2001.963,"mean":1179.482,"median":780.654,"p90":2001.963,"iqr":1246.132,"stdev":581.67},"pipeline":{"count":20,"min":1818.928,"max":2115.368,"mean":1927.589,"median":1891.813,"p90":2055.131,"iqr":149.211,"stdev":95.293},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":18.33401405},"text_decoder":{"count":65,"ms":1882.6945878499998}},"totalMs":1901.0286019},"units":64,"detail":{"timeToFirstTokenMs":109.192,"prefillMs":109,"promptTokens":17},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"statusName":"none","status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":423},"thermalValid":true,"holdsTimedOut":0,"coolingMs":508,"memory":{"baselineMb":41.31,"loadedMb":749.39,"peakMb":612.31,"disposedMb":471.3}} +{"id":"llm/phi4-mini-xnnpack-8-da4-w","task":"llm","model":"llm.PHI4_MINI.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2833963001,"status":"skipped","mode":"async","error":"Skipped on iPhone at user request due to RAM concerns.","progress":{"caseIndex":2,"caseCount":7,"repeat":1,"repeats":1},"downloadMs":0,"taskLoadMs":0} +{"id":"llm/phi4-mini-mlx-int4","task":"llm","model":"llm.PHI4_MINI.MLX_INT4","backend":"mlx","precision":"int4","bytes":3283172473,"status":"skipped","mode":"async","error":"Skipped on iPhone at user request due to RAM concerns.","progress":{"caseIndex":6,"caseCount":7,"repeat":1,"repeats":1},"downloadMs":0,"taskLoadMs":0} diff --git a/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-ios-iPhone18-3.notes.json b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-ios-iPhone18-3.notes.json new file mode 100644 index 0000000000..89d04ffe89 --- /dev/null +++ b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-ios-iPhone18-3.notes.json @@ -0,0 +1,15 @@ +{ + "procedure": "Each saved measurement is followed by app termination and relaunch. Completed cases are skipped by the collector. Timing and memory iteration counts are unchanged.", + "launches": [ + { + "caseId": "llm/gemma4-omni-e2-b-mlx-int4", + "procedure": "fresh process before this case", + "launchedAtUtc": "2026-10-26T18:28:38.181524+00:00" + }, + { + "caseId": "llm/phi4-mini-xnnpack-8-da4-w", + "procedure": "fresh process before this case", + "launchedAtUtc": "2026-10-26T18:30:20.602376+00:00" + } + ] +} diff --git a/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-no-phi4-ios-iPhone18-3.jsonl b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-no-phi4-ios-iPhone18-3.jsonl new file mode 100644 index 0000000000..52af5858fb --- /dev/null +++ b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-no-phi4-ios-iPhone18-3.jsonl @@ -0,0 +1,5 @@ +{"type":"run","label":"v0.11.0-missing-20261026-isolated-no-phi4","platform":"ios","device":{"model":"iPhone18,3","osVersion":"26.6","totalMemoryBytes":8044216320,"soc":"iPhone18,3","isEmulator":false,"cpuCores":6},"inputSpecVersion":1,"buildType":"release","clocksPinned":false,"settings":{"suite":"llm/qwen3-4-b-mlx-int4,llm/lfm2-5-vl-1-6-b-mlx-int8,llm/gemma4-e2-b-xnnpack-8-da4-w,llm/gemma4-omni-e2-b-xnnpack-8-da4-w","iterations":20,"warmup":3,"memoryIterations":5,"loadIterations":3,"repeats":1,"maxTempC":37,"maxBytes":6000000000,"urlMap":{}},"startedAt":"2026-10-26T18:34:24.444Z"} +{"id":"llm/qwen3-4-b-mlx-int4","task":"llm","model":"llm.QWEN3_4B.MLX_INT4","backend":"mlx","precision":"int4","bytes":2926890664,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":1,"caseCount":4,"repeat":1,"repeats":1},"status":"ok","downloadMs":55305.088,"taskLoadMs":236.253,"taskLoad":{"count":3,"min":227.815,"max":494.229,"mean":319.433,"median":236.253,"p90":494.229,"iqr":266.414,"stdev":123.648},"pipeline":{"count":20,"min":2677.706,"max":3321.982,"mean":2871.853,"median":2828.863,"p90":3144.445,"iqr":301.764,"stdev":192.734},"execution":{"perIteration":{"forward":{"count":65,"ms":2847.97120225}},"totalMs":2847.97120225},"units":64,"detail":{"timeToFirstTokenMs":180.916,"prefillMs":180,"promptTokens":16},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","batteryTemperatureC":-1,"status":0},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":1452},"thermalValid":true,"holdsTimedOut":0,"coolingMs":90485,"memory":{"baselineMb":43.8,"loadedMb":839.99,"peakMb":815.55,"disposedMb":342.81}} +{"id":"llm/lfm2-5-vl-1-6-b-mlx-int8","task":"llm","model":"llm.LFM2_5_VL_1_6B.MLX_INT8","backend":"mlx","precision":"int8","bytes":2960510731,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":2,"caseCount":4,"repeat":1,"repeats":1},"status":"ok","downloadMs":62401.724,"taskLoadMs":92.751,"taskLoad":{"count":3,"min":89.299,"max":125.618,"mean":102.556,"median":92.751,"p90":125.618,"iqr":36.32,"stdev":16.368},"pipeline":{"count":20,"min":1487.359,"max":1622.929,"mean":1514.596,"median":1504.482,"p90":1536.406,"iqr":33.035,"stdev":32.024},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":11.83152035},"text_decoder":{"count":65,"ms":1492.30833605}},"totalMs":1504.1398564},"units":64,"detail":{"timeToFirstTokenMs":69.798,"prefillMs":70,"promptTokens":17},"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"light","batteryTemperatureC":-1,"status":1},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":335},"thermalValid":true,"holdsTimedOut":0,"coolingMs":390,"memory":{"baselineMb":40.13,"loadedMb":200.05,"peakMb":216.44,"disposedMb":216.42}} +{"id":"llm/gemma4-e2-b-xnnpack-8-da4-w","task":"llm","model":"llm.GEMMA4_E2B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":3247469492,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":3,"caseCount":4,"repeat":1,"repeats":1},"status":"ok","downloadMs":62591.546,"taskLoadMs":2150.484,"taskLoad":{"count":3,"min":2085.863,"max":2258.045,"mean":2164.797,"median":2150.484,"p90":2258.045,"iqr":172.182,"stdev":71.018},"pipeline":{"count":20,"min":5547.365,"max":6848.167,"mean":6280.152,"median":6377.982,"p90":6771.84,"iqr":567.203,"stdev":396.557},"execution":{"perIteration":{"forward":{"count":65,"ms":6251.0382644500005}},"totalMs":6251.0382644500005},"units":64,"detail":{"timeToFirstTokenMs":255.627,"prefillMs":255,"promptTokens":17},"gate":{"kind":"device","waitedS":70,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"batteryTemperatureC":-1,"statusName":"none"},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":4165},"thermalValid":true,"holdsTimedOut":0,"coolingMs":310702,"memory":{"baselineMb":42.44,"loadedMb":2073.32,"peakMb":1933.71,"disposedMb":602.72}} +{"id":"llm/gemma4-omni-e2-b-xnnpack-8-da4-w","task":"llm","model":"llm.GEMMA4_OMNI_E2B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":3651367476,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":4,"caseCount":4,"repeat":1,"repeats":1},"status":"ok","downloadMs":55922.123,"taskLoadMs":2636.022,"taskLoad":{"count":3,"min":2623.778,"max":3718.426,"mean":2992.742,"median":2636.022,"p90":3718.426,"iqr":1094.648,"stdev":513.16},"pipeline":{"count":20,"min":5548.318,"max":6812.044,"mean":6195.531,"median":6248.999,"p90":6657.515,"iqr":625.424,"stdev":377.182},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":6.79186505},"text_decoder":{"count":65,"ms":6158.18550375}},"totalMs":6164.9773688000005},"units":64,"detail":{"timeToFirstTokenMs":281.066,"prefillMs":280,"promptTokens":17},"gate":{"kind":"device","waitedS":70,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"status":1,"statusName":"light"},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":null,"samples":4268},"thermalValid":true,"holdsTimedOut":0,"coolingMs":324604,"memory":{"baselineMb":42.08,"loadedMb":2528.11,"peakMb":2393.77,"disposedMb":711.27}} diff --git a/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-no-phi4-ios-iPhone18-3.notes.json b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-no-phi4-ios-iPhone18-3.notes.json new file mode 100644 index 0000000000..db7c4c1f0e --- /dev/null +++ b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-missing-20261026-isolated-no-phi4-ios-iPhone18-3.notes.json @@ -0,0 +1,25 @@ +{ + "procedure": "Each saved measurement is followed by app termination and relaunch. Completed cases are skipped by the collector. Timing and memory iteration counts are unchanged.", + "launches": [ + { + "caseId": "llm/qwen3-4-b-mlx-int4", + "procedure": "fresh process before this case", + "launchedAtUtc": "2026-10-26T18:34:25.131877+00:00" + }, + { + "caseId": "llm/lfm2-5-vl-1-6-b-mlx-int8", + "procedure": "fresh process before this case", + "launchedAtUtc": "2026-10-26T18:38:15.791811+00:00" + }, + { + "caseId": "llm/gemma4-e2-b-xnnpack-8-da4-w", + "procedure": "fresh process before this case", + "launchedAtUtc": "2026-10-26T18:40:05.264007+00:00" + }, + { + "caseId": "llm/gemma4-omni-e2-b-xnnpack-8-da4-w", + "procedure": "fresh process before this case", + "launchedAtUtc": "2026-10-26T18:50:33.720441+00:00" + } + ] +} diff --git a/docs/docs/04-benchmarks/runs/s20plus/v0.11.0-missing-20261026-android-SM-G986B.jsonl b/docs/docs/04-benchmarks/runs/s20plus/v0.11.0-missing-20261026-android-SM-G986B.jsonl new file mode 100644 index 0000000000..d5e609c9a0 --- /dev/null +++ b/docs/docs/04-benchmarks/runs/s20plus/v0.11.0-missing-20261026-android-SM-G986B.jsonl @@ -0,0 +1,38 @@ +{"type":"run","label":"v0.11.0-missing-20261026","platform":"android","device":{"model":"SM-G986B","osVersion":"13","soc":"Exynos 990","isEmulator":false,"cpuCores":8,"totalMemoryBytes":11430510592},"inputSpecVersion":1,"buildType":"release","clocksPinned":false,"settings":{"suite":"depth-estimation/depth-anything-v2-small-xnnpack-int8,depth-estimation/depth-anything-v2-small-xnnpack-fp32,depth-estimation/depth-anything-v2-small-vulkan-fp16,depth-estimation/depth-anything-v3-small-xnnpack-int8,depth-estimation/depth-anything-v3-small-xnnpack-fp32,depth-estimation/depth-anything-v3-small-vulkan-fp16,depth-estimation/zipdepth-base-xnnpack-fp32,depth-estimation/zipdepth-base-vulkan-fp16,llm/lfm2-5-1-2-b-xnnpack-8-da4-w,llm/lfm2-5-2-6-b-xnnpack-8-da4-w,llm/lfm2-5-350-m-xnnpack-8-da4-w,llm/lfm2-5-vl-450-m-xnnpack-8-da4-w,llm/lfm2-5-vl-450-m-vulkan-8-da4-w,llm/lfm2-5-vl-1-6-b-xnnpack-8-da4-w,llm/lfm2-5-vl-1-6-b-vulkan-8-da4-w,llm/bielik-v3-1-5-b-xnnpack-8-da4-w,llm/llama3-2-1-b-xnnpack-spinquant,llm/llama3-2-3-b-xnnpack-spinquant,llm/smollm2-135-m-xnnpack-8-da8-w,llm/smollm2-360-m-xnnpack-8-da8-w,llm/smollm2-1-7-b-xnnpack-8-da8-w,llm/hammer2-1-0-5-b-xnnpack-8-da4-w,llm/hammer2-1-1-5-b-xnnpack-8-da4-w,llm/hammer2-1-3-b-xnnpack-8-da4-w,llm/phi4-mini-xnnpack-8-da4-w,llm/qwen2-5-0-5-b-xnnpack-8-da4-w,llm/qwen2-5-1-5-b-xnnpack-8-da4-w,llm/qwen2-5-3-b-xnnpack-8-da4-w,llm/qwen3-0-6-b-xnnpack-8-da4-w,llm/qwen3-1-7-b-xnnpack-8-da4-w,llm/qwen3-4-b-xnnpack-8-da4-w,llm/gemma4-e2-b-xnnpack-8-da4-w,llm/gemma4-e2-b-vulkan-8-da4-w,llm/gemma4-omni-e2-b-xnnpack-8-da4-w,llm/gemma4-omni-e2-b-vulkan-8-da4-w,colbert-embeddings/lfm2-5-colbert-350-m-xnnpack-8-da4-w","iterations":20,"warmup":3,"memoryIterations":5,"loadIterations":3,"repeats":1,"maxTempC":37,"maxBytes":6000000000,"urlMap":{}},"startedAt":"2026-10-26T15:48:45.854Z"} +{"id":"depth-estimation/zipdepth-base-vulkan-fp16","task":"depthEstimation","model":"depthEstimation.ZIPDEPTH_BASE.VULKAN_FP16","backend":"vulkan","precision":"fp16","bytes":12794626,"mode":"worklet","progress":{"caseIndex":1,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":1824.123,"taskLoadMs":62.567,"taskLoad":{"count":3,"min":49.597,"max":1016.76,"mean":376.308,"median":62.567,"p90":1016.76,"iqr":967.164,"stdev":452.899},"pipeline":{"count":20,"min":59.307,"max":90.149,"mean":68.747,"median":66.526,"p90":75.808,"iqr":4.728,"stdev":7.913},"execution":{"perIteration":{"forward":{"count":1,"ms":59.3320924}},"totalMs":59.3320924},"units":262144,"gate":{"kind":"host","waitedS":0,"temperatureC":29.1,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":29.2},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":29.7,"samples":17},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":158.56,"loadedMb":180.13,"peakMb":198.37,"disposedMb":197.63}} +{"id":"depth-estimation/zipdepth-base-xnnpack-fp32","task":"depthEstimation","model":"depthEstimation.ZIPDEPTH_BASE.XNNPACK_FP32","backend":"xnnpack","precision":"fp32","bytes":24609344,"mode":"worklet","progress":{"caseIndex":2,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":1112.012,"taskLoadMs":45.532,"taskLoad":{"count":3,"min":44.354,"max":75.512,"mean":55.133,"median":45.532,"p90":75.512,"iqr":31.158,"stdev":14.418},"pipeline":{"count":20,"min":101.268,"max":231.168,"mean":161.693,"median":146.824,"p90":211.052,"iqr":62.514,"stdev":38.582},"execution":{"perIteration":{"forward":{"count":1,"ms":156.38245755}},"totalMs":156.38245755},"units":262144,"gate":{"kind":"host","waitedS":0,"temperatureC":29.2,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":29.2},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":29.2,"samples":37},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":176.05,"loadedMb":207.99,"peakMb":271.23,"disposedMb":270.72}} +{"id":"depth-estimation/depth-anything-v2-small-xnnpack-int8","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V2_SMALL.XNNPACK_INT8","backend":"xnnpack","precision":"int8","bytes":35481732,"mode":"worklet","progress":{"caseIndex":3,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":1448.651,"taskLoadMs":72.619,"taskLoad":{"count":3,"min":67.379,"max":94.518,"mean":78.172,"median":72.619,"p90":94.518,"iqr":27.14,"stdev":11.755},"pipeline":{"count":20,"min":863.097,"max":1120.91,"mean":938.874,"median":926.984,"p90":972.687,"iqr":42.165,"stdev":54.834},"execution":{"perIteration":{"forward":{"count":1,"ms":930.2002185499999}},"totalMs":930.2002185499999},"units":262144,"gate":{"kind":"host","waitedS":0,"temperatureC":29.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":30.4},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":30.4,"samples":200},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":272.29,"loadedMb":246.18,"peakMb":358.52,"disposedMb":288.18}} +{"id":"depth-estimation/depth-anything-v3-small-xnnpack-int8","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V3_SMALL.XNNPACK_INT8","backend":"xnnpack","precision":"int8","bytes":40603272,"mode":"worklet","progress":{"caseIndex":4,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":2157.175,"taskLoadMs":107.868,"taskLoad":{"count":3,"min":103.814,"max":113.19,"mean":108.291,"median":107.868,"p90":113.19,"iqr":9.376,"stdev":3.839},"pipeline":{"count":20,"min":1047.256,"max":1109.27,"mean":1083.939,"median":1083.152,"p90":1107.261,"iqr":34.69,"stdev":19.687},"execution":{"perIteration":{"forward":{"count":1,"ms":1074.55949925}},"totalMs":1074.55949925},"units":262144,"gate":{"kind":"host","waitedS":0,"temperatureC":31.1,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":31.7},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":31.7,"samples":232},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":290.88,"loadedMb":315.06,"peakMb":478.8,"disposedMb":274.81}} +{"id":"depth-estimation/depth-anything-v2-small-vulkan-fp16","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V2_SMALL.VULKAN_FP16","backend":"vulkan","precision":"fp16","bytes":49595524,"mode":"worklet","progress":{"caseIndex":5,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":2112.472,"taskLoadMs":82.377,"taskLoad":{"count":3,"min":81.077,"max":664.959,"mean":276.138,"median":82.377,"p90":664.959,"iqr":583.881,"stdev":274.939},"pipeline":{"count":20,"min":878.45,"max":903.275,"mean":887.158,"median":886.566,"p90":893.05,"iqr":8.718,"stdev":6.267},"execution":{"perIteration":{"forward":{"count":1,"ms":870.8217455500001}},"totalMs":870.8217455500001},"units":262144,"gate":{"kind":"host","waitedS":0,"temperatureC":32.5,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":31.7},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":31.7,"samples":191},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":276.11,"loadedMb":204.96,"peakMb":214.24,"disposedMb":207.23}} +{"id":"depth-estimation/depth-anything-v3-small-vulkan-fp16","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V3_SMALL.VULKAN_FP16","backend":"vulkan","precision":"fp16","bytes":56407810,"mode":"worklet","progress":{"caseIndex":6,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":2010.666,"taskLoadMs":84.16,"taskLoad":{"count":3,"min":82.51,"max":623.658,"mean":263.443,"median":84.16,"p90":623.658,"iqr":541.148,"stdev":254.712},"pipeline":{"count":20,"min":1121.417,"max":1143.626,"mean":1131.265,"median":1131.899,"p90":1136.803,"iqr":5.271,"stdev":5.679},"execution":{"perIteration":{"forward":{"count":1,"ms":1113.9749029500001}},"totalMs":1113.9749029500001},"units":262144,"gate":{"kind":"host","waitedS":0,"temperatureC":32.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":32.9},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":32.9,"samples":243},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":208.75,"loadedMb":207.21,"peakMb":218.53,"disposedMb":217.63}} +{"id":"depth-estimation/depth-anything-v2-small-xnnpack-fp32","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V2_SMALL.XNNPACK_FP32","backend":"xnnpack","precision":"fp32","bytes":98979332,"mode":"worklet","progress":{"caseIndex":7,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":3106.868,"taskLoadMs":97.795,"taskLoad":{"count":3,"min":72.313,"max":122.802,"mean":97.637,"median":97.795,"p90":122.802,"iqr":50.489,"stdev":20.612},"pipeline":{"count":20,"min":1202.689,"max":1394.095,"mean":1322.946,"median":1332.498,"p90":1369.383,"iqr":44.727,"stdev":43.812},"execution":{"perIteration":{"forward":{"count":1,"ms":1313.43115105}},"totalMs":1313.43115105},"units":262144,"gate":{"kind":"host","waitedS":30,"temperatureC":33.4,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":33.4},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":33.4,"samples":279},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":213.05,"loadedMb":310.19,"peakMb":438.38,"disposedMb":362.89}} +{"id":"depth-estimation/depth-anything-v3-small-xnnpack-fp32","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V3_SMALL.XNNPACK_FP32","backend":"xnnpack","precision":"fp32","bytes":106480776,"mode":"worklet","progress":{"caseIndex":8,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":3802.077,"taskLoadMs":112.417,"taskLoad":{"count":3,"min":108.345,"max":118.054,"mean":112.939,"median":112.417,"p90":118.054,"iqr":9.709,"stdev":3.981},"pipeline":{"count":20,"min":1409.343,"max":1531.443,"mean":1467.808,"median":1472.759,"p90":1513.718,"iqr":52.484,"stdev":33.606},"execution":{"perIteration":{"forward":{"count":1,"ms":1458.1320316}},"totalMs":1458.1320316},"units":262144,"gate":{"kind":"host","waitedS":30,"temperatureC":34,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":34.5},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":34.5,"samples":312},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":371.29,"loadedMb":406.27,"peakMb":572.12,"disposedMb":349.41}} +{"id":"colbert-embeddings/lfm2-5-colbert-350-m-xnnpack-8-da4-w","task":"colbertEmbeddings","model":"colbertEmbeddings.LFM2_5_COLBERT_350M.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":377361478,"mode":"worklet","progress":{"caseIndex":11,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":9334.399,"taskLoadMs":1506.111,"taskLoad":{"count":3,"min":1476.83,"max":1598.885,"mean":1527.275,"median":1506.111,"p90":1598.885,"iqr":122.055,"stdev":52.027},"pipeline":{"count":20,"min":207.961,"max":479.761,"mean":260.773,"median":214.96,"p90":443.577,"iqr":14.834,"stdev":93.884},"execution":{"perIteration":{"forward":{"count":1,"ms":259.44660749999997}},"totalMs":259.44660749999997},"units":56,"gate":{"kind":"host","waitedS":46,"temperatureC":35.6,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":34.8},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":35.5,"samples":56},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":333.08,"loadedMb":658.02,"peakMb":673.97,"disposedMb":572.06}} +{"type":"run","label":"v0.11.0-missing-20261026","platform":"android","device":{"model":"SM-G986B","osVersion":"13","soc":"Exynos 990","isEmulator":false,"cpuCores":8,"totalMemoryBytes":11430510592},"inputSpecVersion":1,"buildType":"release","clocksPinned":false,"settings":{"suite":"depth-estimation/depth-anything-v2-small-xnnpack-int8,depth-estimation/depth-anything-v2-small-xnnpack-fp32,depth-estimation/depth-anything-v2-small-vulkan-fp16,depth-estimation/depth-anything-v3-small-xnnpack-int8,depth-estimation/depth-anything-v3-small-xnnpack-fp32,depth-estimation/depth-anything-v3-small-vulkan-fp16,depth-estimation/zipdepth-base-xnnpack-fp32,depth-estimation/zipdepth-base-vulkan-fp16,llm/lfm2-5-1-2-b-xnnpack-8-da4-w,llm/lfm2-5-2-6-b-xnnpack-8-da4-w,llm/lfm2-5-350-m-xnnpack-8-da4-w,llm/lfm2-5-vl-450-m-xnnpack-8-da4-w,llm/lfm2-5-vl-450-m-vulkan-8-da4-w,llm/lfm2-5-vl-1-6-b-xnnpack-8-da4-w,llm/lfm2-5-vl-1-6-b-vulkan-8-da4-w,llm/bielik-v3-1-5-b-xnnpack-8-da4-w,llm/llama3-2-1-b-xnnpack-spinquant,llm/llama3-2-3-b-xnnpack-spinquant,llm/smollm2-135-m-xnnpack-8-da8-w,llm/smollm2-360-m-xnnpack-8-da8-w,llm/smollm2-1-7-b-xnnpack-8-da8-w,llm/hammer2-1-0-5-b-xnnpack-8-da4-w,llm/hammer2-1-1-5-b-xnnpack-8-da4-w,llm/hammer2-1-3-b-xnnpack-8-da4-w,llm/phi4-mini-xnnpack-8-da4-w,llm/qwen2-5-0-5-b-xnnpack-8-da4-w,llm/qwen2-5-1-5-b-xnnpack-8-da4-w,llm/qwen2-5-3-b-xnnpack-8-da4-w,llm/qwen3-0-6-b-xnnpack-8-da4-w,llm/qwen3-1-7-b-xnnpack-8-da4-w,llm/qwen3-4-b-xnnpack-8-da4-w,llm/gemma4-e2-b-xnnpack-8-da4-w,llm/gemma4-e2-b-vulkan-8-da4-w,llm/gemma4-omni-e2-b-xnnpack-8-da4-w,llm/gemma4-omni-e2-b-vulkan-8-da4-w,colbert-embeddings/lfm2-5-colbert-350-m-xnnpack-8-da4-w","iterations":20,"warmup":3,"memoryIterations":5,"loadIterations":3,"repeats":1,"maxTempC":37,"maxBytes":6000000000,"urlMap":{}},"startedAt":"2026-10-26T16:21:48.041Z"} +{"id":"llm/smollm2-135-m-xnnpack-8-da8-w","task":"llm","model":"llm.SMOLLM2_135M.XNNPACK_8DA8W","backend":"xnnpack","precision":"8da8w","bytes":175114287,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":9,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":6580.048,"taskLoadMs":749.776,"taskLoad":{"count":3,"min":745.523,"max":769.348,"mean":754.882,"median":749.776,"p90":769.348,"iqr":23.825,"stdev":10.375},"pipeline":{"count":20,"min":631.394,"max":680.221,"mean":647.449,"median":645.749,"p90":663.329,"iqr":16.824,"stdev":12.038},"execution":{"perIteration":{"forward":{"count":65,"ms":616.03780785}},"totalMs":616.03780785},"units":64,"detail":{"timeToFirstTokenMs":53.036,"prefillMs":53,"promptTokens":38},"gate":{"kind":"host","waitedS":0,"temperatureC":31.4,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":32.7},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":32.7,"samples":186},"thermalValid":true,"holdsTimedOut":0,"coolingMs":5042,"memory":{"baselineMb":142.94,"loadedMb":1081.77,"peakMb":1104.01,"disposedMb":242.33}} +{"id":"llm/lfm2-5-350-m-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_350M.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":296972650,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":10,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":7123.444,"taskLoadMs":973.239,"taskLoad":{"count":3,"min":971.453,"max":1017.394,"mean":987.362,"median":973.239,"p90":1017.394,"iqr":45.941,"stdev":21.249},"pipeline":{"count":20,"min":736.247,"max":759.521,"mean":747.549,"median":747.28,"p90":756.832,"iqr":9.25,"stdev":6.671},"execution":{"perIteration":{"forward":{"count":65,"ms":711.16892535}},"totalMs":711.16892535},"units":64,"detail":{"timeToFirstTokenMs":51.709,"prefillMs":52,"promptTokens":17},"gate":{"kind":"host","waitedS":0,"temperatureC":33.5,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":34.1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":34.1,"samples":210},"thermalValid":true,"holdsTimedOut":0,"coolingMs":5247,"memory":{"baselineMb":248.84,"loadedMb":930.46,"peakMb":992.86,"disposedMb":204.55}} +{"id":"llm/smollm2-360-m-xnnpack-8-da8-w","task":"llm","model":"llm.SMOLLM2_360M.XNNPACK_8DA8W","backend":"xnnpack","precision":"8da8w","bytes":421904303,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":12,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":407.097,"taskLoadMs":1535.842,"taskLoad":{"count":3,"min":1519.462,"max":1646.091,"mean":1567.132,"median":1535.842,"p90":1646.091,"iqr":126.629,"stdev":56.232},"pipeline":{"count":20,"min":1327.226,"max":1445.262,"mean":1370.188,"median":1367.592,"p90":1397.994,"iqr":17.725,"stdev":24.812},"execution":{"perIteration":{"forward":{"count":65,"ms":1337.44276375}},"totalMs":1337.44276375},"units":64,"detail":{"timeToFirstTokenMs":107.917,"prefillMs":108,"promptTokens":38},"gate":{"kind":"host","waitedS":0,"temperatureC":34.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":35.3},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":35.3,"samples":619},"thermalValid":true,"holdsTimedOut":0,"coolingMs":35117,"memory":{"baselineMb":204.48,"loadedMb":1917.95,"peakMb":1947.62,"disposedMb":372.6}} +{"id":"llm/qwen2-5-0-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN2_5_0_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":439214950,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":13,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":12786.938,"taskLoadMs":2373.361,"taskLoad":{"count":3,"min":2344.239,"max":2408.296,"mean":2375.299,"median":2373.361,"p90":2408.296,"iqr":64.057,"stdev":26.187},"pipeline":{"count":20,"min":1190.239,"max":1345.918,"mean":1241.425,"median":1224.247,"p90":1288.552,"iqr":47.594,"stdev":39.168},"execution":{"perIteration":{"forward":{"count":65,"ms":1168.5948839500002}},"totalMs":1168.5948839500002},"units":64,"detail":{"timeToFirstTokenMs":159.595,"prefillMs":159,"promptTokens":37},"gate":{"kind":"host","waitedS":31,"temperatureC":34.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":35.3},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":35.3,"samples":312},"thermalValid":true,"holdsTimedOut":0,"coolingMs":5082,"memory":{"baselineMb":383.99,"loadedMb":1117.06,"peakMb":1159.31,"disposedMb":253.99}} +{"id":"llm/hammer2-1-0-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.HAMMER2_1_0_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":443224596,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":14,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":10737.033,"taskLoadMs":3595.79,"taskLoad":{"count":3,"min":3534.5,"max":3620.04,"mean":3583.443,"median":3595.79,"p90":3620.04,"iqr":85.54,"stdev":35.996},"pipeline":{"count":20,"min":1236.87,"max":1308.302,"mean":1266.13,"median":1260.943,"p90":1291.113,"iqr":29.549,"stdev":21.303},"execution":{"perIteration":{"forward":{"count":65,"ms":1187.1069202}},"totalMs":1187.1069202},"units":64,"detail":{"timeToFirstTokenMs":123.758,"prefillMs":123,"promptTokens":27},"gate":{"kind":"host","waitedS":46,"temperatureC":34.8,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":35.2},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":35.2,"samples":318},"thermalValid":true,"holdsTimedOut":0,"coolingMs":5081,"memory":{"baselineMb":262.57,"loadedMb":1104.89,"peakMb":1150.37,"disposedMb":245.97}} +{"id":"llm/qwen3-0-6-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN3_0_6B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":529702696,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":15,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":12211.666,"taskLoadMs":4864.639,"taskLoad":{"count":3,"min":4811.507,"max":5028.548,"mean":4901.564,"median":4864.639,"p90":5028.548,"iqr":217.041,"stdev":92.374},"pipeline":{"count":20,"min":1557.892,"max":1612.486,"mean":1581.169,"median":1580.73,"p90":1596.103,"iqr":17.179,"stdev":13.496},"execution":{"perIteration":{"forward":{"count":65,"ms":1500.01963175}},"totalMs":1500.01963175},"units":64,"detail":{"timeToFirstTokenMs":104.184,"prefillMs":103,"promptTokens":16},"gate":{"kind":"host","waitedS":30,"temperatureC":35.6,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":36.3},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":36.3,"samples":386},"thermalValid":true,"holdsTimedOut":0,"coolingMs":4931,"memory":{"baselineMb":252.07,"loadedMb":2565.34,"peakMb":2637.64,"disposedMb":334.01}} +{"id":"llm/lfm2-5-vl-450-m-vulkan-8-da4-w","task":"llm","model":"llm.LFM2_5_VL_450M.VULKAN_8DA4W","backend":"vulkan","precision":"8da4w","bytes":627140107,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":16,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":14459.845,"taskLoadMs":2004.668,"taskLoad":{"count":3,"min":1877.531,"max":3431.383,"mean":2437.861,"median":2004.668,"p90":3431.383,"iqr":1553.852,"stdev":704.441},"pipeline":{"count":20,"min":3686.125,"max":3802.578,"mean":3734.642,"median":3735.846,"p90":3769.275,"iqr":46.289,"stdev":29.741},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":62.95086775},"text_decoder":{"count":65,"ms":3598.3410892499996}},"totalMs":3661.291957},"units":64,"detail":{"timeToFirstTokenMs":278.825,"prefillMs":278,"promptTokens":17},"gate":{"kind":"host","waitedS":30,"temperatureC":35.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":35.2},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":36.3,"samples":856},"thermalValid":true,"holdsTimedOut":0,"coolingMs":5129,"memory":{"baselineMb":331.7,"loadedMb":646.18,"peakMb":652.31,"disposedMb":619.22}} +{"id":"llm/lfm2-5-vl-450-m-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_VL_450M.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":653603851,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":17,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":16372.09,"taskLoadMs":2493.083,"taskLoad":{"count":3,"min":2336.001,"max":2533.043,"mean":2454.042,"median":2493.083,"p90":2533.043,"iqr":197.043,"stdev":85.047},"pipeline":{"count":20,"min":1638.155,"max":1758.782,"mean":1687.418,"median":1691.663,"p90":1725.293,"iqr":48.991,"stdev":32.85},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":1.2413871},"text_decoder":{"count":65,"ms":1631.25037785}},"totalMs":1632.49176495},"units":64,"detail":{"timeToFirstTokenMs":79.672,"prefillMs":79,"promptTokens":17},"gate":{"kind":"host","waitedS":0,"temperatureC":35.2,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":36.5},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":36.5,"samples":409},"thermalValid":true,"holdsTimedOut":0,"coolingMs":4975,"memory":{"baselineMb":614.54,"loadedMb":1192.56,"peakMb":1247.05,"disposedMb":607.82}} +{"id":"llm/lfm2-5-1-2-b-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_1_2B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":815187806,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":18,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":17993.58,"taskLoadMs":4404.559,"taskLoad":{"count":3,"min":4215.166,"max":4521.004,"mean":4380.243,"median":4404.559,"p90":4521.004,"iqr":305.838,"stdev":126.036},"pipeline":{"count":20,"min":2222.971,"max":2305.394,"mean":2246.537,"median":2237.587,"p90":2270.542,"iqr":31.546,"stdev":21.393},"execution":{"perIteration":{"forward":{"count":65,"ms":2198.5553552}},"totalMs":2198.5553552},"units":64,"detail":{"timeToFirstTokenMs":186.689,"prefillMs":186,"promptTokens":17},"gate":{"kind":"host","waitedS":45,"temperatureC":35.5,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":36.4},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":36.6,"samples":946},"thermalValid":true,"holdsTimedOut":0,"coolingMs":50208,"memory":{"baselineMb":609.55,"loadedMb":1499.71,"peakMb":1606.79,"disposedMb":259.98}} +{"id":"llm/bielik-v3-1-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.BIELIK_V3_1_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":939299957,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":19,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":22120.465,"taskLoadMs":5135.741,"taskLoad":{"count":3,"min":5104.167,"max":5223.302,"mean":5154.403,"median":5135.741,"p90":5223.302,"iqr":119.134,"stdev":50.395},"pipeline":{"count":20,"min":3198.058,"max":3366.728,"mean":3242.84,"median":3214.331,"p90":3347.752,"iqr":69.801,"stdev":54.823},"execution":{"perIteration":{"forward":{"count":65,"ms":3205.7403182}},"totalMs":3205.7403182},"units":64,"detail":{"timeToFirstTokenMs":387.851,"prefillMs":387,"promptTokens":26},"gate":{"kind":"host","waitedS":46,"temperatureC":35.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":36.4},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":36.4,"samples":1725},"thermalValid":true,"holdsTimedOut":0,"coolingMs":111148,"memory":{"baselineMb":264.18,"loadedMb":1894.17,"peakMb":1971.87,"disposedMb":522.24}} +{"id":"llm/llama3-2-1-b-xnnpack-spinquant","task":"llm","model":"llm.LLAMA3_2_1B.XNNPACK_SPINQUANT","backend":"xnnpack","precision":"spinquant","bytes":1092444893,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":20,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":25508.274,"taskLoadMs":6865.117,"taskLoad":{"count":3,"min":6217.891,"max":7039.058,"mean":6707.355,"median":6865.117,"p90":7039.058,"iqr":821.167,"stdev":353.313},"pipeline":{"count":20,"min":2990.707,"max":3031.356,"mean":3010.251,"median":3009.68,"p90":3024.439,"iqr":17.838,"stdev":11.584},"execution":{"perIteration":{"forward":{"count":65,"ms":2940.0728628}},"totalMs":2940.0728628},"units":64,"detail":{"timeToFirstTokenMs":435.419,"prefillMs":435,"promptTokens":44},"gate":{"kind":"host","waitedS":46,"temperatureC":35.8,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":36.4},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":36.4,"samples":1537},"thermalValid":true,"holdsTimedOut":0,"coolingMs":95504,"memory":{"baselineMb":521.81,"loadedMb":2348.46,"peakMb":2424.66,"disposedMb":301.86}} +{"id":"llm/qwen2-5-1-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN2_5_1_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":1172577510,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":21,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":26035.925,"taskLoadMs":6699.883,"taskLoad":{"count":3,"min":5171.827,"max":6905.367,"mean":6259.026,"median":6699.883,"p90":6905.367,"iqr":1733.54,"stdev":773.329},"pipeline":{"count":20,"min":3298.189,"max":3506.743,"mean":3349.975,"median":3331.432,"p90":3417.854,"iqr":47.71,"stdev":49.812},"execution":{"perIteration":{"forward":{"count":65,"ms":3275.12680695}},"totalMs":3275.12680695},"units":64,"detail":{"timeToFirstTokenMs":466.561,"prefillMs":466,"promptTokens":37},"gate":{"kind":"host","waitedS":46,"temperatureC":35.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":37.5},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":37.5,"samples":1746},"thermalValid":true,"holdsTimedOut":0,"coolingMs":110877,"memory":{"baselineMb":306.01,"loadedMb":2348.74,"peakMb":2401.55,"disposedMb":452.46}} +{"id":"llm/hammer2-1-1-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.HAMMER2_1_1_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":1176265276,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":22,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":25964.267,"taskLoadMs":7026.277,"taskLoad":{"count":3,"min":6954.38,"max":7031.012,"mean":7003.89,"median":7026.277,"p90":7031.012,"iqr":76.632,"stdev":35.062},"pipeline":{"count":20,"min":3162.036,"max":3285.319,"mean":3200.338,"median":3187.456,"p90":3270.408,"iqr":25.144,"stdev":37.912},"execution":{"perIteration":{"forward":{"count":65,"ms":3122.59448365}},"totalMs":3122.59448365},"units":64,"detail":{"timeToFirstTokenMs":344.382,"prefillMs":343,"promptTokens":27},"gate":{"kind":"host","waitedS":61,"temperatureC":35.6,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":36.4},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":36.4,"samples":1715},"thermalValid":true,"holdsTimedOut":0,"coolingMs":110452,"memory":{"baselineMb":454.56,"loadedMb":2338.93,"peakMb":2398.34,"disposedMb":449.83}} +{"id":"llm/qwen3-1-7-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN3_1_7B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":1319364648,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":23,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":32493.9,"taskLoadMs":7410.071,"taskLoad":{"count":3,"min":7215.488,"max":7441.577,"mean":7355.712,"median":7410.071,"p90":7441.577,"iqr":226.089,"stdev":99.984},"pipeline":{"count":20,"min":3380.875,"max":3475.045,"mean":3412.993,"median":3406.906,"p90":3462.431,"iqr":16.594,"stdev":24.681},"execution":{"perIteration":{"forward":{"count":65,"ms":3331.1150519000003}},"totalMs":3331.1150519000003},"units":64,"detail":{"timeToFirstTokenMs":235.771,"prefillMs":234,"promptTokens":16},"gate":{"kind":"host","waitedS":46,"temperatureC":35.8,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":37.5},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":36.4,"samples":1759},"thermalValid":true,"holdsTimedOut":0,"coolingMs":110935,"memory":{"baselineMb":446.27,"loadedMb":2371.49,"peakMb":2465.26,"disposedMb":638.83}} +{"id":"llm/lfm2-5-2-6-b-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_2_6B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":1807639919,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":24,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":38560.624,"taskLoadMs":10316.994,"taskLoad":{"count":3,"min":9988.904,"max":10410.795,"mean":10238.898,"median":10316.994,"p90":10410.795,"iqr":421.891,"stdev":180.872},"pipeline":{"count":20,"min":4910.611,"max":5053.619,"mean":4952.265,"median":4935.975,"p90":5040.496,"iqr":36.309,"stdev":45.156},"execution":{"perIteration":{"forward":{"count":65,"ms":4880.5507514500005}},"totalMs":4880.5507514500005},"units":64,"detail":{"timeToFirstTokenMs":427.986,"prefillMs":428,"promptTokens":18},"gate":{"kind":"host","waitedS":61,"temperatureC":35.8,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":37.3},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":37.5,"samples":3073},"thermalValid":true,"holdsTimedOut":0,"coolingMs":216606,"memory":{"baselineMb":641.03,"loadedMb":2090.67,"peakMb":2103.1,"disposedMb":950.63}} +{"id":"llm/smollm2-1-7-b-xnnpack-8-da8-w","task":"llm","model":"llm.SMOLLM2_1_7B.XNNPACK_8DA8W","backend":"xnnpack","precision":"8da8w","bytes":1818983855,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":25,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":40378.446,"taskLoadMs":8495.949,"taskLoad":{"count":3,"min":6582,"max":8776.701,"mean":7951.55,"median":8495.949,"p90":8776.701,"iqr":2194.7,"stdev":975.177},"pipeline":{"count":20,"min":5443.373,"max":5530.055,"mean":5478.191,"median":5481.171,"p90":5497.293,"iqr":29.889,"stdev":21.706},"execution":{"perIteration":{"forward":{"count":65,"ms":5433.31072485}},"totalMs":5433.31072485},"units":64,"detail":{"timeToFirstTokenMs":379.657,"prefillMs":379,"promptTokens":38},"gate":{"kind":"host","waitedS":61,"temperatureC":36,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":36.5},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":37.6,"samples":3066},"thermalValid":true,"holdsTimedOut":0,"coolingMs":201666,"memory":{"baselineMb":952.33,"loadedMb":2910.22,"peakMb":2990.65,"disposedMb":788.8}} +{"id":"llm/hammer2-1-3-b-xnnpack-8-da4-w","task":"llm","model":"llm.HAMMER2_1_3B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2076552284,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":26,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":45438.283,"taskLoadMs":11142.924,"taskLoad":{"count":3,"min":11000.171,"max":11189.613,"mean":11110.903,"median":11142.924,"p90":11189.613,"iqr":189.442,"stdev":80.586},"pipeline":{"count":20,"min":5903.637,"max":6323.261,"mean":5977.026,"median":5937.639,"p90":6059.568,"iqr":64.849,"stdev":93.891},"execution":{"perIteration":{"forward":{"count":65,"ms":5902.00230135}},"totalMs":5902.00230135},"units":64,"detail":{"timeToFirstTokenMs":700.681,"prefillMs":699,"promptTokens":27},"gate":{"kind":"host","waitedS":61,"temperatureC":35.8,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":37.3},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":37.6,"samples":4026},"thermalValid":true,"holdsTimedOut":0,"coolingMs":294002,"memory":{"baselineMb":794.53,"loadedMb":2940.97,"peakMb":3043.46,"disposedMb":399.1}} +{"id":"llm/qwen2-5-3-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN2_5_3B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2111318246,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":27,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":45059.349,"taskLoadMs":10801.202,"taskLoad":{"count":3,"min":10751.087,"max":10896.134,"mean":10816.141,"median":10801.202,"p90":10896.134,"iqr":145.047,"stdev":60.15},"pipeline":{"count":20,"min":6184.928,"max":8363.701,"mean":6377.183,"median":6247.277,"p90":6362.362,"iqr":61.029,"stdev":464.573},"execution":{"perIteration":{"forward":{"count":65,"ms":6299.77498645}},"totalMs":6299.77498645},"units":64,"detail":{"timeToFirstTokenMs":950.751,"prefillMs":950,"promptTokens":37},"gate":{"kind":"host","waitedS":45,"temperatureC":35.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":37.6},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":37.6,"samples":4079},"thermalValid":true,"holdsTimedOut":0,"coolingMs":292625,"memory":{"baselineMb":396.27,"loadedMb":2943.13,"peakMb":3042.62,"disposedMb":403.72}} +{"id":"llm/llama3-2-3-b-xnnpack-spinquant","task":"llm","model":"llm.LLAMA3_2_3B.XNNPACK_SPINQUANT","backend":"xnnpack","precision":"spinquant","bytes":2389281117,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":28,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":52550.929,"taskLoadMs":12027.412,"taskLoad":{"count":3,"min":9519.144,"max":12228.123,"mean":11258.226,"median":12027.412,"p90":12228.123,"iqr":2708.978,"stdev":1232.444},"pipeline":{"count":20,"min":7132.986,"max":7350.143,"mean":7191.567,"median":7174.198,"p90":7286.645,"iqr":27.377,"stdev":59.757},"execution":{"perIteration":{"forward":{"count":65,"ms":7117.11881115}},"totalMs":7117.11881115},"units":64,"detail":{"timeToFirstTokenMs":1117.616,"prefillMs":1117,"promptTokens":44},"gate":{"kind":"host","waitedS":46,"temperatureC":36.1,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":36.3},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":37.5,"samples":4682},"thermalValid":true,"holdsTimedOut":0,"coolingMs":338083,"memory":{"baselineMb":399.66,"loadedMb":3372.8,"peakMb":3461.32,"disposedMb":1003.86}} +{"id":"llm/lfm2-5-vl-1-6-b-vulkan-8-da4-w","task":"llm","model":"llm.LFM2_5_VL_1_6B.VULKAN_8DA4W","backend":"vulkan","precision":"8da4w","bytes":2470992395,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":29,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":72217.052,"taskLoadMs":5949.561,"taskLoad":{"count":3,"min":5580.008,"max":6128.349,"mean":5885.973,"median":5949.561,"p90":6128.349,"iqr":548.341,"stdev":228.33},"pipeline":{"count":20,"min":7538.438,"max":7809.433,"mean":7702.3,"median":7708.82,"p90":7782.662,"iqr":76.999,"stdev":68.192},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":123.8011356},"text_decoder":{"count":65,"ms":7467.88616965}},"totalMs":7591.68730525},"units":64,"detail":{"timeToFirstTokenMs":714.558,"prefillMs":714,"promptTokens":17},"gate":{"kind":"host","waitedS":0,"temperatureC":36,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":36.3},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":36.3,"samples":1695},"thermalValid":true,"holdsTimedOut":0,"coolingMs":5371,"memory":{"baselineMb":1006.55,"loadedMb":1993.97,"peakMb":2000.35,"disposedMb":876.83}} +{"id":"llm/lfm2-5-vl-1-6-b-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_VL_1_6B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2479599371,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":30,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":73679.006,"taskLoadMs":7149.988,"taskLoad":{"count":3,"min":6532.406,"max":7286.572,"mean":6989.655,"median":7149.988,"p90":7286.572,"iqr":754.167,"stdev":328.097},"pipeline":{"count":20,"min":2920.928,"max":3028.699,"mean":2970.592,"median":2967.066,"p90":3022.445,"iqr":71.47,"stdev":36.876},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":3.41785955},"text_decoder":{"count":65,"ms":2911.10571275}},"totalMs":2914.5235723},"units":64,"detail":{"timeToFirstTokenMs":187.367,"prefillMs":186,"promptTokens":17},"gate":{"kind":"host","waitedS":0,"temperatureC":35.8,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":36.3},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":37.5,"samples":1685},"thermalValid":true,"holdsTimedOut":0,"coolingMs":110887,"memory":{"baselineMb":863.87,"loadedMb":3136.75,"peakMb":3210.46,"disposedMb":664.69}} +{"id":"llm/qwen3-4-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN3_4B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2697273128,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":31,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":59198.142,"taskLoadMs":14155.11,"taskLoad":{"count":3,"min":13892.8,"max":14443.919,"mean":14163.943,"median":14155.11,"p90":14443.919,"iqr":551.119,"stdev":225.08},"pipeline":{"count":20,"min":7384.173,"max":7567.759,"mean":7455.251,"median":7443.8,"p90":7543.295,"iqr":51.793,"stdev":49.039},"execution":{"perIteration":{"forward":{"count":65,"ms":7375.73373365}},"totalMs":7375.73373365},"units":64,"detail":{"timeToFirstTokenMs":529.109,"prefillMs":528,"promptTokens":16},"gate":{"kind":"host","waitedS":46,"temperatureC":35.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":36.1},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":37.5,"samples":4326},"thermalValid":true,"holdsTimedOut":0,"coolingMs":292504,"memory":{"baselineMb":656.6,"loadedMb":4012.35,"peakMb":4122.11,"disposedMb":448.48}} +{"id":"llm/phi4-mini-xnnpack-8-da4-w","task":"llm","model":"llm.PHI4_MINI.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2833963001,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":32,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":58612.059,"taskLoadMs":14275.288,"taskLoad":{"count":3,"min":14065.409,"max":14287.62,"mean":14209.439,"median":14275.288,"p90":14287.62,"iqr":222.211,"stdev":101.969},"pipeline":{"count":20,"min":6793.108,"max":7756.081,"mean":6867.26,"median":6812.341,"p90":6860.038,"iqr":31.004,"stdev":205.105},"execution":{"perIteration":{"forward":{"count":65,"ms":6771.33118175}},"totalMs":6771.33118175},"units":64,"detail":{"timeToFirstTokenMs":376.304,"prefillMs":375,"promptTokens":11},"gate":{"kind":"host","waitedS":0,"temperatureC":36.1,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":36.5},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":37.6,"samples":4756},"thermalValid":true,"holdsTimedOut":0,"coolingMs":353087,"memory":{"baselineMb":434.86,"loadedMb":3738.66,"peakMb":3779.83,"disposedMb":273.96}} +{"id":"llm/gemma4-e2-b-vulkan-8-da4-w","task":"llm","model":"llm.GEMMA4_E2B.VULKAN_8DA4W","backend":"vulkan","precision":"8da4w","bytes":2988556216,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":33,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":78777.714,"taskLoadMs":14579.614,"taskLoad":{"count":3,"min":13480.938,"max":15144.549,"mean":14401.7,"median":14579.614,"p90":15144.549,"iqr":1663.611,"stdev":690.719},"pipeline":{"count":20,"min":22212.006,"max":22876.734,"mean":22627.661,"median":22629.21,"p90":22781.577,"iqr":181.962,"stdev":152.716},"execution":{"perIteration":{"forward":{"count":65,"ms":22441.8796936}},"totalMs":22441.8796936},"units":64,"detail":{"timeToFirstTokenMs":1479.449,"prefillMs":1477,"promptTokens":17},"gate":{"kind":"host","waitedS":0,"temperatureC":36.2,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":35.4},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":36.5,"samples":4900},"thermalValid":true,"holdsTimedOut":0,"coolingMs":5387,"memory":{"baselineMb":276.46,"loadedMb":411.57,"peakMb":326.55,"disposedMb":322.16}} +{"id":"llm/gemma4-e2-b-xnnpack-8-da4-w","task":"llm","model":"llm.GEMMA4_E2B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":3247469492,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":34,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":71833.312,"taskLoadMs":15037.426,"taskLoad":{"count":3,"min":15024.221,"max":15269.846,"mean":15110.498,"median":15037.426,"p90":15269.846,"iqr":245.625,"stdev":112.805},"pipeline":{"count":20,"min":19177.906,"max":22341.332,"mean":21977.796,"median":22126.716,"p90":22289.684,"iqr":164.116,"stdev":651.117},"execution":{"perIteration":{"forward":{"count":65,"ms":21842.76147045}},"totalMs":21842.76147045},"units":64,"detail":{"timeToFirstTokenMs":1201.794,"prefillMs":1200,"promptTokens":17},"gate":{"kind":"host","waitedS":46,"temperatureC":35.4,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":37.6},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":36.5,"samples":5914},"thermalValid":true,"holdsTimedOut":0,"coolingMs":128991,"memory":{"baselineMb":284.5,"loadedMb":2205.52,"peakMb":2123.75,"disposedMb":918.8}} +{"id":"llm/gemma4-omni-e2-b-xnnpack-8-da4-w","task":"llm","model":"llm.GEMMA4_OMNI_E2B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":3651367476,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":35,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":79459.169,"taskLoadMs":17011.058,"taskLoad":{"count":3,"min":16600.677,"max":17129.232,"mean":16913.656,"median":17011.058,"p90":17129.232,"iqr":528.554,"stdev":226.507},"pipeline":{"count":20,"min":21985.867,"max":22352.865,"mean":22159.633,"median":22170.493,"p90":22279.118,"iqr":114.054,"stdev":94.754},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":23.77158085},"text_decoder":{"count":65,"ms":21985.7592117}},"totalMs":22009.530792550002},"units":64,"detail":{"timeToFirstTokenMs":1261.062,"prefillMs":1259,"promptTokens":17},"gate":{"kind":"host","waitedS":61,"temperatureC":35.4,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":36.1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":36.2,"samples":5097},"thermalValid":true,"holdsTimedOut":0,"coolingMs":37311,"memory":{"baselineMb":911.13,"loadedMb":2695.26,"peakMb":2602.01,"disposedMb":349.84}} +{"id":"llm/gemma4-omni-e2-b-vulkan-8-da4-w","task":"llm","model":"llm.GEMMA4_OMNI_E2B.VULKAN_8DA4W","backend":"vulkan","precision":"8da4w","bytes":3838710580,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":36,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":79476.456,"taskLoadMs":17601.543,"taskLoad":{"count":3,"min":16856.437,"max":20709.021,"mean":18389,"median":17601.543,"p90":20709.021,"iqr":3852.584,"stdev":1668.466},"pipeline":{"count":20,"min":21745.577,"max":22043.663,"mean":21882.109,"median":21888.715,"p90":21986.396,"iqr":117.01,"stdev":82.022},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":619.5385043},"text_decoder":{"count":65,"ms":21038.48950625}},"totalMs":21658.02801055},"units":64,"detail":{"timeToFirstTokenMs":1336,"prefillMs":1335,"promptTokens":17},"gate":{"kind":"host","waitedS":46,"temperatureC":35.6,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":35},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":36.1,"samples":4747},"thermalValid":true,"holdsTimedOut":0,"coolingMs":5628,"memory":{"baselineMb":345.77,"loadedMb":691.71,"peakMb":623.44,"disposedMb":400.08}} diff --git a/docs/docs/04-benchmarks/runs/s24/v0.11.0-missing-20261026-android-SM-S921B.jsonl b/docs/docs/04-benchmarks/runs/s24/v0.11.0-missing-20261026-android-SM-S921B.jsonl new file mode 100644 index 0000000000..503b803e9e --- /dev/null +++ b/docs/docs/04-benchmarks/runs/s24/v0.11.0-missing-20261026-android-SM-S921B.jsonl @@ -0,0 +1,38 @@ +{"type":"run","label":"v0.11.0-missing-20261026","platform":"android","device":{"model":"SM-S921B","osVersion":"16","soc":"s5e9945","isEmulator":false,"cpuCores":10,"totalMemoryBytes":7574827008},"inputSpecVersion":1,"buildType":"release","clocksPinned":false,"settings":{"suite":"depth-estimation/depth-anything-v2-small-xnnpack-int8,depth-estimation/depth-anything-v2-small-xnnpack-fp32,depth-estimation/depth-anything-v2-small-vulkan-fp16,depth-estimation/depth-anything-v3-small-xnnpack-int8,depth-estimation/depth-anything-v3-small-xnnpack-fp32,depth-estimation/depth-anything-v3-small-vulkan-fp16,depth-estimation/zipdepth-base-xnnpack-fp32,depth-estimation/zipdepth-base-vulkan-fp16,llm/lfm2-5-1-2-b-xnnpack-8-da4-w,llm/lfm2-5-2-6-b-xnnpack-8-da4-w,llm/lfm2-5-350-m-xnnpack-8-da4-w,llm/lfm2-5-vl-450-m-xnnpack-8-da4-w,llm/lfm2-5-vl-450-m-vulkan-8-da4-w,llm/lfm2-5-vl-1-6-b-xnnpack-8-da4-w,llm/lfm2-5-vl-1-6-b-vulkan-8-da4-w,llm/bielik-v3-1-5-b-xnnpack-8-da4-w,llm/llama3-2-1-b-xnnpack-spinquant,llm/llama3-2-3-b-xnnpack-spinquant,llm/smollm2-135-m-xnnpack-8-da8-w,llm/smollm2-360-m-xnnpack-8-da8-w,llm/smollm2-1-7-b-xnnpack-8-da8-w,llm/hammer2-1-0-5-b-xnnpack-8-da4-w,llm/hammer2-1-1-5-b-xnnpack-8-da4-w,llm/hammer2-1-3-b-xnnpack-8-da4-w,llm/phi4-mini-xnnpack-8-da4-w,llm/qwen2-5-0-5-b-xnnpack-8-da4-w,llm/qwen2-5-1-5-b-xnnpack-8-da4-w,llm/qwen2-5-3-b-xnnpack-8-da4-w,llm/qwen3-0-6-b-xnnpack-8-da4-w,llm/qwen3-1-7-b-xnnpack-8-da4-w,llm/qwen3-4-b-xnnpack-8-da4-w,llm/gemma4-e2-b-xnnpack-8-da4-w,llm/gemma4-e2-b-vulkan-8-da4-w,llm/gemma4-omni-e2-b-xnnpack-8-da4-w,llm/gemma4-omni-e2-b-vulkan-8-da4-w,colbert-embeddings/lfm2-5-colbert-350-m-xnnpack-8-da4-w","iterations":20,"warmup":3,"memoryIterations":5,"loadIterations":3,"repeats":1,"maxTempC":37,"maxBytes":6000000000,"urlMap":{}},"startedAt":"2026-10-26T15:42:39.606Z"} +{"id":"depth-estimation/zipdepth-base-vulkan-fp16","task":"depthEstimation","model":"depthEstimation.ZIPDEPTH_BASE.VULKAN_FP16","backend":"vulkan","precision":"fp16","bytes":12794626,"mode":"worklet","progress":{"caseIndex":1,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":1831.515,"taskLoadMs":17.703,"taskLoad":{"count":3,"min":17.377,"max":609.813,"mean":214.964,"median":17.703,"p90":609.813,"iqr":592.436,"stdev":279.2},"pipeline":{"count":20,"min":16.688,"max":28.146,"mean":20.159,"median":19.084,"p90":23.375,"iqr":2.937,"stdev":2.994},"execution":{"perIteration":{"forward":{"count":1,"ms":16.67307035}},"totalMs":16.67307035},"units":262144,"gate":{"kind":"host","waitedS":0,"temperatureC":29.5,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":29.7},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":29.7,"samples":7},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":146.39,"loadedMb":255.37,"peakMb":263.6,"disposedMb":201.62}} +{"id":"depth-estimation/zipdepth-base-xnnpack-fp32","task":"depthEstimation","model":"depthEstimation.ZIPDEPTH_BASE.XNNPACK_FP32","backend":"xnnpack","precision":"fp32","bytes":24609344,"mode":"worklet","progress":{"caseIndex":2,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":972.182,"taskLoadMs":36.226,"taskLoad":{"count":3,"min":25.062,"max":47.965,"mean":36.417,"median":36.226,"p90":47.965,"iqr":22.903,"stdev":9.351},"pipeline":{"count":20,"min":52.468,"max":117.738,"mean":74.737,"median":76.594,"p90":86.784,"iqr":25.79,"stdev":16.386},"execution":{"perIteration":{"forward":{"count":1,"ms":72.64470109999999}},"totalMs":72.64470109999999},"units":262144,"gate":{"kind":"host","waitedS":0,"temperatureC":29.5,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":29.7},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":29.7,"samples":16},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":215.14,"loadedMb":243.92,"peakMb":299.47,"disposedMb":287.05}} +{"id":"depth-estimation/depth-anything-v2-small-xnnpack-int8","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V2_SMALL.XNNPACK_INT8","backend":"xnnpack","precision":"int8","bytes":35481732,"mode":"worklet","progress":{"caseIndex":3,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":1053.14,"taskLoadMs":31.488,"taskLoad":{"count":3,"min":30.407,"max":46.187,"mean":36.027,"median":31.488,"p90":46.187,"iqr":15.78,"stdev":7.198},"pipeline":{"count":20,"min":477.819,"max":507.925,"mean":492.995,"median":493.128,"p90":505.388,"iqr":12.065,"stdev":8.078},"execution":{"perIteration":{"forward":{"count":1,"ms":489.04829099999995}},"totalMs":489.04829099999995},"units":262144,"gate":{"kind":"host","waitedS":0,"temperatureC":29.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":30.9},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":30.9,"samples":97},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":305.34,"loadedMb":274.12,"peakMb":391.5,"disposedMb":302.65}} +{"id":"depth-estimation/depth-anything-v3-small-xnnpack-int8","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V3_SMALL.XNNPACK_INT8","backend":"xnnpack","precision":"int8","bytes":40603272,"mode":"worklet","progress":{"caseIndex":4,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":1438.366,"taskLoadMs":53.432,"taskLoad":{"count":3,"min":49.762,"max":56.584,"mean":53.259,"median":53.432,"p90":56.584,"iqr":6.821,"stdev":2.787},"pipeline":{"count":20,"min":581.865,"max":628.356,"mean":595.105,"median":591.372,"p90":607.351,"iqr":15.747,"stdev":11.069},"execution":{"perIteration":{"forward":{"count":1,"ms":591.65635555}},"totalMs":591.65635555},"units":262144,"gate":{"kind":"host","waitedS":0,"temperatureC":30.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":33.7},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":33.7,"samples":125},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":307.21,"loadedMb":329.29,"peakMb":497.73,"disposedMb":294.2}} +{"id":"depth-estimation/depth-anything-v2-small-vulkan-fp16","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V2_SMALL.VULKAN_FP16","backend":"vulkan","precision":"fp16","bytes":49595524,"mode":"worklet","progress":{"caseIndex":5,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":1785.481,"taskLoadMs":58.751,"taskLoad":{"count":3,"min":47.743,"max":363.858,"mean":156.784,"median":58.751,"p90":363.858,"iqr":316.116,"stdev":146.493},"pipeline":{"count":20,"min":258.874,"max":264.999,"mean":261.339,"median":261.118,"p90":263.049,"iqr":1.999,"stdev":1.538},"execution":{"perIteration":{"forward":{"count":1,"ms":256.53707625000004}},"totalMs":256.53707625000004},"units":262144,"gate":{"kind":"host","waitedS":0,"temperatureC":33.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":34.9},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":34.9,"samples":58},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":292.09,"loadedMb":369.3,"peakMb":369.82,"disposedMb":235.88}} +{"id":"depth-estimation/depth-anything-v3-small-vulkan-fp16","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V3_SMALL.VULKAN_FP16","backend":"vulkan","precision":"fp16","bytes":56407810,"mode":"worklet","progress":{"caseIndex":6,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":2167.317,"taskLoadMs":80.964,"taskLoad":{"count":3,"min":78.549,"max":413.719,"mean":191.077,"median":80.964,"p90":413.719,"iqr":335.17,"stdev":157.434},"pipeline":{"count":20,"min":318.944,"max":346.942,"mean":328.196,"median":324.277,"p90":344.792,"iqr":11.776,"stdev":9.471},"execution":{"perIteration":{"forward":{"count":1,"ms":317.62733579999997}},"totalMs":317.62733579999997},"units":262144,"gate":{"kind":"host","waitedS":0,"temperatureC":34.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":36.3},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":36.3,"samples":69},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":246.21,"loadedMb":473.06,"peakMb":476.46,"disposedMb":263.12}} +{"id":"depth-estimation/depth-anything-v2-small-xnnpack-fp32","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V2_SMALL.XNNPACK_FP32","backend":"xnnpack","precision":"fp32","bytes":98979332,"mode":"worklet","progress":{"caseIndex":7,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":2454.317,"taskLoadMs":31.795,"taskLoad":{"count":3,"min":27.348,"max":79.892,"mean":46.345,"median":31.795,"p90":79.892,"iqr":52.544,"stdev":23.791},"pipeline":{"count":20,"min":715.837,"max":786.308,"mean":753.326,"median":751.057,"p90":773.895,"iqr":23.737,"stdev":17.419},"execution":{"perIteration":{"forward":{"count":1,"ms":750.3458342}},"totalMs":750.3458342},"units":262144,"gate":{"kind":"host","waitedS":30,"temperatureC":36.2,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":36.2},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":36.2,"samples":160},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":257.87,"loadedMb":333.12,"peakMb":459.72,"disposedMb":380.48}} +{"id":"depth-estimation/depth-anything-v3-small-xnnpack-fp32","task":"depthEstimation","model":"depthEstimation.DEPTH_ANYTHING_V3_SMALL.XNNPACK_FP32","backend":"xnnpack","precision":"fp32","bytes":106480776,"mode":"worklet","progress":{"caseIndex":8,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":3237.814,"taskLoadMs":54.49,"taskLoad":{"count":3,"min":51.921,"max":59.295,"mean":55.235,"median":54.49,"p90":59.295,"iqr":7.375,"stdev":3.056},"pipeline":{"count":20,"min":784.728,"max":859.867,"mean":831.493,"median":839.164,"p90":847.575,"iqr":16.217,"stdev":19.504},"execution":{"perIteration":{"forward":{"count":1,"ms":827.9224843}},"totalMs":827.9224843},"units":262144,"gate":{"kind":"host","waitedS":31,"temperatureC":36.8,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":37.2},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":37.9,"samples":177},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":385.43,"loadedMb":410.96,"peakMb":581.38,"disposedMb":377.29}} +{"id":"colbert-embeddings/lfm2-5-colbert-350-m-xnnpack-8-da4-w","task":"colbertEmbeddings","model":"colbertEmbeddings.LFM2_5_COLBERT_350M.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":377361478,"mode":"worklet","progress":{"caseIndex":11,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":7712.173,"taskLoadMs":502.682,"taskLoad":{"count":3,"min":502.077,"max":587.289,"mean":530.683,"median":502.682,"p90":587.289,"iqr":85.212,"stdev":40.027},"pipeline":{"count":20,"min":82.306,"max":99.276,"mean":84.579,"median":82.693,"p90":87.626,"iqr":0.775,"stdev":4.616},"execution":{"perIteration":{"forward":{"count":1,"ms":83.8619434}},"totalMs":83.8619434},"units":56,"gate":{"kind":"host","waitedS":30,"temperatureC":37,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":37.2},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":37.2,"samples":20},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":414.25,"loadedMb":668.69,"peakMb":679.89,"disposedMb":565.16}} +{"type":"run","label":"v0.11.0-missing-20261026","platform":"android","device":{"model":"SM-S921B","osVersion":"16","soc":"s5e9945","isEmulator":false,"cpuCores":10,"totalMemoryBytes":7574827008},"inputSpecVersion":1,"buildType":"release","clocksPinned":false,"settings":{"suite":"depth-estimation/depth-anything-v2-small-xnnpack-int8,depth-estimation/depth-anything-v2-small-xnnpack-fp32,depth-estimation/depth-anything-v2-small-vulkan-fp16,depth-estimation/depth-anything-v3-small-xnnpack-int8,depth-estimation/depth-anything-v3-small-xnnpack-fp32,depth-estimation/depth-anything-v3-small-vulkan-fp16,depth-estimation/zipdepth-base-xnnpack-fp32,depth-estimation/zipdepth-base-vulkan-fp16,llm/lfm2-5-1-2-b-xnnpack-8-da4-w,llm/lfm2-5-2-6-b-xnnpack-8-da4-w,llm/lfm2-5-350-m-xnnpack-8-da4-w,llm/lfm2-5-vl-450-m-xnnpack-8-da4-w,llm/lfm2-5-vl-450-m-vulkan-8-da4-w,llm/lfm2-5-vl-1-6-b-xnnpack-8-da4-w,llm/lfm2-5-vl-1-6-b-vulkan-8-da4-w,llm/bielik-v3-1-5-b-xnnpack-8-da4-w,llm/llama3-2-1-b-xnnpack-spinquant,llm/llama3-2-3-b-xnnpack-spinquant,llm/smollm2-135-m-xnnpack-8-da8-w,llm/smollm2-360-m-xnnpack-8-da8-w,llm/smollm2-1-7-b-xnnpack-8-da8-w,llm/hammer2-1-0-5-b-xnnpack-8-da4-w,llm/hammer2-1-1-5-b-xnnpack-8-da4-w,llm/hammer2-1-3-b-xnnpack-8-da4-w,llm/phi4-mini-xnnpack-8-da4-w,llm/qwen2-5-0-5-b-xnnpack-8-da4-w,llm/qwen2-5-1-5-b-xnnpack-8-da4-w,llm/qwen2-5-3-b-xnnpack-8-da4-w,llm/qwen3-0-6-b-xnnpack-8-da4-w,llm/qwen3-1-7-b-xnnpack-8-da4-w,llm/qwen3-4-b-xnnpack-8-da4-w,llm/gemma4-e2-b-xnnpack-8-da4-w,llm/gemma4-e2-b-vulkan-8-da4-w,llm/gemma4-omni-e2-b-xnnpack-8-da4-w,llm/gemma4-omni-e2-b-vulkan-8-da4-w,colbert-embeddings/lfm2-5-colbert-350-m-xnnpack-8-da4-w","iterations":20,"warmup":3,"memoryIterations":5,"loadIterations":3,"repeats":1,"maxTempC":37,"maxBytes":6000000000,"urlMap":{}},"startedAt":"2026-10-26T16:23:01.879Z"} +{"id":"llm/smollm2-135-m-xnnpack-8-da8-w","task":"llm","model":"llm.SMOLLM2_135M.XNNPACK_8DA8W","backend":"xnnpack","precision":"8da8w","bytes":175114287,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":9,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":3832.089,"taskLoadMs":386.657,"taskLoad":{"count":3,"min":379.703,"max":439.213,"mean":401.858,"median":386.657,"p90":439.213,"iqr":59.511,"stdev":26.567},"pipeline":{"count":20,"min":349.327,"max":366.337,"mean":357.871,"median":359,"p90":363.896,"iqr":5.944,"stdev":4.491},"execution":{"perIteration":{"forward":{"count":65,"ms":339.7336491}},"totalMs":339.7336491},"units":64,"detail":{"timeToFirstTokenMs":27.432,"prefillMs":27,"promptTokens":38},"gate":{"kind":"host","waitedS":0,"temperatureC":31.2,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":31.8},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":31.8,"samples":116},"thermalValid":true,"holdsTimedOut":0,"coolingMs":4151,"memory":{"baselineMb":172.99,"loadedMb":1111.96,"peakMb":1102.05,"disposedMb":292.79}} +{"id":"llm/lfm2-5-350-m-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_350M.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":296972650,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":10,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":6622.177,"taskLoadMs":553.714,"taskLoad":{"count":3,"min":550.97,"max":598.651,"mean":567.778,"median":553.714,"p90":598.651,"iqr":47.681,"stdev":21.859},"pipeline":{"count":20,"min":423.712,"max":452.345,"mean":433.23,"median":431.899,"p90":439.279,"iqr":7.92,"stdev":6.657},"execution":{"perIteration":{"forward":{"count":65,"ms":413.94203045000006}},"totalMs":413.94203045000006},"units":64,"detail":{"timeToFirstTokenMs":20.206,"prefillMs":20,"promptTokens":17},"gate":{"kind":"host","waitedS":0,"temperatureC":33.1,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":34.8},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":34,"samples":133},"thermalValid":true,"holdsTimedOut":0,"coolingMs":4268,"memory":{"baselineMb":300.01,"loadedMb":946.35,"peakMb":1007,"disposedMb":336.92}} +{"id":"llm/smollm2-360-m-xnnpack-8-da8-w","task":"llm","model":"llm.SMOLLM2_360M.XNNPACK_8DA8W","backend":"xnnpack","precision":"8da8w","bytes":421904303,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":12,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":7687.854,"taskLoadMs":1037.663,"taskLoad":{"count":3,"min":973.063,"max":1245.605,"mean":1085.444,"median":1037.663,"p90":1245.605,"iqr":272.542,"stdev":116.282},"pipeline":{"count":20,"min":777.898,"max":1255.778,"mean":884.882,"median":841.617,"p90":988.877,"iqr":106.584,"stdev":116.691},"execution":{"perIteration":{"forward":{"count":65,"ms":860.95271335}},"totalMs":860.95271335},"units":64,"detail":{"timeToFirstTokenMs":54.567,"prefillMs":54,"promptTokens":38},"gate":{"kind":"host","waitedS":0,"temperatureC":35.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":37.1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":35.9,"samples":227},"thermalValid":true,"holdsTimedOut":0,"coolingMs":4487,"memory":{"baselineMb":354.09,"loadedMb":1925.46,"peakMb":1922.52,"disposedMb":360.71}} +{"id":"llm/qwen2-5-0-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN2_5_0_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":439214950,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":13,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":8046.496,"taskLoadMs":876.287,"taskLoad":{"count":3,"min":871.232,"max":899.522,"mean":882.347,"median":876.287,"p90":899.522,"iqr":28.29,"stdev":12.319},"pipeline":{"count":20,"min":625.669,"max":812.225,"mean":671.511,"median":638.094,"p90":788.161,"iqr":14.047,"stdev":64.296},"execution":{"perIteration":{"forward":{"count":65,"ms":636.6627998}},"totalMs":636.6627998},"units":64,"detail":{"timeToFirstTokenMs":69.613,"prefillMs":69,"promptTokens":37},"gate":{"kind":"host","waitedS":30,"temperatureC":35.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":35.7},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":35.9,"samples":184},"thermalValid":true,"holdsTimedOut":0,"coolingMs":4223,"memory":{"baselineMb":363.52,"loadedMb":1110.53,"peakMb":1129.38,"disposedMb":375.21}} +{"id":"llm/hammer2-1-0-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.HAMMER2_1_0_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":443224596,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":14,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":6487.192,"taskLoadMs":1245.711,"taskLoad":{"count":3,"min":1178.397,"max":1258.356,"mean":1227.488,"median":1245.711,"p90":1258.356,"iqr":79.959,"stdev":35.095},"pipeline":{"count":20,"min":608.33,"max":905.702,"mean":731.012,"median":705.119,"p90":872.835,"iqr":239.174,"stdev":113.826},"execution":{"perIteration":{"forward":{"count":65,"ms":692.9105719500001}},"totalMs":692.9105719500001},"units":64,"detail":{"timeToFirstTokenMs":53.661,"prefillMs":54,"promptTokens":27},"gate":{"kind":"host","waitedS":30,"temperatureC":36.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":37.6},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":37.6,"samples":170},"thermalValid":true,"holdsTimedOut":0,"coolingMs":4394,"memory":{"baselineMb":386.08,"loadedMb":1147.22,"peakMb":1137.53,"disposedMb":379.91}} +{"id":"llm/qwen3-0-6-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN3_0_6B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":529702696,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":15,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":6.126,"taskLoadMs":1531.805,"taskLoad":{"count":3,"min":1491.238,"max":1578.986,"mean":1534.01,"median":1531.805,"p90":1578.986,"iqr":87.748,"stdev":35.857},"pipeline":{"count":20,"min":782.851,"max":1190.073,"mean":825.034,"median":790.375,"p90":911.003,"iqr":12.782,"stdev":94.982},"execution":{"perIteration":{"forward":{"count":65,"ms":791.3545573}},"totalMs":791.3545573},"units":64,"detail":{"timeToFirstTokenMs":38.256,"prefillMs":38,"promptTokens":16},"gate":{"kind":"host","waitedS":46,"temperatureC":36.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":37.5},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":37.5,"samples":636},"thermalValid":true,"holdsTimedOut":0,"coolingMs":49614,"memory":{"baselineMb":381.38,"loadedMb":2598.83,"peakMb":2620.83,"disposedMb":337.49}} +{"id":"llm/lfm2-5-vl-450-m-vulkan-8-da4-w","task":"llm","model":"llm.LFM2_5_VL_450M.VULKAN_8DA4W","backend":"vulkan","precision":"8da4w","bytes":627140107,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":16,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":11384.62,"taskLoadMs":430.118,"taskLoad":{"count":3,"min":423.402,"max":901.387,"mean":584.969,"median":430.118,"p90":901.387,"iqr":477.985,"stdev":223.758},"pipeline":{"count":20,"min":847.059,"max":890.739,"mean":867.22,"median":864.704,"p90":887.253,"iqr":20.288,"stdev":13.046},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":42.82048605},"text_decoder":{"count":65,"ms":726.62071745}},"totalMs":769.4412035},"units":64,"detail":{"timeToFirstTokenMs":41.46,"prefillMs":40,"promptTokens":17},"gate":{"kind":"host","waitedS":46,"temperatureC":36.4,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":36.2},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":37.5,"samples":227},"thermalValid":true,"holdsTimedOut":0,"coolingMs":4675,"memory":{"baselineMb":347.21,"loadedMb":1092.47,"peakMb":1059.62,"disposedMb":610.72}} +{"id":"llm/lfm2-5-vl-450-m-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_VL_450M.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":653603851,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":17,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":14949.062,"taskLoadMs":584.595,"taskLoad":{"count":3,"min":581.655,"max":627.256,"mean":597.835,"median":584.595,"p90":627.256,"iqr":45.601,"stdev":20.838},"pipeline":{"count":20,"min":663.02,"max":690.559,"mean":677.88,"median":677.911,"p90":689.068,"iqr":13.984,"stdev":8.765},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":0.52804255},"text_decoder":{"count":65,"ms":654.17576745}},"totalMs":654.70381},"units":64,"detail":{"timeToFirstTokenMs":20.644,"prefillMs":21,"promptTokens":17},"gate":{"kind":"host","waitedS":0,"temperatureC":36.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":36.6},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":38,"samples":611},"thermalValid":true,"holdsTimedOut":0,"coolingMs":49694,"memory":{"baselineMb":610.89,"loadedMb":1212.88,"peakMb":1276.62,"disposedMb":659.18}} +{"id":"llm/lfm2-5-1-2-b-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_1_2B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":815187806,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":18,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":12653.432,"taskLoadMs":1126.627,"taskLoad":{"count":3,"min":1124.267,"max":1245.645,"mean":1165.513,"median":1126.627,"p90":1245.645,"iqr":121.378,"stdev":56.67},"pipeline":{"count":20,"min":1167.49,"max":1391.631,"mean":1226.768,"median":1193.679,"p90":1309.726,"iqr":102.891,"stdev":68.16},"execution":{"perIteration":{"forward":{"count":65,"ms":1203.5345817}},"totalMs":1203.5345817},"units":64,"detail":{"timeToFirstTokenMs":73.533,"prefillMs":73,"promptTokens":17},"gate":{"kind":"host","waitedS":46,"temperatureC":36.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":38.2},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":38.3,"samples":888},"thermalValid":true,"holdsTimedOut":0,"coolingMs":64910,"memory":{"baselineMb":659.44,"loadedMb":1520.1,"peakMb":1610.02,"disposedMb":786.75}} +{"id":"llm/bielik-v3-1-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.BIELIK_V3_1_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":939299957,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":19,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":19052.922,"taskLoadMs":1262.411,"taskLoad":{"count":3,"min":1245.548,"max":1400.829,"mean":1302.929,"median":1262.411,"p90":1400.829,"iqr":155.281,"stdev":69.567},"pipeline":{"count":20,"min":1625.651,"max":1899.856,"mean":1699.803,"median":1673.145,"p90":1844.1,"iqr":59.835,"stdev":79.436},"execution":{"perIteration":{"forward":{"count":65,"ms":1680.62944835}},"totalMs":1680.62944835},"units":64,"detail":{"timeToFirstTokenMs":186.39,"prefillMs":186,"promptTokens":26},"gate":{"kind":"host","waitedS":61,"temperatureC":37,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":38.3},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":38.3,"samples":1524},"thermalValid":true,"holdsTimedOut":0,"coolingMs":125347,"memory":{"baselineMb":787.34,"loadedMb":1881.23,"peakMb":1925.04,"disposedMb":881.84}} +{"id":"llm/llama3-2-1-b-xnnpack-spinquant","task":"llm","model":"llm.LLAMA3_2_1B.XNNPACK_SPINQUANT","backend":"xnnpack","precision":"spinquant","bytes":1092444893,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":20,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":18177.734,"taskLoadMs":1877.059,"taskLoad":{"count":3,"min":1726.496,"max":1880.123,"mean":1827.893,"median":1877.059,"p90":1880.123,"iqr":153.627,"stdev":71.709},"pipeline":{"count":20,"min":1470.966,"max":1570.961,"mean":1506.593,"median":1507.218,"p90":1523.59,"iqr":20.298,"stdev":20.992},"execution":{"perIteration":{"forward":{"count":65,"ms":1474.1945558}},"totalMs":1474.1945558},"units":64,"detail":{"timeToFirstTokenMs":127.908,"prefillMs":127,"promptTokens":43},"gate":{"kind":"host","waitedS":76,"temperatureC":36.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":37},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":38.5,"samples":2059},"thermalValid":true,"holdsTimedOut":0,"coolingMs":185631,"memory":{"baselineMb":889.66,"loadedMb":2401.63,"peakMb":2422.38,"disposedMb":765.5}} +{"id":"llm/qwen2-5-1-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN2_5_1_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":1172577510,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":21,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":18106.794,"taskLoadMs":1879.9,"taskLoad":{"count":3,"min":1781.328,"max":1935.807,"mean":1865.679,"median":1879.9,"p90":1935.807,"iqr":154.479,"stdev":63.863},"pipeline":{"count":20,"min":1623.733,"max":1968.323,"mean":1723.337,"median":1685.745,"p90":1857.014,"iqr":75.576,"stdev":94.545},"execution":{"perIteration":{"forward":{"count":65,"ms":1688.5431469}},"totalMs":1688.5431469},"units":64,"detail":{"timeToFirstTokenMs":151.694,"prefillMs":152,"promptTokens":37},"gate":{"kind":"host","waitedS":61,"temperatureC":36.8,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":37},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":38.6,"samples":2245},"thermalValid":true,"holdsTimedOut":0,"coolingMs":200687,"memory":{"baselineMb":763.57,"loadedMb":2356.38,"peakMb":2290.68,"disposedMb":817.49}} +{"id":"llm/hammer2-1-1-5-b-xnnpack-8-da4-w","task":"llm","model":"llm.HAMMER2_1_1_5B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":1176265276,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":22,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":18465.382,"taskLoadMs":1820.332,"taskLoad":{"count":3,"min":1805.991,"max":1827.282,"mean":1817.868,"median":1820.332,"p90":1827.282,"iqr":21.291,"stdev":8.865},"pipeline":{"count":20,"min":1604.707,"max":2162.725,"mean":1689.901,"median":1641.224,"p90":1737.764,"iqr":34.222,"stdev":140.646},"execution":{"perIteration":{"forward":{"count":65,"ms":1654.72291865}},"totalMs":1654.72291865},"units":64,"detail":{"timeToFirstTokenMs":179.466,"prefillMs":179,"promptTokens":27},"gate":{"kind":"host","waitedS":61,"temperatureC":36.6,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":38.3},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":38.5,"samples":1557},"thermalValid":true,"holdsTimedOut":0,"coolingMs":125741,"memory":{"baselineMb":829.64,"loadedMb":2363.19,"peakMb":2285.74,"disposedMb":813.9}} +{"id":"llm/qwen3-1-7-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN3_1_7B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":1319364648,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":23,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":19869.11,"taskLoadMs":2068.207,"taskLoad":{"count":3,"min":1963.205,"max":2135.351,"mean":2055.588,"median":2068.207,"p90":2135.351,"iqr":172.146,"stdev":70.843},"pipeline":{"count":20,"min":1771.758,"max":1982.474,"mean":1820.762,"median":1800.624,"p90":1898.541,"iqr":41.994,"stdev":56.008},"execution":{"perIteration":{"forward":{"count":65,"ms":1785.83859795}},"totalMs":1785.83859795},"units":64,"detail":{"timeToFirstTokenMs":89.746,"prefillMs":89,"promptTokens":16},"gate":{"kind":"host","waitedS":76,"temperatureC":36.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":38.3},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":38.5,"samples":2146},"thermalValid":true,"holdsTimedOut":0,"coolingMs":185951,"memory":{"baselineMb":812.15,"loadedMb":2405.08,"peakMb":2439.43,"disposedMb":678.36}} +{"id":"llm/lfm2-5-2-6-b-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_2_6B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":1807639919,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":24,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":26529.474,"taskLoadMs":2802.393,"taskLoad":{"count":3,"min":2752.652,"max":2845.721,"mean":2800.255,"median":2802.393,"p90":2845.721,"iqr":93.068,"stdev":38.025},"pipeline":{"count":20,"min":2595.42,"max":3702.075,"mean":2739.303,"median":2669.933,"p90":2833.771,"iqr":58.078,"stdev":235.397},"execution":{"perIteration":{"forward":{"count":65,"ms":2706.60034455}},"totalMs":2706.60034455},"units":64,"detail":{"timeToFirstTokenMs":152.393,"prefillMs":152,"promptTokens":18},"gate":{"kind":"host","waitedS":76,"temperatureC":36.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":39.3},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":39.1,"samples":3162},"thermalValid":true,"holdsTimedOut":0,"coolingMs":276413,"memory":{"baselineMb":683.33,"loadedMb":2152.55,"peakMb":2067.91,"disposedMb":967.55}} +{"id":"llm/smollm2-1-7-b-xnnpack-8-da8-w","task":"llm","model":"llm.SMOLLM2_1_7B.XNNPACK_8DA8W","backend":"xnnpack","precision":"8da8w","bytes":1818983855,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":25,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":27757.958,"taskLoadMs":2430.546,"taskLoad":{"count":3,"min":2097.106,"max":3366.398,"mean":2631.35,"median":2430.546,"p90":3366.398,"iqr":1269.292,"stdev":537.288},"pipeline":{"count":20,"min":2843.869,"max":3052.529,"mean":2903.195,"median":2899.656,"p90":2931.804,"iqr":38.938,"stdev":43.427},"execution":{"perIteration":{"forward":{"count":65,"ms":2880.1710707499997}},"totalMs":2880.1710707499997},"units":64,"detail":{"timeToFirstTokenMs":117.809,"prefillMs":118,"promptTokens":38},"gate":{"kind":"host","waitedS":91,"temperatureC":36.8,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":39.4},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":39,"samples":3458},"thermalValid":true,"holdsTimedOut":0,"coolingMs":306636,"memory":{"baselineMb":969.73,"loadedMb":2858.47,"peakMb":2943.11,"disposedMb":1368.29}} +{"id":"llm/hammer2-1-3-b-xnnpack-8-da4-w","task":"llm","model":"llm.HAMMER2_1_3B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2076552284,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":26,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":41095.029,"taskLoadMs":3366.916,"taskLoad":{"count":3,"min":3159.009,"max":4234.199,"mean":3586.708,"median":3366.916,"p90":4234.199,"iqr":1075.189,"stdev":465.646},"pipeline":{"count":20,"min":3061.394,"max":3190.406,"mean":3125.522,"median":3132.276,"p90":3171.113,"iqr":58.023,"stdev":37.833},"execution":{"perIteration":{"forward":{"count":65,"ms":3088.92735705}},"totalMs":3088.92735705},"units":64,"detail":{"timeToFirstTokenMs":223.782,"prefillMs":223,"promptTokens":27},"gate":{"kind":"host","waitedS":76,"temperatureC":37,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":38.3},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":38.4,"samples":4640},"thermalValid":true,"holdsTimedOut":0,"coolingMs":427480,"memory":{"baselineMb":1380.87,"loadedMb":2959.23,"peakMb":2901.24,"disposedMb":935.12}} +{"id":"llm/qwen2-5-3-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN2_5_3B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2111318246,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":27,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":30120.515,"taskLoadMs":3078.211,"taskLoad":{"count":3,"min":3016.641,"max":3548.389,"mean":3214.413,"median":3078.211,"p90":3548.389,"iqr":531.748,"stdev":237.49},"pipeline":{"count":20,"min":3125.07,"max":4096.023,"mean":3293.932,"median":3217.7,"p90":3499.515,"iqr":94.919,"stdev":220.41},"execution":{"perIteration":{"forward":{"count":65,"ms":3256.6658436999996}},"totalMs":3256.6658436999996},"units":64,"detail":{"timeToFirstTokenMs":287.543,"prefillMs":287,"promptTokens":37},"gate":{"kind":"host","waitedS":91,"temperatureC":36.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":37.8},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":38.7,"samples":4280},"thermalValid":true,"holdsTimedOut":0,"coolingMs":382352,"memory":{"baselineMb":938.45,"loadedMb":2934.49,"peakMb":2903.09,"disposedMb":934.89}} +{"id":"llm/llama3-2-3-b-xnnpack-spinquant","task":"llm","model":"llm.LLAMA3_2_3B.XNNPACK_SPINQUANT","backend":"xnnpack","precision":"spinquant","bytes":2389281117,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":28,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":35350.912,"taskLoadMs":3943.181,"taskLoad":{"count":3,"min":3927.127,"max":4727.91,"mean":4199.406,"median":3943.181,"p90":4727.91,"iqr":800.783,"stdev":373.766},"pipeline":{"count":20,"min":3576.918,"max":4133.875,"mean":3763.782,"median":3683.294,"p90":4035.132,"iqr":263.789,"stdev":171.863},"execution":{"perIteration":{"forward":{"count":65,"ms":3726.1109582999998}},"totalMs":3726.1109582999998},"units":64,"detail":{"timeToFirstTokenMs":340.142,"prefillMs":340,"promptTokens":43},"gate":{"kind":"host","waitedS":76,"temperatureC":36.8,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":39.3},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":39.3,"samples":4953},"thermalValid":true,"holdsTimedOut":0,"coolingMs":442928,"memory":{"baselineMb":928.97,"loadedMb":3364.19,"peakMb":3307.09,"disposedMb":1153.65}} +{"id":"llm/lfm2-5-vl-1-6-b-vulkan-8-da4-w","task":"llm","model":"llm.LFM2_5_VL_1_6B.VULKAN_8DA4W","backend":"vulkan","precision":"8da4w","bytes":2470992395,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":29,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":51361.217,"taskLoadMs":2173.456,"taskLoad":{"count":3,"min":2040.758,"max":3836.069,"mean":2683.428,"median":2173.456,"p90":3836.069,"iqr":1795.311,"stdev":816.839},"pipeline":{"count":20,"min":1546.428,"max":3170.87,"mean":1954.078,"median":1808.85,"p90":2390.858,"iqr":416.255,"stdev":375.906},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":63.663991450000005},"text_decoder":{"count":65,"ms":1668.9247471499998}},"totalMs":1732.5887386},"units":64,"detail":{"timeToFirstTokenMs":77.109,"prefillMs":76,"promptTokens":17},"gate":{"kind":"host","waitedS":91,"temperatureC":36.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":36.8},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":38,"samples":1279},"thermalValid":true,"holdsTimedOut":0,"coolingMs":95342,"memory":{"baselineMb":1152.77,"loadedMb":2968.25,"peakMb":2962.26,"disposedMb":1431.69}} +{"id":"llm/lfm2-5-vl-1-6-b-xnnpack-8-da4-w","task":"llm","model":"llm.LFM2_5_VL_1_6B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2479599371,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":30,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":46654.71,"taskLoadMs":2450.623,"taskLoad":{"count":3,"min":2146.553,"max":3345.904,"mean":2647.693,"median":2450.623,"p90":3345.904,"iqr":1199.351,"stdev":509.076},"pipeline":{"count":20,"min":1397.328,"max":1621.142,"mean":1431.45,"median":1422.021,"p90":1442.739,"iqr":13.81,"stdev":45.252},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":5.3948552},"text_decoder":{"count":65,"ms":1400.12574635}},"totalMs":1405.52060155},"units":64,"detail":{"timeToFirstTokenMs":53.639,"prefillMs":53,"promptTokens":17},"gate":{"kind":"host","waitedS":45,"temperatureC":36.5,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":37.2},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":38.5,"samples":2043},"thermalValid":true,"holdsTimedOut":0,"coolingMs":185413,"memory":{"baselineMb":1431.86,"loadedMb":3063.05,"peakMb":3145.71,"disposedMb":1411.46}} +{"id":"llm/qwen3-4-b-xnnpack-8-da4-w","task":"llm","model":"llm.QWEN3_4B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2697273128,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":31,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":56033.359,"taskLoadMs":5896.722,"taskLoad":{"count":3,"min":5469.382,"max":6786.504,"mean":6050.869,"median":5896.722,"p90":6786.504,"iqr":1317.121,"stdev":548.649},"pipeline":{"count":20,"min":3927.754,"max":5604.339,"mean":4177.727,"median":4044.966,"p90":4358.371,"iqr":276.891,"stdev":362.704},"execution":{"perIteration":{"forward":{"count":65,"ms":4136.9751458}},"totalMs":4136.9751458},"units":64,"detail":{"timeToFirstTokenMs":159.537,"prefillMs":159,"promptTokens":16},"gate":{"kind":"host","waitedS":46,"temperatureC":37,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":39.2},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":39.1,"samples":5179},"thermalValid":true,"holdsTimedOut":0,"coolingMs":458198,"memory":{"baselineMb":1420.02,"loadedMb":4006.06,"peakMb":3979.96,"disposedMb":1151.03}} +{"id":"llm/phi4-mini-xnnpack-8-da4-w","task":"llm","model":"llm.PHI4_MINI.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":2833963001,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":32,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":42169.689,"taskLoadMs":5326.695,"taskLoad":{"count":3,"min":5019.666,"max":6166.394,"mean":5504.251,"median":5326.695,"p90":6166.394,"iqr":1146.727,"stdev":484.693},"pipeline":{"count":20,"min":3655.352,"max":3907.154,"mean":3732.141,"median":3720.356,"p90":3819.942,"iqr":64.728,"stdev":59.307},"execution":{"perIteration":{"forward":{"count":65,"ms":3687.9745445}},"totalMs":3687.9745445},"units":64,"detail":{"timeToFirstTokenMs":146.162,"prefillMs":146,"promptTokens":11},"gate":{"kind":"host","waitedS":106,"temperatureC":36.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":1,"statusName":"light","batteryTemperatureC":39.8},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":38.3,"samples":5366},"thermalValid":true,"holdsTimedOut":0,"coolingMs":488300,"memory":{"baselineMb":1147.13,"loadedMb":3770.29,"peakMb":3677.17,"disposedMb":1154.4}} +{"id":"llm/gemma4-e2-b-vulkan-8-da4-w","task":"llm","model":"llm.GEMMA4_E2B.VULKAN_8DA4W","backend":"vulkan","precision":"8da4w","bytes":2988556216,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":33,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":49332.228,"taskLoadMs":5768.968,"taskLoad":{"count":3,"min":5580.237,"max":5936.74,"mean":5761.982,"median":5768.968,"p90":5936.74,"iqr":356.502,"stdev":145.625},"pipeline":{"count":20,"min":4221.859,"max":4369.878,"mean":4312.48,"median":4323.733,"p90":4354.988,"iqr":60.586,"stdev":42.624},"execution":{"perIteration":{"forward":{"count":65,"ms":3874.0955859}},"totalMs":3874.0955859},"units":64,"detail":{"timeToFirstTokenMs":185.776,"prefillMs":183,"promptTokens":17},"gate":{"kind":"host","waitedS":91,"temperatureC":36.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":37.3},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":37.3,"samples":2935},"thermalValid":true,"holdsTimedOut":0,"coolingMs":216144,"memory":{"baselineMb":1154.44,"loadedMb":3906.29,"peakMb":3667.01,"disposedMb":298.23}} +{"id":"llm/gemma4-e2-b-xnnpack-8-da4-w","task":"llm","model":"llm.GEMMA4_E2B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":3247469492,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":34,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":58325.538,"taskLoadMs":3831.15,"taskLoad":{"count":3,"min":3814.127,"max":3878.162,"mean":3841.146,"median":3831.15,"p90":3878.162,"iqr":64.035,"stdev":27.081},"pipeline":{"count":20,"min":8553.459,"max":13362.534,"mean":9642.699,"median":9060.464,"p90":12680.278,"iqr":1209.487,"stdev":1476.169},"execution":{"perIteration":{"forward":{"count":65,"ms":9587.631437799999}},"totalMs":9587.631437799999},"units":64,"detail":{"timeToFirstTokenMs":441.691,"prefillMs":441,"promptTokens":17},"gate":{"kind":"host","waitedS":31,"temperatureC":36.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":38.6},"thermalPeak":{"status":1,"statusName":"light","batteryTemperatureC":39.2,"samples":7316},"thermalValid":true,"holdsTimedOut":0,"coolingMs":564104,"memory":{"baselineMb":291.86,"loadedMb":2403.71,"peakMb":2162.33,"disposedMb":969.12}} +{"id":"llm/gemma4-omni-e2-b-xnnpack-8-da4-w","task":"llm","model":"llm.GEMMA4_OMNI_E2B.XNNPACK_8DA4W","backend":"xnnpack","precision":"8da4w","bytes":3651367476,"note":"independent single-user chat prompt; KV cache reset each iteration; 64 generated tokens with EOS ignored; multimodal models are driven text-only","mode":"async","progress":{"caseIndex":35,"caseCount":36,"repeat":1,"repeats":1},"status":"ok","downloadMs":53793.179,"taskLoadMs":4726.15,"taskLoad":{"count":3,"min":4669.735,"max":5076.23,"mean":4824.038,"median":4726.15,"p90":5076.23,"iqr":406.495,"stdev":179.807},"pipeline":{"count":20,"min":8548.398,"max":13466.461,"mean":9477.603,"median":9029.076,"p90":11913.281,"iqr":736.411,"stdev":1465.979},"execution":{"perIteration":{"token_embedding":{"count":65,"ms":11.840393},"text_decoder":{"count":65,"ms":9405.6850317}},"totalMs":9417.525424700001},"units":64,"detail":{"timeToFirstTokenMs":479.227,"prefillMs":479,"promptTokens":17},"gate":{"kind":"host","waitedS":91,"temperatureC":36.6,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":38},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":38.5,"samples":7718},"thermalValid":true,"holdsTimedOut":0,"coolingMs":612939,"memory":{"baselineMb":954.94,"loadedMb":2835.28,"peakMb":2609.13,"disposedMb":362.26}} +{"id":"llm/gemma4-omni-e2-b-vulkan-8-da4-w","status":"error","error":"app process died during load","progress":{"repeat":1}} diff --git a/docs/docs/04-benchmarks/runs/s24/v0.11.0-missing-20261026-retry-vulkan-android-SM-S921B.jsonl b/docs/docs/04-benchmarks/runs/s24/v0.11.0-missing-20261026-retry-vulkan-android-SM-S921B.jsonl new file mode 100644 index 0000000000..a7ed44cc3c --- /dev/null +++ b/docs/docs/04-benchmarks/runs/s24/v0.11.0-missing-20261026-retry-vulkan-android-SM-S921B.jsonl @@ -0,0 +1,2 @@ +{"type":"run","label":"v0.11.0-missing-20261026-retry-vulkan","platform":"android","device":{"model":"SM-S921B","osVersion":"16","soc":"s5e9945","isEmulator":false,"cpuCores":10,"totalMemoryBytes":7574827008},"inputSpecVersion":1,"buildType":"release","clocksPinned":false,"settings":{"suite":"llm/gemma4-omni-e2-b-vulkan-8-da4-w","iterations":20,"warmup":3,"memoryIterations":5,"loadIterations":3,"repeats":1,"maxTempC":37,"maxBytes":6000000000,"urlMap":{}},"startedAt":"2026-10-26T19:38:42.066Z"} +{"id":"llm/gemma4-omni-e2-b-vulkan-8-da4-w","status":"error","error":"app process died during load","progress":{"repeat":1}} diff --git a/docs/docs/04-benchmarks/runs/s24/v0.11.0-missing-20261026-retry-vulkan-android-SM-S921B.notes.json b/docs/docs/04-benchmarks/runs/s24/v0.11.0-missing-20261026-retry-vulkan-android-SM-S921B.notes.json new file mode 100644 index 0000000000..a3500eba16 --- /dev/null +++ b/docs/docs/04-benchmarks/runs/s24/v0.11.0-missing-20261026-retry-vulkan-android-SM-S921B.notes.json @@ -0,0 +1,5 @@ +{ + "case": "llm/gemma4-omni-e2-b-vulkan-8-da4-w", + "outcome": "Failed during loading in the original batch and a fresh-process retry. No measurement published.", + "evidence": "2026-10-26 20:38:51 device log: lmkd reclaimed com.anonymous.benchmarks (pid 30877, oom_score_adj 0), reason: min2x watermark is breached even after kill; level: 2. Confirms system memory pressure during the retry." +} diff --git a/docs/sidebars.js b/docs/sidebars.js index 26bb08a4e5..a23528a8e0 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -8,6 +8,15 @@ const sidebars = { label: 'Fundamentals', items: [{ type: 'autogenerated', dirName: '01-fundamentals' }], }, + { + type: 'category', + label: 'Benchmarks', + className: 'hide-api-items', + collapsible: true, + collapsed: true, + link: { type: 'doc', id: 'benchmarks/benchmarks' }, + items: [{ type: 'doc', id: 'benchmarks/benchmarks', label: 'Benchmarks' }], + }, { type: 'category', label: 'Extensions', @@ -18,15 +27,6 @@ const sidebars = { }, items: [{ type: 'autogenerated', dirName: '02-extensions' }], }, - { - type: 'category', - label: 'Benchmarks', - className: 'hide-api-items', - collapsible: true, - collapsed: true, - link: { type: 'doc', id: 'benchmarks/benchmarks' }, - items: [{ type: 'doc', id: 'benchmarks/benchmarks', label: 'Benchmarks' }], - }, { type: 'category', label: 'Core & Advanced', diff --git a/docs/src/data/benchmarks.ts b/docs/src/data/benchmarks.ts index 2e1fe17f89..be0bca6f35 100644 --- a/docs/src/data/benchmarks.ts +++ b/docs/src/data/benchmarks.ts @@ -103,7 +103,7 @@ export const METRICS: Metric[] = [ label: 'Peak Process Memory', unit: 'MB', description: - 'Peak process footprint during inference, including the app baseline: total PSS on Android and phys_footprint on iOS.', + 'Sampled process footprint during inference, including the app baseline: total PSS on Android and phys_footprint on iOS. GPU memory may be under-counted; loading peaks and brief peaks are not captured reliably.', lowerIsBetter: true, }, { From 3c16e3f8b17da43ccacfa05853c08f643504538a Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 27 Oct 2026 13:54:00 +0100 Subject: [PATCH 3/4] Simplify benchmark memory caveat --- docs/docs/04-benchmarks/01-benchmarks.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/04-benchmarks/01-benchmarks.mdx b/docs/docs/04-benchmarks/01-benchmarks.mdx index 60a43c3871..9901f253af 100644 --- a/docs/docs/04-benchmarks/01-benchmarks.mdx +++ b/docs/docs/04-benchmarks/01-benchmarks.mdx @@ -53,7 +53,7 @@ Measurements were collected on physical iOS and Android devices spanning flagshi Memory values report peak process footprint during a separate inference pass, including the app baseline: total PSS on Android and `phys_footprint` on iOS. Some supplemental iPhone LLM measurements use a fresh process for each model. :::caution GPU memory accounting -These sampled process counters do not measure total GPU memory or the peak during model loading. GPU backends can report unexpectedly low values: for example, MLX LFM2.5-VL-1.6B (int8) reports 216 MB despite a roughly 2.8 GB download, and Gemma4 E2B on Vulkan reports 327 MB on S20+ versus 3667 MB on S24. These discrepancies suggest incomplete or device-dependent memory accounting; the exact cause has not been established. Treat the figures as observed process-counter readings, not total model memory requirements or a reliable comparison of GPU memory across devices and backends. Sampling can also miss brief peaks. +Memory figures reflect sampled process usage during inference. They may under-report GPU memory usage and miss brief peaks, and they exclude peaks during model loading. Memory accounting also varies across devices and backends, so use these values as a guide rather than as minimum RAM requirements or a direct comparison of GPU memory usage. ::: ## Results @@ -187,7 +187,7 @@ These benchmarks measure execution latency, memory utilization, and initializati - **Variants:** Quantized LLMs only. - **Collection:** The S26 Ultra LLM results retain the September measurements. Supplemental S20+, S24, and iPhone 17 LLM runs were collected in October using the fixed-prompt harness described below. - **Input:** Prompt (`List three uses for a paperclip.`), 64 tokens generated with EOS ignored, temperature 0 (greedy decoding), and the KV cache reset before each independent prompt. -- **Prefill vs. Decode & Memory:** The short prompt isolates autoregressive token generation throughput (tokens/second) and time to first token (TTFT). Autoregressive decoding executes one forward pass per generated token. Memory requirements include model weights, allocated key-value (KV) cache buffers, and intermediate tensors. Observed process memory reaches 4122 MB for Qwen3 4B on S20+; GPU readings have the accounting limitations described above. +- **Prefill vs. Decode & Memory:** The short prompt isolates autoregressive token generation throughput (tokens/second) and time to first token (TTFT). Autoregressive decoding executes one forward pass per generated token. Memory requirements include model weights, allocated key-value (KV) cache buffers, and intermediate tensors. ::: On iPhone 17, Phi-4 Mini was omitted. Qwen3 4B (XNNPACK) terminated after critical memory-pressure warnings, including in a fresh-process retry. On Galaxy S24, Android's low-memory killer terminated Gemma4 Omni E2B (Vulkan) during loading in a fresh-process retry. Only successful measurements are charted. From 378dea7e84bb4f7d3e2c5c05ffb40088ed4c7557 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 27 Oct 2026 14:34:35 +0100 Subject: [PATCH 4/4] Add MIGAN inpainting benchmarks across Android and iOS --- apps/benchmarks/BENCHMARK_SPEC.md | 1 + apps/benchmarks/scripts/variant-sizes.json | 11 +- apps/benchmarks/src/drivers.ts | 11 + apps/benchmarks/src/inputs.ts | 7 + docs/docs/04-benchmarks/01-benchmarks.mdx | 17 +- docs/docs/04-benchmarks/measured.json | 252 ++++++++++++++++++ ...-libs-0.10.7-20261027-ios-iPhone18-3.jsonl | 7 + ...-0.10.7-20261027-ios-iPhone18-3.notes.json | 14 + ...ibs-0.10.7-20261027-android-SM-G986B.jsonl | 7 + ....10.7-20261027-android-SM-G986B.notes.json | 14 + ...ibs-0.10.7-20261027-android-SM-S921B.jsonl | 7 + ....10.7-20261027-android-SM-S921B.notes.json | 14 + 12 files changed, 357 insertions(+), 5 deletions(-) create mode 100644 docs/docs/04-benchmarks/runs/iphone17/v0.11.0-migan-libs-0.10.7-20261027-ios-iPhone18-3.jsonl create mode 100644 docs/docs/04-benchmarks/runs/iphone17/v0.11.0-migan-libs-0.10.7-20261027-ios-iPhone18-3.notes.json create mode 100644 docs/docs/04-benchmarks/runs/s20plus/v0.11.0-migan-libs-0.10.7-20261027-android-SM-G986B.jsonl create mode 100644 docs/docs/04-benchmarks/runs/s20plus/v0.11.0-migan-libs-0.10.7-20261027-android-SM-G986B.notes.json create mode 100644 docs/docs/04-benchmarks/runs/s24/v0.11.0-migan-libs-0.10.7-20261027-android-SM-S921B.jsonl create mode 100644 docs/docs/04-benchmarks/runs/s24/v0.11.0-migan-libs-0.10.7-20261027-android-SM-S921B.notes.json diff --git a/apps/benchmarks/BENCHMARK_SPEC.md b/apps/benchmarks/BENCHMARK_SPEC.md index 1621627ca8..39f9d7998e 100644 --- a/apps/benchmarks/BENCHMARK_SPEC.md +++ b/apps/benchmarks/BENCHMARK_SPEC.md @@ -152,6 +152,7 @@ report and the comparator refuses to diff across a change to it. | -------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | | Vision (classification, depth estimation, detection, segmentation, style transfer, OCR, keypoints, image embeddings) | A fixed 512x512 or 640x640 scene: a vertical gradient, three solid ellipses, and a seeded dither | | ColBERT embeddings | The fixed text-embedding paragraph, encoded as a document; query encoding and MaxSim scoring are not timed | +| Image inpainting | The fixed 512x512 RGBA scene with a central 256x256 hole (25% of the image), marked by a gray mask; includes resizing, inference, and blending | | Text embeddings, privacy filter | Fixed prose, with the PII text seeded with the entity types a detector should find | | VAD, speech-to-text | 10 s of a voice-shaped waveform: a 130 Hz pulse train through three formant resonators, 0.7 s bursts alternating with 0.3 s of near-silence | | Text-to-speech | Fixed text; the voice is the alphabetically first the config publishes, so the choice is a property of the voice set rather than of declaration order | diff --git a/apps/benchmarks/scripts/variant-sizes.json b/apps/benchmarks/scripts/variant-sizes.json index 9d495eb984..5977a18ee0 100644 --- a/apps/benchmarks/scripts/variant-sizes.json +++ b/apps/benchmarks/scripts/variant-sizes.json @@ -491,5 +491,14 @@ "https://huggingface.co/software-mansion/react-native-executorch-yolo26/resolve/v0.11.0/x/xnnpack/yolo26x_640_xnnpack_int8.pte": 66161216, "https://huggingface.co/software-mansion/react-native-executorch-zipdepth/resolve/v0.11.0/base/coreml/zipdepth_base_384_coreml_fp32.pte": 24858521, "https://huggingface.co/software-mansion/react-native-executorch-zipdepth/resolve/v0.11.0/base/vulkan/zipdepth_base_384_vulkan_fp16.pte": 12794626, - "https://huggingface.co/software-mansion/react-native-executorch-zipdepth/resolve/v0.11.0/base/xnnpack/zipdepth_base_384_xnnpack_fp32.pte": 24609344 + "https://huggingface.co/software-mansion/react-native-executorch-zipdepth/resolve/v0.11.0/base/xnnpack/zipdepth_base_384_xnnpack_fp32.pte": 24609344, + "https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/xnnpack/faces_256/migan_faces_256_xnnpack_fp16.pte": 24002700, + "https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/xnnpack/512/migan_512_xnnpack_fp16.pte": 25181068, + "https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/xnnpack/256/migan_256_xnnpack_fp16.pte": 24002700, + "https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/vulkan/faces_256/migan_faces_256_vulkan_fp16.pte": 12411650, + "https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/vulkan/512/migan_512_vulkan_fp16.pte": 14058114, + "https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/vulkan/256/migan_256_vulkan_fp16.pte": 12411650, + "https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/coreml/256/migan_256_coreml_fp16.pte": 12769586, + "https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/coreml/512/migan_512_coreml_fp16.pte": 13968400, + "https://huggingface.co/software-mansion/react-native-executorch-migan/resolve/v0.11.0/coreml/faces_256/migan_faces_256_coreml_fp16.pte": 12769586 } diff --git a/apps/benchmarks/src/drivers.ts b/apps/benchmarks/src/drivers.ts index f3da5e109d..09bf50c84f 100644 --- a/apps/benchmarks/src/drivers.ts +++ b/apps/benchmarks/src/drivers.ts @@ -19,6 +19,7 @@ import { createDepthEstimator, createFsmnVoiceActivityDetector, createImageEmbedder, + createInpainter, createInstanceSegmenter, createKeypointDetector, createKokoroTextToSpeech, @@ -45,6 +46,7 @@ import RNBlobUtil from 'react-native-blob-util'; import { IMAGE_512, IMAGE_640, + INPAINTING_MASK, LLM_MAX_NEW_TOKENS, LLM_PROMPT, SAMPLE_PII_TEXT, @@ -161,6 +163,14 @@ const depthEstimation: Driver = { }, }; +const inpainting: Driver = { + create: createInpainter, + run: (instance) => () => { + 'worklet'; + return instance.inpaintWorklet(IMAGE_512, INPAINTING_MASK).data.length; + }, +}; + const colbertEmbeddings: Driver = { create: createColbertEmbedder, run: (instance) => () => { @@ -378,6 +388,7 @@ const BY_TASK: Readonly> = { imageEmbeddings, textEmbeddings, depthEstimation, + inpainting, colbertEmbeddings, privacyFilter, voiceActivityDetection, diff --git a/apps/benchmarks/src/inputs.ts b/apps/benchmarks/src/inputs.ts index 30b274251b..5e5929c09f 100644 --- a/apps/benchmarks/src/inputs.ts +++ b/apps/benchmarks/src/inputs.ts @@ -201,6 +201,13 @@ export const INPUT_SPEC_VERSION = 1; * construction into the numbers. */ export const IMAGE_512 = syntheticImage(512, 512); + +/** Fixed central 256x256 hole, covering one quarter of the 512x512 frame. */ +export const INPAINTING_MASK: ImageBuffer & { readonly format: 'gray' } = (() => { + const data = new Uint8Array(512 * 512); + for (let y = 128; y < 384; y++) data.fill(255, y * 512 + 128, y * 512 + 384); + return { data, width: 512, height: 512, format: 'gray', layout: 'hwc' }; +})(); export const IMAGE_640 = syntheticImage(640, 640); /** Fixed text for the text-to-speech pipelines. Long enough to span chunks. */ diff --git a/docs/docs/04-benchmarks/01-benchmarks.mdx b/docs/docs/04-benchmarks/01-benchmarks.mdx index 9901f253af..adb36b5336 100644 --- a/docs/docs/04-benchmarks/01-benchmarks.mdx +++ b/docs/docs/04-benchmarks/01-benchmarks.mdx @@ -39,15 +39,15 @@ Explore the charts below to compare backends and find the optimal balance for yo ## Devices -Measurements were collected on physical iOS and Android devices spanning flagship, modern mid-tier, and legacy hardware tiers. All numbers come from release builds of React Native ExecuTorch 0.11.0-dev (`main` shortly after v0.10.0), measured in September 2026, with supplemental quantized LLM, depth estimation, and ColBERT runs in October 2026. +Measurements were collected on physical iOS and Android devices spanning flagship, modern mid-tier, and legacy hardware tiers. All numbers come from release builds of React Native ExecuTorch 0.11.0-dev (`main` shortly after v0.10.0), measured in September 2026, with supplemental quantized LLM, depth estimation, ColBERT, and inpainting runs in October 2026. | Device | Backends tested | Variants | Notes | |---|---|---|---| | Samsung Galaxy S26 Ultra | XNNPACK (121), Vulkan (24) | 145 | Snapdragon SM8850; historical standard-task results and 21 quantized LLM variants | | Google Pixel 10 | XNNPACK (100), Vulkan (24) | 124 | Google Tensor G5; historical standard-task measurements | -| Samsung Galaxy S20+ | XNNPACK (129), Vulkan (31) | 160 | Exynos 990; historical results plus supplemental quantized LLM, depth, and ColBERT measurements | -| Samsung Galaxy S24 | XNNPACK (29), Vulkan (6) | 35 | Supplemental quantized LLM, depth estimation, and ColBERT measurements | -| iPhone 17 | XNNPACK (127), Core ML (78), MLX (38) | 243 | A19; historical results plus supplemental quantized LLM, depth, and ColBERT measurements | +| Samsung Galaxy S20+ | XNNPACK (132), Vulkan (34) | 166 | Exynos 990; historical results plus supplemental quantized LLM, depth, ColBERT, and inpainting measurements | +| Samsung Galaxy S24 | XNNPACK (32), Vulkan (9) | 41 | Supplemental quantized LLM, depth estimation, ColBERT, and inpainting measurements | +| iPhone 17 | XNNPACK (130), Core ML (81), MLX (38) | 249 | A19; historical results plus supplemental quantized LLM, depth, ColBERT, and inpainting measurements | | iPhone SE (3rd gen) | XNNPACK (99), Core ML (73), MLX (15) | 187 | A15 Bionic with 4 GB RAM; historical standard-task measurements | Memory values report peak process footprint during a separate inference pass, including the app baseline: total PSS on Android and `phys_footprint` on iOS. Some supplemental iPhone LLM measurements use a fresh process for each model. @@ -126,6 +126,15 @@ These benchmarks measure execution latency, memory utilization, and initializati +### [Image Inpainting](../02-extensions/computer-vision/11-image-inpainting.md) + +:::info Input & Pipeline Details +- **Input:** Fixed 512x512 RGBA synthetic frame and a single-channel mask marking the central 256x256 square as the region to fill (25% of the image). +- **Pipeline Structure:** MI-GAN resizes the image and mask to the model's 256x256 or 512x512 input, normalizes the image, and reconstructs the masked region. The output is resized back to 512x512 and blended with the original image through the mask. Timings include preprocessing, inference, and blending for the Places2 and Faces checkpoints. +::: + + + ### [Image Embeddings](../02-extensions/computer-vision/09-image-embeddings.md) :::info Input & Pipeline Details diff --git a/docs/docs/04-benchmarks/measured.json b/docs/docs/04-benchmarks/measured.json index 069ffdb931..d903fc8bd7 100644 --- a/docs/docs/04-benchmarks/measured.json +++ b/docs/docs/04-benchmarks/measured.json @@ -1,4 +1,88 @@ [ + { + "task": "inpainting", + "model": "migan-size-256", + "backend": "coreml", + "precision": "fp16", + "device": "iphone17", + "medianMs": 13.42, + "meanMs": 13.46, + "etMs": 12.43, + "share": 92, + "peakMb": 89, + "loadMs": 41, + "sizeMb": 12 + }, + { + "task": "inpainting", + "model": "migan-faces-256", + "backend": "coreml", + "precision": "fp16", + "device": "iphone17", + "medianMs": 13.34, + "meanMs": 13.35, + "etMs": 12.39, + "share": 93, + "peakMb": 81, + "loadMs": 42, + "sizeMb": 12 + }, + { + "task": "inpainting", + "model": "migan-size-512", + "backend": "coreml", + "precision": "fp16", + "device": "iphone17", + "medianMs": 31.95, + "meanMs": 31.99, + "etMs": 30.07, + "share": 94, + "peakMb": 96, + "loadMs": 61, + "sizeMb": 13 + }, + { + "task": "inpainting", + "model": "migan-size-256", + "backend": "xnnpack", + "precision": "fp16", + "device": "iphone17", + "medianMs": 45.79, + "meanMs": 45.87, + "etMs": 44.95, + "share": 98, + "peakMb": 138, + "loadMs": 22, + "sizeMb": 23 + }, + { + "task": "inpainting", + "model": "migan-faces-256", + "backend": "xnnpack", + "precision": "fp16", + "device": "iphone17", + "medianMs": 46.52, + "meanMs": 46.5, + "etMs": 45.58, + "share": 98, + "peakMb": 137, + "loadMs": 21, + "sizeMb": 23 + }, + { + "task": "inpainting", + "model": "migan-size-512", + "backend": "xnnpack", + "precision": "fp16", + "device": "iphone17", + "medianMs": 83.5, + "meanMs": 83.13, + "etMs": 81.68, + "share": 98, + "peakMb": 262, + "loadMs": 24, + "sizeMb": 24 + }, { "task": "depth-estimation", "model": "zipdepth-base", @@ -741,6 +825,90 @@ "loadMs": 2636, "sizeMb": 3482 }, + { + "task": "inpainting", + "model": "migan-size-256", + "backend": "vulkan", + "precision": "fp16", + "device": "s20plus", + "medianMs": 250, + "meanMs": 251.52, + "etMs": 240.33, + "share": 96, + "peakMb": 210, + "loadMs": 48, + "sizeMb": 12 + }, + { + "task": "inpainting", + "model": "migan-faces-256", + "backend": "vulkan", + "precision": "fp16", + "device": "s20plus", + "medianMs": 248.69, + "meanMs": 249.05, + "etMs": 238.94, + "share": 96, + "peakMb": 190, + "loadMs": 32, + "sizeMb": 12 + }, + { + "task": "inpainting", + "model": "migan-size-512", + "backend": "vulkan", + "precision": "fp16", + "device": "s20plus", + "medianMs": 399.33, + "meanMs": 400.49, + "etMs": 380.53, + "share": 95, + "peakMb": 209, + "loadMs": 40, + "sizeMb": 13 + }, + { + "task": "inpainting", + "model": "migan-size-256", + "backend": "xnnpack", + "precision": "fp16", + "device": "s20plus", + "medianMs": 257.6, + "meanMs": 261.41, + "etMs": 253.74, + "share": 97, + "peakMb": 277, + "loadMs": 49, + "sizeMb": 23 + }, + { + "task": "inpainting", + "model": "migan-faces-256", + "backend": "xnnpack", + "precision": "fp16", + "device": "s20plus", + "medianMs": 262.43, + "meanMs": 263.2, + "etMs": 254.68, + "share": 97, + "peakMb": 283, + "loadMs": 49, + "sizeMb": 23 + }, + { + "task": "inpainting", + "model": "migan-size-512", + "backend": "xnnpack", + "precision": "fp16", + "device": "s20plus", + "medianMs": 364.97, + "meanMs": 364.18, + "etMs": 356.45, + "share": 98, + "peakMb": 416, + "loadMs": 49, + "sizeMb": 24 + }, { "task": "depth-estimation", "model": "zipdepth-base", @@ -1245,6 +1413,90 @@ "loadMs": 17602, "sizeMb": 3661 }, + { + "task": "inpainting", + "model": "migan-size-256", + "backend": "vulkan", + "precision": "fp16", + "device": "s24", + "medianMs": 90.57, + "meanMs": 91.49, + "etMs": 84.26, + "share": 92, + "peakMb": 349, + "loadMs": 26, + "sizeMb": 12 + }, + { + "task": "inpainting", + "model": "migan-faces-256", + "backend": "vulkan", + "precision": "fp16", + "device": "s24", + "medianMs": 90.9, + "meanMs": 91.6, + "etMs": 84.59, + "share": 92, + "peakMb": 329, + "loadMs": 37, + "sizeMb": 12 + }, + { + "task": "inpainting", + "model": "migan-size-512", + "backend": "vulkan", + "precision": "fp16", + "device": "s24", + "medianMs": 175.22, + "meanMs": 174.78, + "etMs": 165.84, + "share": 95, + "peakMb": 438, + "loadMs": 45, + "sizeMb": 13 + }, + { + "task": "inpainting", + "model": "migan-size-256", + "backend": "xnnpack", + "precision": "fp16", + "device": "s24", + "medianMs": 176.79, + "meanMs": 185.28, + "etMs": 181.29, + "share": 98, + "peakMb": 287, + "loadMs": 46, + "sizeMb": 23 + }, + { + "task": "inpainting", + "model": "migan-faces-256", + "backend": "xnnpack", + "precision": "fp16", + "device": "s24", + "medianMs": 188.48, + "meanMs": 188.45, + "etMs": 184.98, + "share": 98, + "peakMb": 296, + "loadMs": 48, + "sizeMb": 23 + }, + { + "task": "inpainting", + "model": "migan-size-512", + "backend": "xnnpack", + "precision": "fp16", + "device": "s24", + "medianMs": 235.15, + "meanMs": 234.74, + "etMs": 230.79, + "share": 98, + "peakMb": 436, + "loadMs": 29, + "sizeMb": 24 + }, { "task": "depth-estimation", "model": "zipdepth-base", diff --git a/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-migan-libs-0.10.7-20261027-ios-iPhone18-3.jsonl b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-migan-libs-0.10.7-20261027-ios-iPhone18-3.jsonl new file mode 100644 index 0000000000..e2f9a11c40 --- /dev/null +++ b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-migan-libs-0.10.7-20261027-ios-iPhone18-3.jsonl @@ -0,0 +1,7 @@ +{"type":"run","label":"v0.11.0-migan-libs-0.10.7-20261027","platform":"ios","device":{"totalMemoryBytes":8044216320,"isEmulator":false,"osVersion":"26.6","cpuCores":6,"model":"iPhone18,3","soc":"iPhone18,3"},"inputSpecVersion":1,"buildType":"release","clocksPinned":false,"settings":{"suite":"everything","iterations":20,"warmup":3,"memoryIterations":5,"loadIterations":3,"repeats":1,"maxTempC":37,"maxBytes":6000000000,"urlMap":{}},"startedAt":"2026-10-27T13:23:00.880Z"} +{"id":"inpainting/migan-size-256-coreml-fp16","task":"inpainting","model":"inpainting.MIGAN.SIZE_256.COREML_FP16","backend":"coreml","precision":"fp16","bytes":12769586,"mode":"worklet","progress":{"caseIndex":1,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1438.64,"taskLoadMs":41.463,"taskLoad":{"count":3,"min":40.893,"max":685.884,"mean":256.08,"median":41.463,"p90":685.884,"iqr":644.991,"stdev":303.917},"pipeline":{"count":20,"min":13.246,"max":13.852,"mean":13.458,"median":13.424,"p90":13.586,"iqr":0.149,"stdev":0.134},"execution":{"perIteration":{"forward":{"count":1,"ms":12.4283459}},"totalMs":12.4283459},"units":1048576,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","status":0,"batteryTemperatureC":-1},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":4},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":47.41,"loadedMb":72.49,"peakMb":88.91,"disposedMb":87.49}} +{"id":"inpainting/migan-faces-256-coreml-fp16","task":"inpainting","model":"inpainting.MIGAN.FACES_256.COREML_FP16","backend":"coreml","precision":"fp16","bytes":12769586,"mode":"worklet","progress":{"caseIndex":2,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1418.106,"taskLoadMs":41.815,"taskLoad":{"count":3,"min":41.253,"max":657.101,"mean":246.723,"median":41.815,"p90":657.101,"iqr":615.848,"stdev":290.181},"pipeline":{"count":20,"min":13.02,"max":13.685,"mean":13.351,"median":13.337,"p90":13.505,"iqr":0.099,"stdev":0.151},"execution":{"perIteration":{"forward":{"count":1,"ms":12.3859437}},"totalMs":12.3859437},"units":1048576,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"status":0,"statusName":"none"},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":4},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":75.83,"loadedMb":79.03,"peakMb":81.14,"disposedMb":79.72}} +{"id":"inpainting/migan-size-512-coreml-fp16","task":"inpainting","model":"inpainting.MIGAN.SIZE_512.COREML_FP16","backend":"coreml","precision":"fp16","bytes":13968400,"mode":"worklet","progress":{"caseIndex":3,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1447.5,"taskLoadMs":60.757,"taskLoad":{"count":3,"min":60.447,"max":788.478,"mean":303.227,"median":60.757,"p90":788.478,"iqr":728.031,"stdev":343.124},"pipeline":{"count":20,"min":31.563,"max":32.715,"mean":31.991,"median":31.947,"p90":32.408,"iqr":0.352,"stdev":0.304},"execution":{"perIteration":{"forward":{"count":1,"ms":30.073402299999998}},"totalMs":30.073402299999998},"units":1048576,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"status":0,"statusName":"none"},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":8},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":72.5,"loadedMb":92.16,"peakMb":96.19,"disposedMb":90.64}} +{"id":"inpainting/migan-size-256-xnnpack-fp16","task":"inpainting","model":"inpainting.MIGAN.SIZE_256.XNNPACK_FP16","backend":"xnnpack","precision":"fp16","bytes":24002700,"mode":"worklet","progress":{"caseIndex":4,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1726.685,"taskLoadMs":21.569,"taskLoad":{"count":3,"min":16.852,"max":53.109,"mean":30.51,"median":21.569,"p90":53.109,"iqr":36.257,"stdev":16.095},"pipeline":{"count":20,"min":44.776,"max":47.738,"mean":45.869,"median":45.786,"p90":46.879,"iqr":0.94,"stdev":0.765},"execution":{"perIteration":{"forward":{"count":1,"ms":44.95303745}},"totalMs":44.95303745},"units":1048576,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"statusName":"none","status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":12},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":73.14,"loadedMb":84.3,"peakMb":137.59,"disposedMb":129.99}} +{"id":"inpainting/migan-faces-256-xnnpack-fp16","task":"inpainting","model":"inpainting.MIGAN.FACES_256.XNNPACK_FP16","backend":"xnnpack","precision":"fp16","bytes":24002700,"mode":"worklet","progress":{"caseIndex":5,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1855.706,"taskLoadMs":21.449,"taskLoad":{"count":3,"min":16.347,"max":49.657,"mean":29.151,"median":21.449,"p90":49.657,"iqr":33.31,"stdev":14.649},"pipeline":{"count":20,"min":44.743,"max":47.521,"mean":46.498,"median":46.517,"p90":46.952,"iqr":0.386,"stdev":0.585},"execution":{"perIteration":{"forward":{"count":1,"ms":45.58426455}},"totalMs":45.58426455},"units":1048576,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"statusName":"none","batteryTemperatureC":-1,"status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":11},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":64.45,"loadedMb":72.11,"peakMb":136.53,"disposedMb":128.94}} +{"id":"inpainting/migan-size-512-xnnpack-fp16","task":"inpainting","model":"inpainting.MIGAN.SIZE_512.XNNPACK_FP16","backend":"xnnpack","precision":"fp16","bytes":25181068,"mode":"worklet","progress":{"caseIndex":6,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1810.732,"taskLoadMs":23.818,"taskLoad":{"count":3,"min":16.298,"max":51.373,"mean":30.496,"median":23.818,"p90":51.373,"iqr":35.075,"stdev":15.078},"pipeline":{"count":20,"min":80.235,"max":84.852,"mean":83.127,"median":83.497,"p90":84.383,"iqr":1.774,"stdev":1.173},"execution":{"perIteration":{"forward":{"count":1,"ms":81.67539165}},"totalMs":81.67539165},"units":1048576,"gate":{"kind":"device","waitedS":0,"temperatureC":null,"thermalStatus":0,"timedOut":false},"thermal":{"batteryTemperatureC":-1,"statusName":"none","status":0},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":null,"samples":19},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":56.31,"loadedMb":83.03,"peakMb":262.39,"disposedMb":254.75}} diff --git a/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-migan-libs-0.10.7-20261027-ios-iPhone18-3.notes.json b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-migan-libs-0.10.7-20261027-ios-iPhone18-3.notes.json new file mode 100644 index 0000000000..06ddd05652 --- /dev/null +++ b/docs/docs/04-benchmarks/runs/iphone17/v0.11.0-migan-libs-0.10.7-20261027-ios-iPhone18-3.notes.json @@ -0,0 +1,14 @@ +{ + "nativeLibsRelease": "v0.10.7-libs", + "source": "https://github.com/software-mansion/react-native-executorch/releases/tag/v0.10.7-libs", + "artifactSha256": { + "headers.tar.gz": "0a8a2380c1952e0284c2cdd831f5e2c062af056ace861f84671868170c174b73", + "coreml-ios.tar.gz": "01af0ceb04c92e915c7a5af54ac702045f8f5f1a53006e7be13638d84562c5a1", + "mlx-ios.tar.gz": "f1ff0526d7461a25e45e47c8883bc404479f4aa0d7314b0dbf3d98eb373ac9ef", + "core-ios.tar.gz": "d522dfd04bf096d6f9b330528c6e47cea5ecd96cf46a0af17af20d9051d828e5", + "xnnpack-ios.tar.gz": "09536717c94248d828e798ad7497029293fba1a889f6eabc0d37abf8f4a77e3e" + }, + "build": "Release; freshly downloaded artifacts verified against release checksums; native build outputs cleaned before rebuilding. S24 reused the refreshed Android native build with device-specific benchmark settings rebundled.", + "input": "Fixed 512x512 synthetic RGBA frame and a central 256x256 hole marked with a gray mask (25% of the image).", + "results": "All six platform-supported MIGAN variants completed with 20 timed iterations, 3 warmups, 3 load iterations and 5 memory iterations." +} diff --git a/docs/docs/04-benchmarks/runs/s20plus/v0.11.0-migan-libs-0.10.7-20261027-android-SM-G986B.jsonl b/docs/docs/04-benchmarks/runs/s20plus/v0.11.0-migan-libs-0.10.7-20261027-android-SM-G986B.jsonl new file mode 100644 index 0000000000..823420dd12 --- /dev/null +++ b/docs/docs/04-benchmarks/runs/s20plus/v0.11.0-migan-libs-0.10.7-20261027-android-SM-G986B.jsonl @@ -0,0 +1,7 @@ +{"type":"run","label":"v0.11.0-migan-libs-0.10.7-20261027","platform":"android","device":{"model":"SM-G986B","osVersion":"13","soc":"Exynos 990","isEmulator":false,"cpuCores":8,"totalMemoryBytes":11430510592},"inputSpecVersion":1,"buildType":"release","clocksPinned":false,"settings":{"suite":"everything","iterations":20,"warmup":3,"memoryIterations":5,"loadIterations":3,"repeats":1,"maxTempC":37,"maxBytes":6000000000,"urlMap":{}},"startedAt":"2026-10-27T13:26:32.916Z"} +{"id":"inpainting/migan-size-256-vulkan-fp16","task":"inpainting","model":"inpainting.MIGAN.SIZE_256.VULKAN_FP16","backend":"vulkan","precision":"fp16","bytes":12411650,"mode":"worklet","progress":{"caseIndex":1,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1859.687,"taskLoadMs":47.674,"taskLoad":{"count":3,"min":31.211,"max":570.163,"mean":216.349,"median":47.674,"p90":570.163,"iqr":538.952,"stdev":250.274},"pipeline":{"count":20,"min":245.253,"max":262.842,"mean":251.519,"median":249.999,"p90":257.775,"iqr":7.843,"stdev":5.316},"execution":{"perIteration":{"forward":{"count":1,"ms":240.33134034999998}},"totalMs":240.33134034999998},"units":1048576,"gate":{"kind":"host","waitedS":0,"temperatureC":27.1,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":27.3},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":27.3,"samples":56},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":166.43,"loadedMb":187.25,"peakMb":210.08,"disposedMb":177.69}} +{"id":"inpainting/migan-faces-256-vulkan-fp16","task":"inpainting","model":"inpainting.MIGAN.FACES_256.VULKAN_FP16","backend":"vulkan","precision":"fp16","bytes":12411650,"mode":"worklet","progress":{"caseIndex":2,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":830.271,"taskLoadMs":32.425,"taskLoad":{"count":3,"min":28.275,"max":34.272,"mean":31.657,"median":32.425,"p90":34.272,"iqr":5.998,"stdev":2.508},"pipeline":{"count":20,"min":244.566,"max":254.869,"mean":249.051,"median":248.694,"p90":253.169,"iqr":4.465,"stdev":2.93},"execution":{"perIteration":{"forward":{"count":1,"ms":238.93624595}},"totalMs":238.93624595},"units":1048576,"gate":{"kind":"host","waitedS":0,"temperatureC":27.3,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":27.9},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":27.9,"samples":55},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":176.05,"loadedMb":172.54,"peakMb":190.33,"disposedMb":189.82}} +{"id":"inpainting/migan-size-512-vulkan-fp16","task":"inpainting","model":"inpainting.MIGAN.SIZE_512.VULKAN_FP16","backend":"vulkan","precision":"fp16","bytes":14058114,"mode":"worklet","progress":{"caseIndex":3,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":967.118,"taskLoadMs":39.596,"taskLoad":{"count":3,"min":38.636,"max":109.312,"mean":62.515,"median":39.596,"p90":109.312,"iqr":70.677,"stdev":33.093},"pipeline":{"count":20,"min":392.347,"max":422.624,"mean":400.486,"median":399.329,"p90":406.521,"iqr":6.493,"stdev":6.575},"execution":{"perIteration":{"forward":{"count":1,"ms":380.52785365}},"totalMs":380.52785365},"units":1048576,"gate":{"kind":"host","waitedS":0,"temperatureC":27.9,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":28.5},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":28.5,"samples":87},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":191.09,"loadedMb":208.55,"peakMb":209.08,"disposedMb":207.57}} +{"id":"inpainting/migan-size-256-xnnpack-fp16","task":"inpainting","model":"inpainting.MIGAN.SIZE_256.XNNPACK_FP16","backend":"xnnpack","precision":"fp16","bytes":24002700,"mode":"worklet","progress":{"caseIndex":4,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1153.875,"taskLoadMs":48.644,"taskLoad":{"count":3,"min":46.123,"max":80.983,"mean":58.583,"median":48.644,"p90":80.983,"iqr":34.859,"stdev":15.872},"pipeline":{"count":20,"min":242.26,"max":286.647,"mean":261.407,"median":257.595,"p90":275.104,"iqr":17.863,"stdev":12.414},"execution":{"perIteration":{"forward":{"count":1,"ms":253.73963444999998}},"totalMs":253.73963444999998},"units":1048576,"gate":{"kind":"host","waitedS":0,"temperatureC":29,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":29},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":29,"samples":58},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":210.24,"loadedMb":196.5,"peakMb":276.6,"disposedMb":211.12}} +{"id":"inpainting/migan-faces-256-xnnpack-fp16","task":"inpainting","model":"inpainting.MIGAN.FACES_256.XNNPACK_FP16","backend":"xnnpack","precision":"fp16","bytes":24002700,"mode":"worklet","progress":{"caseIndex":5,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1318.975,"taskLoadMs":49.384,"taskLoad":{"count":3,"min":46.466,"max":55.006,"mean":50.286,"median":49.384,"p90":55.006,"iqr":8.54,"stdev":3.544},"pipeline":{"count":20,"min":222.536,"max":289.254,"mean":263.204,"median":262.43,"p90":284.103,"iqr":26.629,"stdev":18.385},"execution":{"perIteration":{"forward":{"count":1,"ms":254.68301515}},"totalMs":254.68301515},"units":1048576,"gate":{"kind":"host","waitedS":0,"temperatureC":29,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":29.5},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":29.5,"samples":58},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":213.89,"loadedMb":196.66,"peakMb":282.6,"disposedMb":217.15}} +{"id":"inpainting/migan-size-512-xnnpack-fp16","task":"inpainting","model":"inpainting.MIGAN.SIZE_512.XNNPACK_FP16","backend":"xnnpack","precision":"fp16","bytes":25181068,"mode":"worklet","progress":{"caseIndex":6,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1335.73,"taskLoadMs":49.1,"taskLoad":{"count":3,"min":47.776,"max":60.488,"mean":52.455,"median":49.1,"p90":60.488,"iqr":12.712,"stdev":5.706},"pipeline":{"count":20,"min":335.07,"max":392.739,"mean":364.176,"median":364.968,"p90":381.099,"iqr":25.228,"stdev":15.775},"execution":{"perIteration":{"forward":{"count":1,"ms":356.44624795}},"totalMs":356.44624795},"units":1048576,"gate":{"kind":"host","waitedS":0,"temperatureC":29.5,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":30.4},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":30,"samples":80},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":214.06,"loadedMb":221.7,"peakMb":416.39,"disposedMb":269.29}} diff --git a/docs/docs/04-benchmarks/runs/s20plus/v0.11.0-migan-libs-0.10.7-20261027-android-SM-G986B.notes.json b/docs/docs/04-benchmarks/runs/s20plus/v0.11.0-migan-libs-0.10.7-20261027-android-SM-G986B.notes.json new file mode 100644 index 0000000000..851fef2962 --- /dev/null +++ b/docs/docs/04-benchmarks/runs/s20plus/v0.11.0-migan-libs-0.10.7-20261027-android-SM-G986B.notes.json @@ -0,0 +1,14 @@ +{ + "nativeLibsRelease": "v0.10.7-libs", + "source": "https://github.com/software-mansion/react-native-executorch/releases/tag/v0.10.7-libs", + "artifactSha256": { + "headers.tar.gz": "0a8a2380c1952e0284c2cdd831f5e2c062af056ace861f84671868170c174b73", + "vulkan-android-arm64-v8a.tar.gz": "a7ad34ac5f41146c339ace2cd808173973d3703893cc567edf41bb3564642ba1", + "core-android-arm64-v8a.tar.gz": "0bb757a4cfd06aee282018d98a9904f55bb0cc1fff5bcc16bb9a8a9a64a90145", + "xnnpack-android-arm64-v8a.tar.gz": "c50e17ccfe0feff0357aef6745f32c2980b029c6a49ec0e16bcd7f0c1c94baba", + "opencv-android-arm64-v8a.tar.gz": "1db60340773320b4f9bda166f407cf5d24153269f08c39a6af25eb42aec2835a" + }, + "build": "Release; freshly downloaded artifacts verified against release checksums; native build outputs cleaned before rebuilding. S24 reused the refreshed Android native build with device-specific benchmark settings rebundled.", + "input": "Fixed 512x512 synthetic RGBA frame and a central 256x256 hole marked with a gray mask (25% of the image).", + "results": "All six platform-supported MIGAN variants completed with 20 timed iterations, 3 warmups, 3 load iterations and 5 memory iterations." +} diff --git a/docs/docs/04-benchmarks/runs/s24/v0.11.0-migan-libs-0.10.7-20261027-android-SM-S921B.jsonl b/docs/docs/04-benchmarks/runs/s24/v0.11.0-migan-libs-0.10.7-20261027-android-SM-S921B.jsonl new file mode 100644 index 0000000000..5bae53c5a3 --- /dev/null +++ b/docs/docs/04-benchmarks/runs/s24/v0.11.0-migan-libs-0.10.7-20261027-android-SM-S921B.jsonl @@ -0,0 +1,7 @@ +{"type":"run","label":"v0.11.0-migan-libs-0.10.7-20261027","platform":"android","device":{"model":"SM-S921B","osVersion":"16","soc":"s5e9945","isEmulator":false,"cpuCores":10,"totalMemoryBytes":7574827008},"inputSpecVersion":1,"buildType":"release","clocksPinned":false,"settings":{"suite":"everything","iterations":20,"warmup":3,"memoryIterations":5,"loadIterations":3,"repeats":1,"maxTempC":37,"maxBytes":6000000000,"urlMap":{}},"startedAt":"2026-10-27T13:27:06.777Z"} +{"id":"inpainting/migan-size-256-vulkan-fp16","task":"inpainting","model":"inpainting.MIGAN.SIZE_256.VULKAN_FP16","backend":"vulkan","precision":"fp16","bytes":12411650,"mode":"worklet","progress":{"caseIndex":1,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1492.362,"taskLoadMs":26.317,"taskLoad":{"count":3,"min":24.986,"max":383.055,"mean":144.786,"median":26.317,"p90":383.055,"iqr":358.07,"stdev":168.483},"pipeline":{"count":20,"min":87.326,"max":115.564,"mean":91.486,"median":90.572,"p90":92.1,"iqr":1.721,"stdev":5.687},"execution":{"perIteration":{"forward":{"count":1,"ms":84.2633769}},"totalMs":84.2633769},"units":1048576,"gate":{"kind":"host","waitedS":0,"temperatureC":26.5,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":27.2},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":27.2,"samples":22},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":176.81,"loadedMb":331.51,"peakMb":348.59,"disposedMb":209.59}} +{"id":"inpainting/migan-faces-256-vulkan-fp16","task":"inpainting","model":"inpainting.MIGAN.FACES_256.VULKAN_FP16","backend":"vulkan","precision":"fp16","bytes":12411650,"mode":"worklet","progress":{"caseIndex":2,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":854.898,"taskLoadMs":36.507,"taskLoad":{"count":3,"min":31.813,"max":38.804,"mean":35.708,"median":36.507,"p90":38.804,"iqr":6.991,"stdev":2.909},"pipeline":{"count":20,"min":89.026,"max":104.793,"mean":91.602,"median":90.903,"p90":92.639,"iqr":1.787,"stdev":3.24},"execution":{"perIteration":{"forward":{"count":1,"ms":84.5885547}},"totalMs":84.5885547},"units":1048576,"gate":{"kind":"host","waitedS":0,"temperatureC":26.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":27.2},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":27.2,"samples":19},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":211.52,"loadedMb":311.29,"peakMb":328.59,"disposedMb":223.49}} +{"id":"inpainting/migan-size-512-vulkan-fp16","task":"inpainting","model":"inpainting.MIGAN.SIZE_512.VULKAN_FP16","backend":"vulkan","precision":"fp16","bytes":14058114,"mode":"worklet","progress":{"caseIndex":3,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1074.956,"taskLoadMs":45.308,"taskLoad":{"count":3,"min":44.594,"max":74.808,"mean":54.903,"median":45.308,"p90":74.808,"iqr":30.215,"stdev":14.078},"pipeline":{"count":20,"min":167.157,"max":182.003,"mean":174.783,"median":175.217,"p90":178.362,"iqr":4.34,"stdev":3.106},"execution":{"perIteration":{"forward":{"count":1,"ms":165.83594920000002}},"totalMs":165.83594920000002},"units":1048576,"gate":{"kind":"host","waitedS":0,"temperatureC":26.7,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":28},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":27.2,"samples":35},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":234.47,"loadedMb":435.11,"peakMb":438.13,"disposedMb":243.21}} +{"id":"inpainting/migan-size-256-xnnpack-fp16","task":"inpainting","model":"inpainting.MIGAN.SIZE_256.XNNPACK_FP16","backend":"xnnpack","precision":"fp16","bytes":24002700,"mode":"worklet","progress":{"caseIndex":4,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1201.58,"taskLoadMs":46.079,"taskLoad":{"count":3,"min":27.183,"max":55.685,"mean":42.982,"median":46.079,"p90":55.685,"iqr":28.502,"stdev":11.84},"pipeline":{"count":20,"min":132.396,"max":243.13,"mean":185.284,"median":176.792,"p90":223.849,"iqr":51.378,"stdev":31.243},"execution":{"perIteration":{"forward":{"count":1,"ms":181.29225194999998}},"totalMs":181.29225194999998},"units":1048576,"gate":{"kind":"host","waitedS":0,"temperatureC":28,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":28.6},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":28.6,"samples":39},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":235.76,"loadedMb":218.18,"peakMb":286.54,"disposedMb":236.81}} +{"id":"inpainting/migan-faces-256-xnnpack-fp16","task":"inpainting","model":"inpainting.MIGAN.FACES_256.XNNPACK_FP16","backend":"xnnpack","precision":"fp16","bytes":24002700,"mode":"worklet","progress":{"caseIndex":5,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1052.767,"taskLoadMs":47.552,"taskLoad":{"count":3,"min":26.572,"max":50.327,"mean":41.484,"median":47.552,"p90":50.327,"iqr":23.756,"stdev":10.605},"pipeline":{"count":20,"min":132.139,"max":247.519,"mean":188.448,"median":188.482,"p90":242.01,"iqr":45.23,"stdev":32.849},"execution":{"perIteration":{"forward":{"count":1,"ms":184.9795332}},"totalMs":184.9795332},"units":1048576,"gate":{"kind":"host","waitedS":0,"temperatureC":28.6,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":30.2},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":28.6,"samples":36},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":245.82,"loadedMb":227.98,"peakMb":295.54,"disposedMb":231.26}} +{"id":"inpainting/migan-size-512-xnnpack-fp16","task":"inpainting","model":"inpainting.MIGAN.SIZE_512.XNNPACK_FP16","backend":"xnnpack","precision":"fp16","bytes":25181068,"mode":"worklet","progress":{"caseIndex":6,"caseCount":6,"repeat":1,"repeats":1},"status":"ok","downloadMs":1120.705,"taskLoadMs":28.883,"taskLoad":{"count":3,"min":28.401,"max":38.642,"mean":31.975,"median":28.883,"p90":38.642,"iqr":10.241,"stdev":4.718},"pipeline":{"count":20,"min":193.254,"max":262.467,"mean":234.744,"median":235.149,"p90":256.179,"iqr":28.645,"stdev":19.046},"execution":{"perIteration":{"forward":{"count":1,"ms":230.7937871}},"totalMs":230.7937871},"units":1048576,"gate":{"kind":"host","waitedS":0,"temperatureC":30.2,"thermalStatus":0,"timedOut":false},"thermal":{"status":0,"statusName":"none","batteryTemperatureC":30.2},"thermalPeak":{"status":0,"statusName":"none","batteryTemperatureC":30.2,"samples":48},"thermalValid":true,"holdsTimedOut":0,"coolingMs":0,"memory":{"baselineMb":245.49,"loadedMb":252.51,"peakMb":435.8,"disposedMb":303.72}} diff --git a/docs/docs/04-benchmarks/runs/s24/v0.11.0-migan-libs-0.10.7-20261027-android-SM-S921B.notes.json b/docs/docs/04-benchmarks/runs/s24/v0.11.0-migan-libs-0.10.7-20261027-android-SM-S921B.notes.json new file mode 100644 index 0000000000..851fef2962 --- /dev/null +++ b/docs/docs/04-benchmarks/runs/s24/v0.11.0-migan-libs-0.10.7-20261027-android-SM-S921B.notes.json @@ -0,0 +1,14 @@ +{ + "nativeLibsRelease": "v0.10.7-libs", + "source": "https://github.com/software-mansion/react-native-executorch/releases/tag/v0.10.7-libs", + "artifactSha256": { + "headers.tar.gz": "0a8a2380c1952e0284c2cdd831f5e2c062af056ace861f84671868170c174b73", + "vulkan-android-arm64-v8a.tar.gz": "a7ad34ac5f41146c339ace2cd808173973d3703893cc567edf41bb3564642ba1", + "core-android-arm64-v8a.tar.gz": "0bb757a4cfd06aee282018d98a9904f55bb0cc1fff5bcc16bb9a8a9a64a90145", + "xnnpack-android-arm64-v8a.tar.gz": "c50e17ccfe0feff0357aef6745f32c2980b029c6a49ec0e16bcd7f0c1c94baba", + "opencv-android-arm64-v8a.tar.gz": "1db60340773320b4f9bda166f407cf5d24153269f08c39a6af25eb42aec2835a" + }, + "build": "Release; freshly downloaded artifacts verified against release checksums; native build outputs cleaned before rebuilding. S24 reused the refreshed Android native build with device-specific benchmark settings rebundled.", + "input": "Fixed 512x512 synthetic RGBA frame and a central 256x256 hole marked with a gray mask (25% of the image).", + "results": "All six platform-supported MIGAN variants completed with 20 timed iterations, 3 warmups, 3 load iterations and 5 memory iterations." +}