Skip to content

Commit d76ceaf

Browse files
msluszniakmsluszniak
authored
feat: add Qwen3-ASR speech-to-text (#1554)
## Description - New `createQwen3AsrSpeechToText` task for [Qwen3-ASR 0.6B](https://huggingface.co/Qwen/Qwen3-ASR-0.6B), same `transcribe`/`stream` API as Whisper; language is auto-detected or forced via `QWEN3_ASR_LANGUAGES`. - `useSpeechToText` accepts Qwen3-ASR models; registry entry `models.speechToText.QWEN3_ASR.SIZE_0_6B` (XNNPACK 8da4w; iOS default Core ML audio tower + MLX decoder; HF `react-native-executorch-qwen-3-asr` @ v0.11.0). - A variant key naming two backends (`MLX_COREML_INT4`) is the default only when both are linked; loading a method whose backend is not linked now names it. - Speech demo screens list the model; docs section added. ### Introduces a breaking change? - [ ] Yes - [x] No ### Type of change - [ ] Bug fix (change which fixes an issue) - [x] New feature (change which adds functionality) - [x] Documentation update (improves or adds clarity to existing documentation) - [ ] Other (chores, tests, code style improvements etc.) ### Tested on - [x] iOS - [x] Android ### Testing instructions - `apps/speech`: pick "Qwen3-ASR 0.6B (CPU)" on the audio file or microphone screen and transcribe. ### Screenshots ### Related issues #769 ### Checklist - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have updated the documentation accordingly - [x] My changes generate no new warnings ### Additional notes - WER on 100 LibriSpeech test-clean clips: 2.46% (fp32 2.14%). Decoder layer 0 must stay 8-bit or transcripts derail. - iPhone 16: Core ML + MLX 293 ms per clip vs XNNPACK 468, WER 2.51%. - No Vulkan: on a Galaxy S26 Ultra it wins prefill but loses encoder and decode (whole clip 451 vs 360 ms). - Exporters and findings: export-scripts !41. ExecuTorch fixes found on the way: pytorch/executorch#24055, #24056.
1 parent 9eaf9ac commit d76ceaf

14 files changed

Lines changed: 1197 additions & 43 deletions

File tree

‎apps/speech/app/audio-file-transcription/index.tsx‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,12 @@ import {
88
TouchableOpacity,
99
TextInput,
1010
} from 'react-native';
11-
import { useSpeechToText, models, WHISPER_SAMPLE_RATE_HZ } from 'react-native-executorch';
11+
import {
12+
useSpeechToText,
13+
models,
14+
QWEN3_ASR_LANGUAGES,
15+
WHISPER_SAMPLE_RATE_HZ,
16+
} from 'react-native-executorch';
1217
import { decodeAudioData } from 'react-native-audio-api';
1318

1419
import ScreenWrapper from '../../components/ScreenWrapper';
@@ -126,8 +131,24 @@ const MODELS = [
126131
config: models.speechToText.WHISPER.SMALL.MLX_INT8,
127132
disabled: Platform.OS !== 'ios',
128133
},
134+
{
135+
name: 'Qwen3-ASR 0.6B (CPU)',
136+
config: models.speechToText.QWEN3_ASR.SIZE_0_6B.XNNPACK_8DA4W,
137+
},
138+
{
139+
name: 'Qwen3-ASR 0.6B (MLX + Core ML)',
140+
config: models.speechToText.QWEN3_ASR.SIZE_0_6B.MLX_COREML_INT4,
141+
disabled: Platform.OS !== 'ios',
142+
},
129143
];
130144

145+
// Qwen3-ASR detects the language itself; 'auto' leaves it to the model.
146+
const AUTO = 'auto';
147+
const languagesOf = (config: (typeof MODELS)[number]['config']): readonly string[] =>
148+
'supportedLanguages' in config
149+
? config.supportedLanguages
150+
: [AUTO, ...Object.keys(QWEN3_ASR_LANGUAGES)];
151+
131152
function STTAudioContent() {
132153
const [selectedModel, setSelectedModel] = useState(MODELS[0]!);
133154
const [selectedLanguage, setSelectedLanguage] = useState<string>('en');
@@ -145,7 +166,7 @@ function STTAudioContent() {
145166
error: modelError,
146167
} = useSpeechToText(selectedModel.config);
147168

148-
const supportedLanguages = selectedModel.config.supportedLanguages;
169+
const supportedLanguages = languagesOf(selectedModel.config);
149170

150171
useEffect(() => {
151172
return () => {
@@ -167,7 +188,8 @@ function STTAudioContent() {
167188
setStatus('Transcribing...');
168189

169190
let currentText = '';
170-
const text = await transcribe(samples, { language: selectedLanguage as any }, (token) => {
191+
const language = selectedLanguage === AUTO ? undefined : selectedLanguage;
192+
const text = await transcribe(samples, { language: language as any }, (token) => {
171193
currentText += token;
172194
setAudioText(currentText);
173195
});
@@ -196,7 +218,7 @@ function STTAudioContent() {
196218
<ScrollView style={styles.container} contentContainerStyle={styles.content}>
197219
<View style={styles.card}>
198220
<ModelPicker
199-
label="Whisper Model"
221+
label="Speech-to-Text Model"
200222
options={MODELS.map((m) => ({
201223
label: m.name,
202224
value: m,
@@ -205,14 +227,14 @@ function STTAudioContent() {
205227
selectedValue={selectedModel}
206228
onValueChange={(m) => {
207229
setSelectedModel(m);
208-
setSelectedLanguage(m.config.supportedLanguages[0]!);
230+
setSelectedLanguage(languagesOf(m.config)[0]!);
209231
}}
210232
/>
211233
<ModelStatus
212234
isReady={isSttReady}
213235
downloadProgress={downloadProgress}
214236
error={modelError ? modelError.message : null}
215-
modelTypeLabel="Whisper model"
237+
modelTypeLabel="Speech-to-text model"
216238
/>
217239
{supportedLanguages.length > 1 && (
218240
<ModelPicker

‎apps/speech/app/microphone-transcription/index.tsx‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
import React, { useEffect, useRef, useState } from 'react';
22
import { Platform, View, Text, StyleSheet, ScrollView, TouchableOpacity } from 'react-native';
3-
import { useSpeechToText, models, WHISPER_SAMPLE_RATE_HZ } from 'react-native-executorch';
3+
import {
4+
useSpeechToText,
5+
models,
6+
QWEN3_ASR_LANGUAGES,
7+
WHISPER_SAMPLE_RATE_HZ,
8+
} from 'react-native-executorch';
49
import { AudioManager, AudioRecorder } from 'react-native-audio-api';
510
import DeviceInfo from 'react-native-device-info';
611

@@ -119,8 +124,24 @@ const MODELS = [
119124
config: models.speechToText.WHISPER.SMALL.MLX_INT8,
120125
disabled: Platform.OS !== 'ios',
121126
},
127+
{
128+
name: 'Qwen3-ASR 0.6B (CPU)',
129+
config: models.speechToText.QWEN3_ASR.SIZE_0_6B.XNNPACK_8DA4W,
130+
},
131+
{
132+
name: 'Qwen3-ASR 0.6B (MLX + Core ML)',
133+
config: models.speechToText.QWEN3_ASR.SIZE_0_6B.MLX_COREML_INT4,
134+
disabled: Platform.OS !== 'ios',
135+
},
122136
];
123137

138+
// Qwen3-ASR detects the language itself; 'auto' leaves it to the model.
139+
const AUTO = 'auto';
140+
const languagesOf = (config: (typeof MODELS)[number]['config']): readonly string[] =>
141+
'supportedLanguages' in config
142+
? config.supportedLanguages
143+
: [AUTO, ...Object.keys(QWEN3_ASR_LANGUAGES)];
144+
124145
const isSimulator = DeviceInfo.isEmulatorSync();
125146

126147
function STTContent() {
@@ -141,7 +162,7 @@ function STTContent() {
141162
error: modelError,
142163
} = useSpeechToText(selectedModel.config);
143164

144-
const supportedLanguages = selectedModel.config.supportedLanguages;
165+
const supportedLanguages = languagesOf(selectedModel.config);
145166

146167
const recorderRef = useRef<AudioRecorder | null>(null);
147168

@@ -168,7 +189,8 @@ function STTContent() {
168189

169190
(async () => {
170191
try {
171-
const textStream = stream({ language: selectedLanguage as any });
192+
const language = selectedLanguage === AUTO ? undefined : selectedLanguage;
193+
const textStream = stream({ language: language as any });
172194
for await (const result of textStream) {
173195
setCommittedText(result.committed);
174196
setNonCommittedText(result.nonCommitted);
@@ -216,7 +238,7 @@ function STTContent() {
216238
<ScrollView style={styles.container} contentContainerStyle={styles.content}>
217239
<View style={styles.card}>
218240
<ModelPicker
219-
label="Whisper Model"
241+
label="Speech-to-Text Model"
220242
options={MODELS.map((m) => ({
221243
label: m.name,
222244
value: m,
@@ -225,14 +247,14 @@ function STTContent() {
225247
selectedValue={selectedModel}
226248
onValueChange={(m) => {
227249
setSelectedModel(m);
228-
setSelectedLanguage(m.config.supportedLanguages[0]!);
250+
setSelectedLanguage(languagesOf(m.config)[0]!);
229251
}}
230252
/>
231253
<ModelStatus
232254
isReady={isSttReady}
233255
downloadProgress={downloadProgress}
234256
error={modelError ? modelError.message : null}
235-
modelTypeLabel="Whisper model"
257+
modelTypeLabel="Speech-to-text model"
236258
/>
237259
{supportedLanguages.length > 1 && (
238260
<ModelPicker

‎docs/docs/02-extensions/speech/03-speech-to-text.md‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
---
22
title: Speech-to-Text (STT)
33
slug: /extensions/speech-to-text
4-
description: 'Transcribe spoken audio in real time on-device in React Native using OpenAI Whisper with integrated Voice Activity Detection.'
4+
description: 'Transcribe spoken audio in real time on-device in React Native using OpenAI Whisper or Qwen3-ASR with integrated Voice Activity Detection.'
55
keywords:
66
[
77
react native,
88
speech to text,
99
stt,
1010
asr,
1111
whisper,
12+
qwen3-asr,
1213
voice recognition,
1314
transcription,
1415
vad,
@@ -20,7 +21,7 @@ keywords:
2021

2122
# Speech-to-Text (STT)
2223

23-
The Speech-to-Text extension transcribes spoken audio into text directly on-device using OpenAI's Whisper model paired with an integrated FSMN Voice Activity Detector (VAD).
24+
The Speech-to-Text extension transcribes spoken audio into text directly on-device using OpenAI's Whisper or Alibaba's [Qwen3-ASR](https://huggingface.co/Qwen/Qwen3-ASR-0.6B) paired with an integrated FSMN Voice Activity Detector (VAD).
2425

2526
The pipeline supports two primary workflows:
2627

@@ -206,18 +207,20 @@ See [Worklets & Threading](../../03-core-and-advanced/06-worklets-and-threading.
206207

207208
The library provides ready-to-use Whisper models from the [Software Mansion HuggingFace Whisper Collection](https://huggingface.co/collections/software-mansion/whisper), available in [`models.speechToText`](../../06-api-reference/variables/models.md#speechtotext):
208209

209-
| Model Family | Variants | Size Range | Supported Backends | Languages | Notes |
210-
| :---------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :----------------- | :------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------- |
211-
| **Whisper Tiny** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispertiny), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperentiny) | 57.1 MB – 221.8 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Ultra-fast transcription with minimal RAM usage. |
212-
| **Whisper Base** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhisperbase), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperenbase) | 97.8 MB – 380.2 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Balanced accuracy and speed for general voice dictation. |
213-
| **Whisper Small** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispersmall), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperensmall) | 276.0 MB – 1.05 GB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | High-capacity model for complex, noisy, or multi-speaker audio. |
210+
| Model Family | Variants | Size Range | Supported Backends | Languages | Notes |
211+
| :----------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :------------------ | :------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------- |
212+
| **Whisper Tiny** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispertiny), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperentiny) | 57.1 MB – 221.8 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Ultra-fast transcription with minimal RAM usage. |
213+
| **Whisper Base** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhisperbase), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperenbase) | 97.8 MB – 380.2 MB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | Balanced accuracy and speed for general voice dictation. |
214+
| **Whisper Small** | [`Multilingual`](../../06-api-reference/variables/models.md#speechtotextwhispersmall), [`English`](../../06-api-reference/variables/models.md#speechtotextwhisperensmall) | 276.0 MB – 1.05 GB | XNNPACK (CPU), Core ML (Apple), MLX (Apple), Vulkan (Android) | English / [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) (99+ languages) | High-capacity model for complex, noisy, or multi-speaker audio. |
215+
| **Qwen3-ASR 0.6B** | [`0.6B`](../../06-api-reference/variables/models.md#speechtotextqwen3_asr) | 707.7 MB – 808.1 MB | XNNPACK (CPU), Core ML + MLX (Apple) | [`QWEN3_ASR_LANGUAGES`](../../06-api-reference/variables/QWEN3_ASR_LANGUAGES.md) (30 languages) | Detects the language; pass `language` only to force one. |
214216

215217
## API Reference
216218

217219
### Hooks & Pipelines
218220

219221
- [`useSpeechToText()`](../../06-api-reference/functions/useSpeechToText.md) — React hook for Whisper model loading, downloading, and live transcription state.
220222
- [`createWhisperSpeechToText()`](../../06-api-reference/functions/createWhisperSpeechToText.md) — Imperative factory for Whisper Speech-to-Text pipelines.
223+
- [`createQwen3AsrSpeechToText()`](../../06-api-reference/functions/createQwen3AsrSpeechToText.md) — Imperative factory for Qwen3-ASR Speech-to-Text pipelines.
221224

222225
### Types & Options
223226

@@ -226,15 +229,18 @@ The library provides ready-to-use Whisper models from the [Software Mansion Hugg
226229
- [`WhisperSttOptions`](../../06-api-reference/type-aliases/WhisperSttOptions.md) — Per-call transcription options (`language`).
227230
- [`WhisperStreamOptions`](../../06-api-reference/type-aliases/WhisperStreamOptions.md) — Live microphone streaming options (`language`, `vadOptions`).
228231
- [`WhisperLanguage`](../../06-api-reference/type-aliases/WhisperLanguage.md) — Union of supported Whisper language codes.
232+
- [`Qwen3AsrSttModel`](../../06-api-reference/type-aliases/Qwen3AsrSttModel.md), [`Qwen3AsrSttOptions`](../../06-api-reference/type-aliases/Qwen3AsrSttOptions.md), [`Qwen3AsrLanguage`](../../06-api-reference/type-aliases/Qwen3AsrLanguage.md) — Qwen3-ASR counterparts.
229233

230234
### Constants & Model Presets
231235

232236
- [`WHISPER_SAMPLE_RATE_HZ`](../../06-api-reference/variables/WHISPER_SAMPLE_RATE_HZ.md) — Target audio sample rate expected by Whisper models (16000 Hz).
233237
- [`WHISPER_LANGUAGES`](../../06-api-reference/variables/WHISPER_LANGUAGES.md) — Array of 99+ supported language codes.
238+
- [`QWEN3_ASR_LANGUAGES`](../../06-api-reference/variables/QWEN3_ASR_LANGUAGES.md) — Language codes Qwen3-ASR can be forced to.
234239
- [`models.speechToText`](../../06-api-reference/variables/models.md#speechtotext) — Pre-configured Whisper models registry.
235240

236241
:::info Source Code
237242
View the implementation on GitHub:
238243

239244
- [`src/extensions/speech/tasks/whisperSpeechToText.ts` ↗](https://github.com/software-mansion/react-native-executorch/blob/main/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts)
245+
- [`src/extensions/speech/tasks/qwen3AsrSpeechToText.ts` ↗](https://github.com/software-mansion/react-native-executorch/blob/main/packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts)
240246
:::

‎packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,8 @@ exports[`public API surface matches the recorded export list 1`] = `
129129
"PASCAL_VOC_LABELS",
130130
"PRIVACY_FILTER_NEMOTRON_LABELS",
131131
"PRIVACY_FILTER_OPENAI_LABELS",
132+
"QWEN3_ASR_LANGUAGES",
133+
"QWEN3_ASR_SAMPLE_RATE_HZ",
132134
"RnExecuTorchError",
133135
"SUPERTONIC_DEFAULT_VOICE_NAMES",
134136
"SUPERTONIC_SAMPLE_RATE",
@@ -148,6 +150,7 @@ exports[`public API surface matches the recorded export list 1`] = `
148150
"createObjectDetector",
149151
"createPaddleOcr",
150152
"createPrivacyFilter",
153+
"createQwen3AsrSpeechToText",
151154
"createResourceScope",
152155
"createSdxsTextToImage",
153156
"createSemanticSegmenter",

‎packages/react-native-executorch/__tests__/api/modelVariants.test.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,17 @@ describe('variant selection rules', () => {
325325
expect(defaultKeyOf(distiluse(registry))).toBe('XNNPACK_8DA4W');
326326
});
327327

328+
it('picks a variant naming two backends only when both are linked', () => {
329+
// Qwen3-ASR's MLX file runs its audio tower on Core ML.
330+
const qwen3Asr = (registry: Node): Node =>
331+
((registry.speechToText as Node).QWEN3_ASR as Node).SIZE_0_6B as Node;
332+
333+
expect(defaultKeyOf(qwen3Asr(registryFor({ os: 'ios' })))).toBe('MLX_COREML_INT4');
334+
expect(
335+
defaultKeyOf(qwen3Asr(registryFor({ os: 'ios', backends: ['XnnpackBackend', 'MLXBackend'] })))
336+
).toBe('XNNPACK_8DA4W');
337+
});
338+
328339
it('applies a pin only on the platform it names', () => {
329340
// The same group pins MLX for iOS only; Android resolves by its own order.
330341
expect(defaultKeyOf(distiluse(registryFor({ os: 'android' })))).toBe('VULKAN_FP16');
@@ -443,6 +454,13 @@ describe('feature map', () => {
443454
if (typeof value === 'string') {
444455
const match = value.match(/\/(xnnpack|coreml|mlx|vulkan)\//);
445456
if (match) found.add(match[1]!);
457+
// A file can need a second backend, which only its name records:
458+
// `mlx/qwen3_asr_0_6b_coreml_mlx_4w.pte` runs its audio tower on Core ML.
459+
if (value.endsWith('.pte')) {
460+
for (const part of value.split('/').pop()!.split('_')) {
461+
if (/^(xnnpack|coreml|mlx|vulkan)$/.test(part)) found.add(part);
462+
}
463+
}
446464
} else if (Array.isArray(value)) value.forEach(walk);
447465
else if (isObject(value)) Object.values(value).forEach(walk);
448466
};

0 commit comments

Comments
 (0)