Skip to content

Commit 4a3bc34

Browse files
Metal backend: Enable Voxtral Realtime (pytorch#17536)
This pull request adds Metal backend support for the Voxtral Realtime model (offline mode, streaming support is coming next) **Voxtral Realtime Metal backend support:** * Added Metal backend support to `export_voxtral_rt.py`, including custom decompositions and dynamic shape handling for Metal/AOTI compatibility. Also added validation for Metal-specific quantization (`fpa4w`). * Updated `export_model_artifact.sh` and `test_model_e2e.sh` scripts to support a new `mode` parameter. This is used in the Voxtral Realtime model for selecting between streaming/offline export modes. **CI/CD and workflow updates:** * Added quantized `Voxtral-Mini-4B-Realtime-2602` to Metal workflow matrix. * Added new CMake presets for building and testing the Voxtral Realtime runner with Metal backend.
1 parent cd32e42 commit 4a3bc34

9 files changed

Lines changed: 491 additions & 42 deletions

File tree

‎.ci/scripts/export_model_artifact.sh‎

Lines changed: 60 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
show_help() {
1111
cat << EOF
12-
Usage: export_model_artifact.sh <device> <hf_model> [quant_name] [output_dir]
12+
Usage: export_model_artifact.sh <device> <hf_model> [quant_name] [output_dir] [mode]
1313
1414
Export a HuggingFace model to CUDA/Metal/XNNPACK format with optional quantization.
1515
@@ -34,13 +34,22 @@ Arguments:
3434
3535
output_dir Output directory for artifacts (optional, default: current directory)
3636
37+
mode Export mode (optional, default: auto-detect based on model and device)
38+
Supported modes:
39+
- vr-streaming: Voxtral Realtime streaming mode
40+
- vr-offline: Voxtral Realtime offline mode
41+
3742
Examples:
3843
export_model_artifact.sh metal "openai/whisper-small"
3944
export_model_artifact.sh metal "nvidia/parakeet-tdt" "quantized-int4-metal"
45+
export_model_artifact.sh metal "mistralai/Voxtral-Mini-4B-Realtime-2602" "quantized-int4-metal"
46+
export_model_artifact.sh metal "mistralai/Voxtral-Mini-4B-Realtime-2602" "non-quantized" "." "vr-streaming"
4047
export_model_artifact.sh cuda "mistralai/Voxtral-Mini-3B-2507" "quantized-int4-tile-packed"
4148
export_model_artifact.sh cuda "google/gemma-3-4b-it" "non-quantized" "./output"
4249
export_model_artifact.sh cuda "nvidia/parakeet-tdt" "non-quantized" "./output"
4350
export_model_artifact.sh xnnpack "nvidia/parakeet-tdt" "quantized-8da4w" "./output"
51+
export_model_artifact.sh xnnpack "mistralai/Voxtral-Mini-4B-Realtime-2602" "quantized-8da4w" "./output"
52+
export_model_artifact.sh xnnpack "mistralai/Voxtral-Mini-4B-Realtime-2602" "non-quantized" "./output" "vr-offline"
4453
EOF
4554
}
4655

@@ -61,6 +70,26 @@ DEVICE="$1"
6170
HF_MODEL="$2"
6271
QUANT_NAME="${3:-non-quantized}"
6372
OUTPUT_DIR="${4:-.}"
73+
MODE="${5:-}"
74+
75+
# Validate mode if specified
76+
if [ -n "$MODE" ]; then
77+
case "$MODE" in
78+
vr-streaming|vr-offline)
79+
# Voxtral Realtime modes require Voxtral Realtime model
80+
if [ "$HF_MODEL" != "mistralai/Voxtral-Mini-4B-Realtime-2602" ]; then
81+
echo "Error: Mode '$MODE' can only be used with Voxtral Realtime model"
82+
echo "Provided model: $HF_MODEL"
83+
exit 1
84+
fi
85+
;;
86+
*)
87+
echo "Error: Unsupported mode '$MODE'"
88+
echo "Supported modes: vr-streaming, vr-offline"
89+
exit 1
90+
;;
91+
esac
92+
fi
6493

6594
case "$DEVICE" in
6695
cuda)
@@ -210,7 +239,7 @@ if [ "$MODEL_NAME" = "parakeet" ]; then
210239
exit 0
211240
fi
212241

213-
# Voxtral Realtime uses a custom export script (streaming mode)
242+
# Voxtral Realtime uses a custom export script
214243
if [ "$MODEL_NAME" = "voxtral_realtime" ]; then
215244
pip install safetensors huggingface_hub
216245

@@ -222,20 +251,42 @@ if [ "$MODEL_NAME" = "voxtral_realtime" ]; then
222251
VR_QUANT_ARGS=""
223252
if [ "$QUANT_NAME" = "quantized-8da4w" ]; then
224253
VR_QUANT_ARGS="--qlinear-encoder 8da4w --qlinear 8da4w --qlinear-group-size 32 --qembedding 8w"
254+
elif [ "$QUANT_NAME" = "quantized-int4-metal" ]; then
255+
VR_QUANT_ARGS="--qlinear-encoder fpa4w --qlinear fpa4w"
256+
fi
257+
258+
# Determine streaming mode based on MODE parameter
259+
USE_STREAMING="false"
260+
if [ "$MODE" = "vr-streaming" ]; then
261+
USE_STREAMING="true"
262+
elif [ "$MODE" = "vr-offline" ]; then
263+
USE_STREAMING="false"
264+
elif [ -z "$MODE" ]; then
265+
# Auto-detect: XNNPACK uses streaming, others use offline
266+
if [ "$DEVICE" = "xnnpack" ]; then
267+
USE_STREAMING="true"
268+
fi
269+
fi
270+
271+
# Configure export and preprocessor based on streaming mode
272+
STREAMING_ARG=""
273+
PREPROCESSOR_ARGS="--feature_size 128 --output_file ${OUTPUT_DIR}/preprocessor.pte"
274+
if [ "$USE_STREAMING" = "true" ]; then
275+
STREAMING_ARG="--streaming"
276+
PREPROCESSOR_ARGS="$PREPROCESSOR_ARGS --streaming"
277+
else
278+
PREPROCESSOR_ARGS="$PREPROCESSOR_ARGS --stack_output --max_audio_len 300"
225279
fi
226280

227281
python -m executorch.examples.models.voxtral_realtime.export_voxtral_rt \
228282
--model-path "$LOCAL_MODEL_DIR" \
229-
--backend xnnpack \
230-
--streaming \
283+
--backend "$DEVICE" \
284+
${STREAMING_ARG} \
231285
--output-dir "${OUTPUT_DIR}" \
232286
${VR_QUANT_ARGS}
233287

234-
# Export streaming preprocessor (no chunk padding)
235-
python -m executorch.extension.audio.mel_spectrogram \
236-
--feature_size 128 \
237-
--streaming \
238-
--output_file "${OUTPUT_DIR}/preprocessor.pte"
288+
# Export preprocessor
289+
python -m executorch.extension.audio.mel_spectrogram ${PREPROCESSOR_ARGS}
239290

240291
test -f "${OUTPUT_DIR}/model.pte"
241292
test -f "${OUTPUT_DIR}/preprocessor.pte"

‎.ci/scripts/test_model_e2e.sh‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
show_help() {
1111
cat << EOF
12-
Usage: test_model_e2e.sh <device> <hf_model> <quant_name> [model_dir]
12+
Usage: test_model_e2e.sh <device> <hf_model> <quant_name> [model_dir] [mode]
1313
1414
Build and run end-to-end tests for CUDA/Metal/XNNPACK models.
1515
@@ -35,11 +35,18 @@ Arguments:
3535
Expected files: model.pte, aoti_cuda_blob.ptd (CUDA only)
3636
Tokenizers and test files will be downloaded to this directory
3737
38+
mode Test mode (optional, default: auto-detect based on model and device)
39+
Supported modes:
40+
- vr-streaming: Voxtral Realtime streaming mode
41+
- vr-offline: Voxtral Realtime offline mode
42+
3843
Examples:
3944
test_model_e2e.sh metal "openai/whisper-small" "non-quantized"
4045
test_model_e2e.sh cuda "mistralai/Voxtral-Mini-3B-2507" "quantized-int4-tile-packed" "./model_output"
4146
test_model_e2e.sh cuda "nvidia/parakeet-tdt" "non-quantized" "./model_output"
4247
test_model_e2e.sh xnnpack "nvidia/parakeet-tdt" "quantized-8da4w" "./model_output"
48+
test_model_e2e.sh metal "mistralai/Voxtral-Mini-4B-Realtime-2602" "non-quantized" "." "vr-streaming"
49+
test_model_e2e.sh xnnpack "mistralai/Voxtral-Mini-4B-Realtime-2602" "quantized-8da4w" "./model_output" "vr-offline"
4350
EOF
4451
}
4552

@@ -67,6 +74,26 @@ HF_MODEL="$2"
6774
QUANT_NAME="$3"
6875
# Download tokenizers, audio, and image files to this directory
6976
MODEL_DIR="${4:-.}"
77+
MODE="${5:-}"
78+
79+
# Validate mode if specified
80+
if [ -n "$MODE" ]; then
81+
case "$MODE" in
82+
vr-streaming|vr-offline)
83+
# Voxtral Realtime modes require Voxtral Realtime model
84+
if [ "$HF_MODEL" != "mistralai/Voxtral-Mini-4B-Realtime-2602" ]; then
85+
echo "Error: Mode '$MODE' can only be used with Voxtral Realtime model"
86+
echo "Provided model: $HF_MODEL"
87+
exit 1
88+
fi
89+
;;
90+
*)
91+
echo "Error: Unsupported mode '$MODE'"
92+
echo "Supported modes: vr-streaming, vr-offline"
93+
exit 1
94+
;;
95+
esac
96+
fi
7097

7198
echo "Testing model: $HF_MODEL (quantization: $QUANT_NAME)"
7299

@@ -243,7 +270,23 @@ case "$MODEL_NAME" in
243270
fi
244271
;;
245272
voxtral_realtime)
246-
RUNNER_ARGS="--model_path ${MODEL_DIR}/model.pte --tokenizer_path ${MODEL_DIR}/$TOKENIZER_FILE --preprocessor_path ${MODEL_DIR}/$PREPROCESSOR --audio_path ${MODEL_DIR}/$AUDIO_FILE --temperature 0 --streaming"
273+
RUNNER_ARGS="--model_path ${MODEL_DIR}/model.pte --tokenizer_path ${MODEL_DIR}/$TOKENIZER_FILE --preprocessor_path ${MODEL_DIR}/$PREPROCESSOR --audio_path ${MODEL_DIR}/$AUDIO_FILE --temperature 0"
274+
# Determine streaming mode based on MODE parameter
275+
USE_STREAMING="false"
276+
if [ "$MODE" = "vr-streaming" ]; then
277+
USE_STREAMING="true"
278+
elif [ "$MODE" = "vr-offline" ]; then
279+
USE_STREAMING="false"
280+
elif [ -z "$MODE" ]; then
281+
# Auto-detect: XNNPACK uses streaming, others use offline
282+
if [ "$DEVICE" = "xnnpack" ]; then
283+
USE_STREAMING="true"
284+
fi
285+
fi
286+
# Add streaming flag if needed
287+
if [ "$USE_STREAMING" = "true" ]; then
288+
RUNNER_ARGS="$RUNNER_ARGS --streaming"
289+
fi
247290
;;
248291
esac
249292

‎.github/workflows/metal.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ jobs:
6464
model:
6565
- repo: "mistralai"
6666
name: "Voxtral-Mini-3B-2507"
67+
- repo: "mistralai"
68+
name: "Voxtral-Mini-4B-Realtime-2602"
6769
- repo: "openai"
6870
name: "whisper-small"
6971
- repo: "openai"
@@ -73,6 +75,12 @@ jobs:
7375
quant:
7476
- "non-quantized"
7577
- "quantized-int4-metal"
78+
exclude:
79+
# Exclude non-quantized for Voxtral Realtime (too large)
80+
- model:
81+
repo: "mistralai"
82+
name: "Voxtral-Mini-4B-Realtime-2602"
83+
quant: "non-quantized"
7684
with:
7785
runner: macos-m2-stable
7886
python-version: '3.11'
@@ -115,6 +123,8 @@ jobs:
115123
model:
116124
- repo: "mistralai"
117125
name: "Voxtral-Mini-3B-2507"
126+
- repo: "mistralai"
127+
name: "Voxtral-Mini-4B-Realtime-2602"
118128
- repo: "openai"
119129
name: "whisper-small"
120130
- repo: "openai"
@@ -124,6 +134,12 @@ jobs:
124134
quant:
125135
- "non-quantized"
126136
- "quantized-int4-metal"
137+
exclude:
138+
# Exclude non-quantized for Voxtral Realtime (too large)
139+
- model:
140+
repo: "mistralai"
141+
name: "Voxtral-Mini-4B-Realtime-2602"
142+
quant: "non-quantized"
127143
with:
128144
runner: macos-m2-stable
129145
python-version: '3.11'

‎Makefile‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,14 +91,15 @@
9191
#
9292
# ==============================================================================
9393

94-
.PHONY: voxtral-cuda voxtral-cpu voxtral-metal voxtral_realtime-cpu whisper-cuda whisper-cuda-debug whisper-cpu whisper-metal parakeet-cuda parakeet-cuda-debug parakeet-cpu parakeet-metal sortformer-cpu silero-vad-cpu llama-cpu llava-cpu gemma3-cuda gemma3-cpu clean help
94+
.PHONY: voxtral-cuda voxtral-cpu voxtral-metal voxtral_realtime-cpu voxtral_realtime-metal whisper-cuda whisper-cuda-debug whisper-cpu whisper-metal parakeet-cuda parakeet-cuda-debug parakeet-cpu parakeet-metal sortformer-cpu silero-vad-cpu llama-cpu llava-cpu gemma3-cuda gemma3-cpu clean help
9595

9696
help:
9797
@echo "This Makefile adds targets to build runners for various models on various backends. Run using \`make <target>\`. Available targets:"
9898
@echo " voxtral-cuda - Build Voxtral runner with CUDA backend"
9999
@echo " voxtral-cpu - Build Voxtral runner with CPU backend"
100100
@echo " voxtral-metal - Build Voxtral runner with Metal backend (macOS only)"
101101
@echo " voxtral_realtime-cpu - Build Voxtral Realtime runner with CPU backend"
102+
@echo " voxtral_realtime-metal - Build Voxtral Realtime runner with Metal backend (macOS only)"
102103
@echo " whisper-cuda - Build Whisper runner with CUDA backend"
103104
@echo " whisper-cuda-debug - Build Whisper runner with CUDA backend (debug mode)"
104105
@echo " whisper-cpu - Build Whisper runner with CPU backend"
@@ -232,6 +233,15 @@ voxtral_realtime-cpu:
232233
@echo "✓ Build complete!"
233234
@echo " Binary: cmake-out/examples/models/voxtral_realtime/voxtral_realtime_runner"
234235

236+
voxtral_realtime-metal:
237+
@echo "==> Building and installing ExecuTorch with Metal (stats enabled)..."
238+
cmake --workflow --preset llm-metal-stats
239+
@echo "==> Building Voxtral Realtime runner with Metal..."
240+
cd examples/models/voxtral_realtime && cmake --workflow --preset voxtral-realtime-metal
241+
@echo ""
242+
@echo "✓ Build complete!"
243+
@echo " Binary: cmake-out/examples/models/voxtral_realtime/voxtral_realtime_runner"
244+
235245
silero-vad-cpu:
236246
@echo "==> Building and installing ExecuTorch..."
237247
cmake --workflow --preset llm-release

‎examples/models/voxtral_realtime/CMakePresets.json‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,19 @@
1515
"name": "voxtral-realtime-cpu",
1616
"displayName": "Voxtral Realtime runner (CPU)",
1717
"inherits": ["voxtral-realtime-base"]
18+
},
19+
{
20+
"name": "voxtral-realtime-metal",
21+
"displayName": "Voxtral Realtime runner (Metal)",
22+
"inherits": ["voxtral-realtime-base"],
23+
"cacheVariables": {
24+
"EXECUTORCH_BUILD_METAL": "ON"
25+
},
26+
"condition": {
27+
"lhs": "${hostSystemName}",
28+
"type": "equals",
29+
"rhs": "Darwin"
30+
}
1831
}
1932
],
2033
"buildPresets": [
@@ -23,6 +36,13 @@
2336
"displayName": "Build Voxtral Realtime runner (CPU)",
2437
"configurePreset": "voxtral-realtime-cpu",
2538
"targets": ["voxtral_realtime_runner"]
39+
},
40+
{
41+
"name": "voxtral-realtime-metal",
42+
"displayName": "Build Voxtral Realtime runner (Metal)",
43+
"configurePreset": "voxtral-realtime-metal",
44+
"configuration": "Release",
45+
"targets": ["voxtral_realtime_runner"]
2646
}
2747
],
2848
"workflowPresets": [
@@ -39,6 +59,20 @@
3959
"name": "voxtral-realtime-cpu"
4060
}
4161
]
62+
},
63+
{
64+
"name": "voxtral-realtime-metal",
65+
"displayName": "Configure and build Voxtral Realtime runner (Metal)",
66+
"steps": [
67+
{
68+
"type": "configure",
69+
"name": "voxtral-realtime-metal"
70+
},
71+
{
72+
"type": "build",
73+
"name": "voxtral-realtime-metal"
74+
}
75+
]
4276
}
4377
]
4478
}

0 commit comments

Comments
 (0)