Skip to content

Commit 6a8eae9

Browse files
authored
Experimental Voxtral-realtime support for AMD ROCm through AOTI (pytorch#21977)
1 parent d756e7d commit 6a8eae9

14 files changed

Lines changed: 885 additions & 82 deletions

File tree

‎.ci/scripts/test-rocm-voxtral.sh‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
#!/usr/bin/env bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -euo pipefail
9+
10+
ROCM_VERSION="${ROCM_VERSION:-7.1}"
11+
ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
12+
PYTORCH_ROCM_INDEX="${PYTORCH_ROCM_INDEX:-https://download.pytorch.org/whl/test/rocm${ROCM_VERSION}}"
13+
TORCHAO_ROCM_WHEEL_BASE="${TORCHAO_ROCM_WHEEL_BASE:-https://download.pytorch.org/whl/nightly/rocm${ROCM_VERSION}}"
14+
VOXTRAL_CI_TMP_ROOT="${RUNNER_TEMP:-/tmp}"
15+
mkdir -p "${VOXTRAL_CI_TMP_ROOT}" 2>/dev/null || VOXTRAL_CI_TMP_ROOT=/tmp
16+
VOXTRAL_CI_TMPDIR="$(mktemp -d "${VOXTRAL_CI_TMP_ROOT}/executorch-rocm-voxtral.XXXXXX")"
17+
trap 'rm -rf "${VOXTRAL_CI_TMPDIR}"' EXIT
18+
VOXTRAL_START_SECONDS="${SECONDS}"
19+
20+
export ROCM_PATH
21+
export HIP_VISIBLE_DEVICES=0
22+
export CUDA_VISIBLE_DEVICES=0
23+
export HF_HOME="${VOXTRAL_CI_TMP_ROOT}/hf-cache"
24+
export HF_HUB_DISABLE_XET=1
25+
mkdir -p "${HF_HOME}"
26+
HF_TOKEN="$(printf '%s' "${SECRET_EXECUTORCH_HF_TOKEN}" | tr -d '\r\n')"
27+
export HF_TOKEN
28+
29+
read -r TORCH_VERSION TORCHAO_VERSION < <(
30+
python - <<'PY'
31+
from install_requirements import TORCHAO_NIGHTLY_VERSION
32+
from torch_pin import TORCH_VERSION
33+
34+
print(TORCH_VERSION, TORCHAO_NIGHTLY_VERSION)
35+
PY
36+
)
37+
# TorchAO ROCm wheels are not exposed by the per-version pip index.
38+
TORCHAO_WHEEL="${TORCHAO_ROCM_WHEEL_BASE}/torchao-${TORCHAO_VERSION}"
39+
TORCHAO_WHEEL+="%2Brocm${ROCM_VERSION}-cp310-abi3-manylinux_2_28_x86_64.whl"
40+
python -m pip install "torch==${TORCH_VERSION}" \
41+
--index-url "${PYTORCH_ROCM_INDEX}"
42+
python -m pip install -r requirements-dev.txt \
43+
"${TORCHAO_WHEEL}"
44+
python -m pip install --editable . --no-build-isolation
45+
46+
if ! command -v conda >/dev/null; then
47+
echo "The ROCm CI image must provide conda for its runtime libraries"
48+
exit 1
49+
fi
50+
conda install -y -c conda-forge ffmpeg 'libstdcxx-ng>=12'
51+
52+
python -m pip install 'fsspec[http]<=2025.3.0'
53+
python -m pip install datasets huggingface_hub librosa mistral-common safetensors soundfile
54+
python -m pip install torchcodec==0.11.0 \
55+
--extra-index-url https://download.pytorch.org/whl/test/cpu
56+
57+
python - <<'PY'
58+
import torch
59+
import torchao
60+
61+
assert torch.version.hip is not None, "PyTorch is not a ROCm build"
62+
assert torch.version.cuda is None, "PyTorch unexpectedly reports a CUDA runtime"
63+
assert torch.cuda.is_available(), "No AMD GPU is visible through PyTorch"
64+
assert "+rocm" in torchao.__version__, "TorchAO is not a ROCm build"
65+
66+
device = torch.cuda.get_device_properties(0)
67+
arch = device.gcnArchName.split(":", 1)[0]
68+
assert arch == "gfx950", f"Expected gfx950, got {device.gcnArchName}"
69+
assert device.warp_size == 64, f"Expected wave64, got {device.warp_size}"
70+
print(
71+
torch.__version__,
72+
torchao.__version__,
73+
torch.version.hip,
74+
device.name,
75+
device.gcnArchName,
76+
)
77+
PY
78+
79+
python -m pytest -v -o 'addopts=' examples/models/voxtral_realtime/tests
80+
81+
VOXTRAL_MODEL_DIR="${VOXTRAL_CI_TMPDIR}/model"
82+
VOXTRAL_AUDIO="${VOXTRAL_CI_TMPDIR}/test_audio.wav"
83+
VOXTRAL_OUTPUT="${VOXTRAL_CI_TMPDIR}/output"
84+
VOXTRAL_LOG="${VOXTRAL_CI_TMPDIR}/voxtral.log"
85+
86+
python - "${VOXTRAL_MODEL_DIR}" <<'PY'
87+
import sys
88+
89+
from huggingface_hub import snapshot_download
90+
91+
snapshot_download(
92+
"mistralai/Voxtral-Mini-4B-Realtime-2602",
93+
local_dir=sys.argv[1],
94+
allow_patterns=["params.json", "tekken.json", "consolidated.safetensors"],
95+
)
96+
PY
97+
98+
python - "${VOXTRAL_AUDIO}" <<'PY'
99+
import sys
100+
101+
import soundfile as sf
102+
from datasets import load_dataset
103+
104+
sample = load_dataset("distil-whisper/librispeech_long", "clean", split="validation")[0][
105+
"audio"
106+
]
107+
sf.write(sys.argv[1], sample["array"][: sample["sampling_rate"] * 30], sample["sampling_rate"])
108+
PY
109+
110+
CMAKE_PREFIX_PATH="$(python -c 'import torch; print(torch.utils.cmake_prefix_path)')"
111+
export CMAKE_PREFIX_PATH
112+
VOXTRAL_PYTHON=python \
113+
bash examples/models/voxtral_realtime/run_rocm_e2e.sh \
114+
"${VOXTRAL_MODEL_DIR}" \
115+
"${VOXTRAL_AUDIO}" \
116+
w4-bf16 \
117+
streaming \
118+
"${VOXTRAL_OUTPUT}" 2>&1 | tee "${VOXTRAL_LOG}"
119+
120+
if ! grep -qi 'Quilter' "${VOXTRAL_LOG}"; then
121+
echo "Voxtral transcript did not contain the expected word: Quilter"
122+
exit 1
123+
fi
124+
echo "Voxtral transcript contains the expected word: Quilter"
125+
126+
for VOXTRAL_METRIC in 'Voxtral export metrics:' 'Voxtral RTF:'; do
127+
if ! grep -q "${VOXTRAL_METRIC}" "${VOXTRAL_LOG}"; then
128+
echo "Voxtral CI did not report ${VOXTRAL_METRIC}"
129+
exit 1
130+
fi
131+
done
132+
133+
du -sh "${VOXTRAL_MODEL_DIR}" "${VOXTRAL_OUTPUT}"
134+
echo "Voxtral ROCm CI wall time: $((SECONDS - VOXTRAL_START_SECONDS)) seconds"

‎.github/workflows/rocm.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ on:
1717
- .github/workflows/_ci-run-decision.yml
1818
- .github/workflows/_get-changed-files.yml
1919
- .ci/scripts/test-rocm-aoti.sh
20+
- .ci/scripts/test-rocm-voxtral.sh
2021
- CMakeLists.txt
2122
- CMakePresets.json
2223
- install_requirements.py
@@ -28,6 +29,8 @@ on:
2829
- extension/cuda/**
2930
- examples/cuda/**
3031
- '!examples/cuda/**.md'
32+
- examples/models/voxtral_realtime/**
33+
- '!examples/models/voxtral_realtime/**.md'
3134
workflow_dispatch:
3235

3336
concurrency:
@@ -93,6 +96,48 @@ jobs:
9396
9497
echo "run-aoti=$RUN_AOTI" >> "$GITHUB_OUTPUT"
9598
99+
voxtral-run-decision:
100+
name: Voxtral CI run decision
101+
needs: [changed-files, run-decision]
102+
runs-on: ubuntu-latest
103+
outputs:
104+
run-voxtral: ${{ steps.decide.outputs.run-voxtral }}
105+
steps:
106+
- name: Select Voxtral runs
107+
id: decide
108+
env:
109+
CHANGED_FILES: ${{ needs.changed-files.outputs.changed-files }}
110+
FULL_RUN: ${{ needs.run-decision.outputs.is-full-run }}
111+
run: |
112+
set -efu
113+
RUN_VOXTRAL=false
114+
115+
if [[ "$FULL_RUN" == "true" ]]; then
116+
RUN_VOXTRAL=true
117+
fi
118+
119+
for path in $CHANGED_FILES; do
120+
case "$path" in
121+
examples/models/voxtral_realtime/*.md)
122+
continue
123+
;;
124+
examples/models/voxtral_realtime/* | \
125+
backends/cuda/aoti_packed_int4_tensor.py | \
126+
backends/cuda/triton/kernels/int4_matmul.py | \
127+
.ci/scripts/test-rocm-voxtral.sh | \
128+
.github/workflows/_ci-run-decision.yml | \
129+
.github/workflows/_get-changed-files.yml | \
130+
install_requirements.py | \
131+
torch_pin.py | \
132+
.github/workflows/rocm.yml)
133+
RUN_VOXTRAL=true
134+
break
135+
;;
136+
esac
137+
done
138+
139+
echo "run-voxtral=$RUN_VOXTRAL" >> "$GITHUB_OUTPUT"
140+
96141
unittest-rocm-gfx950:
97142
name: unittest-rocm-gfx950-rocm${{ matrix.rocm-version }}
98143
needs: [aoti-run-decision]
@@ -121,3 +166,37 @@ jobs:
121166
conda create -n executorch-rocm-ci python=3.10 -y
122167
conda activate executorch-rocm-ci
123168
ROCM_VERSION=${{ matrix.rocm-version }} bash .ci/scripts/test-rocm-aoti.sh
169+
170+
test-voxtral-realtime-rocm-gfx950:
171+
name: test-voxtral-realtime-rocm-gfx950-rocm${{ matrix.rocm-version }}
172+
needs: [voxtral-run-decision]
173+
if: |
174+
needs.voxtral-run-decision.outputs.run-voxtral == 'true' &&
175+
(github.event_name != 'pull_request' ||
176+
github.event.pull_request.head.repo.full_name == github.repository)
177+
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
178+
permissions:
179+
id-token: write
180+
contents: read
181+
secrets: inherit
182+
strategy:
183+
fail-fast: false
184+
matrix:
185+
rocm-version: ["7.1"]
186+
with:
187+
timeout: 180
188+
no-sudo: true
189+
secrets-env: EXECUTORCH_HF_TOKEN
190+
runner: linux.rocm.gpu.gfx950.1
191+
gpu-arch-type: rocm
192+
gpu-arch-version: ${{ matrix.rocm-version }}
193+
use-custom-docker-registry: false
194+
docker-image: pytorch/manylinux2_28-builder:rocm${{ matrix.rocm-version }}
195+
submodules: recursive
196+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
197+
script: |
198+
set -eux
199+
eval "$(conda shell.bash hook)"
200+
conda create -n executorch-rocm-ci python=3.10 -y
201+
conda activate executorch-rocm-ci
202+
ROCM_VERSION=${{ matrix.rocm-version }} bash .ci/scripts/test-rocm-voxtral.sh

‎Makefile‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@
9191
#
9292
# ==============================================================================
9393

94-
.PHONY: voxtral-cuda voxtral-cpu voxtral-metal voxtral-mlx voxtral_realtime-cuda voxtral_realtime-cpu voxtral_realtime-metal voxtral_realtime-mlx voxtral_tts-cpu voxtral_tts-cuda whisper-cuda whisper-cuda-debug whisper-cpu whisper-metal parakeet-cuda parakeet-cuda-debug parakeet-cpu parakeet-metal parakeet-mlx parakeet-vulkan dinov2-cuda dinov2-cuda-debug sortformer-cuda sortformer-cpu silero-vad-cpu llama-cuda llama-cuda-debug llama-cpu lfm_2_5-mlx llava-cpu gemma3-cuda gemma3-cpu gemma4_31b-cuda gemma4_31b-mlx muse-glimmer-cuda muse-glimmer-mlx qwen3_5_moe-cuda qwen3_5_moe-metal qwen3_5_moe-mlx clean help
94+
.PHONY: voxtral-cuda voxtral-cpu voxtral-metal voxtral-mlx voxtral_realtime-cuda voxtral_realtime-rocm voxtral_realtime-cpu voxtral_realtime-metal voxtral_realtime-mlx voxtral_tts-cpu voxtral_tts-cuda whisper-cuda whisper-cuda-debug whisper-cpu whisper-metal parakeet-cuda parakeet-cuda-debug parakeet-cpu parakeet-metal parakeet-mlx parakeet-vulkan dinov2-cuda dinov2-cuda-debug sortformer-cuda sortformer-cpu silero-vad-cpu llama-cuda llama-cuda-debug llama-cpu lfm_2_5-mlx llava-cpu gemma3-cuda gemma3-cpu gemma4_31b-cuda gemma4_31b-mlx muse-glimmer-cuda muse-glimmer-mlx qwen3_5_moe-cuda qwen3_5_moe-metal qwen3_5_moe-mlx clean help
9595

9696
help:
9797
@echo "This Makefile adds targets to build runners for various models on various backends. Run using \`make <target>\`. Available targets:"
@@ -100,6 +100,7 @@ help:
100100
@echo " voxtral-metal - Build Voxtral runner with Metal backend (macOS only)"
101101
@echo " voxtral-mlx - Build Voxtral runner with MLX backend"
102102
@echo " voxtral_realtime-cuda - Build Voxtral Realtime runner with CUDA backend"
103+
@echo " voxtral_realtime-rocm - Build Voxtral Realtime runner with ROCm backend"
103104
@echo " voxtral_realtime-cpu - Build Voxtral Realtime runner with CPU backend"
104105
@echo " voxtral_realtime-metal - Build Voxtral Realtime runner with Metal backend (macOS only)"
105106
@echo " voxtral_realtime-mlx - Build Voxtral Realtime runner with MLX backend"
@@ -326,6 +327,15 @@ voxtral_realtime-cuda:
326327
@echo "✓ Build complete!"
327328
@echo " Binary: cmake-out/examples/models/voxtral_realtime/voxtral_realtime_runner"
328329

330+
voxtral_realtime-rocm:
331+
@echo "==> Building and installing ExecuTorch with ROCm..."
332+
cmake --workflow --preset llm-release-rocm
333+
@echo "==> Building Voxtral Realtime runner with ROCm..."
334+
cd examples/models/voxtral_realtime && cmake --workflow --preset voxtral-realtime-rocm
335+
@echo ""
336+
@echo "✓ Build complete!"
337+
@echo " Binary: cmake-out-rocm-llm/examples/models/voxtral_realtime/voxtral_realtime_runner"
338+
329339
voxtral_realtime-mlx:
330340
@echo "==> Building and installing ExecuTorch with MLX..."
331341
cmake --workflow --preset mlx-release

‎backends/cuda/rocm.md‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,11 @@ MI300X (`gfx942`); CI is configured to exercise `gfx950`.
1818
- Python pybindings cannot allocate ROCm device memory; use a native runner such
1919
as `executor_runner`.
2020
- Installed CMake consumers must be able to find the HIP package.
21-
- Model runners do not yet claim ROCm support. The gfx950 CI job covers the
22-
backend build, native runtime, AOTI export and execution, and Triton W4 tests.
21+
- Voxtral Realtime has an explicit ROCm workflow. Other model runners do not
22+
claim ROCm support. The gfx950 CI job covers the backend build, native
23+
runtime, AOTI export and execution, and Triton W4 tests. Sampled CI is
24+
configured to run the streaming packed-W4/BF16 Voxtral path; offline and
25+
performance validation remain manual.
2326

2427
## Build
2528

@@ -37,3 +40,7 @@ A `libcudart` dependency is a configuration error.
3740

3841
For an end-to-end export that proves Inductor emitted a Triton kernel, see
3942
[../../examples/cuda/README.md](../../examples/cuda/README.md).
43+
44+
Voxtral Realtime additionally covers BF16 and packed W4/BF16 execution. See
45+
[../../examples/models/voxtral_realtime/README.md](../../examples/models/voxtral_realtime/README.md)
46+
and its `run_rocm_e2e.sh` example.

‎backends/cuda/runtime/cuda_allocator.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -137,13 +137,13 @@ CudaAllocator::allocate(size_t nbytes, DeviceIndex index, size_t alignment) {
137137
const bool switch_device = index >= 0 && prev_device_err == cudaSuccess &&
138138
static_cast<int>(index) != prev_device;
139139
if (switch_device) {
140-
cudaSetDevice(index);
140+
(void)cudaSetDevice(index);
141141
}
142142

143143
cudaError_t err = cudaMalloc(&ptr, nbytes);
144144

145145
if (switch_device) {
146-
cudaSetDevice(prev_device);
146+
(void)cudaSetDevice(prev_device);
147147
}
148148

149149
if (err != cudaSuccess) {
@@ -165,7 +165,7 @@ CudaAllocator::allocate(size_t nbytes, DeviceIndex index, size_t alignment) {
165165
"cudaMalloc returned pointer %p not aligned to %zu bytes",
166166
ptr,
167167
alignment);
168-
cudaFree(ptr);
168+
(void)cudaFree(ptr);
169169
return Error::MemoryAllocationFailed;
170170
}
171171

@@ -183,14 +183,14 @@ void CudaAllocator::deallocate(void* ptr, DeviceIndex index) {
183183
if (index >= 0) {
184184
prev_device_err = cudaGetDevice(&prev_device);
185185
if (prev_device_err == cudaSuccess) {
186-
cudaSetDevice(index);
186+
(void)cudaSetDevice(index);
187187
}
188188
}
189189

190190
cudaError_t err = cudaFree(ptr);
191191

192192
if (index >= 0 && prev_device_err == cudaSuccess) {
193-
cudaSetDevice(prev_device);
193+
(void)cudaSetDevice(prev_device);
194194
}
195195

196196
if (err != cudaSuccess) {

‎backends/cuda/runtime/cuda_delegate_handle.h‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ namespace cuda {
2222
struct CudaStreamDeleter {
2323
void operator()(cudaStream_t* stream) const {
2424
if (stream != nullptr && *stream != nullptr) {
25-
cudaStreamDestroy(*stream);
25+
(void)cudaStreamDestroy(*stream);
2626
}
2727
delete stream;
2828
}
@@ -86,16 +86,16 @@ struct CudaGraphState {
8686

8787
~CudaGraphState() {
8888
if (graph_exec) {
89-
cudaGraphExecDestroy(graph_exec);
89+
(void)cudaGraphExecDestroy(graph_exec);
9090
}
9191
if (graph) {
92-
cudaGraphDestroy(graph);
92+
(void)cudaGraphDestroy(graph);
9393
}
9494
// Only free input buffers — output buffers are owned by the AOTI runtime
9595
// (allocated during graph capture via the caching allocator).
9696
for (auto* ptr : static_input_ptrs) {
9797
if (ptr)
98-
cudaFree(ptr);
98+
(void)cudaFree(ptr);
9999
}
100100
}
101101

@@ -121,12 +121,12 @@ struct CudaGraphState {
121121
if (this != &other) {
122122
// Clean up existing resources
123123
if (graph_exec)
124-
cudaGraphExecDestroy(graph_exec);
124+
(void)cudaGraphExecDestroy(graph_exec);
125125
if (graph)
126-
cudaGraphDestroy(graph);
126+
(void)cudaGraphDestroy(graph);
127127
for (auto* ptr : static_input_ptrs) {
128128
if (ptr)
129-
cudaFree(ptr);
129+
(void)cudaFree(ptr);
130130
}
131131

132132
phase = other.phase;

0 commit comments

Comments
 (0)