Skip to content

Commit 36c22e9

Browse files
mergennachinmnachin
andauthored
Add Qwen3.5 MoE (35B-A3B) model export and runner for CUDA backend (pytorch#18169)
Memory-efficient loading using meta-device construction + lazy safetensors shard-by-shard loading + assign=True state dict loading, following the voxtral_realtime pattern. Peak CPU memory during loading is ~1x model size instead of ~3x. Expert weights are structured as grouped nn.Linear modules (16 groups of 16 experts each) so quantize_model_() handles them automatically. Layer-by-layer quantization on CUDA avoids loading the full bf16 model onto GPU at once. Includes C++ runner using the shared TextLLMRunner, Makefile target, and CMake presets. Reference implementations: - https://github.com/mergennachin/nano_qwen35_moe/ - vLLM: vllm/model_executor/models/qwen3_5.py <img width="1249" height="722" alt="Screenshot 2026-03-18 at 8 03 56 PM" src="https://github.com/user-attachments/assets/60fdb018-16af-4a33-a1e4-cd69bef7675c" /> 6.5 token/s for decode -- can improve later --------- Co-authored-by: mnachin <mnachin@fb.com>
1 parent 3b95f7e commit 36c22e9

11 files changed

Lines changed: 1553 additions & 1 deletion

File tree

‎.github/workflows/cuda.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,9 @@ jobs:
135135
# Run CUDA backend Python tests
136136
python -m pytest backends/cuda/tests backends/cuda/passes/tests -v -o "addopts="
137137
138+
# Build Qwen3.5 MoE runner (ExecuTorch already built above)
139+
cd examples/models/qwen3_5_moe && cmake --workflow --preset qwen3-5-moe-cuda
140+
138141
export-model-cuda-artifact:
139142
name: export-model-cuda-artifact
140143
# Skip this job if the pull request is from a fork (HuggingFace secrets are not available)

‎Makefile‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@
9191
#
9292
# ==============================================================================
9393

94-
.PHONY: voxtral-cuda voxtral-cpu voxtral-metal voxtral_realtime-cuda voxtral_realtime-cpu voxtral_realtime-metal whisper-cuda whisper-cuda-debug whisper-cpu whisper-metal parakeet-cuda parakeet-cuda-debug parakeet-cpu parakeet-metal parakeet-vulkan dinov2-cuda dinov2-cuda-debug sortformer-cuda sortformer-cpu silero-vad-cpu llama-cuda llama-cuda-debug llama-cpu llava-cpu gemma3-cuda gemma3-cpu clean help
94+
.PHONY: voxtral-cuda voxtral-cpu voxtral-metal voxtral_realtime-cuda voxtral_realtime-cpu voxtral_realtime-metal whisper-cuda whisper-cuda-debug whisper-cpu whisper-metal parakeet-cuda parakeet-cuda-debug parakeet-cpu parakeet-metal parakeet-vulkan dinov2-cuda dinov2-cuda-debug sortformer-cuda sortformer-cpu silero-vad-cpu llama-cuda llama-cuda-debug llama-cpu llava-cpu gemma3-cuda gemma3-cpu qwen3_5_moe-cuda clean help
9595

9696
help:
9797
@echo "This Makefile adds targets to build runners for various models on various backends. Run using \`make <target>\`. Available targets:"
@@ -121,6 +121,7 @@ help:
121121
@echo " llava-cpu - Build Llava runner with CPU backend"
122122
@echo " gemma3-cuda - Build Gemma3 runner with CUDA backend"
123123
@echo " gemma3-cpu - Build Gemma3 runner with CPU backend"
124+
@echo " qwen3_5_moe-cuda - Build Qwen3.5 MoE runner with CUDA backend"
124125
@echo " clean - Clean build artifacts"
125126

126127
voxtral-cuda:
@@ -362,6 +363,15 @@ gemma3-cpu:
362363
@echo "✓ Build complete!"
363364
@echo " Binary: cmake-out/examples/models/gemma3/gemma3_e2e_runner"
364365

366+
qwen3_5_moe-cuda:
367+
@echo "==> Building and installing ExecuTorch with CUDA..."
368+
cmake --workflow --preset llm-release-cuda
369+
@echo "==> Building Qwen3.5 MoE runner with CUDA..."
370+
cd examples/models/qwen3_5_moe && cmake --workflow --preset qwen3-5-moe-cuda
371+
@echo ""
372+
@echo "✓ Build complete!"
373+
@echo " Binary: cmake-out/examples/models/qwen3_5_moe/qwen3_5_moe_runner"
374+
365375
clean:
366376
rm -rf cmake-out \
367377
extension/llm/tokenizers/build \
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
cmake_minimum_required(VERSION 3.24)
8+
project(qwen3_5_moe)
9+
10+
set(CMAKE_CXX_STANDARD 17)
11+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
12+
13+
set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
14+
15+
include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake)
16+
17+
set(_common_include_directories ${EXECUTORCH_ROOT}/..)
18+
19+
# gflags
20+
set(gflags_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/gflags)
21+
find_package(gflags REQUIRED)
22+
23+
# executorch
24+
list(APPEND CMAKE_FIND_ROOT_PATH ${CMAKE_CURRENT_BINARY_DIR}/../../..)
25+
find_package(executorch CONFIG REQUIRED FIND_ROOT_PATH_BOTH)
26+
executorch_target_link_options_shared_lib(executorch)
27+
28+
set(link_libraries executorch gflags)
29+
30+
# CPU ops
31+
list(APPEND link_libraries optimized_native_cpu_ops_lib cpublas eigen_blas)
32+
executorch_target_link_options_shared_lib(optimized_native_cpu_ops_lib)
33+
34+
# Extensions
35+
list(
36+
APPEND
37+
link_libraries
38+
extension_llm_runner
39+
extension_module
40+
extension_data_loader
41+
extension_tensor
42+
extension_flat_tensor
43+
)
44+
45+
# CUDA backend (required)
46+
find_package(CUDAToolkit REQUIRED)
47+
list(APPEND link_libraries aoti_cuda_backend)
48+
executorch_target_link_options_shared_lib(aoti_cuda_backend)
49+
50+
# Tokenizer
51+
list(APPEND link_libraries tokenizers::tokenizers)
52+
53+
add_executable(qwen3_5_moe_runner main.cpp)
54+
target_include_directories(
55+
qwen3_5_moe_runner PUBLIC ${_common_include_directories}
56+
)
57+
target_link_libraries(qwen3_5_moe_runner PUBLIC ${link_libraries})
58+
59+
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
60+
target_link_options_gc_sections(qwen3_5_moe_runner)
61+
target_link_options(qwen3_5_moe_runner PRIVATE "LINKER:-s")
62+
endif()
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{
2+
"version": 6,
3+
"configurePresets": [
4+
{
5+
"name": "qwen3-5-moe-base",
6+
"hidden": true,
7+
"binaryDir": "${sourceDir}/../../../cmake-out/examples/models/qwen3_5_moe",
8+
"cacheVariables": {
9+
"CMAKE_BUILD_TYPE": "Release",
10+
"CMAKE_FIND_ROOT_PATH": "${sourceDir}/../../../cmake-out",
11+
"CMAKE_PREFIX_PATH": "${sourceDir}/../../../cmake-out"
12+
}
13+
},
14+
{
15+
"name": "qwen3-5-moe-cuda",
16+
"displayName": "Qwen3.5 MoE runner (CUDA)",
17+
"inherits": ["qwen3-5-moe-base"],
18+
"cacheVariables": {
19+
"EXECUTORCH_BUILD_CUDA": "ON"
20+
},
21+
"condition": {
22+
"type": "inList",
23+
"string": "${hostSystemName}",
24+
"list": ["Linux", "Windows"]
25+
}
26+
}
27+
],
28+
"buildPresets": [
29+
{
30+
"name": "qwen3-5-moe-cuda",
31+
"displayName": "Build Qwen3.5 MoE runner (CUDA)",
32+
"configurePreset": "qwen3-5-moe-cuda",
33+
"targets": ["qwen3_5_moe_runner"]
34+
}
35+
],
36+
"workflowPresets": [
37+
{
38+
"name": "qwen3-5-moe-cuda",
39+
"displayName": "Configure and build Qwen3.5 MoE runner (CUDA)",
40+
"steps": [
41+
{
42+
"type": "configure",
43+
"name": "qwen3-5-moe-cuda"
44+
},
45+
{
46+
"type": "build",
47+
"name": "qwen3-5-moe-cuda"
48+
}
49+
]
50+
}
51+
]
52+
}
Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# Qwen 3.5 MoE
2+
3+
Self-contained ExecuTorch implementation of
4+
[Qwen3.5-35B-A3B](https://huggingface.co/Qwen/Qwen3.5-35B-A3B),
5+
a ~35B total / ~3B active parameter Mixture-of-Experts language model.
6+
Weights are loaded directly from the HuggingFace safetensors checkpoint.
7+
CUDA backend only. See [model.md](model.md) for architecture and
8+
implementation details.
9+
10+
## Overview
11+
12+
The pipeline has two stages: **export** (Python, once) and **inference**
13+
(C++ runner, repeated). Export converts the HuggingFace checkpoint into a
14+
`model.pte` file with int4 quantization. At inference time, the C++ runner
15+
loads the `.pte`, `.ptd`, and a HuggingFace tokenizer, then generates text.
16+
17+
## Prerequisites
18+
19+
- ExecuTorch installed from source (see [building from source](../../../docs/source/using-executorch-building-from-source.md))
20+
- [safetensors](https://pypi.org/project/safetensors/) (`pip install safetensors`)
21+
- NVIDIA GPU with CUDA toolkit (tested on A100 80GB)
22+
- Python dependencies: `pip install -r requirements.txt` (installs FLA / Flash Linear Attention)
23+
- Model weights downloaded from HuggingFace. The directory should contain
24+
`config.json`, `model.safetensors.index.json`, safetensors shards, and
25+
`tokenizer.json`.
26+
27+
## Export
28+
29+
Export produces a `model.pte` and `aoti_cuda_blob.ptd` containing the
30+
compiled CUDA kernels and quantized weights. Int4 quantization is
31+
recommended — the model is too large to fit in VRAM at bf16.
32+
33+
```bash
34+
python export.py \
35+
--model-dir ~/models/Qwen3.5-MoE-A3B \
36+
--output-dir ./qwen35_moe_exports \
37+
--qlinear 4w \
38+
--qembedding 8w
39+
```
40+
41+
### Options
42+
43+
| Flag | Default | Description |
44+
|------|---------|-------------|
45+
| `--model-dir` | (required) | HuggingFace model directory with `config.json` + safetensors |
46+
| `--output-dir` | `./qwen35_moe_exports` | Output directory |
47+
| `--max-seq-len` | `4096` | KV cache length |
48+
| `--qlinear` | (none) | Linear layer quantization: `4w`, `8w`, `8da4w`, `8da8w` |
49+
| `--qlinear-group-size` | `32` | Group size for linear quantization |
50+
| `--qembedding` | (none) | Embedding quantization: `8w` |
51+
52+
## Build
53+
54+
ExecuTorch must be installed from source first (see
55+
[Prerequisites](#prerequisites)). The `make` target handles building
56+
core libraries and the runner binary.
57+
58+
```bash
59+
make qwen3_5_moe-cuda
60+
```
61+
62+
This builds ExecuTorch with CUDA backend support, then the runner binary
63+
at `cmake-out/examples/models/qwen3_5_moe/qwen3_5_moe_runner`.
64+
65+
## Run
66+
67+
The runner requires:
68+
- `model.pte` — exported model (see [Export](#export))
69+
- `aoti_cuda_blob.ptd` — CUDA delegate data file (produced during export)
70+
- `tokenizer.json` — HuggingFace tokenizer from the model weights directory
71+
72+
```bash
73+
cmake-out/examples/models/qwen3_5_moe/qwen3_5_moe_runner \
74+
--model_path qwen35_moe_exports/model.pte \
75+
--data_path qwen35_moe_exports/aoti_cuda_blob.ptd \
76+
--tokenizer_path ~/models/Qwen3.5-MoE-A3B/tokenizer.json \
77+
--prompt "The meaning of life is" \
78+
--max_new_tokens 128
79+
```
80+
81+
### Options
82+
83+
| Flag | Default | Description |
84+
|------|---------|-------------|
85+
| `--model_path` | (required) | Path to exported `.pte` model |
86+
| `--data_path` | (none) | Path to `.ptd` delegate data file (required for CUDA) |
87+
| `--tokenizer_path` | (required) | Path to HuggingFace `tokenizer.json` |
88+
| `--prompt` | `"Hello"` | Input prompt text |
89+
| `--temperature` | `0.8` | Sampling temperature (0 = greedy) |
90+
| `--max_new_tokens` | `128` | Maximum tokens to generate |
91+
92+
## Troubleshooting
93+
94+
- **OOM during export**: The model requires significant GPU memory even
95+
with int4 quantization. Try reducing `--max-seq-len` or using a GPU
96+
with more VRAM.
97+
- **OOM during loading**: The 35B parameter model requires ~70 GB RAM to
98+
load from safetensors before quantization. Ensure sufficient system
99+
memory.
100+
- **Missing `aoti_cuda_blob.ptd`**: This file is produced during export
101+
alongside the `.pte`. Both files are required for inference.

‎examples/models/qwen3_5_moe/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)