Skip to content

Commit 1ebd320

Browse files
authored
[executorch][cuda] Add Muse Glimmer export and runtime CI (pytorch#21758)
Add self-contained CUDA coverage for representative Muse Glimmer text and vision configurations. Introduce an opt-in same-job export path so large-model CI can avoid artifact upload and download handoffs, and the time for executorch rebuild. This is for basic output check and perf guard will be added afterwards
1 parent 4a29383 commit 1ebd320

4 files changed

Lines changed: 278 additions & 11 deletions

File tree

‎.ci/scripts/export_model_artifact.sh‎

Lines changed: 114 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ Arguments:
2525
- nvidia/diar_streaming_sortformer_4spk-v2
2626
- nvidia/parakeet-tdt
2727
- facebook/dinov2-small-imagenet1k-1-layer
28+
- meta-models/Muse-Glimmer-30B-GGUF
2829
2930
quant_name Quantization type (optional, default: non-quantized)
3031
Options:
@@ -33,13 +34,17 @@ Arguments:
3334
- quantized-int4-weight-only (CUDA only)
3435
- quantized-int4-metal (Metal only)
3536
- quantized-8da4w (XNNPACK only)
37+
- kquant-17gb (Muse Glimmer only)
38+
- kquant-dynamic (Muse Glimmer only)
3639
3740
output_dir Output directory for artifacts (optional, default: current directory)
3841
3942
mode Export mode (optional, default: vr-streaming)
4043
Supported modes:
4144
- vr-streaming: Voxtral Realtime streaming mode
4245
- vr-offline: Voxtral Realtime offline mode
46+
- solo-text: Muse Glimmer solo text mode
47+
- dflash-image: Muse Glimmer DFlash vision mode
4348
4449
Examples:
4550
export_model_artifact.sh metal "openai/whisper-small"
@@ -53,6 +58,7 @@ Examples:
5358
export_model_artifact.sh xnnpack "nvidia/parakeet-tdt" "quantized-8da4w" "./output"
5459
export_model_artifact.sh xnnpack "mistralai/Voxtral-Mini-4B-Realtime-2602" "quantized-8da4w" "./output"
5560
export_model_artifact.sh xnnpack "mistralai/Voxtral-Mini-4B-Realtime-2602" "non-quantized" "./output" "vr-offline"
61+
export_model_artifact.sh cuda "meta-models/Muse-Glimmer-30B-GGUF" "kquant-17gb" "./output" "solo-text"
5662
EOF
5763
}
5864

@@ -89,9 +95,16 @@ if [ -n "$MODE" ]; then
8995
exit 1
9096
fi
9197
;;
98+
solo-text|dflash-image)
99+
if [ "$HF_MODEL" != "meta-models/Muse-Glimmer-30B-GGUF" ]; then
100+
echo "Error: Mode '$MODE' can only be used with Muse Glimmer model"
101+
echo "Provided model: $HF_MODEL"
102+
exit 1
103+
fi
104+
;;
92105
*)
93106
echo "Error: Unsupported mode '$MODE'"
94-
echo "Supported modes: vr-streaming, vr-offline"
107+
echo "Supported modes: vr-streaming, vr-offline, solo-text, dflash-image"
95108
exit 1
96109
;;
97110
esac
@@ -203,9 +216,17 @@ case "$HF_MODEL" in
203216
PREPROCESSOR_FEATURE_SIZE=""
204217
PREPROCESSOR_OUTPUT=""
205218
;;
219+
meta-models/Muse-Glimmer-30B-GGUF)
220+
MODEL_NAME="muse_glimmer"
221+
TASK=""
222+
MAX_SEQ_LEN=""
223+
EXTRA_PIP=""
224+
PREPROCESSOR_FEATURE_SIZE=""
225+
PREPROCESSOR_OUTPUT=""
226+
;;
206227
*)
207228
echo "Error: Unsupported model '$HF_MODEL'"
208-
echo "Supported models: mistralai/Voxtral-Mini-3B-2507, mistralai/Voxtral-Mini-4B-Realtime-2602, openai/whisper-{small, medium, large, large-v2, large-v3, large-v3-turbo}, google/gemma-3-4b-it, Qwen/Qwen3-0.6B, nvidia/diar_streaming_sortformer_4spk-v2, nvidia/parakeet-tdt, facebook/dinov2-small-imagenet1k-1-layer, SocialLocalMobile/Qwen3.5-35B-A3B-HQQ-INT4, unsloth/gemma-4-31B-it-GGUF"
229+
echo "Supported models: mistralai/Voxtral-Mini-3B-2507, mistralai/Voxtral-Mini-4B-Realtime-2602, openai/whisper-{small, medium, large, large-v2, large-v3, large-v3-turbo}, google/gemma-3-4b-it, Qwen/Qwen3-0.6B, nvidia/diar_streaming_sortformer_4spk-v2, nvidia/parakeet-tdt, facebook/dinov2-small-imagenet1k-1-layer, SocialLocalMobile/Qwen3.5-35B-A3B-HQQ-INT4, unsloth/gemma-4-31B-it-GGUF, meta-models/Muse-Glimmer-30B-GGUF"
209230
exit 1
210231
;;
211232
esac
@@ -243,13 +264,36 @@ case "$QUANT_NAME" in
243264
fi
244265
EXTRA_ARGS="--qlinear 8da4w --qlinear_group_size 32 --qlinear_encoder 8da4w --qlinear_encoder_group_size 32"
245266
;;
267+
kquant-17gb|kquant-dynamic)
268+
if [ "$HF_MODEL" != "meta-models/Muse-Glimmer-30B-GGUF" ]; then
269+
echo "Error: Quantization '$QUANT_NAME' can only be used with Muse Glimmer model"
270+
echo "Provided model: $HF_MODEL"
271+
exit 1
272+
fi
273+
EXTRA_ARGS=""
274+
;;
246275
*)
247276
echo "Error: Unsupported quantization '$QUANT_NAME'"
248-
echo "Supported quantizations: non-quantized, quantized-int4-tile-packed, quantized-int4-weight-only, quantized-int4-metal, quantized-8da4w"
277+
echo "Supported quantizations: non-quantized, quantized-int4-tile-packed, quantized-int4-weight-only, quantized-int4-metal, quantized-8da4w, kquant-17gb, kquant-dynamic"
249278
exit 1
250279
;;
251280
esac
252281

282+
if [ "$MODEL_NAME" = "muse_glimmer" ]; then
283+
if [ "$DEVICE" != "cuda" ]; then
284+
echo "Error: Muse Glimmer is only supported with the cuda device"
285+
exit 1
286+
fi
287+
if [ "$QUANT_NAME" != "kquant-17gb" ] && [ "$QUANT_NAME" != "kquant-dynamic" ]; then
288+
echo "Error: Muse Glimmer requires quantization 'kquant-17gb' or 'kquant-dynamic'"
289+
exit 1
290+
fi
291+
if [ "$MODE" != "solo-text" ] && [ "$MODE" != "dflash-image" ]; then
292+
echo "Error: Muse Glimmer requires mode 'solo-text' or 'dflash-image'"
293+
exit 1
294+
fi
295+
fi
296+
253297
echo "::group::Export $MODEL_NAME"
254298

255299
if [ -n "$EXTRA_PIP" ]; then
@@ -470,6 +514,73 @@ if [ "$MODEL_NAME" = "qwen3_5_moe" ]; then
470514
exit 0
471515
fi
472516

517+
# Muse Glimmer: download the selected GGUFs and export the requested CUDA configuration.
518+
if [ "$MODEL_NAME" = "muse_glimmer" ]; then
519+
pip install safetensors huggingface_hub gguf
520+
521+
LOCAL_MODEL_DIR=$(mktemp -d)
522+
INDUCTOR_CACHE=$(mktemp -d "${RUNNER_TEMP:-/tmp}/inductor_cache_XXXXXX")
523+
INDUCTOR_TMPDIR=$(mktemp -d "${RUNNER_TEMP:-/tmp}/tmpdir_XXXXXX")
524+
trap 'rm -rf "$LOCAL_MODEL_DIR" "$INDUCTOR_CACHE" "$INDUCTOR_TMPDIR"' EXIT
525+
526+
case "$QUANT_NAME" in
527+
kquant-17gb)
528+
TARGET_GGUF_FILE="muse-glimmer-30B-kquant-17gb.gguf"
529+
;;
530+
kquant-dynamic)
531+
TARGET_GGUF_FILE="muse-glimmer-30B-kquant-dynamic.gguf"
532+
;;
533+
esac
534+
535+
python -c "from huggingface_hub import hf_hub_download; hf_hub_download('${HF_MODEL}', '${TARGET_GGUF_FILE}', local_dir='${LOCAL_MODEL_DIR}')"
536+
TARGET_GGUF_PATH="${LOCAL_MODEL_DIR}/${TARGET_GGUF_FILE}"
537+
538+
echo "::group::Export"
539+
case "$MODE" in
540+
solo-text)
541+
EXPORT_START_SECONDS=$SECONDS
542+
TMPDIR="$INDUCTOR_TMPDIR" \
543+
TORCHINDUCTOR_CACHE_DIR="$INDUCTOR_CACHE" \
544+
python -m executorch.examples.models.muse_glimmer.export.export_solo \
545+
--gguf "$TARGET_GGUF_PATH" \
546+
--backend cuda \
547+
--output-dir "${OUTPUT_DIR}"
548+
;;
549+
dflash-image)
550+
DRAFT_GGUF_FILE="dflash-kquant.gguf"
551+
MMPROJ_GGUF_FILE="mmproj-kquant.gguf"
552+
python -c "from huggingface_hub import hf_hub_download; hf_hub_download('${HF_MODEL}', '${DRAFT_GGUF_FILE}', local_dir='${LOCAL_MODEL_DIR}')"
553+
python -c "from huggingface_hub import hf_hub_download; hf_hub_download('${HF_MODEL}', '${MMPROJ_GGUF_FILE}', local_dir='${LOCAL_MODEL_DIR}')"
554+
EXPORT_START_SECONDS=$SECONDS
555+
TMPDIR="$INDUCTOR_TMPDIR" \
556+
TORCHINDUCTOR_CACHE_DIR="$INDUCTOR_CACHE" \
557+
python -m executorch.examples.models.muse_glimmer.export.export_dflash \
558+
--target-gguf "$TARGET_GGUF_PATH" \
559+
--draft-gguf "${LOCAL_MODEL_DIR}/${DRAFT_GGUF_FILE}" \
560+
--mmproj "${LOCAL_MODEL_DIR}/${MMPROJ_GGUF_FILE}" \
561+
--backend cuda \
562+
--output-dir "${OUTPUT_DIR}"
563+
;;
564+
*)
565+
echo "Error: Muse Glimmer requires mode 'solo-text' or 'dflash-image'"
566+
exit 1
567+
;;
568+
esac
569+
EXPORT_DURATION_SECONDS=$((SECONDS - EXPORT_START_SECONDS))
570+
EXPORT_DURATION_MINUTES=$(awk -v seconds="$EXPORT_DURATION_SECONDS" 'BEGIN {printf "%.2f", seconds / 60}')
571+
echo "Muse Glimmer took ${EXPORT_DURATION_MINUTES} minutes to export."
572+
echo "::endgroup::"
573+
574+
test -f "${OUTPUT_DIR}/model.pte"
575+
test -f "${OUTPUT_DIR}/aoti_cuda_blob.ptd"
576+
if [ "$MODE" = "dflash-image" ]; then
577+
test -f "${OUTPUT_DIR}/pos_embed.bin"
578+
fi
579+
ls -al "${OUTPUT_DIR}"
580+
581+
exit 0
582+
fi
583+
473584
# Gemma 4 31B: download the Q4_K_M GGUF and export via the GGUF loader
474585
if [ "$MODEL_NAME" = "gemma4_31b" ]; then
475586
pip install safetensors huggingface_hub gguf

‎.ci/scripts/test_model_e2e.sh‎

Lines changed: 80 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
# This source code is licensed under the BSD-style license found in the
66
# LICENSE file in the root directory of this source tree.
77

8-
# Test CUDA/Metal/XNNPACK model end-to-end, need to run .ci/scripts/export_model_artifact.sh first
8+
# Test CUDA/Metal/XNNPACK models end-to-end, optionally exporting artifacts first.
99

1010
show_help() {
1111
cat << EOF
@@ -26,13 +26,16 @@ Arguments:
2626
- nvidia/parakeet-tdt
2727
- facebook/dinov2-small-imagenet1k-1-layer
2828
- mistralai/Voxtral-Mini-4B-Realtime-2602
29+
- meta-models/Muse-Glimmer-30B-GGUF
2930
3031
quant_name Quantization type (required)
3132
Options:
3233
- non-quantized
3334
- quantized-int4-tile-packed
3435
- quantized-int4-weight-only
3536
- quantized-8da4w (XNNPACK only)
37+
- kquant-17gb (Muse Glimmer only)
38+
- kquant-dynamic (Muse Glimmer only)
3639
3740
model_dir Directory containing model artifacts (optional, default: current directory)
3841
Expected files: model.pte, aoti_cuda_blob.ptd (CUDA only)
@@ -42,6 +45,11 @@ Arguments:
4245
Supported modes:
4346
- vr-streaming: Voxtral Realtime streaming mode
4447
- vr-offline: Voxtral Realtime offline mode
48+
- solo-text: Muse Glimmer solo text mode
49+
- dflash-image: Muse Glimmer DFlash vision mode
50+
51+
Environment:
52+
RUN_EXPORT Set to 1 to export artifacts with export_model_artifact.sh before testing
4553
4654
Examples:
4755
test_model_e2e.sh metal "openai/whisper-small" "non-quantized"
@@ -51,6 +59,7 @@ Examples:
5159
test_model_e2e.sh xnnpack "nvidia/parakeet-tdt" "quantized-8da4w" "./model_output"
5260
test_model_e2e.sh metal "mistralai/Voxtral-Mini-4B-Realtime-2602" "non-quantized" "." "vr-streaming"
5361
test_model_e2e.sh xnnpack "mistralai/Voxtral-Mini-4B-Realtime-2602" "quantized-8da4w" "./model_output" "vr-offline"
62+
RUN_EXPORT=1 test_model_e2e.sh cuda "meta-models/Muse-Glimmer-30B-GGUF" "kquant-17gb" "./model_output" "solo-text"
5463
EOF
5564
}
5665

@@ -80,6 +89,17 @@ QUANT_NAME="$3"
8089
MODEL_DIR="${4:-.}"
8190
MODE="${5:-}"
8291

92+
# Locate EXECUTORCH_ROOT from the directory of this script.
93+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
94+
EXECUTORCH_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
95+
RUN_EXPORT="${RUN_EXPORT:-0}"
96+
if [ "$RUN_EXPORT" = "1" ]; then
97+
mkdir -p "$MODEL_DIR"
98+
fi
99+
if [ -d "$MODEL_DIR" ]; then
100+
MODEL_DIR="$(cd "$MODEL_DIR" && pwd)"
101+
fi
102+
83103
# Validate mode if specified
84104
if [ -n "$MODE" ]; then
85105
case "$MODE" in
@@ -91,16 +111,35 @@ if [ -n "$MODE" ]; then
91111
exit 1
92112
fi
93113
;;
114+
solo-text|dflash-image)
115+
if [ "$HF_MODEL" != "meta-models/Muse-Glimmer-30B-GGUF" ]; then
116+
echo "Error: Mode '$MODE' can only be used with Muse Glimmer model"
117+
echo "Provided model: $HF_MODEL"
118+
exit 1
119+
fi
120+
;;
94121
*)
95122
echo "Error: Unsupported mode '$MODE'"
96-
echo "Supported modes: vr-streaming, vr-offline"
123+
echo "Supported modes: vr-streaming, vr-offline, solo-text, dflash-image"
97124
exit 1
98125
;;
99126
esac
100127
fi
101128

102129
echo "Testing model: $HF_MODEL (quantization: $QUANT_NAME)"
103130

131+
if [ "$HF_MODEL" = "meta-models/Muse-Glimmer-30B-GGUF" ] && [ "$DEVICE" != "cuda" ]; then
132+
echo "Error: Muse Glimmer is only supported with the cuda device"
133+
exit 1
134+
fi
135+
136+
if [ "$RUN_EXPORT" = "1" ]; then
137+
(
138+
cd "$EXECUTORCH_ROOT"
139+
bash "$SCRIPT_DIR/export_model_artifact.sh" "$DEVICE" "$HF_MODEL" "$QUANT_NAME" "$MODEL_DIR" "$MODE"
140+
)
141+
fi
142+
104143
# Make sure model.pte exists
105144
if [ ! -f "$MODEL_DIR/model.pte" ]; then
106145
echo "Error: model.pte not found in $MODEL_DIR"
@@ -111,10 +150,6 @@ if [ "$DEVICE" = "cuda" ] && [ ! -f "$MODEL_DIR/aoti_cuda_blob.ptd" ]; then
111150
echo "Error: aoti_cuda_blob.ptd not found in $MODEL_DIR"
112151
exit 1
113152
fi
114-
# Locate EXECUTORCH_ROOT from the directory of this script
115-
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
116-
EXECUTORCH_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
117-
118153
pushd "$EXECUTORCH_ROOT"
119154

120155
# Determine model configuration based on HF model ID
@@ -240,9 +275,35 @@ case "$HF_MODEL" in
240275
AUDIO_FILE=""
241276
IMAGE_PATH=""
242277
;;
278+
meta-models/Muse-Glimmer-30B-GGUF)
279+
MODEL_NAME="muse_glimmer"
280+
RUNNER_PATH="muse-glimmer"
281+
PREPROCESSOR=""
282+
TOKENIZER_URL="https://huggingface.co/meta-models/Muse-Glimmer-30B/resolve/main" # @lint-ignore
283+
TOKENIZER_FILE="tokenizer.json"
284+
AUDIO_URL=""
285+
AUDIO_FILE=""
286+
IMAGE_PATH=""
287+
case "$MODE" in
288+
solo-text)
289+
RUNNER_TARGET="solo_runner"
290+
EXPECTED_OUTPUT="Paris"
291+
IMAGE_URL=""
292+
;;
293+
dflash-image)
294+
RUNNER_TARGET="dflash_runner"
295+
EXPECTED_OUTPUT="dog"
296+
IMAGE_URL="https://github.com/pytorch/hub/raw/master/images/dog.jpg"
297+
;;
298+
*)
299+
echo "Error: Muse Glimmer requires mode 'solo-text' or 'dflash-image'"
300+
exit 1
301+
;;
302+
esac
303+
;;
243304
*)
244305
echo "Error: Unsupported model '$HF_MODEL'"
245-
echo "Supported models: mistralai/Voxtral-Mini-3B-2507, mistralai/Voxtral-Mini-4B-Realtime-2602, nvidia/diar_streaming_sortformer_4spk-v2, openai/whisper series (whisper-{small, medium, large, large-v2, large-v3, large-v3-turbo}), google/gemma-3-4b-it, Qwen/Qwen3-0.6B, nvidia/parakeet-tdt, facebook/dinov2-small-imagenet1k-1-layer, SocialLocalMobile/Qwen3.5-35B-A3B-HQQ-INT4, unsloth/gemma-4-31B-it-GGUF"
306+
echo "Supported models: mistralai/Voxtral-Mini-3B-2507, mistralai/Voxtral-Mini-4B-Realtime-2602, nvidia/diar_streaming_sortformer_4spk-v2, openai/whisper series (whisper-{small, medium, large, large-v2, large-v3, large-v3-turbo}), google/gemma-3-4b-it, Qwen/Qwen3-0.6B, nvidia/parakeet-tdt, facebook/dinov2-small-imagenet1k-1-layer, SocialLocalMobile/Qwen3.5-35B-A3B-HQQ-INT4, unsloth/gemma-4-31B-it-GGUF, meta-models/Muse-Glimmer-30B-GGUF"
246307
exit 1
247308
;;
248309
esac
@@ -383,6 +444,18 @@ EOF
383444
gemma4_31b)
384445
RUNNER_ARGS="$RUNNER_ARGS --tokenizer_path ${MODEL_DIR}/$TOKENIZER_FILE --prompt 'What is the capital of France?' --max_new_tokens 128 --temperature 0 --cuda_graph"
385446
;;
447+
muse_glimmer)
448+
PROMPT_FILE="${MODEL_DIR}/muse_glimmer_prompt.txt"
449+
if [ "$MODE" = "solo-text" ]; then
450+
printf '%s' '<|start|>user<|message|>What is the capital of France?<|eot|><|start|>assistant' > "$PROMPT_FILE"
451+
else
452+
printf '%s' '<|start|>user<|message|>What animal is in this image? <img><|eot|><|start|>assistant' > "$PROMPT_FILE"
453+
fi
454+
RUNNER_ARGS="$RUNNER_ARGS --tokenizer_path ${MODEL_DIR}/$TOKENIZER_FILE --prompt_file \"$PROMPT_FILE\" --max_new_tokens 512 --cuda_graph"
455+
if [ "$MODE" = "dflash-image" ]; then
456+
RUNNER_ARGS="$RUNNER_ARGS --image_path ${MODEL_DIR}/test_image.jpg"
457+
fi
458+
;;
386459
voxtral_realtime)
387460
RUNNER_ARGS="--model_path ${MODEL_DIR}/model.pte --tokenizer_path ${MODEL_DIR}/$TOKENIZER_FILE --preprocessor_path ${MODEL_DIR}/$PREPROCESSOR --audio_path ${MODEL_DIR}/$AUDIO_FILE --temperature 0"
388461
# Add CUDA data path if present

0 commit comments

Comments
 (0)