Skip to content

Commit 5c5c9ee

Browse files
authored
Stop uploading Voxtral checkpoints and partial exports when CI exports fail (pytorch#22622)
### Summary Internal context: https://docs.google.com/document/d/1CnGfiP5SU0PG0IdJWn9Ca43LS8bGlEMi3l2gjgC7k48/edit When a job fails, the output dir is uploaded to GHA storage. This causes a significant increase in storage for Voxtral models: up to 0.7 TB/day. Like [Qwen3.5](https://github.com/pytorch/executorch/blob/9b3f660b4a74afa87726fbbb4480196d02fcb9e9/.ci/scripts/export_model_artifact.sh#L452-L455) and [Muse Glimmer](https://github.com/pytorch/executorch/blob/9b3f660b4a74afa87726fbbb4480196d02fcb9e9/.ci/scripts/export_model_artifact.sh#L524-L527), let's upload the Voxtral models to a temporary directory instead. In this PR we generalize the download location and cleanup for the models. ### Test plan CI
1 parent d9b3e45 commit 5c5c9ee

1 file changed

Lines changed: 28 additions & 13 deletions

File tree

‎.ci/scripts/export_model_artifact.sh‎

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,26 @@ if [ "$MODEL_NAME" = "muse_glimmer" ]; then
294294
fi
295295
fi
296296

297+
# Downloads and compiler caches go in scratch dirs outside OUTPUT_DIR because the CI job
298+
# templates upload OUTPUT_DIR even when the job fails. A failed export also empties
299+
# OUTPUT_DIR, but only if it started out empty, so a local run with output_dir=. cannot
300+
# delete the checkout. Scratch goes under RUNNER_TEMP, which the runner wipes between
301+
# jobs, with a fallback for containers where RUNNER_TEMP is not writable.
302+
LOCAL_MODEL_DIR=$(mktemp -d "${RUNNER_TEMP:-/tmp}/model_XXXXXX" 2>/dev/null || mktemp -d)
303+
SCRATCH_DIRS=("$LOCAL_MODEL_DIR")
304+
OUTPUT_DIR_WAS_EMPTY=0
305+
[ -n "$(ls -A -- "$OUTPUT_DIR" 2>/dev/null)" ] || OUTPUT_DIR_WAS_EMPTY=1
306+
cleanup() {
307+
local rc=$?
308+
set +e
309+
rm -rf "${SCRATCH_DIRS[@]}"
310+
if [ "$rc" -ne 0 ] && [ "$OUTPUT_DIR_WAS_EMPTY" = 1 ] && [ -d "$OUTPUT_DIR" ]; then
311+
echo "Export failed with exit code $rc; removing partial output from ${OUTPUT_DIR}"
312+
(cd -- "$OUTPUT_DIR" && find . -mindepth 1 -delete)
313+
fi
314+
}
315+
trap cleanup EXIT
316+
297317
echo "::group::Export $MODEL_NAME"
298318

299319
if [ -n "$EXTRA_PIP" ]; then
@@ -386,8 +406,7 @@ fi
386406
if [ "$MODEL_NAME" = "voxtral_realtime" ]; then
387407
pip install safetensors huggingface_hub
388408

389-
# Download model weights from HuggingFace (requires HF_TOKEN for gated model)
390-
LOCAL_MODEL_DIR="${OUTPUT_DIR}/model_weights"
409+
# Download model weights outside OUTPUT_DIR to avoid uploading on failure (requires HF_TOKEN for gated model)
391410
python -c "from huggingface_hub import snapshot_download; snapshot_download('${HF_MODEL}', local_dir='${LOCAL_MODEL_DIR}')"
392411

393412
# Per-component quantization flags
@@ -437,7 +456,6 @@ if [ "$MODEL_NAME" = "voxtral_realtime" ]; then
437456
fi
438457
# Copy tokenizer from downloaded model weights
439458
cp "$LOCAL_MODEL_DIR/tekken.json" "${OUTPUT_DIR}/tekken.json"
440-
rm -rf "$LOCAL_MODEL_DIR"
441459
ls -al "${OUTPUT_DIR}"
442460
echo "::endgroup::"
443461
exit 0
@@ -448,12 +466,11 @@ if [ "$MODEL_NAME" = "qwen3_5_moe" ]; then
448466
pip install safetensors huggingface_hub
449467
pip install -r examples/models/qwen3_5_moe/requirements.txt
450468

451-
# Download prequantized model outside OUTPUT_DIR to avoid uploading on failure
452-
LOCAL_MODEL_DIR=$(mktemp -d)
453469
INDUCTOR_CACHE=$(mktemp -d "${RUNNER_TEMP:-/tmp}/inductor_cache_XXXXXX")
454470
INDUCTOR_TMPDIR=$(mktemp -d "${RUNNER_TEMP:-/tmp}/tmpdir_XXXXXX")
455-
trap 'rm -rf "$LOCAL_MODEL_DIR" "$INDUCTOR_CACHE" "$INDUCTOR_TMPDIR"' EXIT
471+
SCRATCH_DIRS+=("$INDUCTOR_CACHE" "$INDUCTOR_TMPDIR")
456472

473+
# Download prequantized model outside OUTPUT_DIR to avoid uploading on failure
457474
python -c "from huggingface_hub import snapshot_download; snapshot_download('${HF_MODEL}', local_dir='${LOCAL_MODEL_DIR}')"
458475

459476
# Sanity check: run inference on the prequantized model
@@ -521,10 +538,9 @@ fi
521538
if [ "$MODEL_NAME" = "muse_glimmer" ]; then
522539
pip install safetensors huggingface_hub gguf
523540

524-
LOCAL_MODEL_DIR=$(mktemp -d)
525541
INDUCTOR_CACHE=$(mktemp -d "${RUNNER_TEMP:-/tmp}/inductor_cache_XXXXXX")
526542
INDUCTOR_TMPDIR=$(mktemp -d "${RUNNER_TEMP:-/tmp}/tmpdir_XXXXXX")
527-
trap 'rm -rf "$LOCAL_MODEL_DIR" "$INDUCTOR_CACHE" "$INDUCTOR_TMPDIR"' EXIT
543+
SCRATCH_DIRS+=("$INDUCTOR_CACHE" "$INDUCTOR_TMPDIR")
528544

529545
case "$QUANT_NAME" in
530546
kquant-17gb)
@@ -588,14 +604,13 @@ fi
588604
if [ "$MODEL_NAME" = "gemma4_31b" ]; then
589605
pip install safetensors huggingface_hub gguf
590606

591-
# Download GGUF + tokenizer outside OUTPUT_DIR to avoid uploading on failure.
592-
# The unsloth GGUF repo ships the .gguf but no tokenizer.json, so the tokenizer
593-
# is fetched from the (non-GGUF) unsloth/gemma-4-31B-it repo.
594-
LOCAL_MODEL_DIR=$(mktemp -d)
595607
INDUCTOR_CACHE=$(mktemp -d "${RUNNER_TEMP:-/tmp}/inductor_cache_XXXXXX")
596608
INDUCTOR_TMPDIR=$(mktemp -d "${RUNNER_TEMP:-/tmp}/tmpdir_XXXXXX")
597-
trap 'rm -rf "$LOCAL_MODEL_DIR" "$INDUCTOR_CACHE" "$INDUCTOR_TMPDIR"' EXIT
609+
SCRATCH_DIRS+=("$INDUCTOR_CACHE" "$INDUCTOR_TMPDIR")
598610

611+
# Download GGUF + tokenizer outside OUTPUT_DIR to avoid uploading on failure.
612+
# The unsloth GGUF repo ships the .gguf but no tokenizer.json, so the tokenizer
613+
# is fetched from the (non-GGUF) unsloth/gemma-4-31B-it repo.
599614
GGUF_FILE="gemma-4-31B-it-Q4_K_M.gguf"
600615
python -c "from huggingface_hub import hf_hub_download; hf_hub_download('unsloth/gemma-4-31B-it-GGUF', '${GGUF_FILE}', local_dir='${LOCAL_MODEL_DIR}')"
601616
python -c "from huggingface_hub import hf_hub_download; hf_hub_download('unsloth/gemma-4-31B-it', 'tokenizer.json', local_dir='${LOCAL_MODEL_DIR}')"

0 commit comments

Comments
 (0)