Skip to content

Commit 9f209bd

Browse files
authored
Collapse metal export and test jobs (pytorch#22848)
### Summary **Problem:** The metal workflow currently exports each model in one job, uploads it to GitHub as an artifact, then downloads it in a second job just to run the test. Nothing else ever touches these artifacts, and they add up to about **18 GB per run**. **Solution:** I moved the export into the test job so the model never leaves the runner. `test_model_e2e.sh` already supports this with `RUN_EXPORT=1`. Also, each matrix cell is independent now, so one broken export doesn't skip all nine tests anymore. ### Test plan CI
1 parent c89046c commit 9f209bd

2 files changed

Lines changed: 31 additions & 88 deletions

File tree

‎.github/workflows/_ci-run-decision.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ name: CI Run Decision
2525
# workflow. The other seven callers (cuda, cuda-windows, cuda-perf,
2626
# metal, mlx, rocm, vulkan) do have ciflow tags, but none of them listens
2727
# for ``ciflow/trunk``, so a promotion tag never starts them, and they
28-
# hold 32 of the 48 jobs whose ``if:`` gates on ``is-full-run``. Most of
28+
# hold 32 of the 47 jobs whose ``if:`` gates on ``is-full-run``. Most of
2929
# those 32 also have a changed-files branch and still run on main when
3030
# their own paths are touched; the 12 in mlx.yml do not, so for those the
3131
# depth sample below is the only thing that runs them automatically on a

‎.github/workflows/metal.yml‎

Lines changed: 30 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -201,8 +201,8 @@ jobs:
201201
echo "Success: prefill ($PROMPT_TOKENS tokens) + decode completed"
202202
echo "::endgroup::"
203203
204-
export-model-metal-artifact:
205-
name: export-model-metal-artifact
204+
test-model-metal-e2e:
205+
name: test-model-metal-e2e
206206
# Skip this job if the pull request is from a fork (HuggingFace secrets are not available).
207207
# Path-filtered on push: mirrors the workflow-level pull_request `paths:`
208208
# filter so push commits that don't touch metal-relevant paths skip
@@ -251,10 +251,33 @@ jobs:
251251
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
252252
timeout: 90
253253
secrets-env: EXECUTORCH_HF_TOKEN
254-
upload-artifact: ${{ matrix.model.repo }}-${{ matrix.model.name }}-metal-${{ matrix.quant }}
255254
script: |
256255
set -eux
257256
257+
echo "::group::Print machine info"
258+
uname -a
259+
if [ $(uname -s) == Darwin ]; then
260+
sw_vers
261+
# Print RAM in GB
262+
RAM_BYTES=$(sysctl -n hw.memsize)
263+
RAM_GB=$(echo "scale=2; $RAM_BYTES/1024/1024/1024" | bc)
264+
echo "Available RAM (GB): $RAM_GB"
265+
sysctl machdep.cpu.brand_string
266+
sysctl machdep.cpu.core_count
267+
# Print number of GPU cores (Apple Silicon)
268+
if command -v system_profiler &> /dev/null; then
269+
GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Total Number of Cores/ {print $5; exit}')
270+
if [ -z "$GPU_CORES" ]; then
271+
# Fallback: try to parse "Core Count" from Apple GPU section
272+
GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Core Count/ {print $3; exit}')
273+
fi
274+
echo "GPU Cores: ${GPU_CORES:-Unknown}"
275+
else
276+
echo "system_profiler not available, cannot determine GPU cores."
277+
fi
278+
fi
279+
echo "::endgroup::"
280+
258281
echo "::group::Setup Huggingface"
259282
${CONDA_RUN} pip install -U "huggingface_hub[cli]<1.0" accelerate
260283
${CONDA_RUN} huggingface-cli login --token $SECRET_EXECUTORCH_HF_TOKEN
@@ -284,87 +307,7 @@ jobs:
284307
if [ "${{ matrix.model.name }}" = "Voxtral-Mini-4B-Realtime-2602" ]; then
285308
VR_MODE="vr-offline"
286309
fi
287-
${CONDA_RUN} bash .ci/scripts/export_model_artifact.sh metal "${{ matrix.model.repo }}/${{ matrix.model.name }}" "${{ matrix.quant }}" "${RUNNER_ARTIFACT_DIR}" "${VR_MODE}"
288-
289-
test-model-metal-e2e:
290-
name: test-model-metal-e2e
291-
# Same path filter as export-model-metal-artifact above. Also
292-
# explicitly gated on the export job succeeding — when needs: jobs
293-
# are *skipped* (e.g. fork PR), GitHub still evaluates this if:,
294-
# so without the explicit success-check this job would run and then
295-
# fail trying to download an artifact that was never produced.
296-
needs: [changed-files, export-model-metal-artifact, run-decision]
297-
if: |
298-
needs.export-model-metal-artifact.result == 'success' &&
299-
(
300-
contains(needs.changed-files.outputs.changed-files, 'backends/apple/metal') ||
301-
contains(needs.changed-files.outputs.changed-files, 'backends/aoti') ||
302-
contains(needs.changed-files.outputs.changed-files, 'examples/models/qwen3_5_moe') ||
303-
contains(needs.changed-files.outputs.changed-files, 'extension/llm/export') ||
304-
contains(needs.changed-files.outputs.changed-files, '.github/workflows/metal.yml') ||
305-
needs.run-decision.outputs.is-full-run == 'true'
306-
)
307-
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
308-
strategy:
309-
fail-fast: false
310-
matrix:
311-
model:
312-
- repo: "mistralai"
313-
name: "Voxtral-Mini-3B-2507"
314-
- repo: "mistralai"
315-
name: "Voxtral-Mini-4B-Realtime-2602"
316-
- repo: "openai"
317-
name: "whisper-small"
318-
- repo: "openai"
319-
name: "whisper-large-v3-turbo"
320-
- repo: "nvidia"
321-
name: "parakeet-tdt"
322-
quant:
323-
- "non-quantized"
324-
- "quantized-int4-metal"
325-
exclude:
326-
# Exclude non-quantized for Voxtral Realtime (too large)
327-
- model:
328-
repo: "mistralai"
329-
name: "Voxtral-Mini-4B-Realtime-2602"
330-
quant: "non-quantized"
331-
with:
332-
default-packages: ""
333-
runner: macos-m2-stable
334-
python-version: '3.11'
335-
submodules: 'recursive'
336-
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
337-
timeout: 90
338-
download-artifact: ${{ matrix.model.repo }}-${{ matrix.model.name }}-metal-${{ matrix.quant }}
339-
script: |
340-
set -eux
341-
342-
echo "::group::Print machine info"
343-
uname -a
344-
if [ $(uname -s) == Darwin ]; then
345-
sw_vers
346-
# Print RAM in GB
347-
RAM_BYTES=$(sysctl -n hw.memsize)
348-
RAM_GB=$(echo "scale=2; $RAM_BYTES/1024/1024/1024" | bc)
349-
echo "Available RAM (GB): $RAM_GB"
350-
sysctl machdep.cpu.brand_string
351-
sysctl machdep.cpu.core_count
352-
# Print number of GPU cores (Apple Silicon)
353-
if command -v system_profiler &> /dev/null; then
354-
GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Total Number of Cores/ {print $5; exit}')
355-
if [ -z "$GPU_CORES" ]; then
356-
# Fallback: try to parse "Core Count" from Apple GPU section
357-
GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Core Count/ {print $3; exit}')
358-
fi
359-
echo "GPU Cores: ${GPU_CORES:-Unknown}"
360-
else
361-
echo "system_profiler not available, cannot determine GPU cores."
362-
fi
363-
fi
364-
echo "::endgroup::"
365-
366-
VR_MODE=""
367-
if [ "${{ matrix.model.name }}" = "Voxtral-Mini-4B-Realtime-2602" ]; then
368-
VR_MODE="vr-offline"
369-
fi
370-
${CONDA_RUN} bash .ci/scripts/test_model_e2e.sh metal "${{ matrix.model.repo }}/${{ matrix.model.name }}" "${{ matrix.quant }}" "${RUNNER_ARTIFACT_DIR}" "${VR_MODE}"
310+
# Export and test in the same job so the model never leaves the runner
311+
# (no GitHub Actions artifact upload/download).
312+
MODEL_DIR="$(mktemp -d "${RUNNER_TEMP}/metal_model_XXXXXX")"
313+
${CONDA_RUN} RUN_EXPORT=1 bash .ci/scripts/test_model_e2e.sh metal "${{ matrix.model.repo }}/${{ matrix.model.name }}" "${{ matrix.quant }}" "${MODEL_DIR}" "${VR_MODE}"

0 commit comments

Comments
 (0)