Skip to content

Commit 0aa375a

Browse files
authored
Collapse cuda-perf export and test jobs (pytorch#22854)
### Summary Same idea as pytorch#22848, for cuda-perf. **Problem:** The cuda-perf workflow currently exports each model in one job, uploads it to GitHub as an artifact, then downloads it in a second job to build the runner and benchmark it. Nothing else ever touches these artifacts, and across the 19 matrix cells they add up to about **42 GB per run**. **Solution:** I moved the export into the benchmark job so the model never leaves the runner. `test_model_e2e.sh` already supports this with `RUN_EXPORT=1`, and the benchmark itself didn't need to change since it reads from the same directory. The job timeout goes from 90 to 120 minutes because one job now does both halves. A failed export also no longer spawns a benchmark cell that just fails two minutes in. ### Test plan CI
1 parent 11cd1b8 commit 0aa375a

1 file changed

Lines changed: 22 additions & 122 deletions

File tree

‎.github/workflows/cuda-perf.yml‎

Lines changed: 22 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -121,9 +121,21 @@ jobs:
121121
echo "Generated benchmark configs:"
122122
echo "$CONFIGS" | python -m json.tool
123123
124-
export-models:
125-
name: export-models
126-
needs: set-parameters
124+
benchmark-cuda:
125+
name: benchmark-cuda
126+
needs:
127+
- changed-files
128+
- run-decision
129+
- set-parameters
130+
# Same gate as set-parameters, repeated so the job reads on its own;
131+
# the needs on set-parameters already skips it when the gate is closed.
132+
if: |
133+
contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda-perf.yml') ||
134+
contains(needs.changed-files.outputs.changed-files, '.ci/scripts/cuda_benchmark.py') ||
135+
contains(needs.changed-files.outputs.changed-files, '.ci/scripts/cuda_perf_prompts') ||
136+
contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') ||
137+
contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e.sh') ||
138+
needs.run-decision.outputs.is-full-run == 'true'
127139
uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
128140
permissions:
129141
id-token: write
@@ -133,14 +145,14 @@ jobs:
133145
matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }}
134146
fail-fast: false
135147
with:
136-
timeout: 90
148+
timeout: 120
137149
secrets-env: EXECUTORCH_HF_TOKEN
138150
runner: ${{ contains(matrix.model, 'Qwen3.5-35B-A3B') && 'mt-l-x86iavx512-11-125-a100' || 'mt-l-x86aavx2-29-113-a10g' }}
139151
gpu-arch-type: cuda
140152
gpu-arch-version: "13.0"
141153
use-custom-docker-registry: false
142154
submodules: recursive
143-
upload-artifact: model-${{ matrix.model_safe }}-${{ matrix.quant }}
155+
upload-artifact: results-${{ matrix.model_safe }}-${{ matrix.quant }}
144156
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
145157
script: |
146158
set -eux
@@ -172,118 +184,10 @@ jobs:
172184
pip install --no-deps git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
173185
echo "::endgroup::"
174186
175-
echo "::group::Exporting model ${{ matrix.model }} with quantization ${{ matrix.quant }}"
176-
OUTPUT_DIR="model_artifacts"
177-
mkdir -p "$OUTPUT_DIR"
178-
179-
bash .ci/scripts/export_model_artifact.sh cuda "${{ matrix.model }}" "${{ matrix.quant }}" "$OUTPUT_DIR"
180-
181-
# Move artifacts to RUNNER_ARTIFACT_DIR for upload
182-
mv "$OUTPUT_DIR"/* "${RUNNER_ARTIFACT_DIR}/"
183-
ls -lah "${RUNNER_ARTIFACT_DIR}"
184-
echo "::endgroup::"
185-
186-
benchmark-cuda:
187-
name: benchmark-cuda
188-
needs:
189-
- changed-files
190-
- run-decision
191-
- set-parameters
192-
- export-models
193-
# Inherit the gate from set-parameters/export-models (they cascade-
194-
# skip when the gate evaluates false). `always()` keeps benchmark-
195-
# cuda running even when some export-models matrix cells fail —
196-
# but only if the gate itself is open. Without the explicit gate
197-
# here, `always()` would fire benchmark-cuda even when set-
198-
# parameters was gated out.
199-
if: |
200-
always() &&
201-
(
202-
contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda-perf.yml') ||
203-
contains(needs.changed-files.outputs.changed-files, '.ci/scripts/cuda_benchmark.py') ||
204-
contains(needs.changed-files.outputs.changed-files, '.ci/scripts/cuda_perf_prompts') ||
205-
contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') ||
206-
contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e.sh') ||
207-
needs.run-decision.outputs.is-full-run == 'true'
208-
)
209-
uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
210-
permissions:
211-
id-token: write
212-
contents: read
213-
strategy:
214-
matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }}
215-
fail-fast: false
216-
with:
217-
timeout: 90
218-
runner: ${{ contains(matrix.model, 'Qwen3.5-35B-A3B') && 'mt-l-x86iavx512-11-125-a100' || 'mt-l-x86aavx2-29-113-a10g' }}
219-
gpu-arch-type: cuda
220-
gpu-arch-version: "13.0"
221-
use-custom-docker-registry: false
222-
submodules: recursive
223-
download-artifact: model-${{ matrix.model_safe }}-${{ matrix.quant }}
224-
upload-artifact: results-${{ matrix.model_safe }}-${{ matrix.quant }}
225-
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
226-
script: |
227-
set -eux
228-
229-
# OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
230-
# (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
231-
export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
232-
mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
233-
mkdir -p "${HF_HOME}"
234-
235-
echo "::group::Setup environment"
236-
# OSDC runners can't reach the public PyPI CDN that download.pytorch.org's
237-
# transitive deps resolve to. Pre-install torch's pure-python deps from the
238-
# in-cluster pypi-cache and drop the default cpu extra-index so the cuda
239-
# torch wheel is the only candidate.
240-
export PIP_EXTRA_INDEX_URL=
241-
# fsspec is pinned to satisfy datasets' fsspec[http]<=2025.3.0 so the later
242-
# examples install doesn't try to downgrade it from the public CDN.
243-
pip install filelock typing-extensions "setuptools<82" sympy networkx jinja2 "fsspec[http]<=2025.3.0" numpy pillow
244-
./install_requirements.sh
245-
pip list
246-
echo "::endgroup::"
247-
248-
echo "::group::Prepare model artifacts"
249-
mkdir -p model_artifacts
250-
cp "${RUNNER_ARTIFACT_DIR}/model.pte" model_artifacts/model.pte
251-
cp "${RUNNER_ARTIFACT_DIR}/aoti_cuda_blob.ptd" model_artifacts/aoti_cuda_blob.ptd
252-
253-
# Copy additional files if they exist
254-
if [ -f "${RUNNER_ARTIFACT_DIR}/voxtral_preprocessor.pte" ]; then
255-
cp "${RUNNER_ARTIFACT_DIR}/voxtral_preprocessor.pte" model_artifacts/
256-
fi
257-
if [ -f "${RUNNER_ARTIFACT_DIR}/whisper_preprocessor.pte" ]; then
258-
cp "${RUNNER_ARTIFACT_DIR}/whisper_preprocessor.pte" model_artifacts/
259-
fi
260-
if [ -f "${RUNNER_ARTIFACT_DIR}/tekken.json" ]; then
261-
cp "${RUNNER_ARTIFACT_DIR}/tekken.json" model_artifacts/
262-
fi
263-
if [ -f "${RUNNER_ARTIFACT_DIR}/poem.wav" ]; then
264-
cp "${RUNNER_ARTIFACT_DIR}/poem.wav" model_artifacts/
265-
fi
266-
if [ -f "${RUNNER_ARTIFACT_DIR}/output.wav" ]; then
267-
cp "${RUNNER_ARTIFACT_DIR}/output.wav" model_artifacts/
268-
fi
269-
if [ -f "${RUNNER_ARTIFACT_DIR}/tokenizer.model" ]; then
270-
cp "${RUNNER_ARTIFACT_DIR}/tokenizer.model" model_artifacts/
271-
fi
272-
if [ -f "${RUNNER_ARTIFACT_DIR}/test_audio.wav" ]; then
273-
cp "${RUNNER_ARTIFACT_DIR}/test_audio.wav" model_artifacts/
274-
fi
275-
# Copy tokenizer files
276-
for file in tokenizer.json tokenizer_config.json special_tokens_map.json; do
277-
if [ -f "${RUNNER_ARTIFACT_DIR}/$file" ]; then
278-
cp "${RUNNER_ARTIFACT_DIR}/$file" model_artifacts/
279-
fi
280-
done
281-
282-
ls -lah model_artifacts/
283-
echo "::endgroup::"
284-
285-
echo "::group::Build runner"
286-
bash .ci/scripts/test_model_e2e.sh cuda "${{ matrix.model }}" "${{ matrix.quant }}" model_artifacts
187+
echo "::group::Export model and build runner"
188+
# Export and benchmark in the same job so the model never leaves the runner
189+
# (no GitHub Actions artifact upload/download).
190+
RUN_EXPORT=1 bash .ci/scripts/test_model_e2e.sh cuda "${{ matrix.model }}" "${{ matrix.quant }}" model_artifacts
287191
echo "::endgroup::"
288192
289193
echo "::group::Running benchmark for ${{ matrix.model }} (${{ matrix.quant }}) with ${{ matrix.num_runs }} runs"
@@ -377,11 +281,7 @@ jobs:
377281
}
378282
EOF
379283
380-
# Only copy benchmark results to RUNNER_ARTIFACT_DIR for upload (not the entire model)
381-
# First, clean up the downloaded model artifacts from RUNNER_ARTIFACT_DIR
382-
rm -rf "${RUNNER_ARTIFACT_DIR}"/*
383-
384-
# Then copy only the benchmark result JSON files
284+
# Upload only the benchmark result JSON files
385285
cp "$RESULTS_DIR"/*.json "${RUNNER_ARTIFACT_DIR}/"
386286
echo "Benchmark results prepared for upload:"
387287
ls -lah "${RUNNER_ARTIFACT_DIR}"

0 commit comments

Comments
 (0)