@@ -121,9 +121,21 @@ jobs:
121121 echo "Generated benchmark configs:"
122122 echo "$CONFIGS" | python -m json.tool
123123
124- export-models :
125- name : export-models
126- needs : set-parameters
124+ benchmark-cuda :
125+ name : benchmark-cuda
126+ needs :
127+ - changed-files
128+ - run-decision
129+ - set-parameters
130+ # Same gate as set-parameters, repeated so the job reads on its own;
131+ # the needs on set-parameters already skips it when the gate is closed.
132+ if : |
133+ contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda-perf.yml') ||
134+ contains(needs.changed-files.outputs.changed-files, '.ci/scripts/cuda_benchmark.py') ||
135+ contains(needs.changed-files.outputs.changed-files, '.ci/scripts/cuda_perf_prompts') ||
136+ contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') ||
137+ contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e.sh') ||
138+ needs.run-decision.outputs.is-full-run == 'true'
127139 uses : pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
128140 permissions :
129141 id-token : write
@@ -133,14 +145,14 @@ jobs:
133145 matrix : ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }}
134146 fail-fast : false
135147 with :
136- timeout : 90
148+ timeout : 120
137149 secrets-env : EXECUTORCH_HF_TOKEN
138150 runner : ${{ contains(matrix.model, 'Qwen3.5-35B-A3B') && 'mt-l-x86iavx512-11-125-a100' || 'mt-l-x86aavx2-29-113-a10g' }}
139151 gpu-arch-type : cuda
140152 gpu-arch-version : " 13.0"
141153 use-custom-docker-registry : false
142154 submodules : recursive
143- upload-artifact : model -${{ matrix.model_safe }}-${{ matrix.quant }}
155+ upload-artifact : results -${{ matrix.model_safe }}-${{ matrix.quant }}
144156 ref : ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
145157 script : |
146158 set -eux
@@ -172,118 +184,10 @@ jobs:
172184 pip install --no-deps git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
173185 echo "::endgroup::"
174186
175- echo "::group::Exporting model ${{ matrix.model }} with quantization ${{ matrix.quant }}"
176- OUTPUT_DIR="model_artifacts"
177- mkdir -p "$OUTPUT_DIR"
178-
179- bash .ci/scripts/export_model_artifact.sh cuda "${{ matrix.model }}" "${{ matrix.quant }}" "$OUTPUT_DIR"
180-
181- # Move artifacts to RUNNER_ARTIFACT_DIR for upload
182- mv "$OUTPUT_DIR"/* "${RUNNER_ARTIFACT_DIR}/"
183- ls -lah "${RUNNER_ARTIFACT_DIR}"
184- echo "::endgroup::"
185-
186- benchmark-cuda :
187- name : benchmark-cuda
188- needs :
189- - changed-files
190- - run-decision
191- - set-parameters
192- - export-models
193- # Inherit the gate from set-parameters/export-models (they cascade-
194- # skip when the gate evaluates false). `always()` keeps benchmark-
195- # cuda running even when some export-models matrix cells fail —
196- # but only if the gate itself is open. Without the explicit gate
197- # here, `always()` would fire benchmark-cuda even when set-
198- # parameters was gated out.
199- if : |
200- always() &&
201- (
202- contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda-perf.yml') ||
203- contains(needs.changed-files.outputs.changed-files, '.ci/scripts/cuda_benchmark.py') ||
204- contains(needs.changed-files.outputs.changed-files, '.ci/scripts/cuda_perf_prompts') ||
205- contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') ||
206- contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e.sh') ||
207- needs.run-decision.outputs.is-full-run == 'true'
208- )
209- uses : pytorch/test-infra/.github/workflows/linux_job_v3.yml@main
210- permissions :
211- id-token : write
212- contents : read
213- strategy :
214- matrix : ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }}
215- fail-fast : false
216- with :
217- timeout : 90
218- runner : ${{ contains(matrix.model, 'Qwen3.5-35B-A3B') && 'mt-l-x86iavx512-11-125-a100' || 'mt-l-x86aavx2-29-113-a10g' }}
219- gpu-arch-type : cuda
220- gpu-arch-version : " 13.0"
221- use-custom-docker-registry : false
222- submodules : recursive
223- download-artifact : model-${{ matrix.model_safe }}-${{ matrix.quant }}
224- upload-artifact : results-${{ matrix.model_safe }}-${{ matrix.quant }}
225- ref : ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
226- script : |
227- set -eux
228-
229- # OSDC mounts HF_HOME read-only at /mnt/hf_cache; redirect to a writable dir
230- # (RUNNER_TEMP, or /tmp when RUNNER_TEMP isn't writable inside the container).
231- export HF_HOME="${RUNNER_TEMP:-/tmp}/hf_cache"
232- mkdir -p "${HF_HOME}" 2>/dev/null || export HF_HOME=/tmp/hf_cache
233- mkdir -p "${HF_HOME}"
234-
235- echo "::group::Setup environment"
236- # OSDC runners can't reach the public PyPI CDN that download.pytorch.org's
237- # transitive deps resolve to. Pre-install torch's pure-python deps from the
238- # in-cluster pypi-cache and drop the default cpu extra-index so the cuda
239- # torch wheel is the only candidate.
240- export PIP_EXTRA_INDEX_URL=
241- # fsspec is pinned to satisfy datasets' fsspec[http]<=2025.3.0 so the later
242- # examples install doesn't try to downgrade it from the public CDN.
243- pip install filelock typing-extensions "setuptools<82" sympy networkx jinja2 "fsspec[http]<=2025.3.0" numpy pillow
244- ./install_requirements.sh
245- pip list
246- echo "::endgroup::"
247-
248- echo "::group::Prepare model artifacts"
249- mkdir -p model_artifacts
250- cp "${RUNNER_ARTIFACT_DIR}/model.pte" model_artifacts/model.pte
251- cp "${RUNNER_ARTIFACT_DIR}/aoti_cuda_blob.ptd" model_artifacts/aoti_cuda_blob.ptd
252-
253- # Copy additional files if they exist
254- if [ -f "${RUNNER_ARTIFACT_DIR}/voxtral_preprocessor.pte" ]; then
255- cp "${RUNNER_ARTIFACT_DIR}/voxtral_preprocessor.pte" model_artifacts/
256- fi
257- if [ -f "${RUNNER_ARTIFACT_DIR}/whisper_preprocessor.pte" ]; then
258- cp "${RUNNER_ARTIFACT_DIR}/whisper_preprocessor.pte" model_artifacts/
259- fi
260- if [ -f "${RUNNER_ARTIFACT_DIR}/tekken.json" ]; then
261- cp "${RUNNER_ARTIFACT_DIR}/tekken.json" model_artifacts/
262- fi
263- if [ -f "${RUNNER_ARTIFACT_DIR}/poem.wav" ]; then
264- cp "${RUNNER_ARTIFACT_DIR}/poem.wav" model_artifacts/
265- fi
266- if [ -f "${RUNNER_ARTIFACT_DIR}/output.wav" ]; then
267- cp "${RUNNER_ARTIFACT_DIR}/output.wav" model_artifacts/
268- fi
269- if [ -f "${RUNNER_ARTIFACT_DIR}/tokenizer.model" ]; then
270- cp "${RUNNER_ARTIFACT_DIR}/tokenizer.model" model_artifacts/
271- fi
272- if [ -f "${RUNNER_ARTIFACT_DIR}/test_audio.wav" ]; then
273- cp "${RUNNER_ARTIFACT_DIR}/test_audio.wav" model_artifacts/
274- fi
275- # Copy tokenizer files
276- for file in tokenizer.json tokenizer_config.json special_tokens_map.json; do
277- if [ -f "${RUNNER_ARTIFACT_DIR}/$file" ]; then
278- cp "${RUNNER_ARTIFACT_DIR}/$file" model_artifacts/
279- fi
280- done
281-
282- ls -lah model_artifacts/
283- echo "::endgroup::"
284-
285- echo "::group::Build runner"
286- bash .ci/scripts/test_model_e2e.sh cuda "${{ matrix.model }}" "${{ matrix.quant }}" model_artifacts
187+ echo "::group::Export model and build runner"
188+ # Export and benchmark in the same job so the model never leaves the runner
189+ # (no GitHub Actions artifact upload/download).
190+ RUN_EXPORT=1 bash .ci/scripts/test_model_e2e.sh cuda "${{ matrix.model }}" "${{ matrix.quant }}" model_artifacts
287191 echo "::endgroup::"
288192
289193 echo "::group::Running benchmark for ${{ matrix.model }} (${{ matrix.quant }}) with ${{ matrix.num_runs }} runs"
@@ -377,11 +281,7 @@ jobs:
377281 }
378282 EOF
379283
380- # Only copy benchmark results to RUNNER_ARTIFACT_DIR for upload (not the entire model)
381- # First, clean up the downloaded model artifacts from RUNNER_ARTIFACT_DIR
382- rm -rf "${RUNNER_ARTIFACT_DIR}"/*
383-
384- # Then copy only the benchmark result JSON files
284+ # Upload only the benchmark result JSON files
385285 cp "$RESULTS_DIR"/*.json "${RUNNER_ARTIFACT_DIR}/"
386286 echo "Benchmark results prepared for upload:"
387287 ls -lah "${RUNNER_ARTIFACT_DIR}"
0 commit comments