Skip to content

Commit c4eb0ea

Browse files
authored
Merge cuda-perf.yml benchmark into cuda.yml e2e tests (pytorch#23385)
Summary: cuda-perf.yml is retired. Its benchmark + upload pipeline now runs inside cuda.yml's test-model-cuda-e2e cells, reusing each cell's exported pte/ptd and built runner in place (new helper .ci/scripts/cuda_benchmark_from_e2e.sh, allowlisted to the 7 perf models; whisper-medium backfilled into the accuracy matrix). Reviewed By: shoumikhin Differential Revision: D123135327 Pull Request resolved: pytorch#23385
1 parent 07e1b50 commit c4eb0ea

8 files changed

Lines changed: 518 additions & 516 deletions

File tree

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
# Run the CUDA perf benchmark for one (model, quant) pair, reusing the
9+
# .pte/.ptd artifacts and the already-built runner left behind by
10+
# test_model_e2e.sh in the same CI job.
11+
#
12+
# Only the (model, quant) pairs in the perf allowlist below are benchmarked.
13+
# Anything else exits 0 after printing BENCHMARK_SKIP, so the caller can tell
14+
# "not covered by perf" apart from a real failure. The optional
15+
# models/quantizations filters (from workflow_dispatch) narrow the benchmark
16+
# stage further without touching the accuracy stage.
17+
#
18+
# Usage:
19+
# cuda_benchmark_from_e2e.sh <hf_model> <quant> <model_dir> <results_dir>
20+
# [num_runs] [git_sha] [run_id] [run_url] [models_filter] [quants_filter]
21+
#
22+
# Outputs (in <results_dir>):
23+
# benchmark_results.json, benchmark_results_v3.json, metadata.json
24+
25+
set -euo pipefail
26+
27+
HF_MODEL="${1:?hf_model required, e.g. openai/whisper-small}"
28+
QUANT_NAME="${2:?quant required, e.g. non-quantized}"
29+
MODEL_DIR="${3:?model_dir required}"
30+
RESULTS_DIR="${4:?results_dir required}"
31+
NUM_RUNS="${5:-50}"
32+
GIT_SHA="${6:-unknown}"
33+
WORKFLOW_RUN_ID="${7:-0}"
34+
WORKFLOW_RUN_URL="${8:-}"
35+
MODELS_FILTER="${9:-}"
36+
QUANTS_FILTER="${10:-}"
37+
38+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
39+
EXECUTORCH_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
40+
41+
# (model, quant) pairs covered by perf tracking. Keep in sync with the
42+
# test-model-cuda-e2e matrix in .github/workflows/cuda.yml.
43+
is_bench_pair() {
44+
case "${HF_MODEL} ${QUANT_NAME}" in
45+
"mistralai/Voxtral-Mini-3B-2507 "* | \
46+
"openai/whisper-small non-quantized" | \
47+
"openai/whisper-medium non-quantized" | \
48+
"openai/whisper-large-v3-turbo quantized-"* | \
49+
"google/gemma-3-4b-it non-quantized" | \
50+
"google/gemma-3-4b-it quantized-int4-tile-packed" | \
51+
"nvidia/parakeet-tdt non-quantized" | \
52+
"nvidia/parakeet-tdt quantized-int4-tile-packed" | \
53+
"SocialLocalMobile/Qwen3.5-35B-A3B-HQQ-INT4 quantized-int4-tile-packed")
54+
return 0
55+
;;
56+
*)
57+
return 1
58+
;;
59+
esac
60+
}
61+
62+
if ! is_bench_pair; then
63+
echo "BENCHMARK_SKIP: ${HF_MODEL} ${QUANT_NAME} is not in the perf allowlist"
64+
exit 0
65+
fi
66+
67+
# workflow_dispatch filters: comma-separated HF model IDs / quant names.
68+
# Empty means "benchmark every allowlisted pair".
69+
case ",${MODELS_FILTER}," in
70+
*",${HF_MODEL},"*) ;;
71+
*)
72+
if [ -n "${MODELS_FILTER}" ]; then
73+
echo "BENCHMARK_SKIP: ${HF_MODEL} not in dispatch models filter"
74+
exit 0
75+
fi
76+
;;
77+
esac
78+
case ",${QUANTS_FILTER}," in
79+
*",${QUANT_NAME},"*) ;;
80+
*)
81+
if [ -n "${QUANTS_FILTER}" ]; then
82+
echo "BENCHMARK_SKIP: ${QUANT_NAME} not in dispatch quantizations filter"
83+
exit 0
84+
fi
85+
;;
86+
esac
87+
88+
cd "${EXECUTORCH_ROOT}"
89+
export LD_LIBRARY_PATH="/opt/conda/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
90+
91+
# Runner command mapping, ported from the retired cuda-perf.yml so the
92+
# benchmark prompts and flags (and therefore the numbers) stay identical.
93+
case "${HF_MODEL}" in
94+
mistralai/Voxtral-Mini-3B-2507)
95+
RUNNER="cmake-out/examples/models/voxtral/voxtral_runner"
96+
PREPROCESSOR="${MODEL_DIR}/voxtral_preprocessor.pte"
97+
TOKENIZER="${MODEL_DIR}/tekken.json"
98+
AUDIO="${MODEL_DIR}/poem.wav"
99+
RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer_path $TOKENIZER --audio_path $AUDIO --processor_path $PREPROCESSOR --temperature 0"
100+
MODEL_NAME="voxtral_${QUANT_NAME}"
101+
;;
102+
openai/whisper-*)
103+
RUNNER="cmake-out/examples/models/whisper/whisper_runner"
104+
PREPROCESSOR="${MODEL_DIR}/whisper_preprocessor.pte"
105+
AUDIO="${MODEL_DIR}/output.wav"
106+
RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer_path ${MODEL_DIR}/ --audio_path $AUDIO --processor_path $PREPROCESSOR --temperature 0"
107+
MODEL_NAME="${HF_MODEL#openai/}_${QUANT_NAME}"
108+
;;
109+
google/gemma-3-4b-it)
110+
RUNNER="cmake-out/examples/models/gemma3/gemma3_e2e_runner"
111+
IMAGE="docs/source/_static/img/et-logo.png"
112+
RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer_path ${MODEL_DIR}/ --image_path $IMAGE --temperature 0"
113+
MODEL_NAME="gemma3_${QUANT_NAME}"
114+
;;
115+
nvidia/parakeet-tdt)
116+
RUNNER="cmake-out/examples/models/parakeet/parakeet_runner"
117+
AUDIO="${MODEL_DIR}/test_audio.wav"
118+
TOKENIZER="${MODEL_DIR}/tokenizer.model"
119+
RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --audio_path $AUDIO --tokenizer_path $TOKENIZER"
120+
MODEL_NAME="parakeet_${QUANT_NAME}"
121+
;;
122+
SocialLocalMobile/Qwen3.5-35B-A3B-HQQ-INT4)
123+
RUNNER="cmake-out/examples/models/qwen3_5_moe/qwen3_5_moe_runner"
124+
TOKENIZER="${MODEL_DIR}/tokenizer.json"
125+
# A checked-in long prompt (>1000 tokens). A static, meaningful prompt
126+
# avoids the degenerate / repetitive outputs that can result from
127+
# synthetic prompts built by repeating the same sentence.
128+
PROMPT_FILE=".ci/scripts/cuda_perf_prompts/qwen3_5_moe_long_prompt.txt"
129+
RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer_path $TOKENIZER --prompt_file $PROMPT_FILE --max_new_tokens 512 --temperature 0"
130+
MODEL_NAME="qwen3_5_moe_${QUANT_NAME}"
131+
;;
132+
*)
133+
echo "BENCHMARK_SKIP: no runner mapping for '${HF_MODEL}'"
134+
exit 0
135+
;;
136+
esac
137+
138+
echo "::group::Running benchmark for ${HF_MODEL} (${QUANT_NAME}) with ${NUM_RUNS} runs"
139+
140+
GPU_NAME=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)
141+
echo "Detected GPU: $GPU_NAME"
142+
CUDA_DRIVER_VERSION=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -1)
143+
echo "CUDA Driver Version: $CUDA_DRIVER_VERSION"
144+
145+
mkdir -p "${RESULTS_DIR}"
146+
147+
# Run benchmark using cuda_benchmark.py
148+
python .ci/scripts/cuda_benchmark.py \
149+
--runner_command "$RUNNER_CMD" \
150+
--model_name "$MODEL_NAME" \
151+
--num_runs "${NUM_RUNS}" \
152+
--output_json "${RESULTS_DIR}/benchmark_results.json" \
153+
--output_v3 "${RESULTS_DIR}/benchmark_results_v3.json" \
154+
--model "${HF_MODEL}" \
155+
--quantization "${QUANT_NAME}" \
156+
--git_sha "${GIT_SHA}" \
157+
--workflow_run_id "${WORKFLOW_RUN_ID}" \
158+
--workflow_run_url "${WORKFLOW_RUN_URL}" \
159+
--gpu_name "$GPU_NAME" \
160+
--cuda_driver_version "$CUDA_DRIVER_VERSION"
161+
162+
# Save additional metadata (written via a serializer, never string-built).
163+
BENCH_MODEL="${HF_MODEL}" BENCH_QUANT="${QUANT_NAME}" BENCH_NUM_RUNS="${NUM_RUNS}" \
164+
BENCH_RUNNER="${RUNNER}" BENCH_SHA="${GIT_SHA}" BENCH_RUN_ID="${WORKFLOW_RUN_ID}" \
165+
BENCH_RUN_URL="${WORKFLOW_RUN_URL}" BENCH_GPU="${GPU_NAME}" \
166+
RESULTS_DIR="${RESULTS_DIR}" python3 - <<'PY'
167+
import datetime
168+
import json
169+
import os
170+
171+
metadata = {
172+
"model": os.environ["BENCH_MODEL"],
173+
"quantization": os.environ["BENCH_QUANT"],
174+
"num_runs": int(os.environ["BENCH_NUM_RUNS"]),
175+
"runner": os.environ["BENCH_RUNNER"],
176+
"timestamp": datetime.datetime.now(datetime.timezone.utc).strftime(
177+
"%Y-%m-%dT%H:%M:%SZ"
178+
),
179+
"git_sha": os.environ["BENCH_SHA"],
180+
"workflow_run_id": os.environ["BENCH_RUN_ID"],
181+
"workflow_run_url": os.environ["BENCH_RUN_URL"],
182+
"gpu_name": os.environ["BENCH_GPU"],
183+
}
184+
with open(os.path.join(os.environ.get("RESULTS_DIR", "."), "metadata.json"), "w") as f:
185+
json.dump(metadata, f, indent=2)
186+
PY
187+
echo "::endgroup::"

‎.ci/scripts/tests/test_cuda_workflow.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,14 @@ def test_pybind_runs_inline_for_the_expected_matrix_cells(self):
184184
self.assertIn("--run_only", script)
185185
self.assertGreaterEqual(script.count('"${MODEL_DIR}"'), 2)
186186

187-
def test_model_e2e_does_not_transfer_artifacts(self):
187+
def test_model_e2e_only_uploads_benchmark_results(self):
188188
self.assertNotIn("test-cuda-pybind", WORKFLOW["jobs"])
189-
keys = set(_all_keys(WORKFLOW["jobs"]["test-model-cuda-e2e"]))
190-
self.assertNotIn("upload-artifact", keys)
191-
self.assertNotIn("download-artifact", keys)
189+
job = WORKFLOW["jobs"]["test-model-cuda-e2e"]
190+
self.assertNotIn("download-artifact", set(_all_keys(job)))
191+
self.assertTrue(job["with"]["upload-artifact"].startswith("cuda-bench-"))
192+
# Exported .pte/.ptd live outside RUNNER_ARTIFACT_DIR; only the
193+
# benchmark JSON under bench/ is uploaded.
194+
script = job["with"]["script"]
195+
self.assertIn('MODEL_DIR="$(mktemp -d "${RUNNER_TEMP:-/tmp}/', script)
196+
self.assertEqual(1, script.count("RUNNER_ARTIFACT_DIR"))
197+
self.assertIn('BENCH_RESULTS_DIR="${RUNNER_ARTIFACT_DIR}/bench"', script)

‎.github/pytorch-probot.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ ciflow_push_tags:
55
- ciflow/apple
66
- ciflow/coreai
77
- ciflow/cuda
8-
- ciflow/cuda-perf
98
- ciflow/docker
109
- ciflow/metal
1110
- ciflow/mlx
Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
# Quick script to trigger the CUDA workflow (accuracy + benchmark, merged
9+
# from the retired cuda-perf.yml) via GitHub CLI.
10+
# Benchmarks reuse each accuracy cell's exported .pte/.ptd in place; the
11+
# models/quantizations filters below only narrow the benchmark stage.
12+
# Usage:
13+
# ./trigger_cuda_benchmark.sh # Use defaults (all allowlisted pairs)
14+
# ./trigger_cuda_benchmark.sh "openai/whisper-medium" # Single model
15+
# ./trigger_cuda_benchmark.sh "openai/whisper-small,google/gemma-3-4b-it" "non-quantized,quantized-int4-tile-packed" "100"
16+
#
17+
# NOTE: perf allowlist lives in .ci/scripts/cuda_benchmark_from_e2e.sh and the
18+
# test-model-cuda-e2e matrix in .github/workflows/cuda.yml. Models outside the
19+
# accuracy matrix cannot be benchmarked via this script.
20+
21+
set -e
22+
23+
# Models covered by both accuracy e2e and perf tracking (see
24+
# cuda_benchmark_from_e2e.sh is_bench_pair). whisper-medium is the one
25+
# perf-first model backfilled into the accuracy matrix.
26+
ALL_MODELS="mistralai/Voxtral-Mini-3B-2507,openai/whisper-small,openai/whisper-medium,openai/whisper-large-v3-turbo,google/gemma-3-4b-it,nvidia/parakeet-tdt,SocialLocalMobile/Qwen3.5-35B-A3B-HQQ-INT4"
27+
ALL_QUANTIZATIONS="non-quantized,quantized-int4-tile-packed,quantized-int4-weight-only"
28+
29+
# Check if gh CLI is installed
30+
if ! command -v gh &> /dev/null; then
31+
echo "Error: GitHub CLI (gh) is not installed."
32+
echo "Install it from: https://cli.github.com/"
33+
echo ""
34+
echo "Quick install:"
35+
echo " macOS: brew install gh"
36+
echo " Linux: See https://github.com/cli/cli/blob/trunk/docs/install_linux.md"
37+
exit 1
38+
fi
39+
40+
MODELS="${1:-}"
41+
QUANT="${2:-}"
42+
NUM_RUNS="${3:-50}"
43+
44+
# Display configuration
45+
echo "========================================="
46+
echo "Triggering cuda workflow (accuracy + benchmark)"
47+
echo "========================================="
48+
if [ -z "$MODELS" ]; then
49+
echo "Models: (all allowlisted: $ALL_MODELS)"
50+
else
51+
echo "Models: $MODELS"
52+
fi
53+
if [ -z "$QUANT" ]; then
54+
echo "Quantizations: (all allowlisted: $ALL_QUANTIZATIONS)"
55+
else
56+
echo "Quantizations: $QUANT"
57+
fi
58+
echo "Num runs: $NUM_RUNS"
59+
echo "========================================="
60+
61+
echo ""
62+
63+
# Trigger workflow (dispatch inputs only narrow the benchmark stage)
64+
gh workflow run cuda.yml \
65+
-R pytorch/executorch \
66+
-f models="$MODELS" \
67+
-f quantizations="$QUANT" \
68+
-f num_runs="$NUM_RUNS"
69+
70+
if [ $? -eq 0 ]; then
71+
echo "✓ Workflow triggered successfully!"
72+
echo ""
73+
echo "View status:"
74+
echo " gh run list --workflow=cuda.yml"
75+
echo ""
76+
echo "Watch the latest run:"
77+
echo " gh run watch \$(gh run list --workflow=cuda.yml --limit 1 --json databaseId --jq '.[0].databaseId')"
78+
else
79+
echo "✗ Failed to trigger workflow"
80+
exit 1
81+
fi

‎.github/scripts/trigger_cuda_perf.sh‎

Lines changed: 0 additions & 100 deletions
This file was deleted.

0 commit comments

Comments
 (0)