Skip to content

Commit e754f90

Browse files
Neil4561Neil4561
authored
Merge pull request #8 from NVIDIA-Omniverse/neil/enable-perf-smoke-seed-dispatch
Register perf-smoke seed-baselines workflow for dispatch
2 parents 452827c + 03ade6d commit e754f90

1 file changed

Lines changed: 187 additions & 0 deletions

File tree

‎.github/workflows/perf-smoke-seed-baselines.yaml‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
2+
# All rights reserved.
3+
#
4+
# SPDX-License-Identifier: BSD-3-Clause
5+
6+
# Seed the perf-baselines branch from real commit history.
7+
#
8+
# This is the deployment-simulation counterpart to the live gate: instead of
9+
# benchmarking one PR head, it walks a slice of a protected branch's history,
10+
# re-runs each commit's own benchmark inside the CI image (source-mounted so the
11+
# container git-tags the real commit), and appends the results to perf-baselines.
12+
# The samples are keyed by their real commit SHA, so the gate's merge-base /
13+
# ancestry isolation has a populated, branch-correct baseline to compare against.
14+
#
15+
# Uses the prebuilt CI image published to nvcr.io (the same image the gate pulls, so
16+
# baselines and PR runs share an identical environment). Defaults to
17+
# nvcr.io/nvidian/isaac-lab:latest-perf; set repo variable PERF_SMOKE_CI_IMAGE only to
18+
# override that (e.g. to pin a sha-<short> tag). If the image is not published yet the
19+
# job fails fast with a clear message (publish it first via perf-smoke-publish-image).
20+
#
21+
# Manual dispatch only — seeding writes to perf-baselines and burns GPU time, so
22+
# it should never run automatically.
23+
24+
name: Performance Smoke - Seed Baselines
25+
26+
on:
27+
workflow_dispatch:
28+
inputs:
29+
branches:
30+
description: "Branches to seed in one run (comma/space separated). Use 'branch:target' to override the stamp. Empty = use commit_branch/commits. NOTE: only seed branches whose code matches the prebuilt CI image's era (the image is built from unified-POC); seeding a divergent branch such as main/develop yields incompatible extensions or unregistered task ids."
31+
required: false
32+
default: "unified-POC"
33+
commits:
34+
description: "Explicit commit SHAs/refs (space/comma separated). Used only when branches is empty."
35+
required: false
36+
default: ""
37+
commit_branch:
38+
description: "Single branch to seed when branches and commits are empty."
39+
required: false
40+
default: "unified-POC"
41+
commit_count:
42+
description: "Number of recent commits to seed from commit_branch."
43+
required: false
44+
default: "5"
45+
samples_per_commit:
46+
description: "Benchmark repetitions per commit/backend."
47+
required: false
48+
default: "5"
49+
tasks:
50+
description: "Comma-separated task_id allowlist (empty = all tasks.json tasks)."
51+
required: false
52+
default: "Isaac-Cartpole-Direct"
53+
backends:
54+
description: "Comma-separated backend_key allowlist (empty = all backends)."
55+
required: false
56+
default: ""
57+
target_branch:
58+
description: "Protected branch stamped onto each seeded sample."
59+
required: false
60+
default: "unified-POC"
61+
dry_run:
62+
description: "Run benchmarks + build samples but DO NOT push to perf-baselines."
63+
type: boolean
64+
required: false
65+
default: false
66+
67+
concurrency:
68+
# Serialize seeding so concurrent pushes can't race the append-only baseline branch.
69+
group: perf-smoke-seed
70+
cancel-in-progress: false
71+
72+
permissions:
73+
contents: write # push appended samples to the perf-baselines branch
74+
75+
env:
76+
NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
77+
CI_IMAGE_TAG: isaac-lab-ci:seed-${{ github.run_id }}
78+
79+
jobs:
80+
seed:
81+
name: Seed baselines (${{ inputs.commit_branch }} x${{ inputs.commit_count }})
82+
runs-on: ${{ fromJSON(vars.PERF_SMOKE_RUNS_ON || '["linux-amd64-gpu-rtxpro6000-latest-1"]') }}
83+
timeout-minutes: 600
84+
85+
steps:
86+
- name: Checkout Code
87+
uses: actions/checkout@v6
88+
with:
89+
# Full history so historical commits are resolvable and ancestry checks work.
90+
fetch-depth: 0
91+
lfs: true
92+
93+
# Make the seed/target branch history and the baseline branch available to the
94+
# orchestrator. The repo only auto-fetches the checked-out ref, so pull these
95+
# explicitly; the clone the orchestrator makes draws its objects from here.
96+
- name: Fetch seed + baseline refs
97+
env:
98+
SEED_BRANCHES: ${{ inputs.branches }}
99+
SEED_COMMIT_BRANCH: ${{ inputs.commit_branch }}
100+
run: |
101+
set -euo pipefail
102+
# Collect every branch we might seed from: the multi-branch list (stripping
103+
# any ':target' suffix) plus the single-branch fallback. Fetch each so the
104+
# orchestrator's clone can resolve their commits offline.
105+
BRANCHES="${SEED_BRANCHES//,/ } ${SEED_COMMIT_BRANCH}"
106+
for entry in ${BRANCHES}; do
107+
BRANCH="${entry%%:*}"
108+
[ -z "${BRANCH}" ] && continue
109+
git fetch --no-tags origin "+refs/heads/${BRANCH}:refs/remotes/origin/${BRANCH}" || \
110+
echo "::warning::Could not fetch ${BRANCH}; skipping (explicit commits may still work)"
111+
done
112+
git fetch --no-tags origin "+refs/heads/perf-baselines:refs/remotes/origin/perf-baselines" || \
113+
echo "::notice::perf-baselines not found yet; first seed run will create it"
114+
115+
# Pull the published CI image and retag it locally (the exact image the gate pulls,
116+
# so seeded baselines and PR runs share one environment). Defaults to
117+
# nvcr.io/nvidian/isaac-lab:latest-perf; set PERF_SMOKE_CI_IMAGE to override (e.g. to
118+
# pin a sha-<short> tag). Unlike the gate there is no build fallback: seeding is a
119+
# deliberate, baseline-writing op, so a missing image should fail fast (publish first).
120+
- name: Pull prebuilt CI image
121+
env:
122+
CI_IMAGE_REF: ${{ vars.PERF_SMOKE_CI_IMAGE || 'nvcr.io/nvidian/isaac-lab:latest-perf' }}
123+
run: |
124+
set -euo pipefail
125+
126+
# The runner's docker credential store backend is broken ("not implemented"),
127+
# so disable credsStore before any login (same trick as ecr-build-push-pull).
128+
DOCKER_CONFIG_DIR="$(mktemp -d)"
129+
echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json"
130+
export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}"
131+
132+
REGISTRY="${CI_IMAGE_REF%%/*}"
133+
case "${REGISTRY}" in
134+
nvcr.io)
135+
if [ -n "${NGC_API_KEY:-}" ]; then
136+
echo "🔵 Logging into nvcr.io..."
137+
echo "${NGC_API_KEY}" | docker login nvcr.io -u '$oauthtoken' --password-stdin
138+
else
139+
echo "::warning::NGC_API_KEY not set; attempting anonymous pull from nvcr.io"
140+
fi
141+
;;
142+
*)
143+
echo "::notice::No known login for registry '${REGISTRY}'; attempting anonymous pull"
144+
;;
145+
esac
146+
147+
echo "🔵 Pulling prebuilt image ${CI_IMAGE_REF}..."
148+
docker pull "${CI_IMAGE_REF}" || {
149+
echo "::error::Failed to pull ${CI_IMAGE_REF}. Publish it first via perf-smoke-publish-image"
150+
echo "::error::(default tag latest-perf), or set PERF_SMOKE_CI_IMAGE to an existing image."
151+
exit 1
152+
}
153+
docker tag "${CI_IMAGE_REF}" "${{ env.CI_IMAGE_TAG }}"
154+
echo "🟢 Tagged ${CI_IMAGE_REF} as ${{ env.CI_IMAGE_TAG }}"
155+
rm -rf "${DOCKER_CONFIG_DIR}"
156+
157+
- name: Seed baselines from commit history
158+
run: |
159+
set -euo pipefail
160+
GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)"
161+
python3 tools/perf_smoke_test/seed_baselines.py \
162+
--branches "${{ inputs.branches }}" \
163+
--commits "${{ inputs.commits }}" \
164+
--commit_branch "${{ inputs.commit_branch }}" \
165+
--commit_count "${{ inputs.commit_count }}" \
166+
--samples_per_commit "${{ inputs.samples_per_commit }}" \
167+
--tasks "${{ inputs.tasks }}" \
168+
--backends "${{ inputs.backends }}" \
169+
--image "${{ env.CI_IMAGE_TAG }}" \
170+
--gpu_model "${GPU_MODEL}" \
171+
--target_branch "${{ inputs.target_branch }}" \
172+
--baseline_branch perf-baselines \
173+
--baseline_remote origin \
174+
--baseline_push_retries 3 \
175+
--workdir "${{ github.workspace }}" \
176+
--artifacts_root seed-artifacts \
177+
--source_mount true \
178+
--dry_run "${{ inputs.dry_run }}"
179+
180+
- name: Upload seed artifacts
181+
if: always()
182+
uses: actions/upload-artifact@v7
183+
with:
184+
name: seed-baselines-${{ github.run_id }}
185+
path: seed-artifacts/
186+
retention-days: 7
187+
if-no-files-found: warn

0 commit comments

Comments
 (0)