Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 187 additions & 0 deletions .github/workflows/perf-smoke-seed-baselines.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause

# Seed the perf-baselines branch from real commit history.
#
# This is the deployment-simulation counterpart to the live gate: instead of
# benchmarking one PR head, it walks a slice of a protected branch's history,
# re-runs each commit's own benchmark inside the CI image (source-mounted so the
# container git-tags the real commit), and appends the results to perf-baselines.
# The samples are keyed by their real commit SHA, so the gate's merge-base /
# ancestry isolation has a populated, branch-correct baseline to compare against.
#
# Uses the prebuilt CI image published to nvcr.io (the same image the gate pulls, so
# baselines and PR runs share an identical environment). Defaults to
# nvcr.io/nvidian/isaac-lab:latest-perf; set repo variable PERF_SMOKE_CI_IMAGE only to
# override that (e.g. to pin a sha-<short> tag). If the image is not published yet the
# job fails fast with a clear message (publish it first via perf-smoke-publish-image).
#
# Manual dispatch only — seeding writes to perf-baselines and burns GPU time, so
# it should never run automatically.

name: Performance Smoke - Seed Baselines

on:
workflow_dispatch:
inputs:
branches:
description: "Branches to seed in one run (comma/space separated). Use 'branch:target' to override the stamp. Empty = use commit_branch/commits. NOTE: only seed branches whose code matches the prebuilt CI image's era (the image is built from unified-POC); seeding a divergent branch such as main/develop yields incompatible extensions or unregistered task ids."
required: false
default: "unified-POC"
commits:
description: "Explicit commit SHAs/refs (space/comma separated). Used only when branches is empty."
required: false
default: ""
commit_branch:
description: "Single branch to seed when branches and commits are empty."
required: false
default: "unified-POC"
commit_count:
description: "Number of recent commits to seed from commit_branch."
required: false
default: "5"
samples_per_commit:
description: "Benchmark repetitions per commit/backend."
required: false
default: "5"
tasks:
description: "Comma-separated task_id allowlist (empty = all tasks.json tasks)."
required: false
default: "Isaac-Cartpole-Direct"
backends:
description: "Comma-separated backend_key allowlist (empty = all backends)."
required: false
default: ""
target_branch:
description: "Protected branch stamped onto each seeded sample."
required: false
default: "unified-POC"
dry_run:
description: "Run benchmarks + build samples but DO NOT push to perf-baselines."
type: boolean
required: false
default: false

concurrency:
# Serialize seeding so concurrent pushes can't race the append-only baseline branch.
group: perf-smoke-seed
cancel-in-progress: false

permissions:
contents: write # push appended samples to the perf-baselines branch

env:
NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
CI_IMAGE_TAG: isaac-lab-ci:seed-${{ github.run_id }}

jobs:
seed:
name: Seed baselines (${{ inputs.commit_branch }} x${{ inputs.commit_count }})
runs-on: ${{ fromJSON(vars.PERF_SMOKE_RUNS_ON || '["linux-amd64-gpu-rtxpro6000-latest-1"]') }}
timeout-minutes: 600

steps:
- name: Checkout Code
uses: actions/checkout@v6
with:
# Full history so historical commits are resolvable and ancestry checks work.
fetch-depth: 0
lfs: true

# Make the seed/target branch history and the baseline branch available to the
# orchestrator. The repo only auto-fetches the checked-out ref, so pull these
# explicitly; the clone the orchestrator makes draws its objects from here.
- name: Fetch seed + baseline refs
env:
SEED_BRANCHES: ${{ inputs.branches }}
SEED_COMMIT_BRANCH: ${{ inputs.commit_branch }}
run: |
set -euo pipefail
# Collect every branch we might seed from: the multi-branch list (stripping
# any ':target' suffix) plus the single-branch fallback. Fetch each so the
# orchestrator's clone can resolve their commits offline.
BRANCHES="${SEED_BRANCHES//,/ } ${SEED_COMMIT_BRANCH}"
for entry in ${BRANCHES}; do
BRANCH="${entry%%:*}"
[ -z "${BRANCH}" ] && continue
git fetch --no-tags origin "+refs/heads/${BRANCH}:refs/remotes/origin/${BRANCH}" || \
echo "::warning::Could not fetch ${BRANCH}; skipping (explicit commits may still work)"
done
git fetch --no-tags origin "+refs/heads/perf-baselines:refs/remotes/origin/perf-baselines" || \
echo "::notice::perf-baselines not found yet; first seed run will create it"

# Pull the published CI image and retag it locally (the exact image the gate pulls,
# so seeded baselines and PR runs share one environment). Defaults to
# nvcr.io/nvidian/isaac-lab:latest-perf; set PERF_SMOKE_CI_IMAGE to override (e.g. to
# pin a sha-<short> tag). Unlike the gate there is no build fallback: seeding is a
# deliberate, baseline-writing op, so a missing image should fail fast (publish first).
- name: Pull prebuilt CI image
env:
CI_IMAGE_REF: ${{ vars.PERF_SMOKE_CI_IMAGE || 'nvcr.io/nvidian/isaac-lab:latest-perf' }}
run: |
set -euo pipefail

# The runner's docker credential store backend is broken ("not implemented"),
# so disable credsStore before any login (same trick as ecr-build-push-pull).
DOCKER_CONFIG_DIR="$(mktemp -d)"
echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json"
export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}"

REGISTRY="${CI_IMAGE_REF%%/*}"
case "${REGISTRY}" in
nvcr.io)
if [ -n "${NGC_API_KEY:-}" ]; then
echo "🔵 Logging into nvcr.io..."
echo "${NGC_API_KEY}" | docker login nvcr.io -u '$oauthtoken' --password-stdin
else
echo "::warning::NGC_API_KEY not set; attempting anonymous pull from nvcr.io"
fi
;;
*)
echo "::notice::No known login for registry '${REGISTRY}'; attempting anonymous pull"
;;
esac

echo "🔵 Pulling prebuilt image ${CI_IMAGE_REF}..."
docker pull "${CI_IMAGE_REF}" || {
echo "::error::Failed to pull ${CI_IMAGE_REF}. Publish it first via perf-smoke-publish-image"
echo "::error::(default tag latest-perf), or set PERF_SMOKE_CI_IMAGE to an existing image."
exit 1
}
docker tag "${CI_IMAGE_REF}" "${{ env.CI_IMAGE_TAG }}"
echo "🟢 Tagged ${CI_IMAGE_REF} as ${{ env.CI_IMAGE_TAG }}"
rm -rf "${DOCKER_CONFIG_DIR}"

- name: Seed baselines from commit history
run: |
set -euo pipefail
GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)"
python3 tools/perf_smoke_test/seed_baselines.py \
--branches "${{ inputs.branches }}" \
--commits "${{ inputs.commits }}" \
--commit_branch "${{ inputs.commit_branch }}" \
--commit_count "${{ inputs.commit_count }}" \
--samples_per_commit "${{ inputs.samples_per_commit }}" \
--tasks "${{ inputs.tasks }}" \
--backends "${{ inputs.backends }}" \
--image "${{ env.CI_IMAGE_TAG }}" \
--gpu_model "${GPU_MODEL}" \
--target_branch "${{ inputs.target_branch }}" \
--baseline_branch perf-baselines \
--baseline_remote origin \
--baseline_push_retries 3 \
--workdir "${{ github.workspace }}" \
--artifacts_root seed-artifacts \
--source_mount true \
--dry_run "${{ inputs.dry_run }}"

- name: Upload seed artifacts
if: always()
uses: actions/upload-artifact@v7
with:
name: seed-baselines-${{ github.run_id }}
path: seed-artifacts/
retention-days: 7
if-no-files-found: warn
Loading