diff --git a/.github/workflows/perf-smoke-seed-baselines.yaml b/.github/workflows/perf-smoke-seed-baselines.yaml new file mode 100644 index 000000000000..b52b3c98d6be --- /dev/null +++ b/.github/workflows/perf-smoke-seed-baselines.yaml @@ -0,0 +1,187 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Seed the perf-baselines branch from real commit history. +# +# This is the deployment-simulation counterpart to the live gate: instead of +# benchmarking one PR head, it walks a slice of a protected branch's history, +# re-runs each commit's own benchmark inside the CI image (source-mounted so the +# container git-tags the real commit), and appends the results to perf-baselines. +# The samples are keyed by their real commit SHA, so the gate's merge-base / +# ancestry isolation has a populated, branch-correct baseline to compare against. +# +# Uses the prebuilt CI image published to nvcr.io (the same image the gate pulls, so +# baselines and PR runs share an identical environment). Defaults to +# nvcr.io/nvidian/isaac-lab:latest-perf; set repo variable PERF_SMOKE_CI_IMAGE only to +# override that (e.g. to pin a sha- tag). If the image is not published yet the +# job fails fast with a clear message (publish it first via perf-smoke-publish-image). +# +# Manual dispatch only — seeding writes to perf-baselines and burns GPU time, so +# it should never run automatically. + +name: Performance Smoke - Seed Baselines + +on: + workflow_dispatch: + inputs: + branches: + description: "Branches to seed in one run (comma/space separated). Use 'branch:target' to override the stamp. Empty = use commit_branch/commits. NOTE: only seed branches whose code matches the prebuilt CI image's era (the image is built from unified-POC); seeding a divergent branch such as main/develop yields incompatible extensions or unregistered task ids." + required: false + default: "unified-POC" + commits: + description: "Explicit commit SHAs/refs (space/comma separated). Used only when branches is empty." + required: false + default: "" + commit_branch: + description: "Single branch to seed when branches and commits are empty." + required: false + default: "unified-POC" + commit_count: + description: "Number of recent commits to seed from commit_branch." + required: false + default: "5" + samples_per_commit: + description: "Benchmark repetitions per commit/backend." + required: false + default: "5" + tasks: + description: "Comma-separated task_id allowlist (empty = all tasks.json tasks)." + required: false + default: "Isaac-Cartpole-Direct" + backends: + description: "Comma-separated backend_key allowlist (empty = all backends)." + required: false + default: "" + target_branch: + description: "Protected branch stamped onto each seeded sample." + required: false + default: "unified-POC" + dry_run: + description: "Run benchmarks + build samples but DO NOT push to perf-baselines." + type: boolean + required: false + default: false + +concurrency: + # Serialize seeding so concurrent pushes can't race the append-only baseline branch. + group: perf-smoke-seed + cancel-in-progress: false + +permissions: + contents: write # push appended samples to the perf-baselines branch + +env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + CI_IMAGE_TAG: isaac-lab-ci:seed-${{ github.run_id }} + +jobs: + seed: + name: Seed baselines (${{ inputs.commit_branch }} x${{ inputs.commit_count }}) + runs-on: ${{ fromJSON(vars.PERF_SMOKE_RUNS_ON || '["linux-amd64-gpu-rtxpro6000-latest-1"]') }} + timeout-minutes: 600 + + steps: + - name: Checkout Code + uses: actions/checkout@v6 + with: + # Full history so historical commits are resolvable and ancestry checks work. + fetch-depth: 0 + lfs: true + + # Make the seed/target branch history and the baseline branch available to the + # orchestrator. The repo only auto-fetches the checked-out ref, so pull these + # explicitly; the clone the orchestrator makes draws its objects from here. + - name: Fetch seed + baseline refs + env: + SEED_BRANCHES: ${{ inputs.branches }} + SEED_COMMIT_BRANCH: ${{ inputs.commit_branch }} + run: | + set -euo pipefail + # Collect every branch we might seed from: the multi-branch list (stripping + # any ':target' suffix) plus the single-branch fallback. Fetch each so the + # orchestrator's clone can resolve their commits offline. + BRANCHES="${SEED_BRANCHES//,/ } ${SEED_COMMIT_BRANCH}" + for entry in ${BRANCHES}; do + BRANCH="${entry%%:*}" + [ -z "${BRANCH}" ] && continue + git fetch --no-tags origin "+refs/heads/${BRANCH}:refs/remotes/origin/${BRANCH}" || \ + echo "::warning::Could not fetch ${BRANCH}; skipping (explicit commits may still work)" + done + git fetch --no-tags origin "+refs/heads/perf-baselines:refs/remotes/origin/perf-baselines" || \ + echo "::notice::perf-baselines not found yet; first seed run will create it" + + # Pull the published CI image and retag it locally (the exact image the gate pulls, + # so seeded baselines and PR runs share one environment). Defaults to + # nvcr.io/nvidian/isaac-lab:latest-perf; set PERF_SMOKE_CI_IMAGE to override (e.g. to + # pin a sha- tag). Unlike the gate there is no build fallback: seeding is a + # deliberate, baseline-writing op, so a missing image should fail fast (publish first). + - name: Pull prebuilt CI image + env: + CI_IMAGE_REF: ${{ vars.PERF_SMOKE_CI_IMAGE || 'nvcr.io/nvidian/isaac-lab:latest-perf' }} + run: | + set -euo pipefail + + # The runner's docker credential store backend is broken ("not implemented"), + # so disable credsStore before any login (same trick as ecr-build-push-pull). + DOCKER_CONFIG_DIR="$(mktemp -d)" + echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json" + export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}" + + REGISTRY="${CI_IMAGE_REF%%/*}" + case "${REGISTRY}" in + nvcr.io) + if [ -n "${NGC_API_KEY:-}" ]; then + echo "🔵 Logging into nvcr.io..." + echo "${NGC_API_KEY}" | docker login nvcr.io -u '$oauthtoken' --password-stdin + else + echo "::warning::NGC_API_KEY not set; attempting anonymous pull from nvcr.io" + fi + ;; + *) + echo "::notice::No known login for registry '${REGISTRY}'; attempting anonymous pull" + ;; + esac + + echo "🔵 Pulling prebuilt image ${CI_IMAGE_REF}..." + docker pull "${CI_IMAGE_REF}" || { + echo "::error::Failed to pull ${CI_IMAGE_REF}. Publish it first via perf-smoke-publish-image" + echo "::error::(default tag latest-perf), or set PERF_SMOKE_CI_IMAGE to an existing image." + exit 1 + } + docker tag "${CI_IMAGE_REF}" "${{ env.CI_IMAGE_TAG }}" + echo "🟢 Tagged ${CI_IMAGE_REF} as ${{ env.CI_IMAGE_TAG }}" + rm -rf "${DOCKER_CONFIG_DIR}" + + - name: Seed baselines from commit history + run: | + set -euo pipefail + GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)" + python3 tools/perf_smoke_test/seed_baselines.py \ + --branches "${{ inputs.branches }}" \ + --commits "${{ inputs.commits }}" \ + --commit_branch "${{ inputs.commit_branch }}" \ + --commit_count "${{ inputs.commit_count }}" \ + --samples_per_commit "${{ inputs.samples_per_commit }}" \ + --tasks "${{ inputs.tasks }}" \ + --backends "${{ inputs.backends }}" \ + --image "${{ env.CI_IMAGE_TAG }}" \ + --gpu_model "${GPU_MODEL}" \ + --target_branch "${{ inputs.target_branch }}" \ + --baseline_branch perf-baselines \ + --baseline_remote origin \ + --baseline_push_retries 3 \ + --workdir "${{ github.workspace }}" \ + --artifacts_root seed-artifacts \ + --source_mount true \ + --dry_run "${{ inputs.dry_run }}" + + - name: Upload seed artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: seed-baselines-${{ github.run_id }} + path: seed-artifacts/ + retention-days: 7 + if-no-files-found: warn