Add the perf-smoke performance regression gate

.github/workflows/perf-smoke-seed-baselines.yaml

@@ -0,0 +1,345 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# Seed the perf-baselines branch from real commit history.
+#
+# This is the deployment-simulation counterpart to the live gate: instead of
+# benchmarking one PR head, it walks a slice of a protected branch's history,
+# re-runs each commit's own benchmark inside the CI image (source-mounted so the
+# container git-tags the real commit), and appends the results to perf-baselines.
+# The samples are keyed by their real commit SHA, so the gate's merge-base /
+# ancestry isolation has a populated, branch-correct baseline to compare against.
+# A preflight drops any seed commit that is not an ancestor of the target branch
+# tip (the gate would silently ignore it), and a post-publish step replays the
+# gate's per-bucket match logic to confirm every bucket has MIN_BASELINE_SAMPLES
+# usable samples -- i.e. that the seeded baselines will actually be used.
+#
+# Resolves the CI image exactly like the gate: the era-pinned immutable image for
+# the seeded commit's container era when recorded in the manifest on perf-baselines,
+# otherwise the moving prebuilt tag. Seeding through the same resolver guarantees
+# baselines and later PR runs share an identical environment. Set repo variable
+# PERF_SMOKE_CI_IMAGE only to override that default, e.g. to pin an immutable tag.
+#
+# Manual dispatch defaults to dry-run so maintainers can validate the L40S
+# runner/image path before explicitly choosing dry_run=false to publish samples.
+# There is no push trigger: seeding is expensive, so it runs only when a
+# maintainer dispatches it or when the gate calls it to refill an under-filled
+# bucket.
+
+name: Performance Smoke - Seed Baselines
+
+on:
+  workflow_dispatch:
+    inputs:
+      branches:
+        description: "Branches to seed in one run (comma/space separated). Use 'branch:target' to override the stamp. Empty = use commit_branch/commits. NOTE: only seed branches whose code matches the prebuilt CI image's era."
+        required: false
+        default: "develop"
+      commits:
+        description: "Explicit commit SHAs/refs (space/comma separated). Used only when branches is empty."
+        required: false
+        default: ""
+      commit_branch:
+        description: "Single branch to seed when branches and commits are empty."
+        required: false
+        default: "develop"
+      commit_count:
+        description: "Number of recent commits to seed from commit_branch."
+        required: false
+        default: "5"
+      samples_per_commit:
+        description: "Benchmark repetitions per commit/backend."
+        required: false
+        default: "5"
+      tasks:
+        description: "Comma-separated task_id allowlist (empty = all tasks.json tasks)."
+        required: false
+        default: "Isaac-Cartpole-Direct"
+      backends:
+        description: "Comma-separated backend_key allowlist (empty = all backends)."
+        required: false
+        default: ""
+      target_branch:
+        description: "Protected branch stamped onto each seeded sample."
+        required: false
+        default: "develop"
+      strict_ancestry:
+        description: "Abort if any seed commit is not an ancestor of the target branch tip (default: skip+warn). Enable when you require every seeded sample to be gate-usable."
+        type: boolean
+        required: false
+        default: false
+      dry_run:
+        description: "Run benchmarks + build samples but DO NOT push to perf-baselines."
+        type: boolean
+        required: false
+        default: true
+  # Reusable: the gate's reseed job calls this to fill under-filled buckets.
+  # Callers provide the NGC credential explicitly or via `secrets: inherit`.
+  # The booleans are declared explicitly so a missing value can never coerce to
+  # "publish".
+  workflow_call:
+    inputs:
+      branches:
+        description: "Branches/refs to seed ('ref:target' overrides the stamp)."
+        required: false
+        default: "develop"
+        type: string
+      commits:
+        description: "Explicit commit SHAs/refs (space/comma separated). Used only when branches is empty."
+        required: false
+        default: ""
+        type: string
+      commit_branch:
+        description: "Single branch to seed when branches and commits are empty."
+        required: false
+        default: "develop"
+        type: string
+      commit_count:
+        description: "Number of recent commits to seed per branch/ref."
+        required: false
+        default: "5"
+        type: string
+      samples_per_commit:
+        description: "Benchmark repetitions per commit/backend."
+        required: false
+        default: "5"
+        type: string
+      tasks:
+        description: "Comma-separated task_id allowlist (empty = all tasks.json tasks)."
+        required: false
+        default: "Isaac-Cartpole-Direct"
+        type: string
+      backends:
+        description: "Comma-separated backend_key allowlist (empty = all backends)."
+        required: false
+        default: ""
+        type: string
+      target_branch:
+        description: "Protected branch stamped onto each seeded sample."
+        required: false
+        default: "develop"
+        type: string
+      strict_ancestry:
+        description: "Abort if any seed commit is not an ancestor of the target branch tip."
+        type: boolean
+        required: false
+        default: false
+      dry_run:
+        description: "Run benchmarks + build samples but DO NOT push to perf-baselines."
+        type: boolean
+        required: false
+        default: true
+    secrets:
+      NGC_API_KEY:
+        description: "Optional credential for pulling private NGC images."
+        required: false
+
+concurrency:
+  # Serialize seeding so concurrent runs can't race the append-only baseline branch.
+  group: perf-smoke-seed
+  cancel-in-progress: false
+
+permissions:
+  contents: write   # push appended samples to the perf-baselines branch
+
+env:
+  NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
+  CI_IMAGE_TAG: isaac-lab-ci:seed-${{ github.run_id }}
+
+jobs:
+  config:
+    name: Load Config
+    runs-on: ubuntu-latest
+    outputs:
+      isaaclab_image_ref: ${{ steps.load.outputs.isaaclab_image_ref }}
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 1
+        sparse-checkout: .github/workflows/config.yaml
+        sparse-checkout-cone-mode: false
+    - id: load
+      run: |
+        set -euo pipefail
+        f=.github/workflows/config.yaml
+        ISAACLAB_IMAGE_NAME="$(yq -r .isaaclab_image_name "$f")"
+        echo "isaaclab_image_ref=${ISAACLAB_IMAGE_NAME}:latest-develop" >> "$GITHUB_OUTPUT"
+
+  seed:
+    name: Seed baselines (${{ inputs.commit_branch || github.ref_name || 'develop' }} x${{ inputs.commit_count || '1' }})
+    runs-on: ${{ fromJSON(vars.PERF_SMOKE_RUNS_ON || '["self-hosted","gpu"]') }}
+    needs: [config]
+    timeout-minutes: 600
+
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v6
+      with:
+        # Full history so historical commits are resolvable and ancestry checks work.
+        fetch-depth: 0
+        lfs: true
+
+    # Make the seed/target branch history and the baseline branch available to the
+    # orchestrator. The repo only auto-fetches the checked-out ref, so pull these
+    # explicitly; the clone the orchestrator makes draws its objects from here.
+    - name: Fetch seed + baseline refs
+      env:
+        SEED_BRANCHES: ${{ inputs.branches }}
+        SEED_COMMIT_BRANCH: ${{ inputs.commit_branch || 'develop' }}
+      run: |
+        set -euo pipefail
+        # Collect every branch we might seed from: the multi-branch list (stripping
+        # any ':target' suffix) plus the single-branch fallback. Fetch each so the
+        # orchestrator's clone can resolve their commits offline.
+        BRANCHES="${SEED_BRANCHES//,/ } ${SEED_COMMIT_BRANCH}"
+        for entry in ${BRANCHES}; do
+          BRANCH="${entry%%:*}"
+          [ -z "${BRANCH}" ] && continue
+          git fetch --no-tags origin "+refs/heads/${BRANCH}:refs/remotes/origin/${BRANCH}" || \
+            echo "::warning::Could not fetch ${BRANCH}; skipping (explicit commits may still work)"
+        done
+        git fetch --no-tags origin "+refs/heads/perf-baselines:refs/remotes/origin/perf-baselines" || \
+          echo "::notice::perf-baselines not found yet; first seed run will create it"
+
+    # Resolve the era-pinned immutable image so seeded baselines share the exact
+    # image the gate will later pin for this era (no seed-vs-gate environment
+    # offset). An explicit PERF_SMOKE_CI_IMAGE wins outright; otherwise resolve the
+    # container era (docker/.env.base) against the manifest on perf-baselines and
+    # fall back to the config job's moving tag on a miss.
+    - name: Resolve era-pinned CI image
+      id: era
+      env:
+        FALLBACK_REF: ${{ needs.config.outputs.isaaclab_image_ref }}
+        EXPLICIT_IMAGE: ${{ vars.PERF_SMOKE_CI_IMAGE }}
+      run: |
+        set -euo pipefail
+        if [ -n "${EXPLICIT_IMAGE}" ]; then
+          echo "🔵 Using explicit PERF_SMOKE_CI_IMAGE=${EXPLICIT_IMAGE}"
+          echo "image_ref=${EXPLICIT_IMAGE}" >> "$GITHUB_OUTPUT"
+          exit 0
+        fi
+        ERA_JSON="$(python3 tools/perf_smoke_test/image_era.py \
+          --source_root . \
+          --manifest_from_git \
+          --branch perf-baselines \
+          --remote origin \
+          --fallback_image "${FALLBACK_REF}")"
+        echo "${ERA_JSON}"
+        IMAGE_REF="$(echo "${ERA_JSON}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["image"])')"
+        echo "image_ref=${IMAGE_REF}" >> "$GITHUB_OUTPUT"
+
+    # Pull the published CI image and retag it locally (the exact image the gate pulls,
+    # so seeded baselines and PR runs share one environment). Unlike the gate there is no
+    # build fallback: seeding is a deliberate, baseline-writing op, so a missing image
+    # should fail fast.
+    - name: Pull prebuilt CI image
+      env:
+        CI_IMAGE_REF: ${{ steps.era.outputs.image_ref }}
+      run: |
+        set -euo pipefail
+
+        # The runner's docker credential store backend is broken ("not implemented"),
+        # so disable credsStore before any login (same trick as ecr-build-push-pull).
+        # With credsStore disabled, docker writes the NGC key to config.json in clear
+        # text, so wipe the temp dir on ANY exit -- a failed docker pull must not leave
+        # the long-lived credential on the self-hosted runner's filesystem.
+        DOCKER_CONFIG_DIR="$(mktemp -d)"
+        trap 'rm -rf "${DOCKER_CONFIG_DIR}"' EXIT
+        echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json"
+        export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}"
+
+        REGISTRY="${CI_IMAGE_REF%%/*}"
+        case "${REGISTRY}" in
+          nvcr.io)
+            if [ -n "${NGC_API_KEY:-}" ]; then
+              echo "🔵 Logging into nvcr.io..."
+              echo "${NGC_API_KEY}" | docker login nvcr.io -u '$oauthtoken' --password-stdin
+            else
+              echo "::warning::NGC_API_KEY not set; attempting anonymous pull from nvcr.io"
+            fi
+            ;;
+          *)
+            echo "::notice::No known login for registry '${REGISTRY}'; attempting anonymous pull"
+            ;;
+        esac
+
+        echo "🔵 Pulling prebuilt image ${CI_IMAGE_REF}..."
+        docker pull "${CI_IMAGE_REF}" || {
+          echo "::error::Failed to pull ${CI_IMAGE_REF}. Publish the main IsaacLab CI image or set PERF_SMOKE_CI_IMAGE."
+          exit 1
+        }
+        docker tag "${CI_IMAGE_REF}" "${{ env.CI_IMAGE_TAG }}"
+        echo "🟢 Tagged ${CI_IMAGE_REF} as ${{ env.CI_IMAGE_TAG }}"
+
+    - name: Seed baselines from commit history
+      env:
+        SEED_BRANCHES: ${{ inputs.branches }}
+        SEED_COMMITS: ${{ inputs.commits || '' }}
+        SEED_COMMIT_BRANCH: ${{ inputs.commit_branch || 'develop' }}
+        SEED_COMMIT_COUNT: ${{ inputs.commit_count || '1' }}
+        SEED_SAMPLES_PER_COMMIT: ${{ inputs.samples_per_commit || '3' }}
+        SEED_TASKS: ${{ inputs.tasks }}
+        SEED_BACKENDS: ${{ inputs.backends || '' }}
+        SEED_TARGET_BRANCH: ${{ inputs.target_branch || 'develop' }}
+        SEED_STRICT_ANCESTRY: ${{ inputs.strict_ancestry == true && 'true' || 'false' }}
+        # Fail safe: anything other than an explicit dry_run=false stays a dry run,
+        # so a missing or malformed input can never publish to perf-baselines.
+        SEED_DRY_RUN: ${{ inputs.dry_run == false && 'false' || 'true' }}
+        PERF_SMOKE_RUNNER_NAME: ${{ runner.name }}
+      run: |
+        set -euo pipefail
+        GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)"
+        python3 tools/perf_smoke_test/seed_baselines.py \
+          --branches "${SEED_BRANCHES}" \
+          --commits "${SEED_COMMITS}" \
+          --commit_branch "${SEED_COMMIT_BRANCH}" \
+          --commit_count "${SEED_COMMIT_COUNT}" \
+          --samples_per_commit "${SEED_SAMPLES_PER_COMMIT}" \
+          --tasks "${SEED_TASKS}" \
+          --backends "${SEED_BACKENDS}" \
+          --image "${{ env.CI_IMAGE_TAG }}" \
+          --gpu_model "${GPU_MODEL}" \
+          --target_branch "${SEED_TARGET_BRANCH}" \
+          --strict_ancestry "${SEED_STRICT_ANCESTRY}" \
+          --baseline_branch perf-baselines \
+          --baseline_remote origin \
+          --baseline_push_retries 3 \
+          --workdir "${{ github.workspace }}" \
+          --artifacts_root seed-artifacts \
+          --source_mount true \
+          --dry_run "${SEED_DRY_RUN}"
+
+    # Prove the just-pushed samples will actually be used by the gate: replay the
+    # gate's per-bucket match logic (fingerprint + commit ancestry) against the
+    # target branch tip and require MIN_BASELINE_SAMPLES usable samples per bucket.
+    # Only meaningful on a real publish (dry-run pushes nothing to verify).
+    - name: Verify seeded baselines are gate-usable
+      if: ${{ inputs.dry_run != true }}
+      env:
+        SEED_TASKS: ${{ inputs.tasks }}
+        SEED_BACKENDS: ${{ inputs.backends || '' }}
+        SEED_TARGET_BRANCH: ${{ inputs.target_branch || 'develop' }}
+      run: |
+        set -euo pipefail
+        GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)"
+        python3 tools/perf_smoke_test/verify_baselines.py \
+          --baseline_branch perf-baselines \
+          --baseline_remote origin \
+          --gpu_model "${GPU_MODEL}" \
+          --target_branch "${SEED_TARGET_BRANCH}" \
+          --expected_records seed-artifacts/seed_records.json \
+          --tasks "${SEED_TASKS}" \
+          --backends "${SEED_BACKENDS}" \
+          --repo_dir "${{ github.workspace }}" \
+          --require
+
+    - name: Upload seed artifacts
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: seed-baselines-${{ github.run_id }}
+        path: seed-artifacts/
+        retention-days: 7
+        if-no-files-found: warn