diff --git a/.github/copy-pr-bot.yaml b/.github/copy-pr-bot.yaml new file mode 100644 index 000000000000..54682e0dada9 --- /dev/null +++ b/.github/copy-pr-bot.yaml @@ -0,0 +1,6 @@ +enabled: true +# Don't auto-copy draft PRs to pull-request/* branches — avoids burning GPU +# runner time on every push to an in-progress PR. Ready-for-review PRs sync +# automatically; drafts require an explicit `/ok to test` from a vetter. +auto_sync_draft: false +auto_sync_ready: true diff --git a/.github/workflows/perf-regression.yml b/.github/workflows/perf-regression.yml new file mode 100644 index 000000000000..b2a80debf756 --- /dev/null +++ b/.github/workflows/perf-regression.yml @@ -0,0 +1,128 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Performance regression CI gate (OmniPerf). +# +# IMPORTANT: this runs on NVIDIA's self-hosted NVKS fleet, where ProdSec forbids +# `pull_request` / `pull_request_target` triggers. Code reaches us only via +# `copy-pr-bot`, which copies trusted (or `/ok to test`-approved) PRs onto a +# `pull-request/` branch in the source repo. So we trigger on `push` to those +# branches, NOT on `pull_request`. See: +# https://docs.gha-runners.nvidia.com/platform/onboarding/pull-request-testing/ +# +# Because the event is a `push` (not a `pull_request`), `github.event.pull_request.*` +# is unavailable — we recover PR context via the nv-gha-runners/get-pr-info action. + +name: Performance Smoke + +on: + push: + branches: + - "pull-request/[0-9]+" + +# One run per PR branch; cancel superseded pushes. +concurrency: + group: perf-regression-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write # post the perf summary comment + statuses: write # report the gate status back to the PR commit + checks: write + +env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + ISAACSIM_BASE_IMAGE: ${{ vars.ISAACSIM_BASE_IMAGE || 'nvcr.io/nvidia/isaac-sim' }} + ISAACSIM_BASE_VERSION: ${{ vars.ISAACSIM_BASE_VERSION || '5.1.0' }} + DOCKER_IMAGE_TAG: isaac-lab-perf:${{ github.sha }} + +jobs: + # Recover PR metadata (number, base branch, head repo) from the push event. + pr-info: + runs-on: ubuntu-latest + outputs: + pr_number: ${{ steps.get-pr.outputs.number }} + base_sha: ${{ steps.get-pr.outputs.base_sha }} + steps: + - name: Get PR info + id: get-pr + uses: nv-gha-runners/get-pr-info@main + + benchmark: + needs: pr-info + # RTX PRO 6000, group `nv-gpu-amd64-rtxpro6000-1gpu` (driver 595, 16 cores, + # 64G). Only the `latest` driver alias exists for this GPU — there is no + # `earliest`. Confirmed against runner-groups.md. + runs-on: linux-amd64-gpu-rtxpro6000-latest-1 + timeout-minutes: 120 + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + lfs: true + + - name: Build Docker Image + uses: ./.github/actions/docker-build + with: + image-tag: ${{ env.DOCKER_IMAGE_TAG }} + isaacsim-base-image: ${{ env.ISAACSIM_BASE_IMAGE }} + isaacsim-version: ${{ env.ISAACSIM_BASE_VERSION }} + + # --- Angelina's layer: WARM benchmark execution ------------------------- + # Runs the perf smoke suite (G1, H1, Anymal-C, cartpole, camera tasks) and + # emits a machine-readable result (FPS / frame time per task). + # TODO(Angelina): replace with the pytest-based perf module once it lands. + - name: Run Performance Benchmarks (WARM) + run: | + mkdir -p reports + docker run --rm --gpus all \ + -v ${{ github.workspace }}/reports:/workspace/isaaclab/reports \ + ${{ env.DOCKER_IMAGE_TAG }} \ + bash -lc './isaaclab.sh -p scripts/benchmarks/benchmark_non_rl.py \ + --tasks G1 H1 Anymal-C cartpole \ + --warm \ + --output /workspace/isaaclab/reports/perf-results.json' + + # --- Neil's layer: threshold check vs. baseline ------------------------- + # Baselines MUST be keyed by (runner-type, GPU, env) — an AWS-L40S baseline + # is NOT comparable to an RTX-PRO-6000/NVKS run (see #nv-gha-runners thread). + # TODO(Neil): replace with the threshold-checker (load baseline, compute + # delta, fail if any metric regresses beyond the per-task threshold). + - name: Check Against Baseline + id: gate + run: | + echo "PR #${{ needs.pr-info.outputs.pr_number }} base ${{ needs.pr-info.outputs.base_sha }}" + python3 tools/perf/check_thresholds.py \ + --results reports/perf-results.json \ + --baseline-key "nvks-rtxpro6000-1gpu" \ + --pr "${{ needs.pr-info.outputs.pr_number }}" + + - name: Upload Perf Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: perf-results-pr-${{ needs.pr-info.outputs.pr_number }} + path: reports/perf-results.json + retention-days: 7 + compression-level: 9 + + # Status is reported against github.sha, which (per copy-pr-bot) matches the + # PR's head commit, so it surfaces on the PR automatically. + - name: Report Perf Gate Status + if: always() + uses: actions/github-script@v7 + with: + script: | + const ok = '${{ steps.gate.outcome }}' === 'success'; + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.sha, + state: ok ? 'success' : 'failure', + context: 'perf-regression', + description: ok ? 'No perf regression detected' : 'Perf regression detected — see artifact', + });