Add perf-smoke comparator scaffolding under tools/

.github/copy-pr-bot.yaml

@@ -0,0 +1,25 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# copy-pr-bot configuration for NVIDIA self-hosted GPU runners.
+#
+# NVIDIA's self-hosted runners do NOT run workflows triggered by `pull_request`
+# events (a ProdSec requirement so only NVIDIA employees can trigger GPU jobs).
+# Instead, copy-pr-bot mirrors a vetted PR's HEAD commit onto a
+# `pull-request/<PR_NUMBER>` branch in this repository, and workflows trigger on
+# `push` to that branch (see .github/workflows/perf-gate.yml). Because the
+# mirrored commit SHA matches the PR's HEAD SHA, the resulting check statuses are
+# reported back onto the originating pull request.
+#
+# IMPORTANT: this file only takes effect once it is committed to the repo's
+# DEFAULT branch (it is ignored while sitting on a PR), and the copy-pr-bot
+# GitHub app must be installed on the org/repo. See the runner docs:
+#   https://docs.gha-runners.nvidia.com/  (Pull Request Testing)
+enabled: true
+
+# Trusted users' PRs are mirrored automatically when marked ready for review.
+# Untrusted PRs require a vetter to comment `/ok to test <SHA>` first.
+auto_sync_draft: false
+auto_sync_ready: true

.github/workflows/perf-gate.yml

@@ -0,0 +1,223 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+name: Perf Smoke Gate
+
+# -----------------------------------------------------------------------------
+# Phase 1 fork trial on NVIDIA's *shared* self-hosted GPU fleet (arm64 L40S).
+#
+# Those runners do NOT run `pull_request`-triggered workflows (ProdSec rule).
+# copy-pr-bot (.github/copy-pr-bot.yaml) mirrors a vetted PR onto a
+# `pull-request/<N>` branch; we trigger on `push` to that branch. The mirrored
+# SHA equals the PR HEAD SHA, so statuses report back onto the PR.
+#
+# NOTE (upstream variant): IsaacLab's own GPU CI (build.yaml) instead triggers
+# on `pull_request` with generic `[self-hosted, gpu]` labels and runs tests in
+# an ECR-built container. If/when this gate is promoted into the official repo,
+# switch the trigger + `runs-on` to match build.yaml. This file targets the
+# shared arm64 fleet because that is what the trial has access to.
+# -----------------------------------------------------------------------------
+on:
+  push:
+    branches:
+      - "pull-request/[0-9]+"
+  workflow_dispatch:
+    inputs:
+      tasks:
+        description: 'Space-separated gate task names (default: all baseline.json tasks).'
+        required: false
+        default: ''
+      cache_dir:
+        description: 'Optional persistent dir for the warm JIT-cache sidecar (empty = cold run).'
+        required: false
+        default: ''
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: read
+  statuses: write  # post the per-task gate verdict back onto the PR head commit
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Emit the task matrix from baseline.json (single source of truth) so the
+  # matrix can never drift from the calibrated tasks. Runs on a cheap GitHub-
+  # hosted runner; arch-independent.
+  # ---------------------------------------------------------------------------
+  setup:
+    name: Build Task Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      tasks: ${{ steps.tasks.outputs.tasks }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
+        with:
+          fetch-depth: 1
+          sparse-checkout: tools/perf_smoke/baseline.json
+          sparse-checkout-cone-mode: false
+
+      - id: tasks
+        env:
+          DISPATCH_TASKS: ${{ github.event_name == 'workflow_dispatch' && inputs.tasks || '' }}
+        run: |
+          set -euo pipefail
+          if [ -n "${DISPATCH_TASKS}" ]; then
+            tasks_json="$(printf '%s\n' ${DISPATCH_TASKS} | jq -R . | jq -cs .)"
+          else
+            tasks_json="$(jq -c '[keys[] | select(startswith("_") | not)]' tools/perf_smoke/baseline.json)"
+          fi
+          echo "tasks=$tasks_json" >> "$GITHUB_OUTPUT"
+          echo "Matrix tasks: $tasks_json"
+
+  # ---------------------------------------------------------------------------
+  # Pure-logic tests for the comparator, orchestrator, rebaseline tool, and the
+  # history-bucketing helpers. Pure stdlib unittest -- no GPU, no Isaac Sim, and
+  # arch-independent -- so this validates the PASS/WARN/BLOCK verdict logic, the
+  # launch-config plumbing, and the rolling-window writer on every trigger and
+  # gives fast signal before the GPU job is scheduled.
+  # ---------------------------------------------------------------------------
+  comparator-unit-tests:
+    name: Comparator Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
+      - name: Run unittest suite
+        run: |
+          set -e
+          for t in test_check_perf_regression test_history_fingerprint test_run_perf_gate test_rebaseline; do
+            echo "::group::$t"
+            python3 "tools/perf_smoke/$t.py"
+            echo "::endgroup::"
+          done
+
+  # ---------------------------------------------------------------------------
+  # The GPU gate. One job PER TASK (matrix from baseline.json): each task is its
+  # own check, parallelizing across the pool, and re-runnable in isolation.
+  # pytest orchestrates (D1) -- shells out the benchmark as its own Isaac Sim
+  # subprocess, then runs the comparator. Advisory (continue-on-error) for the
+  # fork trial; flip to required once cross-runner variance is confirmed.
+  #
+  # Runs on the shared arm64 L40S fleet. The aarch64 Isaac Sim stack is
+  # supported (see docs/source/setup/installation/pip_installation.rst) but two
+  # deps (imgui-bundle, nlopt) build from source on arm64, hence the dev headers
+  # installed below -- mirroring docker/Dockerfile.base's arm64 branch.
+  # ---------------------------------------------------------------------------
+  perf-gate:
+    name: Perf Gate (${{ matrix.task }})
+    needs: [setup]
+    runs-on: linux-arm64-gpu-l40s-latest-1
+    timeout-minutes: 60
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        task: ${{ fromJSON(needs.setup.outputs.tasks) }}
+    env:
+      GATE_CACHE_DIR: ${{ github.event_name == 'workflow_dispatch' && inputs.cache_dir || '' }}
+      # arm64 Isaac Sim runtime requirements (verified on the L40S dev box):
+      #  - libgomp must be preloaded or the benchmark aborts before launch
+      #    (documented aarch64 workaround, docs/.../pip_installation.rst).
+      #  - kit's EULA prompt is non-interactive under CI; accept it up front.
+      LD_PRELOAD: /lib/aarch64-linux-gnu/libgomp.so.1
+      OMNI_KIT_ACCEPT_EULA: "YES"
+    steps:
+      # Recover PR metadata from the mirrored commit so check statuses associate
+      # with the originating PR. Required on the shared fleet's push model.
+      # TODO(bringup): these two nv-gha-runners actions still need repo-admin
+      # allowlist approval. Pinned to main@<sha> below (no release tags exist).
+      - name: Get PR info
+        # Only meaningful on the copy-pr-bot push model (recovers PR metadata from
+        # the mirrored commit). A manual workflow_dispatch has no originating PR, so
+        # skip it there to keep dispatch-triggered trial runs unblocked.
+        if: github.event_name == 'push'
+        uses: nv-gha-runners/get-pr-info@090577647b8ddc4e06e809e264f7881650ecdccf  # main
+
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
+        with:
+          lfs: true
+
+      # Routes pip/apt through NVIDIA's internal proxy (runners have no public
+      # network). TODO(bringup): allowlist as above.
+      - name: Setup proxy cache
+        uses: nv-gha-runners/setup-proxy-cache@14229018fe157c83e03c008f27d183d8e99bc67c  # main
+
+      # arm64-only build deps for imgui-bundle / nlopt (no prebuilt aarch64
+      # wheels). Mirrors docker/Dockerfile.base. Assumes the runner grants apt;
+      # if it does not, switch this job to run inside the arm64 isaac-lab
+      # container instead (see tools/perf_smoke/RUNNER_BRINGUP.md).
+      - name: Install arm64 build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            python3.12-dev libgl1-mesa-dev libopengl-dev libglx-dev \
+            libx11-dev libxcursor-dev libxi-dev libxinerama-dev libxrandr-dev swig
+
+      - name: Install Isaac Lab
+        run: ./isaaclab.sh --install
+
+      # Re-expose pre-1.13 Warp internals so omni.replicator.core (RTX/camera
+      # path) imports under Warp >=1.13. Idempotent; arch-independent.
+      - name: Install Warp/replicator compatibility shim
+        run: ./isaaclab.sh -p tools/perf_smoke/warp_replicator_shim.py --check || ./isaaclab.sh -p tools/perf_smoke/warp_replicator_shim.py
+
+      - name: Verify GPU availability
+        run: |
+          echo "=== GPU Info ==="
+          nvidia-smi --query-gpu=index,name,driver_version --format=csv
+          GPU_COUNT=$(./isaaclab.sh -p -c "import torch; print(torch.cuda.device_count())")
+          echo "Detected $GPU_COUNT GPU(s)"
+          if [ "$GPU_COUNT" -lt 1 ]; then
+            echo "::error::Perf gate requires a GPU, found $GPU_COUNT"
+            exit 1
+          fi
+
+      # pytest orchestrates: one parametrized test for this task, shelled out as
+      # its own Isaac Sim subprocess, then the comparator. Judged against the
+      # rolling-window store (tools/perf_smoke/perf_history) + in-tree overrides.
+      # BLOCK fails the test; WARN/PASS pass.
+      - name: Run perf gate
+        id: gate
+        env:
+          GATE_RUN: "1"
+          GATE_TASKS: ${{ matrix.task }}
+          GATE_OUTPUT_DIR: ${{ github.workspace }}/perf-output
+        run: ./isaaclab.sh -p -m pytest -v tools/perf_smoke/test_perf_gate.py
+
+      - name: Upload gate output
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: perf-gate-output-${{ github.run_id }}-${{ strategy.job-index }}
+          path: perf-output/
+          if-no-files-found: ignore
+          retention-days: 14
+
+      # Surface the verdict on the PR head commit. On the shared fleet's push
+      # model, github.sha equals the PR HEAD (copy-pr-bot mirrors it), so a
+      # per-task status here shows up as its own check on the originating PR.
+      # One context per matrix task keeps tasks independently visible.
+      - name: Report perf gate status
+        if: always()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const ok = '${{ steps.gate.outcome }}' === 'success';
+            await github.rest.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: context.sha,
+              state: ok ? 'success' : 'failure',
+              context: `perf-gate (${{ matrix.task }})`,
+              description: ok
+                ? 'No perf regression detected'
+                : 'Perf gate failed (BLOCK or benchmark error) — see artifact',
+            });

.github/workflows/perf-rebaseline.yml

@@ -0,0 +1,94 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+name: Perf Baseline Refresh
+
+# Manual rolling re-baseline: measure each task N times on the current main,
+# recompute the in-tree baseline.json values, and open a REVIEWED PR with the
+# diff. Baselines therefore track recent reality, but every change is a visible,
+# approvable diff -- never a silent auto-commit. The boiling-frog guard in
+# rebaseline.py flags/refuses suspicious downward drift.
+on:
+  workflow_dispatch:
+    inputs:
+      repeat:
+        description: 'Runs per task in the rolling window.'
+        required: false
+        default: '5'
+      tasks:
+        description: 'Space-separated tasks (default: all baseline.json tasks).'
+        required: false
+        default: ''
+      cache_dir:
+        description: 'Optional warm JIT-cache dir (empty = cold).'
+        required: false
+        default: ''
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  rebaseline:
+    name: Refresh Perf Baselines
+    runs-on: [self-hosted, gpu]
+    timeout-minutes: 180
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
+        with:
+          lfs: true
+
+      - name: Install Isaac Lab
+        run: ./isaaclab.sh --install
+
+      - name: Install Warp/replicator compatibility shim
+        run: ./isaaclab.sh -p tools/perf_smoke/warp_replicator_shim.py --check || ./isaaclab.sh -p tools/perf_smoke/warp_replicator_shim.py
+
+      - name: Run rolling re-baseline
+        env:
+          R_TASKS: ${{ inputs.tasks }}
+          R_REPEAT: ${{ inputs.repeat }}
+          R_CACHE: ${{ inputs.cache_dir }}
+        run: |
+          set -euo pipefail
+          ARGS="--repeat ${R_REPEAT} --apply --output-dir ${{ github.workspace }}/perf-output-rebaseline"
+          if [ -n "${R_TASKS}" ]; then ARGS="${ARGS} --tasks ${R_TASKS}"; fi
+          if [ -n "${R_CACHE}" ]; then ARGS="${ARGS} --cache-dir ${R_CACHE}"; fi
+          # Capture the report (window stats + guard flags) for the PR body.
+          ./isaaclab.sh -p tools/perf_smoke/rebaseline.py ${ARGS} | tee /tmp/rebaseline_report.txt
+
+      - name: Open re-baseline PR
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          if git diff --quiet -- tools/perf_smoke/baseline.json tools/perf_smoke/perf_history; then
+            echo "No baseline/window changes; nothing to propose."
+            exit 0
+          fi
+          BRANCH="perf/rebaseline-${{ github.run_id }}"
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout -b "${BRANCH}"
+          # The rolling window (perf_history/) is the primary store; baseline.json
+          # carries the refreshed static fallback. Commit both.
+          git add tools/perf_smoke/baseline.json tools/perf_smoke/perf_history
+          git commit -m "perf: refresh rolling baselines (run ${{ github.run_id }})"
+          git push origin "${BRANCH}"
+          {
+            echo "Automated rolling re-baseline of \`tools/perf_smoke/baseline.json\`."
+            echo
+            echo "Review the window stats and any ⚠️/❌ guard flags below before merging."
+            echo
+            echo '```'
+            cat /tmp/rebaseline_report.txt
+            echo '```'
+          } > /tmp/pr_body.md
+          gh pr create \
+            --title "perf: refresh rolling baselines (run ${{ github.run_id }})" \
+            --body-file /tmp/pr_body.md \
+            --base "${{ github.ref_name }}" \
+            --head "${BRANCH}"

scripts/benchmarks/benchmark_non_rl.py

@@ -61,6 +61,7 @@
 
 from scripts.benchmarks.utils import (
     get_backend_type,
+    get_physics_string,
     get_preset_string,
     log_app_start_time,
     log_python_imports_time,
@@ -111,6 +112,7 @@
             {"name": "num_envs", "data": args_cli.num_envs},
             {"name": "num_frames", "data": args_cli.num_frames},
             {"name": "presets", "data": get_preset_string(hydra_args)},
+            {"name": "physics", "data": get_physics_string(hydra_args)},
         ]
     },
 )

scripts/benchmarks/utils.py

@@ -136,6 +136,25 @@ def get_preset_string(hydra_args: list[str]) -> str:
     return os.environ.get("ISAACLAB_BENCHMARK_PRESET", "") or "default"
 
 
+def get_physics_string(hydra_args: list[str]) -> str:
+    """Extract the selected physics backend from CLI hydra args or an environment variable.
+
+    The ``physics=`` Hydra group selects the simulation backend (e.g. ``physx``,
+    ``newton_mjwarp``); unlike rendering/observation modes it is a distinct group
+    from ``presets=``, so it is recorded separately for run provenance.
+
+    Checks (in order):
+        1. ``physics=...`` in *hydra_args* (e.g. ``physics=physx``)
+        2. ``ISAACLAB_BENCHMARK_PHYSICS`` environment variable
+        3. Falls back to ``"default"`` (the task's configured backend)
+    """
+    for arg in hydra_args:
+        if arg.startswith("physics="):
+            value = arg.split("=", 1)[1]
+            return value if value else "default"
+    return os.environ.get("ISAACLAB_BENCHMARK_PHYSICS", "") or "default"
+
+
 def log_rl_policy_rewards(benchmark: BaseIsaacLabBenchmark, value: list):
     measurement = ListMeasurement(name="Rewards", value=value)
     benchmark.add_measurement("train", measurement=measurement)