From 4d0b4585959f1f33c0af22ad8137dcc6679805f8 Mon Sep 17 00:00:00 2001 From: Neil4561 <283821122+Neil4561@users.noreply.github.com> Date: Thu, 11 Jun 2026 20:20:45 +0000 Subject: [PATCH 1/3] Add performance regression smoke gate for CI Add a PR-time performance gate that benchmarks a small fixed set of Isaac Lab tasks on the arm64 L40S fleet and compares each result against a rolling baseline, reporting PASS/WARN/BLOCK back onto the PR. Includes the gate orchestrator, the runtime-free comparator (median+MAD) with its unit and stress tests, the per-task baseline config and seeded rolling-window history, re-baselining helpers, the Warp/replicator arm64 shim, and the GitHub Actions workflow (push to pull-request/* with copy-pr-bot, per-task matrix, commit-status reporting). Advisory to start. --- .github/copy-pr-bot.yaml | 25 + .github/workflows/perf-gate.yml | 211 ++++++ .github/workflows/perf-rebaseline.yml | 94 +++ tools/perf_smoke/README.md | 85 +++ tools/perf_smoke/baseline.json | 134 ++++ tools/perf_smoke/baseline_overrides.json | 24 + tools/perf_smoke/check_perf_regression.py | 636 ++++++++++++++++++ tools/perf_smoke/demo_regression.py | 153 +++++ ...Isaac-Cartpole-v0@newton__NVIDIA_L40S.json | 40 ++ .../Isaac-Cartpole-v0__NVIDIA_L40S.json | 40 ++ ...ctory-GearMesh-Direct-v0__NVIDIA_L40S.json | 40 ++ ...-Shadow-Vision-Direct-v0__NVIDIA_L40S.json | 40 ++ ...locity-Flat-G1-v0@newton__NVIDIA_L40S.json | 40 ++ ...saac-Velocity-Flat-G1-v0__NVIDIA_L40S.json | 40 ++ tools/perf_smoke/pytest.ini | 11 + tools/perf_smoke/rebaseline.py | 365 ++++++++++ tools/perf_smoke/run_perf_gate.py | 300 +++++++++ tools/perf_smoke/seed_history.py | 104 +++ .../perf_smoke/test_check_perf_regression.py | 606 +++++++++++++++++ tools/perf_smoke/test_perf_gate.py | 86 +++ .../test_stress_check_perf_regression.py | 381 +++++++++++ tools/perf_smoke/warp_replicator_shim.py | 125 ++++ 22 files changed, 3580 insertions(+) create mode 100644 .github/copy-pr-bot.yaml create mode 100644 .github/workflows/perf-gate.yml create mode 100644 .github/workflows/perf-rebaseline.yml create mode 100644 tools/perf_smoke/README.md create mode 100644 tools/perf_smoke/baseline.json create mode 100644 tools/perf_smoke/baseline_overrides.json create mode 100644 tools/perf_smoke/check_perf_regression.py create mode 100644 tools/perf_smoke/demo_regression.py create mode 100644 tools/perf_smoke/perf_history/Isaac-Cartpole-v0@newton__NVIDIA_L40S.json create mode 100644 tools/perf_smoke/perf_history/Isaac-Cartpole-v0__NVIDIA_L40S.json create mode 100644 tools/perf_smoke/perf_history/Isaac-Factory-GearMesh-Direct-v0__NVIDIA_L40S.json create mode 100644 tools/perf_smoke/perf_history/Isaac-Repose-Cube-Shadow-Vision-Direct-v0__NVIDIA_L40S.json create mode 100644 tools/perf_smoke/perf_history/Isaac-Velocity-Flat-G1-v0@newton__NVIDIA_L40S.json create mode 100644 tools/perf_smoke/perf_history/Isaac-Velocity-Flat-G1-v0__NVIDIA_L40S.json create mode 100644 tools/perf_smoke/pytest.ini create mode 100644 tools/perf_smoke/rebaseline.py create mode 100644 tools/perf_smoke/run_perf_gate.py create mode 100644 tools/perf_smoke/seed_history.py create mode 100644 tools/perf_smoke/test_check_perf_regression.py create mode 100644 tools/perf_smoke/test_perf_gate.py create mode 100644 tools/perf_smoke/test_stress_check_perf_regression.py create mode 100644 tools/perf_smoke/warp_replicator_shim.py diff --git a/.github/copy-pr-bot.yaml b/.github/copy-pr-bot.yaml new file mode 100644 index 000000000000..3ed8fa893ddd --- /dev/null +++ b/.github/copy-pr-bot.yaml @@ -0,0 +1,25 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# copy-pr-bot configuration for NVIDIA self-hosted GPU runners. +# +# NVIDIA's self-hosted runners do NOT run workflows triggered by `pull_request` +# events (a ProdSec requirement so only NVIDIA employees can trigger GPU jobs). +# Instead, copy-pr-bot mirrors a vetted PR's HEAD commit onto a +# `pull-request/` branch in this repository, and workflows trigger on +# `push` to that branch (see .github/workflows/perf-gate.yml). Because the +# mirrored commit SHA matches the PR's HEAD SHA, the resulting check statuses are +# reported back onto the originating pull request. +# +# IMPORTANT: this file only takes effect once it is committed to the repo's +# DEFAULT branch (it is ignored while sitting on a PR), and the copy-pr-bot +# GitHub app must be installed on the org/repo. See the runner docs: +# https://docs.gha-runners.nvidia.com/ (Pull Request Testing) +enabled: true + +# Trusted users' PRs are mirrored automatically when marked ready for review. +# Untrusted PRs require a vetter to comment `/ok to test ` first. +auto_sync_draft: false +auto_sync_ready: true diff --git a/.github/workflows/perf-gate.yml b/.github/workflows/perf-gate.yml new file mode 100644 index 000000000000..4506fb4cdb21 --- /dev/null +++ b/.github/workflows/perf-gate.yml @@ -0,0 +1,211 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +name: Perf Smoke Gate + +# ----------------------------------------------------------------------------- +# Phase 1 fork trial on NVIDIA's *shared* self-hosted GPU fleet (arm64 L40S). +# +# Those runners do NOT run `pull_request`-triggered workflows (ProdSec rule). +# copy-pr-bot (.github/copy-pr-bot.yaml) mirrors a vetted PR onto a +# `pull-request/` branch; we trigger on `push` to that branch. The mirrored +# SHA equals the PR HEAD SHA, so statuses report back onto the PR. +# +# NOTE (upstream variant): IsaacLab's own GPU CI (build.yaml) instead triggers +# on `pull_request` with generic `[self-hosted, gpu]` labels and runs tests in +# an ECR-built container. If/when this gate is promoted into the official repo, +# switch the trigger + `runs-on` to match build.yaml. This file targets the +# shared arm64 fleet because that is what the trial has access to. +# ----------------------------------------------------------------------------- +on: + push: + branches: + - "pull-request/[0-9]+" + workflow_dispatch: + inputs: + tasks: + description: 'Space-separated gate task names (default: all baseline.json tasks).' + required: false + default: '' + cache_dir: + description: 'Optional persistent dir for the warm JIT-cache sidecar (empty = cold run).' + required: false + default: '' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: read + statuses: write # post the per-task gate verdict back onto the PR head commit + +jobs: + # --------------------------------------------------------------------------- + # Emit the task matrix from baseline.json (single source of truth) so the + # matrix can never drift from the calibrated tasks. Runs on a cheap GitHub- + # hosted runner; arch-independent. + # --------------------------------------------------------------------------- + setup: + name: Build Task Matrix + runs-on: ubuntu-latest + outputs: + tasks: ${{ steps.tasks.outputs.tasks }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + sparse-checkout: tools/perf_smoke/baseline.json + sparse-checkout-cone-mode: false + + - id: tasks + env: + DISPATCH_TASKS: ${{ github.event_name == 'workflow_dispatch' && inputs.tasks || '' }} + run: | + set -euo pipefail + if [ -n "${DISPATCH_TASKS}" ]; then + tasks_json="$(printf '%s\n' ${DISPATCH_TASKS} | jq -R . | jq -cs .)" + else + tasks_json="$(jq -c '[keys[] | select(startswith("_") | not)]' tools/perf_smoke/baseline.json)" + fi + echo "tasks=$tasks_json" >> "$GITHUB_OUTPUT" + echo "Matrix tasks: $tasks_json" + + # --------------------------------------------------------------------------- + # Comparator logic tests. Pure stdlib unittest -- no GPU, no Isaac Sim, and + # arch-independent -- so this validates the PASS/WARN/BLOCK verdict logic on + # every trigger and gives fast signal before the GPU job is scheduled. + # --------------------------------------------------------------------------- + comparator-unit-tests: + name: Comparator Unit Tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Run comparator unittest suite + run: python3 tools/perf_smoke/test_check_perf_regression.py + + # --------------------------------------------------------------------------- + # The GPU gate. One job PER TASK (matrix from baseline.json): each task is its + # own check, parallelizing across the pool, and re-runnable in isolation. + # pytest orchestrates (D1) -- shells out the benchmark as its own Isaac Sim + # subprocess, then runs the comparator. Advisory (continue-on-error) for the + # fork trial; flip to required once cross-runner variance is confirmed. + # + # Runs on the shared arm64 L40S fleet. The aarch64 Isaac Sim stack is + # supported (see docs/source/setup/installation/pip_installation.rst) but two + # deps (imgui-bundle, nlopt) build from source on arm64, hence the dev headers + # installed below -- mirroring docker/Dockerfile.base's arm64 branch. + # --------------------------------------------------------------------------- + perf-gate: + name: Perf Gate (${{ matrix.task }}) + needs: [setup] + runs-on: linux-arm64-gpu-l40s-latest-1 + timeout-minutes: 60 + continue-on-error: true + strategy: + fail-fast: false + matrix: + task: ${{ fromJSON(needs.setup.outputs.tasks) }} + env: + GATE_CACHE_DIR: ${{ github.event_name == 'workflow_dispatch' && inputs.cache_dir || '' }} + # arm64 Isaac Sim runtime requirements (verified on the L40S dev box): + # - libgomp must be preloaded or the benchmark aborts before launch + # (documented aarch64 workaround, docs/.../pip_installation.rst). + # - kit's EULA prompt is non-interactive under CI; accept it up front. + LD_PRELOAD: /lib/aarch64-linux-gnu/libgomp.so.1 + OMNI_KIT_ACCEPT_EULA: "YES" + steps: + # Recover PR metadata from the mirrored commit so check statuses associate + # with the originating PR. Required on the shared fleet's push model. + # TODO(bringup): these two nv-gha-runners actions still need repo-admin + # allowlist approval. Pinned to main@ below (no release tags exist). + - name: Get PR info + uses: nv-gha-runners/get-pr-info@090577647b8ddc4e06e809e264f7881650ecdccf # main + + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + lfs: true + + # Routes pip/apt through NVIDIA's internal proxy (runners have no public + # network). TODO(bringup): allowlist as above. + - name: Setup proxy cache + uses: nv-gha-runners/setup-proxy-cache@14229018fe157c83e03c008f27d183d8e99bc67c # main + + # arm64-only build deps for imgui-bundle / nlopt (no prebuilt aarch64 + # wheels). Mirrors docker/Dockerfile.base. Assumes the runner grants apt; + # if it does not, switch this job to run inside the arm64 isaac-lab + # container instead (see tools/perf_smoke/RUNNER_BRINGUP.md). + - name: Install arm64 build dependencies + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + python3.12-dev libgl1-mesa-dev libopengl-dev libglx-dev \ + libx11-dev libxcursor-dev libxi-dev libxinerama-dev libxrandr-dev swig + + - name: Install Isaac Lab + run: ./isaaclab.sh --install + + # Re-expose pre-1.13 Warp internals so omni.replicator.core (RTX/camera + # path) imports under Warp >=1.13. Idempotent; arch-independent. + - name: Install Warp/replicator compatibility shim + run: ./isaaclab.sh -p tools/perf_smoke/warp_replicator_shim.py --check || ./isaaclab.sh -p tools/perf_smoke/warp_replicator_shim.py + + - name: Verify GPU availability + run: | + echo "=== GPU Info ===" + nvidia-smi --query-gpu=index,name,driver_version --format=csv + GPU_COUNT=$(./isaaclab.sh -p -c "import torch; print(torch.cuda.device_count())") + echo "Detected $GPU_COUNT GPU(s)" + if [ "$GPU_COUNT" -lt 1 ]; then + echo "::error::Perf gate requires a GPU, found $GPU_COUNT" + exit 1 + fi + + # pytest orchestrates: one parametrized test for this task, shelled out as + # its own Isaac Sim subprocess, then the comparator. Judged against the + # rolling-window store (tools/perf_smoke/perf_history) + in-tree overrides. + # BLOCK fails the test; WARN/PASS pass. + - name: Run perf gate + id: gate + env: + GATE_RUN: "1" + GATE_TASKS: ${{ matrix.task }} + GATE_OUTPUT_DIR: ${{ github.workspace }}/perf-output + run: ./isaaclab.sh -p -m pytest -v tools/perf_smoke/test_perf_gate.py + + - name: Upload gate output + if: always() + uses: actions/upload-artifact@v7 + with: + name: perf-gate-output-${{ github.run_id }}-${{ strategy.job-index }} + path: perf-output/ + if-no-files-found: ignore + retention-days: 14 + + # Surface the verdict on the PR head commit. On the shared fleet's push + # model, github.sha equals the PR HEAD (copy-pr-bot mirrors it), so a + # per-task status here shows up as its own check on the originating PR. + # One context per matrix task keeps tasks independently visible. + - name: Report perf gate status + if: always() + uses: actions/github-script@v7 + with: + script: | + const ok = '${{ steps.gate.outcome }}' === 'success'; + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.sha, + state: ok ? 'success' : 'failure', + context: `perf-gate (${{ matrix.task }})`, + description: ok + ? 'No perf regression detected' + : 'Perf gate failed (BLOCK or benchmark error) — see artifact', + }); diff --git a/.github/workflows/perf-rebaseline.yml b/.github/workflows/perf-rebaseline.yml new file mode 100644 index 000000000000..8eac93c74536 --- /dev/null +++ b/.github/workflows/perf-rebaseline.yml @@ -0,0 +1,94 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +name: Perf Baseline Refresh + +# Manual rolling re-baseline: measure each task N times on the current main, +# recompute the in-tree baseline.json values, and open a REVIEWED PR with the +# diff. Baselines therefore track recent reality, but every change is a visible, +# approvable diff -- never a silent auto-commit. The boiling-frog guard in +# rebaseline.py flags/refuses suspicious downward drift. +on: + workflow_dispatch: + inputs: + repeat: + description: 'Runs per task in the rolling window.' + required: false + default: '5' + tasks: + description: 'Space-separated tasks (default: all baseline.json tasks).' + required: false + default: '' + cache_dir: + description: 'Optional warm JIT-cache dir (empty = cold).' + required: false + default: '' + +permissions: + contents: write + pull-requests: write + +jobs: + rebaseline: + name: Refresh Perf Baselines + runs-on: [self-hosted, gpu] + timeout-minutes: 180 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + lfs: true + + - name: Install Isaac Lab + run: ./isaaclab.sh --install + + - name: Install Warp/replicator compatibility shim + run: ./isaaclab.sh -p tools/perf_smoke/warp_replicator_shim.py --check || ./isaaclab.sh -p tools/perf_smoke/warp_replicator_shim.py + + - name: Run rolling re-baseline + env: + R_TASKS: ${{ inputs.tasks }} + R_REPEAT: ${{ inputs.repeat }} + R_CACHE: ${{ inputs.cache_dir }} + run: | + set -euo pipefail + ARGS="--repeat ${R_REPEAT} --apply --output-dir ${{ github.workspace }}/perf-output-rebaseline" + if [ -n "${R_TASKS}" ]; then ARGS="${ARGS} --tasks ${R_TASKS}"; fi + if [ -n "${R_CACHE}" ]; then ARGS="${ARGS} --cache-dir ${R_CACHE}"; fi + # Capture the report (window stats + guard flags) for the PR body. + ./isaaclab.sh -p tools/perf_smoke/rebaseline.py ${ARGS} | tee /tmp/rebaseline_report.txt + + - name: Open re-baseline PR + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + if git diff --quiet -- tools/perf_smoke/baseline.json tools/perf_smoke/perf_history; then + echo "No baseline/window changes; nothing to propose." + exit 0 + fi + BRANCH="perf/rebaseline-${{ github.run_id }}" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git checkout -b "${BRANCH}" + # The rolling window (perf_history/) is the primary store; baseline.json + # carries the refreshed static fallback. Commit both. + git add tools/perf_smoke/baseline.json tools/perf_smoke/perf_history + git commit -m "perf: refresh rolling baselines (run ${{ github.run_id }})" + git push origin "${BRANCH}" + { + echo "Automated rolling re-baseline of \`tools/perf_smoke/baseline.json\`." + echo + echo "Review the window stats and any ⚠️/❌ guard flags below before merging." + echo + echo '```' + cat /tmp/rebaseline_report.txt + echo '```' + } > /tmp/pr_body.md + gh pr create \ + --title "perf: refresh rolling baselines (run ${{ github.run_id }})" \ + --body-file /tmp/pr_body.md \ + --base "${{ github.ref_name }}" \ + --head "${BRANCH}" diff --git a/tools/perf_smoke/README.md b/tools/perf_smoke/README.md new file mode 100644 index 000000000000..e9fbae08cb2f --- /dev/null +++ b/tools/perf_smoke/README.md @@ -0,0 +1,85 @@ +# Performance Regression Smoke Gate + +A small CI gate that catches **GPU performance regressions** in Isaac Lab before +they merge. On a pull request it runs a few short benchmark tasks on a fixed GPU, +compares the measured speed against a rolling baseline, and reports +PASS / WARN / BLOCK back onto the PR. + +This document explains, in plain terms, what each file in this commit does. + +## The idea in one paragraph + +Every PR, we run a handful of cheap, stable Isaac Lab tasks (e.g. Cartpole) on a +known GPU and measure how fast they step (frames per second). We keep a short +history of recent "known-good" numbers per task and GPU. If a PR's measurement +drops well below that history, the gate flags it. Small wobble is expected, so a +mild dip is an advisory **WARN** and only a real, sustained drop is a **BLOCK**. + +## What's in this directory + +### The gate itself (the parts that do the work) + +| File | Plain-English purpose | +|---|---| +| `run_perf_gate.py` | The orchestrator. For each task it launches the Isaac Lab benchmark as its own process, then hands the result to the comparator and prints the verdict. | +| `check_perf_regression.py` | The comparator (pure logic, no GPU needed). Reads a benchmark result + the baseline/history and decides PASS, WARN, or BLOCK using a median + MAD test. | +| `baseline.json` | Per-task run settings (num envs, frames, seed) **and** a static fallback speed used when there isn't enough history yet. Keyed by task and GPU. | +| `baseline_overrides.json` | Manual, in-tree threshold overrides that ride along with a PR when a one-off adjustment is needed. | +| `perf_history/` | The rolling window of recent measurements, one JSON per task+GPU. This is what the comparator normally judges against. | + +### Helper tools + +| File | Plain-English purpose | +|---|---| +| `rebaseline.py` | Re-computes the baseline/history from fresh runs (used when moving to new hardware). | +| `seed_history.py` | Seeds the `perf_history/` window from existing runs. | +| `demo_regression.py` | Injects a fake slowdown so you can watch the gate correctly BLOCK. | +| `warp_replicator_shim.py` | Small compatibility fix so the camera/RTX tasks import under newer Warp. Needed on arm64. | + +### Tests + +| File | Plain-English purpose | +|---|---| +| `test_check_perf_regression.py` | Unit tests for the comparator logic (no GPU). | +| `test_stress_check_perf_regression.py` | Heavier stress/edge-case tests for the comparator. | +| `test_perf_gate.py` | The pytest entry point CI uses to drive a single task end-to-end. | +| `pytest.ini` | Local pytest config for this directory. | + +### CI wiring (in `.github/`) + +| File | Plain-English purpose | +|---|---| +| `workflows/perf-gate.yml` | The GitHub Actions workflow. Runs the gate on the arm64 L40S fleet, one job per task, and posts a status back to the PR. | +| `workflows/perf-rebaseline.yml` | A manual workflow to re-bless baselines on the runner. | +| `copy-pr-bot.yaml` | Enables NVIDIA's copy-pr-bot so PRs are mirrored to `pull-request/*` branches (required for the self-hosted fleet). | + +## The verdict model + +| Verdict | Meaning | Effect | +|---|---|---| +| **PASS** | Within or above expected speed. | Gate succeeds. | +| **WARN** | Mild dip, within noise. | Advisory only; does not fail. | +| **BLOCK** | A real regression, or a structural problem (missing/blank result, unknown GPU, config mismatch). | Gate fails for that task. | + +## Running it locally + +Run the full gate for one task (needs a GPU + Isaac Lab installed): + +```bash +./isaaclab.sh -p tools/perf_smoke/run_perf_gate.py --tasks Isaac-Cartpole-v0 +``` + +Run just the comparator tests (no GPU needed): + +```bash +python3 tools/perf_smoke/test_check_perf_regression.py +``` + +## Current status + +- Target hardware: **arm64 L40S** on NVIDIA's shared self-hosted fleet. +- The gate is **advisory** to start (`continue-on-error: true`): it reports a + verdict on the PR but does not block merges yet. It is flipped to required + once cross-runner variance is confirmed. +- Baselines must be **re-blessed on the runner fleet** before the verdict is + authoritative; the values committed here are calibration starting points. diff --git a/tools/perf_smoke/baseline.json b/tools/perf_smoke/baseline.json new file mode 100644 index 000000000000..b0786dd9c51d --- /dev/null +++ b/tools/perf_smoke/baseline.json @@ -0,0 +1,134 @@ +{ + "_schema_version": 3, + "_notes": [ + "Phase 1 perf-smoke gate config + static fallback (4-task subset). Implements", + "ci-regression-gate-config-info.md.", + "ROLE OF THIS FILE: (1) the run config each task is launched with (num_envs, num_frames, seed,", + "benchmark_args, warmup_frames) -- read by the gate so a measurement is always taken under the", + "config the window was calibrated with; (2) a STATIC FALLBACK (baseline_fps + warn_pct/", + "max_regression_pct) used only when the rolling-window history store has < 3 samples.", + "THRESHOLDS (normal path): computed at test time from a rolling window via median+MAD (see", + "check_perf_regression.py and the perf_history/ store -- the orphan-branch stand-in). Manual", + "overrides live in baseline_overrides.json (committed with the PR), NOT here and NOT in the window.", + "D6 warm-up exclusion: the gating KPI is the mean post-warm-up effective FPS -- the first", + "warmup_frames are dropped (2 for PhysX tasks; 60 for shadow-vision to clear the camera/JIT window).", + "VARIANTS: a key may be a gym id (e.g. 'Isaac-Cartpole-v0', the PhysX default) or a variant", + "'@' (e.g. 'Isaac-Cartpole-v0@newton') carrying its own 'task_id' (the real gym", + "task the benchmark is launched with) and benchmark_args. Newton uses physics=newton_mjwarp and a", + "5-frame warm-up (JIT). Output/history are keyed by the full variant key so PhysX and Newton don't collide.", + "Gate config: NVIDIA L40S, WARM runs/task, num_frames=300, seed=42.", + "Fallback values were derived from the existing warm L40S runs (500f truncated to 300f, post-warm-up);", + "the perf_history/ window holds the per-run samples. Authoritative re-bless happens on the runner pool.", + "Per-GPU keys substring-match hardware_info.gpu_devices[].name ('NVIDIA L40S' matches that device)." + ], + "Isaac-Cartpole-v0": { + "num_envs": 4096, + "num_frames": 300, + "seed": 42, + "warmup_frames": 2, + "benchmark_args": ["physics=physx"], + "config_note": "cartpole_physx_n4096", + "per_gpu": { + "NVIDIA L40S": { + "baseline_fps": 115235.8, + "warn_pct": 5.0, + "max_regression_pct": 10.0, + "cv_pct": 2.02, + "n_runs": 5 + } + } + }, + "Isaac-Factory-GearMesh-Direct-v0": { + "num_envs": 512, + "num_frames": 300, + "seed": 42, + "warmup_frames": 2, + "benchmark_args": [], + "config_note": "factory_physx_n512", + "per_gpu": { + "NVIDIA L40S": { + "baseline_fps": 880.5, + "warn_pct": 5.0, + "max_regression_pct": 10.0, + "cv_pct": 1.29, + "n_runs": 5 + } + } + }, + "Isaac-Velocity-Flat-G1-v0": { + "num_envs": 2048, + "num_frames": 300, + "seed": 42, + "warmup_frames": 2, + "benchmark_args": ["physics=physx"], + "config_note": "g1_flat_physx_n2048", + "per_gpu": { + "NVIDIA L40S": { + "baseline_fps": 19213.7, + "warn_pct": 5.0, + "max_regression_pct": 10.0, + "cv_pct": 0.81, + "n_runs": 5 + } + } + }, + "Isaac-Cartpole-v0@newton": { + "task_id": "Isaac-Cartpole-v0", + "num_envs": 4096, + "num_frames": 300, + "seed": 42, + "warmup_frames": 5, + "benchmark_args": ["physics=newton_mjwarp"], + "config_note": "cartpole_newton_n4096 (Newton/MJWarp; warm-up 5 frames for JIT)", + "per_gpu": { + "NVIDIA L40S": { + "baseline_fps": 358461.3, + "warn_pct": 6.0, + "max_regression_pct": 12.0, + "cv_pct": 2.9, + "n_runs": 5 + } + } + }, + "Isaac-Velocity-Flat-G1-v0@newton": { + "task_id": "Isaac-Velocity-Flat-G1-v0", + "num_envs": 2048, + "num_frames": 300, + "seed": 42, + "warmup_frames": 5, + "benchmark_args": ["physics=newton_mjwarp"], + "config_note": "g1_flat_newton_n2048 (Newton/MJWarp; warm-up 5 frames for JIT)", + "per_gpu": { + "NVIDIA L40S": { + "baseline_fps": 69660.2, + "warn_pct": 6.0, + "max_regression_pct": 12.0, + "cv_pct": 1.9, + "n_runs": 5 + } + } + }, + "Isaac-Repose-Cube-Shadow-Vision-Direct-v0": { + "num_envs": 128, + "num_frames": 300, + "seed": 42, + "warmup_frames": 60, + "benchmark_args": [ + "--enable_cameras", + "physics=physx", + "presets=isaacsim_rtx_renderer", + "env.tiled_camera.width=64", + "env.tiled_camera.height=64" + ], + "config_note": "shadow_vision_physx_rtx_64x64_n128 (needs the warp/replicator shim)", + "per_gpu": { + "NVIDIA L40S": { + "baseline_fps": 1024.6, + "warn_pct": 5.0, + "max_regression_pct": 10.0, + "cv_pct": 0.86, + "n_runs": 5 + } + } + } +} diff --git a/tools/perf_smoke/baseline_overrides.json b/tools/perf_smoke/baseline_overrides.json new file mode 100644 index 000000000000..497f5166862d --- /dev/null +++ b/tools/perf_smoke/baseline_overrides.json @@ -0,0 +1,24 @@ +{ + "_notes": [ + "Manual threshold overrides, committed WITH the PR (not in the orphan-branch window).", + "This is the escape hatch from ci-regression-gate-config-info.md: when a PR legitimately", + "changes performance, or a task is temporarily flaky, the author adjusts the gate here in the", + "same commit, so the change is reviewed alongside the code.", + "Resolution order: _defaults < < .. Recognized keys:", + " k_warn / k_block -- sigma multipliers on the MAD spread (PASS/WARN/BLOCK bands).", + " min_spread_pct -- floor on spread as % of center (guards tiny windows).", + " pin_center_fps -- pin the baseline center (e.g. an intended perf change).", + " pin_spread_fps -- pin the absolute spread.", + " tail_p99_warn -- opt-in: WARN (never BLOCK) when post-warm-up p99/median step ratio", + " exceeds this. Surfaces tail/spike regressions the FPS mean hides.", + " Leave unset for tasks with real recurring spikes (e.g. g1-flat).", + " skip (true) -- force PASS for a task (quarantine); use sparingly.", + "Example:", + " \"Isaac-Cartpole-v0\": { \"NVIDIA L40S\": { \"k_block\": 8, \"tail_p99_warn\": 1.8 } }" + ], + "_defaults": { + "k_warn": 3.0, + "k_block": 6.0, + "min_spread_pct": 1.5 + } +} diff --git a/tools/perf_smoke/check_perf_regression.py b/tools/perf_smoke/check_perf_regression.py new file mode 100644 index 000000000000..626a2d3f529a --- /dev/null +++ b/tools/perf_smoke/check_perf_regression.py @@ -0,0 +1,636 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Compare a benchmark result JSON against the perf-smoke baseline. + +Used by the Phase 1 perf-smoke CI gate. Implements the test logic of +``ci-regression-gate-config-info.md``: + +Verdict ``{PASS, WARN, BLOCK}`` (the doc's vocabulary), surfaced as a grep-able +``RESULT=`` line and a rich ``$GITHUB_STEP_SUMMARY`` table: + +* ``PASS`` -- measured KPI within threshold of the baseline center. +* ``WARN`` -- medium regression (between the warn and block bands), or a result + that needed a retry. Advisory; does not block merge (exit ``0``). +* ``BLOCK`` -- a hard failure. ``kind=regression`` for a large KPI drop (exit + ``1``); ``kind=hard_failure`` for a structural problem -- missing/malformed + result, missing baseline, NaN/zero FPS, unknown GPU (exit ``2``). Both block. + +KPIs (D-test-logic) +------------------- +* ``fps`` -- ``Mean Environment step effective FPS`` (primary), computed over the + *post-warm-up* frames (D6): the per-frame effective-FPS array with the first + ``warmup_frames`` dropped. This is the identical statistic the backend reports, + just windowed, so it stays comparable to the rolling window. +* ``wall_s`` -- wall-clock seconds of the run (secondary signal; advisory). + +Threshold & baseline strategy (D-threshold) +------------------------------------------- +Thresholds are computed *at test time* from a rolling window of historical +samples using a robust **median + MAD** estimator (tunable ``k``): + + center = median(window) + spread = max(1.4826 * MAD(window), min_spread_pct/100 * center) + WARN when measured < center - k_warn * spread + BLOCK when measured < center - k_block * spread (kind=regression) + +The window lives in an orphan-branch history store (``--history-dir``; the local +stand-in is a plain directory, exactly like the warm-cache sidecar). Manual +overrides (k values, spread floor, pinned center, or skip) come from an in-tree +``baseline_overrides.json`` committed *with the PR*. When no window is available +yet (fresh store) the comparator falls back to the static ``baseline_fps`` and +``warn_pct`` / ``max_regression_pct`` carried in ``baseline.json`` so the gate +still produces a verdict. + +Result formats +-------------- +Both benchmark backends are accepted: OmniPerf (a dict) and JSON (a list of +phase objects, normalized into the dict shape on load). The JSON backend carries +the per-frame arrays the steady metric and debug KPIs need. +""" + +from __future__ import annotations + +import argparse +import glob +import json +import os +import sys +from pathlib import Path + +EXIT_PASS = 0 +EXIT_BLOCK = 1 +EXIT_HARD_FAILURE = 2 + +METRIC_PHASE = "runtime" +# Scalar all-frames fallback (used only when the per-frame array is absent). +METRIC_NAME = "Mean Environment step effective FPS" +# Per-frame arrays (json backend), inside the "Step Frametimes" measurement. +FRAMETIMES_NAME = "Step Frametimes" +EFF_FPS_ARRAY = "Environment step effective FPS" +STEP_MS_ARRAY = "Environment step times" + +# D6: frames dropped before computing the gating KPI (warm-up / first-touch / +# JIT pollution). Per-task value comes from baseline.json; this is the default. +DEFAULT_WARMUP_FRAMES = 2 + +# Robust-threshold defaults (D-threshold); overridable per task/gpu. +DEFAULT_K_WARN = 3.0 +DEFAULT_K_BLOCK = 6.0 +DEFAULT_MIN_SPREAD_PCT = 1.5 # spread floor as % of center (guards tiny windows) +MAD_TO_STD = 1.4826 # MAD -> std-equivalent for ~normal data +MIN_WINDOW = 3 # samples needed before the rolling estimator is trusted +OUTLIER_FACTOR = 2.0 # a step slower than 2x the steady median is an outlier +MAX_REPORTED_OUTLIERS = 8 # cap index/magnitude lists in the report +# Warm-up guardrail: if the first *kept* (post-warm-up) step is slower than this +# multiple of the steady median, warmup_frames is probably too small (pollution +# leaking into the KPI). Advisory only -- never changes the verdict. +WARMUP_GUARD_FACTOR = 3.0 + +DEFAULT_GLOB_TEMPLATE = "benchmark_non_rl_{task}*.json" +_JSON_NAME_PREFIX = "benchmark_non_rl" + + +class CompareError(Exception): + """Raised for any structural problem that should map to ``BLOCK/hard_failure``.""" + + +_RESULT_BADGE = {"PASS": "✅", "WARN": "⚠️", "BLOCK": "❌"} + + +def _emit(result: str, **fields: object) -> None: + """Print the machine-parseable line and a markdown table to the CI summary. + + Args: + result: One of ``PASS``, ``WARN``, ``BLOCK``. + **fields: Additional ``key=value`` pairs (``kind=...`` for ``BLOCK``). + """ + parts = [f"RESULT={result}"] + [f"{k}={v}" for k, v in fields.items()] + print(" ".join(parts)) + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + try: + badge = _RESULT_BADGE.get(result, "") + task = fields.get("task", "") + rows = "\n".join(f"| {k} | {v} |" for k, v in fields.items() if k != "task") + with open(summary_path, "a", encoding="utf-8") as f: + f.write(f"### {badge} Perf Smoke: {result} — `{task}`\n\n| metric | value |\n|---|---|\n{rows}\n\n") + except OSError: + pass + + +# ----------------------------------------------------------------------------- IO + + +def _read_json(path: Path) -> object: + """Read and parse a JSON file, returning whatever top-level value it holds.""" + try: + with open(path, encoding="utf-8") as f: + return json.load(f) + except FileNotFoundError: + raise CompareError(f"file_not_found path={path}") + except json.JSONDecodeError as e: + raise CompareError(f"malformed_json path={path} line={e.lineno} col={e.colno}") + + +def _load_json(path: Path) -> dict: + """Load a JSON file that must be a top-level object.""" + data = _read_json(path) + if not isinstance(data, dict): + raise CompareError(f"json_not_object path={path}") + return data + + +def _normalize_result(data: object) -> dict: + """Normalize a benchmark result into the OmniPerf ``{phase: {metric: value}}`` shape.""" + if isinstance(data, dict): + return data + if not isinstance(data, list): + raise CompareError(f"json_not_object_or_list type={type(data).__name__}") + out: dict[str, dict] = {} + for phase in data: + if not isinstance(phase, dict): + continue + name = phase.get("phase_name") + if not isinstance(name, str) or not name: + continue + prefix = f"{_JSON_NAME_PREFIX} {name} " + bucket: dict[str, object] = {} + for entry in phase.get("measurements", []) or []: + if isinstance(entry, dict) and "name" in entry and "value" in entry: + key = str(entry["name"]) + bucket[key[len(prefix) :] if key.startswith(prefix) else key] = entry["value"] + for entry in phase.get("metadata", []) or []: + if isinstance(entry, dict) and "name" in entry and "data" in entry: + key = str(entry["name"]) + bucket.setdefault(key[len(prefix) :] if key.startswith(prefix) else key, entry["data"]) + out[name] = bucket + return out + + +def _load_result(path: Path) -> dict: + """Load a benchmark result file and normalize it to the OmniPerf shape.""" + return _normalize_result(_read_json(path)) + + +# ------------------------------------------------------------------- KPI extraction + + +def _runtime_arrays(result: dict) -> dict: + """Return the ``Step Frametimes`` per-frame array map, or ``{}`` when absent.""" + runtime = result.get(METRIC_PHASE) + if not isinstance(runtime, dict): + return {} + frametimes = runtime.get(FRAMETIMES_NAME) + return frametimes if isinstance(frametimes, dict) else {} + + +def _floats(seq: object) -> list[float]: + """Coerce a sequence into a list of plain floats (dropping bools/non-numerics).""" + if not isinstance(seq, list): + return [] + return [float(s) for s in seq if isinstance(s, (int, float)) and not isinstance(s, bool)] + + +def steady_fps(result: dict, warmup_frames: int, max_frames: int | None = None) -> float: + """Mean post-warm-up effective FPS -- the gating KPI (D6). + + Computed as the mean of the per-frame ``Environment step effective FPS`` + array after dropping the first ``warmup_frames`` (and truncating to + ``max_frames`` so a longer run is comparable to the calibrated window). Falls + back to the scalar all-frames ``Mean Environment step effective FPS`` when the + per-frame array is unavailable (e.g. the OmniPerf backend). + + Raises: + CompareError: When neither the per-frame array nor the scalar metric + yields a finite positive value. + """ + arr = _floats(_runtime_arrays(result).get(EFF_FPS_ARRAY)) + if arr: + window = arr[:max_frames] if max_frames else arr + steady = window[warmup_frames:] if len(window) > warmup_frames else window + if steady: + value = sum(steady) / len(steady) + if value == value and value > 0: + return float(value) + # Fallback: scalar all-frames metric. + runtime = result.get(METRIC_PHASE) + if isinstance(runtime, dict) and isinstance(runtime.get(METRIC_NAME), (int, float)): + value = float(runtime[METRIC_NAME]) + if value == value and value > 0: + return value + raise CompareError(f"missing_metric phase={METRIC_PHASE} metric={METRIC_NAME!r}") + + +def _debug_kpis(result: dict, warmup_frames: int, max_frames: int | None = None) -> dict[str, object]: + """Advisory per-frame KPIs for triage (never change the verdict). + + Reports the steady step-time distribution plus outlier accounting -- count, + indices and magnitudes of steps slower than ``OUTLIER_FACTOR`` x the steady + median (D-debug-info) -- and GPU memory when present. + """ + arrays = _runtime_arrays(result) + steps = _floats(arrays.get(STEP_MS_ARRAY)) + if not steps: + return {} + window = steps[:max_frames] if max_frames else steps + if len(window) <= warmup_frames + 1: + return {} + steady = window[warmup_frames:] + ordered = sorted(steady) + n = len(ordered) + median = ordered[n // 2] if n % 2 else (ordered[n // 2 - 1] + ordered[n // 2]) / 2.0 + mean = sum(steady) / n + p99 = ordered[min(n - 1, int(round(0.99 * (n - 1))))] + # Outlier indices are reported in the post-warm-up frame coordinate. + outliers = [(i, s) for i, s in enumerate(steady) if median > 0 and s > OUTLIER_FACTOR * median] + out: dict[str, object] = { + "frames": len(window), + "steady_median_ms": f"{median:.3f}", + "steady_mean_ms": f"{mean:.3f}", + "p99_over_median": f"{(p99 / median):.2f}" if median > 0 else "—", + "outlier_count": len(outliers), + } + # Warm-up guardrail: the first *kept* frame should already look steady. If it is + # much slower than the steady median, warmup_frames is likely too small and + # one-time startup cost is leaking into the KPI. Advisory flag only -- the + # verdict is unchanged; it tells a maintainer to re-check warmup_frames. + if median > 0 and steady[0] > WARMUP_GUARD_FACTOR * median: + out["warmup_flag"] = f"first_kept_frame={steady[0] / median:.1f}x_median(warmup={warmup_frames})" + if outliers: + idx = [i for i, _ in outliers[:MAX_REPORTED_OUTLIERS]] + mag = [round(s / median, 2) for _, s in outliers[:MAX_REPORTED_OUTLIERS]] + out["outlier_idx"] = ",".join(str(i) for i in idx) + out["outlier_mag_x"] = ",".join(f"{m:g}" for m in mag) + mem = result.get(METRIC_PHASE, {}) + if isinstance(mem, dict) and isinstance(mem.get("GPU Memory Used"), (int, float)): + out["gpu_mem_gb"] = f"{float(mem['GPU Memory Used']):.2f}" + return out + + +def _benchmark_info(result: dict) -> dict: + """Return the run's self-reported config from the ``benchmark_info`` phase. + + The json backend records the ``task``/``seed``/``num_envs``/``num_frames`` the + run actually used, plus the comma-joined ``presets`` (physics + renderer). Empty + for the OmniPerf backend or older results. + """ + info = result.get("benchmark_info") + return info if isinstance(info, dict) else {} + + +def _expected_presets(task_entry: dict) -> list[str]: + """Physics/renderer preset tokens the gate launches this task with. + + Pulled from the ``physics=`` / ``presets=`` Hydra overrides in ``benchmark_args`` + (e.g. ``physics=newton_mjwarp`` -> ``newton_mjwarp``). The backend echoes these + back, comma-joined, in ``benchmark_info.presets``. + """ + out: list[str] = [] + for arg in task_entry.get("benchmark_args", []) or []: + if isinstance(arg, str) and "=" in arg: + key, _, val = arg.partition("=") + if key in ("physics", "presets") and val: + out.append(val) + return out + + +def _assert_run_config(result: dict, task: str, task_entry: dict) -> None: + """Verify the run used the configured task settings; raise on drift (D1/provenance). + + The gate launches each task with the config carried in ``baseline.json``, and the + backend echoes that config back in ``benchmark_info``. If the two disagree -- e.g. + a PR changes a task's default ``num_envs`` -- the measured FPS is no longer + comparable to the calibrated window, so a *config* change would be silently + misread as a *perf* change. We treat that as a structural failure + (``BLOCK/hard_failure``), not a regression. No-op when the backend reports no + ``benchmark_info`` (OmniPerf / legacy results). + """ + info = _benchmark_info(result) + if not info: + return + want_task = task_entry.get("task_id", task) # forward-compatible with task variants + mismatches: list[str] = [] + ran_task = info.get("task") + if isinstance(ran_task, str) and ran_task and ran_task != want_task: + mismatches.append(f"task(ran={ran_task},want={want_task})") + for field in ("num_envs", "seed"): + want = task_entry.get(field) + got = info.get(field) + if want is not None and isinstance(got, (int, float)) and int(got) != int(want): + mismatches.append(f"{field}(ran={int(got)},want={int(want)})") + # The run must cover at least the calibrated frame count (the KPI truncates to it). + want_frames = task_entry.get("num_frames") + got_frames = info.get("num_frames") + if want_frames is not None and isinstance(got_frames, (int, float)) and int(got_frames) < int(want_frames): + mismatches.append(f"num_frames(ran={int(got_frames)},want>={int(want_frames)})") + # Physics/renderer backend: every physics=/presets= override we launch with must + # appear in the run's reported presets. Catches "ran PhysX when the @newton variant + # was intended" -- a different KPI entirely, invisible to an FPS-only check. + want_presets = _expected_presets(task_entry) + if want_presets: + ran_presets = {p.strip() for p in str(info.get("presets", "")).split(",") if p.strip()} + if ran_presets: # only assert when the backend reported its presets + missing = [p for p in want_presets if p not in ran_presets] + if missing: + mismatches.append(f"presets(ran={','.join(sorted(ran_presets))},missing={','.join(missing)})") + if mismatches: + raise CompareError("config_mismatch " + " ".join(mismatches)) + + +def _extract_provenance(result: dict) -> dict[str, object]: + """Pull version provenance (warp / isaaclab / cuda); best-effort.""" + out: dict[str, object] = {} + version = result.get("version_info") + if isinstance(version, dict): + for src, dst in (("warp_version", "warp"), ("isaaclab_version", "isaaclab")): + val = version.get(src) + if isinstance(val, str) and val: + out[dst] = val + hw = result.get("hardware_info") + if isinstance(hw, dict): + cuda = hw.get("cuda_version") + if isinstance(cuda, str) and cuda: + out["cuda"] = cuda + return out + + +def _extract_gpu_name(result: dict) -> str | None: + """Read the runner's GPU model name from the result's hardware metadata.""" + hw = result.get("hardware_info") + if not isinstance(hw, dict): + return None + devices = hw.get("gpu_devices") + if not isinstance(devices, dict) or not devices: + return None + current = str(hw.get("gpu_current_device", "0")) + device = devices.get(current) or next(iter(devices.values()), None) + if isinstance(device, dict): + name = device.get("name") + return name if isinstance(name, str) and name else None + return None + + +# ------------------------------------------------------------------ robust stats + + +def _median(values: list[float]) -> float: + ordered = sorted(values) + n = len(ordered) + return ordered[n // 2] if n % 2 else (ordered[n // 2 - 1] + ordered[n // 2]) / 2.0 + + +def _median_mad(values: list[float]) -> tuple[float, float]: + """Return ``(median, MAD)`` of ``values`` (MAD = median absolute deviation).""" + center = _median(values) + mad = _median([abs(v - center) for v in values]) + return center, mad + + +# ----------------------------------------------------------------- baseline / store + + +def _resolve_results(results_dir: str, glob_pattern: str, allow_multiple: bool) -> Path: + """Resolve the result JSON path within ``results_dir`` (latest by sort order).""" + matches = sorted(glob.glob(os.path.join(results_dir, glob_pattern))) + if not matches: + raise CompareError(f"no_results_found dir={results_dir!r} glob={glob_pattern!r}") + if len(matches) > 1 and not allow_multiple: + raise CompareError(f"multiple_results n={len(matches)}") + return Path(matches[-1]) + + +def _match_gpu(per_gpu: dict, gpu_key: str) -> tuple[str, dict]: + """Find the baseline entry whose key (sub)string-matches ``gpu_key``.""" + for key, entry in per_gpu.items(): + if key == gpu_key or key in gpu_key or gpu_key in key: + if not isinstance(entry, dict): + raise CompareError(f"malformed_baseline_entry gpu={key!r}") + return key, entry + raise CompareError(f"baseline_gpu_mismatch gpu={gpu_key!r} known={sorted(per_gpu)}") + + +def _resolve_baseline( + baseline: dict, task: str, gpu_name: str | None, gpu_override: str | None +) -> tuple[str, dict, dict]: + """Return ``(gpu_key, task_entry, per_gpu_entry)`` for the task and GPU.""" + task_entry = baseline.get(task) + if not isinstance(task_entry, dict): + raise CompareError(f"missing_baseline_task task={task!r}") + per_gpu = task_entry.get("per_gpu") + if not isinstance(per_gpu, dict) or not per_gpu: + raise CompareError(f"missing_per_gpu task={task!r}") + gpu_key = gpu_override or gpu_name + if not gpu_key: + raise CompareError(f"unknown_gpu task={task!r}") + matched_key, entry = _match_gpu(per_gpu, gpu_key) + if "baseline_fps" not in entry: + raise CompareError(f"missing_baseline_field task={task!r} gpu={matched_key!r} field=baseline_fps") + return matched_key, task_entry, entry + + +def _history_window(history_dir: str | None, fingerprint: str | None, task: str, gpu_key: str) -> dict: + """Load the rolling-window samples for ``(task, gpu)`` from the history store. + + The store mirrors the orphan branch: ``//__.json`` + with a flat ``/__.json`` fallback (the seeded + "default" bucket). Returns ``{}`` when no window exists yet. + """ + if not history_dir: + return {} + safe = f"{task}__{gpu_key}".replace("/", "_").replace(" ", "_") + candidates = [] + if fingerprint: + candidates.append(Path(history_dir) / fingerprint / f"{safe}.json") + candidates.append(Path(history_dir) / f"{safe}.json") + for path in candidates: + if path.exists(): + data = _read_json(path) + if isinstance(data, dict): + return data + return {} + + +def _safe_float(val: object) -> float | None: + """Parse a float, returning ``None`` for missing / non-numeric values (e.g. ``"—"``).""" + try: + return float(val) # type: ignore[arg-type] + except (TypeError, ValueError): + return None + + +def _window_values(window: dict, key: str) -> list[float]: + """Pull a numeric series (``fps`` / ``wall_s``) out of a window's samples.""" + samples = window.get("samples") + if not isinstance(samples, list): + return [] + return [float(s[key]) for s in samples if isinstance(s, dict) and isinstance(s.get(key), (int, float))] + + +def _overrides_for(overrides: dict, task: str, gpu_key: str) -> dict: + """Merge global defaults < per-task < per-task/gpu override blocks.""" + merged: dict[str, object] = {} + defaults = overrides.get("_defaults") + if isinstance(defaults, dict): + merged.update(defaults) + task_block = overrides.get(task) + if isinstance(task_block, dict): + merged.update({k: v for k, v in task_block.items() if not isinstance(v, dict)}) + gpu_block = task_block.get(gpu_key) + if isinstance(gpu_block, dict): + merged.update(gpu_block) + return merged + + +def _thresholds(window: dict, entry: dict, ov: dict) -> tuple[float, float, float, float, str]: + """Compute ``(center, spread, k_warn, k_block, source)`` for the FPS KPI. + + Uses the rolling-window median+MAD when enough samples exist; otherwise + falls back to the static ``baseline_fps`` plus ``warn_pct`` / ``max_regression_pct``. + Manual overrides (k values, spread floor, pinned center/spread) win. + """ + k_warn = float(ov.get("k_warn", DEFAULT_K_WARN)) + k_block = float(ov.get("k_block", DEFAULT_K_BLOCK)) + min_spread_pct = float(ov.get("min_spread_pct", DEFAULT_MIN_SPREAD_PCT)) + + fps_window = _window_values(window, "fps") + if len(fps_window) >= MIN_WINDOW: + center, mad = _median_mad(fps_window) + spread = max(MAD_TO_STD * mad, min_spread_pct / 100.0 * center) + source = f"window(n={len(fps_window)})" + else: + center = float(entry["baseline_fps"]) + # Map the static percent bands onto k-sigma so PASS/WARN/BLOCK math is uniform. + warn_pct = float(entry.get("warn_pct", min_spread_pct * k_warn)) + block_pct = float(entry.get("max_regression_pct", min_spread_pct * k_block)) + spread = warn_pct / 100.0 * center / k_warn if k_warn else min_spread_pct / 100.0 * center + # Honor the static block band exactly if it implies a different spread. + spread = max(spread, block_pct / 100.0 * center / k_block if k_block else spread) + source = "static_baseline" + + if "pin_center_fps" in ov: + center = float(ov["pin_center_fps"]) + source = "override_pin" + if "pin_spread_fps" in ov: + spread = float(ov["pin_spread_fps"]) + return center, spread, k_warn, k_block, source + + +def main(argv: list[str] | None = None) -> int: + """CLI entry point. Returns the process exit code.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else None) + parser.add_argument( + "--task", + required=True, + help="Gate key for baseline/history lookup, e.g. Isaac-Cartpole-v0 or Isaac-Cartpole-v0@newton.", + ) + parser.add_argument( + "--task-id", + default=None, + help="Gym task id used for the result glob and config check (defaults to --task; set for @variant keys).", + ) + parser.add_argument("--results-dir", required=True, help="Directory containing the benchmark JSON.") + parser.add_argument("--baseline", required=True, help="Path to baseline.json (run config + static fallback).") + parser.add_argument("--history-dir", default=None, help="Rolling-window store (orphan-branch checkout).") + parser.add_argument("--overrides", default=None, help="Path to baseline_overrides.json (committed with the PR).") + parser.add_argument("--fingerprint", default=None, help="History bucket key (git-subtree+deps hash).") + parser.add_argument("--measured-wall-s", type=float, default=None, help="Wall-clock seconds of the run.") + parser.add_argument("--results-glob", default=None, help=f"Result glob (defaults to {DEFAULT_GLOB_TEMPLATE!r}).") + parser.add_argument("--gpu-override", default=None, help="Override the GPU name read from the result JSON.") + parser.add_argument("--allow-multiple", action="store_true", help="Permit multiple result files; pick the latest.") + args = parser.parse_args(argv) + + # The gate key (--task) may be a variant like "Isaac-Cartpole-v0@newton"; the gym + # task id (--task-id, defaulting to the part before "@") drives the file glob and + # the config check, while the gate key drives baseline / history / overrides lookup. + task_id = args.task_id or args.task.split("@", 1)[0] + glob_pattern = args.results_glob or DEFAULT_GLOB_TEMPLATE.format(task=task_id) + + try: + result_path = _resolve_results(args.results_dir, glob_pattern, args.allow_multiple) + result = _load_result(result_path) + baseline = _load_json(Path(args.baseline)) + gpu_name = _extract_gpu_name(result) + gpu_key, task_entry, entry = _resolve_baseline(baseline, args.task, gpu_name, args.gpu_override) + _assert_run_config(result, task_id, task_entry) + warmup_frames = int(task_entry.get("warmup_frames", DEFAULT_WARMUP_FRAMES)) + max_frames = task_entry.get("num_frames") + max_frames = int(max_frames) if isinstance(max_frames, (int, float)) else None + measured_fps = steady_fps(result, warmup_frames, max_frames) + except CompareError as e: + _emit("BLOCK", kind="hard_failure", reason=str(e), task=args.task) + return EXIT_HARD_FAILURE + + overrides = {} + if args.overrides and Path(args.overrides).exists(): + overrides = _load_json(Path(args.overrides)) + ov = _overrides_for(overrides, args.task, gpu_key) + + if ov.get("skip"): + _emit("PASS", task=args.task, gpu=gpu_key, note="skipped_by_override") + return EXIT_PASS + + window = _history_window(args.history_dir, args.fingerprint, args.task, gpu_key) + center, spread, k_warn, k_block, source = _thresholds(window, entry, ov) + delta_pct = (measured_fps - center) / center * 100.0 + warn_floor = center - k_warn * spread + block_floor = center - k_block * spread + + common: dict[str, object] = { + "task": args.task, + "gpu": gpu_key, + "thresholds": source, + "center_fps": f"{center:.0f}", + "measured_fps": f"{measured_fps:.0f}", + "delta_pct": f"{delta_pct:+.2f}", + "warmup_frames": warmup_frames, + "k_warn": f"{k_warn:g}", + "k_block": f"{k_block:g}", + "warn_below_fps": f"{warn_floor:.0f}", + "block_below_fps": f"{block_floor:.0f}", + } + + # Secondary signal (advisory): wall-clock vs the window's wall median. + if args.measured_wall_s is not None: + common["wall_s"] = f"{args.measured_wall_s:.0f}" + wall_window = _window_values(window, "wall_s") + if len(wall_window) >= MIN_WINDOW: + wcenter, wmad = _median_mad(wall_window) + wspread = max(MAD_TO_STD * wmad, DEFAULT_MIN_SPREAD_PCT / 100.0 * wcenter) + common["wall_center_s"] = f"{wcenter:.0f}" + common["wall_delta_pct"] = f"{(args.measured_wall_s - wcenter) / wcenter * 100.0:+.2f}" + if args.measured_wall_s > wcenter + k_warn * wspread: + common["wall_flag"] = "slow" + + common.update(_debug_kpis(result, warmup_frames, max_frames)) + common.update(_extract_provenance(result)) + + # Advisory tail signal (opt-in): a per-task override can WARN when the post-warm-up + # p99/median step-time ratio exceeds a ceiling -- a tail/spike regression the scalar + # FPS mean hides. Off unless tail_p99_warn is set (tasks with real recurring spikes, + # e.g. g1-flat first-ground-contact, would otherwise flag every run). Never blocks. + tail_warn = False + tail_ceiling = ov.get("tail_p99_warn") + if tail_ceiling is not None: + p99_ratio = _safe_float(common.get("p99_over_median")) + if p99_ratio is not None and p99_ratio > float(tail_ceiling): + common["tail_flag"] = f"p99_over_median={p99_ratio:g}>{float(tail_ceiling):g}" + tail_warn = True + + if measured_fps < block_floor: + _emit("BLOCK", kind="regression", **common) + return EXIT_BLOCK + if measured_fps < warn_floor: + _emit("WARN", **common) + return EXIT_PASS + if tail_warn: + _emit("WARN", reason="tail", **common) + return EXIT_PASS + _emit("PASS", **common) + return EXIT_PASS + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_smoke/demo_regression.py b/tools/perf_smoke/demo_regression.py new file mode 100644 index 000000000000..f3a8d8e44516 --- /dev/null +++ b/tools/perf_smoke/demo_regression.py @@ -0,0 +1,153 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Deterministically demonstrate the gate's PASS vs BLOCK discrimination. + +Waiting for a real regression to land is a poor demo. Instead this takes a *real* +benchmark result for a task and runs the *real* comparator CLI twice: + +1. the unmodified result -> expected **PASS**; +2. a copy whose FPS has been scaled down by ``--factor`` (a synthetic but + realistically-shaped slowdown) -> expected **REGRESSION** once the drop + exceeds the task's block threshold. + +Only the result is synthetic; the comparator, baseline, thresholds, and exit +codes are exactly what CI uses, so this exercises the production decision path. + +Usage (uses a result already produced by the gate; no GPU needed):: + + python3 tools/perf_smoke/demo_regression.py \\ + --task Isaac-Cartpole-v0 --results-dir perf-output/Isaac-Cartpole-v0 +""" + +from __future__ import annotations + +import argparse +import copy +import json +import subprocess +import sys +from pathlib import Path + +_THIS_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(_THIS_DIR)) + +import check_perf_regression as cpr # noqa: E402 + +_FPS_METRIC = f"benchmark_non_rl runtime {cpr.METRIC_NAME}" + + +def _scale_result(data: object, factor: float) -> object: + """Return a copy of a result with the FPS metric (and per-frame arrays) scaled. + + Handles both result shapes. Scaling <1 makes the run look slower. + """ + data = copy.deepcopy(data) + if isinstance(data, dict): # OmniPerf shape + runtime = data.get("runtime") + if isinstance(runtime, dict) and cpr.METRIC_NAME in runtime: + runtime[cpr.METRIC_NAME] = float(runtime[cpr.METRIC_NAME]) * factor + return data + if isinstance(data, list): # json-backend shape + for phase in data: + if not isinstance(phase, dict) or phase.get("phase_name") != "runtime": + continue + for m in phase.get("measurements", []) or []: + if not isinstance(m, dict): + continue + if m.get("name") == _FPS_METRIC and isinstance(m.get("value"), (int, float)): + m["value"] = float(m["value"]) * factor + # Scale the per-frame arrays: effective FPS (the gating metric reads + # this) down, and step times up, so the slowdown is self-consistent. + if m.get("name", "").endswith(cpr.FRAMETIMES_NAME) and isinstance(m.get("value"), dict): + eff = m["value"].get(cpr.EFF_FPS_ARRAY) + if isinstance(eff, list): + m["value"][cpr.EFF_FPS_ARRAY] = [ + (v * factor if isinstance(v, (int, float)) else v) for v in eff + ] + steps = m["value"].get(cpr.STEP_MS_ARRAY) + if isinstance(steps, list): + m["value"][cpr.STEP_MS_ARRAY] = [ + (s / factor if isinstance(s, (int, float)) else s) for s in steps + ] + return data + + +def _run_comparator( + task: str, + results_dir: Path, + baseline: str, + gpu_override: str | None, + history_dir: str | None, + overrides: str | None, +) -> int: + cmd = [ + sys.executable, + str(_THIS_DIR / "check_perf_regression.py"), + "--task", + task, + "--results-dir", + str(results_dir), + "--baseline", + baseline, + "--allow-multiple", + ] + if gpu_override: + cmd += ["--gpu-override", gpu_override] + if history_dir: + cmd += ["--history-dir", history_dir] + if overrides: + cmd += ["--overrides", overrides] + return subprocess.run(cmd).returncode + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else None) + parser.add_argument("--task", required=True, help="Task name (must have a real result + baseline entry).") + parser.add_argument("--results-dir", required=True, help="Directory with a real benchmark result for the task.") + parser.add_argument("--baseline", default=str(_THIS_DIR / "baseline.json"), help="Path to baseline.json.") + parser.add_argument("--history-dir", default=str(_THIS_DIR / "perf_history"), help="Rolling-window store.") + parser.add_argument("--overrides", default=str(_THIS_DIR / "baseline_overrides.json"), help="Overrides file.") + parser.add_argument("--factor", type=float, default=0.7, help="FPS scale for the slowed copy (0.7 = 30%% slower).") + parser.add_argument("--gpu-override", default=None, help="Force the baseline GPU key.") + args = parser.parse_args(argv) + + pattern = cpr.DEFAULT_GLOB_TEMPLATE.format(task=args.task) + try: + real_path = cpr._resolve_results(args.results_dir, pattern, allow_multiple=True) + except cpr.CompareError as e: + print(f"[demo] no real result found: {e}") + print( + f"[demo] produce one first, e.g.:\n ./isaaclab.sh -p tools/perf_smoke/run_perf_gate.py --tasks {args.task}" + ) + return 2 + raw = cpr._read_json(real_path) + + print("\n[demo] === 1) real result -> expect PASS ===") + rc_pass = _run_comparator( + args.task, Path(args.results_dir), args.baseline, args.gpu_override, args.history_dir, args.overrides + ) + + slowed_dir = Path(args.results_dir).parent / f"{args.task}__demo_slowed" + slowed_dir.mkdir(parents=True, exist_ok=True) + slowed_path = slowed_dir / real_path.name + slowed_path.write_text(json.dumps(_scale_result(raw, args.factor)), encoding="utf-8") + + pct = (1.0 - args.factor) * 100.0 + print(f"\n[demo] === 2) same result, {pct:.0f}% slower -> expect BLOCK ===") + rc_block = _run_comparator( + args.task, slowed_dir, args.baseline, args.gpu_override, args.history_dir, args.overrides + ) + + print("\n[demo] === SUMMARY ===") + print(f"[demo] real -> exit {rc_pass} ({cpr.EXIT_PASS}=PASS)") + print(f"[demo] slowed -> exit {rc_block} ({cpr.EXIT_BLOCK}=BLOCK)") + ok = rc_pass == cpr.EXIT_PASS and rc_block == cpr.EXIT_BLOCK + print(f"[demo] discrimination {'OK' if ok else 'UNEXPECTED'}") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_smoke/perf_history/Isaac-Cartpole-v0@newton__NVIDIA_L40S.json b/tools/perf_smoke/perf_history/Isaac-Cartpole-v0@newton__NVIDIA_L40S.json new file mode 100644 index 000000000000..38a1d8e2fc7a --- /dev/null +++ b/tools/perf_smoke/perf_history/Isaac-Cartpole-v0@newton__NVIDIA_L40S.json @@ -0,0 +1,40 @@ +{ + "task": "Isaac-Cartpole-v0@newton", + "gpu": "NVIDIA L40S", + "num_frames": 300, + "warmup_frames": 5, + "window": 20, + "_note": "Seeded from L40S warm calibration runs (500f truncated to 300f, post-warm-up).", + "samples": [ + { + "fps": 376554.2, + "wall_s": 35.06, + "source": "cartpole_newton_n4096/warm_round1", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 356547.6, + "wall_s": 34.11, + "source": "cartpole_newton_n4096/warm_round2", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 348334.8, + "wall_s": 34.45, + "source": "cartpole_newton_n4096/warm_round3", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 367661.8, + "wall_s": 34.6, + "source": "cartpole_newton_n4096/warm_round4", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 358461.3, + "wall_s": 35.91, + "source": "cartpole_newton_n4096/warm_round5", + "ts": "2026-06-10T00:49:53+00:00" + } + ] +} diff --git a/tools/perf_smoke/perf_history/Isaac-Cartpole-v0__NVIDIA_L40S.json b/tools/perf_smoke/perf_history/Isaac-Cartpole-v0__NVIDIA_L40S.json new file mode 100644 index 000000000000..a086a40a5776 --- /dev/null +++ b/tools/perf_smoke/perf_history/Isaac-Cartpole-v0__NVIDIA_L40S.json @@ -0,0 +1,40 @@ +{ + "task": "Isaac-Cartpole-v0", + "gpu": "NVIDIA L40S", + "num_frames": 300, + "warmup_frames": 2, + "window": 20, + "_note": "Seeded from L40S warm calibration runs (500f truncated to 300f, post-warm-up).", + "samples": [ + { + "fps": 115333.6, + "wall_s": 37.23, + "source": "cartpole_physx_n4096/warm_round1", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 115099.7, + "wall_s": 36.72, + "source": "cartpole_physx_n4096/warm_round2", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 114419.0, + "wall_s": 36.22, + "source": "cartpole_physx_n4096/warm_round3", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 114538.0, + "wall_s": 37.07, + "source": "cartpole_physx_n4096/warm_round4", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 118560.6, + "wall_s": 37.07, + "source": "cartpole_physx_n4096/warm_round5", + "ts": "2026-06-10T00:49:53+00:00" + } + ] +} diff --git a/tools/perf_smoke/perf_history/Isaac-Factory-GearMesh-Direct-v0__NVIDIA_L40S.json b/tools/perf_smoke/perf_history/Isaac-Factory-GearMesh-Direct-v0__NVIDIA_L40S.json new file mode 100644 index 000000000000..83a04f8373da --- /dev/null +++ b/tools/perf_smoke/perf_history/Isaac-Factory-GearMesh-Direct-v0__NVIDIA_L40S.json @@ -0,0 +1,40 @@ +{ + "task": "Isaac-Factory-GearMesh-Direct-v0", + "gpu": "NVIDIA L40S", + "num_frames": 300, + "warmup_frames": 2, + "window": 20, + "_note": "Seeded from L40S warm calibration runs (500f truncated to 300f, post-warm-up).", + "samples": [ + { + "fps": 872.1, + "wall_s": 319.13, + "source": "factory_physx_n512/warm_round1", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 890.0, + "wall_s": 314.54, + "source": "factory_physx_n512/warm_round2", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 869.8, + "wall_s": 322.08, + "source": "factory_physx_n512/warm_round3", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 890.7, + "wall_s": 314.0, + "source": "factory_physx_n512/warm_round4", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 890.3, + "wall_s": 313.48, + "source": "factory_physx_n512/warm_round5", + "ts": "2026-06-10T00:49:53+00:00" + } + ] +} diff --git a/tools/perf_smoke/perf_history/Isaac-Repose-Cube-Shadow-Vision-Direct-v0__NVIDIA_L40S.json b/tools/perf_smoke/perf_history/Isaac-Repose-Cube-Shadow-Vision-Direct-v0__NVIDIA_L40S.json new file mode 100644 index 000000000000..613d680fc4ad --- /dev/null +++ b/tools/perf_smoke/perf_history/Isaac-Repose-Cube-Shadow-Vision-Direct-v0__NVIDIA_L40S.json @@ -0,0 +1,40 @@ +{ + "task": "Isaac-Repose-Cube-Shadow-Vision-Direct-v0", + "gpu": "NVIDIA L40S", + "num_frames": 300, + "warmup_frames": 60, + "window": 20, + "_note": "Seeded from L40S warm calibration runs (500f truncated to 300f, post-warm-up).", + "samples": [ + { + "fps": 1028.6, + "wall_s": 94.58, + "source": "shadow_vision_physx_rtx_64x64_n128/warm_round1", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 1029.1, + "wall_s": 93.83, + "source": "shadow_vision_physx_rtx_64x64_n128/warm_round2", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 997.7, + "wall_s": 97.71, + "source": "shadow_vision_physx_rtx_64x64_n128/warm_round3", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 1038.2, + "wall_s": 92.67, + "source": "shadow_vision_physx_rtx_64x64_n128/warm_round4", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 1035.6, + "wall_s": 91.9, + "source": "shadow_vision_physx_rtx_64x64_n128/warm_round5", + "ts": "2026-06-10T00:49:53+00:00" + } + ] +} diff --git a/tools/perf_smoke/perf_history/Isaac-Velocity-Flat-G1-v0@newton__NVIDIA_L40S.json b/tools/perf_smoke/perf_history/Isaac-Velocity-Flat-G1-v0@newton__NVIDIA_L40S.json new file mode 100644 index 000000000000..4c967e5e639f --- /dev/null +++ b/tools/perf_smoke/perf_history/Isaac-Velocity-Flat-G1-v0@newton__NVIDIA_L40S.json @@ -0,0 +1,40 @@ +{ + "task": "Isaac-Velocity-Flat-G1-v0@newton", + "gpu": "NVIDIA L40S", + "num_frames": 300, + "warmup_frames": 5, + "window": 20, + "_note": "Seeded from L40S warm calibration runs (500f truncated to 300f, post-warm-up).", + "samples": [ + { + "fps": 68934.7, + "wall_s": 65.21, + "source": "g1_flat_newton_n2048/warm_round1", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 70992.0, + "wall_s": 63.75, + "source": "g1_flat_newton_n2048/warm_round2", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 69660.2, + "wall_s": 65.56, + "source": "g1_flat_newton_n2048/warm_round3", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 68564.6, + "wall_s": 65.37, + "source": "g1_flat_newton_n2048/warm_round4", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 71300.9, + "wall_s": 65.17, + "source": "g1_flat_newton_n2048/warm_round5", + "ts": "2026-06-10T00:49:53+00:00" + } + ] +} diff --git a/tools/perf_smoke/perf_history/Isaac-Velocity-Flat-G1-v0__NVIDIA_L40S.json b/tools/perf_smoke/perf_history/Isaac-Velocity-Flat-G1-v0__NVIDIA_L40S.json new file mode 100644 index 000000000000..59a3ad3308af --- /dev/null +++ b/tools/perf_smoke/perf_history/Isaac-Velocity-Flat-G1-v0__NVIDIA_L40S.json @@ -0,0 +1,40 @@ +{ + "task": "Isaac-Velocity-Flat-G1-v0", + "gpu": "NVIDIA L40S", + "num_frames": 300, + "warmup_frames": 2, + "window": 20, + "_note": "Seeded from L40S warm calibration runs (500f truncated to 300f, post-warm-up).", + "samples": [ + { + "fps": 19078.1, + "wall_s": 99.14, + "source": "g1_flat_physx_n2048/warm_round1", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 19263.8, + "wall_s": 94.34, + "source": "g1_flat_physx_n2048/warm_round2", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 19114.7, + "wall_s": 93.81, + "source": "g1_flat_physx_n2048/warm_round3", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 19356.8, + "wall_s": 94.68, + "source": "g1_flat_physx_n2048/warm_round4", + "ts": "2026-06-10T00:49:53+00:00" + }, + { + "fps": 19481.9, + "wall_s": 93.37, + "source": "g1_flat_physx_n2048/warm_round5", + "ts": "2026-06-10T00:49:53+00:00" + } + ] +} diff --git a/tools/perf_smoke/pytest.ini b/tools/perf_smoke/pytest.ini new file mode 100644 index 000000000000..5108449fa4a2 --- /dev/null +++ b/tools/perf_smoke/pytest.ini @@ -0,0 +1,11 @@ +; Local pytest config for the perf-smoke gate. +; +; Its mere presence pins the rootdir to tools/perf_smoke/, which keeps the parent +; tools/conftest.py harness (it hijacks every pytest session under tools/) from +; loading. That lets the gate run as an ordinary pytest module that shells out +; each Isaac Sim run as its own subprocess (D1), exactly the model the doc asks +; for, without disturbing the repo-wide test harness. +[pytest] +testpaths = . +python_files = test_*.py +addopts = -ra diff --git a/tools/perf_smoke/rebaseline.py b/tools/perf_smoke/rebaseline.py new file mode 100644 index 000000000000..2e5bb62fe905 --- /dev/null +++ b/tools/perf_smoke/rebaseline.py @@ -0,0 +1,365 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Measure tasks N times and (optionally) refresh the rolling in-tree baseline. + +One measurement path serves two jobs: + +* **Variance study (report mode, default):** run each gate task ``--repeat`` times, + report robust per-task stats (median / CV / MAD / min / max), and show how the + spread compares to the configured thresholds. Use this to justify thresholds to + reviewers and to confirm baselines transfer to the current environment. +* **Rolling re-baseline (``--apply``):** turn that same window of recent runs into + updated ``baseline.json`` values (``baseline_fps`` = window median; warn/block + from window CV). Values stay **in-tree** and the CI workflow opens a *PR* with + the diff -- the change is always reviewed. + +Boiling-frog guard +------------------ +A rolling baseline must not quietly absorb a real regression. Any task whose new +median would drop the baseline by more than ``--soft-drop-pct`` is **flagged for +review** (applied, but loudly marked in the report / PR body). A drop beyond +``--hard-drop-pct`` is **refused** (old value kept) unless ``--force`` -- a drop +that large is almost certainly a regression, not drift. + +Reuses :mod:`run_perf_gate` for command building and :mod:`check_perf_regression` +for FPS extraction, so a measurement here is identical to a gate measurement. + +Examples:: + + # variance study: 5 reps of all baseline tasks, report only + ./isaaclab.sh -p tools/perf_smoke/rebaseline.py --repeat 5 + + # rolling re-baseline: write proposed values (CI then opens a PR) + ./isaaclab.sh -p tools/perf_smoke/rebaseline.py --repeat 5 --apply +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +import statistics +import sys +from pathlib import Path + +_THIS_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(_THIS_DIR)) + +import check_perf_regression as cpr # noqa: E402 +import run_perf_gate as gate # noqa: E402 + + +def _baseline_tasks(baseline: dict) -> list[str]: + """Return the non-metadata task keys from a baseline document, in order.""" + return [k for k in baseline if not k.startswith("_")] + + +def _measure_fps(task_id: str, run_dir: Path, warmup: int, num_frames: int | None) -> float | None: + """Extract the post-warm-up steady FPS (the gating metric, D6) from a run dir. + + ``task_id`` is the gym task the benchmark was launched with (the result-file name), + which for a variant key like ``Isaac-Cartpole-v0@newton`` is just ``Isaac-Cartpole-v0``. + """ + pattern = cpr.DEFAULT_GLOB_TEMPLATE.format(task=task_id) + try: + result_path = cpr._resolve_results(str(run_dir), pattern, allow_multiple=True) + return cpr.steady_fps(cpr._load_result(result_path), warmup, num_frames) + except cpr.CompareError: + return None + + +def _window_stats(samples: list[float]) -> dict | None: + """Robust summary of a measurement window. ``None`` when there are no samples.""" + if not samples: + return None + n = len(samples) + median = statistics.median(samples) + mean = statistics.fmean(samples) + # Sample CV (needs >= 2 points); single-sample windows report 0 spread. + cv_pct = (statistics.stdev(samples) / mean * 100.0) if n >= 2 and mean > 0 else 0.0 + mad = statistics.median([abs(s - median) for s in samples]) + return { + "n": n, + "median": median, + "mean": mean, + "cv_pct": cv_pct, + "mad": mad, + "min": min(samples), + "max": max(samples), + "samples": samples, + } + + +def _proposed_entry(stats: dict) -> dict: + """Map window stats to baseline fields (median anchor; CV-derived thresholds).""" + cv = stats["cv_pct"] + return { + "baseline_fps": round(stats["median"], 1), + "warn_pct": round(max(3.0 * cv, 5.0), 1), + "max_regression_pct": round(max(6.0 * cv, 10.0), 1), + "cv_pct": round(cv, 2), + "n_runs": stats["n"], + } + + +def measure_task( + task: str, + baseline: dict, + out_root: Path, + repeat: int, + cache_dir: str | None, + seed_override: int | None = None, + tag: str = "", +) -> dict | None: + """Run ``task`` ``repeat`` times and return its window stats (or ``None``). + + The returned stats also carry the per-run ``wall`` samples so ``--apply`` can + append both FPS and wall-clock to the rolling-window store. + + ``seed_override`` replaces the baseline seed (used by the seed-sweep study to vary + the random scene); ``tag`` namespaces the run dirs so concurrent seeds/reps don't + collide. + """ + try: + cfg = gate._task_run_config(baseline, task) + except KeyError as e: + print(f"[rebaseline] {task}: {e} -> skipped", flush=True) + return None + if seed_override is not None: + cfg["seed"] = seed_override + entry = baseline.get(task, {}) + task_id = cfg.get("task_id", task) # gym id used for the result-file glob (vs the @variant gate key) + warmup = int(entry.get("warmup_frames", cpr.DEFAULT_WARMUP_FRAMES)) + num_frames = entry.get("num_frames") + num_frames = int(num_frames) if isinstance(num_frames, (int, float)) else None + samples: list[float] = [] + walls: list[float] = [] + for rep in range(1, repeat + 1): + run_dir = out_root / task / f"{tag}rep{rep}" + print(f"\n[rebaseline] === {task} {tag}rep {rep}/{repeat} (seed={cfg['seed']}) ===", flush=True) + wall = gate._run_benchmark(task, cfg, run_dir, retries=1, dry_run=False, cache_dir=cache_dir) + if wall is None: + print(f"[rebaseline] {task} rep {rep}: run failed -> dropped from window", flush=True) + continue + fps = _measure_fps(task_id, run_dir, warmup, num_frames) + if fps is None: + print(f"[rebaseline] {task} rep {rep}: could not read FPS -> dropped", flush=True) + continue + print(f"[rebaseline] {task} rep {rep}: {fps:.0f} FPS ({wall:.0f}s)", flush=True) + samples.append(fps) + walls.append(wall) + stats = _window_stats(samples) + if stats is not None: + stats["walls"] = walls + return stats + + +def _append_window(history_dir: Path, task: str, gpu_key: str, stats: dict, cap: int = 20) -> int: + """Append this study's samples to the rolling-window store; prune to ``cap``. + + This is the orphan-branch update in the doc's model: each study contributes + its runs to ``/__.json``, oldest dropped past ``cap``. + Returns the resulting window length. + """ + history_dir.mkdir(parents=True, exist_ok=True) + path = history_dir / f"{task}__{gpu_key}.json".replace(" ", "_") + store = {"task": task, "gpu": gpu_key, "window": cap, "samples": []} + if path.exists(): + existing = json.loads(path.read_text(encoding="utf-8")) + if isinstance(existing, dict) and isinstance(existing.get("samples"), list): + store = existing + now = dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds") + walls = stats.get("walls") or [None] * len(stats["samples"]) + for fps, wall in zip(stats["samples"], walls): + store["samples"].append({"fps": round(fps, 1), "wall_s": wall, "ts": now, "source": "rebaseline"}) + store["samples"] = store["samples"][-cap:] + store["window"] = cap + path.write_text(json.dumps(store, indent=2) + "\n", encoding="utf-8") + return len(store["samples"]) + + +def _current_baseline_fps(baseline: dict, task: str, gpu_key: str) -> float | None: + entry = baseline.get(task, {}).get("per_gpu", {}).get(gpu_key) + if isinstance(entry, dict) and "baseline_fps" in entry: + return float(entry["baseline_fps"]) + return None + + +def _emit_summary(line: str) -> None: + """Print and append to the CI step summary when available.""" + print(line) + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + try: + with open(summary_path, "a", encoding="utf-8") as f: + f.write(line + "\n") + except OSError: + pass + + +def _seed_study(args: argparse.Namespace, baseline: dict, out_root: Path, tasks: list[str]) -> int: + """Report-only seed-sensitivity study (never touches the window or baseline). + + For each task it compares two coefficients of variation: + + * **same-seed CV** -- ``--repeat`` reps at the baseline (gate) seed: the run-to-run + noise the gate already lives with; + * **cross-seed CV** -- one run per ``--seeds`` value: how much the random scene alone + moves FPS. + + If cross-seed CV is in the same ballpark as same-seed CV, FPS is seed-insensitive and + hard-coding the gate seed costs nothing. If it is much larger, the task's band should + widen (or the gate should average a few seeds). + """ + _emit_summary("\n## Perf seed-sensitivity study\n") + _emit_summary( + f"GPU key `{args.gpu_key}`; same-seed = {args.repeat} reps @ baseline seed, cross-seed = {args.seeds}.\n" + ) + _emit_summary( + "| Task | gate seed | same-seed n / median / CV% | cross-seed n / median / CV% | cross/same | verdict |" + ) + _emit_summary("|---|---:|---|---|---:|---|") + for task in tasks: + gate_seed = gate._task_run_config(baseline, task).get("seed", 42) + same = measure_task(task, baseline, out_root, args.repeat, args.cache_dir, tag="sameseed_") + cross_fps: list[float] = [] + for seed in args.seeds: + s = measure_task(task, baseline, out_root, 1, args.cache_dir, seed_override=seed, tag=f"seed{seed}_") + if s is not None: + cross_fps.append(s["median"]) + cross = _window_stats(cross_fps) + if same is None or cross is None: + _emit_summary( + f"| {task} | {gate_seed} | {'—' if same is None else 'ok'} | measurement failed | — | ⚠️skip |" + ) + continue + ratio = cross["cv_pct"] / same["cv_pct"] if same["cv_pct"] > 0 else float("inf") + # Seed is "safe to fix" when the scene adds no more spread than ordinary run noise + # (allow a small absolute floor so near-zero same-seed CVs don't explode the ratio). + safe = cross["cv_pct"] <= max(same["cv_pct"] * 1.5, same["cv_pct"] + 0.3) + verdict = "✅ seed-insensitive" if safe else "⚠️ widen band / avg seeds" + _emit_summary( + f"| {task} | {gate_seed} | {same['n']} / {same['median']:.0f} / {same['cv_pct']:.2f} | " + f"{cross['n']} / {cross['median']:.0f} / {cross['cv_pct']:.2f} | {ratio:.2f}x | {verdict} |" + ) + _emit_summary("\n_Report only: the seed sweep is never appended to the window or baseline._") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else None) + parser.add_argument("--tasks", nargs="+", default=None, help="Tasks to measure (default: all baseline tasks).") + parser.add_argument("--baseline", default=str(_THIS_DIR / "baseline.json"), help="Path to baseline.json.") + parser.add_argument("--repeat", type=int, default=5, help="Runs per task in the window (default 5).") + parser.add_argument( + "--output-dir", default=str(gate._REPO_ROOT / "perf-output-rebaseline"), help="Run output root." + ) + parser.add_argument("--cache-dir", default=None, help="Optional warm JIT-cache dir (see run_perf_gate.py).") + parser.add_argument("--gpu-key", default="NVIDIA L40S", help="Baseline per-GPU key to read/update.") + parser.add_argument( + "--history-dir", default=str(_THIS_DIR / "perf_history"), help="Rolling-window store to append to on --apply." + ) + parser.add_argument("--window-cap", type=int, default=20, help="Max samples kept per task in the window.") + parser.add_argument("--apply", action="store_true", help="Append to the window + refresh baseline.json fallback.") + parser.add_argument("--soft-drop-pct", type=float, default=5.0, help="Drops beyond this are flagged for review.") + parser.add_argument( + "--hard-drop-pct", type=float, default=15.0, help="Drops beyond this are refused (unless --force)." + ) + parser.add_argument("--force", action="store_true", help="Apply even hard-limit drops.") + parser.add_argument("--stats-out", default=None, help="Optional path to write the raw window stats JSON.") + parser.add_argument("--from-stats", default=None, help="Reuse a previous --stats-out window instead of measuring.") + parser.add_argument( + "--seeds", + nargs="+", + type=int, + default=None, + help="Seed-sensitivity study (report-only): measure same-seed CV (--repeat reps at the baseline seed) " + "vs cross-seed CV (one run per given seed). Justifies hard-coding the gate seed.", + ) + args = parser.parse_args(argv) + + baseline_path = Path(args.baseline).resolve() + baseline = gate._load_baseline(baseline_path) + out_root = Path(args.output_dir).resolve() + tasks = args.tasks or _baseline_tasks(baseline) + + if args.seeds: + return _seed_study(args, baseline, out_root, tasks) + + all_stats: dict[str, dict] = {} + if args.from_stats: + # Apply a window measured earlier (no GPU): measure once, review/apply later. + cached = json.loads(Path(args.from_stats).read_text(encoding="utf-8")) + all_stats = {t: cached[t] for t in tasks if t in cached} + else: + for task in tasks: + stats = measure_task(task, baseline, out_root, args.repeat, args.cache_dir) + if stats is not None: + all_stats[task] = stats + + if args.stats_out: + Path(args.stats_out).write_text(json.dumps(all_stats, indent=2), encoding="utf-8") + + # --- report ------------------------------------------------------------- + _emit_summary("\n## Perf baseline window study\n") + source = "cached window" if args.from_stats else f"{args.repeat} run(s)/task" + _emit_summary(f"GPU key `{args.gpu_key}`, {source}.\n") + _emit_summary("| Task | n | median FPS | CV% | MAD | min | max | current | Δ vs current |") + _emit_summary("|---|---:|---:|---:|---:|---:|---:|---:|---:|") + flagged: list[str] = [] + refused: list[str] = [] + proposals: dict[str, dict] = {} + for task, stats in all_stats.items(): + prop = _proposed_entry(stats) + cur = _current_baseline_fps(baseline, task, args.gpu_key) + delta = ((prop["baseline_fps"] - cur) / cur * 100.0) if cur else None + delta_str = f"{delta:+.2f}%" if delta is not None else "—" + cur_str = f"{cur:.0f}" if cur else "—" + guard = "" + if delta is not None and delta < -args.hard_drop_pct and not args.force: + guard = " ❌refused" + refused.append(task) + elif delta is not None and delta < -args.soft_drop_pct: + guard = " ⚠️review" + flagged.append(task) + proposals[task] = prop + _emit_summary( + f"| {task} | {stats['n']} | {stats['median']:.0f} | {stats['cv_pct']:.2f} | " + f"{stats['mad']:.1f} | {stats['min']:.0f} | {stats['max']:.0f} | {cur_str} | {delta_str}{guard} |" + ) + + if flagged: + _emit_summary(f"\n⚠️ **Flagged for review (drop > {args.soft_drop_pct}%):** {', '.join(flagged)}") + if refused: + _emit_summary( + f"\n❌ **Refused (drop > {args.hard_drop_pct}%, likely a real regression):** {', '.join(refused)}" + ) + + if not args.apply: + _emit_summary("\n_Report only. Re-run with `--apply` to write these into baseline.json._") + return 0 + + # --- apply -------------------------------------------------------------- + # Append to the rolling window (primary store) and refresh the static + # fallback in baseline.json so both track reality. + history_dir = Path(args.history_dir).resolve() + for task, prop in proposals.items(): + if task in refused: + print(f"[rebaseline] {task}: refused (hard-limit drop) -> keeping current value", flush=True) + continue + n_window = _append_window(history_dir, task, args.gpu_key, all_stats[task], cap=args.window_cap) + print(f"[rebaseline] {task}: window now n={n_window}", flush=True) + entry = baseline.setdefault(task, {}).setdefault("per_gpu", {}).setdefault(args.gpu_key, {}) + entry.update(prop) + baseline_path.write_text(json.dumps(baseline, indent=4) + "\n", encoding="utf-8") + applied = len(proposals) - len(refused) + _emit_summary(f"\n✅ Appended to window `{history_dir}` and refreshed `{baseline_path.name}` ({applied} task(s)).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_smoke/run_perf_gate.py b/tools/perf_smoke/run_perf_gate.py new file mode 100644 index 000000000000..d728ab106613 --- /dev/null +++ b/tools/perf_smoke/run_perf_gate.py @@ -0,0 +1,300 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Phase 1 perf-smoke gate runner. + +Orchestrates the gate for one or more tasks: + +1. For each task, read its run config (``num_envs``, ``num_frames``, ``seed``, + optional ``benchmark_args``) from ``baseline.json`` so the gate and the + baseline can never drift apart -- a measurement is always taken under the same + config the baseline was calibrated with. +2. Launch ``scripts/benchmarks/benchmark_non_rl.py`` for that task as its own + subprocess (Isaac Sim must own the process) with ``--benchmark_backend json`` + (the format the baselines were calibrated from), retrying once on a + launch/crash failure. +3. Run the pure-logic comparator (``check_perf_regression.py``) against the + produced result JSON and map its exit code to a per-task verdict. +4. Aggregate: the gate exits non-zero if *any* task is REGRESSION or + HARD_FAILURE. + +Optional warm cache (``--cache-dir``) +------------------------------------- +The dominant cold-start cost is Newton/Warp JIT compilation. Pointing +``WARP_CACHE_PATH`` / ``CUDA_CACHE_PATH`` at a persistent directory turns the +second run on a host into a warm run. ``--cache-dir DIR`` does exactly that; it +is the local stand-in for the S3 "sidecar" (see ``CACHING_SIDECAR.md``) -- in CI +the same directory would be restored from / saved to object storage around the +run. It is purely additive: omit it and the gate runs cold, which only shifts +FPS by ~0-3% and never changes the verdict. + +This is deliberately a standalone script, not a pytest module: ``tools/conftest.py`` +disables pytest collection under ``tools/`` and runs each Isaac Sim entry point as +its own process. The gate follows that same model. + +Run it via the Isaac Lab launcher so the benchmark subprocess inherits the env:: + + ./isaaclab.sh -p tools/perf_smoke/run_perf_gate.py --tasks Isaac-Cartpole-v0 + +The runner itself only needs the standard library, so ``--dry-run`` (which prints +the commands without launching anything) works under any Python. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +# Verdict exit codes mirror check_perf_regression.py so the gate's own exit code +# is meaningful when a single task is run. +EXIT_PASS = 0 +EXIT_BLOCK = 1 +EXIT_HARD_FAILURE = 2 + +_THIS_DIR = Path(__file__).resolve().parent +_REPO_ROOT = _THIS_DIR.parents[1] +_COMPARATOR = _THIS_DIR / "check_perf_regression.py" +_BENCHMARK = _REPO_ROOT / "scripts" / "benchmarks" / "benchmark_non_rl.py" +_LAUNCHER = _REPO_ROOT / "isaaclab.sh" +_DEFAULT_HISTORY = _THIS_DIR / "perf_history" +_DEFAULT_OVERRIDES = _THIS_DIR / "baseline_overrides.json" + +_VERDICT_NAME = {EXIT_PASS: "PASS", EXIT_BLOCK: "BLOCK", EXIT_HARD_FAILURE: "HARD_FAILURE"} + + +def _load_baseline(path: Path) -> dict: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _task_run_config(baseline: dict, task: str) -> dict: + """Pull the launch config for ``task`` out of the baseline document. + + Returns a dict with ``num_envs``, ``num_frames``, ``seed`` and an optional + ``benchmark_args`` list. Falls back to the calibration defaults + (``num_frames=300``, ``seed=42``) when a field is absent. + """ + entry = baseline.get(task) + if not isinstance(entry, dict): + raise KeyError(f"task {task!r} not present in baseline") + # The gate key may be a variant like "Isaac-Cartpole-v0@newton"; ``task_id`` is the + # real gym task the benchmark is launched with (the part before "@" by default). + return { + "task_id": entry.get("task_id", task.split("@", 1)[0]), + "num_envs": entry.get("num_envs"), + "num_frames": entry.get("num_frames", 300), + "seed": entry.get("seed", 42), + "benchmark_args": entry.get("benchmark_args", []), + } + + +def _benchmark_cmd(task: str, cfg: dict, task_out: Path) -> list[str]: + """Build the benchmark subprocess command for a task. + + The benchmark is always launched with the real gym id (``cfg['task_id']``), so a + variant gate key like ``Isaac-Cartpole-v0@newton`` still runs the right task. + """ + task_id = cfg.get("task_id", task) + cmd = [str(_LAUNCHER), "-p", str(_BENCHMARK), "--task", task_id, "--headless"] + if cfg.get("num_envs") is not None: + cmd += ["--num_envs", str(cfg["num_envs"])] + cmd += ["--num_frames", str(cfg["num_frames"]), "--seed", str(cfg["seed"])] + # json backend: same shape the baselines were calibrated from; carries the + # per-frame step-time array the comparator uses for advisory debug KPIs. + cmd += ["--benchmark_backend", "json"] + cmd += list(cfg.get("benchmark_args", [])) + cmd += ["--output_path", str(task_out)] + return cmd + + +def _cache_env(cache_dir: str | None) -> dict[str, str] | None: + """Build the JIT-cache env overlay for a warm run, or ``None`` to run cold. + + Persisting ``WARP_CACHE_PATH`` / ``CUDA_CACHE_PATH`` across runs is the whole + of the sidecar mechanism: the first run populates the dir (cold), later runs + reuse it (warm). In CI this dir is what gets restored from / saved to S3. + """ + if not cache_dir: + return None + root = Path(cache_dir).resolve() + warp_dir = root / "warp" + cuda_dir = root / "nv" + warp_dir.mkdir(parents=True, exist_ok=True) + cuda_dir.mkdir(parents=True, exist_ok=True) + env = dict(os.environ) + env["WARP_CACHE_PATH"] = str(warp_dir) + env["CUDA_CACHE_PATH"] = str(cuda_dir) + return env + + +def _run_benchmark( + task: str, cfg: dict, task_out: Path, retries: int, dry_run: bool, cache_dir: str | None +) -> float | None: + """Launch the benchmark for a task, retrying once on failure. + + Returns the wall-clock seconds of the successful run, or ``None`` if every + attempt failed. + """ + cmd = _benchmark_cmd(task, cfg, task_out) + print(f"[gate] benchmark cmd: {' '.join(cmd)}", flush=True) + if cache_dir: + print(f"[gate] {task}: warm-cache dir = {Path(cache_dir).resolve()}", flush=True) + if dry_run: + return 0.0 + env = _cache_env(cache_dir) + task_out.mkdir(parents=True, exist_ok=True) + for attempt in range(1, retries + 2): # 1 initial + `retries` extra + t0 = time.time() + proc = subprocess.run(cmd, cwd=str(_REPO_ROOT), env=env) + dt = time.time() - t0 + if proc.returncode == 0: + print(f"[gate] {task}: benchmark ok in {dt:.0f}s (attempt {attempt})", flush=True) + return dt + print( + f"[gate] {task}: benchmark FAILED rc={proc.returncode} in {dt:.0f}s (attempt {attempt})", + flush=True, + ) + return None + + +def _run_comparator( + task: str, + results_dir: Path, + baseline_path: Path, + gpu_override: str | None, + history_dir: str | None = None, + overrides_path: Path | None = None, + wall_s: float | None = None, + fingerprint: str | None = None, + task_id: str | None = None, +) -> tuple[int, str]: + """Run the pure-logic comparator. Returns ``(exit_code, result_label)``. + + ``result_label`` is the ``RESULT=...`` token parsed from the comparator's + output (PASS / WARN / BLOCK); it distinguishes an advisory WARN from a clean + PASS, both of which exit 0. The rolling-window store (``history_dir``), the + in-tree overrides, the measured wall-clock and the history fingerprint are + forwarded so the comparator applies the doc's median+MAD test logic. + """ + cmd = [ + sys.executable, + str(_COMPARATOR), + "--task", + task, + "--results-dir", + str(results_dir), + "--baseline", + str(baseline_path), + ] + if task_id and task_id != task: + cmd += ["--task-id", task_id] + if gpu_override: + cmd += ["--gpu-override", gpu_override] + if history_dir: + cmd += ["--history-dir", str(history_dir)] + if overrides_path: + cmd += ["--overrides", str(overrides_path)] + if wall_s is not None: + cmd += ["--measured-wall-s", str(wall_s)] + if fingerprint: + cmd += ["--fingerprint", fingerprint] + proc = subprocess.run(cmd, capture_output=True, text=True) + out = (proc.stdout or "") + (proc.stderr or "") + print(out.rstrip(), flush=True) + label = _VERDICT_NAME.get(proc.returncode, str(proc.returncode)) + for tok in out.split(): + if tok.startswith("RESULT="): + label = tok.split("=", 1)[1] + break + return proc.returncode, label + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else None) + parser.add_argument("--tasks", nargs="+", required=True, help="Gate task name(s).") + parser.add_argument("--baseline", default=str(_THIS_DIR / "baseline.json"), help="Path to baseline.json.") + parser.add_argument("--output-dir", default=str(_REPO_ROOT / "perf-output"), help="Root output directory.") + parser.add_argument("--retries", type=int, default=1, help="Extra benchmark retries on failure (default 1).") + parser.add_argument("--gpu-override", default=None, help="Force the baseline GPU key (e.g. 'NVIDIA L40S').") + parser.add_argument( + "--cache-dir", + default=None, + help="Persist Warp/CUDA JIT caches here (warm runs). Local stand-in for the S3 sidecar; " + "omit to run cold. Purely additive -- never changes the verdict.", + ) + parser.add_argument( + "--history-dir", + default=str(_DEFAULT_HISTORY), + help="Rolling-window store (orphan-branch checkout / local stand-in). Empty disables it.", + ) + parser.add_argument("--overrides", default=str(_DEFAULT_OVERRIDES), help="Path to baseline_overrides.json.") + parser.add_argument("--fingerprint", default=None, help="History bucket key (git-subtree+deps hash).") + parser.add_argument("--dry-run", action="store_true", help="Print commands without launching anything.") + args = parser.parse_args(argv) + + baseline_path = Path(args.baseline).resolve() + out_root = Path(args.output_dir).resolve() + baseline = _load_baseline(baseline_path) + + verdicts: dict[str, int] = {} + labels: dict[str, str] = {} + walls: dict[str, float | None] = {} + for task in args.tasks: + print(f"\n[gate] === {task} ===", flush=True) + try: + cfg = _task_run_config(baseline, task) + except KeyError as e: + print(f"[gate] {task}: {e} -> HARD_FAILURE", flush=True) + verdicts[task] = EXIT_HARD_FAILURE + labels[task] = "HARD_FAILURE" + continue + + task_out = out_root / task + wall = _run_benchmark(task, cfg, task_out, args.retries, args.dry_run, args.cache_dir) + walls[task] = wall + if args.dry_run: + verdicts[task] = EXIT_PASS + labels[task] = "DRY_RUN" + continue + if wall is None: + # Benchmark could not produce a result after retries -> structural failure. + print(f"[gate] {task}: benchmark unrunnable after retries -> HARD_FAILURE", flush=True) + verdicts[task] = EXIT_HARD_FAILURE + labels[task] = "HARD_FAILURE" + continue + + code, label = _run_comparator( + task, + task_out, + baseline_path, + args.gpu_override, + history_dir=args.history_dir or None, + overrides_path=Path(args.overrides) if args.overrides else None, + wall_s=wall, + fingerprint=args.fingerprint, + task_id=cfg.get("task_id"), + ) + verdicts[task] = code + labels[task] = label + + # Aggregate: worst verdict wins (HARD_FAILURE > BLOCK > PASS/WARN). + print("\n[gate] === SUMMARY ===", flush=True) + worst = EXIT_PASS + for task, code in verdicts.items(): + wall = walls.get(task) + wall_str = f" wall={wall:.0f}s" if isinstance(wall, float) and wall > 0 else "" + print(f"[gate] {task}: {labels.get(task, _VERDICT_NAME.get(code, code))}{wall_str}", flush=True) + worst = max(worst, code) + print(f"[gate] OVERALL: {_VERDICT_NAME.get(worst, worst)}", flush=True) + return worst + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_smoke/seed_history.py b/tools/perf_smoke/seed_history.py new file mode 100644 index 000000000000..317a1df0e333 --- /dev/null +++ b/tools/perf_smoke/seed_history.py @@ -0,0 +1,104 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Seed the rolling-window history store from existing L40S calibration runs. + +The perf gate judges a run against a rolling window of historical samples +(median+MAD; see ``check_perf_regression.py``). On a fresh store there is no +window yet, so this one-off seeds it from the warm calibration runs already in +``exploration_matrix/output/`` -- the same L40S data the static baselines came +from -- computing each sample's post-warm-up steady FPS with the *identical* +function the gate uses, so the window and the gate agree by construction. + +In production the window is the orphan branch and is appended to by +``rebaseline.py`` on the runner pool; this just gives the POC a realistic, +non-empty starting window. Run with any Python:: + + python3 tools/perf_smoke/seed_history.py +""" + +from __future__ import annotations + +import contextlib +import csv +import datetime as dt +import glob +import json +import os +from pathlib import Path + +from check_perf_regression import _load_result, steady_fps + +_THIS_DIR = Path(__file__).resolve().parent +_MATRIX = _THIS_DIR / "exploration_matrix" / "output" +_RESULTS_CSV = _MATRIX / "results_full.csv" +_HISTORY_DIR = _THIS_DIR / "perf_history" +_GPU = "NVIDIA L40S" +_NUM_FRAMES = 300 + +# gate key -> (calibration cell, warmup_frames). The key may be a gym id (PhysX +# default) or a "@" variant; the window file is named by the key. +_TASKS = { + "Isaac-Cartpole-v0": ("cartpole_physx_n4096", 2), + "Isaac-Factory-GearMesh-Direct-v0": ("factory_physx_n512", 2), + "Isaac-Velocity-Flat-G1-v0": ("g1_flat_physx_n2048", 2), + "Isaac-Repose-Cube-Shadow-Vision-Direct-v0": ("shadow_vision_physx_rtx_64x64_n128", 60), + "Isaac-Cartpole-v0@newton": ("cartpole_newton_n4096", 5), + "Isaac-Velocity-Flat-G1-v0@newton": ("g1_flat_newton_n2048", 5), +} + + +def _wall_by_round(cell: str) -> dict[int, float]: + """Map warm run_index -> wall_sec for a cell from results_full.csv.""" + out: dict[int, float] = {} + with open(_RESULTS_CSV, newline="") as f: + for row in csv.DictReader(f): + if row["cell"] == cell and row["cache_state"] == "warm" and row["failure_type"] == "OK": + with contextlib.suppress(ValueError, KeyError): + out[int(row["run_index"])] = float(row["wall_sec"]) + return out + + +def main() -> int: + _HISTORY_DIR.mkdir(parents=True, exist_ok=True) + now = dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds") + for task, (cell, warmup) in _TASKS.items(): + walls = _wall_by_round(cell) + samples = [] + for rnd_dir in sorted(glob.glob(str(_MATRIX / cell / "warm_round*"))): + run_index = int(os.path.basename(rnd_dir).replace("warm_round", "")) + jsons = [p for p in sorted(glob.glob(os.path.join(rnd_dir, "*.json"))) if "meta" not in os.path.basename(p)] + if not jsons: + continue + result = _load_result(Path(jsons[-1])) + fps = steady_fps(result, warmup, _NUM_FRAMES) + samples.append( + { + "fps": round(fps, 1), + "wall_s": walls.get(run_index), + "source": f"{cell}/{os.path.basename(rnd_dir)}", + "ts": now, + } + ) + store = { + "task": task, + "gpu": _GPU, + "num_frames": _NUM_FRAMES, + "warmup_frames": warmup, + "window": 20, + "_note": "Seeded from L40S warm calibration runs (500f truncated to 300f, post-warm-up).", + "samples": samples, + } + out_path = _HISTORY_DIR / f"{task}__{_GPU}.json".replace(" ", "_") + with open(out_path, "w", encoding="utf-8") as f: + json.dump(store, f, indent=2) + f.write("\n") + fpss = [s["fps"] for s in samples] + print(f"{task}: n={len(fpss)} fps={[round(x) for x in fpss]} -> {out_path.name}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/perf_smoke/test_check_perf_regression.py b/tools/perf_smoke/test_check_perf_regression.py new file mode 100644 index 000000000000..515282ddc17a --- /dev/null +++ b/tools/perf_smoke/test_check_perf_regression.py @@ -0,0 +1,606 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for ``check_perf_regression.py`` (the perf-smoke comparator). + +Covers the test logic of ``ci-regression-gate-config-info.md``: the post-warm-up +steady metric (D6), rolling-window median+MAD thresholds with a static fallback, +the ``PASS/WARN/BLOCK`` vocabulary, the advisory wall-clock signal, outlier +index/magnitude reporting, and manual overrides. + +Stdlib ``unittest``; also collectable by pytest via this directory's ``pytest.ini``. +Run directly:: + + python3 tools/perf_smoke/test_check_perf_regression.py +""" + +from __future__ import annotations + +import io +import json +import os +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import check_perf_regression as cpr # noqa: E402 + +TASK = "Isaac-Cartpole-Direct-v0" +GPU = "NVIDIA L40" +BASELINE_FPS = 100000 + + +def _omni_doc(fps: float | int | str | None, gpu: str | None = GPU) -> dict: + """OmniPerf-shaped result (scalar metric, no per-frame array).""" + doc: dict = {"runtime": {cpr.METRIC_NAME: fps}} + if gpu is not None: + doc["hardware_info"] = {"gpu_current_device": 0, "gpu_devices": {"0": {"name": gpu}}} + return doc + + +def _eff_doc( + eff_fps: list[float] | None, + gpu: str = GPU, + step_times: list[float] | None = None, + scalar: float | None = None, +) -> list: + """json-backend (list-of-phases) result carrying the per-frame arrays.""" + ft: dict = {} + if eff_fps is not None: + ft[cpr.EFF_FPS_ARRAY] = eff_fps + if step_times is not None: + ft[cpr.STEP_MS_ARRAY] = step_times + runtime_meas: list = [] + if scalar is not None: + runtime_meas.append({"name": f"benchmark_non_rl runtime {cpr.METRIC_NAME}", "value": scalar}) + if ft: + runtime_meas.append({"name": f"benchmark_non_rl runtime {cpr.FRAMETIMES_NAME}", "value": ft}) + return [ + {"phase_name": "runtime", "measurements": runtime_meas, "metadata": []}, + { + "phase_name": "hardware_info", + "measurements": [], + "metadata": [ + {"name": "benchmark_non_rl hardware_info gpu_current_device", "data": 0}, + {"name": "benchmark_non_rl hardware_info gpu_devices", "data": {"0": {"name": gpu}}}, + ], + }, + ] + + +def _baseline( + fps: int = BASELINE_FPS, + warn: float = 5.0, + block: float = 10.0, + gpu_key: str = GPU, + warmup: int | None = None, + num_frames: int | None = None, +) -> dict: + """Minimal baseline document (run config + static fallback thresholds).""" + task_entry: dict = {"per_gpu": {gpu_key: {"baseline_fps": fps, "warn_pct": warn, "max_regression_pct": block}}} + if warmup is not None: + task_entry["warmup_frames"] = warmup + if num_frames is not None: + task_entry["num_frames"] = num_frames + return {TASK: task_entry} + + +class _GateTestBase(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + self.results_dir = self.tmp / "perf-output" + self.results_dir.mkdir() + self.baseline_path = self.tmp / "baseline.json" + self.history_dir = self.tmp / "history" + self.overrides_path = self.tmp / "overrides.json" + + def tearDown(self) -> None: + self._tmp.cleanup() + + def _write_result(self, doc: dict | list | str, name: str | None = None) -> None: + path = self.results_dir / (name or f"benchmark_non_rl_{TASK}_x.json") + path.write_text(doc if isinstance(doc, str) else json.dumps(doc), encoding="utf-8") + + def _write_baseline(self, doc: dict) -> None: + self.baseline_path.write_text(json.dumps(doc), encoding="utf-8") + + def _write_window(self, fps: list[float], wall: list[float] | None = None, gpu_key: str = GPU) -> None: + self.history_dir.mkdir(exist_ok=True) + samples = [{"fps": v} for v in fps] + if wall is not None: + for s, w in zip(samples, wall): + s["wall_s"] = w + safe = f"{TASK}__{gpu_key}".replace(" ", "_") + (self.history_dir / f"{safe}.json").write_text(json.dumps({"samples": samples}), encoding="utf-8") + + def _write_overrides(self, doc: dict) -> None: + self.overrides_path.write_text(json.dumps(doc), encoding="utf-8") + + def _run(self, *extra: str) -> tuple[int, str]: + argv = ["--task", TASK, "--results-dir", str(self.results_dir), "--baseline", str(self.baseline_path), *extra] + buf = io.StringIO() + with redirect_stdout(buf): + code = cpr.main(argv) + return code, buf.getvalue().strip() + + +class StaticFallbackTests(_GateTestBase): + """With no rolling window, the static baseline_fps + pct bands govern.""" + + def test_pass_within_band(self) -> None: + self._write_result(_omni_doc(int(BASELINE_FPS * 0.97))) + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn("RESULT=PASS", out) + self.assertIn("thresholds=static_baseline", out) + + def test_improvement_passes(self) -> None: + self._write_result(_omni_doc(int(BASELINE_FPS * 1.08))) + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn("delta_pct=+8.00", out) + + def test_warn_band_is_advisory(self) -> None: + # 7% drop: past the 5% warn band, short of the 10% block band -> WARN, exit 0. + self._write_result(_omni_doc(int(BASELINE_FPS * 0.93))) + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn("RESULT=WARN", out) + + def test_block_on_large_regression(self) -> None: + self._write_result(_omni_doc(int(BASELINE_FPS * 0.80))) + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_BLOCK) + self.assertIn("RESULT=BLOCK", out) + self.assertIn("kind=regression", out) + + +class WindowThresholdTests(_GateTestBase): + """With >= MIN_WINDOW samples, median+MAD over the window governs.""" + + def test_uses_window_when_available(self) -> None: + self._write_result(_omni_doc(10000)) + self._write_baseline(_baseline(fps=999999)) # static would BLOCK; window must win + self._write_window([10000, 10010, 9990, 10005, 9995]) + code, out = self._run("--history-dir", str(self.history_dir)) + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn("thresholds=window(n=5)", out) + self.assertIn("center_fps=10000", out) + + def test_block_below_window(self) -> None: + self._write_result(_omni_doc(8000)) + self._write_baseline(_baseline(fps=10000)) + self._write_window([10000, 10010, 9990, 10005, 9995]) + code, out = self._run("--history-dir", str(self.history_dir)) + self.assertEqual(code, cpr.EXIT_BLOCK) + self.assertIn("kind=regression", out) + + def test_small_window_falls_back_to_static(self) -> None: + self._write_result(_omni_doc(9700)) + self._write_baseline(_baseline(fps=10000)) + self._write_window([10000, 9990]) # < MIN_WINDOW + code, out = self._run("--history-dir", str(self.history_dir)) + self.assertIn("thresholds=static_baseline", out) + + +class SteadyMetricTests(_GateTestBase): + """D6: the gating KPI is mean effective FPS after dropping warmup_frames.""" + + def test_drops_default_warmup(self) -> None: + # Two tiny warm-up frames then steady at the baseline; dropping 2 -> PASS. + eff = [1.0, 1.0] + [BASELINE_FPS] * 100 + self._write_result(_eff_doc(eff)) + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn(f"measured_fps={BASELINE_FPS}", out) + + def test_task_warmup_override_honored(self) -> None: + # First 5 frames are slow; with warmup_frames=5 they are excluded -> PASS. + eff = [1.0] * 5 + [BASELINE_FPS] * 100 + self._write_result(_eff_doc(eff)) + self._write_baseline(_baseline(warmup=5)) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn("warmup_frames=5", out) + + def test_scalar_fallback_when_no_array(self) -> None: + self._write_result(_eff_doc(None, scalar=BASELINE_FPS)) + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + + def test_num_frames_truncation(self) -> None: + # Trailing frames beyond num_frames are ignored so a longer run stays comparable. + eff = [1.0, 1.0] + [BASELINE_FPS] * 8 + [1.0] * 100 + self._write_result(_eff_doc(eff)) + self._write_baseline(_baseline(num_frames=10)) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn(f"measured_fps={BASELINE_FPS}", out) + + +class DebugAndWallTests(_GateTestBase): + """Advisory KPIs: outlier index/magnitude and the wall-clock signal.""" + + def test_outlier_index_and_magnitude(self) -> None: + steps = [300.0, 140.0] + [10.0] * 20 + steps[7] = 30.0 # 3x the steady median at steady-frame index 5 (after dropping 2) + self._write_result(_eff_doc([BASELINE_FPS] * len(steps), step_times=steps)) + self._write_baseline(_baseline()) + _, out = self._run() + self.assertIn("outlier_count=1", out) + self.assertIn("outlier_idx=5", out) + self.assertIn("outlier_mag_x=3", out) + + def test_wall_signal_reported(self) -> None: + self._write_result(_omni_doc(BASELINE_FPS)) + self._write_baseline(_baseline()) + self._write_window([BASELINE_FPS] * 5, wall=[100, 101, 99, 100, 100]) + _, out = self._run("--history-dir", str(self.history_dir), "--measured-wall-s", "100") + self.assertIn("wall_center_s=100", out) + self.assertIn("wall_delta_pct=", out) + + def test_wall_flag_when_slow(self) -> None: + self._write_result(_omni_doc(BASELINE_FPS)) + self._write_baseline(_baseline()) + self._write_window([BASELINE_FPS] * 5, wall=[100, 101, 99, 100, 100]) + _, out = self._run("--history-dir", str(self.history_dir), "--measured-wall-s", "130") + self.assertIn("wall_flag=slow", out) + + +class WarmupGuardTests(_GateTestBase): + """Advisory warm-up guardrail: flag a hot first kept frame (stale warmup_frames).""" + + def test_flag_when_first_kept_frame_hot(self) -> None: + # Drop 2 warm-up frames; the first KEPT step is 5x the steady median -> flag. + steps = [300.0, 280.0] + [50.0] + [10.0] * 30 + self._write_result(_eff_doc([BASELINE_FPS] * len(steps), step_times=steps)) + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_PASS) # advisory only -- never changes the verdict + self.assertIn("warmup_flag=", out) + + def test_no_flag_when_first_kept_frame_steady(self) -> None: + steps = [300.0, 280.0] + [10.0] * 30 + self._write_result(_eff_doc([BASELINE_FPS] * len(steps), step_times=steps)) + self._write_baseline(_baseline()) + _, out = self._run() + self.assertNotIn("warmup_flag=", out) + + +class TailKpiTests(_GateTestBase): + """Opt-in advisory tail signal: WARN (never BLOCK) on a high p99/median ratio.""" + + def _doc_with_p99(self, ratio: float) -> list: + # Steady median 10ms; a top ~3% of steps at ratio*median pushes p99/median ~= ratio. + steps = [10.0] * 100 + for i in (50, 51, 52): # >= 2% so the p99 index lands on a spike + steps[i] = 10.0 * ratio + return _eff_doc([BASELINE_FPS] * len(steps), step_times=steps) + + def test_tail_warn_when_over_ceiling(self) -> None: + self._write_result(self._doc_with_p99(3.0)) + self._write_baseline(_baseline()) + self._write_overrides({TASK: {GPU: {"tail_p99_warn": 1.5}}}) + code, out = self._run("--overrides", str(self.overrides_path)) + self.assertEqual(code, cpr.EXIT_PASS) # WARN is advisory -> exit 0 + self.assertIn("RESULT=WARN", out) + self.assertIn("reason=tail", out) + self.assertIn("tail_flag=", out) + + def test_no_tail_warn_under_ceiling(self) -> None: + self._write_result(self._doc_with_p99(1.2)) + self._write_baseline(_baseline()) + self._write_overrides({TASK: {GPU: {"tail_p99_warn": 2.0}}}) + code, out = self._run("--overrides", str(self.overrides_path)) + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn("RESULT=PASS", out) + self.assertNotIn("tail_flag=", out) + + def test_tail_disabled_by_default(self) -> None: + # Without the override, a spiky run still PASSes (no tail gating). + self._write_result(self._doc_with_p99(5.0)) + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn("RESULT=PASS", out) + + def test_fps_block_wins_over_tail(self) -> None: + # A real FPS regression must BLOCK even with a tail ceiling set. + steps = [10.0] * 100 + steps[50] = 50.0 + self._write_result(_eff_doc([int(BASELINE_FPS * 0.8)] * len(steps), step_times=steps)) + self._write_baseline(_baseline()) + self._write_overrides({TASK: {GPU: {"tail_p99_warn": 1.5}}}) + code, out = self._run("--overrides", str(self.overrides_path)) + self.assertEqual(code, cpr.EXIT_BLOCK) + self.assertIn("kind=regression", out) + + +class OverrideTests(_GateTestBase): + """Manual overrides (committed with the PR) adjust or bypass the gate.""" + + def test_skip_forces_pass(self) -> None: + self._write_result(_omni_doc(1)) # would BLOCK + self._write_baseline(_baseline()) + self._write_overrides({TASK: {GPU: {"skip": True}}}) + code, out = self._run("--overrides", str(self.overrides_path)) + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn("skipped_by_override", out) + + def test_pin_center(self) -> None: + self._write_result(_omni_doc(120000)) + self._write_baseline(_baseline()) + self._write_overrides({TASK: {GPU: {"pin_center_fps": 120000}}}) + code, out = self._run("--overrides", str(self.overrides_path)) + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn("thresholds=override_pin", out) + self.assertIn("center_fps=120000", out) + + def test_k_block_override_tightens(self) -> None: + # A 4% drop passes the default k_block but a tightened spread should BLOCK. + self._write_result(_omni_doc(int(BASELINE_FPS * 0.96))) + self._write_baseline(_baseline()) + self._write_window([BASELINE_FPS] * 5) + self._write_overrides({TASK: {GPU: {"k_block": 1.0, "min_spread_pct": 1.0}}}) + code, _ = self._run("--history-dir", str(self.history_dir), "--overrides", str(self.overrides_path)) + self.assertEqual(code, cpr.EXIT_BLOCK) + + +class HardFailureTests(_GateTestBase): + """Structural problems map to BLOCK/hard_failure (exit 2).""" + + def test_no_results_file(self) -> None: + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + self.assertIn("kind=hard_failure", out) + self.assertIn("no_results_found", out) + + def test_malformed_json(self) -> None: + self._write_result("{not valid json") + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + self.assertIn("malformed_json", out) + + def test_missing_task(self) -> None: + self._write_result(_omni_doc(BASELINE_FPS)) + self._write_baseline({"other-task": {"per_gpu": {GPU: {"baseline_fps": 1}}}}) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + self.assertIn("missing_baseline_task", out) + + def test_missing_baseline_fps(self) -> None: + self._write_result(_omni_doc(BASELINE_FPS)) + self._write_baseline({TASK: {"per_gpu": {GPU: {"warn_pct": 5.0}}}}) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + self.assertIn("missing_baseline_field", out) + + def test_gpu_mismatch(self) -> None: + self._write_result(_omni_doc(BASELINE_FPS, gpu="NVIDIA Other")) + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + self.assertIn("baseline_gpu_mismatch", out) + + def test_unknown_gpu_without_override(self) -> None: + self._write_result(_omni_doc(BASELINE_FPS, gpu=None)) + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + self.assertIn("unknown_gpu", out) + + def test_zero_metric(self) -> None: + self._write_result(_omni_doc(0)) + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + self.assertIn("missing_metric", out) + + def test_string_metric(self) -> None: + self._write_result(_omni_doc("nope")) + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + + def test_nan_metric(self) -> None: + path = self.results_dir / f"benchmark_non_rl_{TASK}.json" + path.write_text( + '{"runtime": {"' + cpr.METRIC_NAME + '": NaN}, ' + '"hardware_info": {"gpu_current_device": 0, "gpu_devices": {"0": {"name": "' + GPU + '"}}}}', + encoding="utf-8", + ) + self._write_baseline(_baseline()) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + + +def _info_phase( + task: str = TASK, num_envs: int = 512, seed: int = 42, num_frames: int = 300, presets: str | None = None +) -> dict: + """A ``benchmark_info`` phase echoing the run config (for config-assert tests).""" + meta = [ + {"name": "benchmark_non_rl benchmark_info task", "data": task}, + {"name": "benchmark_non_rl benchmark_info seed", "data": seed}, + {"name": "benchmark_non_rl benchmark_info num_envs", "data": num_envs}, + {"name": "benchmark_non_rl benchmark_info num_frames", "data": num_frames}, + ] + if presets is not None: + meta.append({"name": "benchmark_non_rl benchmark_info presets", "data": presets}) + return {"phase_name": "benchmark_info", "measurements": [], "metadata": meta} + + +class ConfigAssertTests(_GateTestBase): + """The run's self-reported config must match baseline.json, else hard_failure.""" + + def _doc_with_info(self, info: dict) -> list: + doc = _eff_doc([BASELINE_FPS] * 50) + doc.append(info) + return doc + + def test_matching_config_passes(self) -> None: + self._write_result(self._doc_with_info(_info_phase(num_envs=512, seed=42, num_frames=300))) + self._write_baseline( + {TASK: {"num_envs": 512, "seed": 42, "num_frames": 300, "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}}} + ) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + + def test_num_envs_mismatch_blocks(self) -> None: + self._write_result(self._doc_with_info(_info_phase(num_envs=4096))) + self._write_baseline({TASK: {"num_envs": 512, "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}}}) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + self.assertIn("config_mismatch", out) + self.assertIn("num_envs(ran=4096,want=512)", out) + + def test_too_few_frames_blocks(self) -> None: + self._write_result(self._doc_with_info(_info_phase(num_frames=100))) + self._write_baseline({TASK: {"num_frames": 300, "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}}}) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + self.assertIn("config_mismatch", out) + + def test_no_benchmark_info_is_noop(self) -> None: + # OmniPerf / legacy results carry no benchmark_info -> assertion is skipped. + self._write_result(_omni_doc(BASELINE_FPS)) + self._write_baseline({TASK: {"num_envs": 512, "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}}}) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + + def test_matching_presets_pass(self) -> None: + self._write_result(self._doc_with_info(_info_phase(presets="newton_mjwarp"))) + self._write_baseline( + {TASK: {"benchmark_args": ["physics=newton_mjwarp"], "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}}} + ) + code, _ = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + + def test_wrong_backend_blocks(self) -> None: + # Baseline expects Newton but the run reported PhysX -> a different KPI, hard_failure. + self._write_result(self._doc_with_info(_info_phase(presets="physx"))) + self._write_baseline( + {TASK: {"benchmark_args": ["physics=newton_mjwarp"], "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}}} + ) + code, out = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + self.assertIn("config_mismatch", out) + self.assertIn("presets(", out) + self.assertIn("missing=newton_mjwarp", out) + + def test_multi_preset_subset_match(self) -> None: + # Each expected token (physx + renderer) must appear in the comma-joined presets. + self._write_result(self._doc_with_info(_info_phase(presets="physx,isaacsim_rtx_renderer"))) + self._write_baseline( + { + TASK: { + "benchmark_args": ["physics=physx", "presets=isaacsim_rtx_renderer"], + "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}, + } + } + ) + code, _ = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + + def test_presets_unreported_is_noop(self) -> None: + # Older results omit presets -> we don't assert (no false BLOCK). + self._write_result(self._doc_with_info(_info_phase(presets=None))) + self._write_baseline( + {TASK: {"benchmark_args": ["physics=newton_mjwarp"], "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}}} + ) + code, _ = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + + +class VariantKeyTests(_GateTestBase): + """A '@' gate key resolves via task_id for glob + config check.""" + + VARIANT = f"{TASK}@newton" + + def _run_variant(self, *extra: str) -> tuple[int, str]: + argv = [ + "--task", + self.VARIANT, + "--task-id", + TASK, + "--results-dir", + str(self.results_dir), + "--baseline", + str(self.baseline_path), + *extra, + ] + buf = io.StringIO() + with redirect_stdout(buf): + code = cpr.main(argv) + return code, buf.getvalue().strip() + + def test_variant_resolves_and_passes(self) -> None: + # Result file is named by the gym id; baseline is keyed by the variant key. + self._write_result(_eff_doc([BASELINE_FPS] * 50), name=f"benchmark_non_rl_{TASK}_x.json") + self._write_baseline({self.VARIANT: {"task_id": TASK, "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}}}) + code, out = self._run_variant() + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn(f"task={self.VARIANT}", out) + + def test_task_id_defaults_to_prefix_before_at(self) -> None: + # Without --task-id, the gym id is inferred as the part before "@". + self._write_result(_eff_doc([BASELINE_FPS] * 50), name=f"benchmark_non_rl_{TASK}_x.json") + self._write_baseline({self.VARIANT: {"task_id": TASK, "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}}}) + argv = [ + "--task", + self.VARIANT, + "--results-dir", + str(self.results_dir), + "--baseline", + str(self.baseline_path), + ] + buf = io.StringIO() + with redirect_stdout(buf): + code = cpr.main(argv) + self.assertEqual(code, cpr.EXIT_PASS) + + +class HelperTests(unittest.TestCase): + """Direct unit tests for helpers where edge cases are easier to express.""" + + def test_steady_fps_drops_warmup(self) -> None: + result = {"runtime": {cpr.FRAMETIMES_NAME: {cpr.EFF_FPS_ARRAY: [1.0, 1.0, 100.0, 100.0]}}} + self.assertEqual(cpr.steady_fps(result, warmup_frames=2), 100.0) + + def test_steady_fps_scalar_fallback(self) -> None: + self.assertEqual(cpr.steady_fps({"runtime": {cpr.METRIC_NAME: 1234}}, warmup_frames=2), 1234.0) + + def test_steady_fps_rejects_empty(self) -> None: + with self.assertRaises(cpr.CompareError): + cpr.steady_fps({"runtime": {}}, warmup_frames=2) + + def test_median_mad(self) -> None: + center, mad = cpr._median_mad([10.0, 12.0, 14.0]) + self.assertEqual(center, 12.0) + self.assertEqual(mad, 2.0) + + def test_match_gpu_substring(self) -> None: + key, _ = cpr._match_gpu({"L40": {"baseline_fps": 1.0}}, "NVIDIA L40") + self.assertEqual(key, "L40") + + def test_overrides_precedence(self) -> None: + ov = cpr._overrides_for({"_defaults": {"k_warn": 3}, TASK: {"k_warn": 4, GPU: {"k_warn": 5}}}, TASK, GPU) + self.assertEqual(ov["k_warn"], 5) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tools/perf_smoke/test_perf_gate.py b/tools/perf_smoke/test_perf_gate.py new file mode 100644 index 000000000000..20ca46283c98 --- /dev/null +++ b/tools/perf_smoke/test_perf_gate.py @@ -0,0 +1,86 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Pytest orchestrator for the perf-smoke gate (D1). + +The doc asks for the gate to be *shelled out by pytest*: pytest is the test +framework, but it never launches Isaac Sim in-process (Isaac Sim is a process +singleton). This module honors that exactly -- each task is one parametrized +test that launches the benchmark as its own subprocess and then runs the +pure-logic comparator, asserting the verdict is not ``BLOCK``. + +It runs as an ordinary pytest module because ``pytest.ini`` in this directory +pins the rootdir here, so the repo-wide ``tools/conftest.py`` harness (which +hijacks every pytest session under ``tools/``) is not loaded. + +Launching Isaac Sim needs a GPU, so the gate tests self-skip unless +``GATE_RUN=1`` -- a plain ``pytest tools/perf_smoke`` still runs the comparator +unit tests but does not try to start the simulator. On the runner:: + + GATE_RUN=1 GATE_TASKS="Isaac-Cartpole-v0 ..." \ + ./isaaclab.sh -p -m pytest tools/perf_smoke/test_perf_gate.py + +Recognized environment: + +* ``GATE_TASKS`` -- space-separated task list (default: all tasks in baseline.json). +* ``GATE_CACHE_DIR`` -- warm Warp/CUDA JIT cache dir (optional; additive). +* ``GATE_OUTPUT_DIR``-- where benchmark JSON is written (default: /perf-output). +* ``GATE_GPU`` -- baseline GPU key override (e.g. "NVIDIA L40S"). +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest +import run_perf_gate as gate + +_THIS = Path(__file__).resolve().parent +_BASELINE = _THIS / "baseline.json" +_HISTORY = _THIS / "perf_history" +_OVERRIDES = _THIS / "baseline_overrides.json" + +pytestmark = pytest.mark.skipif( + os.environ.get("GATE_RUN") != "1", + reason="set GATE_RUN=1 to launch the perf gate (needs a GPU + Isaac Sim)", +) + + +def _gate_tasks() -> list[str]: + env = os.environ.get("GATE_TASKS", "").split() + if env: + return env + with open(_BASELINE, encoding="utf-8") as f: + return [k for k in json.load(f) if not k.startswith("_")] + + +@pytest.mark.parametrize("task", _gate_tasks()) +def test_perf_gate(task: str) -> None: + """Run one task end-to-end (benchmark subprocess -> comparator) and assert no BLOCK.""" + baseline = gate._load_baseline(_BASELINE) + cfg = gate._task_run_config(baseline, task) + + out_root = Path(os.environ.get("GATE_OUTPUT_DIR", gate._REPO_ROOT / "perf-output")).resolve() + task_out = out_root / task + cache_dir = os.environ.get("GATE_CACHE_DIR") or None + gpu = os.environ.get("GATE_GPU") or None + + wall = gate._run_benchmark(task, cfg, task_out, retries=1, dry_run=False, cache_dir=cache_dir) + assert wall is not None, f"{task}: benchmark did not produce a result after retries (BLOCK/hard_failure)" + + code, label = gate._run_comparator( + task, + task_out, + _BASELINE, + gpu, + history_dir=str(_HISTORY), + overrides_path=_OVERRIDES, + wall_s=wall, + task_id=cfg.get("task_id"), + ) + # PASS and WARN both exit 0; BLOCK (regression) and hard_failure are non-zero. + assert code == gate.EXIT_PASS, f"{task}: gate verdict {label} (exit {code})" diff --git a/tools/perf_smoke/test_stress_check_perf_regression.py b/tools/perf_smoke/test_stress_check_perf_regression.py new file mode 100644 index 000000000000..b504143ee554 --- /dev/null +++ b/tools/perf_smoke/test_stress_check_perf_regression.py @@ -0,0 +1,381 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Adversarial / stress tests for ``check_perf_regression.py`` (the perf-smoke comparator). + +The companion suite ``test_check_perf_regression.py`` exercises the *happy paths* +of the comparator. This file deliberately attacks the edges -- malformed history, +hostile GPU strings, degenerate baselines, truncated runs, poisoned inputs -- to +surface correctness and safety gaps. Findings are written up in +``POC_LIMITATIONS_REPORT.md``. + +Two kinds of tests live here: + +* **Lock tests** assert the comparator's *current, observed* behavior on an edge + case (so a future change is forced to acknowledge it). +* **``@unittest.expectedFailure`` tests** assert the behavior we *want* but the + comparator does **not** yet deliver. They pass as "expected failures" today; if + the bug is ever fixed, they flip to "unexpected success" and prompt removing the + marker. Each carries a ``BUG:`` note keyed to the report. + +Run directly:: + + python3 tools/perf_smoke/test_stress_check_perf_regression.py +""" + +from __future__ import annotations + +import io +import json +import os +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import check_perf_regression as cpr # noqa: E402 + +TASK = "Isaac-Cartpole-v0" +GPU = "NVIDIA L40S" +BASELINE_FPS = 100000 + + +def _omni(fps: float | int | str | None, gpu: str | None = GPU) -> dict: + doc: dict = {"runtime": {cpr.METRIC_NAME: fps}} + if gpu is not None: + doc["hardware_info"] = {"gpu_current_device": 0, "gpu_devices": {"0": {"name": gpu}}} + return doc + + +def _eff(eff: list[float] | None, gpu: str = GPU, steps: list[float] | None = None) -> list: + ft: dict = {} + if eff is not None: + ft[cpr.EFF_FPS_ARRAY] = eff + if steps is not None: + ft[cpr.STEP_MS_ARRAY] = steps + return [ + { + "phase_name": "runtime", + "measurements": [{"name": f"benchmark_non_rl runtime {cpr.FRAMETIMES_NAME}", "value": ft}], + "metadata": [], + }, + { + "phase_name": "hardware_info", + "measurements": [], + "metadata": [ + {"name": "benchmark_non_rl hardware_info gpu_current_device", "data": 0}, + {"name": "benchmark_non_rl hardware_info gpu_devices", "data": {"0": {"name": gpu}}}, + ], + }, + ] + + +def _info(**fields: object) -> dict: + meta = [{"name": f"benchmark_non_rl benchmark_info {k}", "data": v} for k, v in fields.items()] + return {"phase_name": "benchmark_info", "measurements": [], "metadata": meta} + + +def _baseline( + fps: float = BASELINE_FPS, warn: float = 5.0, block: float = 10.0, gpu_key: str = GPU, **extra: object +) -> dict: + entry: dict = {"per_gpu": {gpu_key: {"baseline_fps": fps, "warn_pct": warn, "max_regression_pct": block}}} + entry.update(extra) + return {TASK: entry} + + +class _Base(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + self.results_dir = self.tmp / "out" + self.results_dir.mkdir() + self.baseline_path = self.tmp / "baseline.json" + self.history_dir = self.tmp / "history" + self.overrides_path = self.tmp / "overrides.json" + + def tearDown(self) -> None: + self._tmp.cleanup() + + def _write_result(self, doc, name: str | None = None) -> None: + path = self.results_dir / (name or f"benchmark_non_rl_{TASK}_x.json") + path.write_text(doc if isinstance(doc, str) else json.dumps(doc), encoding="utf-8") + + def _write_baseline(self, doc: dict) -> None: + self.baseline_path.write_text(json.dumps(doc), encoding="utf-8") + + def _write_window_raw(self, samples: list[dict], gpu_key: str = GPU) -> None: + self.history_dir.mkdir(exist_ok=True) + safe = f"{TASK}__{gpu_key}".replace(" ", "_") + # allow_nan=True keeps this realistic: json.dump writes NaN/Infinity by default. + (self.history_dir / f"{safe}.json").write_text(json.dumps({"samples": samples}), encoding="utf-8") + + def _write_overrides(self, doc: dict) -> None: + self.overrides_path.write_text(json.dumps(doc), encoding="utf-8") + + def _run(self, *extra: str, task: str = TASK): + argv = ["--task", task, "--results-dir", str(self.results_dir), "--baseline", str(self.baseline_path), *extra] + buf = io.StringIO() + exc = None + with redirect_stdout(buf): + try: + code = cpr.main(argv) + except Exception as e: # noqa: BLE001 -- we WANT to see uncaught exceptions + code, exc = None, e + return code, buf.getvalue().strip(), exc + + +# --------------------------------------------------------------------------- A +class DegenerateBaselineTests(_Base): + """A zero / tiny baseline center must not crash the comparator.""" + + @unittest.expectedFailure + def test_zero_center_should_be_hard_failure_not_crash(self) -> None: + # BUG A: baseline_fps=0 -> delta_pct = (m-0)/0 -> uncaught ZeroDivisionError. + # Desired: a structural BLOCK/hard_failure, never a Python traceback. + self._write_result(_omni(50000)) + self._write_baseline(_baseline(fps=0)) + code, _out, exc = self._run() + self.assertIsNone(exc, f"comparator crashed instead of degrading: {exc!r}") + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + + def test_zero_center_currently_crashes(self) -> None: + # Lock the present (bad) behavior so the report stays accurate. + self._write_result(_omni(50000)) + self._write_baseline(_baseline(fps=0)) + _code, _out, exc = self._run() + self.assertIsInstance(exc, ZeroDivisionError) + + +# --------------------------------------------------------------------------- B +class GpuMatchingTests(_Base): + """Substring GPU matching conflates distinct GPUs.""" + + @unittest.expectedFailure + def test_l40_must_not_match_l40s(self) -> None: + # BUG B: result GPU 'NVIDIA L40' substring-matches baseline 'NVIDIA L40S' + # (gpu_key in key), so an L40 run is judged against an L40S window. + self._write_result(_omni(50000, gpu="NVIDIA L40")) + self._write_baseline(_baseline(fps=50000, gpu_key="NVIDIA L40S")) + code, out, _ = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE, f"expected GPU mismatch, got: {out}") + + @unittest.expectedFailure + def test_desktop_4090_must_not_match_laptop_4090(self) -> None: + # BUG B: 'RTX 4090' is a substring of 'RTX 4090 Laptop GPU' -> false match + # between two very differently-performing GPUs. + self._write_result(_omni(50000, gpu="NVIDIA GeForce RTX 4090")) + self._write_baseline(_baseline(fps=50000, gpu_key="NVIDIA GeForce RTX 4090 Laptop GPU")) + code, _out, _ = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + + def test_substring_match_is_current_behavior(self) -> None: + self._write_result(_omni(50000, gpu="NVIDIA L40")) + self._write_baseline(_baseline(fps=50000, gpu_key="NVIDIA L40S")) + code, out, _ = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn("gpu=NVIDIA L40S", out) + + +# --------------------------------------------------------------------------- C +class TruncatedRunTests(_Base): + """A run with fewer frames than warmup_frames silently skips warm-up exclusion.""" + + @unittest.expectedFailure + def test_short_run_should_not_silently_keep_warmup(self) -> None: + # BUG C: only 3 frames exist but warmup_frames=60. steady_fps keeps the whole + # (warm-up-polluted) window instead of erroring, so a truncated/crashed run is + # accepted as a valid steady measurement. + self._write_result(_eff([10.0, 10.0, 10.0])) # all warm-up frames + self._write_baseline(_baseline(fps=10, warmup_frames=60, num_frames=300)) + code, out, _ = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE, f"truncated run accepted as valid: {out}") + + def test_short_run_currently_passes(self) -> None: + self._write_result(_eff([10.0, 10.0, 10.0])) + self._write_baseline(_baseline(fps=10, warmup_frames=60, num_frames=300)) + code, _out, _ = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + + +# --------------------------------------------------------------------------- D +class StaticBandFidelityTests(_Base): + """The static fallback cannot honor warn_pct and max_regression_pct independently.""" + + @unittest.expectedFailure + def test_block_band_honored_when_not_2x_warn(self) -> None: + # BUG D: warn_pct=5, max_regression_pct=8. A -9% drop is past the configured + # 8% block band and should BLOCK -- but the spread is driven by the warn band, + # so the effective block floor slips to -10% and the run only WARNs. + self._write_result(_omni(int(BASELINE_FPS * 0.91))) # -9% + self._write_baseline(_baseline(warn=5.0, block=8.0)) + code, out, _ = self._run() + self.assertEqual(code, cpr.EXIT_BLOCK, f"configured 8% block band not honored: {out}") + + @unittest.expectedFailure + def test_warn_band_honored_when_block_is_large(self) -> None: + # BUG D (other direction): warn_pct=5, max_regression_pct=20. A -8% drop is + # past the configured 5% warn band and should WARN -- but the block-derived + # spread widens the warn floor to -10%, so this PASSes silently. + self._write_result(_omni(int(BASELINE_FPS * 0.92))) # -8% + self._write_baseline(_baseline(warn=5.0, block=20.0)) + code, out, _ = self._run() + self.assertIn("RESULT=WARN", out, f"configured 5% warn band not honored: {out}") + + +# --------------------------------------------------------------------------- E +class PoisonedHistoryTests(_Base): + """A single corrupt history sample silently disables the gate.""" + + @unittest.expectedFailure + def test_nan_in_window_should_not_blind_the_gate(self) -> None: + # BUG E: one NaN fps at the median position -> median=nan -> all thresholds nan + # -> every '<' comparison is False -> the task can never BLOCK again. Silent. + # (NaN at index 2 of a 5-sample window reliably poisons the median; whether a + # NaN poisons at all is position-dependent, which is itself the bug.) + self._write_result(_omni(1)) # a catastrophic regression + self._write_baseline(_baseline(fps=BASELINE_FPS)) + self._write_window_raw([{"fps": BASELINE_FPS}] * 2 + [{"fps": float("nan")}] + [{"fps": BASELINE_FPS}] * 2) + code, out, _ = self._run("--history-dir", str(self.history_dir)) + self.assertNotIn("center_fps=nan", out) + self.assertEqual(code, cpr.EXIT_BLOCK, f"NaN window let a real regression PASS: {out}") + + def test_nan_window_currently_passes_everything(self) -> None: + # Lock the current behavior: a median-position NaN disables the gate (PASS). + self._write_result(_omni(1)) + self._write_baseline(_baseline(fps=BASELINE_FPS)) + self._write_window_raw([{"fps": BASELINE_FPS}] * 2 + [{"fps": float("nan")}] + [{"fps": BASELINE_FPS}] * 2) + code, out, _ = self._run("--history-dir", str(self.history_dir)) + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn("center_fps=nan", out) + + def test_nan_effect_is_position_dependent(self) -> None: + # The same poisoned window judged with the NaN at a tail position yields a + # REAL median -> the gate works. Identical data, different order, opposite + # verdict: NaN handling is undefined, not merely lenient. + self._write_result(_omni(1)) + self._write_baseline(_baseline(fps=BASELINE_FPS)) + self._write_window_raw([{"fps": BASELINE_FPS}] * 4 + [{"fps": float("nan")}]) # NaN at tail + code, out, _ = self._run("--history-dir", str(self.history_dir)) + self.assertEqual(code, cpr.EXIT_BLOCK) + self.assertNotIn("center_fps=nan", out) + + +# --------------------------------------------------------------------------- F +class PoisonedMeasurementTests(_Base): + """Negative per-frame FPS values are averaged into the KPI instead of rejected.""" + + @unittest.expectedFailure + def test_negative_frames_should_be_caught(self) -> None: + # BUG F: steady_fps only checks the *final mean* > 0; individual negative + # (impossible) per-frame FPS values silently drag the mean instead of failing. + arr = [BASELINE_FPS] * 100 + [-BASELINE_FPS] * 5 + self._write_result(_eff(arr)) + self._write_baseline(_baseline(fps=BASELINE_FPS)) + code, out, _ = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE, f"negative frames silently averaged: {out}") + + +# --------------------------------------------------------------------------- G +class GlobCollisionTests(_Base): + """The result glob is prefix-greedy; sibling-prefixed task names collide.""" + + def test_sibling_prefix_causes_false_hard_failure(self) -> None: + # BUG G: glob 'benchmark_non_rl_Isaac-Cartpole-v0*.json' also matches a sibling + # task 'Isaac-Cartpole-v0-Camera'. Without --allow-multiple this is a spurious + # multiple_results hard_failure (a FALSE BLOCK) even though the real result is present. + self._write_result(_omni(BASELINE_FPS), name="benchmark_non_rl_Isaac-Cartpole-v0_a.json") + self._write_result(_omni(1000), name="benchmark_non_rl_Isaac-Cartpole-v0-Camera_b.json") + self._write_baseline(_baseline()) + code, out, _ = self._run() + # Lock the (undesirable) current behavior. + self.assertEqual(code, cpr.EXIT_HARD_FAILURE) + self.assertIn("multiple_results", out) + + +# --------------------------------------------------------------------------- H +class QuarantineTests(_Base): + """`skip` cannot quarantine a task that crashed (no result produced).""" + + @unittest.expectedFailure + def test_skip_should_quarantine_a_broken_task(self) -> None: + # BUG H: the docs sell `skip:true` as the escape hatch for a "temporarily + # flaky" task -- but flaky tasks usually crash, and hard_failure (missing + # result) is evaluated BEFORE the skip override, so skip cannot rescue them. + self._write_baseline(_baseline()) # NOTE: no result file written -> crash + self._write_overrides({TASK: {GPU: {"skip": True}}}) + code, out, _ = self._run("--overrides", str(self.overrides_path), "--gpu-override", GPU) + self.assertEqual(code, cpr.EXIT_PASS, f"skip did not quarantine a broken task: {out}") + + +# --------------------------------------------------------------------------- I +class PinCenterTests(_Base): + """Pinning a center without pinning spread yields nonsensical bands.""" + + @unittest.expectedFailure + def test_pin_center_should_scale_spread(self) -> None: + # BUG I: pin_center_fps=300000 but spread is still computed from the OLD window + # center (~100000), so the band becomes ~0.5% of the new center. A run within + # 5% of the intended new center then falsely BLOCKs -- the opposite of the + # override's purpose (accepting an intended perf change). + self._write_result(_omni(285000)) # -5% vs the pinned center + self._write_baseline(_baseline(fps=BASELINE_FPS)) + self._write_window_raw([{"fps": BASELINE_FPS}] * 5) + self._write_overrides({TASK: {GPU: {"pin_center_fps": 300000}}}) + code, out, _ = self._run("--history-dir", str(self.history_dir), "--overrides", str(self.overrides_path)) + self.assertEqual(code, cpr.EXIT_PASS, f"pinned-center bands collapsed: {out}") + + +# --------------------------------------------------------------------------- J +class ConfigAssertTypeTests(_Base): + """Config assertion is skipped when the backend serializes numbers as strings.""" + + @unittest.expectedFailure + def test_string_num_envs_should_still_be_checked(self) -> None: + # BUG J: benchmark_info.num_envs reported as "4096" (str). _assert_run_config + # guards on isinstance(got,(int,float)), so a string slips past unchecked and a + # config change is silently misread as a perf number. + doc = _eff([BASELINE_FPS] * 50) + doc.append(_info(num_envs="4096")) + self.baseline_path.write_text( + json.dumps({TASK: {"num_envs": 512, "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}}}), encoding="utf-8" + ) + self._write_result(doc) + code, out, _ = self._run() + self.assertEqual(code, cpr.EXIT_HARD_FAILURE, f"string num_envs bypassed config check: {out}") + + +# --------------------------------------------------------------------------- misc +class BoundaryTests(_Base): + """Smaller, lower-severity edge observations worth locking down.""" + + def test_block_floor_is_exclusive(self) -> None: + # A measurement landing *exactly* on the block floor does not BLOCK (strict <). + # Documented here so the off-by-epsilon boundary is intentional, not accidental. + self._write_result(_omni(90000)) # exactly -10% with a 10% block band + self._write_baseline(_baseline(warn=5.0, block=10.0)) + code, _out, _ = self._run() + self.assertEqual(code, cpr.EXIT_PASS) + + def test_window_warmup_metadata_is_ignored(self) -> None: + # The window file records the warmup_frames it was computed with, but the + # comparator never checks it against the baseline's current warmup_frames, so a + # later warmup change compares new-warmup measurements to old-warmup history. + self._write_result(_omni(BASELINE_FPS)) + self._write_baseline(_baseline(fps=BASELINE_FPS, warmup_frames=2)) + self.history_dir.mkdir(exist_ok=True) + safe = f"{TASK}__{GPU}".replace(" ", "_") + (self.history_dir / f"{safe}.json").write_text( + json.dumps({"warmup_frames": 60, "samples": [{"fps": BASELINE_FPS}] * 5}), encoding="utf-8" + ) + code, out, _ = self._run("--history-dir", str(self.history_dir)) + self.assertEqual(code, cpr.EXIT_PASS) + self.assertIn("warmup_frames=2", out) # uses baseline's, ignores window's 60 + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tools/perf_smoke/warp_replicator_shim.py b/tools/perf_smoke/warp_replicator_shim.py new file mode 100644 index 000000000000..0cab93ca06f4 --- /dev/null +++ b/tools/perf_smoke/warp_replicator_shim.py @@ -0,0 +1,125 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Install a compatibility shim so Warp >=1.13 and ``omni.replicator.core`` coexist. + +Why this exists +--------------- +The perf gate must run all four gate tasks in a *single* environment. Two of them +pull in conflicting Warp expectations: + +* Newton rough-terrain tasks need ``wp.tile_query_valid`` (added in Warp 1.13.0). +* ``omni.replicator.core-1.13.4`` (the RTX/camera path used by the shadow-vision + task) was built against the pre-1.13 layout and still references + ``wp.context`` and a handful of ``warp.types.*`` symbols that Warp 1.13.0 + relocated into ``warp._src``. + +No single PyPI Warp release satisfies both, so we pin Warp 1.13.0 and re-expose +the relocated symbols. This is a temporary bridge: it can be deleted once Isaac +Sim ships a replicator built for Warp >=1.13. + +What it does +------------ +Idempotently appends a small shim block to the installed ``warp/__init__.py`` +that re-publishes ``warp.context`` (both as an attribute and in ``sys.modules``) +and any missing ``warp.types`` helpers from ``warp._src``. Safe no-op when the +symbols already resolve or the shim is already present. + +Usage:: + + ./isaaclab.sh -p tools/perf_smoke/warp_replicator_shim.py # install + ./isaaclab.sh -p tools/perf_smoke/warp_replicator_shim.py --check # verify only +""" + +from __future__ import annotations + +import argparse +import importlib.util +import sys +from pathlib import Path + +_MARKER = "# >>> isaaclab perf-gate warp/replicator compatibility shim >>>" +_END_MARKER = "# <<< isaaclab perf-gate warp/replicator compatibility shim <<<" + +_SHIM = f""" + +{_MARKER} +# Bridge omni.replicator.core (built for the pre-1.13 Warp layout) onto Warp +# >=1.13, which moved internals into warp._src. Installed by +# tools/perf_smoke/warp_replicator_shim.py. Remove once replicator targets +# Warp >=1.13 natively. +try: + import sys as _sys + + from warp import _src as _src + + context = _src.context # noqa: F811 (re-export for `wp.context.*`) + _sys.modules.setdefault("warp.context", _src.context) # for `import warp.context` + + import warp.types as _pub_types # noqa + + for _n in ("array", "type_size_in_bytes", "warp_type_to_np_dtype", "np_dtype_to_warp_type"): + if not hasattr(_pub_types, _n) and hasattr(_src.types, _n): + setattr(_pub_types, _n, getattr(_src.types, _n)) +except Exception: # pragma: no cover - the shim must never break `import warp` + pass +{_END_MARKER} +""" + + +def _warp_init_path() -> Path: + """Locate the installed ``warp/__init__.py`` without importing warp.""" + spec = importlib.util.find_spec("warp") + if spec is None or not spec.origin: + raise SystemExit("warp is not importable in this interpreter") + return Path(spec.origin) + + +def _is_installed(init_path: Path) -> bool: + return _MARKER in init_path.read_text(encoding="utf-8") + + +def install() -> bool: + """Append the shim if absent. Returns True if it wrote, False if already present.""" + init_path = _warp_init_path() + if _is_installed(init_path): + print(f"shim already present: {init_path}") + return False + with open(init_path, "a", encoding="utf-8") as f: + f.write(_SHIM) + print(f"shim installed: {init_path}") + return True + + +def check() -> bool: + """Verify the relocated symbols resolve. Returns True when the env is usable.""" + import warp as wp # noqa + + ok = True + try: + import warp.context # noqa + except Exception as e: # noqa: BLE001 + print(f"FAIL: import warp.context -> {e}") + ok = False + if not hasattr(wp, "context"): + print("FAIL: wp.context attribute missing") + ok = False + if ok: + print(f"OK: warp {wp.__version__} exposes context for replicator") + return ok + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else None) + parser.add_argument("--check", action="store_true", help="Only verify the shim resolves; do not edit files.") + args = parser.parse_args(argv) + if args.check: + return 0 if check() else 1 + install() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 587409b61c45532919b36c9c117eaac77a8426be Mon Sep 17 00:00:00 2001 From: Horde Date: Thu, 11 Jun 2026 21:41:54 +0000 Subject: [PATCH 2/3] Fix perf gate task-id drift and record physics backend Reconcile the perf smoke gate with the current develop registrations and benchmark provenance, surfaced by a 1-task L40S dry run: - Point the cartpole baseline keys at the registered Isaac-Cartpole gym id via task_id (the manager-based env dropped its -v0 suffix), keeping the stable history/baseline keys intact. - Record the physics= backend in benchmark_info (new get_physics_string) so the comparator can verify the simulation backend. Previously only presets= was echoed, so a physics-only task reported presets=default and the config check spuriously hard-failed. - Split the comparator config check: physics= is matched against benchmark_info.physics, presets= against benchmark_info.presets. - Re-baseline Isaac-Cartpole (physx) to the current build (~276k FPS on L40S); the prior 115k was ~2.4x stale. Marked provisional pending a full re-baseline on the runner fleet. - Skip the get-pr-info step on workflow_dispatch (no originating PR), so manual single-task trial runs are not blocked. --- .github/workflows/perf-gate.yml | 4 +++ scripts/benchmarks/benchmark_non_rl.py | 2 ++ scripts/benchmarks/utils.py | 19 +++++++++++ tools/perf_smoke/baseline.json | 16 +++++---- tools/perf_smoke/check_perf_regression.py | 31 +++++++++++------ .../Isaac-Cartpole-v0__NVIDIA_L40S.json | 34 +++---------------- .../perf_smoke/test_check_perf_regression.py | 32 ++++++++++------- 7 files changed, 80 insertions(+), 58 deletions(-) diff --git a/.github/workflows/perf-gate.yml b/.github/workflows/perf-gate.yml index 4506fb4cdb21..a53ad5d283da 100644 --- a/.github/workflows/perf-gate.yml +++ b/.github/workflows/perf-gate.yml @@ -125,6 +125,10 @@ jobs: # TODO(bringup): these two nv-gha-runners actions still need repo-admin # allowlist approval. Pinned to main@ below (no release tags exist). - name: Get PR info + # Only meaningful on the copy-pr-bot push model (recovers PR metadata from + # the mirrored commit). A manual workflow_dispatch has no originating PR, so + # skip it there to keep dispatch-triggered trial runs unblocked. + if: github.event_name == 'push' uses: nv-gha-runners/get-pr-info@090577647b8ddc4e06e809e264f7881650ecdccf # main - name: Checkout diff --git a/scripts/benchmarks/benchmark_non_rl.py b/scripts/benchmarks/benchmark_non_rl.py index ed8702cfbc7e..7d3b49a7395d 100644 --- a/scripts/benchmarks/benchmark_non_rl.py +++ b/scripts/benchmarks/benchmark_non_rl.py @@ -61,6 +61,7 @@ from scripts.benchmarks.utils import ( get_backend_type, + get_physics_string, get_preset_string, log_app_start_time, log_python_imports_time, @@ -111,6 +112,7 @@ {"name": "num_envs", "data": args_cli.num_envs}, {"name": "num_frames", "data": args_cli.num_frames}, {"name": "presets", "data": get_preset_string(hydra_args)}, + {"name": "physics", "data": get_physics_string(hydra_args)}, ] }, ) diff --git a/scripts/benchmarks/utils.py b/scripts/benchmarks/utils.py index 05effa524172..3811318b34bb 100644 --- a/scripts/benchmarks/utils.py +++ b/scripts/benchmarks/utils.py @@ -136,6 +136,25 @@ def get_preset_string(hydra_args: list[str]) -> str: return os.environ.get("ISAACLAB_BENCHMARK_PRESET", "") or "default" +def get_physics_string(hydra_args: list[str]) -> str: + """Extract the selected physics backend from CLI hydra args or an environment variable. + + The ``physics=`` Hydra group selects the simulation backend (e.g. ``physx``, + ``newton_mjwarp``); unlike rendering/observation modes it is a distinct group + from ``presets=``, so it is recorded separately for run provenance. + + Checks (in order): + 1. ``physics=...`` in *hydra_args* (e.g. ``physics=physx``) + 2. ``ISAACLAB_BENCHMARK_PHYSICS`` environment variable + 3. Falls back to ``"default"`` (the task's configured backend) + """ + for arg in hydra_args: + if arg.startswith("physics="): + value = arg.split("=", 1)[1] + return value if value else "default" + return os.environ.get("ISAACLAB_BENCHMARK_PHYSICS", "") or "default" + + def log_rl_policy_rewards(benchmark: BaseIsaacLabBenchmark, value: list): measurement = ListMeasurement(name="Rewards", value=value) benchmark.add_measurement("train", measurement=measurement) diff --git a/tools/perf_smoke/baseline.json b/tools/perf_smoke/baseline.json index b0786dd9c51d..b9e7400ae38e 100644 --- a/tools/perf_smoke/baseline.json +++ b/tools/perf_smoke/baseline.json @@ -12,9 +12,11 @@ "overrides live in baseline_overrides.json (committed with the PR), NOT here and NOT in the window.", "D6 warm-up exclusion: the gating KPI is the mean post-warm-up effective FPS -- the first", "warmup_frames are dropped (2 for PhysX tasks; 60 for shadow-vision to clear the camera/JIT window).", - "VARIANTS: a key may be a gym id (e.g. 'Isaac-Cartpole-v0', the PhysX default) or a variant", - "'@' (e.g. 'Isaac-Cartpole-v0@newton') carrying its own 'task_id' (the real gym", - "task the benchmark is launched with) and benchmark_args. Newton uses physics=newton_mjwarp and a", + "VARIANTS: a key is a stable history/baseline label. When it differs from the live gym id, the", + "entry carries an explicit 'task_id' (the real gym task the benchmark is launched with) -- e.g.", + "'Isaac-Cartpole-v0' maps to the registered 'Isaac-Cartpole', and the '@' Newton", + "variants (e.g. 'Isaac-Cartpole-v0@newton') reuse the same 'task_id' with their own benchmark_args.", + "Newton uses physics=newton_mjwarp and a", "5-frame warm-up (JIT). Output/history are keyed by the full variant key so PhysX and Newton don't collide.", "Gate config: NVIDIA L40S, WARM runs/task, num_frames=300, seed=42.", "Fallback values were derived from the existing warm L40S runs (500f truncated to 300f, post-warm-up);", @@ -22,6 +24,7 @@ "Per-GPU keys substring-match hardware_info.gpu_devices[].name ('NVIDIA L40S' matches that device)." ], "Isaac-Cartpole-v0": { + "task_id": "Isaac-Cartpole", "num_envs": 4096, "num_frames": 300, "seed": 42, @@ -30,11 +33,12 @@ "config_note": "cartpole_physx_n4096", "per_gpu": { "NVIDIA L40S": { - "baseline_fps": 115235.8, + "baseline_fps": 276401.7, "warn_pct": 5.0, "max_regression_pct": 10.0, "cv_pct": 2.02, - "n_runs": 5 + "n_runs": 1, + "_provisional": "Single-run local re-baseline (isaaclab 6.6.1 / warp 1.13.0). Prior 115235.8 was ~2.4x stale (older build). Re-record a full window on the runner fleet; cv_pct/warn/block kept from the prior calibration until then." } } }, @@ -73,7 +77,7 @@ } }, "Isaac-Cartpole-v0@newton": { - "task_id": "Isaac-Cartpole-v0", + "task_id": "Isaac-Cartpole", "num_envs": 4096, "num_frames": 300, "seed": 42, diff --git a/tools/perf_smoke/check_perf_regression.py b/tools/perf_smoke/check_perf_regression.py index 626a2d3f529a..e28e441a87cc 100644 --- a/tools/perf_smoke/check_perf_regression.py +++ b/tools/perf_smoke/check_perf_regression.py @@ -281,19 +281,20 @@ def _benchmark_info(result: dict) -> dict: return info if isinstance(info, dict) else {} -def _expected_presets(task_entry: dict) -> list[str]: - """Physics/renderer preset tokens the gate launches this task with. +def _expected_overrides(task_entry: dict, keys: tuple[str, ...]) -> list[str]: + """Hydra-override values the gate launches this task with, for the given ``keys``. - Pulled from the ``physics=`` / ``presets=`` Hydra overrides in ``benchmark_args`` - (e.g. ``physics=newton_mjwarp`` -> ``newton_mjwarp``). The backend echoes these - back, comma-joined, in ``benchmark_info.presets``. + Pulled from ``benchmark_args`` (e.g. ``physics=newton_mjwarp`` -> ``newton_mjwarp``; + ``presets=physx,rgb`` -> ``physx``, ``rgb``). ``physics=`` and ``presets=`` are + distinct Hydra groups echoed back in distinct ``benchmark_info`` fields, so each is + extracted independently. """ out: list[str] = [] for arg in task_entry.get("benchmark_args", []) or []: if isinstance(arg, str) and "=" in arg: key, _, val = arg.partition("=") - if key in ("physics", "presets") and val: - out.append(val) + if key in keys and val: + out.extend(v.strip() for v in val.split(",") if v.strip()) return out @@ -326,10 +327,18 @@ def _assert_run_config(result: dict, task: str, task_entry: dict) -> None: got_frames = info.get("num_frames") if want_frames is not None and isinstance(got_frames, (int, float)) and int(got_frames) < int(want_frames): mismatches.append(f"num_frames(ran={int(got_frames)},want>={int(want_frames)})") - # Physics/renderer backend: every physics=/presets= override we launch with must - # appear in the run's reported presets. Catches "ran PhysX when the @newton variant - # was intended" -- a different KPI entirely, invisible to an FPS-only check. - want_presets = _expected_presets(task_entry) + # Physics backend: the physics= override we launch with must match the run's + # reported backend (benchmark_info.physics). Catches "ran PhysX when the @newton + # variant was intended" -- a different KPI entirely, invisible to an FPS-only check. + want_physics = _expected_overrides(task_entry, ("physics",)) + ran_physics = str(info.get("physics", "")).strip() + if want_physics and ran_physics: # only assert when the backend reported it + missing = [p for p in want_physics if p != ran_physics] + if missing: + mismatches.append(f"physics(ran={ran_physics},want={','.join(want_physics)})") + # Renderer/observation presets: every presets= override we launch with must appear + # in the run's reported presets (benchmark_info.presets, comma-joined). + want_presets = _expected_overrides(task_entry, ("presets",)) if want_presets: ran_presets = {p.strip() for p in str(info.get("presets", "")).split(",") if p.strip()} if ran_presets: # only assert when the backend reported its presets diff --git a/tools/perf_smoke/perf_history/Isaac-Cartpole-v0__NVIDIA_L40S.json b/tools/perf_smoke/perf_history/Isaac-Cartpole-v0__NVIDIA_L40S.json index a086a40a5776..ff13badf2bfd 100644 --- a/tools/perf_smoke/perf_history/Isaac-Cartpole-v0__NVIDIA_L40S.json +++ b/tools/perf_smoke/perf_history/Isaac-Cartpole-v0__NVIDIA_L40S.json @@ -4,37 +4,13 @@ "num_frames": 300, "warmup_frames": 2, "window": 20, - "_note": "Seeded from L40S warm calibration runs (500f truncated to 300f, post-warm-up).", + "_note": "PROVISIONAL single-run local baseline (isaaclab 6.6.1 / warp 1.13.0, L40S dev box). The prior 5-sample window (~115k FPS) was calibrated on an older build and is ~2.4x stale. With <3 samples the comparator uses the baseline.json static fallback; re-record a full window on the runner fleet to restore window-based gating.", "samples": [ { - "fps": 115333.6, - "wall_s": 37.23, - "source": "cartpole_physx_n4096/warm_round1", - "ts": "2026-06-10T00:49:53+00:00" - }, - { - "fps": 115099.7, - "wall_s": 36.72, - "source": "cartpole_physx_n4096/warm_round2", - "ts": "2026-06-10T00:49:53+00:00" - }, - { - "fps": 114419.0, - "wall_s": 36.22, - "source": "cartpole_physx_n4096/warm_round3", - "ts": "2026-06-10T00:49:53+00:00" - }, - { - "fps": 114538.0, - "wall_s": 37.07, - "source": "cartpole_physx_n4096/warm_round4", - "ts": "2026-06-10T00:49:53+00:00" - }, - { - "fps": 118560.6, - "wall_s": 37.07, - "source": "cartpole_physx_n4096/warm_round5", - "ts": "2026-06-10T00:49:53+00:00" + "fps": 276401.7, + "wall_s": 17.0, + "source": "local_l40s/Isaac-Cartpole_physx_n4096/single", + "ts": "2026-06-11T21:00:48+00:00" } ] } diff --git a/tools/perf_smoke/test_check_perf_regression.py b/tools/perf_smoke/test_check_perf_regression.py index 515282ddc17a..c558e86b4300 100644 --- a/tools/perf_smoke/test_check_perf_regression.py +++ b/tools/perf_smoke/test_check_perf_regression.py @@ -430,7 +430,12 @@ def test_nan_metric(self) -> None: def _info_phase( - task: str = TASK, num_envs: int = 512, seed: int = 42, num_frames: int = 300, presets: str | None = None + task: str = TASK, + num_envs: int = 512, + seed: int = 42, + num_frames: int = 300, + presets: str | None = None, + physics: str | None = None, ) -> dict: """A ``benchmark_info`` phase echoing the run config (for config-assert tests).""" meta = [ @@ -441,6 +446,8 @@ def _info_phase( ] if presets is not None: meta.append({"name": "benchmark_non_rl benchmark_info presets", "data": presets}) + if physics is not None: + meta.append({"name": "benchmark_non_rl benchmark_info physics", "data": physics}) return {"phase_name": "benchmark_info", "measurements": [], "metadata": meta} @@ -482,8 +489,9 @@ def test_no_benchmark_info_is_noop(self) -> None: code, out = self._run() self.assertEqual(code, cpr.EXIT_PASS) - def test_matching_presets_pass(self) -> None: - self._write_result(self._doc_with_info(_info_phase(presets="newton_mjwarp"))) + def test_matching_physics_pass(self) -> None: + # physics= is reported in benchmark_info.physics, not folded into presets. + self._write_result(self._doc_with_info(_info_phase(physics="newton_mjwarp"))) self._write_baseline( {TASK: {"benchmark_args": ["physics=newton_mjwarp"], "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}}} ) @@ -492,19 +500,19 @@ def test_matching_presets_pass(self) -> None: def test_wrong_backend_blocks(self) -> None: # Baseline expects Newton but the run reported PhysX -> a different KPI, hard_failure. - self._write_result(self._doc_with_info(_info_phase(presets="physx"))) + self._write_result(self._doc_with_info(_info_phase(physics="physx"))) self._write_baseline( {TASK: {"benchmark_args": ["physics=newton_mjwarp"], "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}}} ) code, out = self._run() self.assertEqual(code, cpr.EXIT_HARD_FAILURE) self.assertIn("config_mismatch", out) - self.assertIn("presets(", out) - self.assertIn("missing=newton_mjwarp", out) + self.assertIn("physics(", out) + self.assertIn("want=newton_mjwarp", out) - def test_multi_preset_subset_match(self) -> None: - # Each expected token (physx + renderer) must appear in the comma-joined presets. - self._write_result(self._doc_with_info(_info_phase(presets="physx,isaacsim_rtx_renderer"))) + def test_physics_and_presets_both_checked(self) -> None: + # physics= is matched against benchmark_info.physics; presets= against presets. + self._write_result(self._doc_with_info(_info_phase(physics="physx", presets="isaacsim_rtx_renderer"))) self._write_baseline( { TASK: { @@ -516,9 +524,9 @@ def test_multi_preset_subset_match(self) -> None: code, _ = self._run() self.assertEqual(code, cpr.EXIT_PASS) - def test_presets_unreported_is_noop(self) -> None: - # Older results omit presets -> we don't assert (no false BLOCK). - self._write_result(self._doc_with_info(_info_phase(presets=None))) + def test_physics_unreported_is_noop(self) -> None: + # Older results omit the physics field -> we don't assert (no false BLOCK). + self._write_result(self._doc_with_info(_info_phase(physics=None))) self._write_baseline( {TASK: {"benchmark_args": ["physics=newton_mjwarp"], "per_gpu": {GPU: {"baseline_fps": BASELINE_FPS}}}} ) From 1d977c31ceb756097f581d360a68e142422fe118 Mon Sep 17 00:00:00 2001 From: Neil4561 <283821122+Neil4561@users.noreply.github.com> Date: Mon, 15 Jun 2026 22:44:25 +0000 Subject: [PATCH 3/3] Clean up and add tests to perf-smoke POC Finish env-fingerprint bucketing and per-sample provenance for the rolling-window history, add unit tests for the orchestrator, rebaseline tool, and fingerprint/provenance helpers, and document the architecture in DESIGN.md. --- .github/workflows/perf-gate.yml | 18 +- tools/perf_smoke/DESIGN.md | 143 +++++++++++++ tools/perf_smoke/README.md | 6 + tools/perf_smoke/check_perf_regression.py | 82 +++++++- tools/perf_smoke/rebaseline.py | 48 ++++- tools/perf_smoke/seed_history.py | 27 ++- tools/perf_smoke/test_history_fingerprint.py | 153 ++++++++++++++ tools/perf_smoke/test_rebaseline.py | 209 +++++++++++++++++++ tools/perf_smoke/test_run_perf_gate.py | 177 ++++++++++++++++ 9 files changed, 840 insertions(+), 23 deletions(-) create mode 100644 tools/perf_smoke/DESIGN.md create mode 100644 tools/perf_smoke/test_history_fingerprint.py create mode 100644 tools/perf_smoke/test_rebaseline.py create mode 100644 tools/perf_smoke/test_run_perf_gate.py diff --git a/.github/workflows/perf-gate.yml b/.github/workflows/perf-gate.yml index a53ad5d283da..a1a3a3d275da 100644 --- a/.github/workflows/perf-gate.yml +++ b/.github/workflows/perf-gate.yml @@ -76,9 +76,11 @@ jobs: echo "Matrix tasks: $tasks_json" # --------------------------------------------------------------------------- - # Comparator logic tests. Pure stdlib unittest -- no GPU, no Isaac Sim, and - # arch-independent -- so this validates the PASS/WARN/BLOCK verdict logic on - # every trigger and gives fast signal before the GPU job is scheduled. + # Pure-logic tests for the comparator, orchestrator, rebaseline tool, and the + # history-bucketing helpers. Pure stdlib unittest -- no GPU, no Isaac Sim, and + # arch-independent -- so this validates the PASS/WARN/BLOCK verdict logic, the + # launch-config plumbing, and the rolling-window writer on every trigger and + # gives fast signal before the GPU job is scheduled. # --------------------------------------------------------------------------- comparator-unit-tests: name: Comparator Unit Tests @@ -86,8 +88,14 @@ jobs: steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Run comparator unittest suite - run: python3 tools/perf_smoke/test_check_perf_regression.py + - name: Run unittest suite + run: | + set -e + for t in test_check_perf_regression test_history_fingerprint test_run_perf_gate test_rebaseline; do + echo "::group::$t" + python3 "tools/perf_smoke/$t.py" + echo "::endgroup::" + done # --------------------------------------------------------------------------- # The GPU gate. One job PER TASK (matrix from baseline.json): each task is its diff --git a/tools/perf_smoke/DESIGN.md b/tools/perf_smoke/DESIGN.md new file mode 100644 index 000000000000..98fa5c28fd22 --- /dev/null +++ b/tools/perf_smoke/DESIGN.md @@ -0,0 +1,143 @@ +# Perf Smoke Gate — Architecture + +This document explains *how* the gate is put together: the data flow, the +baseline/history model, and the verdict logic. For a plain-English file-by-file +tour, see [`README.md`](README.md). + +## Design goals + +1. **Cheap and per-PR.** A handful of short, stable tasks on a fixed GPU — fast + enough to run on every pull request. +2. **Robust to noise.** Small run-to-run wobble must not flake the gate; only a + real, sustained drop blocks. +3. **In-tree and reviewed.** Baselines and history live in the repo and only + change through a normal, reviewed PR — never a silent side-write from CI. This + keeps the gate auditable and lets `git blame` answer "why did the bar move?". +4. **No new dependencies.** Everything is standard-library Python so the + comparator and its tests run on any runner without Isaac Sim or a GPU. + +## Components + +| Layer | Module | Responsibility | +|---|---|---| +| Orchestration | `run_perf_gate.py` | Per task: resolve launch config from `baseline.json`, launch the benchmark as its own Isaac Sim subprocess (retry once), hand the result to the comparator, aggregate verdicts (worst wins). | +| Decision (pure logic) | `check_perf_regression.py` | Read one benchmark result + baseline + rolling window, compute the KPI, and return PASS / WARN / BLOCK. No GPU, no Isaac Sim. | +| Stored state | `baseline.json`, `perf_history/`, `baseline_overrides.json` | The launch config + static fallback, the rolling window of recent samples, and manual threshold overrides. | +| Maintenance | `rebaseline.py`, `seed_history.py` | Produce/refresh the stored state from fresh or existing runs (always via a reviewed PR). | + +## Data flow (one PR) + +``` +baseline.json ─┐ + ├─► run_perf_gate ─► benchmark_non_rl.py (subprocess) ─► result.json +perf_history/ ─┤ │ +overrides ─────┘ ▼ + check_perf_regression ◄─────────────────┘ + │ + RESULT=PASS|WARN|BLOCK + $GITHUB_STEP_SUMMARY table +``` + +The orchestrator only builds commands and aggregates; **all** of the regression +judgement lives in the comparator, which is why the comparator is independently +unit-testable without hardware. + +## The KPI + +The gating metric is the **post-warm-up steady FPS** (`steady_fps`): the +benchmark's per-frame effective-FPS array with the first `warmup_frames` dropped. +Using the same statistic the backend already reports — just windowed — keeps the +measured value directly comparable to the stored history. Wall-clock seconds are +carried as a secondary, advisory signal only. + +## Baseline & history model + +There are two stores, deliberately layered: + +- **Rolling window (`perf_history/`, primary).** Per `(task, GPU)`, the last + *N* known-good samples. The comparator computes its threshold *at test time* + from this window with a robust **median + MAD** estimator: + + ``` + center = median(window) + spread = max(1.4826 * MAD(window), min_spread_pct/100 * center) + WARN when measured < center - k_warn * spread + BLOCK when measured < center - k_block * spread + ``` + + A `min_spread_pct` floor stops a very low-variance task from blocking on + trivial dips. + +- **Static fallback (`baseline.json`, secondary).** When the window is too small + to trust (`< MIN_WINDOW` samples), the comparator falls back to a static + `baseline_fps` + percentage bands calibrated for that task/GPU. This keeps a + fresh store from silently passing everything before it has accumulated history. + +**Overrides** (`baseline_overrides.json`) are a manual escape hatch keyed by +*stable* test identity (`task` + GPU), applied on top of either source — used for +one-off threshold relaxations or `skip` that ride along in the PR. + +### Environment fingerprint buckets + +Performance is only comparable within the same software stack: a Warp bump or an +Isaac Sim upgrade can legitimately shift FPS, and mixing those samples into one +window would corrupt the baseline. So history is **bucketed by an environment +fingerprint**: + +``` +perf_history/ + __.json # flat "default" bucket (legacy / no provenance) + env-/__.json # one bucket per (warp, isaaclab, cuda) stack +``` + +- `env_fingerprint(result)` hashes the environment-defining provenance + (`warp`, `isaaclab`, `cuda`) into a short, stable `env-` key. GPU is + *not* in the hash because it is already in the file name. +- The comparator derives the fingerprint from the run under test and reads the + matching bucket, **falling back to the flat file** when no bucket exists yet — + so the change is backward-compatible with already-seeded flat history. +- A consistent filename (`history_basename`) is shared by the reader and every + writer so a written bucket is always found again. + +### Per-sample provenance + +Every stored sample carries the context needed to audit or re-bucket it without +re-running the benchmark: `commit`, `warp`, `isaaclab`, `cuda`, plus the +`fingerprint` recorded at the window level. This makes the in-tree history +self-describing — a reviewer reading a `perf_history/` diff can see exactly which +commit and stack produced each number. + +## Re-baselining lifecycle + +`rebaseline.py` is the only writer of the stored state and serves two jobs from +one measurement path: + +1. **Variance study (default).** Run each task `--repeat` times and report robust + stats (median / CV / MAD / min / max) so thresholds can be justified to + reviewers. +2. **Rolling re-baseline (`--apply`).** Append the new samples to the window + (pruned to a cap, stamped with provenance, written into the env bucket) and + refresh the static fallback in `baseline.json`. + +A **boiling-frog guard** keeps a rolling baseline from quietly absorbing a real +regression: a task whose new median drops the baseline by more than +`--soft-drop-pct` is *flagged for review*; a drop beyond `--hard-drop-pct` is +*refused* (old value kept) unless `--force`. Both stores then change only through +the PR that `perf-rebaseline.yml` opens. + +## CI wiring + +- `perf-gate.yml` — on a PR it runs a fast, GPU-free **unit-test job** (comparator, + orchestrator, rebaseline, fingerprint helpers) for early signal, then a + per-task GPU matrix on the L40S fleet that posts the verdict back. Advisory + (`continue-on-error`) until cross-runner variance is confirmed. +- `perf-rebaseline.yml` — manual workflow that runs `rebaseline.py --apply` on the + fleet and opens a PR with the `baseline.json` + `perf_history/` diff. + +## Why these boundaries + +- **Pure-logic comparator** ⇒ the regression rules are fully testable on any + runner; the GPU job only *produces* numbers, it never *decides*. +- **In-tree, reviewed state** ⇒ no opaque external baseline service; every change + to the bar is a diff someone approved. +- **Layered window → static → override** ⇒ robust thresholds when history exists, + a safe floor when it doesn't, and a human escape hatch when neither fits. diff --git a/tools/perf_smoke/README.md b/tools/perf_smoke/README.md index e9fbae08cb2f..117f33c8dfee 100644 --- a/tools/perf_smoke/README.md +++ b/tools/perf_smoke/README.md @@ -42,9 +42,15 @@ mild dip is an advisory **WARN** and only a real, sustained drop is a **BLOCK**. |---|---| | `test_check_perf_regression.py` | Unit tests for the comparator logic (no GPU). | | `test_stress_check_perf_regression.py` | Heavier stress/edge-case tests for the comparator. | +| `test_history_fingerprint.py` | Unit tests for env-fingerprint bucketing + per-sample provenance. | +| `test_run_perf_gate.py` | Unit tests for the orchestrator (config, command building, aggregation). | +| `test_rebaseline.py` | Unit tests for the rebaseline tool (window stats, store writer, boiling-frog guard). | | `test_perf_gate.py` | The pytest entry point CI uses to drive a single task end-to-end. | | `pytest.ini` | Local pytest config for this directory. | +For the architecture (data flow, the baseline/history model, and the verdict +logic), see [`DESIGN.md`](DESIGN.md). + ### CI wiring (in `.github/`) | File | Plain-English purpose | diff --git a/tools/perf_smoke/check_perf_regression.py b/tools/perf_smoke/check_perf_regression.py index e28e441a87cc..5fe64c99f715 100644 --- a/tools/perf_smoke/check_perf_regression.py +++ b/tools/perf_smoke/check_perf_regression.py @@ -55,6 +55,7 @@ import argparse import glob +import hashlib import json import os import sys @@ -366,6 +367,66 @@ def _extract_provenance(result: dict) -> dict[str, object]: return out +def _extract_commit(result: dict) -> str | None: + """Pull the source commit the run was built from (best-effort). + + Reads ``version_info.dev.commit_hash`` (the benchmark backend's dev block) + with a couple of common fallbacks. Returns ``None`` when unavailable. + """ + version = result.get("version_info") + if not isinstance(version, dict): + return None + dev = version.get("dev") + if isinstance(dev, dict): + for key in ("commit_hash", "commit_hash_short", "commit"): + val = dev.get(key) + if isinstance(val, str) and val: + return val + for key in ("commit_hash", "commit"): + val = version.get(key) + if isinstance(val, str) and val: + return val + return None + + +# Provenance keys that define the *environment* (perf regime), and thus the +# history bucket. GPU is already encoded in the file name, so it is excluded. +_FINGERPRINT_KEYS = ("warp", "isaaclab", "cuda") + + +def env_fingerprint(result: dict) -> str | None: + """Compute a short, stable bucket key from a run's environment provenance. + + The fingerprint partitions the rolling-window history so that samples from + incomparable software stacks (e.g. a Warp bump that shifts the perf regime) + never pollute one another's baseline. Returns ``None`` when no provenance is + available, which makes callers fall back to the flat ("default") bucket. + """ + prov = _extract_provenance(result) + parts = {key: prov[key] for key in _FINGERPRINT_KEYS if prov.get(key)} + if not parts: + return None + canonical = json.dumps(parts, sort_keys=True, separators=(",", ":")) + return "env-" + hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:12] + + +def sample_provenance(result: dict) -> dict[str, object]: + """Return the per-sample provenance stamped into each rolling-window record. + + Bundles the environment versions (warp / isaaclab / cuda), the source commit, + and the derived :func:`env_fingerprint` so every stored sample is auditable + and re-bucketable without re-running the benchmark. + """ + prov = _extract_provenance(result) + commit = _extract_commit(result) + if commit: + prov["commit"] = commit + fingerprint = env_fingerprint(result) + if fingerprint: + prov["fingerprint"] = fingerprint + return prov + + def _extract_gpu_name(result: dict) -> str | None: """Read the runner's GPU model name from the result's hardware metadata.""" hw = result.get("hardware_info") @@ -440,6 +501,15 @@ def _resolve_baseline( return matched_key, task_entry, entry +def history_basename(task: str, gpu_key: str) -> str: + """Filesystem-safe ``__`` stem shared by the reader and writers. + + Centralising this keeps the comparator (reader) and rebaseline/seed scripts + (writers) byte-for-byte consistent so bucketed history is always found. + """ + return f"{task}__{gpu_key}".replace("/", "_").replace(" ", "_") + + def _history_window(history_dir: str | None, fingerprint: str | None, task: str, gpu_key: str) -> dict: """Load the rolling-window samples for ``(task, gpu)`` from the history store. @@ -449,7 +519,7 @@ def _history_window(history_dir: str | None, fingerprint: str | None, task: str, """ if not history_dir: return {} - safe = f"{task}__{gpu_key}".replace("/", "_").replace(" ", "_") + safe = history_basename(task, gpu_key) candidates = [] if fingerprint: candidates.append(Path(history_dir) / fingerprint / f"{safe}.json") @@ -544,7 +614,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--baseline", required=True, help="Path to baseline.json (run config + static fallback).") parser.add_argument("--history-dir", default=None, help="Rolling-window store (orphan-branch checkout).") parser.add_argument("--overrides", default=None, help="Path to baseline_overrides.json (committed with the PR).") - parser.add_argument("--fingerprint", default=None, help="History bucket key (git-subtree+deps hash).") + parser.add_argument( + "--fingerprint", + default=None, + help="History bucket key; overrides the env fingerprint auto-derived from the result.", + ) parser.add_argument("--measured-wall-s", type=float, default=None, help="Wall-clock seconds of the run.") parser.add_argument("--results-glob", default=None, help=f"Result glob (defaults to {DEFAULT_GLOB_TEMPLATE!r}).") parser.add_argument("--gpu-override", default=None, help="Override the GPU name read from the result JSON.") @@ -581,7 +655,8 @@ def main(argv: list[str] | None = None) -> int: _emit("PASS", task=args.task, gpu=gpu_key, note="skipped_by_override") return EXIT_PASS - window = _history_window(args.history_dir, args.fingerprint, args.task, gpu_key) + fingerprint = args.fingerprint or env_fingerprint(result) + window = _history_window(args.history_dir, fingerprint, args.task, gpu_key) center, spread, k_warn, k_block, source = _thresholds(window, entry, ov) delta_pct = (measured_fps - center) / center * 100.0 warn_floor = center - k_warn * spread @@ -590,6 +665,7 @@ def main(argv: list[str] | None = None) -> int: common: dict[str, object] = { "task": args.task, "gpu": gpu_key, + "bucket": fingerprint or "flat", "thresholds": source, "center_fps": f"{center:.0f}", "measured_fps": f"{measured_fps:.0f}", diff --git a/tools/perf_smoke/rebaseline.py b/tools/perf_smoke/rebaseline.py index 2e5bb62fe905..f5cc04aa2556 100644 --- a/tools/perf_smoke/rebaseline.py +++ b/tools/perf_smoke/rebaseline.py @@ -72,6 +72,20 @@ def _measure_fps(task_id: str, run_dir: Path, warmup: int, num_frames: int | Non return None +def _measure_provenance(task_id: str, run_dir: Path) -> dict: + """Read the run's per-sample provenance (commit / versions / fingerprint). + + Best-effort: returns ``{}`` when the result is missing or unreadable so a + measurement still contributes its FPS sample to an un-bucketed window. + """ + pattern = cpr.DEFAULT_GLOB_TEMPLATE.format(task=task_id) + try: + result_path = cpr._resolve_results(str(run_dir), pattern, allow_multiple=True) + return cpr.sample_provenance(cpr._load_result(result_path)) + except cpr.CompareError: + return {} + + def _window_stats(samples: list[float]) -> dict | None: """Robust summary of a measurement window. ``None`` when there are no samples.""" if not samples: @@ -138,6 +152,7 @@ def measure_task( num_frames = int(num_frames) if isinstance(num_frames, (int, float)) else None samples: list[float] = [] walls: list[float] = [] + provenance: dict = {} for rep in range(1, repeat + 1): run_dir = out_root / task / f"{tag}rep{rep}" print(f"\n[rebaseline] === {task} {tag}rep {rep}/{repeat} (seed={cfg['seed']}) ===", flush=True) @@ -152,32 +167,55 @@ def measure_task( print(f"[rebaseline] {task} rep {rep}: {fps:.0f} FPS ({wall:.0f}s)", flush=True) samples.append(fps) walls.append(wall) + # The environment is constant across reps; keep the latest non-empty stamp. + prov = _measure_provenance(task_id, run_dir) + if prov: + provenance = prov stats = _window_stats(samples) if stats is not None: stats["walls"] = walls + stats["provenance"] = provenance return stats +# Provenance fields stamped onto every sample, mirroring cpr.sample_provenance. +_SAMPLE_PROVENANCE_KEYS = ("commit", "warp", "isaaclab", "cuda") + + def _append_window(history_dir: Path, task: str, gpu_key: str, stats: dict, cap: int = 20) -> int: """Append this study's samples to the rolling-window store; prune to ``cap``. This is the orphan-branch update in the doc's model: each study contributes - its runs to ``/__.json``, oldest dropped past ``cap``. + its runs to the bucketed store ``//__.json`` + (or the flat ``/__.json`` when the run carries no + environment provenance), oldest dropped past ``cap``. Each sample is stamped + with the commit + env versions so the window stays auditable and the reader + (:func:`check_perf_regression._history_window`) finds the matching bucket. Returns the resulting window length. """ - history_dir.mkdir(parents=True, exist_ok=True) - path = history_dir / f"{task}__{gpu_key}.json".replace(" ", "_") - store = {"task": task, "gpu": gpu_key, "window": cap, "samples": []} + provenance = stats.get("provenance") or {} + fingerprint = provenance.get("fingerprint") + bucket = history_dir / fingerprint if fingerprint else history_dir + bucket.mkdir(parents=True, exist_ok=True) + path = bucket / f"{cpr.history_basename(task, gpu_key)}.json" + store: dict = {"task": task, "gpu": gpu_key, "window": cap, "samples": []} + if fingerprint: + store["fingerprint"] = fingerprint if path.exists(): existing = json.loads(path.read_text(encoding="utf-8")) if isinstance(existing, dict) and isinstance(existing.get("samples"), list): store = existing now = dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds") + stamp = {key: provenance[key] for key in _SAMPLE_PROVENANCE_KEYS if provenance.get(key)} walls = stats.get("walls") or [None] * len(stats["samples"]) for fps, wall in zip(stats["samples"], walls): - store["samples"].append({"fps": round(fps, 1), "wall_s": wall, "ts": now, "source": "rebaseline"}) + sample = {"fps": round(fps, 1), "wall_s": wall, "ts": now, "source": "rebaseline"} + sample.update(stamp) + store["samples"].append(sample) store["samples"] = store["samples"][-cap:] store["window"] = cap + if fingerprint: + store["fingerprint"] = fingerprint path.write_text(json.dumps(store, indent=2) + "\n", encoding="utf-8") return len(store["samples"]) diff --git a/tools/perf_smoke/seed_history.py b/tools/perf_smoke/seed_history.py index 317a1df0e333..7f9cf7cd8623 100644 --- a/tools/perf_smoke/seed_history.py +++ b/tools/perf_smoke/seed_history.py @@ -29,7 +29,7 @@ import os from pathlib import Path -from check_perf_regression import _load_result, steady_fps +from check_perf_regression import _load_result, env_fingerprint, history_basename, sample_provenance, steady_fps _THIS_DIR = Path(__file__).resolve().parent _MATRIX = _THIS_DIR / "exploration_matrix" / "output" @@ -67,6 +67,7 @@ def main() -> int: for task, (cell, warmup) in _TASKS.items(): walls = _wall_by_round(cell) samples = [] + fingerprint: str | None = None for rnd_dir in sorted(glob.glob(str(_MATRIX / cell / "warm_round*"))): run_index = int(os.path.basename(rnd_dir).replace("warm_round", "")) jsons = [p for p in sorted(glob.glob(os.path.join(rnd_dir, "*.json"))) if "meta" not in os.path.basename(p)] @@ -74,14 +75,16 @@ def main() -> int: continue result = _load_result(Path(jsons[-1])) fps = steady_fps(result, warmup, _NUM_FRAMES) - samples.append( - { - "fps": round(fps, 1), - "wall_s": walls.get(run_index), - "source": f"{cell}/{os.path.basename(rnd_dir)}", - "ts": now, - } - ) + prov = sample_provenance(result) + fingerprint = env_fingerprint(result) or fingerprint + sample = { + "fps": round(fps, 1), + "wall_s": walls.get(run_index), + "source": f"{cell}/{os.path.basename(rnd_dir)}", + "ts": now, + } + sample.update({k: prov[k] for k in ("commit", "warp", "isaaclab", "cuda") if prov.get(k)}) + samples.append(sample) store = { "task": task, "gpu": _GPU, @@ -91,7 +94,11 @@ def main() -> int: "_note": "Seeded from L40S warm calibration runs (500f truncated to 300f, post-warm-up).", "samples": samples, } - out_path = _HISTORY_DIR / f"{task}__{_GPU}.json".replace(" ", "_") + if fingerprint: + store["fingerprint"] = fingerprint + bucket = _HISTORY_DIR / fingerprint if fingerprint else _HISTORY_DIR + bucket.mkdir(parents=True, exist_ok=True) + out_path = bucket / f"{history_basename(task, _GPU)}.json" with open(out_path, "w", encoding="utf-8") as f: json.dump(store, f, indent=2) f.write("\n") diff --git a/tools/perf_smoke/test_history_fingerprint.py b/tools/perf_smoke/test_history_fingerprint.py new file mode 100644 index 000000000000..958fe8c6c22a --- /dev/null +++ b/tools/perf_smoke/test_history_fingerprint.py @@ -0,0 +1,153 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the env-fingerprint + per-sample provenance helpers in +``check_perf_regression.py`` and the bucketed history lookup. + +These back the rolling-window bucketing: a run's environment (warp / isaaclab / +cuda) is hashed into a stable bucket key so incomparable software stacks never +share a baseline, and each stored sample is stamped with commit + versions for +auditability. The comparator reads the bucketed file in preference to the flat +("default") one. + +Stdlib ``unittest``; also collectable by pytest via this directory's ``pytest.ini``. +Run directly:: + + python3 tools/perf_smoke/test_history_fingerprint.py +""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import check_perf_regression as cpr # noqa: E402 + + +def _result(warp="1.13.0", isaaclab="6.6.1", cuda="12.4", commit="abc1234"): + """Minimal normalized-result shape carrying version + hardware provenance.""" + version: dict = {} + if warp is not None: + version["warp_version"] = warp + if isaaclab is not None: + version["isaaclab_version"] = isaaclab + if commit is not None: + version["dev"] = {"commit_hash": commit} + hw: dict = {} + if cuda is not None: + hw["cuda_version"] = cuda + out: dict = {} + if version: + out["version_info"] = version + if hw: + out["hardware_info"] = hw + return out + + +class EnvFingerprint(unittest.TestCase): + def test_deterministic_and_prefixed(self): + fp1 = cpr.env_fingerprint(_result()) + fp2 = cpr.env_fingerprint(_result()) + self.assertEqual(fp1, fp2) + assert fp1 is not None + self.assertTrue(fp1.startswith("env-")) + self.assertEqual(len(fp1), len("env-") + 12) + + def test_changes_with_environment(self): + base = cpr.env_fingerprint(_result(warp="1.13.0")) + bumped = cpr.env_fingerprint(_result(warp="1.14.0")) + self.assertNotEqual(base, bumped) + + def test_commit_does_not_affect_fingerprint(self): + # The fingerprint is the *environment*, not the code under test. + a = cpr.env_fingerprint(_result(commit="aaaa")) + b = cpr.env_fingerprint(_result(commit="bbbb")) + self.assertEqual(a, b) + + def test_none_without_provenance(self): + self.assertIsNone(cpr.env_fingerprint({})) + self.assertIsNone(cpr.env_fingerprint(_result(warp=None, isaaclab=None, cuda=None))) + + +class ExtractCommit(unittest.TestCase): + def test_reads_dev_commit_hash(self): + self.assertEqual(cpr._extract_commit(_result(commit="deadbee")), "deadbee") + + def test_fallback_to_top_level_commit(self): + self.assertEqual(cpr._extract_commit({"version_info": {"commit_hash": "top123"}}), "top123") + + def test_none_when_absent(self): + self.assertIsNone(cpr._extract_commit({})) + self.assertIsNone(cpr._extract_commit(_result(commit=None))) + + +class SampleProvenance(unittest.TestCase): + def test_bundles_versions_commit_and_fingerprint(self): + prov = cpr.sample_provenance(_result()) + self.assertEqual(prov["warp"], "1.13.0") + self.assertEqual(prov["isaaclab"], "6.6.1") + self.assertEqual(prov["cuda"], "12.4") + self.assertEqual(prov["commit"], "abc1234") + self.assertEqual(prov["fingerprint"], cpr.env_fingerprint(_result())) + + def test_omits_missing_fields(self): + prov = cpr.sample_provenance(_result(warp=None, cuda=None, commit=None)) + self.assertNotIn("warp", prov) + self.assertNotIn("cuda", prov) + self.assertNotIn("commit", prov) + self.assertEqual(prov["isaaclab"], "6.6.1") + + +class HistoryBasename(unittest.TestCase): + def test_sanitizes_slashes_and_spaces(self): + self.assertEqual(cpr.history_basename("Isaac-Cartpole-v0", "NVIDIA L40S"), "Isaac-Cartpole-v0__NVIDIA_L40S") + self.assertEqual(cpr.history_basename("a/b", "g g"), "a_b__g_g") + + +class HistoryWindowLookup(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.history = Path(self._tmp.name) + self.fp = "env-deadbeef0001" + self.task = "Isaac-Cartpole-v0" + self.gpu = "NVIDIA L40S" + safe = cpr.history_basename(self.task, self.gpu) + self.flat = self.history / f"{safe}.json" + self.bucket = self.history / self.fp / f"{safe}.json" + self.bucket.parent.mkdir(parents=True, exist_ok=True) + self.flat.write_text(json.dumps({"samples": [{"fps": 1.0}], "marker": "flat"}), encoding="utf-8") + self.bucket.write_text(json.dumps({"samples": [{"fps": 2.0}], "marker": "bucket"}), encoding="utf-8") + + def tearDown(self): + self._tmp.cleanup() + + def test_bucket_preferred_when_fingerprint_given(self): + window = cpr._history_window(str(self.history), self.fp, self.task, self.gpu) + self.assertEqual(window["marker"], "bucket") + + def test_flat_fallback_when_bucket_missing(self): + window = cpr._history_window(str(self.history), "env-nonexistent", self.task, self.gpu) + self.assertEqual(window["marker"], "flat") + + def test_flat_used_when_no_fingerprint(self): + window = cpr._history_window(str(self.history), None, self.task, self.gpu) + self.assertEqual(window["marker"], "flat") + + def test_empty_when_nothing_exists(self): + window = cpr._history_window(str(self.history), None, "Other-Task", self.gpu) + self.assertEqual(window, {}) + + def test_empty_when_history_dir_disabled(self): + self.assertEqual(cpr._history_window(None, self.fp, self.task, self.gpu), {}) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tools/perf_smoke/test_rebaseline.py b/tools/perf_smoke/test_rebaseline.py new file mode 100644 index 000000000000..7de9d88571b1 --- /dev/null +++ b/tools/perf_smoke/test_rebaseline.py @@ -0,0 +1,209 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for ``rebaseline.py`` (the variance study / rolling re-baseline tool). + +Covers the pure-logic and store-level surface that needs no GPU: window stats, +the window-stats -> baseline-field mapping, current-baseline lookup, the rolling +window writer (flat vs env-fingerprint bucket, provenance stamping, cap), and the +boiling-frog guard end-to-end via ``main --from-stats --apply``. + +Stdlib ``unittest``; also collectable by pytest via this directory's ``pytest.ini``. +Run directly:: + + python3 tools/perf_smoke/test_rebaseline.py +""" + +from __future__ import annotations + +import io +import json +import os +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import rebaseline # noqa: E402 + + +class WindowStats(unittest.TestCase): + def test_none_for_empty_window(self): + self.assertIsNone(rebaseline._window_stats([])) + + def test_robust_summary_of_known_window(self): + stats = rebaseline._window_stats([100.0, 110.0, 90.0]) + assert stats is not None + self.assertEqual(stats["n"], 3) + self.assertEqual(stats["median"], 100.0) + self.assertEqual(stats["min"], 90.0) + self.assertEqual(stats["max"], 110.0) + self.assertGreater(stats["cv_pct"], 0.0) + + def test_single_sample_reports_zero_spread(self): + stats = rebaseline._window_stats([1234.0]) + assert stats is not None + self.assertEqual(stats["n"], 1) + self.assertEqual(stats["cv_pct"], 0.0) + self.assertEqual(stats["mad"], 0.0) + + +class ProposedEntry(unittest.TestCase): + def test_thresholds_hit_floor_for_low_variance(self): + prop = rebaseline._proposed_entry({"cv_pct": 0.0, "median": 100000.0, "n": 5}) + self.assertEqual(prop["baseline_fps"], 100000.0) + self.assertEqual(prop["warn_pct"], 5.0) + self.assertEqual(prop["max_regression_pct"], 10.0) + self.assertEqual(prop["n_runs"], 5) + + def test_thresholds_scale_with_variance(self): + prop = rebaseline._proposed_entry({"cv_pct": 2.0, "median": 100.0, "n": 3}) + self.assertEqual(prop["warn_pct"], 6.0) # max(3*cv, 5) + self.assertEqual(prop["max_regression_pct"], 12.0) # max(6*cv, 10) + + +class CurrentBaselineFps(unittest.TestCase): + _BASELINE = {"T": {"per_gpu": {"NVIDIA L40S": {"baseline_fps": 4242.0}}}} + + def test_reads_nested_per_gpu_value(self): + self.assertEqual(rebaseline._current_baseline_fps(self._BASELINE, "T", "NVIDIA L40S"), 4242.0) + + def test_none_when_task_or_gpu_absent(self): + self.assertIsNone(rebaseline._current_baseline_fps(self._BASELINE, "T", "NVIDIA A100")) + self.assertIsNone(rebaseline._current_baseline_fps(self._BASELINE, "Other", "NVIDIA L40S")) + + +class AppendWindow(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.history = Path(self._tmp.name) + + def tearDown(self): + self._tmp.cleanup() + + def test_flat_write_when_no_provenance(self): + stats = {"samples": [100.0, 101.0], "walls": [10.0, 11.0]} + n = rebaseline._append_window(self.history, "Isaac-Cartpole-v0", "NVIDIA L40S", stats, cap=20) + self.assertEqual(n, 2) + path = self.history / "Isaac-Cartpole-v0__NVIDIA_L40S.json" + self.assertTrue(path.exists()) + store = json.loads(path.read_text()) + self.assertNotIn("fingerprint", store) + self.assertEqual(store["samples"][0]["fps"], 100.0) + self.assertEqual(store["samples"][0]["source"], "rebaseline") + + def test_bucketed_write_and_provenance_stamp(self): + stats = { + "samples": [100.0, 101.0], + "walls": [10.0, 11.0], + "provenance": { + "fingerprint": "env-deadbeef0001", + "commit": "abc1234", + "warp": "1.13.0", + "isaaclab": "6.6.1", + "cuda": "12.4", + }, + } + rebaseline._append_window(self.history, "Isaac-Cartpole-v0", "NVIDIA L40S", stats, cap=20) + path = self.history / "env-deadbeef0001" / "Isaac-Cartpole-v0__NVIDIA_L40S.json" + self.assertTrue(path.exists(), "bucketed history file should be created under the fingerprint dir") + store = json.loads(path.read_text()) + self.assertEqual(store["fingerprint"], "env-deadbeef0001") + sample = store["samples"][0] + self.assertEqual(sample["commit"], "abc1234") + self.assertEqual(sample["warp"], "1.13.0") + self.assertEqual(sample["isaaclab"], "6.6.1") + self.assertEqual(sample["cuda"], "12.4") + + def test_appends_accumulate_and_prune_to_cap(self): + first = {"samples": [1.0, 2.0], "walls": [1.0, 1.0]} + rebaseline._append_window(self.history, "T", "G", first, cap=3) + second = {"samples": [3.0, 4.0], "walls": [1.0, 1.0]} + n = rebaseline._append_window(self.history, "T", "G", second, cap=3) + self.assertEqual(n, 3) + store = json.loads((self.history / "T__G.json").read_text()) + self.assertEqual([s["fps"] for s in store["samples"]], [2.0, 3.0, 4.0]) + + +class BoilingFrogGuard(unittest.TestCase): + """``main --from-stats --apply``: small drops applied, large drops refused.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + root = Path(self._tmp.name) + self.baseline_path = root / "baseline.json" + self.history = root / "perf_history" + self.stats_path = root / "stats.json" + self.out_dir = root / "out" + baseline = { + "TaskNormal": {"num_envs": 1, "per_gpu": {"NVIDIA L40S": {"baseline_fps": 100000.0}}}, + "TaskRegress": {"num_envs": 1, "per_gpu": {"NVIDIA L40S": {"baseline_fps": 100000.0}}}, + } + self.baseline_path.write_text(json.dumps(baseline), encoding="utf-8") + cached = { + # ~1% drop -> within soft band -> applied. + "TaskNormal": { + "n": 3, "median": 99000.0, "mean": 99000.0, "cv_pct": 0.2, "mad": 50.0, + "min": 98900.0, "max": 99100.0, + "samples": [99000.0, 99050.0, 98950.0], "walls": [10.0, 10.0, 10.0], + }, + # 20% drop -> beyond hard-drop -> refused (old value kept, no window write). + "TaskRegress": { + "n": 1, "median": 80000.0, "mean": 80000.0, "cv_pct": 0.0, "mad": 0.0, + "min": 80000.0, "max": 80000.0, + "samples": [80000.0], "walls": [10.0], + }, + } + self.stats_path.write_text(json.dumps(cached), encoding="utf-8") + + def tearDown(self): + self._tmp.cleanup() + + def _run_apply(self) -> int: + argv = [ + "--baseline", str(self.baseline_path), + "--history-dir", str(self.history), + "--output-dir", str(self.out_dir), + "--from-stats", str(self.stats_path), + "--apply", + ] + with redirect_stdout(io.StringIO()): + return rebaseline.main(argv) + + def test_small_drop_applied_large_drop_refused(self): + self.assertEqual(self._run_apply(), 0) + baseline = json.loads(self.baseline_path.read_text()) + normal = baseline["TaskNormal"]["per_gpu"]["NVIDIA L40S"]["baseline_fps"] + regress = baseline["TaskRegress"]["per_gpu"]["NVIDIA L40S"]["baseline_fps"] + self.assertEqual(normal, 99000.0, "small drop should be written into baseline.json") + self.assertEqual(regress, 100000.0, "hard-limit drop should keep the old baseline value") + + def test_refused_task_window_is_not_written(self): + self._run_apply() + self.assertTrue((self.history / "TaskNormal__NVIDIA_L40S.json").exists()) + self.assertFalse( + (self.history / "TaskRegress__NVIDIA_L40S.json").exists(), + "refused task must not contribute to the rolling window", + ) + + def test_force_applies_even_hard_drop(self): + with redirect_stdout(io.StringIO()): + rc = rebaseline.main([ + "--baseline", str(self.baseline_path), + "--history-dir", str(self.history), + "--output-dir", str(self.out_dir), + "--from-stats", str(self.stats_path), + "--apply", "--force", + ]) + self.assertEqual(rc, 0) + baseline = json.loads(self.baseline_path.read_text()) + self.assertEqual(baseline["TaskRegress"]["per_gpu"]["NVIDIA L40S"]["baseline_fps"], 80000.0) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tools/perf_smoke/test_run_perf_gate.py b/tools/perf_smoke/test_run_perf_gate.py new file mode 100644 index 000000000000..7f44720e250b --- /dev/null +++ b/tools/perf_smoke/test_run_perf_gate.py @@ -0,0 +1,177 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for ``run_perf_gate.py`` (the Phase 1 perf-smoke orchestrator). + +Covers the pure-logic surface that needs no GPU: launch-config resolution from +``baseline.json``, benchmark command construction, the warm-cache env overlay, +comparator ``RESULT=`` label parsing, and the worst-verdict aggregation in +``main`` via ``--dry-run``. + +Stdlib ``unittest``; also collectable by pytest via this directory's ``pytest.ini``. +Run directly:: + + python3 tools/perf_smoke/test_run_perf_gate.py +""" + +from __future__ import annotations + +import io +import json +import os +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import run_perf_gate as gate # noqa: E402 + +_BASELINE = { + "_meta": {"note": "ignored"}, + "Isaac-Cartpole-v0": { + "num_envs": 4096, + "num_frames": 300, + "seed": 42, + "per_gpu": {"NVIDIA L40S": {"baseline_fps": 100000.0}}, + }, + "Isaac-Cartpole-v0@newton": { + "task_id": "Isaac-Cartpole-v0", + "num_envs": 4096, + "benchmark_args": ["presets=newton"], + "per_gpu": {"NVIDIA L40S": {"baseline_fps": 90000.0}}, + }, +} + + +class TaskRunConfig(unittest.TestCase): + def test_defaults_filled_for_absent_fields(self): + cfg = gate._task_run_config(_BASELINE, "Isaac-Cartpole-v0") + self.assertEqual(cfg["task_id"], "Isaac-Cartpole-v0") + self.assertEqual(cfg["num_envs"], 4096) + self.assertEqual(cfg["num_frames"], 300) + self.assertEqual(cfg["seed"], 42) + self.assertEqual(cfg["benchmark_args"], []) + + def test_variant_key_resolves_to_gym_task_id(self): + cfg = gate._task_run_config(_BASELINE, "Isaac-Cartpole-v0@newton") + self.assertEqual(cfg["task_id"], "Isaac-Cartpole-v0") + self.assertEqual(cfg["benchmark_args"], ["presets=newton"]) + # seed/num_frames fall back to calibration defaults when unset. + self.assertEqual(cfg["num_frames"], 300) + self.assertEqual(cfg["seed"], 42) + + def test_missing_task_raises_keyerror(self): + with self.assertRaises(KeyError): + gate._task_run_config(_BASELINE, "Nope") + + +class BenchmarkCmd(unittest.TestCase): + def test_cmd_carries_task_id_frames_seed_and_json_backend(self): + cfg = gate._task_run_config(_BASELINE, "Isaac-Cartpole-v0") + cmd = gate._benchmark_cmd("Isaac-Cartpole-v0", cfg, Path("/tmp/out")) + self.assertIn("--task", cmd) + self.assertEqual(cmd[cmd.index("--task") + 1], "Isaac-Cartpole-v0") + self.assertEqual(cmd[cmd.index("--num_frames") + 1], "300") + self.assertEqual(cmd[cmd.index("--seed") + 1], "42") + self.assertEqual(cmd[cmd.index("--num_envs") + 1], "4096") + self.assertEqual(cmd[cmd.index("--benchmark_backend") + 1], "json") + self.assertEqual(cmd[cmd.index("--output_path") + 1], "/tmp/out") + self.assertIn("--headless", cmd) + + def test_variant_launches_real_gym_id_with_benchmark_args(self): + cfg = gate._task_run_config(_BASELINE, "Isaac-Cartpole-v0@newton") + cmd = gate._benchmark_cmd("Isaac-Cartpole-v0@newton", cfg, Path("/tmp/out")) + self.assertEqual(cmd[cmd.index("--task") + 1], "Isaac-Cartpole-v0") + self.assertIn("presets=newton", cmd) + + def test_num_envs_omitted_when_absent(self): + cfg = {"task_id": "T", "num_envs": None, "num_frames": 10, "seed": 1, "benchmark_args": []} + cmd = gate._benchmark_cmd("T", cfg, Path("/tmp/out")) + self.assertNotIn("--num_envs", cmd) + + +class CacheEnv(unittest.TestCase): + def test_none_when_no_cache_dir(self): + self.assertIsNone(gate._cache_env(None)) + self.assertIsNone(gate._cache_env("")) + + def test_sets_warp_and_cuda_paths_and_creates_dirs(self): + with tempfile.TemporaryDirectory() as tmp: + env = gate._cache_env(tmp) + self.assertIsNotNone(env) + assert env is not None # narrow for type-checkers + warp = Path(env["WARP_CACHE_PATH"]) + cuda = Path(env["CUDA_CACHE_PATH"]) + self.assertTrue(warp.is_dir()) + self.assertTrue(cuda.is_dir()) + # The overlay preserves the ambient environment. + self.assertIn("PATH", env) + + +class _FakeProc: + def __init__(self, returncode: int, stdout: str = "", stderr: str = ""): + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + +class RunComparator(unittest.TestCase): + def test_parses_result_token_over_exit_code(self): + # WARN exits 0 but must be distinguished from a clean PASS. + fake = _FakeProc(0, stdout="RESULT=WARN task=T delta_pct=-4.0\n") + with mock.patch.object(gate.subprocess, "run", return_value=fake): + with redirect_stdout(io.StringIO()): + code, label = gate._run_comparator("T", Path("/tmp"), Path("/tmp/baseline.json"), None) + self.assertEqual(code, 0) + self.assertEqual(label, "WARN") + + def test_falls_back_to_verdict_name_without_result_token(self): + fake = _FakeProc(gate.EXIT_HARD_FAILURE, stderr="boom, no token here\n") + with mock.patch.object(gate.subprocess, "run", return_value=fake): + with redirect_stdout(io.StringIO()): + code, label = gate._run_comparator("T", Path("/tmp"), Path("/tmp/baseline.json"), None) + self.assertEqual(code, gate.EXIT_HARD_FAILURE) + self.assertEqual(label, "HARD_FAILURE") + + +class MainAggregation(unittest.TestCase): + """Exercise ``main`` end-to-end in ``--dry-run`` (no GPU, no subprocess).""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.baseline_path = Path(self._tmp.name) / "baseline.json" + self.baseline_path.write_text(json.dumps(_BASELINE), encoding="utf-8") + self.out_dir = Path(self._tmp.name) / "out" + + def tearDown(self): + self._tmp.cleanup() + + def _run(self, tasks: list[str]) -> int: + argv = [ + "--tasks", *tasks, + "--baseline", str(self.baseline_path), + "--output-dir", str(self.out_dir), + "--dry-run", + ] + with redirect_stdout(io.StringIO()): + return gate.main(argv) + + def test_all_valid_tasks_pass(self): + self.assertEqual(self._run(["Isaac-Cartpole-v0"]), gate.EXIT_PASS) + + def test_missing_task_is_hard_failure(self): + self.assertEqual(self._run(["Nope"]), gate.EXIT_HARD_FAILURE) + + def test_worst_verdict_wins_in_mixed_run(self): + # One valid (PASS) + one missing (HARD_FAILURE) -> overall HARD_FAILURE. + self.assertEqual(self._run(["Isaac-Cartpole-v0", "Nope"]), gate.EXIT_HARD_FAILURE) + + +if __name__ == "__main__": + unittest.main(verbosity=2)