From 47bfc79d28eceb15b54d04f0bc3f557d0dc95801 Mon Sep 17 00:00:00 2001 From: Angelina Hu Date: Wed, 10 Jun 2026 16:37:30 -0700 Subject: [PATCH 1/2] CI performance regresion gate local POC working --- .github/workflows/perf-regression-gate.yaml | 408 +++++++++++ tools/perf_regression_gate/.gitignore | 6 + tools/perf_regression_gate/aggregate.py | 234 ++++++ .../perf_regression_gate/baseline_manager.py | 210 ++++++ .../build_bench_result.py | 333 +++++++++ .../dev/sim_regression.py | 149 ++++ .../dev/stub_benchmark.py | 70 ++ .../docs/module-interfaces.md | 665 ++++++++++++++++++ .../docs/system-design.md | 446 ++++++++++++ tools/perf_regression_gate/gate_config.py | 10 + tools/perf_regression_gate/local_runner.py | 321 +++++++++ tools/perf_regression_gate/oracle.py | 265 +++++++ tools/perf_regression_gate/task_config.py | 180 +++++ tools/perf_regression_gate/tasks.json | 77 ++ .../tasks_to_ci_matrix.py | 37 + tools/subprocess_runner.py | 324 +++++++++ 16 files changed, 3735 insertions(+) create mode 100644 .github/workflows/perf-regression-gate.yaml create mode 100644 tools/perf_regression_gate/.gitignore create mode 100644 tools/perf_regression_gate/aggregate.py create mode 100644 tools/perf_regression_gate/baseline_manager.py create mode 100644 tools/perf_regression_gate/build_bench_result.py create mode 100644 tools/perf_regression_gate/dev/sim_regression.py create mode 100644 tools/perf_regression_gate/dev/stub_benchmark.py create mode 100644 tools/perf_regression_gate/docs/module-interfaces.md create mode 100644 tools/perf_regression_gate/docs/system-design.md create mode 100644 tools/perf_regression_gate/gate_config.py create mode 100644 tools/perf_regression_gate/local_runner.py create mode 100644 tools/perf_regression_gate/oracle.py create mode 100644 tools/perf_regression_gate/task_config.py create mode 100644 tools/perf_regression_gate/tasks.json create mode 100644 tools/perf_regression_gate/tasks_to_ci_matrix.py create mode 100644 tools/subprocess_runner.py diff --git a/.github/workflows/perf-regression-gate.yaml b/.github/workflows/perf-regression-gate.yaml new file mode 100644 index 000000000000..a623401b339b --- /dev/null +++ b/.github/workflows/perf-regression-gate.yaml @@ -0,0 +1,408 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Performance Regression Gate +# +# Runs a matrix of benchmark tasks on self-hosted runners, compares +# results to the rolling baseline in angehu/perf-baselines, and posts a verdict +# table to the job summary. Non-blocking by default (gate_config.json). +# +# Architecture: +# config → load isaacsim version from config.yaml (ubuntu-latest) +# bench → per-task/backend matrix, each pulling the CI image and running +# benchmark_non_rl.py inside Docker (self-hosted, gpu) +# aggregate → download all bench artifacts, run oracle, update baselines +# (self-hosted, gpu) +# +# Baseline updates are pushed to angehu/perf-baselines only from main, +# develop, release or angehu/perf-gate-poc branches. +# +# backend_key: "{physics_backend}_{render_backend}" when render_backend is +# set, otherwise just physics_backend. Matches TaskConfig.backend_key. +# TODO: replace angehu/perf-baselines w/ real branch name + +name: Performance Regression Gate + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: [main, develop, 'release/**'] + workflow_dispatch: + +concurrency: + group: perf-gate-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: write + +env: + CI_IMAGE_TAG: >- + isaac-lab-ci:${{ + github.event_name == 'pull_request' + && format('pr-{0}', github.event.pull_request.number) + || github.ref_name + }}-${{ github.sha }} + +jobs: + # --------------------------------------------------------------------------- + # Load shared image config (matches build.yaml pattern) + # --------------------------------------------------------------------------- + config: + name: Load Config + runs-on: ubuntu-latest + outputs: + isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }} + isaacsim_image_tag: ${{ steps.load.outputs.isaacsim_image_tag }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + sparse-checkout: + - .github/workflows/config.yaml + - tools/perf_regression_gate/tasks.json + - tools/perf_regression_gate/task_config.py + sparse-checkout-cone-mode: false + - id: load + run: | + set -euo pipefail + f=.github/workflows/config.yaml + echo "isaacsim_image_name=$(yq -r .isaacsim_image_name "$f")" >> "$GITHUB_OUTPUT" + echo "isaacsim_image_tag=$(yq -r .isaacsim_image_tag "$f")" >> "$GITHUB_OUTPUT" + - id: build_matrix + run: | + set -euo pipefail + MATRIX=$(python3 tools/perf_regression_gate/tasks_to_ci_matrix.py) + echo "bench_matrix=$MATRIX" >> "$GITHUB_OUTPUT" + + # --------------------------------------------------------------------------- + # Per-task/backend benchmark jobs + # --------------------------------------------------------------------------- + bench: + name: Bench / ${{ matrix.task_id }} / ${{ matrix.physics_backend }}${{ matrix.render_backend != '' && format('_{0}', matrix.render_backend) || '' }} + runs-on: [self-hosted, gpu] + needs: [config] + continue-on-error: true + timeout-minutes: ${{ matrix.job_timeout_minutes }} + strategy: + fail-fast: false + matrix: + # Load the benchmark matrix from the canonical tasks.json task definition file. + include: ${{ fromJson(needs.config.outputs.bench_matrix) }} + + steps: + - name: Checkout Code + uses: actions/checkout@v6 + with: + fetch-depth: 1 + lfs: true + + # Pull the CI image built (or cached) by the build workflow for this commit. + # ecr-build-push-pull will build the image if it is not already in ECR. + - name: Pull CI image + uses: ./.github/actions/ecr-build-push-pull + with: + image-tag: ${{ env.CI_IMAGE_TAG }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + + - name: Run benchmark in Docker + id: bench_run + run: | + set -euo pipefail + TASK_ID="${{ matrix.task_id }}" + PHYSICS_BACKEND="${{ matrix.physics_backend }}" + RENDER_BACKEND="${{ matrix.render_backend }}" + + # Compute backend_key: "{physics}_{render}" when render is set, else "{physics}" + if [ -n "${RENDER_BACKEND}" ]; then + BACKEND_KEY="${PHYSICS_BACKEND}_${RENDER_BACKEND}" + else + BACKEND_KEY="${PHYSICS_BACKEND}" + fi + + ARTIFACT_DIR="$(pwd)/artifacts/${TASK_ID}/${BACKEND_KEY}" + + # Sanitize task_id and backend_key for Docker container name + SAFE_TASK_ID="${TASK_ID//[^a-zA-Z0-9]/-}" + SAFE_BACKEND="${BACKEND_KEY//[^a-zA-Z0-9]/-}" + CONTAINER_NAME="perf-bench-${SAFE_TASK_ID}-${SAFE_BACKEND}-${{ github.run_id }}" + LOG_FILE="${ARTIFACT_DIR}/benchmark.log" + + mkdir -p "${ARTIFACT_DIR}" + + # Remove any stale container from a previous run attempt + docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + + # Build optional render backend preset token for the CLI. + # benchmark_non_rl.py accepts Hydra-style preset tokens (no leading dashes). + # Physics is selected via the sim_backend token; render via renderer= token. + # TODO: Update these tokens once benchmark_non_rl.py CLI flags are finalised. + PRESET_TOKENS="" + if [ -n "${RENDER_BACKEND}" ]; then + PRESET_TOKENS="renderer=${RENDER_BACKEND}" + fi + + # Start the benchmark container in detached mode + docker run -d --name "${CONTAINER_NAME}" \ + --init --stop-timeout 10 \ + --entrypoint bash --gpus all --network=host \ + --security-opt=no-new-privileges:true \ + --ulimit nofile=65536:65536 \ + --ulimit nproc=4096:4096 \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e PYTHONUNBUFFERED=1 \ + -v "${ARTIFACT_DIR}:/tmp/bench_out" \ + "${{ env.CI_IMAGE_TAG }}" \ + -c " + set -e + cd /workspace/isaaclab + rm -f _isaac_sim + ln -s /isaac-sim _isaac_sim + ./isaaclab.sh -p scripts/benchmarks/benchmark_non_rl.py \ + --task '${TASK_ID}' \ + --num_envs ${{ matrix.num_envs }} \ + --num_frames ${{ matrix.num_frames }} \ + --benchmark_backend json \ + --output_path /tmp/bench_out \ + ${PRESET_TOKENS} + " + + START=$(date +%s) + + # Stream container logs to file and terminal while benchmark runs + docker logs -f "${CONTAINER_NAME}" 2>&1 | tee "${LOG_FILE}" & + LOGS_PID=$! + + # Wait for the container to exit, with a hard wall-clock timeout + BENCH_EXIT=1 + if docker_exit=$(timeout ${{ matrix.bench_timeout_s }} docker wait "${CONTAINER_NAME}" 2>/dev/null); then + BENCH_EXIT="${docker_exit:-1}" + else + echo "::warning::Benchmark for ${TASK_ID}/${BACKEND_KEY} timed out after ${{ matrix.bench_timeout_s }}s" + fi + + END=$(date +%s) + WALL_TIME_S=$((END - START)) + + kill "${LOGS_PID}" 2>/dev/null || true + wait "${LOGS_PID}" 2>/dev/null || true + + docker kill "${CONTAINER_NAME}" 2>/dev/null || true + docker rm "${CONTAINER_NAME}" 2>/dev/null || true + + echo "exit_code=${BENCH_EXIT}" >> "$GITHUB_OUTPUT" + echo "wall_time_s=${WALL_TIME_S}" >> "$GITHUB_OUTPUT" + echo "artifact_dir=${ARTIFACT_DIR}" >> "$GITHUB_OUTPUT" + echo "backend_key=${BACKEND_KEY}" >> "$GITHUB_OUTPUT" + + # Retry once on first-attempt failure before giving up. + # Runs only when bench_run reports a non-zero exit code. + - name: Retry benchmark on failure + id: bench_retry + if: always() && steps.bench_run.outputs.exit_code != '0' && steps.bench_run.outputs.exit_code != '' + run: | + set -uo pipefail + TASK_ID="${{ matrix.task_id }}" + PHYSICS_BACKEND="${{ matrix.physics_backend }}" + RENDER_BACKEND="${{ matrix.render_backend }}" + + if [ -n "${RENDER_BACKEND}" ]; then + BACKEND_KEY="${PHYSICS_BACKEND}_${RENDER_BACKEND}" + else + BACKEND_KEY="${PHYSICS_BACKEND}" + fi + + ARTIFACT_DIR="$(pwd)/artifacts/${TASK_ID}/${BACKEND_KEY}" + SAFE_TASK_ID="${TASK_ID//[^a-zA-Z0-9]/-}" + SAFE_BACKEND="${BACKEND_KEY//[^a-zA-Z0-9]/-}" + CONTAINER_NAME="perf-bench-${SAFE_TASK_ID}-${SAFE_BACKEND}-${{ github.run_id }}-retry" + LOG_FILE="${ARTIFACT_DIR}/benchmark.log" + + echo "::notice::First attempt failed (exit=${{ steps.bench_run.outputs.exit_code }}); retrying benchmark for ${TASK_ID}/${BACKEND_KEY}" + + # Remove any partial perf output from the failed first attempt so the retry's + # output is picked up cleanly by build_bench_result's normalize step. + rm -f "${ARTIFACT_DIR}/perf_regression_gate_info.json" + + docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + + PRESET_TOKENS="" + if [ -n "${RENDER_BACKEND}" ]; then + PRESET_TOKENS="renderer=${RENDER_BACKEND}" + fi + + docker run -d --name "${CONTAINER_NAME}" \ + --init --stop-timeout 10 \ + --entrypoint bash --gpus all --network=host \ + --security-opt=no-new-privileges:true \ + --ulimit nofile=65536:65536 \ + --ulimit nproc=4096:4096 \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e PYTHONUNBUFFERED=1 \ + -v "${ARTIFACT_DIR}:/tmp/bench_out" \ + "${{ env.CI_IMAGE_TAG }}" \ + -c " + set -e + cd /workspace/isaaclab + rm -f _isaac_sim + ln -s /isaac-sim _isaac_sim + ./isaaclab.sh -p scripts/benchmarks/benchmark_non_rl.py \ + --task '${TASK_ID}' \ + --num_envs ${{ matrix.num_envs }} \ + --num_frames ${{ matrix.num_frames }} \ + --benchmark_backend json \ + --output_path /tmp/bench_out \ + ${PRESET_TOKENS} + " + + START=$(date +%s) + + docker logs -f "${CONTAINER_NAME}" 2>&1 | tee "${LOG_FILE}" & + LOGS_PID=$! + + BENCH_EXIT=1 + if docker_exit=$(timeout ${{ matrix.bench_timeout_s }} docker wait "${CONTAINER_NAME}" 2>/dev/null); then + BENCH_EXIT="${docker_exit:-1}" + else + echo "::warning::Retry benchmark for ${TASK_ID}/${BACKEND_KEY} timed out after ${{ matrix.bench_timeout_s }}s" + fi + + END=$(date +%s) + WALL_TIME_S=$((END - START)) + + kill "${LOGS_PID}" 2>/dev/null || true + wait "${LOGS_PID}" 2>/dev/null || true + + docker kill "${CONTAINER_NAME}" 2>/dev/null || true + docker rm "${CONTAINER_NAME}" 2>/dev/null || true + + echo "exit_code=${BENCH_EXIT}" >> "$GITHUB_OUTPUT" + echo "wall_time_s=${WALL_TIME_S}" >> "$GITHUB_OUTPUT" + + # Build bench_result.json from the Docker run outputs. + # Runs even when bench_run/bench_retry fail so partial results are always uploaded. + # Uses the retry's exit code and wall time when a retry ran, and sets was_retried accordingly. + - name: Build bench_result + if: always() + run: | + TASK_ID="${{ matrix.task_id }}" + PHYSICS_BACKEND="${{ matrix.physics_backend }}" + RENDER_BACKEND="${{ matrix.render_backend }}" + if [ -n "${RENDER_BACKEND}" ]; then + BACKEND_KEY="${PHYSICS_BACKEND}_${RENDER_BACKEND}" + else + BACKEND_KEY="${PHYSICS_BACKEND}" + fi + ARTIFACT_DIR="$(pwd)/artifacts/${TASK_ID}/${BACKEND_KEY}" + + # If a retry ran, use its exit code/wall time and mark was_retried. + RETRY_EXIT="${{ steps.bench_retry.outputs.exit_code }}" + if [ -n "${RETRY_EXIT}" ]; then + FINAL_EXIT="${RETRY_EXIT}" + FINAL_WALL="${{ steps.bench_retry.outputs.wall_time_s || '0' }}" + EXTRA_FLAGS="--was_retried --attempt 2" + else + FINAL_EXIT="${{ steps.bench_run.outputs.exit_code || '1' }}" + FINAL_WALL="${{ steps.bench_run.outputs.wall_time_s || '0' }}" + EXTRA_FLAGS="" + fi + + python3 tools/perf_regression_gate/build_bench_result.py \ + --task_id "${TASK_ID}" \ + --physics_backend "${PHYSICS_BACKEND}" \ + --render_backend "${RENDER_BACKEND}" \ + --artifact_dir "${ARTIFACT_DIR}" \ + --exit_code "${FINAL_EXIT}" \ + --wall_time_s "${FINAL_WALL}" \ + --timeout_s "${{ matrix.bench_timeout_s }}" \ + --log_file "${ARTIFACT_DIR}/benchmark.log" \ + ${EXTRA_FLAGS} + + - name: Upload bench artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: bench-${{ matrix.task_id }}-${{ matrix.physics_backend }}${{ matrix.render_backend != '' && format('_{0}', matrix.render_backend) || '' }}-${{ github.run_id }} + path: artifacts/${{ matrix.task_id }}/${{ matrix.physics_backend }}${{ matrix.render_backend != '' && format('_{0}', matrix.render_backend) || '' }}/ + retention-days: 7 + if-no-files-found: warn + + # Force-clean the container if the job is cancelled mid-run + - name: Cleanup container on cancellation + if: cancelled() + run: | + TASK_ID="${{ matrix.task_id }}" + PHYSICS_BACKEND="${{ matrix.physics_backend }}" + RENDER_BACKEND="${{ matrix.render_backend }}" + if [ -n "${RENDER_BACKEND}" ]; then + BACKEND_KEY="${PHYSICS_BACKEND}_${RENDER_BACKEND}" + else + BACKEND_KEY="${PHYSICS_BACKEND}" + fi + SAFE_TASK_ID="${TASK_ID//[^a-zA-Z0-9]/-}" + SAFE_BACKEND="${BACKEND_KEY//[^a-zA-Z0-9]/-}" + CONTAINER_NAME="perf-bench-${SAFE_TASK_ID}-${SAFE_BACKEND}-${{ github.run_id }}" + docker kill "${CONTAINER_NAME}" 2>/dev/null || true + docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + + # --------------------------------------------------------------------------- + # Aggregate: oracle verdicts, baseline update, step summary + # --------------------------------------------------------------------------- + aggregate: + name: Aggregate + Verdict + runs-on: [self-hosted, gpu] + needs: [bench] + # Always run even when some bench jobs fail or are skipped + if: always() + + steps: + - name: Checkout Code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + # Create a local branch so _baseline_worktree can commit to it + - name: Fetch baselines branch + run: | + git fetch origin angehu/perf-baselines:angehu/perf-baselines || \ + echo "::warning::Baseline branch not found; gate will run as a seed run (no baseline comparison)" + + - name: Download bench artifacts + uses: actions/download-artifact@v4 + with: + pattern: bench-*-${{ github.run_id }} + path: artifacts/ + merge-multiple: false + + - name: Run aggregate oracle + id: aggregate + run: | + ALLOW_UPDATE="false" + if [[ "${{ github.ref }}" == "refs/heads/develop" || \ + "${{ github.ref }}" == "refs/heads/angehu/perf-gate-poc" ]]; then + ALLOW_UPDATE="true" + fi + python3 tools/perf_regression_gate/aggregate.py \ + --artifacts_dir artifacts/ \ + --gpu_model L40S \ + --gate_config tools/perf_regression_gate/gate_config.json \ + --baseline_branch angehu/perf-baselines \ + --allow_baseline_update "${ALLOW_UPDATE}" \ + --summary_file "${GITHUB_STEP_SUMMARY}" + + # Push baseline updates only from eligible branches + - name: Push baseline updates + if: steps.aggregate.outputs.baselines_updated == 'true' + run: | + git push origin angehu/perf-baselines diff --git a/tools/perf_regression_gate/.gitignore b/tools/perf_regression_gate/.gitignore new file mode 100644 index 000000000000..84ec617f93d3 --- /dev/null +++ b/tools/perf_regression_gate/.gitignore @@ -0,0 +1,6 @@ +# Generated at runtime — not source +artifacts/ +local_baselines/ +__pycache__/ +*.pyc +.pytest_cache/ diff --git a/tools/perf_regression_gate/aggregate.py b/tools/perf_regression_gate/aggregate.py new file mode 100644 index 000000000000..3e5f7ae1111f --- /dev/null +++ b/tools/perf_regression_gate/aggregate.py @@ -0,0 +1,234 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""CI aggregate script: load per-task perf_regression_gate_result.json, run oracle, update baselines. + +Scans the local (downloaded) artifacts directory for perf_regression_gate_result.json files, runs the +oracle for each, writes a GitHub Step Summary table, and optionally updates the baselines branch. + +Usage:: + + python3 tools/perf_regression_gate/aggregate.py \\ + --artifacts_dir artifacts/ \\ + --gpu_model L40S \\ + --gate_config tools/perf_regression_gate/gate_config.json \\ + --baseline_branch angehu/perf-baselines \\ + --allow_baseline_update true \\ + --summary_file "$GITHUB_STEP_SUMMARY" + +For offline/test use, pass ``--baselines_dir`` to read/write flat files instead +of the git baselines branch:: + + python3 tools/perf_regression_gate/aggregate.py \\ + --artifacts_dir artifacts/ \\ + --gpu_model L40S \\ + --baselines_dir local_baselines/ +""" + +import argparse +import json +import os +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +_TOOLS_DIR = _MODULE_DIR.parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) +DEFAULT_BASELINE_BRANCH = "angehu/perf-baselines" # TODO: replace w/ real branch name + +from oracle import OracleVerdict, compare # noqa: E402 +from baseline_manager import load_baseline, load_baseline_git, update_baseline, update_baseline_git # noqa: E402 +from gate_config import load_gate_config # noqa: E402 +from task_config import get_task # noqa: E402 + + +def _parse_args(): + p = argparse.ArgumentParser(description="Aggregate bench results and run oracle.") + p.add_argument( + "--artifacts_dir", + required=True, + type=Path, + help="Root directory containing per-task artifact subdirectories", + ) + p.add_argument("--gpu_model", default="L40S") + p.add_argument("--gate_config", type=Path, default=_MODULE_DIR / "gate_config.json") + p.add_argument( + "--baseline_branch", + default=f"{DEFAULT_BASELINE_BRANCH}", + help=f"Git branch for baseline storage (default: {DEFAULT_BASELINE_BRANCH})", + ) + p.add_argument( + "--baselines_dir", + type=Path, + default=None, + help="Flat-file baseline directory; bypasses git (use for offline testing)", + ) + p.add_argument( + "--allow_baseline_update", + default="false", + help="Update baselines for PASS/WARN results ('true'/'false', default: false)", + ) + p.add_argument( + "--summary_file", + default=None, + help="Append step-summary markdown to this path (set to $GITHUB_STEP_SUMMARY in CI)", + ) + return p.parse_args() + + +def _find_bench_results(artifacts_dir: Path) -> list[tuple[Path, dict]]: + """Return list of (artifact_dir: Path, perf_regression_gate_result: dict) sorted by task_id.""" + found = [] + for p in sorted(artifacts_dir.rglob("perf_regression_gate_result.json")): + with p.open() as fh: + bench_result = json.load(fh) + found.append((p.parent, bench_result)) + return found + + +def _excluded_frames(bench_result: dict) -> frozenset: + """Expand excluded_frames_raw from the task_config_snapshot into a frozenset.""" + raw = (bench_result.get("task_config_snapshot") or {}).get("excluded_frames_raw", []) + indices: set[int] = set() + for entry in raw: + if isinstance(entry, list): + indices.update(range(entry[0], entry[1] + 1)) + else: + indices.add(int(entry)) + return frozenset(indices) + + +def _fmt(v, decimals: int = 1) -> str: + return f"{v:.{decimals}f}" if v is not None else "N/A" + + +def _build_summary_table(rows: list) -> str: + lines = [ + "| Task | Backend | Verdict | FPS (mean) | Baseline | Regression% | Wall (s) |", + "|------|---------|---------|------------|----------|-------------|----------|", + ] + for r in rows: + lines.append( + f"| {r.task_id} | {r.backend} | {r.verdict.value}" + f" | {_fmt(r.measured_fps)} | {_fmt(r.baseline_fps)}" + f" | {_fmt(r.regression_pct, 2)} | {_fmt(r.wall_time_s, 1)} |" + ) + return "\n".join(lines) + + +def main() -> int: + args = _parse_args() + use_flat = args.baselines_dir is not None + allow_update = args.allow_baseline_update.strip().lower() in ("true", "1", "yes") + + gate_config = load_gate_config(args.gate_config) + blocking = gate_config.get("blocking", False) + + items = _find_bench_results(args.artifacts_dir) + if not items: + print(f"[aggregate] No perf_regression_gate_result.json files found under {args.artifacts_dir}") + return 1 + + oracle_results = [] + has_block = False + has_hard_failure = False + baselines_updated = False + + for artifact_dir, bench_result in items: + task_id = bench_result["task_id"] + backend = bench_result.get("backend_key") + + # Load rolling baseline (None on seed passes/failed load) + baseline = None + try: + if use_flat: + baseline = load_baseline(args.baselines_dir, args.gpu_model, task_id, backend) + else: + baseline = load_baseline_git(args.baseline_branch, args.gpu_model, task_id, backend, None) + except Exception as exc: + print(f"[aggregate] Warning: baseline load failed for {task_id}/{backend}: {exc}") + + # Hard-floor FPS for this task (0.0 = disabled) + try: + task = get_task(task_id, backend) + fps_mean_floor = task.fps_mean_floor.get(args.gpu_model, {}).get(backend, 0.0) + except Exception: + fps_mean_floor = 0.0 + + oracle_result = compare( + bench_result=bench_result, + baseline=baseline, + fps_mean_floor=fps_mean_floor, + excluded_frames=_excluded_frames(bench_result), + artifact_dir=artifact_dir, + ) + oracle_results.append(oracle_result) + + print( + f"[aggregate] {task_id}/{backend}: {oracle_result.verdict.value}" + f" fps={_fmt(oracle_result.measured_fps)} baseline={_fmt(oracle_result.baseline_fps)}" + ) + + if oracle_result.verdict == OracleVerdict.BLOCK: + has_block = True + elif oracle_result.verdict == OracleVerdict.HARD_FAILURE: + has_hard_failure = True + + # Update baseline only for measured PASS/WARN results (never for BLOCK/HARD_FAILURE) pushed + # to designated branches or force update with flag for offline/testing use. + if ( + allow_update + and oracle_result.verdict in (OracleVerdict.PASS, OracleVerdict.WARN) + and oracle_result.measured_fps is not None + ): + try: + if use_flat: + update_baseline(args.baselines_dir, args.gpu_model, task_id, backend, oracle_result.measured_fps) + else: + update_baseline_git( + args.baseline_branch, + args.gpu_model, + task_id, + backend, + oracle_result.measured_fps, + None, + ) + baselines_updated = True + print(f"[aggregate] -> baseline updated: {oracle_result.measured_fps:.1f} FPS") + except Exception as exc: + print(f"[aggregate] Warning: baseline update failed for {task_id}/{backend}: {exc}") + + table = _build_summary_table(oracle_results) + print("\n## Performance Gate Results\n") + print(table) + print() + + if args.summary_file: + with open(args.summary_file, "a") as fh: + fh.write("\n## Performance Gate Results\n\n") + fh.write(table) + fh.write("\n") + + # Signal baseline push to the calling workflow step + if baselines_updated: + github_output = os.environ.get("GITHUB_OUTPUT", "") + if github_output: + with open(github_output, "a") as fh: + fh.write("baselines_updated=true\n") + print(f"[aggregate] Baselines updated; workflow will push {DEFAULT_BASELINE_BRANCH}") + + if blocking: # from gate_config.py, explicit PR to make gate blocking + if has_block: + return 1 + if has_hard_failure: + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_regression_gate/baseline_manager.py b/tools/perf_regression_gate/baseline_manager.py new file mode 100644 index 000000000000..d6da10ea85df --- /dev/null +++ b/tools/perf_regression_gate/baseline_manager.py @@ -0,0 +1,210 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import contextlib +import json +import shutil +import statistics +import subprocess +import tempfile +from pathlib import Path + +from oracle import Baseline # noqa: E402 + +DEFAULT_K_WARN = 2.5 # TODO: decide on real threshold values +DEFAULT_K_BLOCK = 4.0 # TODO: decide on real threshold values + + +def _stats_path(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> Path: + if fingerprint is None: + return baselines_dir / gpu_model / task_id / backend / "stats.json" + return baselines_dir / gpu_model / task_id / backend / fingerprint / "stats.json" + + +def _window_path(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> Path: + if fingerprint is None: + return baselines_dir / gpu_model / task_id / backend / "window.ndjson" + return baselines_dir / gpu_model / task_id / backend / fingerprint / "window.ndjson" + + +def load_baseline(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> Baseline | None: + """Load stats.json for a task/backend pair, or return None if it does not exist""" + sp = _stats_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + if not sp.exists(): + return None + with sp.open() as fh: + d = json.load(fh) + return Baseline( + median_fps=d["median_fps"], + mad_fps=d["mad_fps"], + k_warn=d.get("k_warn", DEFAULT_K_WARN), + k_block=d.get("k_block", DEFAULT_K_BLOCK), + sample_count=d.get("sample_count", 0), + ) + + +def update_baseline(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fps: float, fingerprint=None) -> None: + """Append fps to window.ndjson, recompute stats, and write stats.json""" + wp = _window_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + sp = _stats_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + wp.parent.mkdir(parents=True, exist_ok=True) + + fps_window: list[float] = [] + if wp.exists(): + with wp.open() as fh: + for line in fh: + line = line.strip() + if line: + fps_window.append(float(line)) + + fps_window.append(fps) + + with wp.open("a") as fh: + fh.write(f"{fps}\n") + + median = statistics.median(fps_window) + deviations = [abs(v - median) for v in fps_window] + mad = statistics.median(deviations) if len(deviations) > 1 else 0.0 + + stats = { + "median_fps": median, + "mad_fps": mad, + "k_warn": DEFAULT_K_WARN, + "k_block": DEFAULT_K_BLOCK, + "sample_count": len(fps_window), + } + with sp.open("w") as fh: + json.dump(stats, fh, indent=2) + + +def delete_baseline_files(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> None: + """Delete stats.json and window.ndjson for a task/backend pair""" + for p in ( + _stats_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint), + _window_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint), + ): + if p.exists(): + p.unlink() + + +def seed_baseline_with_spread( + baselines_dir: Path, + gpu_model: str, + task_id: str, + backend: str, + center_fps: float, + noise_fps: float = 5.0, + n_samples: int = 10, + seed: int = 0, + fingerprint=None, +) -> None: + """Populate the baseline window with n_samples of varied FPS data around center_fps and compute stats.json. + For testing tasks/backends with no existing baseline or when a deterministic baseline is needed.""" + import random as _random + + rng = _random.Random(seed) + delete_baseline_files(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + for _ in range(n_samples): + fps = max(1.0, rng.gauss(center_fps, noise_fps)) + update_baseline(baselines_dir, gpu_model, task_id, backend, fps, fingerprint=fingerprint) + + +def _git_show_file(branch: str, rel_path: str) -> str | None: + try: + r = subprocess.run( + ["git", "show", f"{branch}:{rel_path}"], + capture_output=True, + text=True, + check=True, + cwd=str(Path(__file__).parent), + ) + return r.stdout + except subprocess.CalledProcessError: + return None + + +@contextlib.contextmanager +def baseline_worktree(branch: str): + tmpdir = tempfile.mkdtemp(prefix="perf-bl-wt-") + committed = False + try: + subprocess.run( + ["git", "worktree", "add", tmpdir, branch], + check=True, + capture_output=True, + cwd=str(Path(__file__).parent), + ) + yield Path(tmpdir) + status = subprocess.run( + ["git", "-C", tmpdir, "status", "--porcelain"], + capture_output=True, + text=True, + ) + if status.stdout.strip(): + subprocess.run(["git", "-C", tmpdir, "add", "-A"], check=True, capture_output=True) + subprocess.run( + ["git", "-C", tmpdir, "commit", "-m", "[baseline_manager] Update baselines"], + check=True, + capture_output=True, + ) + committed = True + except subprocess.CalledProcessError as exc: + if b"not found" in (exc.stderr or b"") or b"unknown" in (exc.stderr or b""): + raise RuntimeError( + f"Baseline branch {branch!r} not found. " + "Run: git fetch origin angehu/perf-baselines (or use --force-flat-baseline)" # TODO: replace w/ real branch name + ) from exc + raise + finally: + subprocess.run( + ["git", "worktree", "remove", "--force", tmpdir], + cwd=str(Path(__file__).parent), + capture_output=True, + ) + shutil.rmtree(tmpdir, ignore_errors=True) + if committed: + print(f"[baseline_manager] -> committed baseline update to {branch!r}") + + +def load_baseline_git(branch: str, gpu_model: str, task_id: str, backend: str, fingerprint: str | None) -> Baseline | None: + rel = str(_stats_path(Path(""), gpu_model, task_id, backend, fingerprint)) + content = _git_show_file(branch, rel) + if content is None: + return None + d = json.loads(content) + return Baseline( + median_fps=d["median_fps"], + mad_fps=d["mad_fps"], + k_warn=d.get("k_warn", DEFAULT_K_WARN), + k_block=d.get("k_block", DEFAULT_K_BLOCK), + sample_count=d.get("sample_count", 0), + ) + + +def update_baseline_git( + branch: str, gpu_model: str, task_id: str, backend: str, fps: float, fingerprint: str | None +) -> None: + with baseline_worktree(branch) as wt: + update_baseline(wt, gpu_model, task_id, backend, fps, fingerprint) + + +def seed_baseline_with_spread_git( + branch: str, + gpu_model: str, + task_id: str, + backend: str, + center_fps: float, + noise_fps: float, + n_samples: int, + seed: int, + fingerprint: str | None, +) -> None: + with baseline_worktree(branch) as wt: + seed_baseline_with_spread(wt, gpu_model, task_id, backend, center_fps, noise_fps, n_samples, seed, fingerprint) + + +def delete_baseline_files_git(branch: str, gpu_model: str, task_id: str, backend: str, fingerprint: str | None) -> None: + with baseline_worktree(branch) as wt: + delete_baseline_files(wt, gpu_model, task_id, backend, fingerprint) diff --git a/tools/perf_regression_gate/build_bench_result.py b/tools/perf_regression_gate/build_bench_result.py new file mode 100644 index 000000000000..fbad11b66db9 --- /dev/null +++ b/tools/perf_regression_gate/build_bench_result.py @@ -0,0 +1,333 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Post-benchmark script: normalizes benchmark output and writes perf_regression_gate_result.json. + +Locates the timestamped benchmark output file written by benchmark_non_rl.py, renames it +to the canonical ``perf_regression_gate_info.json``, classifies the failure phase from the +captured log, and writes ``perf_regression_gate_result.json`` for the aggregate job. + +Usage:: + + python3 tools/perf_regression_gate/build_bench_result.py \\ + --task_id Isaac-Cartpole-Direct-v0 \\ + --artifact_dir artifacts/Isaac-Cartpole-Direct-v0 \\ + --exit_code 0 \\ + --wall_time_s 48.3 \\ + --timeout_s 600 \\ + --log_file artifacts/Isaac-Cartpole-Direct-v0/benchmark.log +""" + +import argparse +import glob +import json +import shutil +import statistics +import subprocess +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +_TOOLS_DIR = _MODULE_DIR.parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + +from subprocess_runner import classify_failure_phase # noqa: E402 +from task_config import get_task # noqa: E402 + + +def _percentile(sorted_data: list[float], p: float) -> float: + """Linear-interpolation percentile on a pre-sorted list""" + n = len(sorted_data) + if n == 1: + return sorted_data[0] + idx = p / 100.0 * (n - 1) + lo = int(idx) + hi = min(lo + 1, n - 1) + return sorted_data[lo] + (sorted_data[hi] - sorted_data[lo]) * (idx - lo) + + +def _strip_phase_prefix(name: str, phase_name: str) -> str: + """Strip the '{task_name} {phase_name} ' prefix added by JSONFileMetrics.finalize()""" + marker = f" {phase_name} " + idx = name.find(marker) + return name[idx + len(marker) :] if idx >= 0 else name + + +def _gpu_driver_version() -> str | None: + """Return the GPU driver version string from nvidia-smi, or None if unavailable""" + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader,nounits"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + version = result.stdout.strip().splitlines()[0].strip() + return version if version else None + except Exception: + pass + return None + + +def _extract_info_provenance(info_path: Path) -> dict: + """Parse ``perf_regression_gate_info.json`` and return provenance + FPS fields + + Extracts hardware metadata (GPU name/memory/CUDA, CPU, RAM), software versions, + git provenance, FPS distribution statistics, GPU memory used at runtime, and + startup time. All fields are best-effort; missing data is omitted. + + Returns: + Dict with a subset of the following keys: + + - ``raw_fps_{mean,std,min,max,median,p5,p95}``: float + - ``startup_time_s``: float + - ``gpu_diag``: dict with gpu_name, gpu_total_memory_gb, cuda_version, + nvidia_driver_version, gpu_mem_used_mb + - ``provenance``: dict with ``hardware``, ``software``, ``git`` sub-dicts + """ + try: + phases: list[dict] = json.loads(info_path.read_text()) + except Exception: + return {} + + hardware: dict = {} + software: dict = {} + git: dict = {} + fps_stats: dict = {} + gpu_mem_used_gb: float | None = None + startup_time_s: float | None = None + + for phase in phases: + pname: str = phase.get("phase_name", "") + metadata_map = { + _strip_phase_prefix(m["name"], pname): m["data"] + for m in phase.get("metadata", []) + if "name" in m and "data" in m + } + + if pname == "hardware_info": + hardware["cpu_name"] = metadata_map.get("cpu_name") + hardware["cpu_physical_cores"] = metadata_map.get("physical_cores") + hardware["total_ram_gb"] = metadata_map.get("total_ram_gb") + hardware["gpu_device_count"] = metadata_map.get("gpu_device_count") + hardware["cuda_version"] = metadata_map.get("cuda_version") + gpu_devices = metadata_map.get("gpu_devices") + if isinstance(gpu_devices, dict): + current = str(metadata_map.get("gpu_current_device", 0)) + dev = gpu_devices.get(current) or next(iter(gpu_devices.values()), {}) + hardware["gpu_name"] = dev.get("name") + hardware["gpu_total_memory_gb"] = dev.get("total_memory_gb") + hardware["gpu_compute_capability"] = dev.get("compute_capability") + hardware["gpu_multi_processor_count"] = dev.get("multi_processor_count") + + elif pname == "version_info": + dev_data: dict = metadata_map.pop("dev", {}) or {} + for k, v in metadata_map.items(): + # strip trailing "_version" suffix added by VersionInfoRecorder + key = k[: -len("_version")] if k.endswith("_version") else k + if v is not None: + software[key] = v + git = { + k: dev_data[k] + for k in ("commit_hash", "commit_hash_short", "branch", "commit_date", "dirty") + if k in dev_data + } + + elif pname == "runtime": + for m in phase.get("measurements", []): + name: str = m.get("name", "") + value = m.get("value") + # FPS series: DictMeasurement written by BenchmarkMonitor + if name.endswith("Step Frametimes") and isinstance(value, dict): + fps_series: list[float] = value.get("Environment step effective FPS", []) + if fps_series: + sorted_fps = sorted(fps_series) + n = len(sorted_fps) + fps_stats = { + "raw_fps_mean": statistics.mean(fps_series), + "raw_fps_std": statistics.stdev(fps_series) if n > 1 else 0.0, + "raw_fps_min": sorted_fps[0], + "raw_fps_max": sorted_fps[-1], + "raw_fps_median": _percentile(sorted_fps, 50.0), + "raw_fps_p5": _percentile(sorted_fps, 5.0), + "raw_fps_p95": _percentile(sorted_fps, 95.0), + } + # GPU memory used (mean over run, in GB): SingleMeasurement from GPUInfoRecorder + elif name.endswith("GPU Memory Used") and isinstance(value, (int, float)): + gpu_mem_used_gb = float(value) + + elif pname == "startup": + for m in phase.get("measurements", []): + if m.get("name", "").endswith("Total Start Time (Launch to Train)"): + val = m.get("value") + if isinstance(val, (int, float)): + startup_time_s = float(val) + break + + gpu_diag: dict = {k: v for k, v in { + "gpu_name": hardware.get("gpu_name"), + "gpu_total_memory_gb": hardware.get("gpu_total_memory_gb"), + "cuda_version": hardware.get("cuda_version"), + "nvidia_driver_version": _gpu_driver_version(), + "gpu_mem_used_mb": round(gpu_mem_used_gb * 1024, 2) if gpu_mem_used_gb is not None else None, + }.items() if v is not None} + + result: dict = {} + result.update(fps_stats) + if startup_time_s is not None: + result["startup_time_s"] = startup_time_s + if gpu_diag: + result["gpu_diag"] = gpu_diag + result["provenance"] = { + "hardware": {k: v for k, v in hardware.items() if v is not None}, + "software": software, + "git": git, + } + return result + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Build perf_regression_gate_result.json from a benchmark run") + p.add_argument("--task_id", required=True) + p.add_argument("--physics_backend", required=True, help="Physics backend used (e.g. physx, newton)") + p.add_argument("--render_backend", default="", help="Render backend used (e.g. rtx, warp, ovrtx); empty = none") + p.add_argument("--artifact_dir", required=True, type=Path) + p.add_argument("--exit_code", required=True, type=int) + p.add_argument("--wall_time_s", required=True, type=float) + p.add_argument("--timeout_s", required=True, type=float) + p.add_argument("--log_file", type=Path, default=None) + p.add_argument("--attempt", type=int, default=1, help="Attempt number (1 = first run, 2 = after one retry)") + p.add_argument("--was_retried", action="store_true", help="Set when this result comes from a retry of a failed first attempt") + return p.parse_args() + + +def _normalize_benchmark_output(artifact_dir: Path, task_id: str) -> bool: + """Rename the timestamped benchmark JSON to ``perf_regression_gate_info.json`` + + benchmark_non_rl.py writes ``benchmark_non_rl_{task_id}_{timestamp}.json``. + The oracle reads ``perf_regression_gate_info.json``. This function bridges the gap. + + Returns True if perf_regression_gate_info.json exists after call + """ + perf_regression_gate_info = artifact_dir / "perf_regression_gate_info.json" + if perf_regression_gate_info.exists(): + return True + # Primary pattern: exact task_id match + matches = sorted(glob.glob(str(artifact_dir / f"benchmark_non_rl_{task_id}_*.json"))) + if not matches: + # Fallback: any benchmark_non_rl_*.json in the artifact dir + matches = sorted(glob.glob(str(artifact_dir / "benchmark_non_rl_*.json"))) + if not matches: + return False + shutil.copy(matches[-1], perf_regression_gate_info) + return True + + +def main() -> int: + args = _parse_args() + artifact_dir = args.artifact_dir + artifact_dir.mkdir(parents=True, exist_ok=True) + + physics_backend: str = args.physics_backend + render_backend: str | None = args.render_backend if args.render_backend else None + backend_key: str = f"{physics_backend}_{render_backend}" if render_backend else physics_backend + + # Load task config for metadata embedded in bench_result + try: + task = get_task(args.task_id, backend_key) + num_envs = task.num_envs + num_frames = task.num_frames + excluded_frames_raw = task.excluded_frames_raw + timeout_minutes = task.timeout_minutes + preset = task.preset + tags = task.tags + seed = task.seed + except KeyError: + print( + f"[build_bench_result] Warning: ({args.task_id!r}, {backend_key!r}) not found in tasks.json; using defaults" + ) + num_envs = 0 + num_frames = 0 + excluded_frames_raw = [] + timeout_minutes = int(args.timeout_s / 60) + preset = "default" + tags = ["always"] + seed = None + + # Read combined stdout/stderr log for failure classification + log_text = "" + if args.log_file and args.log_file.exists(): + log_text = args.log_file.read_text(errors="replace") + + perf_regression_gate_info_present = _normalize_benchmark_output(artifact_dir, args.task_id) + + failure_phase = classify_failure_phase( + stdout=log_text, + stderr="", + exit_code=args.exit_code, + wall_time_s=args.wall_time_s, + timeout_s=args.timeout_s, + ) + + # Extract FPS stats, startup time, GPU diag, and full provenance from the info artifact + info_provenance: dict = {} + if perf_regression_gate_info_present: + info_provenance = _extract_info_provenance(artifact_dir / "perf_regression_gate_info.json") + + bench_result = { + "task_id": args.task_id, + "backend": backend_key, + "physics_backend": physics_backend, + "render_backend": render_backend, + "backend_key": backend_key, + "preset": preset, + "attempt": args.attempt, + "was_retried": args.was_retried, + "exit_code": args.exit_code, + "failure_phase": failure_phase, + "stdout_tail": log_text[-2000:] if len(log_text) > 2000 else log_text, + "wall_time_s": args.wall_time_s, + "startup_time_s": info_provenance.get("startup_time_s"), + "perf_regression_gate_info_present": perf_regression_gate_info_present, + "raw_fps_mean": info_provenance.get("raw_fps_mean"), + "raw_fps_std": info_provenance.get("raw_fps_std"), + "raw_fps_min": info_provenance.get("raw_fps_min"), + "raw_fps_max": info_provenance.get("raw_fps_max"), + "raw_fps_median": info_provenance.get("raw_fps_median"), + "raw_fps_p5": info_provenance.get("raw_fps_p5"), + "raw_fps_p95": info_provenance.get("raw_fps_p95"), + "outlier_count": None, + "gpu_diag": info_provenance.get("gpu_diag"), + "provenance": info_provenance.get("provenance"), + "task_config_snapshot": { + "task_id": args.task_id, + "backend": backend_key, + "physics_backend": physics_backend, + "render_backend": render_backend, + "backend_key": backend_key, + "preset": preset, + "num_envs": num_envs, + "num_frames": num_frames, + "excluded_frames_raw": excluded_frames_raw, + "timeout_minutes": timeout_minutes, + "tags": tags, + "seed": seed, + }, + } + + out = artifact_dir / "perf_regression_gate_result.json" + out.write_text(json.dumps(bench_result, indent=2)) + + status = f"failure_phase={failure_phase!r}, perf_regression_gate_info_present={perf_regression_gate_info_present}, exit_code={args.exit_code}" + print(f"[build_bench_result] {args.task_id}: {status}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_regression_gate/dev/sim_regression.py b/tools/perf_regression_gate/dev/sim_regression.py new file mode 100644 index 000000000000..ce4f07e1fb23 --- /dev/null +++ b/tools/perf_regression_gate/dev/sim_regression.py @@ -0,0 +1,149 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Regression simulation helper that injects degraded FPS artifacts into a scratch artifacts directory. + +Used to demonstrate that the gate produces BLOCK verdicts when a real regression +is introduced, WITHOUT re-running the full benchmark suite. + + python3 tools/perf_regression_gate/dev/sim_regression.py --fps_scale 0.53 + +Then run aggregate.py against the output to see BLOCK verdicts: + + python3 tools/perf_regression_gate/aggregate.py \\ + --artifacts_dir /tmp/sim_artifacts \\ + --gpu_model L40S \\ + --baselines_dir tools/perf_regression_gate/local_baselines \\ + --allow_baseline_update false +""" + +import argparse +import json +import random +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +_GATE_DIR = _MODULE_DIR.parent # tools/perf_regression_gate/ + +sys.path.insert(0, str(_GATE_DIR)) +sys.path.insert(0, str(_GATE_DIR.parent)) + +from task_config import load_tasks # noqa: E402 + + +def _make_perf_info(task_id: str, fps_mean: float, num_frames: int) -> list: + rng = random.Random(42) + fps_series = [max(1.0, fps_mean + rng.gauss(0, fps_mean * 0.01)) for _ in range(num_frames)] + return [ + { + "phase_name": "runtime", + "measurements": [ + { + "name": f"{task_id} Step Frametimes", + "value": {"Environment step effective FPS": fps_series}, + } + ], + "metadata": [], + } + ] + + +def _make_bench_result(task, fps_mean: float) -> dict: + return { + "task_id": task.task_id, + "backend": task.backend_key, + "backend_key": task.backend_key, + "physics_backend": task.physics_backend, + "render_backend": task.render_backend, + "preset": task.preset, + "attempt": 1, + "was_retried": False, + "exit_code": 0, + "failure_phase": None, + "stdout_tail": "", + "wall_time_s": 60.0, + "startup_time_s": 10.0, + "perf_regression_gate_info_present": True, + "raw_fps_mean": fps_mean, + "raw_fps_std": fps_mean * 0.01, + "raw_fps_min": fps_mean * 0.95, + "raw_fps_max": fps_mean * 1.05, + "raw_fps_median": fps_mean, + "raw_fps_p5": fps_mean * 0.96, + "raw_fps_p95": fps_mean * 1.04, + "outlier_count": 0, + "gpu_diag": None, + "task_config_snapshot": { + "task_id": task.task_id, + "backend": task.backend_key, + "backend_key": task.backend_key, + "physics_backend": task.physics_backend, + "render_backend": task.render_backend, + "preset": task.preset, + "num_envs": task.num_envs, + "num_frames": task.num_frames, + "excluded_frames_raw": task.excluded_frames_raw, + "timeout_minutes": task.timeout_minutes, + "camera_resolution": task.camera_resolution, + "tags": task.tags, + }, + } + + +def main() -> int: + p = argparse.ArgumentParser(description="Inject simulated regression artifacts.") + p.add_argument("--fps_scale", type=float, default=0.53, + help="Multiply baseline FPS by this factor (0.53 = 47%% regression, default)") + p.add_argument("--tags", nargs="+", default=["always"], + help="Task tags to include (default: always)") + p.add_argument("--gpu_model", default="L40S") + p.add_argument("--baselines_dir", type=Path, + default=_GATE_DIR / "local_baselines") + p.add_argument("--out_dir", type=Path, default=Path("/tmp/sim_artifacts")) + args = p.parse_args() + + all_tasks = load_tasks() + tag_set = frozenset(args.tags) + tasks = [t for t in all_tasks if tag_set.intersection(frozenset(t.tags))] + + print(f"\n[sim_regression] fps_scale={args.fps_scale} ({(1-args.fps_scale)*100:.0f}% regression)") + print(f"[sim_regression] writing artifacts to {args.out_dir}\n") + + generated = 0 + for task in tasks: + stats_path = args.baselines_dir / args.gpu_model / task.task_id / task.backend_key / "stats.json" + if not stats_path.exists(): + print(f" SKIP (no baseline): {task.task_id}/{task.backend_key}") + continue + + stats = json.loads(stats_path.read_text()) + baseline_fps = stats["median_fps"] + regressed_fps = baseline_fps * args.fps_scale + + art_dir = args.out_dir / task.task_id / task.backend_key + art_dir.mkdir(parents=True, exist_ok=True) + + perf_info = _make_perf_info(task.task_id, regressed_fps, task.num_frames) + (art_dir / "perf_regression_gate_info.json").write_text(json.dumps(perf_info)) + + bench_result = _make_bench_result(task, regressed_fps) + (art_dir / "perf_regression_gate_result.json").write_text(json.dumps(bench_result)) + + print(f" {task.task_id}/{task.backend_key}: baseline={baseline_fps:.1f} regressed={regressed_fps:.1f}") + generated += 1 + + print(f"\n[sim_regression] wrote {generated} artifact sets") + print(f"\nNow run aggregate.py:") + print(f" python3 {_GATE_DIR}/aggregate.py \\") + print(f" --artifacts_dir {args.out_dir} \\") + print(f" --gpu_model {args.gpu_model} \\") + print(f" --baselines_dir {args.baselines_dir} \\") + print(f" --allow_baseline_update false") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_regression_gate/dev/stub_benchmark.py b/tools/perf_regression_gate/dev/stub_benchmark.py new file mode 100644 index 000000000000..01feb4f03494 --- /dev/null +++ b/tools/perf_regression_gate/dev/stub_benchmark.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +import argparse +import json +import math +import random +import sys +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--task_id") + parser.add_argument("--backend") + parser.add_argument("--num_envs", type=int, default=1) + parser.add_argument("--num_frames", type=int, default=200) + parser.add_argument("--out_dir", required=True) + parser.add_argument("--fps_mean", type=float, default=200.0) + parser.add_argument("--failure_phase", default="none") + args = parser.parse_args() + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + # Simulate failures + if args.failure_phase == "import": + # Emit a fake traceback and exit non-zero without writing perf file + print("Traceback (most recent call last):") + print(" File \"\", line 1, in ") + print("ImportError: simulated import failure") + sys.exit(1) + + if args.failure_phase == "init": + # Emit AppLauncher init message then exit non-zero (no perf file) + print("AppLauncher initialization complete") + sys.exit(2) + + # Prepare FPS series + n = int(args.num_frames) + rng = random.Random(0) + noise = 5.0 + fps_series = [max(0.0, rng.gauss(args.fps_mean, noise)) for _ in range(n)] + + # Write perf_regression_gate_info.json + runtime_phase = { + "phase_name": "runtime", + "measurements": [ + { + "name": "Step Frametimes", + "value": {"Environment step effective FPS": fps_series}, + } + ], + } + info_path = out_dir / "perf_regression_gate_info.json" + with info_path.open("w") as fh: + json.dump([runtime_phase], fh) + + # Print Step Frametimes marker so classify_failure_phase can see it + print("Step Frametimes") + + # For runtime failure emulate crash after printing frame times + if args.failure_phase == "runtime": + print("RuntimeError: simulated crash during runtime") + sys.exit(3) + + # Successful exit + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/tools/perf_regression_gate/docs/module-interfaces.md b/tools/perf_regression_gate/docs/module-interfaces.md new file mode 100644 index 000000000000..aabd2a5091b3 --- /dev/null +++ b/tools/perf_regression_gate/docs/module-interfaces.md @@ -0,0 +1,665 @@ +# Performance Regression Gate — Module Interface Reference + +Full function signatures, CLI arguments, and JSON schemas for every module. +All commands run from the `IsaacLab/` repository root unless otherwise noted. + +--- + +## `oracle.py` + +### `compare()` + +```python +def compare( + bench_result: dict, + baseline: Baseline | None, + fps_mean_floor: float, + excluded_frames: frozenset[int], + artifact_dir: Path, +) -> OracleResult +``` + +The central verdict function. Reads `perf_regression_gate_info.json` from `artifact_dir`, +applies `excluded_frames`, computes mean FPS, and returns an `OracleResult`. + +| Parameter | Type | Description | +|---|---|---| +| `bench_result` | `dict` | Loaded `perf_regression_gate_result.json` | +| `baseline` | `Baseline \| None` | Rolling baseline stats; `None` for seed run | +| `fps_mean_floor` | `float` | Hard minimum FPS; 0.0 = disabled | +| `excluded_frames` | `frozenset[int]` | 0-based frame indices to drop before computing mean | +| `artifact_dir` | `Path` | Directory containing `perf_regression_gate_info.json` | + +### `apply_excluded_frames()` + +```python +def apply_excluded_frames(fps_series: list[float], excluded_frames: frozenset[int]) -> list[float] +``` + +Returns `fps_series` with indices listed in `excluded_frames` removed. + +### `class Baseline` + +```python +@dataclass +class Baseline: + median_fps: float # Median FPS of the baseline window + mad_fps: float # Median absolute deviation of FPS in the window + k_warn: float = 2.5 # MAD multiplier for WARN threshold + k_block: float = 4.0 # MAD multiplier for BLOCK threshold + sample_count: int = 0 # Number of samples in the window +``` + +Thresholds: `warn_thresh = median - k_warn × MAD`, `block_thresh = median - k_block × MAD`. + +### `class OracleResult` + +```python +@dataclass +class OracleResult: + verdict: OracleVerdict # PASS / WARN / BLOCK / HARD_FAILURE + bisect_verdict: str # "GOOD" / "BAD" / "SKIP" + failure_phase: str | None # From bench_result; see failure phase table + measured_fps: float | None # Mean FPS post-filter; None on HARD_FAILURE + baseline_fps: float | None # baseline.median_fps; None if no baseline + regression_pct: float | None # ((measured - baseline) / baseline) × 100; None if no baseline + fps_median: float | None # Median of filtered series [informational] + fps_p5: float | None # 5th-percentile of filtered series [informational] + fps_p95: float | None # 95th-percentile of filtered series [informational] + gpu_mem_used_mb: float | None # From bench_result.gpu_diag [informational] + startup_time_s: float | None # From bench_result [informational] + wall_time_s: float | None # From bench_result [informational] + was_retried: bool # Whether Phase 1 succeeded only after a retry + task_id: str + backend: str +``` + +`fps_median`, `fps_p5`, `fps_p95` are informational — they do not affect the verdict. +`measured_fps` (mean of filtered series) is the blocking metric. + +### `class OracleVerdict` + +```python +class OracleVerdict(str, Enum): + PASS = "PASS" + WARN = "WARN" + BLOCK = "BLOCK" + HARD_FAILURE = "HARD_FAILURE" +``` + +--- + +## `task_config.py` + +### `load_tasks()` + +```python +def load_tasks(tasks_json_path: Path | str | None = None) -> list[TaskConfig] +``` + +Loads all benchmark tasks from `tasks.json`, expanding each task's `backends` array into +one `TaskConfig` per `(task_id, backend)` combination. Applies `defaults` block to each task. + +Default path: `tools/perf_regression_gate/tasks.json` (sibling of the module). + +### `get_task()` + +```python +def get_task(task_id: str, backend_key: str, tasks_json_path: Path | str | None = None) -> TaskConfig +``` + +Returns the `TaskConfig` for a specific `(task_id, backend_key)` pair. +Raises `KeyError` if not found. + +### `caches_for_backend()` + +```python +def caches_for_backend(backend: str) -> list[str] +``` + +Returns cache identifiers needed before benchmarking with a given physics backend. +Currently: `"newton"` → `["mjwarp_jit"]`; all others → `[]`. + +### `class TaskConfig` + +```python +@dataclass +class TaskConfig: + task_id: str + physics_backend: str # "physx" or "newton" + render_backend: str | None # "newton_renderer", "ovrtx_renderer", or None + preset: str # Hydra preset base (usually "default") + num_envs: int + num_frames: int + excluded_frames_raw: list[int | list[int]] # Raw JSON; use .excluded_frames + camera_resolution: tuple[int, int] | None + timeout_minutes: int + fps_mean_floor: dict # {"L40S": {"physx": 100.0, ...}} + caches: list[str] # Cache identifiers from caches_for_backend() + tags: list[str] # ["always"] or ["camera"] + task_type: str # "benchmark" + runs_on: str # "gpu-l40s" + seed: int | None # Random seed for benchmark (default 42) + + @property + def backend_key(self) -> str: + # "{physics}_{render}" if render_backend else "{physics}" + + @property + def excluded_frames(self) -> frozenset[int]: + # Expands excluded_frames_raw ranges to individual indices +``` + +`excluded_frames_raw` supports two entry types: +- `[start, end]` — inclusive range, expanded to `range(start, end+1)` +- `N` — single index + +Default value `[[0, 100]]` expands to indices 0–100 (101 frames excluded from 300 total). + +--- + +## `baseline_manager.py` + +### Flat-file operations (local / testing) + +```python +def load_baseline( + baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None +) -> Baseline | None +``` +Loads `stats.json` for a task/backend pair. Returns `None` if file does not exist (seed run). + +```python +def update_baseline( + baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fps: float, fingerprint=None +) -> None +``` +Appends `fps` to `window.ndjson`, recomputes `median` and `MAD`, writes `stats.json`. +Only call for PASS/WARN results — aggregate.py enforces this policy. + +```python +def delete_baseline_files( + baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None +) -> None +``` +Removes `stats.json` and `window.ndjson` for a task/backend pair. Used in tests and resets. + +```python +def seed_baseline_with_spread( + baselines_dir: Path, gpu_model: str, task_id: str, backend: str, + center_fps: float, noise_fps: float = 5.0, n_samples: int = 10, + seed: int = 0, fingerprint=None +) -> None +``` +Populates a baseline window with `n_samples` Gaussian samples around `center_fps` ± `noise_fps`. +Used in tests to create deterministic baselines without running real benchmarks. + +### Git operations (production) + +```python +def load_baseline_git(branch: str, gpu_model: str, task_id: str, backend: str, fingerprint: str | None) -> Baseline | None +def update_baseline_git(branch: str, gpu_model: str, task_id: str, backend: str, fps: float, fingerprint: str | None) -> None +def seed_baseline_with_spread_git(branch, gpu_model, task_id, backend, center_fps, noise_fps, n_samples, seed, fingerprint) -> None +def delete_baseline_files_git(branch, gpu_model, task_id, backend, fingerprint) -> None +``` + +Git variants use `baseline_worktree()` — a context manager that creates a temporary git +worktree on `branch`, yields the worktree path, then commits any changes and removes the +worktree. Raises `RuntimeError` if `branch` is not found locally. + +### File paths (flat-file) + +``` +{baselines_dir}/{gpu_model}/{task_id}/{backend}/stats.json +{baselines_dir}/{gpu_model}/{task_id}/{backend}/window.ndjson + +With fingerprint: +{baselines_dir}/{gpu_model}/{task_id}/{backend}/{fingerprint}/stats.json +{baselines_dir}/{gpu_model}/{task_id}/{backend}/{fingerprint}/window.ndjson +``` + +--- + +## `subprocess_runner.py` + +### `classify_failure_phase()` + +```python +def classify_failure_phase( + stdout: str, stderr: str, exit_code: int, wall_time_s: float, timeout_s: float +) -> str | None +``` + +Classifies the failure phase of a benchmark run by scanning combined output. Priority order: + +| Priority | Phase | Trigger | +|---|---|---| +| 1 | `"oom"` | `exit_code == 137` OR `"oom-kill"` in `stderr` | +| 2 | `"hang"` | `wall_time_s >= timeout_s * 0.95` | +| 3 | `"import"` | `"Traceback"` in combined AND `"AppLauncher"` NOT in `stdout` | +| 4 | `"driver"` | `"CudaError"` OR `"CUDA_ERROR_"` in combined | +| 5 | `"init"` | `exit_code != 0` AND `"AppLauncher initialization complete"` in `stdout` AND `"Step Frametimes"` NOT in `stdout` | +| 6 | `"runtime"` | `exit_code != 0` AND `"Step Frametimes"` in `stdout` | +| 7 | `None` | `exit_code == 0` | + +### `run_benchmark()` + +```python +def run_benchmark(cmd: list, timeout_s: float) -> dict +``` + +Runs `cmd` with `capture_test_output_with_timeout()` and returns: + +```python +{ + "exit_code": int, + "stdout_tail": str, # last 2000 chars of combined stdout + "wall_time_s": float, + "startup_time_s": float, # stub: 0.0 + "failure_phase": str | None, +} +``` + +### `capture_test_output_with_timeout()` + +```python +def capture_test_output_with_timeout( + cmd, timeout, env, startup_deadline=0, report_file="" +) -> tuple[int, bytes, bytes, str, float, str] +``` + +Returns `(returncode, stdout_bytes, stderr_bytes, kill_reason, wall_time, pre_kill_diag)`. + +`kill_reason` values: `""` (normal), `"timeout"`, `"startup_hang"`, `"shutdown_hang"`. + +Uses `select()` + non-blocking I/O for real-time streaming. Kills the entire process group +(including Kit / Isaac Sim child processes) on timeout. + +--- + +## `tasks_to_ci_matrix.py` + +``` +python3 tools/perf_regression_gate/tasks_to_ci_matrix.py +``` + +No arguments. Reads `tasks.json` via `load_tasks()` and prints a JSON array to stdout, +one object per `(task_id, backend)` combination. Used by the `build_matrix` step in +`perf-regression-gate.yaml` to populate the GitHub Actions job matrix. + +Each object contains: `task_id`, `physics_backend`, `render_backend` (empty string if none), +`num_envs`, `num_frames`, `bench_timeout_s`, `job_timeout_minutes`. + +--- + +## `build_bench_result.py` CLI + +``` +python3 tools/perf_regression_gate/build_bench_result.py \ + --task_id task identifier (required) + --physics_backend "physx" or "newton" (required) + --render_backend render backend name or "" for none (default: "") + --artifact_dir directory for artifacts (required) + --exit_code Phase 1 process exit code (required) + --wall_time_s Phase 1 wall-clock time in seconds (required) + --timeout_s Phase 1 timeout in seconds (required) + --log_file combined stdout+stderr log from Phase 1 (default: none) + --attempt attempt number: 1 = first try, 2 = after retry (default: 1) + --was_retried flag: set when this result comes from a retry +``` + +**Output:** `{artifact_dir}/perf_regression_gate_result.json` (always written). + +**Side effect:** Renames `benchmark_non_rl_{task_id}_{timestamp}.json` to +`perf_regression_gate_info.json` if the canonical name does not already exist. + +When `perf_regression_gate_info.json` is present, `build_bench_result.py` also +calls `_extract_info_provenance()` to populate the FPS distribution, startup time, +GPU diagnostics, and full software/hardware/git provenance directly into the result JSON. +`nvidia-smi` is queried once at post-processing time to capture the driver version. + +--- + +## `aggregate.py` CLI + +``` +python3 tools/perf_regression_gate/aggregate.py \ + --artifacts_dir root directory containing per-task artifact subdirectories (required) + --gpu_model GPU model label for baseline lookup (default: L40S) + --gate_config path to gate_config.json (default: perf_regression_gate/gate_config.json) + --baseline_branch git branch for baseline storage (default: angehu/perf-baselines) + --baselines_dir flat-file baseline directory; bypasses git (default: None = use git) + --allow_baseline_update "true"/"false": extend baseline window for PASS/WARN (default: false) + --summary_file append step-summary markdown to this path (default: none) +``` + +**Exit codes:** +- `0` — gate is non-blocking, or all tasks PASS/WARN +- `1` — any BLOCK verdict and `gate_config.blocking == true` +- `2` — any HARD_FAILURE verdict and `gate_config.blocking == true` + +**Environment variable:** When `GITHUB_OUTPUT` is set, writes `baselines_updated=true` to +it after a successful baseline update. Used by the CI workflow to decide whether to push. + +--- + +## `local_runner.py` CLI + +``` +python3 tools/perf_regression_gate/local_runner.py \ + --tags task tags to run (default: always) + --gpu_model GPU model label for baselines (default: L40S) + --artifacts_dir root for per-task artifacts (default: perf_regression_gate/artifacts/) + --baselines_dir flat-file baseline dir (default: perf_regression_gate/local_baselines/) + --allow_baseline_update extend baseline window for PASS/WARN results + --dry_run print task matrix and exit without running anything + --skip_existing skip tasks whose perf_regression_gate_result.json already exists + --gate_config path to gate_config.json +``` + +Orchestrates Phase 1+2+3 sequentially. For each task: +1. Runs `./isaaclab.sh -p scripts/benchmarks/benchmark_non_rl.py` with the derived command +2. On non-zero exit, retries once (sets `was_retried=True`) +3. Calls `build_bench_result.py` (Phase 2) +4. After all tasks, calls `aggregate.py` (Phase 3) + +Returns aggregate.py's exit code. + +--- + +## `gate_config.py` + +```python +def load_gate_config(path: Path | str) -> dict +``` + +Loads `gate_config.json`. Currently a stub that returns `{"blocking": False}`. +When the CI workflow PR lands, this will read the JSON file from disk. + +The dict contract: `{"blocking": bool}`. `aggregate.py` uses `gate_config.get("blocking", False)`. + +--- + +## `dev/stub_benchmark.py` CLI + +Simulates `benchmark_non_rl.py` for testing. Does not require IsaacSim. + +``` +python3 tools/perf_regression_gate/dev/stub_benchmark.py \ + --task_id + --backend + --num_envs (default: 1) + --num_frames (default: 200) + --out_dir (required) + --fps_mean (default: 200.0) + --failure_phase "none" | "import" | "init" | "runtime" (default: none) +``` + +On success: writes `perf_regression_gate_info.json` with a Gaussian FPS series centered on +`--fps_mean`, prints `"Step Frametimes"` to stdout, exits 0. + +On failure modes: +- `import`: prints traceback-like output, exits 1 (no perf file written) +- `init`: prints `"AppLauncher initialization complete"`, exits 2 (no perf file written) +- `runtime`: writes perf file, prints frametimes, then prints error and exits 3 + +--- + +## `dev/sim_regression.py` CLI + +Injects degraded FPS artifacts for demo/testing without re-running benchmarks. + +``` +python3 tools/perf_regression_gate/dev/sim_regression.py \ + --fps_scale multiply baseline FPS by this factor (default: 0.53 = 47% regression) + --tags task tags to include (default: always) + --gpu_model GPU model label (default: L40S) + --baselines_dir (default: perf_regression_gate/local_baselines) + --out_dir output artifacts directory (default: /tmp/sim_artifacts) +``` + +For each task with an existing baseline, reads `stats.json`, computes +`regressed_fps = baseline.median_fps × fps_scale`, and writes: +- `{out_dir}/{task_id}/{backend_key}/perf_regression_gate_info.json` +- `{out_dir}/{task_id}/{backend_key}/perf_regression_gate_result.json` + +Skips tasks with no baseline (prints `SKIP (no baseline)`). + +--- + +## `tasks.json` Schema + +```json +{ + "defaults": { + "type": "benchmark", + "runs_on": "gpu-l40s", + "preset": "default", + "seed": 42, + "num_envs": 512, + "num_frames": 300, + "excluded_frames": [[0, 100]], + "camera_resolution": null, + "timeout_minutes": 10, + "tags": ["always"] + }, + "tasks": [ + { + "task_id": "", + "num_envs": , // overrides defaults + "timeout_minutes": , // overrides defaults + "tags": ["always" | "camera"], // overrides defaults + "backends": [ + {"physics": "physx"}, + {"physics": "newton"}, + {"physics": "physx", "render": "newton_renderer"}, + {"physics": "newton", "render": "ovrtx_renderer"} + ], + "fps_mean_floor": { + "": { + "": // 0.0 = disabled + } + } + } + ] +} +``` + +Each entry in `backends` becomes one `TaskConfig`. `backend_key = physics` when no +`render` field; `backend_key = {physics}_{render}` when `render` is present. + +--- + +## Artifact Schemas + +### `perf_regression_gate_result.json` (Phase 2 output) + +```json +{ + "task_id": "Isaac-Velocity-Flat-G1-v0", + "backend": "physx", + "backend_key": "physx", + "physics_backend": "physx", + "render_backend": null, + "preset": "default", + "attempt": 1, + "was_retried": false, + "exit_code": 0, + "failure_phase": null, + "stdout_tail": "", + "wall_time_s": 23.7, + "startup_time_s": 12.5, + "perf_regression_gate_info_present": true, + "raw_fps_mean": 1655000.0, + "raw_fps_std": 9800.0, + "raw_fps_min": 1632000.0, + "raw_fps_max": 1680000.0, + "raw_fps_median": 1655000.0, + "raw_fps_p5": 1638000.0, + "raw_fps_p95": 1672000.0, + "outlier_count": null, + "gpu_diag": { + "gpu_name": "NVIDIA L40S", + "gpu_total_memory_gb": 45.62, + "cuda_version": "12.1", + "nvidia_driver_version": "550.54.15", + "gpu_mem_used_mb": 18432.0 + }, + "provenance": { + "hardware": { + "cpu_name": "Intel Xeon Gold 6438Y+", + "cpu_physical_cores": 32, + "total_ram_gb": 251.5, + "gpu_device_count": 1, + "gpu_name": "NVIDIA L40S", + "gpu_total_memory_gb": 45.62, + "gpu_compute_capability": "8.9", + "gpu_multi_processor_count": 142, + "cuda_version": "12.1" + }, + "software": { + "isaaclab": "2.1.0", + "warp": "1.6.0", + "isaacsim": "4.5.0", + "torch": "2.3.0+cu121", + "numpy": "1.26.4", + "newton": "1.0.0", + "mujoco_warp": "0.3.0" + }, + "git": { + "commit_hash": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "commit_hash_short": "a1b2c3d4", + "branch": "develop", + "commit_date": "2024-06-10 09:00:00 +0000", + "dirty": false + } + }, + "task_config_snapshot": { + "task_id": "Isaac-Velocity-Flat-G1-v0", + "backend": "physx", + "backend_key": "physx", + "physics_backend": "physx", + "render_backend": null, + "preset": "default", + "num_envs": 512, + "num_frames": 300, + "excluded_frames_raw": [[0, 100]], + "timeout_minutes": 12, + "tags": ["always"], + "seed": 42 + } +} +``` + +**Fields populated from `perf_regression_gate_info.json`** (all `null` when +`perf_regression_gate_info_present` is false): + +| Field | Source | Notes | +|---|---|---| +| `raw_fps_mean` … `raw_fps_p95` | `runtime` phase, `Step Frametimes` measurement | Full distribution before excluded-frame filtering | +| `startup_time_s` | `startup` phase, `Total Start Time (Launch to Train)` measurement | `null` if startup phase absent | +| `gpu_diag.gpu_mem_used_mb` | `runtime` phase, `GPU Memory Used` measurement | Converted from GB | +| `gpu_diag.gpu_name`, `.cuda_version`, `.gpu_total_memory_gb` | `hardware_info` phase | | +| `gpu_diag.nvidia_driver_version` | `nvidia-smi` subprocess at post-processing time | `null` if nvidia-smi unavailable | +| `provenance.hardware` | `hardware_info` phase | CPU, GPU, RAM identity | +| `provenance.software` | `version_info` phase | Package versions; `_version` suffix stripped | +| `provenance.git` | `version_info` phase, `dev` dict | commit, branch, date, dirty flag | + +Key fields the oracle reads: `perf_regression_gate_info_present`, `failure_phase`, +`was_retried`, `gpu_diag.gpu_mem_used_mb`, `startup_time_s`, `wall_time_s`. + +The `raw_fps_*` fields capture the full unfiltered distribution and are for audit/debug; +`oracle.compare()` recomputes mean FPS independently after applying `excluded_frames`. + +### `perf_regression_gate_info.json` (Phase 1 output, renamed from `benchmark_non_rl_*.json`) + +A list of `TestPhase` objects serialized by `JSONFileMetrics`. Measurement and metadata +names are prefixed with `"{task_id} {phase_name} "` by the serializer. + +```json +[ + { + "phase_name": "hardware_info", + "measurements": [], + "metadata": [ + {"name": " hardware_info cpu_name", "data": "Intel Xeon Gold 6438Y+", "type": "string"}, + {"name": " hardware_info physical_cores", "data": 32, "type": "int"}, + {"name": " hardware_info total_ram_gb", "data": 251.5, "type": "float"}, + {"name": " hardware_info gpu_device_count", "data": 1, "type": "int"}, + {"name": " hardware_info cuda_version", "data": "12.1","type": "string"}, + {"name": " hardware_info gpu_devices", "data": { + "0": {"name": "NVIDIA L40S", "total_memory_gb": 45.62, + "compute_capability": "8.9", "multi_processor_count": 142} + }, "type": "dict"} + ] + }, + { + "phase_name": "version_info", + "measurements": [], + "metadata": [ + {"name": " version_info isaaclab_version", "data": "2.1.0", "type": "string"}, + {"name": " version_info warp_version", "data": "1.6.0", "type": "string"}, + {"name": " version_info dev", "data": { + "commit_hash": "a1b2c3d4...", "commit_hash_short": "a1b2c3d4", + "branch": "develop", "commit_date": "2024-06-10 09:00:00 +0000", "dirty": false + }, "type": "dict"} + ] + }, + { + "phase_name": "runtime", + "measurements": [ + { + "name": " runtime Step Frametimes", + "value": {"Environment step effective FPS": [1644720.5, 1638400.0, ...]}, + "type": "dict" + }, + { + "name": " runtime GPU Memory Used", + "value": 18.0, + "unit": "GB", + "type": "single" + } + ], + "metadata": [] + }, + { + "phase_name": "startup", + "measurements": [ + { + "name": " startup Total Start Time (Launch to Train)", + "value": 12.5, + "unit": "s", + "type": "single" + } + ], + "metadata": [] + } +] +``` + +The oracle looks for `phase_name == "runtime"`, then the measurement whose `name` ends with +`"Step Frametimes"`, then extracts `value["Environment step effective FPS"]` as the raw series. +`build_bench_result.py` reads the remaining phases for provenance extraction. + +### `stats.json` (baseline statistics) + +```json +{ + "median_fps": 1667482.3, + "mad_fps": 8337.4, + "k_warn": 2.5, + "k_block": 4.0, + "sample_count": 8 +} +``` + +### `window.ndjson` (baseline raw window) + +One FPS float per line, append-only: + +``` +1644720.5 +1668000.1 +1655000.0 +... +``` diff --git a/tools/perf_regression_gate/docs/system-design.md b/tools/perf_regression_gate/docs/system-design.md new file mode 100644 index 000000000000..c77268ae7f45 --- /dev/null +++ b/tools/perf_regression_gate/docs/system-design.md @@ -0,0 +1,446 @@ +# IsaacLab CI Performance Regression Gate — System Design +**Status:** POC / MVP complete: pipeline runs end-to-end locally, CI integration pending +**Date:** 2026-06-10 +**Owners:** Angelina Hu, Neil Mehta + +--- + +## 1. Purpose and Use Cases + +The performance regression gate runs a fixed benchmark matrix on every PR and blocks merge +when throughput drops below a MAD-derived threshold relative to a rolling baseline. + +**Use cases:** + +| When | What happens | +|---|---| +| Feature PR touches physics or RL code | 5 "always" benchmarks run automatically | +| PR touches camera/rendering paths | 5 additional Shadow-Vision camera benchmarks added | +| Any task regresses > k_block × MAD below baseline | Aggregate exits 1; GitHub required check fails | +| Gate is in advisory mode (`blocking: false`) | Verdicts print but PR is not blocked | +| Baseline does not yet exist | Seed run: PASS unconditionally (no regression to measure) | +| Protected branch (main/develop) merges | Baseline window extended with the new FPS sample | + +--- + +## 2. Design Principles + +1. **One source of truth** Task and backend parameters live in `tasks.json`. + Python, shell, and GitHub Actions YAML all read from it so there is no duplication of info. + +2. **Modularizable** Logic should be back-end and task agnostic; backend is a data dimension in + `tasks.json`, not a logic branch in `oracle.py`, `subprocess_runner.py`, or `task_config.py`. + Each pipeline stage should have proper separation of concerns. Individual components should + be agnostic to environment/call method as long as contract is maintained. + +3. **Minimal invasiveness** Bench jobs directly call `benchmark_non_rl.py --benchmark_backend json`. + They do not invoke `tools/conftest.py` at runtime, interfere with existing tests, or modify task code. + +4. **Traceability** Every stage leaves informative artifacts. Every bench job writes `perf_regression_gate_result.json` + regardless of success or failure so the aggregator always has a structured artifact to read. + +5. **Lightweight** Bench jobs are only run when necessary and with minimally sufficient configs. + Warmed caches are pulled when needed (Newton). + +--- + +## 3. System Diagram + +``` +PR opened (to main / release / develop) + │ + ▼ +┌─────────────────────────────────────────┐ +│ .github/workflows/perf-gate.yml │ +│ │ +│ 1. Expand task matrix from tasks.json │ +│ 2. Activate tags from changed files │ +│ 3. Fan out: one runner per task │ +└────────────────┬────────────────────────┘ + │ (parallel per task) + ┌──────────┴──────────┐ + ▼ ▼ +┌───────────┐ ┌───────────┐ +│ Phase 1 │ ... │ Phase 1 │ benchmark_non_rl.py +│ Cartpole │ │ G1/newton │ --benchmark_backend json +│ /physx │ │ │ writes benchmark_non_rl_*.json +└─────┬─────┘ └─────┬─────┘ + │ │ + ▼ ▼ +┌───────────┐ ┌───────────┐ +│ Phase 2 │ ... │ Phase 2 │ build_bench_result.py +│ │ │ │ renames → perf_regression_gate_info.json +│ │ │ │ classifies failure_phase +│ │ │ │ writes perf_regression_gate_result.json +└─────┬─────┘ └─────┬─────┘ + │ │ + └──────────┬──────────┘ + │ (artifacts dir) + ▼ + ┌────────────────┐ + │ Phase 3 │ aggregate.py + │ aggregate │ oracle.compare() per task + │ + oracle │ prints verdict table + │ │ updates baseline window (if allowed) + └───────┬────────┘ + │ + ┌───────┴────────┐ + │ exit 0 │ all PASS/WARN, or gate non-blocking + │ exit 1 │ any BLOCK + blocking=true + │ exit 2 │ any HARD_FAILURE + blocking=true + └────────────────┘ +``` + +--- + +## 4. Repository File Layout + +``` +IsaacLab/ +├── .github/ +│ └── workflows/ +│ ├── build.yaml MODIFIED: add image_tag output to build job +│ └── perf-regression-gate.yaml CI gate workflow +│ +└── tools/ + ├── conftest.py MODIFIED: one-line import change + ├── subprocess_runner.py run_benchmark(), classify_failure_phase() + │ capture_test_output_with_timeout() borrowed from + │ existing conftest CI infrastructure + │ + └── perf_regression_gate/ + ├── __init__.py + ├── tasks.json SINGLE SOURCE OF TRUTH — task/backend matrix + ├── task_config.py TaskConfig dataclass, load_tasks(), get_task() + ├── tasks_to_ci_matrix.py Converts tasks.json → GitHub Actions matrix JSON + │ (called by perf-regression-gate.yaml build_matrix step) + ├── oracle.py compare() → OracleResult; PASS/WARN/BLOCK/HARD_FAILURE + ├── build_bench_result.py Phase 2: reads log + benchmark JSON, + │ extracts FPS stats + SW/HW/git provenance, + │ writes perf_regression_gate_result.json + ├── aggregate.py Phase 3: scans result JSONs, calls oracle, + │ prints table, updates baselines, exits 0/1/2 + ├── baseline_manager.py load/update baseline, flat-file + git variants + ├── gate_config.py load_gate_config() — reads {"blocking": bool} + ├── local_runner.py LOCAL END-TO-END RUNNER: orchestrates Phase 1+2+3 + | without Github/Docker/cloud platform dependencies + │ + ├── dev/ + │ ├── stub_benchmark.py Simulates benchmark_non_rl.py for unit tests + │ └── sim_regression.py Injects regressed FPS artifacts for demos + │ + ├── docs/ + │ ├── system-design.md High-level overview + │ ├── module-interfaces.md Full function/CLI interface reference + │ + └── tests/ Unit tests (no GPU) +``` + +**Baseline storage:** + +``` +Local (testing): +tools/perf_regression_gate/local_baselines/ + {gpu_model}/{task_id}/{backend_key}/ + stats.json {"median_fps", "mad_fps", "k_warn", "k_block", "sample_count"} + window.ndjson append-only rolling window, one FPS float per line + +Production (planned): +perf-baselines branch (git orphan) + {gpu_model}/{task_id}/{backend_key}/{backend_version}/{runtime_hash}/{code_fingerprint}/ + stats.json + window.ndjson + meta.json +``` + +--- + +## 5. Three-Phase Pipeline + +``` +Phase 1 — bench tasks.json → matrix → benchmark_non_rl.py (one process per task/backend) +Phase 2 — post-bench build_bench_result.py (reads log + benchmark JSON → writes result JSON) +Phase 3 — aggregate aggregate.py + oracle → verdict table → baseline update +``` + +### Phase 1: `benchmark_non_rl.py` + +Called via `./isaaclab.sh -p scripts/benchmarks/benchmark_non_rl.py`: + +```bash +./isaaclab.sh -p scripts/benchmarks/benchmark_non_rl.py \ + --task Isaac-Cartpole-Direct-v0 \ + --num_envs 4096 \ + --num_frames 300 \ + --benchmark_backend json \ + --output_path \ + [presets=newton_mjwarp] +``` + +Output: `benchmark_non_rl_{task_id}_{timestamp}.json` in `artifact_dir`. + +- Only step that depends on IsaacLab run-time. +- Need the `--benchmark_backend json` because the JSON backend preserves `DictMeasurement` +objects including the raw per-step FPS list but the OmniPerf backend drops these. + +### Phase 2: `build_bench_result.py` + +Runs once per task after Phase 1 completes: + +- Renames `benchmark_non_rl_*.json` → `perf_regression_gate_info.json` +- Classifies failure phase by scanning the benchmark log +- Parses the info artifact to extract FPS distribution statistics, startup time, GPU diagnostics, and full SW/HW/git provenance +- Writes `perf_regression_gate_result.json` (always written, even on failure) + +### Phase 3: `aggregate.py` + +Plain Python. Scans `--artifacts_dir` recursively for `perf_regression_gate_result.json`, +calls `oracle.compare()` for each, prints the verdict table, optionally updates baselines. + +Exit codes: 0 = all clear or non-blocking; 1 = BLOCK + blocking mode; 2 = HARD_FAILURE + blocking mode. + +--- + +## 6. Run Modes and Tag System + +Each task entry in `tasks.json` has a `"tags"` array. CI activates tags from the PR's changed +file list; the benchmark matrix is filtered to tasks whose tags intersect the activated set. + +| Tag | Meaning | Tasks | +|---|---|---| +| `"always"` | Run on every PR | Cartpole ×2, Factory ×1, G1 ×2 (5 tasks) | +| `"camera"` | Run when camera/rendering paths change | Shadow-Vision ×5 | + +Shadow-Vision has `"camera"` rather than `"always"` because its FPS is dominated by rendering +cost, not physics and Factory already covers manipulation and high-contact behavior, so it is +only a signal when camera code changes to save test time cost. + +**Tag activation rules WIP (production):** +- Any changed file → `"always"` always activated +- Files matching `source/isaaclab/sensors/**` or `source/isaaclab/envs/**/*vision*` → `"camera"` also activated + +--- + +## 7. Full Task Matrix + +10 (task, backend) combinations. "Effective FPS" = per-env FPS × num_envs. + +| task_id | backend_key | num_envs | frames | timeout | tags | floor (L40S) | +|---|---|---|---|---|---|---| +| Isaac-Cartpole-Direct-v0 | physx | 4096 | 300 | 10 min | always | 100 | +| Isaac-Cartpole-Direct-v0 | newton | 4096 | 300 | 10 min | always | 0 | +| Isaac-Factory-GearMesh-Direct-v0 | physx | 512 | 300 | 15 min | always | 30 | +| Isaac-Velocity-Flat-G1-v0 | physx | 512 | 300 | 12 min | always | 40 | +| Isaac-Velocity-Flat-G1-v0 | newton | 512 | 300 | 12 min | always | 0 | +| Isaac-Repose-Cube-Shadow-Vision-Direct-v0 | physx | 512 | 300 | 20 min | camera | 20 | +| Isaac-Repose-Cube-Shadow-Vision-Direct-v0 | physx_newton_renderer | 512 | 300 | 20 min | camera | 0 | +| Isaac-Repose-Cube-Shadow-Vision-Direct-v0 | newton | 512 | 300 | 20 min | camera | 0 | +| Isaac-Repose-Cube-Shadow-Vision-Direct-v0 | newton_newton_renderer | 512 | 300 | 20 min | camera | 0 | +| Isaac-Repose-Cube-Shadow-Vision-Direct-v0 | newton_ovrtx_renderer | 512 | 300 | 20 min | camera | 0 | + +`backend_key` = `{physics}` or `{physics}_{render}`. Preset tokens are derived automatically +by `local_runner.py` and the CI workflow. + +Floor = 0 means "no hard floor"; baseline MAD thresholds apply only. + +--- + +## 8. Oracle Logic + +``` +compare(bench_result, baseline, fps_mean_floor, excluded_frames, artifact_dir) + → OracleResult +``` + +**Verdict decision tree:** + +``` +perf_regression_gate_info_present == False? + → HARD_FAILURE (file-based check skipped) + +Load perf_regression_gate_info.json, extract fps_series from runtime phase +Apply excluded_frames filter +filtered empty? + → HARD_FAILURE + +mean_fps = statistics.mean(filtered) + +mean_fps < fps_mean_floor? + → BLOCK + +baseline is None or baseline.sample_count < 2? + → PASS (seed run) + +mean_fps < baseline.median - 4.0 * baseline.mad? + → BLOCK +mean_fps < baseline.median - 2.5 * baseline.mad? + → WARN +else + → PASS + +verdict == PASS and was_retried? + → downgrade to WARN +``` + +**Bisect verdicts:** + +| Oracle verdict | Condition | Bisect | +|---|---|---| +| PASS | clean first attempt | GOOD | +| PASS | was_retried | SKIP | +| WARN | any | SKIP | +| BLOCK | any | BAD | +| HARD_FAILURE | failure_phase in {init, runtime} | BAD | +| HARD_FAILURE | failure_phase in {import, driver, oom, hang, None} | SKIP | + +--- + +## 9. Failure Phase Classification + +`classify_failure_phase()` in `subprocess_runner.py` scans combined stdout+stderr in priority +order. Classification is entirely string-pattern-based — no backend branching. + +| `failure_phase` | Trigger | Bisect | +|---|---|---| +| `"oom"` | exit 137 or `"oom-kill"` in stderr | SKIP | +| `"hang"` | wall_time ≥ timeout × 0.95 | SKIP | +| `"import"` | `"Traceback"` in output, no `"AppLauncher"` seen | SKIP | +| `"driver"` | `"CudaError"` or `"CUDA_ERROR_"` in output | SKIP | +| `"init"` | exit ≠ 0, AppLauncher present, no `"Step Frametimes"` | BAD | +| `"runtime"` | exit ≠ 0, `"Step Frametimes"` present (partial run) | BAD | +| `null` | exit 0, no error markers | (clean) | + +`import` → SKIP because import failures indicate environment mismatch, not code regression. +`init` and `runtime` → BAD because Isaac Sim started normally: the failure is the code's fault. + +--- + +## 10. Baseline Storage + +**Flat-file (local testing):** + +``` +local_baselines/{gpu_model}/{task_id}/{backend_key}/ + stats.json {"median_fps", "mad_fps", "k_warn", "k_block", "sample_count"} + window.ndjson one FPS float per line, append-only +``` + +`baseline_manager.update_baseline()` appends to `window.ndjson`, recomputes median and MAD, +and rewrites `stats.json`. BLOCK results are never written. + +**Git branch (production):** `perf-baselines` orphan branch. `baseline_manager.update_baseline_git()` +uses a temporary git worktree to write and commit atomically. + +**Write policy:** Writes occur only on protected branches (main/develop/release/*) or with +`--allow_baseline_update` (local testing flag). Feature branch runs are read-only. + +--- + +## 11. Artifact Schema (Key Fields) + +`perf_regression_gate_result.json` (Phase 2 output): + +```json +{ + "task_id": "Isaac-Velocity-Flat-G1-v0", + "backend": "physx", + "failure_phase": null, + "wall_time_s": 23.7, + "startup_time_s": 12.5, + "perf_regression_gate_info_present": true, + "raw_fps_mean": 1655000.0, + "raw_fps_std": 9800.0, + "raw_fps_min": 1632000.0, + "raw_fps_max": 1680000.0, + "raw_fps_median": 1655000.0, + "raw_fps_p5": 1638000.0, + "raw_fps_p95": 1672000.0, + "gpu_diag": { + "gpu_name": "NVIDIA L40S", + "gpu_total_memory_gb": 45.62, + "cuda_version": "12.1", + "nvidia_driver_version": "550.54.15", + "gpu_mem_used_mb": 18432.0 + }, + "provenance": { + "hardware": { "cpu_name": "...", "gpu_name": "NVIDIA L40S", "cuda_version": "12.1", "..." : "..." }, + "software": { "isaaclab": "2.1.0", "warp": "1.6.0", "torch": "2.3.0+cu121", "...": "..." }, + "git": { "commit_hash": "a1b2c3d4...", "branch": "develop", "dirty": false } + }, + "task_config_snapshot": { "..." : "..." } +} +``` + +`raw_fps_*` fields capture the unfiltered per-step distribution for audit/debug; +`oracle.compare()` recomputes mean FPS independently after applying `excluded_frames`. +`provenance` enables cross-run comparison: when two baselines diverge, diff their +`provenance` blocks to identify driver, CUDA, or package version changes. + +See `module-interfaces.md` for the full schema with all fields. + +`perf_regression_gate_info.json` (Phase 1 output, renamed from `benchmark_non_rl_*.json`): + +A list of `TestPhase` objects. The oracle reads `"Environment step effective FPS"` from the +`"runtime"` phase's `"Step Frametimes"` measurement. `build_bench_result.py` also reads +the `"hardware_info"`, `"version_info"`, and `"startup"` phases for provenance extraction. + +--- + +## 12. Environment Requirements (Local Testing) + +Validated configuration as of 2026-06-10: + +| Component | Version | +|---|---| +| IsaacSim | 6.0.0.1 (NOT 6.0.0.0 — `omni.physics.tensors.api` moved in 6.0.0.1) | +| isaacsim-extscache-physics | 6.0.0.1 | +| warp-lang | 1.12.0 (NOT 1.13.0 — `warp.context` removed, breaks omni.replicator.core) | +| mujoco-warp | 3.8.1 | +| Python | 3.12 | +| GPU | RTX 5090 (local) / L40S (production CI) | + +**Installation after fresh `./isaaclab.sh -i --extra rl`:** + +```bash +pip install isaacsim==6.0.0.1 isaacsim-extscache-physics==6.0.0.1 +pip install warp-lang==1.12.0 +``` + +--- + +## 13. Alignment with OVPLC Testing Principles + +### 13.1 PR-gated runs vs. nightly-authoritative runs + +**Principle:** Authoritative runs SHOULD execute on a nightly schedule. +**Deviation:** We run per-PR and write baselines from protected branches. +**Defense:** We use wide tolerance bands (k_warn=2.5, k_block=4.0 MAD) to +absorb inter-run variance. and the purpose of our tests is to catch issues live rather +than nightly (existing OmniPerf test suite). + +### 13.2 Single FPS sample per run vs. N=10 iterations + +**Principle:** Minimum N=10 iterations per benchmark. +**Deviation:** Each CI run produces one FPS value (mean of 200 post-warmup frames) +**Defense:** Simulation steps ARE independent samples and this gate is meant to be light-weight. +The baseline window accumulates samples across runs, building MAD statistics over +the true run-to-run variance distribution. + +### 13.3 Memory tracking scope + +**Current state:** `gpu_diag.gpu_mem_used_mb` is captured and surfaced in `OracleResult` +as an informational field. No memory regression threshold yet. + +--- + +## 14. Bisect Engine (Future / TBD) + +`oracle.py` already populates `bisect_verdict` (`GOOD`/`BAD`/`SKIP`) on every run. +A bisect engine would consume it directly and execute a similar workflow to local_runner: + +``` +tools/bisect/ + env_resolver.py resolve_env(commit) → ResolvedEnv { isaaclab_root, python_launcher, … } + bisect_runner.py run_at_commit(env, task_id, backend) → (Path|None, SubprocessResult) + bisect_engine.py git bisect loop using OracleResult.bisect_verdict +``` diff --git a/tools/perf_regression_gate/gate_config.py b/tools/perf_regression_gate/gate_config.py new file mode 100644 index 000000000000..2c1ad7f09cd3 --- /dev/null +++ b/tools/perf_regression_gate/gate_config.py @@ -0,0 +1,10 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import json +from pathlib import Path + +def load_gate_config(path: Path | str) -> dict: + return {"blocking": False} diff --git a/tools/perf_regression_gate/local_runner.py b/tools/perf_regression_gate/local_runner.py new file mode 100644 index 000000000000..0e2f8706210c --- /dev/null +++ b/tools/perf_regression_gate/local_runner.py @@ -0,0 +1,321 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Local end-to-end benchmark runner without Docker/Github actions + +Mirrors the structure of the three-phase CI workflow: + + Phase 1 bench tasks.json → matrix → benchmark_non_rl.py (per task) + Phase 2 post-bench build_bench_result.py (per task, plain Python) + Phase 3 aggregate aggregate.py + oracle → verdict table + +Usage:: + + # Preview the full task matrix without running anything + python3 tools/perf_regression_gate/local_runner.py --dry_run + + # Seed/build baselines (run a few times on stable main) + python3 tools/perf_regression_gate/local_runner.py --allow_baseline_update + + # Check a branch against baselines (no update) + python3 tools/perf_regression_gate/local_runner.py + + # Run only "always"-tagged tasks on L40S label + python3 tools/perf_regression_gate/local_runner.py --tags always --gpu_model L40S + +Preconditions: + IsaacSim must be accessible via ./isaaclab.sh -p. + Activate venv before running +""" + +import argparse +import subprocess +import sys +import time +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +_TOOLS_DIR = _MODULE_DIR.parent +_REPO_ROOT = _TOOLS_DIR.parent + +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + +from task_config import TaskConfig, load_tasks # noqa: E402 + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Local end-to-end benchmark runner (no Docker required).", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument( + "--tags", + nargs="+", + default=["always"], + help="Run only tasks whose tag list overlaps this set (default: always)", + ) + p.add_argument("--gpu_model", default="L40S", help="GPU model label for baseline bucket (default: L40S)") + p.add_argument( + "--artifacts_dir", + type=Path, + default=_MODULE_DIR / "artifacts", + help="Root directory for per-task benchmark artifacts (default: tools/perf_regression_gate/artifacts/)", + ) + p.add_argument( + "--baselines_dir", + type=Path, + default=_MODULE_DIR / "local_baselines", + help="Flat-file baseline directory (default: tools/perf_regression_gate/local_baselines/)", + ) + p.add_argument( + "--allow_baseline_update", + action="store_true", + help="Append this run's FPS to the baseline window (use when building baselines on stable main)", + ) + p.add_argument( + "--dry_run", + action="store_true", + help="Print the expanded task matrix and exit without running anything", + ) + p.add_argument( + "--skip_existing", + action="store_true", + help="Skip tasks whose bench_result.json already exists in --artifacts_dir", + ) + p.add_argument( + "--gate_config", + type=Path, + default=_MODULE_DIR / "gate_config.json", + help="Path to gate_config.json (controls advisory vs blocking mode)", + ) + return p.parse_args() + + +# --------------------------------------------------------------------------- +# Phase 1 helpers: matrix expansion and benchmark execution +# --------------------------------------------------------------------------- + + +def _print_matrix(tasks: list[TaskConfig], tags: list[str]) -> None: + print(f"\n{'=' * 68}") + print(f" Phase 1 — Task Matrix ({len(tasks)} entries, tags={tags})") + print(f"{'=' * 68}") + col = "{:<42} {:<14} {:>8} {:>8} {:>5}" + print(col.format("task_id", "backend_key", "num_envs", "frames", "tmin")) + print("-" * 68) + for t in tasks: + print(col.format(t.task_id, t.backend_key, t.num_envs, t.num_frames, t.timeout_minutes)) + print() + + +def _isaaclab_cmd(bench_script: Path, task: TaskConfig, artifact_dir: Path) -> list[str]: + """Build the ./isaaclab.sh -p benchmark_non_rl.py command for one task/backend.""" + cmd = [ + str(_REPO_ROOT / "isaaclab.sh"), + "-p", + str(bench_script), + "--task", task.task_id, + "--num_envs", str(task.num_envs), + "--num_frames", str(task.num_frames), + "--benchmark_backend", "json", + "--output_path", str(artifact_dir), + ] + # Combine physics and render presets into a single Hydra token. + # newton physics requires newton_mjwarp; render backends use their canonical + # preset name (newton_renderer for Warp, ovrtx_renderer for OV-RTX, absent for + # default RTX). Warp renderer also requires the rgb data-type preset to avoid + # the semantic_segmentation incompatibility in ShadowHandVisionEnvCfg. + presets: list[str] = [] + if task.physics_backend == "newton": + presets.append("newton_mjwarp") + if task.render_backend: + presets.append(task.render_backend) + if task.render_backend == "newton_renderer": + presets.append("rgb") + if presets: + cmd.append(f"presets={','.join(presets)}") + if task.seed is not None: + cmd.extend(["--seed", str(task.seed)]) + return cmd + + +def _run_benchmark(task: TaskConfig, artifact_dir: Path, bench_script: Path) -> tuple[int, float]: + """Run benchmark_non_rl.py for one task, streaming output live + writing to log. + + Returns (exit_code, wall_time_s). + """ + log_file = artifact_dir / "benchmark.log" + cmd = _isaaclab_cmd(bench_script, task, artifact_dir) + + print(f" cmd: {' '.join(cmd)}") + print(f" log: {log_file}") + print() + + start = time.monotonic() + with open(log_file, "w") as log_fh: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + assert proc.stdout is not None + for raw_line in proc.stdout: + line = raw_line.decode(errors="replace") + sys.stdout.write(line) + sys.stdout.flush() + log_fh.write(line) + proc.wait() + wall_time = time.monotonic() - start + return proc.returncode, wall_time + + +# --------------------------------------------------------------------------- +# Phase 2 helper: build_bench_result.py +# --------------------------------------------------------------------------- + + +def _run_build_bench_result( + task: TaskConfig, + artifact_dir: Path, + exit_code: int, + wall_time: float, + *, + was_retried: bool = False, + attempt: int = 1, +) -> None: + """Run build_bench_result.py for one task (plain Python, no IsaacSim needed)""" + build_script = _MODULE_DIR / "build_bench_result.py" + cmd = [ + sys.executable, + str(build_script), + "--task_id", task.task_id, + "--physics_backend", task.physics_backend, + "--render_backend", task.render_backend or "", + "--artifact_dir", str(artifact_dir), + "--exit_code", str(exit_code), + "--wall_time_s", f"{wall_time:.1f}", + "--timeout_s", str(task.timeout_minutes * 60), + "--log_file", str(artifact_dir / "benchmark.log"), + "--attempt", str(attempt), + ] + if was_retried: + cmd.append("--was_retried") + subprocess.run(cmd, check=True) + + +# --------------------------------------------------------------------------- +# Phase 3 helper: aggregate.py +# --------------------------------------------------------------------------- + + +def _run_aggregate(artifacts_dir: Path, baselines_dir: Path, gpu_model: str, + gate_config: Path, allow_baseline_update: bool) -> int: + """Run aggregate.py over all artifacts and return its exit code.""" + agg_script = _MODULE_DIR / "aggregate.py" + cmd = [ + sys.executable, + str(agg_script), + "--artifacts_dir", str(artifacts_dir), + "--gpu_model", gpu_model, + "--baselines_dir", str(baselines_dir), + "--gate_config", str(gate_config), + "--allow_baseline_update", "true" if allow_baseline_update else "false", + ] + result = subprocess.run(cmd) + return result.returncode + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main() -> int: + args = _parse_args() + + # Expand matrix from tasks.json + all_tasks = load_tasks() + tag_set = frozenset(args.tags) + tasks = [t for t in all_tasks if tag_set.intersection(frozenset(t.tags))] + + if not tasks: + print(f"[local_runner] No tasks match tags {args.tags}.") + return 1 + + _print_matrix(tasks, args.tags) + + if args.dry_run: + print("[local_runner] --dry_run: exiting without running benchmarks.") + return 0 + + bench_script = _REPO_ROOT / "scripts" / "benchmarks" / "benchmark_non_rl.py" + if not bench_script.exists(): + print(f"[local_runner] ERROR: benchmark script not found at {bench_script}") + return 1 + + # ----------------------------------------------------------------------- + # Phase 1 + 2: benchmark → build_bench_result, one task at a time + # ----------------------------------------------------------------------- + print(f"{'=' * 68}") + print(" Phase 1+2 — Benchmark + Post-process") + print(f"{'=' * 68}\n") + + for i, task in enumerate(tasks, 1): + artifact_dir = args.artifacts_dir / task.task_id / task.backend_key + artifact_dir.mkdir(parents=True, exist_ok=True) + + bench_result_path = artifact_dir / "bench_result.json" + if args.skip_existing and bench_result_path.exists(): + print(f"[{i}/{len(tasks)}] SKIP (bench_result.json exists): {task.task_id} / {task.backend_key}") + continue + + print(f"[{i}/{len(tasks)}] {task.task_id} / {task.backend_key}") + print(f" envs={task.num_envs} frames={task.num_frames} timeout={task.timeout_minutes}m") + + exit_code, wall_time = _run_benchmark(task, artifact_dir, bench_script) + + was_retried = False + if exit_code != 0: + print(f"\n[{i}/{len(tasks)}] first attempt failed (exit={exit_code}), retrying once...") + # Remove any partial perf output from the failed attempt so the retry is clean + stale = artifact_dir / "perf_regression_gate_info.json" + if stale.exists(): + stale.unlink() + exit_code, wall_time = _run_benchmark(task, artifact_dir, bench_script) + was_retried = True + + attempt = 2 if was_retried else 1 + print(f"\n[{i}/{len(tasks)}] exit={exit_code} wall={wall_time:.0f}s attempt={attempt} → build_bench_result") + _run_build_bench_result(task, artifact_dir, exit_code, wall_time, was_retried=was_retried, attempt=attempt) + print() + + # ----------------------------------------------------------------------- + # Phase 3: aggregate + oracle + # ----------------------------------------------------------------------- + print(f"\n{'=' * 68}") + print(" Phase 3 — Aggregate + Oracle Verdict") + print(f"{'=' * 68}\n") + + if args.allow_baseline_update: + print("[local_runner] --allow_baseline_update: PASS/WARN results will extend the baseline window.\n") + else: + print("[local_runner] Read-only baseline run (pass --allow_baseline_update to extend window).\n") + + rc = _run_aggregate( + args.artifacts_dir, + args.baselines_dir, + args.gpu_model, + args.gate_config, + args.allow_baseline_update, + ) + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_regression_gate/oracle.py b/tools/perf_regression_gate/oracle.py new file mode 100644 index 000000000000..49e88a3ed0f0 --- /dev/null +++ b/tools/perf_regression_gate/oracle.py @@ -0,0 +1,265 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Oracle layer for the CI performance regression gate. + +Provides verdict computation (PASS / WARN / BLOCK / HARD_FAILURE) by comparing +a measured FPS sample against a rolling baseline.\ +""" + +import json +import statistics +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + + +class OracleVerdict(str, Enum): + PASS = "PASS" + WARN = "WARN" + BLOCK = "BLOCK" + HARD_FAILURE = "HARD_FAILURE" + + +@dataclass +class Baseline: + """Rolling-window statistics used as the comparison reference for a single (task, backend) pair. + + Args: + median_fps: Median FPS computed from the baseline window + mad_fps: Median absolute deviation of FPS in the baseline window + k_warn: Number of MADs below the median that triggers a WARN verdict + k_block: Number of MADs below the median that triggers a BLOCK verdict + sample_count: Number of samples in the window used to compute the stats + """ + + median_fps: float + mad_fps: float + k_warn: float = 2.5 # TODO: decide on real threshold values + k_block: float = 4.0 # TODO: decide on real threshold values + sample_count: int = 0 + + +@dataclass +class OracleResult: + """Full verdict record produced by :func:`compare`""" + + verdict: OracleVerdict # High-level verdict: PASS / WARN / BLOCK / HARD_FAILURE + bisect_verdict: str # GOOD / BAD / SKIP for bisect compatibility + failure_phase: str | None # Phase classification from build_bench_result (e.g. "import", "init", "runtime", "oom", "hang", "driver", or None) + measured_fps: float | None # Mean FPS after excluded-frame filtering, or None on hard failure; blocking metric + baseline_fps: float | None # baseline.median_fps, or None + regression_pct: float | None # ((measured_fps - baseline_fps) / baseline_fps) * 100, or None + fps_median: float | None # Median FPS of the filtered series [informational] + fps_p5: float | None # 5th-percentile FPS of the filtered series [informational] + fps_p95: float | None # 95th-percentile FPS of the filtered series [informational] + gpu_mem_used_mb: float | None # GPU memory used at benchmark time [MiB], or None + startup_time_s: float | None # Startup time reported by the benchmark process [s] + wall_time_s: float | None # Wall-clock time of the benchmark run [s] + was_retried: bool # True when the benchmark succeeded only after a retry, False otherwise + task_id: str # Benchmark task identifier + backend: str # Physics/Render backend name + + +# --------------------------------------------------------------------------- +# Bisect verdict mapping (spec section "Bisect verdict mapping") +# --------------------------------------------------------------------------- + +# failure_phase values that map HARD_FAILURE -> "BAD" +_BISECT_BAD_PHASES: frozenset[str] = frozenset({"init", "runtime"}) + +# failure_phase values (and None) that map HARD_FAILURE -> "SKIP" +_BISECT_SKIP_PHASES: frozenset[str | None] = frozenset({"import", "driver", "oom", "hang", None}) + + +def _bisect_verdict(verdict: OracleVerdict, was_retried: bool, failure_phase: str | None) -> str: + """Compute the bisect-friendly label for a given verdict""" + if verdict == OracleVerdict.PASS: + return "SKIP" if was_retried else "GOOD" + if verdict == OracleVerdict.WARN: + return "SKIP" + if verdict == OracleVerdict.BLOCK: + return "BAD" + # HARD_FAILURE + if failure_phase in _BISECT_BAD_PHASES: + return "BAD" + return "SKIP" + + +# --------------------------------------------------------------------------- +# Core functions +# --------------------------------------------------------------------------- + + +def _percentile(sorted_data: list[float], p: float) -> float: + """Linear-interpolation percentile on a pre-sorted list + + Args: + sorted_data: Ascending-sorted FPS values (must be non-empty) + p: Percentile in [0, 100] + + Returns: + Interpolated value at the requested percentile + """ + n = len(sorted_data) + if n == 1: + return sorted_data[0] + idx = p / 100.0 * (n - 1) + lo = int(idx) + hi = min(lo + 1, n - 1) + return sorted_data[lo] + (sorted_data[hi] - sorted_data[lo]) * (idx - lo) + + +def apply_excluded_frames(fps_series: list[float], excluded_frames: frozenset[int]) -> list[float]: + """Return fps_series with frames at 0-based indices listed in excluded_frames removed""" + if not excluded_frames: + return list(fps_series) + return [fps for idx, fps in enumerate(fps_series) if idx not in excluded_frames] + + +def compare( + bench_result: dict, + baseline: "Baseline | None", + fps_mean_floor: float, + excluded_frames: "frozenset[int]", + artifact_dir: "Path", +) -> OracleResult: + """Compare a benchmark result against its baseline and return an :class:`OracleResult` + + Oracle logic (in order): + + 1. If ``failure_phase`` is set **and** ``perf_regression_gate_info_present`` is False, return + ``HARD_FAILURE`` immediately without reading any files + 2. Load ``artifact_dir/perf_regression_gate_info.json`` and extract the FPS series + 3. Filter the series with :func:`apply_excluded_frames`. + 4. Compute mean FPS with :mod:`statistics`. + 5. Apply hard-floor check, then baseline thresholds (or seed-run PASS) + 6. Downgrade PASS to WARN when ``was_retried`` is True + 7. Compute bisect verdict and regression percentage + + Args: + bench_result: Dict matching the ``perf_regression_gate_result.json`` schema + baseline: Rolling baseline statistics, or None for a seed run + fps_mean_floor: Absolute minimum acceptable FPS (hard floor) + excluded_frames: 0-based frame indices to drop before computing mean FPS + artifact_dir: Directory that contains ``perf_regression_gate_info.json`` + + Returns: + Fully populated :class:`OracleResult` + """ + task_id: str = bench_result["task_id"] + backend: str = bench_result["backend"] + failure_phase: str | None = bench_result["failure_phase"] + was_retried: bool = bench_result["was_retried"] + startup_time_s: float | None = bench_result.get("startup_time_s") + wall_time_s: float | None = bench_result.get("wall_time_s") + gpu_mem_used_mb: float | None = (bench_result.get("gpu_diag") or {}).get("gpu_mem_used_mb") + + # Treat missing perf data as HARD_FAILURE regardless of failure_phase + if not bench_result["perf_regression_gate_info_present"]: + bv = _bisect_verdict(OracleVerdict.HARD_FAILURE, was_retried, failure_phase) + return OracleResult( + verdict=OracleVerdict.HARD_FAILURE, + bisect_verdict=bv, + failure_phase=failure_phase, + measured_fps=None, + baseline_fps=None, + regression_pct=None, + fps_median=None, + fps_p5=None, + fps_p95=None, + gpu_mem_used_mb=gpu_mem_used_mb, + startup_time_s=startup_time_s, + wall_time_s=wall_time_s, + was_retried=was_retried, + task_id=task_id, + backend=backend, + ) + + # Load perf_regression_gate_info.json + perf_regression_gate_info_path = Path(artifact_dir) / "perf_regression_gate_info.json" + with perf_regression_gate_info_path.open() as fh: + perf_regression_gate_info = json.load(fh) + + # Extract FPS series + fps_series: list[float] = [] + for phase in perf_regression_gate_info: + if phase.get("phase_name") == "runtime": + for measurement in phase.get("measurements", []): + if measurement.get("name", "").endswith("Step Frametimes"): + fps_series = measurement["value"]["Environment step effective FPS"] + break + break + + # Compute filtered data, mean, and informational statistics + filtered = apply_excluded_frames(fps_series, excluded_frames) + if not filtered: + bv = _bisect_verdict(OracleVerdict.HARD_FAILURE, was_retried, failure_phase) + return OracleResult( + verdict=OracleVerdict.HARD_FAILURE, + bisect_verdict=bv, + failure_phase=failure_phase, + measured_fps=None, + baseline_fps=None, + regression_pct=None, + fps_median=None, + fps_p5=None, + fps_p95=None, + gpu_mem_used_mb=gpu_mem_used_mb, + startup_time_s=startup_time_s, + wall_time_s=wall_time_s, + was_retried=was_retried, + task_id=task_id, + backend=backend, + ) + mean_fps = statistics.mean(filtered) + sorted_filtered = sorted(filtered) + fps_median = _percentile(sorted_filtered, 50.0) + fps_p5 = _percentile(sorted_filtered, 5.0) + fps_p95 = _percentile(sorted_filtered, 95.0) + + # Verdict logic + _MIN_SAMPLES_FOR_MAD = 2 # TODO: remove + + if mean_fps < fps_mean_floor: + verdict = OracleVerdict.BLOCK + elif baseline is None or baseline.sample_count < _MIN_SAMPLES_FOR_MAD: + verdict = OracleVerdict.PASS + else: + block_thresh = baseline.median_fps - baseline.k_block * baseline.mad_fps + warn_thresh = baseline.median_fps - baseline.k_warn * baseline.mad_fps + if mean_fps < block_thresh: + verdict = OracleVerdict.BLOCK + elif mean_fps < warn_thresh: + verdict = OracleVerdict.WARN + else: + verdict = OracleVerdict.PASS + if verdict == OracleVerdict.PASS and was_retried: + verdict = OracleVerdict.WARN + + baseline_fps: float | None = baseline.median_fps if baseline is not None else None + regression_pct: float | None = None + if baseline_fps is not None: + regression_pct = ((mean_fps - baseline_fps) / baseline_fps) * 100.0 + + bisection_verdict = _bisect_verdict(verdict, was_retried, failure_phase) + + return OracleResult( + verdict=verdict, + bisect_verdict=bisection_verdict, + failure_phase=failure_phase, + measured_fps=mean_fps, + baseline_fps=baseline_fps, + regression_pct=regression_pct, + fps_median=fps_median, + fps_p5=fps_p5, + fps_p95=fps_p95, + gpu_mem_used_mb=gpu_mem_used_mb, + startup_time_s=startup_time_s, + wall_time_s=wall_time_s, + was_retried=was_retried, + task_id=task_id, + backend=backend, + ) diff --git a/tools/perf_regression_gate/task_config.py b/tools/perf_regression_gate/task_config.py new file mode 100644 index 000000000000..3b8293f14114 --- /dev/null +++ b/tools/perf_regression_gate/task_config.py @@ -0,0 +1,180 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import json +from dataclasses import dataclass, field +from pathlib import Path + +_DEFAULT_TASKS_JSON = Path(__file__).parent / "tasks.json" + +# Maps backend name to list of cache identifiers that CI needs to pull; absent key defaults to no caches +_BACKEND_CACHES: dict[str, list[str]] = { + "newton": ["mjwarp_jit"], +} + + +def caches_for_backend(backend: str) -> list[str]: + """Return cache identifiers required before benchmarking with a given physics backend. + + The returned identifiers are consumed by the CI cache-pull step to locate and + restore named cache artifacts by name or pattern. An empty list means no + pre-run cache restoration is needed. + + Currently defined identifiers: + ``"mjwarp_jit"``: Newton MJWarp JIT compilation cache. + + Args: + backend: Backend name (e.g. ``"newton"``, ``"physx"``). + + Returns: + List of cache identifier strings. + """ + return list(_BACKEND_CACHES.get(backend, [])) + + +@dataclass +class TaskConfig: + """Configuration for a single benchmark task and backend combination.""" + + task_id: str + physics_backend: str + render_backend: str | None + preset: str + num_envs: int + num_frames: int + excluded_frames_raw: list[int | list[int]] + camera_resolution: tuple[int, int] | None + timeout_minutes: int + fps_mean_floor: dict + caches: list[str] + tags: list[str] = field(default_factory=lambda: ["always"]) + task_type: str = "benchmark" + runs_on: str = "gpu-l40s" + seed: int | None = None + + @property + def backend_key(self) -> str: + """Composite key identifying the backend combination. + + Returns f"{physics_backend}_{render_backend}" when render_backend is set, + otherwise returns physics_backend. + """ + if self.render_backend: + return f"{self.physics_backend}_{self.render_backend}" + return self.physics_backend + + @property + def excluded_frames(self) -> frozenset[int]: + """Expand raw excluded_frames entries (single index or inclusive range) to a + frozenset of integer frame indices. + """ + indices: set[int] = set() + for entry in self.excluded_frames_raw: + if isinstance(entry, list): + if len(entry) != 2: + raise ValueError(f"excluded_frames range entry must have exactly 2 elements, got {entry!r}") + start, end = entry[0], entry[1] + if start > end: + raise ValueError(f"excluded_frames range start must be <= end, got [{start}, {end}]") + indices.update(range(start, end + 1)) + else: + indices.add(int(entry)) + return frozenset(indices) + + +def _load_tasks_json(path: Path) -> tuple[dict, list[dict]]: + with open(path) as f: + raw_data = json.load(f) + + if isinstance(raw_data, dict): + defaults = raw_data.get("defaults", {}) + raw_list = raw_data.get("tasks", []) + if not isinstance(raw_list, list): + raise TypeError(f"'tasks' field in {path} must be a list") + elif isinstance(raw_data, list): + defaults = {} + raw_list = raw_data + else: + raise TypeError(f"{path} must contain a JSON list or an object with a top-level 'tasks' list") + + if not isinstance(defaults, dict): + raise TypeError(f"'defaults' field in {path} must be an object") + + return defaults, raw_list + + +def load_tasks(tasks_json_path: Path | str | None = None) -> list[TaskConfig]: + """Load all benchmark tasks from tasks.json, producing a TaskConfig for each backend combination. + + Args: + tasks_json_path: Path to tasks.json. Defaults to the tasks.json next to this module. + + Returns: + List of TaskConfig objects, one per (task_id, backend) combination. + """ + path = Path(tasks_json_path) if tasks_json_path is not None else _DEFAULT_TASKS_JSON + defaults, raw_list = _load_tasks_json(path) + + tasks: list[TaskConfig] = [] + for raw in raw_list: + if not isinstance(raw, dict): + raise TypeError(f"task entry in {path} must be an object") + merged = {**defaults, **raw} + + camera_raw = merged.get("camera_resolution") + camera_resolution: tuple[int, int] | None = ( + tuple(camera_raw) if camera_raw is not None else None # type: ignore[assignment] + ) + fps_mean_floor: dict = merged.get("fps_mean_floor", {}) + backends: list[dict] = merged.get("backends", []) + + for backend_entry in backends: + physics = backend_entry["physics"] + render = backend_entry.get("render") + tasks.append( + TaskConfig( + task_id=merged["task_id"], + physics_backend=physics, + render_backend=render, + preset=merged["preset"], + num_envs=merged["num_envs"], + num_frames=merged["num_frames"], + excluded_frames_raw=merged["excluded_frames"], + camera_resolution=camera_resolution, + timeout_minutes=int(merged["timeout_minutes"]), + fps_mean_floor=fps_mean_floor, + caches=caches_for_backend(physics), + tags=merged["tags"], + task_type=merged["type"], + runs_on=merged["runs_on"], + seed=merged.get("seed"), + ) + ) + return tasks + + +def get_task( + task_id: str, + backend_key: str, + tasks_json_path: Path | str | None = None, +) -> TaskConfig: + """Return the TaskConfig for the given task_id and backend_key combination. + + Args: + task_id: The task identifier to look up. + backend_key: The backend key (e.g. "physx", "newton", "physx_rtx"). + tasks_json_path: Optional path to tasks.json. + + Returns: + The matching TaskConfig. + + Raises: + KeyError: If no task with the given (task_id, backend_key) exists. + """ + tasks = load_tasks(tasks_json_path) + for task in tasks: + if task.task_id == task_id and task.backend_key == backend_key: + return task + raise KeyError(f"Task not found: task_id={task_id!r} backend_key={backend_key!r}") diff --git a/tools/perf_regression_gate/tasks.json b/tools/perf_regression_gate/tasks.json new file mode 100644 index 000000000000..6a4e1a8776fe --- /dev/null +++ b/tools/perf_regression_gate/tasks.json @@ -0,0 +1,77 @@ +{ + "defaults": { + "type": "benchmark", + "runs_on": "gpu-l40s", + "preset": "default", + "seed": 42, + "num_envs": 512, + "num_frames": 300, + "excluded_frames": [[0, 100]], + "camera_resolution": null, + "timeout_minutes": 10, + "tags": ["always"] + }, + "tasks": [ + { + "task_id": "Isaac-Cartpole-Direct-v0", + "num_envs": 4096, + "backends": [ + {"physics": "physx"}, + {"physics": "newton"} + ], + "fps_mean_floor": { + "L40S": { + "physx": 100.0, + "newton": 0.0 + } + } + }, + { + "task_id": "Isaac-Factory-GearMesh-Direct-v0", + "timeout_minutes": 15, + "backends": [ + {"physics": "physx"} + ], + "fps_mean_floor": { + "L40S": { + "physx": 30.0 + } + } + }, + { + "task_id": "Isaac-Repose-Cube-Shadow-Vision-Direct-v0", + "timeout_minutes": 20, + "tags": ["camera"], + "backends": [ + {"physics": "physx"}, + {"physics": "physx", "render": "newton_renderer"}, + {"physics": "newton"}, + {"physics": "newton", "render": "newton_renderer"}, + {"physics": "newton", "render": "ovrtx_renderer"} + ], + "fps_mean_floor": { + "L40S": { + "physx": 20.0, + "physx_newton_renderer": 0.0, + "newton": 0.0, + "newton_newton_renderer": 0.0, + "newton_ovrtx_renderer": 0.0 + } + } + }, + { + "task_id": "Isaac-Velocity-Flat-G1-v0", + "timeout_minutes": 12, + "backends": [ + {"physics": "physx"}, + {"physics": "newton"} + ], + "fps_mean_floor": { + "L40S": { + "physx": 40.0, + "newton": 0.0 + } + } + } + ] +} diff --git a/tools/perf_regression_gate/tasks_to_ci_matrix.py b/tools/perf_regression_gate/tasks_to_ci_matrix.py new file mode 100644 index 000000000000..04271c78ca87 --- /dev/null +++ b/tools/perf_regression_gate/tasks_to_ci_matrix.py @@ -0,0 +1,37 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Convert tasks.json into the GitHub Actions bench matrix JSON. + +Prints a JSON array to stdout, one object per (task_id, backend) combination, +containing the fields consumed by the ``bench`` job matrix in perf-regression-gate.yaml. + +Usage:: + + python3 tools/perf_regression_gate/tasks_to_ci_matrix.py +""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from task_config import load_tasks # noqa: E402 + +tasks = load_tasks() +rows = [] +for task in tasks: + rows.append({ + "task_id": task.task_id, + "physics_backend": task.physics_backend, + "render_backend": task.render_backend or "", + "num_envs": task.num_envs, + "num_frames": task.num_frames, + "bench_timeout_s": task.timeout_minutes * 60, + "job_timeout_minutes": max(30, task.timeout_minutes + 15), + }) + +print(json.dumps(rows)) diff --git a/tools/subprocess_runner.py b/tools/subprocess_runner.py new file mode 100644 index 000000000000..57a750202647 --- /dev/null +++ b/tools/subprocess_runner.py @@ -0,0 +1,324 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import contextlib +import os +import select +import signal +import subprocess +import sys +import time + + +def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, report_file=""): + """Run a command with timeout and capture all output while streaming in real-time. + + Args: + cmd: Command to execute. + timeout: Maximum wall-clock seconds before the process is killed. + env: Environment variables for the subprocess. + startup_deadline: If > 0, the process is killed early when neither + ``AppLauncher initialization complete`` (stderr) nor ``collected`` + (stdout) appears within this many seconds. + report_file: Path to the JUnit XML report file. When set, the process + is given only :data:`SHUTDOWN_GRACE_PERIOD` seconds to exit after + the file appears on disk. + + Returns: + Tuple of ``(returncode, stdout_bytes, stderr_bytes, kill_reason, + wall_time, pre_kill_diag)``. *kill_reason* is ``""`` for normal exits, + ``"timeout"`` for hard timeouts, ``"startup_hang"`` when the process + did not reach pytest collection in time, or ``"shutdown_hang"`` when + the test completed but the process hung during shutdown. + """ + # Import here to avoid circular dependency; SHUTDOWN_GRACE_PERIOD is defined in conftest. + # We define a local default that matches conftest's constant. + _SHUTDOWN_GRACE_PERIOD = 30 + + stdout_data = b"" + stderr_data = b"" + process = None + + try: + # Each test gets its own session so orphaned Kit/Isaac Sim child + # processes cannot send SIGHUP to the next test's process group. + process = subprocess.Popen( + cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + universal_newlines=False, + start_new_session=True, + ) + pgid = os.getpgid(process.pid) + + stdout_fd = process.stdout.fileno() + stderr_fd = process.stderr.fileno() + + try: + import fcntl + + for fd in [stdout_fd, stderr_fd]: + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + except ImportError: + pass + + start_time = time.time() + startup_done = startup_deadline <= 0 + shutdown_deadline = 0.0 + + while process.poll() is None: + elapsed = time.time() - start_time + + if not startup_done: + if b"AppLauncher initialization complete" in stderr_data or b"collected " in stdout_data: + startup_done = True + + if report_file and not shutdown_deadline and os.path.exists(report_file): + shutdown_deadline = time.time() + _SHUTDOWN_GRACE_PERIOD + + kill_reason = None + if not startup_done and elapsed > startup_deadline: + kill_reason = "startup_hang" + elif shutdown_deadline and time.time() > shutdown_deadline: + kill_reason = "shutdown_hang" + elif elapsed > timeout: + kill_reason = "timeout" + + if kill_reason: + pre_kill_diag = _capture_system_diagnostics() + + # Kill the entire process group (test + any Kit children). + try: + os.killpg(pgid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, OSError): + process.kill() + try: + remaining_stdout, remaining_stderr = process.communicate(timeout=5) + stdout_data += remaining_stdout + stderr_data += remaining_stderr + except subprocess.TimeoutExpired: + pass + wall_time = time.time() - start_time + return -1, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag + + try: + ready_fds, _, _ = select.select([stdout_fd, stderr_fd], [], [], 0.1) + + for fd in ready_fds: + with contextlib.suppress(OSError): + if fd == stdout_fd: + chunk = process.stdout.read(1024) + if chunk: + stdout_data += chunk + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + elif fd == stderr_fd: + chunk = process.stderr.read(1024) + if chunk: + stderr_data += chunk + sys.stderr.buffer.write(chunk) + sys.stderr.buffer.flush() + except OSError: + time.sleep(0.1) + continue + + # Drain any output the process wrote before or just after exiting. + try: + remaining_stdout, remaining_stderr = process.communicate(timeout=10) + stdout_data += remaining_stdout + stderr_data += remaining_stderr + except Exception: + pass + + # Kill any orphaned child processes (Kit, Isaac Sim) left by the test. + try: + os.killpg(pgid, signal.SIGKILL) + time.sleep(1) + except (ProcessLookupError, PermissionError, OSError): + pass + + wall_time = time.time() - start_time + return process.returncode, stdout_data, stderr_data, "", wall_time, "" + + except Exception as e: + if process is not None and process.poll() is None: + process.kill() + with contextlib.suppress(Exception): + rem_out, rem_err = process.communicate(timeout=5) + stdout_data += rem_out + stderr_data += rem_err + stdout_data += f"\n[capture error: {e}]\n".encode() + return -1, stdout_data, stderr_data, "", 0.0, "" + + +def _capture_system_diagnostics(): + """Capture system diagnostics (GPU, memory, processes) for crash investigation. + + All errors are caught and reported inline so this never raises. + """ + sections = [] + + try: + r = subprocess.run(["nvidia-smi"], capture_output=True, text=True, timeout=10) + if r.stdout: + sections.append(f"--- nvidia-smi ---\n{r.stdout.strip()}") + except Exception as e: + sections.append(f"--- nvidia-smi --- FAILED: {e}") + + try: + with open("/proc/meminfo") as f: + lines = f.readlines() + keys = ("MemTotal", "MemFree", "MemAvailable", "Committed_AS", "SwapTotal", "SwapFree") + relevant = [line.strip() for line in lines if any(line.startswith(k) for k in keys)] + if relevant: + sections.append("--- /proc/meminfo ---\n" + "\n".join(relevant)) + except Exception as e: + sections.append(f"--- /proc/meminfo --- FAILED: {e}") + + cgroup_lines = [] + for path in ( + "/sys/fs/cgroup/memory.current", + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory.events", + "/sys/fs/cgroup/memory/memory.usage_in_bytes", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + "/sys/fs/cgroup/memory/memory.oom_control", + ): + try: + with open(path) as f: + cgroup_lines.append(f"{path}: {f.read().strip()}") + except FileNotFoundError: + pass + except Exception as e: + cgroup_lines.append(f"{path}: FAILED ({e})") + if cgroup_lines: + sections.append("--- cgroup memory ---\n" + "\n".join(cgroup_lines)) + + try: + r = subprocess.run(["ps", "auxf"], capture_output=True, text=True, timeout=5) + if r.stdout: + sections.append(f"--- process tree (ps auxf) ---\n{r.stdout.strip()}") + except Exception as e: + sections.append(f"--- process tree --- FAILED: {e}") + + try: + r = subprocess.run(["dmesg", "-T"], capture_output=True, text=True, timeout=5) + if r.stdout: + lines = r.stdout.strip().split("\n") + sections.append("--- dmesg (last 30 lines) ---\n" + "\n".join(lines[-30:])) + except Exception: + pass + + return "\n\n".join(sections) + + +def classify_failure_phase( + stdout: str, stderr: str, exit_code: int, wall_time_s: float, timeout_s: float +) -> str | None: + """Classify the failure phase of a benchmark run. + + Priority order (highest to lowest): + 1. oom: exit_code == 137 or "oom-kill" in stderr + 2. hang: wall_time_s >= timeout_s * 0.95 + 3. import: "Traceback" in stdout/stderr AND no "AppLauncher" in stdout + 4. driver: "CudaError" or "CUDA_ERROR_" in stdout/stderr (case-sensitive) + 5. init: "AppLauncher initialization complete" in stdout but no "Step Frametimes" in stdout + 6. runtime: exit_code != 0 and "Step Frametimes" in stdout + 7. null: exit_code == 0 + + Args: + stdout: Captured standard output as a string. + stderr: Captured standard error as a string. + exit_code: Process exit code. + wall_time_s: Measured wall-clock time in seconds. + timeout_s: Configured timeout in seconds. + + Returns: + A failure phase string or None for a clean exit. + """ + combined = stdout + stderr + + # 1. OOM + if exit_code == 137 or "oom-kill" in stderr: + return "oom" + + # 2. Hang + if wall_time_s >= timeout_s * 0.95: + return "hang" + + # 3. Import error + if "Traceback" in combined and "AppLauncher" not in stdout: + return "import" + + # 4. Driver error + if "CudaError" in combined or "CUDA_ERROR_" in combined: + return "driver" + + # 5. Init failure + if exit_code != 0 and "AppLauncher initialization complete" in stdout and "Step Frametimes" not in stdout: + return "init" + + # 6. Runtime failure (partial run then crash) + if exit_code != 0 and "Step Frametimes" in stdout: + return "runtime" + + # 7. Clean exit + return None + + +def run_benchmark(cmd: list, timeout_s: float) -> dict: + """Run a benchmark command and return structured result. + + Args: + cmd: Command list to execute. + timeout_s: Hard timeout in seconds. + + Returns: + Dict with keys: + - exit_code (int): Process exit code. + - stdout_tail (str): Last 2000 characters of combined stdout. + - wall_time_s (float): Measured wall-clock time in seconds. + - startup_time_s (float): Startup time in seconds (0.0 stub for POC). + - failure_phase (str | None): Classified failure phase. + """ + env = os.environ.copy() + + returncode, stdout_bytes, stderr_bytes, kill_reason, wall_time, _ = capture_test_output_with_timeout( + cmd, + timeout=timeout_s, + env=env, + startup_deadline=0, + report_file="", + ) + + stdout_str = stdout_bytes.decode("utf-8", errors="replace") + stderr_str = stderr_bytes.decode("utf-8", errors="replace") + + # Use kill_reason to override exit_code for hang detection + effective_exit_code = returncode + if kill_reason in ("timeout", "startup_hang"): + effective_exit_code = -1 + + failure_phase = classify_failure_phase( + stdout=stdout_str, + stderr=stderr_str, + exit_code=effective_exit_code, + wall_time_s=wall_time, + timeout_s=timeout_s, + ) + + combined_output = stdout_str + stdout_tail = combined_output[-2000:] if len(combined_output) > 2000 else combined_output + + return { + "exit_code": returncode, + "stdout_tail": stdout_tail, + "wall_time_s": wall_time, + "startup_time_s": 0.0, + "failure_phase": failure_phase, + } From df15280060530e8de8842669474919d8a05d32c3 Mon Sep 17 00:00:00 2001 From: Angelina Hu Date: Fri, 12 Jun 2026 16:37:48 -0700 Subject: [PATCH 2/2] hardening fixes: see slack thread (https://nvidia.slack.com/archives/C0B4J6NEZ3P/p1781234467738349) --- .github/workflows/perf-regression-gate.yaml | 95 ++- tools/perf_regression_gate/aggregate.py | 348 ++++++---- .../perf_regression_gate/backend_identity.py | 147 +++++ .../perf_regression_gate/baseline_manager.py | 623 ++++++++++++++---- .../build_bench_result.py | 285 +++++++- .../dev/sim_regression.py | 8 +- .../dev/stub_benchmark.py | 53 +- .../docs/module-interfaces.md | 150 +++-- .../docs/system-design.md | 76 ++- tools/perf_regression_gate/gate_config.py | 76 ++- tools/perf_regression_gate/gate_types.py | 39 ++ .../github_gate_context.py | 196 ++++++ tools/perf_regression_gate/gpu_identity.py | 87 +++ tools/perf_regression_gate/launch_config.py | 181 +++++ tools/perf_regression_gate/local_runner.py | 88 ++- tools/perf_regression_gate/oracle.py | 317 +++++---- .../perf_regression_gate/runtime_contract.py | 99 +++ tools/perf_regression_gate/task_config.py | 17 +- .../tasks_to_ci_matrix.py | 5 +- .../write_launch_config.py | 55 ++ tools/subprocess_runner.py | 14 +- 21 files changed, 2364 insertions(+), 595 deletions(-) create mode 100644 tools/perf_regression_gate/backend_identity.py create mode 100644 tools/perf_regression_gate/gate_types.py create mode 100644 tools/perf_regression_gate/github_gate_context.py create mode 100644 tools/perf_regression_gate/gpu_identity.py create mode 100644 tools/perf_regression_gate/launch_config.py create mode 100644 tools/perf_regression_gate/runtime_contract.py create mode 100644 tools/perf_regression_gate/write_launch_config.py diff --git a/.github/workflows/perf-regression-gate.yaml b/.github/workflows/perf-regression-gate.yaml index a623401b339b..cefcf6be469a 100644 --- a/.github/workflows/perf-regression-gate.yaml +++ b/.github/workflows/perf-regression-gate.yaml @@ -5,6 +5,10 @@ # Performance Regression Gate # +# GPU jobs default to the RTX PRO 6000 runner label used by the upstream +# deployment. Override with repo variable PERF_GATE_RUNS_ON, e.g. a JSON array +# of runner labels. +# # Runs a matrix of benchmark tasks on self-hosted runners, compares # results to the rolling baseline in angehu/perf-baselines, and posts a verdict # table to the job summary. Non-blocking by default (gate_config.json). @@ -16,19 +20,27 @@ # aggregate → download all bench artifacts, run oracle, update baselines # (self-hosted, gpu) # -# Baseline updates are pushed to angehu/perf-baselines only from main, -# develop, release or angehu/perf-gate-poc branches. +# Baseline updates are published from protected-branch push events +# (main/develop/release) plus angehu/perf-gate-poc while this POC is active. +# Mirrored PR pushes under pull-request/ can run the gate on upstream +# RTX runners; they are read-only unless PERF_GATE_ALLOW_MIRROR_BASELINE_UPDATE +# is explicitly enabled as a temporary deployment escape hatch. # # backend_key: "{physics_backend}_{render_backend}" when render_backend is # set, otherwise just physics_backend. Matches TaskConfig.backend_key. # TODO: replace angehu/perf-baselines w/ real branch name +# TODO: remove angehu/perf-gate-poc after deployment name: Performance Regression Gate on: pull_request: - types: [opened, synchronize, reopened] + types: [opened, synchronize, reopened, ready_for_review] + branches: [main, develop, 'release/**'] + merge_group: branches: [main, develop, 'release/**'] + push: + branches: [main, develop, 'release/**', angehu/perf-gate-poc, 'pull-request/**'] workflow_dispatch: concurrency: @@ -37,13 +49,14 @@ concurrency: permissions: contents: write + pull-requests: read env: CI_IMAGE_TAG: >- isaac-lab-ci:${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) - || github.ref_name + || 'sha' }}-${{ github.sha }} jobs: @@ -52,6 +65,7 @@ jobs: # --------------------------------------------------------------------------- config: name: Load Config + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} runs-on: ubuntu-latest outputs: isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }} @@ -82,7 +96,8 @@ jobs: # --------------------------------------------------------------------------- bench: name: Bench / ${{ matrix.task_id }} / ${{ matrix.physics_backend }}${{ matrix.render_backend != '' && format('_{0}', matrix.render_backend) || '' }} - runs-on: [self-hosted, gpu] + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} + runs-on: ${{ fromJSON(vars.PERF_GATE_RUNS_ON || '["linux-amd64-gpu-rtxpro6000-latest-1"]') }} needs: [config] continue-on-error: true timeout-minutes: ${{ matrix.job_timeout_minutes }} @@ -134,6 +149,16 @@ jobs: LOG_FILE="${ARTIFACT_DIR}/benchmark.log" mkdir -p "${ARTIFACT_DIR}" + GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)" + if [[ -z "${GPU_MODEL}" ]]; then + GPU_MODEL="unknown-gpu" + fi + python3 tools/perf_regression_gate/write_launch_config.py \ + --task_id "${TASK_ID}" \ + --physics_backend "${PHYSICS_BACKEND}" \ + --render_backend "${RENDER_BACKEND}" \ + --gpu_model "${GPU_MODEL}" \ + --artifact_dir "${ARTIFACT_DIR}" # Remove any stale container from a previous run attempt docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true @@ -142,10 +167,7 @@ jobs: # benchmark_non_rl.py accepts Hydra-style preset tokens (no leading dashes). # Physics is selected via the sim_backend token; render via renderer= token. # TODO: Update these tokens once benchmark_non_rl.py CLI flags are finalised. - PRESET_TOKENS="" - if [ -n "${RENDER_BACKEND}" ]; then - PRESET_TOKENS="renderer=${RENDER_BACKEND}" - fi + HYDRA_ARGS="${{ matrix.hydra_args }}" # Start the benchmark container in detached mode docker run -d --name "${CONTAINER_NAME}" \ @@ -172,7 +194,8 @@ jobs: --num_frames ${{ matrix.num_frames }} \ --benchmark_backend json \ --output_path /tmp/bench_out \ - ${PRESET_TOKENS} + ${{ matrix.seed != '' && format('--seed {0}', matrix.seed) || '' }} \ + ${HYDRA_ARGS} " START=$(date +%s) @@ -234,10 +257,7 @@ jobs: docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true - PRESET_TOKENS="" - if [ -n "${RENDER_BACKEND}" ]; then - PRESET_TOKENS="renderer=${RENDER_BACKEND}" - fi + HYDRA_ARGS="${{ matrix.hydra_args }}" docker run -d --name "${CONTAINER_NAME}" \ --init --stop-timeout 10 \ @@ -263,7 +283,8 @@ jobs: --num_frames ${{ matrix.num_frames }} \ --benchmark_backend json \ --output_path /tmp/bench_out \ - ${PRESET_TOKENS} + ${{ matrix.seed != '' && format('--seed {0}', matrix.seed) || '' }} \ + ${HYDRA_ARGS} " START=$(date +%s) @@ -327,6 +348,8 @@ jobs: --wall_time_s "${FINAL_WALL}" \ --timeout_s "${{ matrix.bench_timeout_s }}" \ --log_file "${ARTIFACT_DIR}/benchmark.log" \ + --launch_config "${ARTIFACT_DIR}/launch_config.json" \ + --gate_config tools/perf_regression_gate/gate_config.json \ ${EXTRA_FLAGS} - name: Upload bench artifacts @@ -361,10 +384,10 @@ jobs: # --------------------------------------------------------------------------- aggregate: name: Aggregate + Verdict - runs-on: [self-hosted, gpu] + runs-on: ${{ fromJSON(vars.PERF_GATE_RUNS_ON || '["linux-amd64-gpu-rtxpro6000-latest-1"]') }} needs: [bench] - # Always run even when some bench jobs fail or are skipped - if: always() + # Always run for real gate attempts, even when some bench jobs fail. + if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} steps: - name: Checkout Code @@ -372,10 +395,11 @@ jobs: with: fetch-depth: 0 - # Create a local branch so _baseline_worktree can commit to it + # Prime the remote-tracking baseline ref. aggregate.py refetches immediately + # before reading and again inside the transactional push retry loop. - name: Fetch baselines branch run: | - git fetch origin angehu/perf-baselines:angehu/perf-baselines || \ + git fetch origin +refs/heads/angehu/perf-baselines:refs/remotes/origin/angehu/perf-baselines || \ echo "::warning::Baseline branch not found; gate will run as a seed run (no baseline comparison)" - name: Download bench artifacts @@ -385,24 +409,31 @@ jobs: path: artifacts/ merge-multiple: false + - name: Resolve gate context + id: gate_context + env: + ALLOW_MIRROR_UPDATE: ${{ vars.PERF_GATE_ALLOW_MIRROR_BASELINE_UPDATE || 'false' }} + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 tools/perf_regression_gate/github_gate_context.py + - name: Run aggregate oracle id: aggregate run: | - ALLOW_UPDATE="false" - if [[ "${{ github.ref }}" == "refs/heads/develop" || \ - "${{ github.ref }}" == "refs/heads/angehu/perf-gate-poc" ]]; then - ALLOW_UPDATE="true" + GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)" + if [[ -z "${GPU_MODEL}" ]]; then + GPU_MODEL="unknown-gpu" fi python3 tools/perf_regression_gate/aggregate.py \ --artifacts_dir artifacts/ \ - --gpu_model L40S \ + --gpu_model "${GPU_MODEL}" \ --gate_config tools/perf_regression_gate/gate_config.json \ --baseline_branch angehu/perf-baselines \ - --allow_baseline_update "${ALLOW_UPDATE}" \ + --baseline_remote origin \ + --baseline_push_retries 3 \ + --base_sha "${{ steps.gate_context.outputs.base_sha }}" \ + --target_branch "${{ steps.gate_context.outputs.target_branch }}" \ + --source_branch "${{ steps.gate_context.outputs.source_branch }}" \ + --allow_baseline_update "${{ steps.gate_context.outputs.allow_update }}" \ + --trusted_source "${{ steps.gate_context.outputs.trusted_source }}" \ --summary_file "${GITHUB_STEP_SUMMARY}" - - # Push baseline updates only from eligible branches - - name: Push baseline updates - if: steps.aggregate.outputs.baselines_updated == 'true' - run: | - git push origin angehu/perf-baselines diff --git a/tools/perf_regression_gate/aggregate.py b/tools/perf_regression_gate/aggregate.py index 3e5f7ae1111f..62de74f5e4de 100644 --- a/tools/perf_regression_gate/aggregate.py +++ b/tools/perf_regression_gate/aggregate.py @@ -3,29 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""CI aggregate script: load per-task perf_regression_gate_result.json, run oracle, update baselines. - -Scans the local (downloaded) artifacts directory for perf_regression_gate_result.json files, runs the -oracle for each, writes a GitHub Step Summary table, and optionally updates the baselines branch. - -Usage:: - - python3 tools/perf_regression_gate/aggregate.py \\ - --artifacts_dir artifacts/ \\ - --gpu_model L40S \\ - --gate_config tools/perf_regression_gate/gate_config.json \\ - --baseline_branch angehu/perf-baselines \\ - --allow_baseline_update true \\ - --summary_file "$GITHUB_STEP_SUMMARY" - -For offline/test use, pass ``--baselines_dir`` to read/write flat files instead -of the git baselines branch:: - - python3 tools/perf_regression_gate/aggregate.py \\ - --artifacts_dir artifacts/ \\ - --gpu_model L40S \\ - --baselines_dir local_baselines/ -""" +"""Aggregate benchmark artifacts, run the oracle, and update trusted baselines""" import argparse import json @@ -39,139 +17,217 @@ sys.path.insert(0, str(_MODULE_DIR)) if str(_TOOLS_DIR) not in sys.path: sys.path.insert(0, str(_TOOLS_DIR)) -DEFAULT_BASELINE_BRANCH = "angehu/perf-baselines" # TODO: replace w/ real branch name - -from oracle import OracleVerdict, compare # noqa: E402 -from baseline_manager import load_baseline, load_baseline_git, update_baseline, update_baseline_git # noqa: E402 -from gate_config import load_gate_config # noqa: E402 +DEFAULT_BASELINE_BRANCH = "angehu/perf-baselines" # TODO: replace with actual dorphan branch + +from baseline_manager import ( # noqa: E402 + BaselineUpdateRecord, + load_baseline, + load_baseline_git, + make_sample_metadata, + match_context_from_bench_result, + refresh_baseline_branch, + update_baseline, + update_baselines_git, +) +from gate_config import BASELINE_PUSH_RETRIES, load_gate_config # noqa: E402 +from gpu_identity import canonical_gpu_model, gpu_model_config_keys # noqa: E402 +from gate_types import OracleVerdict # noqa: E402 +from oracle import compare # noqa: E402 from task_config import get_task # noqa: E402 def _parse_args(): - p = argparse.ArgumentParser(description="Aggregate bench results and run oracle.") - p.add_argument( - "--artifacts_dir", - required=True, - type=Path, - help="Root directory containing per-task artifact subdirectories", - ) - p.add_argument("--gpu_model", default="L40S") - p.add_argument("--gate_config", type=Path, default=_MODULE_DIR / "gate_config.json") - p.add_argument( - "--baseline_branch", - default=f"{DEFAULT_BASELINE_BRANCH}", - help=f"Git branch for baseline storage (default: {DEFAULT_BASELINE_BRANCH})", - ) - p.add_argument( - "--baselines_dir", - type=Path, - default=None, - help="Flat-file baseline directory; bypasses git (use for offline testing)", - ) - p.add_argument( - "--allow_baseline_update", - default="false", - help="Update baselines for PASS/WARN results ('true'/'false', default: false)", - ) - p.add_argument( - "--summary_file", - default=None, - help="Append step-summary markdown to this path (set to $GITHUB_STEP_SUMMARY in CI)", - ) - return p.parse_args() + parser = argparse.ArgumentParser(description="Aggregate bench results and run oracle.") + parser.add_argument("--artifacts_dir", required=True, type=Path) + parser.add_argument("--gpu_model", default="L40S") + parser.add_argument("--gate_config", type=Path, default=_MODULE_DIR / "gate_config.json") + parser.add_argument("--baseline_branch", default=DEFAULT_BASELINE_BRANCH) + parser.add_argument("--baseline_remote", default="origin", help="Git remote that owns the baseline branch; empty = local only") + parser.add_argument("--baseline_push_retries", type=int, default=None) + parser.add_argument("--baselines_dir", type=Path, default=None, help="Flat-file baseline directory; bypasses git") + parser.add_argument("--allow_baseline_update", default="false") + parser.add_argument("--summary_file", default=None) + parser.add_argument("--base_sha", default=None, help="PR base SHA for ancestry-aware baseline matching") + parser.add_argument("--target_branch", default=None, help="Target protected branch, e.g. main/develop/release/x") + parser.add_argument("--source_branch", default=None, help="Branch that produced baseline updates") + parser.add_argument("--trusted_source", default="protected_branch", help="Audit label for baseline samples written by this run") + return parser.parse_args() def _find_bench_results(artifacts_dir: Path) -> list[tuple[Path, dict]]: - """Return list of (artifact_dir: Path, perf_regression_gate_result: dict) sorted by task_id.""" found = [] - for p in sorted(artifacts_dir.rglob("perf_regression_gate_result.json")): - with p.open() as fh: - bench_result = json.load(fh) - found.append((p.parent, bench_result)) + for path in sorted(artifacts_dir.rglob("perf_regression_gate_result.json")): + with path.open() as fh: + found.append((path.parent, json.load(fh))) return found -def _excluded_frames(bench_result: dict) -> frozenset: - """Expand excluded_frames_raw from the task_config_snapshot into a frozenset.""" - raw = (bench_result.get("task_config_snapshot") or {}).get("excluded_frames_raw", []) +def _excluded_frames(bench_result: dict) -> frozenset[int]: + launch_config = bench_result.get("launch_config") or {} + raw = launch_config.get("excluded_frames_raw") + if raw is None: + raw = (bench_result.get("task_config_snapshot") or {}).get("excluded_frames_raw", []) indices: set[int] = set() - for entry in raw: + for entry in raw or []: if isinstance(entry, list): - indices.update(range(entry[0], entry[1] + 1)) + indices.update(range(int(entry[0]), int(entry[1]) + 1)) else: indices.add(int(entry)) return frozenset(indices) -def _fmt(v, decimals: int = 1) -> str: - return f"{v:.{decimals}f}" if v is not None else "N/A" +def _fmt(value, decimals: int = 1) -> str: + return f"{value:.{decimals}f}" if value is not None else "N/A" + + +def _short_sha(value: str | None) -> str: + return value[:12] if value else "none" + +def _bench_gpu_model(bench_result: dict, fallback: str) -> str: + launch_config = bench_result.get("launch_config") or {} + gpu_model = canonical_gpu_model(launch_config.get("gpu_model") or launch_config.get("gpu_model_raw")) + return canonical_gpu_model(fallback) if gpu_model == "unknown_gpu" else gpu_model -def _build_summary_table(rows: list) -> str: + +def _hard_floor(bench_result: dict, gpu_model: str, backend: str) -> float: + launch_config = bench_result.get("launch_config") or {} + if launch_config.get("fps_mean_floor") is not None: + return float(launch_config.get("fps_mean_floor") or 0.0) + try: + task = get_task(bench_result["task_id"], backend) + for key in gpu_model_config_keys(gpu_model): + value = task.fps_mean_floor.get(key, {}).get(backend) + if value is not None: + return float(value) + return 0.0 + except Exception: + return 0.0 + + +def _build_summary_table(rows: list[tuple]) -> str: lines = [ - "| Task | Backend | Verdict | FPS (mean) | Baseline | Regression% | Wall (s) |", - "|------|---------|---------|------------|----------|-------------|----------|", + "| Task | Backend | Verdict | FPS | Baseline | Samples | Regression% | Floor | Threshold | Phase | Retry | GPU | Runtime | Note |", + "|---|---|---|---:|---:|---:|---:|---:|---|---|---|---|---|---|", ] - for r in rows: + for result, bench_result in rows: + gpu_diag = bench_result.get("gpu_diag") or {} + launch_config = bench_result.get("launch_config") or {} + gpu_name = gpu_diag.get("gpu_name") or launch_config.get("gpu_model_raw") or launch_config.get("gpu_model", "") + provenance = bench_result.get("provenance") or {} + software = provenance.get("software") or {} + runtime = ", ".join( + part + for part in ( + f"cuda={gpu_diag.get('cuda_version')}" if gpu_diag.get("cuda_version") else "", + f"driver={gpu_diag.get('nvidia_driver_version')}" if gpu_diag.get("nvidia_driver_version") else "", + f"warp={software.get('warp')}" if software.get("warp") else "", + ) + if part + ) + note_parts = [part for part in (result.note, bench_result.get("config_mismatch")) if part] + if bench_result.get("p99_over_median") is not None: + note_parts.append(f"p99/med={bench_result['p99_over_median']}") + if bench_result.get("outlier_count") is not None: + note_parts.append(f"outliers={bench_result['outlier_count']}") lines.append( - f"| {r.task_id} | {r.backend} | {r.verdict.value}" - f" | {_fmt(r.measured_fps)} | {_fmt(r.baseline_fps)}" - f" | {_fmt(r.regression_pct, 2)} | {_fmt(r.wall_time_s, 1)} |" + f"| {result.task_id} | {result.backend} | {result.verdict.value}" + f" | {_fmt(result.measured_fps)} | {_fmt(result.baseline_fps)} | {result.baseline_sample_count}" + f" | {_fmt(result.regression_pct, 2)} | {_fmt(result.hard_floor_fps)} | {result.threshold_source}" + f" | {result.failure_phase or ''} | {result.was_retried} | {gpu_name}" + f" | {runtime} | {'; '.join(note_parts)} |" ) return "\n".join(lines) +def _write_github_output(**values) -> None: + github_output = os.environ.get("GITHUB_OUTPUT", "") + if not github_output: + return + with open(github_output, "a") as fh: + for key, value in values.items(): + if value is not None: + fh.write(f"{key}={value}\n") + + def main() -> int: args = _parse_args() use_flat = args.baselines_dir is not None allow_update = args.allow_baseline_update.strip().lower() in ("true", "1", "yes") + baseline_remote = args.baseline_remote or None gate_config = load_gate_config(args.gate_config) - blocking = gate_config.get("blocking", False) + blocking = bool(gate_config.get("blocking", False)) + min_block_regression_pct = float(gate_config.get("min_block_regression_pct", 3.0)) + baseline_push_retries = int(args.baseline_push_retries or gate_config.get("baseline_push_retries", BASELINE_PUSH_RETRIES)) items = _find_bench_results(args.artifacts_dir) if not items: print(f"[aggregate] No perf_regression_gate_result.json files found under {args.artifacts_dir}") return 1 - oracle_results = [] + baseline_read_sha = None + baseline_read_ref = None + if not use_flat: + try: + baseline_read_sha = refresh_baseline_branch(args.baseline_branch, remote=baseline_remote, allow_missing=True) + baseline_read_ref = baseline_read_sha + except Exception as exc: + print(f"::error::Failed to refresh baseline branch before reading: {exc}") + return 1 + if baseline_read_sha: + print(f"[aggregate] Baseline read snapshot: {args.baseline_branch}@{_short_sha(baseline_read_sha)}") + else: + print(f"[aggregate] Baseline branch {args.baseline_branch!r} not found; treating this as a seed run") + + rows = [] has_block = False has_hard_failure = False baselines_updated = False + baseline_update_failed = False + pending_git_updates: list[BaselineUpdateRecord] = [] for artifact_dir, bench_result in items: task_id = bench_result["task_id"] - backend = bench_result.get("backend_key") + backend = bench_result.get("backend_key") or bench_result.get("backend") + bench_gpu_model = _bench_gpu_model(bench_result, args.gpu_model) + match_context = match_context_from_bench_result( + bench_result, + gpu_model=bench_gpu_model, + base_sha=args.base_sha, + target_branch=args.target_branch, + ) - # Load rolling baseline (None on seed passes/failed load) baseline = None try: if use_flat: - baseline = load_baseline(args.baselines_dir, args.gpu_model, task_id, backend) - else: - baseline = load_baseline_git(args.baseline_branch, args.gpu_model, task_id, backend, None) + baseline = load_baseline(args.baselines_dir, bench_gpu_model, task_id, backend, match_context=match_context) + elif baseline_read_ref: + baseline = load_baseline_git( + baseline_read_ref, + bench_gpu_model, + task_id, + backend, + None, + match_context, + ) except Exception as exc: print(f"[aggregate] Warning: baseline load failed for {task_id}/{backend}: {exc}") - # Hard-floor FPS for this task (0.0 = disabled) - try: - task = get_task(task_id, backend) - fps_mean_floor = task.fps_mean_floor.get(args.gpu_model, {}).get(backend, 0.0) - except Exception: - fps_mean_floor = 0.0 - oracle_result = compare( bench_result=bench_result, baseline=baseline, - fps_mean_floor=fps_mean_floor, + fps_mean_floor=_hard_floor(bench_result, bench_gpu_model, backend), excluded_frames=_excluded_frames(bench_result), artifact_dir=artifact_dir, + min_block_regression_pct=min_block_regression_pct, ) - oracle_results.append(oracle_result) + rows.append((oracle_result, bench_result)) print( f"[aggregate] {task_id}/{backend}: {oracle_result.verdict.value}" f" fps={_fmt(oracle_result.measured_fps)} baseline={_fmt(oracle_result.baseline_fps)}" + f" samples={oracle_result.baseline_sample_count} source={oracle_result.threshold_source}" ) if oracle_result.verdict == OracleVerdict.BLOCK: @@ -179,31 +235,74 @@ def main() -> int: elif oracle_result.verdict == OracleVerdict.HARD_FAILURE: has_hard_failure = True - # Update baseline only for measured PASS/WARN results (never for BLOCK/HARD_FAILURE) pushed - # to designated branches or force update with flag for offline/testing use. if ( allow_update and oracle_result.verdict in (OracleVerdict.PASS, OracleVerdict.WARN) and oracle_result.measured_fps is not None ): - try: - if use_flat: - update_baseline(args.baselines_dir, args.gpu_model, task_id, backend, oracle_result.measured_fps) - else: - update_baseline_git( - args.baseline_branch, - args.gpu_model, + sample_metadata = make_sample_metadata( + gpu_model=bench_gpu_model, + task_id=task_id, + backend=backend, + fps=oracle_result.measured_fps, + bench_result=bench_result, + target_branch=args.target_branch, + source_branch=args.source_branch, + trusted_source=args.trusted_source, + ) + if baseline_read_sha: + sample_metadata["baseline_read_sha"] = baseline_read_sha + + if use_flat: + try: + update_baseline( + args.baselines_dir, + bench_gpu_model, task_id, backend, oracle_result.measured_fps, - None, + sample_metadata=sample_metadata, + ) + baselines_updated = True + print(f"[aggregate] -> baseline updated locally: {oracle_result.measured_fps:.1f} FPS") + except Exception as exc: + baseline_update_failed = True + print(f"::error::Baseline update failed for {task_id}/{backend}: {exc}") + else: + pending_git_updates.append( + BaselineUpdateRecord( + gpu_model=bench_gpu_model, + task_id=task_id, + backend=backend, + fps=oracle_result.measured_fps, + sample_metadata=sample_metadata, ) - baselines_updated = True - print(f"[aggregate] -> baseline updated: {oracle_result.measured_fps:.1f} FPS") - except Exception as exc: - print(f"[aggregate] Warning: baseline update failed for {task_id}/{backend}: {exc}") + ) + print(f"[aggregate] -> baseline update queued: {oracle_result.measured_fps:.1f} FPS") + + baseline_push_result = None + if pending_git_updates: + try: + baseline_push_result = update_baselines_git( + args.baseline_branch, + pending_git_updates, + remote=baseline_remote, + max_retries=baseline_push_retries, + ) + baselines_updated = baseline_push_result.pushed + if baseline_push_result.pushed: + print( + f"[aggregate] Baseline push succeeded: {args.baseline_branch}@" + f"{_short_sha(baseline_push_result.pushed_sha)} " + f"after {baseline_push_result.attempts} attempt(s)" + ) + else: + print("[aggregate] Baseline samples were already present; no push needed") + except Exception as exc: + baseline_update_failed = True + print(f"::error::Baseline push failed: {exc}") - table = _build_summary_table(oracle_results) + table = _build_summary_table(rows) print("\n## Performance Gate Results\n") print(table) print() @@ -211,18 +310,33 @@ def main() -> int: if args.summary_file: with open(args.summary_file, "a") as fh: fh.write("\n## Performance Gate Results\n\n") + if not use_flat: + fh.write(f"Baseline read SHA: `{_short_sha(baseline_read_sha)}`\n\n") + if baseline_push_result and baseline_push_result.pushed_sha: + fh.write( + f"Baseline pushed SHA: `{_short_sha(baseline_push_result.pushed_sha)}` " + f"after {baseline_push_result.attempts} attempt(s)\n\n" + ) fh.write(table) fh.write("\n") - # Signal baseline push to the calling workflow step - if baselines_updated: - github_output = os.environ.get("GITHUB_OUTPUT", "") - if github_output: - with open(github_output, "a") as fh: - fh.write("baselines_updated=true\n") - print(f"[aggregate] Baselines updated; workflow will push {DEFAULT_BASELINE_BRANCH}") + output_values = {"baseline_read_sha": baseline_read_sha} + if baseline_push_result: + output_values.update( + { + "baselines_updated": "true" if baseline_push_result.pushed else "false", + "baseline_pushed_sha": baseline_push_result.pushed_sha, + "baseline_push_attempts": baseline_push_result.attempts, + } + ) + elif baselines_updated: + output_values["baselines_updated"] = "true" + _write_github_output(**output_values) + + if baseline_update_failed: + return 1 - if blocking: # from gate_config.py, explicit PR to make gate blocking + if blocking: if has_block: return 1 if has_hard_failure: diff --git a/tools/perf_regression_gate/backend_identity.py b/tools/perf_regression_gate/backend_identity.py new file mode 100644 index 000000000000..7d4653415995 --- /dev/null +++ b/tools/perf_regression_gate/backend_identity.py @@ -0,0 +1,147 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Canonical backend identity helpers for the performance regression gate""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +_DEFAULT_PHYSICS_BACKEND = "physx" +_EMPTY_VALUES = {"", "none", "null"} +_DEFAULT_VALUES = _EMPTY_VALUES | {"default"} +_PHYSICS_PRESET_TO_BACKEND = { + "physx": "physx", + "newton": "newton", + "newton_mjwarp": "newton", +} +_RENDER_PRESET_TOKENS = frozenset( + { + "newton_renderer", + "ovrtx_renderer", + "warp_renderer", + "rtx_renderer", + } +) +_KNOWN_PHYSICS_BACKENDS = ("physx", "newton") + + +@dataclass(frozen=True) +class BackendIdentity: + physics_backend: str + render_backend: str | None = None + + @property + def backend_key(self) -> str: + return make_backend_key(self.physics_backend, self.render_backend) + + def to_dict(self) -> dict[str, str | None]: + return { + "physics_backend": self.physics_backend, + "render_backend": self.render_backend, + "backend_key": self.backend_key, + } + + +def _clean(value: Any) -> str | None: + if value is None: + return None + cleaned = str(value).strip() + return cleaned or None + + +def normalize_physics_backend(value: Any, *, default: str | None = None) -> str | None: + cleaned = _clean(value) + if cleaned is None: + return default + lowered = cleaned.lower() + if lowered in _DEFAULT_VALUES: + return default + return _PHYSICS_PRESET_TO_BACKEND.get(lowered, lowered) + + +def normalize_render_backend(value: Any) -> str | None: + cleaned = _clean(value) + if cleaned is None: + return None + lowered = cleaned.lower() + if lowered in _DEFAULT_VALUES: + return None + return lowered + + +def make_backend_key(physics_backend: str, render_backend: str | None = None) -> str: + physics = normalize_physics_backend(physics_backend) + if not physics: + raise ValueError("physics_backend is required to build backend_key") + render = normalize_render_backend(render_backend) + return f"{physics}_{render}" if render else physics + + +def identity_from_parts(physics_backend: Any, render_backend: Any = None) -> BackendIdentity | None: + physics = normalize_physics_backend(physics_backend) + if not physics: + return None + return BackendIdentity(physics, normalize_render_backend(render_backend)) + + +def split_backend_key(backend_key: Any) -> BackendIdentity | None: + key = _clean(backend_key) + if not key: + return None + for physics in _KNOWN_PHYSICS_BACKENDS: + if key == physics: + return BackendIdentity(physics, None) + prefix = f"{physics}_" + if key.startswith(prefix): + return BackendIdentity(physics, normalize_render_backend(key[len(prefix):])) + if "_" in key: + physics, render = key.split("_", 1) + return identity_from_parts(physics, render) + return BackendIdentity(key, None) + + +def preset_tokens(value: Any) -> frozenset[str]: + cleaned = _clean(value) + if not cleaned: + return frozenset() + tokens: set[str] = set() + for chunk in cleaned.replace(";", ",").split(","): + token = chunk.strip().lower() + if token: + tokens.add(token) + return frozenset(tokens) + + +def identity_from_presets(presets: Any) -> BackendIdentity | None: + tokens = preset_tokens(presets) + if not tokens: + return None + physics = "newton" if "newton_mjwarp" in tokens else _DEFAULT_PHYSICS_BACKEND + render = next((token for token in sorted(tokens) if token in _RENDER_PRESET_TOKENS), None) + return BackendIdentity(physics, render) + + +def backend_identity_from_launch_config(config: dict[str, Any]) -> BackendIdentity | None: + identity = identity_from_parts(config.get("physics_backend"), config.get("render_backend")) + if identity is not None: + return identity + return split_backend_key(config.get("backend_key") or config.get("backend")) + + +def backend_identity_from_benchmark_info(info: dict[str, Any]) -> BackendIdentity | None: + direct_key = info.get("backend_key") or info.get("backend") + if direct_key: + return split_backend_key(direct_key) + + identity = identity_from_parts( + info.get("physics_backend") or info.get("physics"), + info.get("render_backend") or info.get("render"), + ) + if identity is not None: + return identity + + return identity_from_presets(info.get("presets") or info.get("preset")) diff --git a/tools/perf_regression_gate/baseline_manager.py b/tools/perf_regression_gate/baseline_manager.py index d6da10ea85df..600022a7b566 100644 --- a/tools/perf_regression_gate/baseline_manager.py +++ b/tools/perf_regression_gate/baseline_manager.py @@ -3,90 +3,405 @@ # # SPDX-License-Identifier: BSD-3-Clause +"""Baseline storage for the CI performance regression gate + +The baseline store keeps immutable structured samples in ``samples.ndjson``. +Threshold stats are calculated over the newest compatible samples instead of +truncating history on write. Git-backed updates are append-only transactions: +the manager refetches the remote branch, reapplies queued samples, and retries +the push to prevent races across CI runners +""" + import contextlib +import hashlib import json +import os import shutil import statistics import subprocess import tempfile +from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path +from typing import Any + +try: + from .gate_config import BASELINE_PUSH_RETRIES, DEFAULT_K_BLOCK, DEFAULT_K_WARN, MAX_BASELINE_SAMPLES + from .oracle import Baseline +except ImportError: # pragma: no cover - supports direct script imports + from gate_config import BASELINE_PUSH_RETRIES, DEFAULT_K_BLOCK, DEFAULT_K_WARN, MAX_BASELINE_SAMPLES + from oracle import Baseline + +SAMPLES_FILENAME = "samples.ndjson" +_REPO_DIR = Path(__file__).resolve().parent +_COMMIT_ENV_DEFAULTS = { + "GIT_AUTHOR_NAME": "perf-regression-gate", + "GIT_AUTHOR_EMAIL": "perf-regression-gate@localhost", + "GIT_COMMITTER_NAME": "perf-regression-gate", + "GIT_COMMITTER_EMAIL": "perf-regression-gate@localhost", +} + -from oracle import Baseline # noqa: E402 +@dataclass(frozen=True) +class BaselineUpdateRecord: + gpu_model: str + task_id: str + backend: str + fps: float + fingerprint: str | None = None + sample_metadata: dict[str, Any] | None = None -DEFAULT_K_WARN = 2.5 # TODO: decide on real threshold values -DEFAULT_K_BLOCK = 4.0 # TODO: decide on real threshold values +@dataclass(frozen=True) +class BaselinePushResult: + branch: str + remote: str | None + base_sha: str | None + pushed_sha: str | None + attempts: int + update_count: int + pushed: bool -def _stats_path(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> Path: - if fingerprint is None: - return baselines_dir / gpu_model / task_id / backend / "stats.json" - return baselines_dir / gpu_model / task_id / backend / fingerprint / "stats.json" +def _bucket_dir(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> Path: + base = baselines_dir / gpu_model / task_id / backend + return base if fingerprint is None else base / fingerprint -def _window_path(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> Path: - if fingerprint is None: - return baselines_dir / gpu_model / task_id / backend / "window.ndjson" - return baselines_dir / gpu_model / task_id / backend / fingerprint / "window.ndjson" +def _samples_path(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> Path: + return _bucket_dir(baselines_dir, gpu_model, task_id, backend, fingerprint) / SAMPLES_FILENAME -def load_baseline(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> Baseline | None: - """Load stats.json for a task/backend pair, or return None if it does not exist""" - sp = _stats_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) - if not sp.exists(): + +def _baseline_from_values(values: list[float], *, source: str, total_sample_count: int | None = None) -> Baseline | None: + if not values: return None - with sp.open() as fh: - d = json.load(fh) + selected = values[-MAX_BASELINE_SAMPLES:] + median = statistics.median(selected) + deviations = [abs(v - median) for v in selected] + mad = statistics.median(deviations) if len(deviations) > 1 else 0.0 return Baseline( - median_fps=d["median_fps"], - mad_fps=d["mad_fps"], - k_warn=d.get("k_warn", DEFAULT_K_WARN), - k_block=d.get("k_block", DEFAULT_K_BLOCK), - sample_count=d.get("sample_count", 0), + median_fps=median, + mad_fps=mad, + k_warn=DEFAULT_K_WARN, + k_block=DEFAULT_K_BLOCK, + sample_count=len(selected), + source=source, + total_sample_count=total_sample_count if total_sample_count is not None else len(values), ) -def update_baseline(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fps: float, fingerprint=None) -> None: - """Append fps to window.ndjson, recompute stats, and write stats.json""" - wp = _window_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) - sp = _stats_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) - wp.parent.mkdir(parents=True, exist_ok=True) +def _load_sample_records_from_text(content: str) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for line in content.splitlines(): + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, dict) and isinstance(record.get("fps"), (int, float)): + records.append(record) + return records + + +def _commit_env() -> dict[str, str]: + env = os.environ.copy() + for key, value in _COMMIT_ENV_DEFAULTS.items(): + env.setdefault(key, value) + return env + + +def _git( + args: list[str], + *, + cwd: Path | None = None, + check: bool = False, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess: + result = subprocess.run( + ["git", *args], + cwd=str(cwd or _REPO_DIR), + capture_output=True, + text=True, + env=env, + ) + if check and result.returncode != 0: + raise subprocess.CalledProcessError(result.returncode, ["git", *args], result.stdout, result.stderr) + return result - fps_window: list[float] = [] - if wp.exists(): - with wp.open() as fh: - for line in fh: - line = line.strip() - if line: - fps_window.append(float(line)) - fps_window.append(fps) +def _git_error(result: subprocess.CompletedProcess) -> str: + return (result.stderr or result.stdout or "unknown git error").strip() - with wp.open("a") as fh: - fh.write(f"{fps}\n") - median = statistics.median(fps_window) - deviations = [abs(v - median) for v in fps_window] - mad = statistics.median(deviations) if len(deviations) > 1 else 0.0 +def _remote_ref_missing(result: subprocess.CompletedProcess) -> bool: + message = _git_error(result).lower() + return "couldn't find remote ref" in message or "could not find remote ref" in message + + +def _resolve_git_ref(ref: str, *, repo_dir: Path | None = None) -> str | None: + result = _git(["rev-parse", "--verify", ref], cwd=repo_dir) + if result.returncode != 0: + return None + return result.stdout.strip() + + +def refresh_baseline_branch( + branch: str, + *, + remote: str | None = "origin", + repo_dir: Path | None = None, + allow_missing: bool = True, +) -> str | None: + """Fetch and return the exact baseline branch SHA to read. + + Returning a SHA instead of the branch name makes aggregate comparisons traceable + and prevents a stale local branch from being used after the remote moved. + """ + if not remote: + return _resolve_git_ref(branch, repo_dir=repo_dir) + + remote_ref = f"refs/remotes/{remote}/{branch}" + refspec = f"+refs/heads/{branch}:{remote_ref}" + result = _git(["fetch", remote, refspec], cwd=repo_dir) + if result.returncode != 0: + if allow_missing and _remote_ref_missing(result): + return None + raise RuntimeError(f"Failed to fetch baseline branch {branch!r} from {remote!r}: {_git_error(result)}") + return _resolve_git_ref(remote_ref, repo_dir=repo_dir) + + +def _git_is_ancestor(commit_sha: str, base_sha: str, *, repo_dir: Path | None = None) -> bool: + result = _git(["merge-base", "--is-ancestor", commit_sha, base_sha], cwd=repo_dir) + return result.returncode == 0 + + +def _git_distance(commit_sha: str, base_sha: str, *, repo_dir: Path | None = None) -> int: + result = _git(["rev-list", "--count", f"{commit_sha}..{base_sha}"], cwd=repo_dir) + if result.returncode != 0: + return 10**9 + try: + return int(result.stdout.strip()) + except ValueError: + return 10**9 + + +def _sample_matches(record: dict[str, Any], context: dict[str, Any] | None) -> bool: + if context is None: + return True + exact_fields = ( + "gpu_model", + "task_id", + "backend_key", + "launch_config_hash", + "baseline_epoch", + "benchmark_contract_hash", + "runtime_contract_hash", + ) + for field in exact_fields: + expected = context.get(field) + if expected is not None and record.get(field) != expected: + return False + base_sha = context.get("base_sha") + commit_sha = record.get("commit_sha") + repo_dir = context.get("_repo_dir") + if base_sha: + if not commit_sha: + return False + if not _git_is_ancestor(str(commit_sha), str(base_sha), repo_dir=repo_dir): + return False + return True + + +def _select_records(records: list[dict[str, Any]], context: dict[str, Any] | None) -> list[dict[str, Any]]: + compatible = [r for r in records if _sample_matches(r, context)] + base_sha = context.get("base_sha") if context else None + if base_sha: + repo_dir = context.get("_repo_dir") if context else None + compatible.sort( + key=lambda r: (_git_distance(str(r.get("commit_sha", "")), str(base_sha), repo_dir=repo_dir), r.get("timestamp", "")) + ) + return compatible[:MAX_BASELINE_SAMPLES] + return compatible[-MAX_BASELINE_SAMPLES:] + - stats = { - "median_fps": median, - "mad_fps": mad, - "k_warn": DEFAULT_K_WARN, - "k_block": DEFAULT_K_BLOCK, - "sample_count": len(fps_window), +def _load_baseline_from_contents( + samples_content: str | None, + *, + match_context: dict[str, Any] | None = None, +) -> Baseline | None: + if not samples_content: + return None + records = _load_sample_records_from_text(samples_content) + selected = _select_records(records, match_context) + return _baseline_from_values( + [float(r["fps"]) for r in selected], + source="samples", + total_sample_count=len(records), + ) + + +def load_baseline( + baselines_dir: Path, + gpu_model: str, + task_id: str, + backend: str, + fingerprint=None, + match_context: dict[str, Any] | None = None, +) -> Baseline | None: + """Load compatible baseline stats for a task/backend pair.""" + samples = _samples_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + return _load_baseline_from_contents( + samples.read_text() if samples.exists() else None, + match_context=match_context, + ) + + +def _stable_sample_id(metadata: dict[str, Any]) -> str: + keys = ( + "ci_run_id", + "ci_run_attempt", + "commit_sha", + "task_id", + "backend_key", + "launch_config_hash", + "benchmark_contract_hash", + "runtime_contract_hash", + "baseline_epoch", + "fps", + "attempt", + "was_retried", + ) + payload = {key: metadata.get(key) for key in keys if metadata.get(key) is not None} + if not payload.get("ci_run_id"): + payload["timestamp"] = metadata.get("timestamp") + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:24] + + +def _sample_id_exists(samples_path: Path, sample_id: str | None) -> bool: + if not sample_id or not samples_path.exists(): + return False + for record in _load_sample_records_from_text(samples_path.read_text()): + if record.get("sample_id") == sample_id: + return True + return False + + +def make_sample_metadata( + *, + gpu_model: str, + task_id: str, + backend: str, + fps: float, + bench_result: dict | None = None, + target_branch: str | None = None, + source_branch: str | None = None, + trusted_source: str = "protected_branch", +) -> dict[str, Any]: + bench_result = bench_result or {} + launch_config = bench_result.get("launch_config") or {} + provenance = bench_result.get("provenance") or {} + git_info = provenance.get("git") or {} + software = provenance.get("software") or {} + gpu_diag = bench_result.get("gpu_diag") or {} + metadata = { + "schema_version": 1, + "fps": float(fps), + "timestamp": datetime.now(timezone.utc).isoformat(), + "trusted_source": trusted_source, + "gpu_model": gpu_model, + "task_id": task_id, + "backend_key": backend, + "physics_backend": launch_config.get("physics_backend") or bench_result.get("physics_backend"), + "render_backend": launch_config.get("render_backend") or bench_result.get("render_backend"), + "commit_sha": git_info.get("commit_hash"), + "branch": git_info.get("branch") or source_branch, + "target_branch": target_branch, + "launch_config_hash": bench_result.get("launch_config_hash") or launch_config.get("launch_config_hash"), + "benchmark_contract_hash": bench_result.get("benchmark_contract_hash") + or launch_config.get("benchmark_contract_hash"), + "runtime_contract_hash": bench_result.get("runtime_contract_hash"), + "runtime_contract": bench_result.get("runtime_contract"), + "runtime_info": bench_result.get("runtime_info"), + "baseline_epoch": bench_result.get("baseline_epoch") or launch_config.get("baseline_epoch", 1), + "attempt": bench_result.get("attempt"), + "was_retried": bench_result.get("was_retried"), + "ci_run_id": os.environ.get("GITHUB_RUN_ID"), + "ci_run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"), + "ci_workflow": os.environ.get("GITHUB_WORKFLOW"), + "ci_job": os.environ.get("GITHUB_JOB"), + "launch_config": launch_config, + "runtime": { + "isaacsim": software.get("isaacsim"), + "warp": software.get("warp"), + "cuda": (gpu_diag or {}).get("cuda_version"), + "driver": (gpu_diag or {}).get("nvidia_driver_version"), + }, + } + metadata["sample_id"] = _stable_sample_id(metadata) + return metadata + + +def match_context_from_bench_result( + bench_result: dict, + *, + gpu_model: str, + base_sha: str | None = None, + target_branch: str | None = None, +) -> dict[str, Any]: + launch_config = bench_result.get("launch_config") or {} + return { + "gpu_model": gpu_model, + "task_id": bench_result.get("task_id"), + "backend_key": bench_result.get("backend_key") or bench_result.get("backend"), + "launch_config_hash": bench_result.get("launch_config_hash") or launch_config.get("launch_config_hash"), + "benchmark_contract_hash": bench_result.get("benchmark_contract_hash") + or launch_config.get("benchmark_contract_hash"), + "runtime_contract_hash": bench_result.get("runtime_contract_hash"), + "baseline_epoch": bench_result.get("baseline_epoch") or launch_config.get("baseline_epoch", 1), + "base_sha": base_sha, + "target_branch": target_branch, } - with sp.open("w") as fh: - json.dump(stats, fh, indent=2) + + +def update_baseline( + baselines_dir: Path, + gpu_model: str, + task_id: str, + backend: str, + fps: float, + fingerprint=None, + sample_metadata: dict[str, Any] | None = None, +) -> bool: + """Append a structured baseline sample. Returns False when already present.""" + bucket = _bucket_dir(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + bucket.mkdir(parents=True, exist_ok=True) + samples = _samples_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + + metadata = dict(sample_metadata or {}) + metadata.setdefault("schema_version", 1) + metadata.setdefault("fps", float(fps)) + metadata.setdefault("timestamp", datetime.now(timezone.utc).isoformat()) + metadata.setdefault("gpu_model", gpu_model) + metadata.setdefault("task_id", task_id) + metadata.setdefault("backend_key", backend) + metadata.setdefault("sample_id", _stable_sample_id(metadata)) + + if _sample_id_exists(samples, metadata.get("sample_id")): + return False + + with samples.open("a") as fh: + fh.write(json.dumps(metadata, sort_keys=True) + "\n") + return True def delete_baseline_files(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> None: - """Delete stats.json and window.ndjson for a task/backend pair""" - for p in ( - _stats_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint), - _window_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint), - ): - if p.exists(): - p.unlink() + """Delete structured samples for a task/backend pair.""" + samples = _samples_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + if samples.exists(): + samples.unlink() def seed_baseline_with_spread( @@ -100,8 +415,7 @@ def seed_baseline_with_spread( seed: int = 0, fingerprint=None, ) -> None: - """Populate the baseline window with n_samples of varied FPS data around center_fps and compute stats.json. - For testing tasks/backends with no existing baseline or when a deterministic baseline is needed.""" + """Populate a deterministic structured baseline window for tests and demos.""" import random as _random rng = _random.Random(seed) @@ -111,100 +425,141 @@ def seed_baseline_with_spread( update_baseline(baselines_dir, gpu_model, task_id, backend, fps, fingerprint=fingerprint) -def _git_show_file(branch: str, rel_path: str) -> str | None: - try: - r = subprocess.run( - ["git", "show", f"{branch}:{rel_path}"], - capture_output=True, - text=True, - check=True, - cwd=str(Path(__file__).parent), - ) - return r.stdout - except subprocess.CalledProcessError: - return None +def _git_show_file(ref: str, rel_path: str, *, repo_dir: Path | None = None) -> str | None: + result = _git(["show", f"{ref}:{rel_path}"], cwd=repo_dir) + return result.stdout if result.returncode == 0 else None + + +def load_baseline_git( + ref: str, + gpu_model: str, + task_id: str, + backend: str, + fingerprint: str | None, + match_context: dict[str, Any] | None = None, + *, + repo_dir: Path | None = None, +) -> Baseline | None: + """Load compatible baseline stats from an exact git ref or SHA.""" + samples = str(_samples_path(Path(""), gpu_model, task_id, backend, fingerprint)) + context = dict(match_context or {}) + if repo_dir is not None: + context["_repo_dir"] = repo_dir + return _load_baseline_from_contents( + _git_show_file(ref, samples, repo_dir=repo_dir), + match_context=context or None, + ) @contextlib.contextmanager -def baseline_worktree(branch: str): +def _baseline_update_worktree(base_ref: str | None, *, repo_dir: Path | None = None): + repo_dir = repo_dir or _REPO_DIR tmpdir = tempfile.mkdtemp(prefix="perf-bl-wt-") - committed = False + orphan_branch = None try: - subprocess.run( - ["git", "worktree", "add", tmpdir, branch], - check=True, - capture_output=True, - cwd=str(Path(__file__).parent), - ) + if base_ref: + _git(["worktree", "add", "--detach", tmpdir, base_ref], cwd=repo_dir, check=True) + else: + orphan_branch = f"perf-baseline-seed-{os.getpid()}-{Path(tmpdir).name}" + _git(["worktree", "add", "--detach", tmpdir, "HEAD"], cwd=repo_dir, check=True) + _git(["checkout", "--orphan", orphan_branch], cwd=Path(tmpdir), check=True) + rm_result = _git(["rm", "-rf", "."], cwd=Path(tmpdir)) + if rm_result.returncode not in (0, 128): + raise RuntimeError(f"Failed to clear orphan baseline worktree: {_git_error(rm_result)}") yield Path(tmpdir) - status = subprocess.run( - ["git", "-C", tmpdir, "status", "--porcelain"], - capture_output=True, - text=True, - ) - if status.stdout.strip(): - subprocess.run(["git", "-C", tmpdir, "add", "-A"], check=True, capture_output=True) - subprocess.run( - ["git", "-C", tmpdir, "commit", "-m", "[baseline_manager] Update baselines"], - check=True, - capture_output=True, - ) - committed = True - except subprocess.CalledProcessError as exc: - if b"not found" in (exc.stderr or b"") or b"unknown" in (exc.stderr or b""): - raise RuntimeError( - f"Baseline branch {branch!r} not found. " - "Run: git fetch origin angehu/perf-baselines (or use --force-flat-baseline)" # TODO: replace w/ real branch name - ) from exc - raise finally: - subprocess.run( - ["git", "worktree", "remove", "--force", tmpdir], - cwd=str(Path(__file__).parent), - capture_output=True, - ) + _git(["worktree", "remove", "--force", tmpdir], cwd=repo_dir) shutil.rmtree(tmpdir, ignore_errors=True) - if committed: - print(f"[baseline_manager] -> committed baseline update to {branch!r}") + if orphan_branch: + _git(["branch", "-D", orphan_branch], cwd=repo_dir) -def load_baseline_git(branch: str, gpu_model: str, task_id: str, backend: str, fingerprint: str | None) -> Baseline | None: - rel = str(_stats_path(Path(""), gpu_model, task_id, backend, fingerprint)) - content = _git_show_file(branch, rel) - if content is None: +def _commit_baseline_worktree(worktree: Path) -> str | None: + status = _git(["status", "--porcelain"], cwd=worktree, check=True) + if not status.stdout.strip(): return None - d = json.loads(content) - return Baseline( - median_fps=d["median_fps"], - mad_fps=d["mad_fps"], - k_warn=d.get("k_warn", DEFAULT_K_WARN), - k_block=d.get("k_block", DEFAULT_K_BLOCK), - sample_count=d.get("sample_count", 0), + _git(["add", "-A"], cwd=worktree, check=True) + _git( + ["commit", "-m", "[baseline_manager] Append baseline samples"], + cwd=worktree, + check=True, + env=_commit_env(), ) + commit = _git(["rev-parse", "HEAD"], cwd=worktree, check=True) + return commit.stdout.strip() + + +def _apply_updates(root: Path, updates: list[BaselineUpdateRecord]) -> int: + appended = 0 + for update in updates: + if update_baseline( + root, + update.gpu_model, + update.task_id, + update.backend, + update.fps, + fingerprint=update.fingerprint, + sample_metadata=update.sample_metadata, + ): + appended += 1 + return appended + + +def update_baselines_git( + branch: str, + updates: list[BaselineUpdateRecord], + *, + remote: str | None = "origin", + max_retries: int = BASELINE_PUSH_RETRIES, + repo_dir: Path | None = None, +) -> BaselinePushResult: + """Append samples to a git-backed baseline branch and push safely. + + Each attempt starts from the latest remote branch SHA. If the push loses a + race, the next attempt refetches and reapplies the same sample IDs, making + retries idempotent. + """ + if not updates: + return BaselinePushResult(branch, remote, None, None, 0, 0, False) + if max_retries < 1: + raise ValueError("max_retries must be >= 1") + + last_error = "unknown push failure" + for attempt in range(1, max_retries + 1): + base_sha = refresh_baseline_branch(branch, remote=remote, repo_dir=repo_dir, allow_missing=True) + with _baseline_update_worktree(base_sha, repo_dir=repo_dir) as worktree: + appended = _apply_updates(worktree, updates) + commit_sha = _commit_baseline_worktree(worktree) + if commit_sha is None: + return BaselinePushResult(branch, remote, base_sha, base_sha, attempt, appended, False) + push_ref = f"HEAD:refs/heads/{branch}" + if remote: + push = _git(["push", remote, push_ref], cwd=worktree) + else: + push = _git(["branch", "--force", branch, "HEAD"], cwd=worktree) + if push.returncode == 0: + return BaselinePushResult(branch, remote, base_sha, commit_sha, attempt, appended, bool(remote)) + last_error = _git_error(push) + print( + f"[baseline_manager] baseline push attempt {attempt}/{max_retries} failed; " + "refetching and retrying" + ) - -def update_baseline_git( - branch: str, gpu_model: str, task_id: str, backend: str, fps: float, fingerprint: str | None -) -> None: - with baseline_worktree(branch) as wt: - update_baseline(wt, gpu_model, task_id, backend, fps, fingerprint) + raise RuntimeError(f"Failed to push baseline branch {branch!r} after {max_retries} attempts: {last_error}") -def seed_baseline_with_spread_git( +def update_baseline_git( branch: str, gpu_model: str, task_id: str, backend: str, - center_fps: float, - noise_fps: float, - n_samples: int, - seed: int, + fps: float, fingerprint: str | None, -) -> None: - with baseline_worktree(branch) as wt: - seed_baseline_with_spread(wt, gpu_model, task_id, backend, center_fps, noise_fps, n_samples, seed, fingerprint) - - -def delete_baseline_files_git(branch: str, gpu_model: str, task_id: str, backend: str, fingerprint: str | None) -> None: - with baseline_worktree(branch) as wt: - delete_baseline_files(wt, gpu_model, task_id, backend, fingerprint) + sample_metadata: dict[str, Any] | None = None, + *, + remote: str | None = "origin", + max_retries: int = BASELINE_PUSH_RETRIES, + repo_dir: Path | None = None, +) -> BaselinePushResult: + update = BaselineUpdateRecord(gpu_model, task_id, backend, fps, fingerprint, sample_metadata) + return update_baselines_git(branch, [update], remote=remote, max_retries=max_retries, repo_dir=repo_dir) diff --git a/tools/perf_regression_gate/build_bench_result.py b/tools/perf_regression_gate/build_bench_result.py index fbad11b66db9..99d38ab9a057 100644 --- a/tools/perf_regression_gate/build_bench_result.py +++ b/tools/perf_regression_gate/build_bench_result.py @@ -37,6 +37,19 @@ sys.path.insert(0, str(_TOOLS_DIR)) from subprocess_runner import classify_failure_phase # noqa: E402 +from backend_identity import ( # noqa: E402 + backend_identity_from_benchmark_info, + backend_identity_from_launch_config, + identity_from_parts, + make_backend_key, + normalize_physics_backend, + normalize_render_backend, +) +from gate_config import load_gate_config # noqa: E402 +from gate_types import FailurePhase # noqa: E402 +from gpu_identity import normalize_gpu_fields # noqa: E402 +from launch_config import fallback_launch_config, load_launch_config # noqa: E402 +from runtime_contract import build_runtime_contract, build_runtime_publish_info # noqa: E402 from task_config import get_task # noqa: E402 @@ -58,6 +71,15 @@ def _strip_phase_prefix(name: str, phase_name: str) -> str: return name[idx + len(marker) :] if idx >= 0 else name +def _duration_to_seconds(value: float, unit: str | None) -> float: + normalized = str(unit or "s").strip().lower() + if normalized in {"ms", "millisecond", "milliseconds"}: + return value / 1000.0 + if normalized in {"us", "microsecond", "microseconds"}: + return value / 1_000_000.0 + return value + + def _gpu_driver_version() -> str | None: """Return the GPU driver version string from nvidia-smi, or None if unavailable""" try: @@ -167,7 +189,7 @@ def _extract_info_provenance(info_path: Path) -> dict: if m.get("name", "").endswith("Total Start Time (Launch to Train)"): val = m.get("value") if isinstance(val, (int, float)): - startup_time_s = float(val) + startup_time_s = _duration_to_seconds(float(val), m.get("unit")) break gpu_diag: dict = {k: v for k, v in { @@ -192,6 +214,118 @@ def _extract_info_provenance(info_path: Path) -> dict: return result + +# --------------------------------------------------------------------------- +# Launch/run provenance guard + step-time debug KPIs +# --------------------------------------------------------------------------- + +_OUTLIER_FACTOR = 2.0 +_WARMUP_GUARD_FACTOR = 3.0 +_MAX_REPORTED_OUTLIERS = 8 + + +def _extract_benchmark_info(info_path: Path) -> dict: + """Return the run's self-reported benchmark_info metadata, if present.""" + try: + phases: list[dict] = json.loads(info_path.read_text()) + except Exception: + return {} + for phase in phases: + if phase.get("phase_name") != "benchmark_info": + continue + out = {} + for item in phase.get("metadata", []): + if "name" in item and "data" in item: + out[_strip_phase_prefix(item["name"], "benchmark_info")] = item["data"] + return out + return {} + + +def _coerce_int(value: object) -> int | None: + try: + return int(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return None + + +def _config_drift(benchmark_info: dict, launch_config: dict) -> str | None: + """Return a compact mismatch string when the actual run differs from launch intent.""" + if not benchmark_info: + return None + mismatches: list[str] = [] + + wanted_task = launch_config.get("task_id") + ran_task = benchmark_info.get("task") + if isinstance(ran_task, str) and ran_task and wanted_task and ran_task != wanted_task: + mismatches.append(f"task(ran={ran_task},want={wanted_task})") + + for field in ("num_envs", "seed"): + wanted = _coerce_int(launch_config.get(field)) + ran = _coerce_int(benchmark_info.get(field)) + if wanted is not None and ran is not None and ran != wanted: + mismatches.append(f"{field}(ran={ran},want={wanted})") + + wanted_frames = _coerce_int(launch_config.get("num_frames")) + ran_frames = _coerce_int(benchmark_info.get("num_frames")) + if wanted_frames is not None and ran_frames is not None and ran_frames < wanted_frames: + mismatches.append(f"num_frames(ran={ran_frames},want>={wanted_frames})") + + wanted_backend = backend_identity_from_launch_config(launch_config) + ran_backend = backend_identity_from_benchmark_info(benchmark_info) + if wanted_backend is not None and ran_backend is not None and wanted_backend.backend_key != ran_backend.backend_key: + mismatches.append(f"backend(ran={ran_backend.backend_key},want={wanted_backend.backend_key})") + + return " ".join(mismatches) if mismatches else None + + +def _expand_excluded_frames(raw: list) -> frozenset[int]: + indices: set[int] = set() + for entry in raw or []: + if isinstance(entry, list): + indices.update(range(int(entry[0]), int(entry[1]) + 1)) + else: + indices.add(int(entry)) + return frozenset(indices) + + +def _extract_debug_kpis(info_path: Path, excluded_frames: frozenset[int]) -> dict: + """Return post-warm-up step-time diagnostics for aggregate summaries.""" + try: + phases: list[dict] = json.loads(info_path.read_text()) + except Exception: + return {} + steps: list[float] = [] + for phase in phases: + if phase.get("phase_name") != "runtime": + continue + for measurement in phase.get("measurements", []): + value = measurement.get("value") + if measurement.get("name", "").endswith("Step Frametimes") and isinstance(value, dict): + raw = value.get("Environment step times", []) + steps = [float(v) for v in raw if isinstance(v, (int, float)) and not isinstance(v, bool)] + break + if not steps: + return {} + steady = [value for idx, value in enumerate(steps) if idx not in excluded_frames] + if len(steady) < 2: + return {} + ordered = sorted(steady) + n = len(ordered) + median = ordered[n // 2] if n % 2 else (ordered[n // 2 - 1] + ordered[n // 2]) / 2.0 + p99 = ordered[min(n - 1, int(round(0.99 * (n - 1))))] + out: dict = {"steady_frames": n} + if median > 0: + out["p99_over_median"] = round(p99 / median, 3) + outliers = [(idx, value) for idx, value in enumerate(steady) if value > _OUTLIER_FACTOR * median] + out["outlier_count"] = len(outliers) + if outliers: + out["outlier_idx"] = ",".join(str(idx) for idx, _ in outliers[:_MAX_REPORTED_OUTLIERS]) + out["outlier_mag_x"] = ",".join(f"{value / median:.2g}" for _, value in outliers[:_MAX_REPORTED_OUTLIERS]) + if steady[0] > _WARMUP_GUARD_FACTOR * median: + out["warmup_flag"] = f"first_kept_frame={steady[0] / median:.1f}x_median" + return out + + def _parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description="Build perf_regression_gate_result.json from a benchmark run") p.add_argument("--task_id", required=True) @@ -202,6 +336,8 @@ def _parse_args() -> argparse.Namespace: p.add_argument("--wall_time_s", required=True, type=float) p.add_argument("--timeout_s", required=True, type=float) p.add_argument("--log_file", type=Path, default=None) + p.add_argument("--launch_config", type=Path, default=None, help="Path to launch_config.json (default: artifact_dir/launch_config.json)") + p.add_argument("--gate_config", type=Path, default=_MODULE_DIR / "gate_config.json") p.add_argument("--attempt", type=int, default=1, help="Attempt number (1 = first run, 2 = after one retry)") p.add_argument("--was_retried", action="store_true", help="Set when this result comes from a retry of a failed first attempt") return p.parse_args() @@ -233,39 +369,75 @@ def main() -> int: args = _parse_args() artifact_dir = args.artifact_dir artifact_dir.mkdir(parents=True, exist_ok=True) - - physics_backend: str = args.physics_backend - render_backend: str | None = args.render_backend if args.render_backend else None - backend_key: str = f"{physics_backend}_{render_backend}" if render_backend else physics_backend - - # Load task config for metadata embedded in bench_result - try: - task = get_task(args.task_id, backend_key) - num_envs = task.num_envs - num_frames = task.num_frames - excluded_frames_raw = task.excluded_frames_raw - timeout_minutes = task.timeout_minutes - preset = task.preset - tags = task.tags - seed = task.seed - except KeyError: - print( - f"[build_bench_result] Warning: ({args.task_id!r}, {backend_key!r}) not found in tasks.json; using defaults" + gate_config = load_gate_config(args.gate_config) + runtime_policy = gate_config.get("runtime_compatibility", {}) + + cli_physics_backend = normalize_physics_backend(args.physics_backend) + if cli_physics_backend is None: + raise ValueError("--physics_backend must name a concrete backend") + cli_render_backend = normalize_render_backend(args.render_backend) + cli_backend_key = make_backend_key(cli_physics_backend, cli_render_backend) + + launch_config = load_launch_config(artifact_dir, args.launch_config) + if launch_config is None: + try: + task = get_task(args.task_id, cli_backend_key) + except KeyError: + print( + f"[build_bench_result] Warning: ({args.task_id!r}, {cli_backend_key!r}) not found in tasks.json; using defaults" + ) + task = None + launch_config = fallback_launch_config( + task_id=args.task_id, + physics_backend=cli_physics_backend, + render_backend=cli_render_backend, + backend_key=cli_backend_key, + timeout_s=args.timeout_s, + task=task, ) - num_envs = 0 - num_frames = 0 - excluded_frames_raw = [] - timeout_minutes = int(args.timeout_s / 60) - preset = "default" - tags = ["always"] - seed = None + launch_config = dict(launch_config) + + expected_backend = backend_identity_from_launch_config(launch_config) or identity_from_parts( + cli_physics_backend, cli_render_backend + ) + if expected_backend is None: + raise ValueError("launch_config must define a concrete backend identity") + task_id = str(launch_config.get("task_id") or args.task_id) + physics_backend = expected_backend.physics_backend + render_backend = expected_backend.render_backend + backend_key = expected_backend.backend_key + + phase2_mismatches: list[str] = [] + if args.task_id != task_id: + phase2_mismatches.append(f"phase2_task_arg(arg={args.task_id},want={task_id})") + if cli_backend_key != backend_key: + phase2_mismatches.append(f"phase2_backend_arg(arg={cli_backend_key},want={backend_key})") + phase2_arg_mismatch = " ".join(phase2_mismatches) if phase2_mismatches else None + + gpu_fields = normalize_gpu_fields(launch_config.get("gpu_model_raw") or launch_config.get("gpu_model")) + launch_config["task_id"] = task_id + launch_config["backend_key"] = backend_key + launch_config["backend"] = backend_key + launch_config["physics_backend"] = physics_backend + launch_config["render_backend"] = render_backend + launch_config["gpu_model"] = gpu_fields["gpu_model"] + launch_config["gpu_model_raw"] = gpu_fields["gpu_model_raw"] + + num_envs = launch_config.get("num_envs", 0) + num_frames = launch_config.get("num_frames", 0) + excluded_frames_raw = launch_config.get("excluded_frames_raw", []) + excluded_frames = _expand_excluded_frames(excluded_frames_raw) + timeout_minutes = launch_config.get("timeout_minutes", int(args.timeout_s / 60)) + preset = launch_config.get("preset", "default") + tags = launch_config.get("tags", ["always"]) + seed = launch_config.get("seed") # Read combined stdout/stderr log for failure classification log_text = "" if args.log_file and args.log_file.exists(): log_text = args.log_file.read_text(errors="replace") - perf_regression_gate_info_present = _normalize_benchmark_output(artifact_dir, args.task_id) + perf_regression_gate_info_present = _normalize_benchmark_output(artifact_dir, task_id) failure_phase = classify_failure_phase( stdout=log_text, @@ -275,13 +447,39 @@ def main() -> int: timeout_s=args.timeout_s, ) - # Extract FPS stats, startup time, GPU diag, and full provenance from the info artifact + # Extract FPS stats, startup time, GPU diag, run config, and full provenance. info_provenance: dict = {} + benchmark_info: dict = {} + debug_kpis: dict = {} + config_mismatch: str | None = None + observed_backend = None + runtime_contract = None + runtime_contract_hash = None + runtime_info = None if perf_regression_gate_info_present: - info_provenance = _extract_info_provenance(artifact_dir / "perf_regression_gate_info.json") + info_path = artifact_dir / "perf_regression_gate_info.json" + info_provenance = _extract_info_provenance(info_path) + benchmark_info = _extract_benchmark_info(info_path) + observed_backend = backend_identity_from_benchmark_info(benchmark_info) + debug_kpis = _extract_debug_kpis(info_path, excluded_frames) + runtime_contract, runtime_contract_hash = build_runtime_contract( + provenance=info_provenance.get("provenance"), + gpu_diag=info_provenance.get("gpu_diag"), + backend=expected_backend, + policy=runtime_policy, + ) + runtime_info = build_runtime_publish_info( + provenance=info_provenance.get("provenance"), + gpu_diag=info_provenance.get("gpu_diag"), + policy=runtime_policy, + ) + config_mismatch = _config_drift(benchmark_info, launch_config) + config_mismatch = " ".join(part for part in (phase2_arg_mismatch, config_mismatch) if part) or None + if config_mismatch and failure_phase is None: + failure_phase = FailurePhase.CONFIG_MISMATCH.value bench_result = { - "task_id": args.task_id, + "task_id": task_id, "backend": backend_key, "physics_backend": physics_backend, "render_backend": render_backend, @@ -302,11 +500,23 @@ def main() -> int: "raw_fps_median": info_provenance.get("raw_fps_median"), "raw_fps_p5": info_provenance.get("raw_fps_p5"), "raw_fps_p95": info_provenance.get("raw_fps_p95"), - "outlier_count": None, + "p99_over_median": debug_kpis.get("p99_over_median"), + "outlier_count": debug_kpis.get("outlier_count"), + "debug_kpis": debug_kpis, + "benchmark_info": benchmark_info, + "observed_backend": observed_backend.to_dict() if observed_backend else None, + "config_mismatch": config_mismatch, + "runtime_contract": runtime_contract, + "runtime_contract_hash": runtime_contract_hash, + "runtime_info": runtime_info, "gpu_diag": info_provenance.get("gpu_diag"), "provenance": info_provenance.get("provenance"), + "launch_config": launch_config, + "launch_config_hash": launch_config.get("launch_config_hash"), + "benchmark_contract_hash": launch_config.get("benchmark_contract_hash"), + "baseline_epoch": launch_config.get("baseline_epoch", 1), "task_config_snapshot": { - "task_id": args.task_id, + "task_id": task_id, "backend": backend_key, "physics_backend": physics_backend, "render_backend": render_backend, @@ -318,14 +528,21 @@ def main() -> int: "timeout_minutes": timeout_minutes, "tags": tags, "seed": seed, + "launch_config_hash": launch_config.get("launch_config_hash"), + "benchmark_contract_hash": launch_config.get("benchmark_contract_hash"), + "runtime_contract_hash": runtime_contract_hash, + "baseline_epoch": launch_config.get("baseline_epoch", 1), }, } out = artifact_dir / "perf_regression_gate_result.json" out.write_text(json.dumps(bench_result, indent=2)) - status = f"failure_phase={failure_phase!r}, perf_regression_gate_info_present={perf_regression_gate_info_present}, exit_code={args.exit_code}" - print(f"[build_bench_result] {args.task_id}: {status}") + status = ( + f"failure_phase={failure_phase!r}, perf_regression_gate_info_present={perf_regression_gate_info_present}, " + f"exit_code={args.exit_code}, config_mismatch={config_mismatch!r}" + ) + print(f"[build_bench_result] {task_id}: {status}") return 0 diff --git a/tools/perf_regression_gate/dev/sim_regression.py b/tools/perf_regression_gate/dev/sim_regression.py index ce4f07e1fb23..2e8ec959bdf9 100644 --- a/tools/perf_regression_gate/dev/sim_regression.py +++ b/tools/perf_regression_gate/dev/sim_regression.py @@ -31,6 +31,7 @@ sys.path.insert(0, str(_GATE_DIR)) sys.path.insert(0, str(_GATE_DIR.parent)) +from baseline_manager import load_baseline # noqa: E402 from task_config import load_tasks # noqa: E402 @@ -114,13 +115,12 @@ def main() -> int: generated = 0 for task in tasks: - stats_path = args.baselines_dir / args.gpu_model / task.task_id / task.backend_key / "stats.json" - if not stats_path.exists(): + baseline = load_baseline(args.baselines_dir, args.gpu_model, task.task_id, task.backend_key) + if baseline is None: print(f" SKIP (no baseline): {task.task_id}/{task.backend_key}") continue - stats = json.loads(stats_path.read_text()) - baseline_fps = stats["median_fps"] + baseline_fps = baseline.median_fps regressed_fps = baseline_fps * args.fps_scale art_dir = args.out_dir / task.task_id / task.backend_key diff --git a/tools/perf_regression_gate/dev/stub_benchmark.py b/tools/perf_regression_gate/dev/stub_benchmark.py index 01feb4f03494..8eddf6520c2d 100644 --- a/tools/perf_regression_gate/dev/stub_benchmark.py +++ b/tools/perf_regression_gate/dev/stub_benchmark.py @@ -6,6 +6,34 @@ import sys from pathlib import Path +_MODULE_DIR = Path(__file__).resolve().parent.parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) + +from backend_identity import split_backend_key # noqa: E402 +from launch_config import hydra_args_for_task # noqa: E402 +from task_config import TaskConfig # noqa: E402 + + +def _presets_for_backend(task_id: str, identity) -> str: + task = TaskConfig( + task_id=task_id, + physics_backend=identity.physics_backend, + render_backend=identity.render_backend, + preset="default", + num_envs=1, + num_frames=1, + excluded_frames_raw=[], + camera_resolution=None, + timeout_minutes=1, + fps_mean_floor={}, + caches=[], + ) + args = hydra_args_for_task(task) + if not args: + return "default" + return args[0].split("=", 1)[1] + def main(): parser = argparse.ArgumentParser() @@ -13,6 +41,7 @@ def main(): parser.add_argument("--backend") parser.add_argument("--num_envs", type=int, default=1) parser.add_argument("--num_frames", type=int, default=200) + parser.add_argument("--seed", type=int, default=42) parser.add_argument("--out_dir", required=True) parser.add_argument("--fps_mean", type=float, default=200.0) parser.add_argument("--failure_phase", default="none") @@ -41,18 +70,38 @@ def main(): fps_series = [max(0.0, rng.gauss(args.fps_mean, noise)) for _ in range(n)] # Write perf_regression_gate_info.json + identity = split_backend_key(args.backend) + if identity is None: + raise RuntimeError(f"Cannot parse backend identity from {args.backend!r}") + benchmark_info_phase = { + "phase_name": "benchmark_info", + "metadata": [ + {"name": "stub benchmark_info task", "data": args.task_id}, + {"name": "stub benchmark_info num_envs", "data": args.num_envs}, + {"name": "stub benchmark_info num_frames", "data": args.num_frames}, + {"name": "stub benchmark_info seed", "data": args.seed}, + {"name": "stub benchmark_info physics_backend", "data": identity.physics_backend}, + {"name": "stub benchmark_info render_backend", "data": identity.render_backend}, + {"name": "stub benchmark_info backend_key", "data": identity.backend_key}, + {"name": "stub benchmark_info presets", "data": _presets_for_backend(args.task_id, identity)}, + ], + } + runtime_phase = { "phase_name": "runtime", "measurements": [ { "name": "Step Frametimes", - "value": {"Environment step effective FPS": fps_series}, + "value": { + "Environment step effective FPS": fps_series, + "Environment step times": [1000.0 / max(fps, 1.0) for fps in fps_series], + }, } ], } info_path = out_dir / "perf_regression_gate_info.json" with info_path.open("w") as fh: - json.dump([runtime_phase], fh) + json.dump([benchmark_info_phase, runtime_phase], fh) # Print Step Frametimes marker so classify_failure_phase can see it print("Step Frametimes") diff --git a/tools/perf_regression_gate/docs/module-interfaces.md b/tools/perf_regression_gate/docs/module-interfaces.md index aabd2a5091b3..b142e0e03c81 100644 --- a/tools/perf_regression_gate/docs/module-interfaces.md +++ b/tools/perf_regression_gate/docs/module-interfaces.md @@ -5,6 +5,52 @@ All commands run from the `IsaacLab/` repository root unless otherwise noted. --- +## `backend_identity.py` + +Canonical backend identity helpers. `backend_key` is always derived as `physics_backend` for physics-only runs, or `{physics_backend}_{render_backend}` when a render backend is set. + +```python +def make_backend_key(physics_backend: str, render_backend: str | None = None) -> str +def backend_identity_from_launch_config(config: dict) -> BackendIdentity | None +def backend_identity_from_benchmark_info(info: dict) -> BackendIdentity | None +``` + +This is used by task loading, launch config generation, Phase 2 config-drift checks, aggregate baseline lookup, and baseline sample metadata. Render backends are workload identity, so `physx_newton_renderer`, `newton_newton_renderer`, and `newton_ovrtx_renderer` are separate baseline buckets. + +## `gpu_identity.py` + +Canonical GPU identity helpers. `gpu_model` is the baseline bucket key; `gpu_model_raw` is preserved for display/provenance. Existing `tasks.json` hard-floor maps can keep legacy keys such as `L40S` because floor lookup tries canonical, raw, and known legacy aliases. + +```python +def canonical_gpu_model(value: Any) -> str +def normalize_gpu_fields(value: Any) -> dict[str, str] +def gpu_model_config_keys(value: Any) -> list[str] +``` + +Examples: `NVIDIA L40S -> l40s`, `RTX6000 -> rtx_6000`, `NVIDIA GeForce RTX 5090 -> geforce_rtx_5090`. + +## `runtime_contract.py` + +Builds the runtime compatibility contract used for baseline matching. The matching code only sees `runtime_contract_hash`; package/version field selection stays in `gate_config.py`. + +```python +def build_runtime_contract(*, provenance: dict | None, gpu_diag: dict | None, backend: BackendIdentity, policy: Mapping[str, Any]) -> tuple[dict, str] +def build_runtime_publish_info(*, provenance: dict | None, gpu_diag: dict | None, policy: Mapping[str, Any]) -> dict +``` + +Default compatibility fields gate on top-level active-path packages: IsaacSim, IsaacLab, Torch, Warp, the active physics package (`isaaclab_physx` or `isaaclab_newton`/`newton`), and `isaaclab_ov` for renderer backends. CUDA version, NVIDIA driver, GPU memory, and compute capability are published for humans but do not affect the compatibility hash by default. + +## `github_gate_context.py` + +Resolves GitHub event metadata into the aggregate arguments used for baseline matching and publication policy. + +```python +def resolve_gate_context(env=None, event=None, fetch_pr=None) -> GateContext +``` + +Outputs: `base_sha`, `target_branch`, `source_branch`, `allow_update`, `trusted_source`, and `event_kind`. +Mirrored PR pushes under `pull-request/` use the GitHub PR API to recover the real PR base/source branches. They are read-only unless `PERF_GATE_ALLOW_MIRROR_BASELINE_UPDATE`/`ALLOW_MIRROR_UPDATE` is explicitly true. + ## `oracle.py` ### `compare()` @@ -167,14 +213,14 @@ def load_baseline( baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None ) -> Baseline | None ``` -Loads `stats.json` for a task/backend pair. Returns `None` if file does not exist (seed run). +Loads `samples.ndjson` for a task/backend pair and computes thresholds from compatible samples. Matching can require exact `gpu_model`, `task_id`, `backend_key`, `launch_config_hash`, `benchmark_contract_hash`, `runtime_contract_hash`, and `baseline_epoch`; with `base_sha`, samples must also come from ancestor commits. Returns `None` if no structured compatible samples exist. ```python def update_baseline( baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fps: float, fingerprint=None ) -> None ``` -Appends `fps` to `window.ndjson`, recomputes `median` and `MAD`, writes `stats.json`. +Appends one structured sample to `samples.ndjson`. Thresholds are computed at read time. Only call for PASS/WARN results — aggregate.py enforces this policy. ```python @@ -182,7 +228,7 @@ def delete_baseline_files( baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None ) -> None ``` -Removes `stats.json` and `window.ndjson` for a task/backend pair. Used in tests and resets. +Removes `samples.ndjson` for a task/backend pair. Used in tests and resets. ```python def seed_baseline_with_spread( @@ -197,25 +243,26 @@ Used in tests to create deterministic baselines without running real benchmarks. ### Git operations (production) ```python -def load_baseline_git(branch: str, gpu_model: str, task_id: str, backend: str, fingerprint: str | None) -> Baseline | None -def update_baseline_git(branch: str, gpu_model: str, task_id: str, backend: str, fps: float, fingerprint: str | None) -> None -def seed_baseline_with_spread_git(branch, gpu_model, task_id, backend, center_fps, noise_fps, n_samples, seed, fingerprint) -> None -def delete_baseline_files_git(branch, gpu_model, task_id, backend, fingerprint) -> None +def refresh_baseline_branch(branch: str, remote: str | None = "origin") -> str | None +def load_baseline_git(ref: str, gpu_model: str, task_id: str, backend: str, fingerprint: str | None) -> Baseline | None +def update_baselines_git(branch: str, updates: list[BaselineUpdateRecord], remote: str | None = "origin") -> BaselinePushResult +def update_baseline_git(branch: str, gpu_model: str, task_id: str, backend: str, fps: float, fingerprint: str | None) -> BaselinePushResult ``` -Git variants use `baseline_worktree()` — a context manager that creates a temporary git -worktree on `branch`, yields the worktree path, then commits any changes and removes the -worktree. Raises `RuntimeError` if `branch` is not found locally. +`refresh_baseline_branch()` fetches the remote baseline branch and returns the exact SHA +used for reads. When `base_sha` is supplied, baseline matching requires each selected sample +to have a `commit_sha` that is an ancestor of that base. `update_baselines_git()` is the +production writer: it refetches the branch, applies queued structured samples in a temporary +worktree, commits once, and retries on non-fast-forward push races. Sample IDs make retries +idempotent if a previous push outcome was ambiguous. ### File paths (flat-file) ``` -{baselines_dir}/{gpu_model}/{task_id}/{backend}/stats.json -{baselines_dir}/{gpu_model}/{task_id}/{backend}/window.ndjson +{baselines_dir}/{gpu_model}/{task_id}/{backend}/samples.ndjson With fingerprint: -{baselines_dir}/{gpu_model}/{task_id}/{backend}/{fingerprint}/stats.json -{baselines_dir}/{gpu_model}/{task_id}/{backend}/{fingerprint}/window.ndjson +{baselines_dir}/{gpu_model}/{task_id}/{backend}/{fingerprint}/samples.ndjson ``` --- @@ -288,7 +335,7 @@ one object per `(task_id, backend)` combination. Used by the `build_matrix` step `perf-regression-gate.yaml` to populate the GitHub Actions job matrix. Each object contains: `task_id`, `physics_backend`, `render_backend` (empty string if none), -`num_envs`, `num_frames`, `bench_timeout_s`, `job_timeout_minutes`. +`num_envs`, `num_frames`, `seed`, `hydra_args`, `bench_timeout_s`, and `job_timeout_minutes`. --- @@ -304,6 +351,8 @@ python3 tools/perf_regression_gate/build_bench_result.py \ --wall_time_s Phase 1 wall-clock time in seconds (required) --timeout_s Phase 1 timeout in seconds (required) --log_file combined stdout+stderr log from Phase 1 (default: none) + --launch_config launch_config.json from Phase 1 (default: artifact_dir/launch_config.json) + --gate_config gate_config.json for runtime compatibility policy --attempt attempt number: 1 = first try, 2 = after retry (default: 1) --was_retried flag: set when this result comes from a retry ``` @@ -316,6 +365,8 @@ python3 tools/perf_regression_gate/build_bench_result.py \ When `perf_regression_gate_info.json` is present, `build_bench_result.py` also calls `_extract_info_provenance()` to populate the FPS distribution, startup time, GPU diagnostics, and full software/hardware/git provenance directly into the result JSON. +It also compares observed benchmark identity to `launch_config.json`, records `observed_backend`, +computes `runtime_contract_hash`, and publishes non-matching runtime diagnostics such as CUDA and driver version. `nvidia-smi` is queried once at post-processing time to capture the driver version. --- @@ -328,9 +379,15 @@ python3 tools/perf_regression_gate/aggregate.py \ --gpu_model GPU model label for baseline lookup (default: L40S) --gate_config path to gate_config.json (default: perf_regression_gate/gate_config.json) --baseline_branch git branch for baseline storage (default: angehu/perf-baselines) + --baseline_remote git remote that owns the baseline branch (default: origin; empty = local only) + --baseline_push_retries max retry attempts for transactional baseline pushes (default: config) --baselines_dir flat-file baseline directory; bypasses git (default: None = use git) --allow_baseline_update "true"/"false": extend baseline window for PASS/WARN (default: false) --summary_file append step-summary markdown to this path (default: none) + --base_sha PR/protected branch base SHA for ancestry-aware matching + --target_branch protected target branch name + --source_branch source branch name recorded in baseline metadata + --trusted_source audit label for written samples ``` **Exit codes:** @@ -338,8 +395,10 @@ python3 tools/perf_regression_gate/aggregate.py \ - `1` — any BLOCK verdict and `gate_config.blocking == true` - `2` — any HARD_FAILURE verdict and `gate_config.blocking == true` -**Environment variable:** When `GITHUB_OUTPUT` is set, writes `baselines_updated=true` to -it after a successful baseline update. Used by the CI workflow to decide whether to push. +**Environment variable:** When `GITHUB_OUTPUT` is set, aggregate writes baseline trace +outputs such as `baseline_read_sha`, `baseline_pushed_sha`, `baseline_push_attempts`, and +`baselines_updated`. The push has already happened inside aggregate when these outputs are +written. --- @@ -348,7 +407,7 @@ it after a successful baseline update. Used by the CI workflow to decide whether ``` python3 tools/perf_regression_gate/local_runner.py \ --tags task tags to run (default: always) - --gpu_model GPU model label for baselines (default: L40S) + --gpu_model GPU model label for baselines (default: auto-detect with nvidia-smi) --artifacts_dir root for per-task artifacts (default: perf_regression_gate/artifacts/) --baselines_dir flat-file baseline dir (default: perf_regression_gate/local_baselines/) --allow_baseline_update extend baseline window for PASS/WARN results @@ -373,10 +432,11 @@ Returns aggregate.py's exit code. def load_gate_config(path: Path | str) -> dict ``` -Loads `gate_config.json`. Currently a stub that returns `{"blocking": False}`. -When the CI workflow PR lands, this will read the JSON file from disk. - -The dict contract: `{"blocking": bool}`. `aggregate.py` uses `gate_config.get("blocking", False)`. +Loads `gate_config.json` when present and otherwise returns conservative defaults. +The dict contract includes `blocking`, `min_baseline_samples`, `max_baseline_samples`, +`min_block_regression_pct`, `baseline_push_retries`, and `runtime_compatibility`. +`runtime_compatibility` owns the policy for fields included in `runtime_contract_hash` vs +publish-only diagnostic fields. --- @@ -418,7 +478,7 @@ python3 tools/perf_regression_gate/dev/sim_regression.py \ --out_dir output artifacts directory (default: /tmp/sim_artifacts) ``` -For each task with an existing baseline, reads `stats.json`, computes +For each task with an existing baseline, loads `samples.ndjson`, computes `regressed_fps = baseline.median_fps × fps_scale`, and writes: - `{out_dir}/{task_id}/{backend_key}/perf_regression_gate_info.json` - `{out_dir}/{task_id}/{backend_key}/perf_regression_gate_result.json` @@ -534,6 +594,26 @@ Each entry in `backends` becomes one `TaskConfig`. `backend_key = physics` when "dirty": false } }, + "launch_config": { + "gpu_model": "l40s", + "gpu_model_raw": "NVIDIA L40S", + "launch_config_hash": "...", + "benchmark_contract_hash": "..." + }, + "observed_backend": { + "physics_backend": "physx", + "render_backend": null, + "backend_key": "physx" + }, + "runtime_contract_hash": "...", + "runtime_contract": { + "runtime_contract_version": 1, + "fields": {"software.warp": "1.6.0"} + }, + "runtime_info": { + "software": {"warp": "1.6.0"}, + "publish_only": {"gpu_diag.cuda_version": "12.1"} + }, "task_config_snapshot": { "task_id": "Isaac-Velocity-Flat-G1-v0", "backend": "physx", @@ -641,25 +721,11 @@ The oracle looks for `phase_name == "runtime"`, then the measurement whose `name `"Step Frametimes"`, then extracts `value["Environment step effective FPS"]` as the raw series. `build_bench_result.py` reads the remaining phases for provenance extraction. -### `stats.json` (baseline statistics) +### `samples.ndjson` (baseline history) -```json -{ - "median_fps": 1667482.3, - "mad_fps": 8337.4, - "k_warn": 2.5, - "k_block": 4.0, - "sample_count": 8 -} -``` +One JSON object per accepted baseline sample, append-only. The exact metadata can grow +without changing the file contract; the required fields for threshold calculation are: -### `window.ndjson` (baseline raw window) - -One FPS float per line, append-only: - -``` -1644720.5 -1668000.1 -1655000.0 -... +```json +{"fps": 1667482.3, "gpu_model": "l40s", "task_id": "Isaac-Cartpole-Direct-v0", "backend_key": "physx", "launch_config_hash": "...", "benchmark_contract_hash": "...", "runtime_contract_hash": "...", "baseline_epoch": 1} ``` diff --git a/tools/perf_regression_gate/docs/system-design.md b/tools/perf_regression_gate/docs/system-design.md index c77268ae7f45..7e43d6af7ecc 100644 --- a/tools/perf_regression_gate/docs/system-design.md +++ b/tools/perf_regression_gate/docs/system-design.md @@ -1,6 +1,6 @@ # IsaacLab CI Performance Regression Gate — System Design -**Status:** POC / MVP complete: pipeline runs end-to-end locally, CI integration pending -**Date:** 2026-06-10 +**Status:** POC / MVP running locally, pending productionization, deployment, deployment features +**Date:** 2026-06-15 **Owners:** Angelina Hu, Neil Mehta --- @@ -8,7 +8,7 @@ ## 1. Purpose and Use Cases The performance regression gate runs a fixed benchmark matrix on every PR and blocks merge -when throughput drops below a MAD-derived threshold relative to a rolling baseline. +when throughput drops below an explicit hard floor or a MAD-derived threshold relative to compatible rolling baseline samples. **Use cases:** @@ -18,8 +18,8 @@ when throughput drops below a MAD-derived threshold relative to a rolling baseli | PR touches camera/rendering paths | 5 additional Shadow-Vision camera benchmarks added | | Any task regresses > k_block × MAD below baseline | Aggregate exits 1; GitHub required check fails | | Gate is in advisory mode (`blocking: false`) | Verdicts print but PR is not blocked | -| Baseline does not yet exist | Seed run: PASS unconditionally (no regression to measure) | -| Protected branch (main/develop) merges | Baseline window extended with the new FPS sample | +| Baseline does not yet exist | Seed run: WARN with `no_baseline` (transparent, non-blocking in advisory mode) | +| Protected branch (main/develop/release) merges | Baseline history extended with structured PASS/WARN samples | --- @@ -51,7 +51,7 @@ PR opened (to main / release / develop) │ ▼ ┌─────────────────────────────────────────┐ -│ .github/workflows/perf-gate.yml │ +│ .github/workflows/perf-regression-gate.yaml │ │ │ │ 1. Expand task matrix from tasks.json │ │ 2. Activate tags from changed files │ @@ -112,6 +112,13 @@ IsaacLab/ ├── __init__.py ├── tasks.json SINGLE SOURCE OF TRUTH — task/backend matrix ├── task_config.py TaskConfig dataclass, load_tasks(), get_task() + ├── backend_identity.py canonical physics/render backend identity + ├── gpu_identity.py canonical GPU buckets + legacy floor aliases + ├── runtime_contract.py runtime compatibility contract/hash builder + ├── launch_config.py artifact-carried launch intent + hashes + ├── write_launch_config.py CI/local helper to write launch_config.json + ├── github_gate_context.py PR/merge/push context + baseline write policy + ├── gate_types.py verdict/failure/threshold enums ├── tasks_to_ci_matrix.py Converts tasks.json → GitHub Actions matrix JSON │ (called by perf-regression-gate.yaml build_matrix step) ├── oracle.py compare() → OracleResult; PASS/WARN/BLOCK/HARD_FAILURE @@ -121,7 +128,7 @@ IsaacLab/ ├── aggregate.py Phase 3: scans result JSONs, calls oracle, │ prints table, updates baselines, exits 0/1/2 ├── baseline_manager.py load/update baseline, flat-file + git variants - ├── gate_config.py load_gate_config() — reads {"blocking": bool} + ├── gate_config.py policy constants + runtime compatibility defaults ├── local_runner.py LOCAL END-TO-END RUNNER: orchestrates Phase 1+2+3 | without Github/Docker/cloud platform dependencies │ @@ -142,15 +149,12 @@ IsaacLab/ Local (testing): tools/perf_regression_gate/local_baselines/ {gpu_model}/{task_id}/{backend_key}/ - stats.json {"median_fps", "mad_fps", "k_warn", "k_block", "sample_count"} - window.ndjson append-only rolling window, one FPS float per line + samples.ndjson append-only structured baseline samples -Production (planned): +Production: perf-baselines branch (git orphan) - {gpu_model}/{task_id}/{backend_key}/{backend_version}/{runtime_hash}/{code_fingerprint}/ - stats.json - window.ndjson - meta.json + {gpu_model}/{task_id}/{backend_key}/ + samples.ndjson append-only structured samples; compatibility fields live in each sample ``` --- @@ -190,6 +194,7 @@ Runs once per task after Phase 1 completes: - Renames `benchmark_non_rl_*.json` → `perf_regression_gate_info.json` - Classifies failure phase by scanning the benchmark log - Parses the info artifact to extract FPS distribution statistics, startup time, GPU diagnostics, and full SW/HW/git provenance +- Computes `runtime_contract_hash` and publish-only runtime info - Writes `perf_regression_gate_result.json` (always written, even on failure) ### Phase 3: `aggregate.py` @@ -268,10 +273,12 @@ mean_fps = statistics.mean(filtered) mean_fps < fps_mean_floor? → BLOCK -baseline is None or baseline.sample_count < 2? - → PASS (seed run) +baseline is None? + → WARN (no_baseline) +baseline.sample_count < MIN_BASELINE_SAMPLES? + → WARN (insufficient_baseline) -mean_fps < baseline.median - 4.0 * baseline.mad? +mean_fps < baseline.median - 4.0 * baseline.mad AND regression_pct <= -MIN_BLOCK_REGRESSION_PCT? → BLOCK mean_fps < baseline.median - 2.5 * baseline.mad? → WARN @@ -321,18 +328,24 @@ order. Classification is entirely string-pattern-based — no backend branching. ``` local_baselines/{gpu_model}/{task_id}/{backend_key}/ - stats.json {"median_fps", "mad_fps", "k_warn", "k_block", "sample_count"} - window.ndjson one FPS float per line, append-only + samples.ndjson append-only structured baseline samples ``` -`baseline_manager.update_baseline()` appends to `window.ndjson`, recomputes median and MAD, -and rewrites `stats.json`. BLOCK results are never written. +`baseline_manager.update_baseline()` appends one structured sample per accepted result. +The rolling median/MAD thresholds are computed from the newest compatible samples at read time. +BLOCK results are never written. -**Git branch (production):** `perf-baselines` orphan branch. `baseline_manager.update_baseline_git()` -uses a temporary git worktree to write and commit atomically. +**Git branch (production):** `perf-baselines` orphan branch. `aggregate.py` reads from a +freshly fetched baseline branch SHA. Accepted PASS/WARN samples are pushed through +`baseline_manager.update_baselines_git()`, which refetches before writing, commits +append-only `samples.ndjson` updates in a temporary worktree, and retries the push +if another runner updates the branch first. -**Write policy:** Writes occur only on protected branches (main/develop/release/*) or with -`--allow_baseline_update` (local testing flag). Feature branch runs are read-only. +**Trigger/write policy:** Non-draft PRs and merge-queue candidates run the gate but do +not publish baselines. Protected-branch push events (main/develop/release/*, plus the +POC branch while enabled) publish accepted PASS/WARN samples through the transactional +git writer. Feature branch runs are read-only unless `--allow_baseline_update` is set +for local testing. --- @@ -388,7 +401,7 @@ the `"hardware_info"`, `"version_info"`, and `"startup"` phases for provenance e ## 12. Environment Requirements (Local Testing) -Validated configuration as of 2026-06-10: +Validated local smoke configuration as of 2026-06-15: | Component | Version | |---|---| @@ -397,7 +410,7 @@ Validated configuration as of 2026-06-10: | warp-lang | 1.12.0 (NOT 1.13.0 — `warp.context` removed, breaks omni.replicator.core) | | mujoco-warp | 3.8.1 | | Python | 3.12 | -| GPU | RTX 5090 (local) / L40S (production CI) | +| GPU | RTX 5090 (local) / RTX PRO 6000 target runners / L40S historical reference | **Installation after fresh `./isaaclab.sh -i --extra rl`:** @@ -413,10 +426,11 @@ pip install warp-lang==1.12.0 ### 13.1 PR-gated runs vs. nightly-authoritative runs **Principle:** Authoritative runs SHOULD execute on a nightly schedule. -**Deviation:** We run per-PR and write baselines from protected branches. -**Defense:** We use wide tolerance bands (k_warn=2.5, k_block=4.0 MAD) to -absorb inter-run variance. and the purpose of our tests is to catch issues live rather -than nightly (existing OmniPerf test suite). +**Deviation:** We run on non-draft PRs and merge-queue candidates, and publish baselines +from protected-branch pushes. +**Defense:** The gate is intended to catch merge-time regressions before they land. Baseline +publication remains restricted to trusted protected-branch states, and matching prefers +compatible nearest-ancestor samples when a base SHA is available. ### 13.2 Single FPS sample per run vs. N=10 iterations diff --git a/tools/perf_regression_gate/gate_config.py b/tools/perf_regression_gate/gate_config.py index 2c1ad7f09cd3..30f91cbe1a52 100644 --- a/tools/perf_regression_gate/gate_config.py +++ b/tools/perf_regression_gate/gate_config.py @@ -6,5 +6,79 @@ import json from pathlib import Path + +DEFAULT_K_WARN = 2.5 +DEFAULT_K_BLOCK = 4.0 +MIN_BASELINE_SAMPLES = 5 +MAX_BASELINE_SAMPLES = 20 +MIN_BLOCK_REGRESSION_PCT = 3.0 +BASELINE_PUSH_RETRIES = 3 +DEFAULT_RUNTIME_COMPATIBILITY = { + "contract_version": 1, + "always": [ + "software.isaacsim", + "software.isaaclab", + "software.torch", + "software.warp", + ], + "by_physics_backend": { + "physx": [ + "software.isaaclab_physx", + ], + "newton": [ + "software.isaaclab_newton", + "software.newton", + ], + }, + "by_render_backend": { + "newton_renderer": [ + "software.isaaclab_ov", + ], + "ovrtx_renderer": [ + "software.isaaclab_ov", + ], + "warp_renderer": [ + "software.isaaclab_ov", + ], + "rtx_renderer": [ + "software.isaaclab_ov", + ], + }, + "publish_only": [ + "hardware.gpu_compute_capability", + "hardware.gpu_total_memory_gb", + "gpu_diag.cuda_version", + "gpu_diag.nvidia_driver_version", + ], +} + + +def _merge_runtime_compatibility(raw: dict | None) -> dict: + policy = json.loads(json.dumps(DEFAULT_RUNTIME_COMPATIBILITY)) + if not raw: + return policy + for key, value in raw.items(): + if isinstance(value, dict) and isinstance(policy.get(key), dict): + policy[key].update(value) + else: + policy[key] = value + return policy + + def load_gate_config(path: Path | str) -> dict: - return {"blocking": False} + config = { + "blocking": False, + "min_baseline_samples": MIN_BASELINE_SAMPLES, + "max_baseline_samples": MAX_BASELINE_SAMPLES, + "min_block_regression_pct": MIN_BLOCK_REGRESSION_PCT, + "baseline_push_retries": BASELINE_PUSH_RETRIES, + "runtime_compatibility": _merge_runtime_compatibility(None), + } + p = Path(path) + if p.exists(): + with p.open() as fh: + loaded = json.load(fh) + runtime_policy = _merge_runtime_compatibility(loaded.pop("runtime_compatibility", None)) + config.update(loaded) + config["runtime_compatibility"] = runtime_policy + return config diff --git a/tools/perf_regression_gate/gate_types.py b/tools/perf_regression_gate/gate_types.py new file mode 100644 index 000000000000..ed72b0b61046 --- /dev/null +++ b/tools/perf_regression_gate/gate_types.py @@ -0,0 +1,39 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared model states for the performance regression gate""" + +from enum import Enum + + +class OracleVerdict(str, Enum): + PASS = "PASS" + WARN = "WARN" + BLOCK = "BLOCK" + HARD_FAILURE = "HARD_FAILURE" + + +class BisectVerdict(str, Enum): + GOOD = "GOOD" + BAD = "BAD" + SKIP = "SKIP" + + +class FailurePhase(str, Enum): + IMPORT = "import" + INIT = "init" + RUNTIME = "runtime" + OOM = "oom" + HANG = "hang" + DRIVER = "driver" + CONFIG_MISMATCH = "config_mismatch" + + +class ThresholdSource(str, Enum): + NO_BASELINE = "no_baseline" + INSUFFICIENT_WINDOW = "insufficient_window" + ROLLING_WINDOW = "rolling_window" + HARD_FLOOR = "hard_floor" + NOT_APPLICABLE = "n/a" diff --git a/tools/perf_regression_gate/github_gate_context.py b/tools/perf_regression_gate/github_gate_context.py new file mode 100644 index 000000000000..e418cd121cb4 --- /dev/null +++ b/tools/perf_regression_gate/github_gate_context.py @@ -0,0 +1,196 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Resolve GitHub event context for the performance regression gate""" + +from __future__ import annotations + +import json +import os +import re +import urllib.request +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +_MIRRORED_PR_BRANCH_RE = re.compile(r"^pull-request/(\d+)$") +_BASELINE_PUBLISH_BRANCHES = frozenset({"main", "develop", "angehu/perf-gate-poc"}) +_BASELINE_PUBLISH_PREFIXES = ("release/",) + + +@dataclass(frozen=True) +class GateContext: + base_sha: str + target_branch: str + source_branch: str + allow_update: bool + trusted_source: str + event_kind: str + + def outputs(self) -> dict[str, str]: + return { + "base_sha": self.base_sha, + "target_branch": self.target_branch, + "source_branch": self.source_branch, + "allow_update": "true" if self.allow_update else "false", + "trusted_source": self.trusted_source, + "event_kind": self.event_kind, + } + + +def _strip_heads_ref(value: str) -> str: + prefix = "refs/heads/" + return value[len(prefix):] if value.startswith(prefix) else value + + +def _truthy(value: str | None) -> bool: + return str(value or "").strip().lower() in {"1", "true", "yes"} + + +def _event_value(env: Mapping[str, str], key: str) -> str: + try: + return env[key] + except KeyError as exc: + raise RuntimeError(f"Missing required GitHub environment variable: {key}") from exc + + +def _load_event(event_path: str | None) -> dict[str, Any]: + if not event_path: + return {} + with Path(event_path).open(encoding="utf-8") as fh: + event = json.load(fh) + return event if isinstance(event, dict) else {} + + +def _baseline_publish_branch(ref: str) -> bool: + branch = _strip_heads_ref(ref) + return branch in _BASELINE_PUBLISH_BRANCHES or branch.startswith(_BASELINE_PUBLISH_PREFIXES) + + +def fetch_pr_from_github(repo: str, pr_number: str, env: Mapping[str, str]) -> dict[str, Any]: + token = env.get("GITHUB_TOKEN") + if not token: + raise RuntimeError("GITHUB_TOKEN is required to resolve mirrored PR context") + api_url = env.get("GITHUB_API_URL", "https://api.github.com").rstrip("/") + request = urllib.request.Request( + f"{api_url}/repos/{repo}/pulls/{pr_number}", + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + with urllib.request.urlopen(request, timeout=30) as response: + pr = json.load(response) + if not isinstance(pr, dict): + raise RuntimeError(f"GitHub API returned non-object PR payload for PR {pr_number}") + return pr + + +def _pr_context(pr: Mapping[str, Any], *, allow_update: bool, trusted_source: str, event_kind: str) -> GateContext: + base = pr["base"] + head = pr["head"] + return GateContext( + base_sha=str(base["sha"]), + target_branch=str(base["ref"]), + source_branch=str(head["ref"]), + allow_update=allow_update, + trusted_source=trusted_source, + event_kind=event_kind, + ) + + +def resolve_gate_context( + env: Mapping[str, str] | None = None, + event: Mapping[str, Any] | None = None, + fetch_pr: Callable[[str, str, Mapping[str, str]], Mapping[str, Any]] | None = None, +) -> GateContext: + env = env or os.environ + event = event if event is not None else _load_event(env.get("GITHUB_EVENT_PATH")) + fetch_pr = fetch_pr or fetch_pr_from_github + + event_name = _event_value(env, "GITHUB_EVENT_NAME") + github_ref = _event_value(env, "GITHUB_REF") + ref_name = _event_value(env, "GITHUB_REF_NAME") + github_sha = _event_value(env, "GITHUB_SHA") + + if event_name == "pull_request": + return _pr_context( + event["pull_request"], + allow_update=False, + trusted_source="read_only", + event_kind="pull_request", + ) + + if event_name == "merge_group": + merge_group = event.get("merge_group") or {} + return GateContext( + base_sha=str(merge_group.get("base_sha") or github_sha), + target_branch=_strip_heads_ref(str(merge_group.get("base_ref") or ref_name)), + source_branch=_strip_heads_ref(str(merge_group.get("head_ref") or ref_name)), + allow_update=False, + trusted_source="read_only", + event_kind="merge_group", + ) + + if event_name == "push" and github_ref.startswith("refs/heads/pull-request/"): + match = _MIRRORED_PR_BRANCH_RE.fullmatch(ref_name) + if not match: + raise RuntimeError(f"Cannot parse mirrored PR number from GITHUB_REF_NAME={ref_name!r}") + repo = _event_value(env, "GITHUB_REPOSITORY") + pr = fetch_pr(repo, match.group(1), env) + allow_update = _truthy(env.get("ALLOW_MIRROR_UPDATE") or env.get("PERF_GATE_ALLOW_MIRROR_BASELINE_UPDATE")) + return _pr_context( + pr, + allow_update=allow_update, + trusted_source="mirrored_pr_push" if allow_update else "read_only", + event_kind="mirrored_pr_push", + ) + + if event_name == "push": + allow_update = _baseline_publish_branch(github_ref) + return GateContext( + base_sha=github_sha, + target_branch=ref_name, + source_branch=ref_name, + allow_update=allow_update, + trusted_source="protected_branch" if allow_update else "read_only", + event_kind="protected_push" if allow_update else "push", + ) + + return GateContext( + base_sha=github_sha, + target_branch=ref_name, + source_branch=ref_name, + allow_update=False, + trusted_source="read_only", + event_kind=event_name, + ) + + +def write_github_outputs(outputs: Mapping[str, str], output_path: str | None) -> None: + if not output_path: + return + with Path(output_path).open("a", encoding="utf-8") as fh: + for key, value in outputs.items(): + fh.write(f"{key}={value}\n") + + +def main() -> int: + context = resolve_gate_context() + outputs = context.outputs() + write_github_outputs(outputs, os.environ.get("GITHUB_OUTPUT")) + print( + "gate_context " + f"event={outputs['event_kind']} base={outputs['base_sha'][:12]} " + f"target={outputs['target_branch']} source={outputs['source_branch']} " + f"allow_update={outputs['allow_update']} trusted_source={outputs['trusted_source']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/perf_regression_gate/gpu_identity.py b/tools/perf_regression_gate/gpu_identity.py new file mode 100644 index 000000000000..786929e33999 --- /dev/null +++ b/tools/perf_regression_gate/gpu_identity.py @@ -0,0 +1,87 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Canonical GPU identity helpers for baseline bucket selection""" + +from __future__ import annotations + +import re +from typing import Any + +_UNKNOWN_GPU = "unknown_gpu" + + +def _clean(value: Any) -> str: + return str(value or "").strip() + + +def _slug(value: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_") + return slug or _UNKNOWN_GPU + + +def canonical_gpu_model(value: Any) -> str: + """Return the canonical baseline bucket key for a raw GPU model string.""" + raw = _clean(value) + if not raw: + return _UNKNOWN_GPU + normalized = re.sub(r"\s+", " ", raw.lower()).strip() + compact = re.sub(r"[^a-z0-9]+", "", normalized) + + if "l40s" in compact: + return "l40s" + if compact.endswith("l40") or compact == "nvidial40" or "teslal40" in compact: + return "l40" + if "rtxpro6000" in compact and "blackwell" in compact: + return "rtx_pro_6000_blackwell" + if "rtxpro6000" in compact: + return "rtx_pro_6000" + if "rtx6000adageneration" in compact or ("rtx6000" in compact and "ada" in compact): + return "rtx_6000_ada" + if compact in {"rtx6000", "nvidiartx6000"} or "rtx6000" in compact: + return "rtx_6000" + if "rtxa6000" in compact or "a6000" in compact: + return "rtx_a6000" + if "geforcertx5090" in compact: + return "geforce_rtx_5090" + if "geforcertx4090" in compact: + return "geforce_rtx_4090" + return _slug(raw) + + +def gpu_model_config_keys(value: Any) -> list[str]: + """Return candidate keys for reading existing GPU-keyed config dictionaries. + + `gpu_model` is canonical for new artifacts, but existing task floor configs may + still use legacy display keys such as `L40S`. + """ + raw = _clean(value) + canonical = canonical_gpu_model(raw) + keys: list[str] = [] + for key in (canonical, raw): + if key and key not in keys: + keys.append(key) + + legacy = { + "l40s": ["L40S"], + "l40": ["L40"], + "rtx_pro_6000_blackwell": ["RTX6000", "RTX PRO 6000", "RTX PRO 6000 Blackwell"], + "rtx_pro_6000": ["RTX6000", "RTX PRO 6000"], + "rtx_6000_ada": ["RTX6000", "RTX 6000 Ada"], + "rtx_6000": ["RTX6000", "RTX 6000"], + "rtx_a6000": ["RTXA6000", "RTX A6000"], + } + for key in legacy.get(canonical, []): + if key not in keys: + keys.append(key) + return keys + + +def normalize_gpu_fields(value: Any) -> dict[str, str]: + raw = _clean(value) + return { + "gpu_model": canonical_gpu_model(raw), + "gpu_model_raw": raw or _UNKNOWN_GPU, + } diff --git a/tools/perf_regression_gate/launch_config.py b/tools/perf_regression_gate/launch_config.py new file mode 100644 index 000000000000..4d640701f5c0 --- /dev/null +++ b/tools/perf_regression_gate/launch_config.py @@ -0,0 +1,181 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Launch configuration artifact helpers for the performance regression gate + +``launch_config.json`` is the durable contract between the matrix builder, the +benchmark runner, post-processing, and aggregate. Phase 2 treats this artifact +as the run intent instead of re-reading ``tasks.json``, so it can catch workflow +command bugs and self-hosted-runner handoff issues. +""" + +import hashlib +import json +from pathlib import Path +from typing import Any + +try: + from .backend_identity import make_backend_key, normalize_render_backend + from .gpu_identity import normalize_gpu_fields + from .task_config import TaskConfig +except ImportError: # pragma: no cover (for direct scripting execution/import) + from backend_identity import make_backend_key, normalize_render_backend + from gpu_identity import normalize_gpu_fields + from task_config import TaskConfig + +LAUNCH_CONFIG_SCHEMA_VERSION = 1 +BENCHMARK_CONTRACT_VERSION = 1 +DEFAULT_BASELINE_EPOCH = 1 +LAUNCH_CONFIG_FILENAME = "launch_config.json" + + +def _canonical_json(data: dict[str, Any]) -> str: + return json.dumps(data, sort_keys=True, separators=(",", ":")) + + +def stable_hash(data: dict[str, Any]) -> str: + return hashlib.sha256(_canonical_json(data).encode("utf-8")).hexdigest()[:16] + + +def workload_contract(config: dict[str, Any]) -> dict[str, Any]: + """Return the workload-defining subset used for launch_config_hash.""" + keys = ( + "task_id", + "backend_key", + "physics_backend", + "render_backend", + "preset", + "num_envs", + "num_frames", + "seed", + "excluded_frames_raw", + "camera_resolution", + "benchmark_backend", + "hydra_args", + ) + return {key: config.get(key) for key in keys} + + +def hydra_args_for_task(task: TaskConfig) -> list[str]: + """Return Hydra args used by the current local/CI benchmark launch path.""" + presets: list[str] = [] + if task.physics_backend == "newton": + presets.append("newton_mjwarp") + if task.render_backend: + presets.append(task.render_backend) + if task.render_backend == "newton_renderer": + presets.append("rgb") + return [f"presets={','.join(presets)}"] if presets else [] + + +def task_to_launch_config( + task: TaskConfig, + *, + fps_mean_floor: float, + gpu_model: str | None = None, + hydra_args: list[str] | None = None, + benchmark_backend: str = "json", +) -> dict[str, Any]: + """Build a serializable launch config for one task/backend job""" + gpu_fields = normalize_gpu_fields(gpu_model) + config: dict[str, Any] = { + "schema_version": LAUNCH_CONFIG_SCHEMA_VERSION, + "task_id": task.task_id, + "backend_key": make_backend_key(task.physics_backend, task.render_backend), + "physics_backend": task.physics_backend, + "render_backend": normalize_render_backend(task.render_backend), + "preset": task.preset, + "num_envs": task.num_envs, + "num_frames": task.num_frames, + "seed": task.seed, + "excluded_frames_raw": task.excluded_frames_raw, + "camera_resolution": list(task.camera_resolution) if task.camera_resolution else None, + "timeout_minutes": task.timeout_minutes, + "tags": list(task.tags), + "gpu_model": gpu_fields["gpu_model"], + "gpu_model_raw": gpu_fields["gpu_model_raw"], + "benchmark_backend": benchmark_backend, + "hydra_args": list(hydra_args or []), + "fps_mean_floor": float(fps_mean_floor), + "baseline_epoch": int(getattr(task, "baseline_epoch", DEFAULT_BASELINE_EPOCH)), + "benchmark_contract_version": BENCHMARK_CONTRACT_VERSION, + } + config["launch_config_hash"] = stable_hash(workload_contract(config)) + config["benchmark_contract_hash"] = stable_hash( + { + "benchmark_contract_version": config["benchmark_contract_version"], + "excluded_frames_raw": config["excluded_frames_raw"], + "benchmark_backend": config["benchmark_backend"], + } + ) + return config + + +def fallback_launch_config( + *, + task_id: str, + physics_backend: str, + render_backend: str | None, + backend_key: str, + timeout_s: float, + task: TaskConfig | None = None, +) -> dict[str, Any]: + """Build a launch config for legacy/manual Phase 2 calls without an artifact""" + normalized_backend_key = make_backend_key(physics_backend, render_backend) + gpu_fields = normalize_gpu_fields(None) + + if task is None: + config: dict[str, Any] = { + "schema_version": LAUNCH_CONFIG_SCHEMA_VERSION, + "task_id": task_id, + "backend_key": normalized_backend_key, + "physics_backend": physics_backend, + "render_backend": normalize_render_backend(render_backend), + "preset": "default", + "num_envs": 0, + "num_frames": 0, + "seed": None, + "excluded_frames_raw": [], + "camera_resolution": None, + "timeout_minutes": int(timeout_s / 60), + "tags": ["always"], + "gpu_model": gpu_fields["gpu_model"], + "gpu_model_raw": gpu_fields["gpu_model_raw"], + "benchmark_backend": "json", + "hydra_args": [], + "fps_mean_floor": 0.0, + "baseline_epoch": DEFAULT_BASELINE_EPOCH, + "benchmark_contract_version": BENCHMARK_CONTRACT_VERSION, + } + config["launch_config_hash"] = stable_hash(workload_contract(config)) + config["benchmark_contract_hash"] = stable_hash( + { + "benchmark_contract_version": config["benchmark_contract_version"], + "excluded_frames_raw": config["excluded_frames_raw"], + "benchmark_backend": config["benchmark_backend"], + } + ) + return config + + return task_to_launch_config( + task, + fps_mean_floor=(task.fps_mean_floor.get("L40S", {}) or {}).get(task.backend_key, 0.0), + hydra_args=[], + ) + + +def load_launch_config(artifact_dir: Path, explicit_path: Path | None = None) -> dict[str, Any] | None: + path = explicit_path or artifact_dir / LAUNCH_CONFIG_FILENAME + if not path.exists(): + return None + with path.open() as fh: + return json.load(fh) + + +def write_launch_config(artifact_dir: Path, config: dict[str, Any]) -> Path: + artifact_dir.mkdir(parents=True, exist_ok=True) + path = artifact_dir / LAUNCH_CONFIG_FILENAME + path.write_text(json.dumps(config, indent=2, sort_keys=True)) + return path diff --git a/tools/perf_regression_gate/local_runner.py b/tools/perf_regression_gate/local_runner.py index 0e2f8706210c..94a68d1ef3be 100644 --- a/tools/perf_regression_gate/local_runner.py +++ b/tools/perf_regression_gate/local_runner.py @@ -45,6 +45,8 @@ if str(_TOOLS_DIR) not in sys.path: sys.path.insert(0, str(_TOOLS_DIR)) +from gpu_identity import canonical_gpu_model, gpu_model_config_keys # noqa: E402 +from launch_config import hydra_args_for_task, task_to_launch_config, write_launch_config # noqa: E402 from task_config import TaskConfig, load_tasks # noqa: E402 @@ -64,7 +66,11 @@ def _parse_args() -> argparse.Namespace: default=["always"], help="Run only tasks whose tag list overlaps this set (default: always)", ) - p.add_argument("--gpu_model", default="L40S", help="GPU model label for baseline bucket (default: L40S)") + p.add_argument( + "--gpu_model", + default=None, + help="GPU model label for baseline bucket (default: auto-detect with nvidia-smi)", + ) p.add_argument( "--artifacts_dir", type=Path, @@ -90,7 +96,7 @@ def _parse_args() -> argparse.Namespace: p.add_argument( "--skip_existing", action="store_true", - help="Skip tasks whose bench_result.json already exists in --artifacts_dir", + help="Skip tasks whose perf_regression_gate_result.json already exists in --artifacts_dir", ) p.add_argument( "--gate_config", @@ -106,6 +112,23 @@ def _parse_args() -> argparse.Namespace: # --------------------------------------------------------------------------- +def _detect_gpu_model() -> str: + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"], + capture_output=True, + text=True, + check=True, + ) + except Exception: + return "unknown-gpu" + for line in result.stdout.splitlines(): + gpu_model = line.strip() + if gpu_model: + return gpu_model + return "unknown-gpu" + + def _print_matrix(tasks: list[TaskConfig], tags: list[str]) -> None: print(f"\n{'=' * 68}") print(f" Phase 1 — Task Matrix ({len(tasks)} entries, tags={tags})") @@ -118,6 +141,18 @@ def _print_matrix(tasks: list[TaskConfig], tags: list[str]) -> None: print() +def _hydra_args(task: TaskConfig) -> list[str]: + return hydra_args_for_task(task) + + +def _fps_mean_floor(task: TaskConfig, gpu_model: str) -> float: + for key in gpu_model_config_keys(gpu_model): + value = task.fps_mean_floor.get(key, {}).get(task.backend_key) + if value is not None: + return float(value) + return 0.0 + + def _isaaclab_cmd(bench_script: Path, task: TaskConfig, artifact_dir: Path) -> list[str]: """Build the ./isaaclab.sh -p benchmark_non_rl.py command for one task/backend.""" cmd = [ @@ -130,22 +165,9 @@ def _isaaclab_cmd(bench_script: Path, task: TaskConfig, artifact_dir: Path) -> l "--benchmark_backend", "json", "--output_path", str(artifact_dir), ] - # Combine physics and render presets into a single Hydra token. - # newton physics requires newton_mjwarp; render backends use their canonical - # preset name (newton_renderer for Warp, ovrtx_renderer for OV-RTX, absent for - # default RTX). Warp renderer also requires the rgb data-type preset to avoid - # the semantic_segmentation incompatibility in ShadowHandVisionEnvCfg. - presets: list[str] = [] - if task.physics_backend == "newton": - presets.append("newton_mjwarp") - if task.render_backend: - presets.append(task.render_backend) - if task.render_backend == "newton_renderer": - presets.append("rgb") - if presets: - cmd.append(f"presets={','.join(presets)}") if task.seed is not None: cmd.extend(["--seed", str(task.seed)]) + cmd.extend(_hydra_args(task)) return cmd @@ -188,6 +210,7 @@ def _run_build_bench_result( *, was_retried: bool = False, attempt: int = 1, + gate_config: Path | None = None, ) -> None: """Run build_bench_result.py for one task (plain Python, no IsaacSim needed)""" build_script = _MODULE_DIR / "build_bench_result.py" @@ -202,10 +225,14 @@ def _run_build_bench_result( "--wall_time_s", f"{wall_time:.1f}", "--timeout_s", str(task.timeout_minutes * 60), "--log_file", str(artifact_dir / "benchmark.log"), + "--launch_config", str(artifact_dir / "launch_config.json"), "--attempt", str(attempt), ] + if gate_config is not None: + cmd.extend(["--gate_config", str(gate_config)]) if was_retried: cmd.append("--was_retried") + sys.stdout.flush() subprocess.run(cmd, check=True) @@ -227,6 +254,7 @@ def _run_aggregate(artifacts_dir: Path, baselines_dir: Path, gpu_model: str, "--gate_config", str(gate_config), "--allow_baseline_update", "true" if allow_baseline_update else "false", ] + sys.stdout.flush() result = subprocess.run(cmd) return result.returncode @@ -238,6 +266,8 @@ def _run_aggregate(artifacts_dir: Path, baselines_dir: Path, gpu_model: str, def main() -> int: args = _parse_args() + gpu_model_raw = args.gpu_model or _detect_gpu_model() + gpu_model = canonical_gpu_model(gpu_model_raw) # Expand matrix from tasks.json all_tasks = load_tasks() @@ -250,6 +280,8 @@ def main() -> int: _print_matrix(tasks, args.tags) + print(f"[local_runner] GPU model bucket: {gpu_model} (raw={gpu_model_raw})") + if args.dry_run: print("[local_runner] --dry_run: exiting without running benchmarks.") return 0 @@ -270,11 +302,19 @@ def main() -> int: artifact_dir = args.artifacts_dir / task.task_id / task.backend_key artifact_dir.mkdir(parents=True, exist_ok=True) - bench_result_path = artifact_dir / "bench_result.json" + bench_result_path = artifact_dir / "perf_regression_gate_result.json" if args.skip_existing and bench_result_path.exists(): - print(f"[{i}/{len(tasks)}] SKIP (bench_result.json exists): {task.task_id} / {task.backend_key}") + print(f"[{i}/{len(tasks)}] SKIP (perf_regression_gate_result.json exists): {task.task_id} / {task.backend_key}") continue + launch_config = task_to_launch_config( + task, + fps_mean_floor=_fps_mean_floor(task, gpu_model), + gpu_model=gpu_model_raw, + hydra_args=_hydra_args(task), + ) + write_launch_config(artifact_dir, launch_config) + print(f"[{i}/{len(tasks)}] {task.task_id} / {task.backend_key}") print(f" envs={task.num_envs} frames={task.num_frames} timeout={task.timeout_minutes}m") @@ -292,7 +332,15 @@ def main() -> int: attempt = 2 if was_retried else 1 print(f"\n[{i}/{len(tasks)}] exit={exit_code} wall={wall_time:.0f}s attempt={attempt} → build_bench_result") - _run_build_bench_result(task, artifact_dir, exit_code, wall_time, was_retried=was_retried, attempt=attempt) + _run_build_bench_result( + task, + artifact_dir, + exit_code, + wall_time, + was_retried=was_retried, + attempt=attempt, + gate_config=args.gate_config, + ) print() # ----------------------------------------------------------------------- @@ -310,7 +358,7 @@ def main() -> int: rc = _run_aggregate( args.artifacts_dir, args.baselines_dir, - args.gpu_model, + gpu_model, args.gate_config, args.allow_baseline_update, ) diff --git a/tools/perf_regression_gate/oracle.py b/tools/perf_regression_gate/oracle.py index 49e88a3ed0f0..5a92a27af111 100644 --- a/tools/perf_regression_gate/oracle.py +++ b/tools/perf_regression_gate/oracle.py @@ -3,106 +3,81 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Oracle layer for the CI performance regression gate. - -Provides verdict computation (PASS / WARN / BLOCK / HARD_FAILURE) by comparing -a measured FPS sample against a rolling baseline.\ -""" +"""Oracle layer for the CI performance regression gate.""" import json import statistics from dataclasses import dataclass -from enum import Enum from pathlib import Path - -class OracleVerdict(str, Enum): - PASS = "PASS" - WARN = "WARN" - BLOCK = "BLOCK" - HARD_FAILURE = "HARD_FAILURE" +try: + from .gate_config import DEFAULT_K_BLOCK, DEFAULT_K_WARN, MIN_BASELINE_SAMPLES, MIN_BLOCK_REGRESSION_PCT + from .gate_types import BisectVerdict, FailurePhase, OracleVerdict, ThresholdSource +except ImportError: # pragma: no cover (for direct scripting execution/import) + from gate_config import DEFAULT_K_BLOCK, DEFAULT_K_WARN, MIN_BASELINE_SAMPLES, MIN_BLOCK_REGRESSION_PCT + from gate_types import BisectVerdict, FailurePhase, OracleVerdict, ThresholdSource @dataclass class Baseline: - """Rolling-window statistics used as the comparison reference for a single (task, backend) pair. - - Args: - median_fps: Median FPS computed from the baseline window - mad_fps: Median absolute deviation of FPS in the baseline window - k_warn: Number of MADs below the median that triggers a WARN verdict - k_block: Number of MADs below the median that triggers a BLOCK verdict - sample_count: Number of samples in the window used to compute the stats - """ + """Rolling-window statistics for one compatible benchmark history""" median_fps: float mad_fps: float - k_warn: float = 2.5 # TODO: decide on real threshold values - k_block: float = 4.0 # TODO: decide on real threshold values + k_warn: float = DEFAULT_K_WARN + k_block: float = DEFAULT_K_BLOCK sample_count: int = 0 + source: str = "unknown" + total_sample_count: int | None = None @dataclass class OracleResult: """Full verdict record produced by :func:`compare`""" - verdict: OracleVerdict # High-level verdict: PASS / WARN / BLOCK / HARD_FAILURE - bisect_verdict: str # GOOD / BAD / SKIP for bisect compatibility - failure_phase: str | None # Phase classification from build_bench_result (e.g. "import", "init", "runtime", "oom", "hang", "driver", or None) - measured_fps: float | None # Mean FPS after excluded-frame filtering, or None on hard failure; blocking metric - baseline_fps: float | None # baseline.median_fps, or None - regression_pct: float | None # ((measured_fps - baseline_fps) / baseline_fps) * 100, or None - fps_median: float | None # Median FPS of the filtered series [informational] - fps_p5: float | None # 5th-percentile FPS of the filtered series [informational] - fps_p95: float | None # 95th-percentile FPS of the filtered series [informational] - gpu_mem_used_mb: float | None # GPU memory used at benchmark time [MiB], or None - startup_time_s: float | None # Startup time reported by the benchmark process [s] - wall_time_s: float | None # Wall-clock time of the benchmark run [s] - was_retried: bool # True when the benchmark succeeded only after a retry, False otherwise - task_id: str # Benchmark task identifier - backend: str # Physics/Render backend name - - -# --------------------------------------------------------------------------- -# Bisect verdict mapping (spec section "Bisect verdict mapping") -# --------------------------------------------------------------------------- - -# failure_phase values that map HARD_FAILURE -> "BAD" -_BISECT_BAD_PHASES: frozenset[str] = frozenset({"init", "runtime"}) - -# failure_phase values (and None) that map HARD_FAILURE -> "SKIP" -_BISECT_SKIP_PHASES: frozenset[str | None] = frozenset({"import", "driver", "oom", "hang", None}) + verdict: OracleVerdict + bisect_verdict: str + failure_phase: str | None + measured_fps: float | None + baseline_fps: float | None + regression_pct: float | None + fps_median: float | None + fps_p5: float | None + fps_p95: float | None + gpu_mem_used_mb: float | None + startup_time_s: float | None + wall_time_s: float | None + was_retried: bool + task_id: str + backend: str + baseline_sample_count: int = 0 + baseline_source: str = "none" + threshold_source: str = ThresholdSource.NO_BASELINE.value + warn_threshold_fps: float | None = None + block_threshold_fps: float | None = None + hard_floor_fps: float | None = None + min_block_regression_pct: float = MIN_BLOCK_REGRESSION_PCT + note: str | None = None + + +_BISECT_BAD_PHASES: frozenset[str] = frozenset({FailurePhase.INIT.value, FailurePhase.RUNTIME.value}) def _bisect_verdict(verdict: OracleVerdict, was_retried: bool, failure_phase: str | None) -> str: - """Compute the bisect-friendly label for a given verdict""" + """Compute the bisect-friendly label for a given verdict.""" if verdict == OracleVerdict.PASS: - return "SKIP" if was_retried else "GOOD" + return BisectVerdict.SKIP.value if was_retried else BisectVerdict.GOOD.value if verdict == OracleVerdict.WARN: - return "SKIP" + return BisectVerdict.SKIP.value if verdict == OracleVerdict.BLOCK: - return "BAD" - # HARD_FAILURE + return BisectVerdict.BAD.value if failure_phase in _BISECT_BAD_PHASES: - return "BAD" - return "SKIP" - - -# --------------------------------------------------------------------------- -# Core functions -# --------------------------------------------------------------------------- + return BisectVerdict.BAD.value + return BisectVerdict.SKIP.value def _percentile(sorted_data: list[float], p: float) -> float: - """Linear-interpolation percentile on a pre-sorted list - - Args: - sorted_data: Ascending-sorted FPS values (must be non-empty) - p: Percentile in [0, 100] - - Returns: - Interpolated value at the requested percentile - """ + """Linear-interpolation percentile on a pre-sorted non-empty list.""" n = len(sorted_data) if n == 1: return sorted_data[0] @@ -119,136 +94,138 @@ def apply_excluded_frames(fps_series: list[float], excluded_frames: frozenset[in return [fps for idx, fps in enumerate(fps_series) if idx not in excluded_frames] +def _hard_failure( + bench_result: dict, + failure_phase: str | None, + was_retried: bool, + gpu_mem_used_mb: float | None, + *, + note: str | None = None, +) -> OracleResult: + verdict = OracleVerdict.HARD_FAILURE + return OracleResult( + verdict=verdict, + bisect_verdict=_bisect_verdict(verdict, was_retried, failure_phase), + failure_phase=failure_phase, + measured_fps=None, + baseline_fps=None, + regression_pct=None, + fps_median=None, + fps_p5=None, + fps_p95=None, + gpu_mem_used_mb=gpu_mem_used_mb, + startup_time_s=bench_result.get("startup_time_s"), + wall_time_s=bench_result.get("wall_time_s"), + was_retried=was_retried, + task_id=bench_result["task_id"], + backend=bench_result.get("backend_key") or bench_result["backend"], + threshold_source=ThresholdSource.NOT_APPLICABLE.value, + note=note, + ) + + +def _extract_fps_series(perf_regression_gate_info: list[dict]) -> list[float]: + for phase in perf_regression_gate_info: + if phase.get("phase_name") == "runtime": + for measurement in phase.get("measurements", []): + if measurement.get("name", "").endswith("Step Frametimes"): + value = measurement.get("value", {}) + return list(value.get("Environment step effective FPS", [])) + break + return [] + + def compare( bench_result: dict, baseline: "Baseline | None", fps_mean_floor: float, excluded_frames: "frozenset[int]", artifact_dir: "Path", + *, + min_block_regression_pct: float = MIN_BLOCK_REGRESSION_PCT, ) -> OracleResult: - """Compare a benchmark result against its baseline and return an :class:`OracleResult` - - Oracle logic (in order): - - 1. If ``failure_phase`` is set **and** ``perf_regression_gate_info_present`` is False, return - ``HARD_FAILURE`` immediately without reading any files - 2. Load ``artifact_dir/perf_regression_gate_info.json`` and extract the FPS series - 3. Filter the series with :func:`apply_excluded_frames`. - 4. Compute mean FPS with :mod:`statistics`. - 5. Apply hard-floor check, then baseline thresholds (or seed-run PASS) - 6. Downgrade PASS to WARN when ``was_retried`` is True - 7. Compute bisect verdict and regression percentage - - Args: - bench_result: Dict matching the ``perf_regression_gate_result.json`` schema - baseline: Rolling baseline statistics, or None for a seed run - fps_mean_floor: Absolute minimum acceptable FPS (hard floor) - excluded_frames: 0-based frame indices to drop before computing mean FPS - artifact_dir: Directory that contains ``perf_regression_gate_info.json`` - - Returns: - Fully populated :class:`OracleResult` - """ + """Compare a benchmark result against its baseline and return an OracleResult.""" task_id: str = bench_result["task_id"] - backend: str = bench_result["backend"] - failure_phase: str | None = bench_result["failure_phase"] - was_retried: bool = bench_result["was_retried"] + backend: str = bench_result.get("backend_key") or bench_result["backend"] + failure_phase: str | None = bench_result.get("failure_phase") + was_retried: bool = bool(bench_result.get("was_retried")) startup_time_s: float | None = bench_result.get("startup_time_s") wall_time_s: float | None = bench_result.get("wall_time_s") gpu_mem_used_mb: float | None = (bench_result.get("gpu_diag") or {}).get("gpu_mem_used_mb") - # Treat missing perf data as HARD_FAILURE regardless of failure_phase - if not bench_result["perf_regression_gate_info_present"]: - bv = _bisect_verdict(OracleVerdict.HARD_FAILURE, was_retried, failure_phase) - return OracleResult( - verdict=OracleVerdict.HARD_FAILURE, - bisect_verdict=bv, - failure_phase=failure_phase, - measured_fps=None, - baseline_fps=None, - regression_pct=None, - fps_median=None, - fps_p5=None, - fps_p95=None, - gpu_mem_used_mb=gpu_mem_used_mb, - startup_time_s=startup_time_s, - wall_time_s=wall_time_s, - was_retried=was_retried, - task_id=task_id, - backend=backend, + config_mismatch = bench_result.get("config_mismatch") + if config_mismatch or failure_phase == FailurePhase.CONFIG_MISMATCH.value: + return _hard_failure( + bench_result, + FailurePhase.CONFIG_MISMATCH.value, + was_retried, + gpu_mem_used_mb, + note=str(config_mismatch or "config_mismatch"), ) - # Load perf_regression_gate_info.json - perf_regression_gate_info_path = Path(artifact_dir) / "perf_regression_gate_info.json" - with perf_regression_gate_info_path.open() as fh: - perf_regression_gate_info = json.load(fh) + if not bench_result.get("perf_regression_gate_info_present", False): + return _hard_failure(bench_result, failure_phase, was_retried, gpu_mem_used_mb) - # Extract FPS series - fps_series: list[float] = [] - for phase in perf_regression_gate_info: - if phase.get("phase_name") == "runtime": - for measurement in phase.get("measurements", []): - if measurement.get("name", "").endswith("Step Frametimes"): - fps_series = measurement["value"]["Environment step effective FPS"] - break - break + perf_info_path = Path(artifact_dir) / "perf_regression_gate_info.json" + with perf_info_path.open() as fh: + perf_info = json.load(fh) - # Compute filtered data, mean, and informational statistics - filtered = apply_excluded_frames(fps_series, excluded_frames) + filtered = apply_excluded_frames(_extract_fps_series(perf_info), excluded_frames) if not filtered: - bv = _bisect_verdict(OracleVerdict.HARD_FAILURE, was_retried, failure_phase) - return OracleResult( - verdict=OracleVerdict.HARD_FAILURE, - bisect_verdict=bv, - failure_phase=failure_phase, - measured_fps=None, - baseline_fps=None, - regression_pct=None, - fps_median=None, - fps_p5=None, - fps_p95=None, - gpu_mem_used_mb=gpu_mem_used_mb, - startup_time_s=startup_time_s, - wall_time_s=wall_time_s, - was_retried=was_retried, - task_id=task_id, - backend=backend, - ) + return _hard_failure(bench_result, failure_phase, was_retried, gpu_mem_used_mb, note="empty_fps_series") + mean_fps = statistics.mean(filtered) sorted_filtered = sorted(filtered) fps_median = _percentile(sorted_filtered, 50.0) fps_p5 = _percentile(sorted_filtered, 5.0) fps_p95 = _percentile(sorted_filtered, 95.0) - # Verdict logic - _MIN_SAMPLES_FOR_MAD = 2 # TODO: remove + baseline_fps = baseline.median_fps if baseline is not None else None + baseline_sample_count = baseline.sample_count if baseline is not None else 0 + baseline_source = baseline.source if baseline is not None else "none" + regression_pct = None + if baseline_fps: + regression_pct = ((mean_fps - baseline_fps) / baseline_fps) * 100.0 - if mean_fps < fps_mean_floor: + hard_floor_fps = fps_mean_floor if fps_mean_floor > 0 else None + threshold_source = ThresholdSource.NO_BASELINE.value + warn_threshold = None + block_threshold = None + note = None + + if fps_mean_floor > 0 and mean_fps < fps_mean_floor: verdict = OracleVerdict.BLOCK - elif baseline is None or baseline.sample_count < _MIN_SAMPLES_FOR_MAD: - verdict = OracleVerdict.PASS + threshold_source = ThresholdSource.HARD_FLOOR.value + note = "below_hard_floor" + elif baseline is None: + verdict = OracleVerdict.WARN + note = "no_baseline" + elif baseline.sample_count < MIN_BASELINE_SAMPLES: + verdict = OracleVerdict.WARN + threshold_source = ThresholdSource.INSUFFICIENT_WINDOW.value + note = f"insufficient_baseline(n={baseline.sample_count},min={MIN_BASELINE_SAMPLES})" else: - block_thresh = baseline.median_fps - baseline.k_block * baseline.mad_fps - warn_thresh = baseline.median_fps - baseline.k_warn * baseline.mad_fps - if mean_fps < block_thresh: - verdict = OracleVerdict.BLOCK - elif mean_fps < warn_thresh: + threshold_source = ThresholdSource.ROLLING_WINDOW.value + block_threshold = baseline.median_fps - baseline.k_block * baseline.mad_fps + warn_threshold = baseline.median_fps - baseline.k_warn * baseline.mad_fps + if mean_fps < block_threshold: + if regression_pct is None or regression_pct <= -float(min_block_regression_pct): + verdict = OracleVerdict.BLOCK + else: + verdict = OracleVerdict.WARN + note = "below_mad_block_but_inside_regression_floor" + elif mean_fps < warn_threshold: verdict = OracleVerdict.WARN else: verdict = OracleVerdict.PASS + if verdict == OracleVerdict.PASS and was_retried: verdict = OracleVerdict.WARN - - baseline_fps: float | None = baseline.median_fps if baseline is not None else None - regression_pct: float | None = None - if baseline_fps is not None: - regression_pct = ((mean_fps - baseline_fps) / baseline_fps) * 100.0 - - bisection_verdict = _bisect_verdict(verdict, was_retried, failure_phase) + note = note or "was_retried" return OracleResult( verdict=verdict, - bisect_verdict=bisection_verdict, + bisect_verdict=_bisect_verdict(verdict, was_retried, failure_phase), failure_phase=failure_phase, measured_fps=mean_fps, baseline_fps=baseline_fps, @@ -262,4 +239,12 @@ def compare( was_retried=was_retried, task_id=task_id, backend=backend, + baseline_sample_count=baseline_sample_count, + baseline_source=baseline_source, + threshold_source=threshold_source, + warn_threshold_fps=warn_threshold, + block_threshold_fps=block_threshold, + hard_floor_fps=hard_floor_fps, + min_block_regression_pct=float(min_block_regression_pct), + note=note, ) diff --git a/tools/perf_regression_gate/runtime_contract.py b/tools/perf_regression_gate/runtime_contract.py new file mode 100644 index 000000000000..5c2e02884a85 --- /dev/null +++ b/tools/perf_regression_gate/runtime_contract.py @@ -0,0 +1,99 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Runtime compatibility contract construction for baseline matching""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from typing import Any + +try: + from .backend_identity import BackendIdentity +except ImportError: # pragma: no cover - supports direct script imports + from backend_identity import BackendIdentity + + +def _canonical_json(data: dict[str, Any]) -> str: + return json.dumps(data, sort_keys=True, separators=(",", ":")) + + +def stable_hash(data: dict[str, Any]) -> str: + return hashlib.sha256(_canonical_json(data).encode("utf-8")).hexdigest()[:16] + + +def _get_path(data: Mapping[str, Any], dotted_path: str) -> Any: + cur: Any = data + for part in dotted_path.split("."): + if not isinstance(cur, Mapping) or part not in cur: + return None + cur = cur[part] + return cur + + +def _field_list(policy: Mapping[str, Any], backend: BackendIdentity) -> list[str]: + fields: list[str] = [] + + def add_many(values: Any) -> None: + for value in values or []: + field = str(value) + if field not in fields: + fields.append(field) + + add_many(policy.get("always")) + by_physics = policy.get("by_physics_backend") or {} + if isinstance(by_physics, Mapping): + add_many(by_physics.get(backend.physics_backend)) + by_render = policy.get("by_render_backend") or {} + if isinstance(by_render, Mapping) and backend.render_backend: + add_many(by_render.get(backend.render_backend)) + return fields + + +def runtime_source(provenance: dict[str, Any] | None, gpu_diag: dict[str, Any] | None) -> dict[str, Any]: + provenance = provenance or {} + return { + "software": provenance.get("software") or {}, + "hardware": provenance.get("hardware") or {}, + "gpu_diag": gpu_diag or {}, + } + + +def build_runtime_contract( + *, + provenance: dict[str, Any] | None, + gpu_diag: dict[str, Any] | None, + backend: BackendIdentity, + policy: Mapping[str, Any], +) -> tuple[dict[str, Any], str]: + """Build the compatibility contract and hash used for baseline matching.""" + source = runtime_source(provenance, gpu_diag) + selected = {field: _get_path(source, field) for field in _field_list(policy, backend)} + contract = { + "runtime_contract_version": int(policy.get("contract_version", 1)), + "fields": selected, + } + return contract, stable_hash(contract) + + +def build_runtime_publish_info( + *, + provenance: dict[str, Any] | None, + gpu_diag: dict[str, Any] | None, + policy: Mapping[str, Any], +) -> dict[str, Any]: + """Return debug/runtime fields published for humans but not used for matching.""" + source = runtime_source(provenance, gpu_diag) + publish_only = { + str(field): _get_path(source, str(field)) + for field in policy.get("publish_only", []) + } + publish_only = {key: value for key, value in publish_only.items() if value is not None} + return { + "software": source["software"], + "publish_only": publish_only, + } diff --git a/tools/perf_regression_gate/task_config.py b/tools/perf_regression_gate/task_config.py index 3b8293f14114..35b616c0d93f 100644 --- a/tools/perf_regression_gate/task_config.py +++ b/tools/perf_regression_gate/task_config.py @@ -7,6 +7,11 @@ from dataclasses import dataclass, field from pathlib import Path +try: + from .backend_identity import make_backend_key, normalize_physics_backend, normalize_render_backend +except ImportError: # pragma: no cover - supports direct script imports + from backend_identity import make_backend_key, normalize_physics_backend, normalize_render_backend + _DEFAULT_TASKS_JSON = Path(__file__).parent / "tasks.json" # Maps backend name to list of cache identifiers that CI needs to pull; absent key defaults to no caches @@ -53,6 +58,7 @@ class TaskConfig: task_type: str = "benchmark" runs_on: str = "gpu-l40s" seed: int | None = None + baseline_epoch: int = 1 @property def backend_key(self) -> str: @@ -61,9 +67,7 @@ def backend_key(self) -> str: Returns f"{physics_backend}_{render_backend}" when render_backend is set, otherwise returns physics_backend. """ - if self.render_backend: - return f"{self.physics_backend}_{self.render_backend}" - return self.physics_backend + return make_backend_key(self.physics_backend, self.render_backend) @property def excluded_frames(self) -> frozenset[int]: @@ -131,8 +135,10 @@ def load_tasks(tasks_json_path: Path | str | None = None) -> list[TaskConfig]: backends: list[dict] = merged.get("backends", []) for backend_entry in backends: - physics = backend_entry["physics"] - render = backend_entry.get("render") + physics = normalize_physics_backend(backend_entry["physics"]) + if physics is None: + raise ValueError(f"backend entry in {path} must define a non-default physics backend") + render = normalize_render_backend(backend_entry.get("render")) tasks.append( TaskConfig( task_id=merged["task_id"], @@ -150,6 +156,7 @@ def load_tasks(tasks_json_path: Path | str | None = None) -> list[TaskConfig]: task_type=merged["type"], runs_on=merged["runs_on"], seed=merged.get("seed"), + baseline_epoch=int(merged.get("baseline_epoch", 1)), ) ) return tasks diff --git a/tools/perf_regression_gate/tasks_to_ci_matrix.py b/tools/perf_regression_gate/tasks_to_ci_matrix.py index 04271c78ca87..dab6390ccfdd 100644 --- a/tools/perf_regression_gate/tasks_to_ci_matrix.py +++ b/tools/perf_regression_gate/tasks_to_ci_matrix.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Convert tasks.json into the GitHub Actions bench matrix JSON. +"""Convert tasks.json into the GitHub Actions bench matrix JSON Prints a JSON array to stdout, one object per (task_id, backend) combination, containing the fields consumed by the ``bench`` job matrix in perf-regression-gate.yaml. @@ -19,6 +19,7 @@ sys.path.insert(0, str(Path(__file__).parent)) +from launch_config import hydra_args_for_task # noqa: E402 from task_config import load_tasks # noqa: E402 tasks = load_tasks() @@ -30,6 +31,8 @@ "render_backend": task.render_backend or "", "num_envs": task.num_envs, "num_frames": task.num_frames, + "seed": task.seed if task.seed is not None else "", + "hydra_args": " ".join(hydra_args_for_task(task)), "bench_timeout_s": task.timeout_minutes * 60, "job_timeout_minutes": max(30, task.timeout_minutes + 15), }) diff --git a/tools/perf_regression_gate/write_launch_config.py b/tools/perf_regression_gate/write_launch_config.py new file mode 100644 index 000000000000..a996b9ce9779 --- /dev/null +++ b/tools/perf_regression_gate/write_launch_config.py @@ -0,0 +1,55 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Write launch_config.json for a perf-gate task/backend job""" + +import argparse +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) + +from backend_identity import make_backend_key, normalize_render_backend +from gpu_identity import gpu_model_config_keys +from launch_config import hydra_args_for_task, task_to_launch_config, write_launch_config +from task_config import get_task + + +def _parse_args(): + parser = argparse.ArgumentParser(description="Write launch_config.json for a perf-gate benchmark job") + parser.add_argument("--task_id", required=True) + parser.add_argument("--physics_backend", required=True) + parser.add_argument("--render_backend", default="") + parser.add_argument("--gpu_model", default="L40S") + parser.add_argument("--artifact_dir", required=True, type=Path) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + render_backend = normalize_render_backend(args.render_backend) + backend_key = make_backend_key(args.physics_backend, render_backend) + task = get_task(args.task_id, backend_key) + fps_mean_floor = 0.0 + for gpu_key in gpu_model_config_keys(args.gpu_model): + configured_floor = task.fps_mean_floor.get(gpu_key, {}).get(task.backend_key) + if configured_floor is not None: + fps_mean_floor = float(configured_floor) + break + config = task_to_launch_config( + task, + fps_mean_floor=fps_mean_floor, + gpu_model=args.gpu_model, + hydra_args=hydra_args_for_task(task), + ) + path = write_launch_config(args.artifact_dir, config) + print(path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/subprocess_runner.py b/tools/subprocess_runner.py index 57a750202647..6075eff42844 100644 --- a/tools/subprocess_runner.py +++ b/tools/subprocess_runner.py @@ -11,6 +11,8 @@ import sys import time +from perf_regression_gate.gate_types import FailurePhase + def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, report_file=""): """Run a command with timeout and capture all output while streaming in real-time. @@ -245,27 +247,27 @@ def classify_failure_phase( # 1. OOM if exit_code == 137 or "oom-kill" in stderr: - return "oom" + return FailurePhase.OOM.value # 2. Hang if wall_time_s >= timeout_s * 0.95: - return "hang" + return FailurePhase.HANG.value # 3. Import error if "Traceback" in combined and "AppLauncher" not in stdout: - return "import" + return FailurePhase.IMPORT.value # 4. Driver error if "CudaError" in combined or "CUDA_ERROR_" in combined: - return "driver" + return FailurePhase.DRIVER.value # 5. Init failure if exit_code != 0 and "AppLauncher initialization complete" in stdout and "Step Frametimes" not in stdout: - return "init" + return FailurePhase.INIT.value # 6. Runtime failure (partial run then crash) if exit_code != 0 and "Step Frametimes" in stdout: - return "runtime" + return FailurePhase.RUNTIME.value # 7. Clean exit return None