diff --git a/.github/workflows/perf-regression-gate.yaml b/.github/workflows/perf-regression-gate.yaml new file mode 100644 index 000000000000..cefcf6be469a --- /dev/null +++ b/.github/workflows/perf-regression-gate.yaml @@ -0,0 +1,439 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Performance Regression Gate +# +# GPU jobs default to the RTX PRO 6000 runner label used by the upstream +# deployment. Override with repo variable PERF_GATE_RUNS_ON, e.g. a JSON array +# of runner labels. +# +# Runs a matrix of benchmark tasks on self-hosted runners, compares +# results to the rolling baseline in angehu/perf-baselines, and posts a verdict +# table to the job summary. Non-blocking by default (gate_config.json). +# +# Architecture: +# config → load isaacsim version from config.yaml (ubuntu-latest) +# bench → per-task/backend matrix, each pulling the CI image and running +# benchmark_non_rl.py inside Docker (self-hosted, gpu) +# aggregate → download all bench artifacts, run oracle, update baselines +# (self-hosted, gpu) +# +# Baseline updates are published from protected-branch push events +# (main/develop/release) plus angehu/perf-gate-poc while this POC is active. +# Mirrored PR pushes under pull-request/ can run the gate on upstream +# RTX runners; they are read-only unless PERF_GATE_ALLOW_MIRROR_BASELINE_UPDATE +# is explicitly enabled as a temporary deployment escape hatch. +# +# backend_key: "{physics_backend}_{render_backend}" when render_backend is +# set, otherwise just physics_backend. Matches TaskConfig.backend_key. +# TODO: replace angehu/perf-baselines w/ real branch name +# TODO: remove angehu/perf-gate-poc after deployment + +name: Performance Regression Gate + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: [main, develop, 'release/**'] + merge_group: + branches: [main, develop, 'release/**'] + push: + branches: [main, develop, 'release/**', angehu/perf-gate-poc, 'pull-request/**'] + workflow_dispatch: + +concurrency: + group: perf-gate-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: write + pull-requests: read + +env: + CI_IMAGE_TAG: >- + isaac-lab-ci:${{ + github.event_name == 'pull_request' + && format('pr-{0}', github.event.pull_request.number) + || 'sha' + }}-${{ github.sha }} + +jobs: + # --------------------------------------------------------------------------- + # Load shared image config (matches build.yaml pattern) + # --------------------------------------------------------------------------- + config: + name: Load Config + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} + runs-on: ubuntu-latest + outputs: + isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }} + isaacsim_image_tag: ${{ steps.load.outputs.isaacsim_image_tag }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + sparse-checkout: + - .github/workflows/config.yaml + - tools/perf_regression_gate/tasks.json + - tools/perf_regression_gate/task_config.py + sparse-checkout-cone-mode: false + - id: load + run: | + set -euo pipefail + f=.github/workflows/config.yaml + echo "isaacsim_image_name=$(yq -r .isaacsim_image_name "$f")" >> "$GITHUB_OUTPUT" + echo "isaacsim_image_tag=$(yq -r .isaacsim_image_tag "$f")" >> "$GITHUB_OUTPUT" + - id: build_matrix + run: | + set -euo pipefail + MATRIX=$(python3 tools/perf_regression_gate/tasks_to_ci_matrix.py) + echo "bench_matrix=$MATRIX" >> "$GITHUB_OUTPUT" + + # --------------------------------------------------------------------------- + # Per-task/backend benchmark jobs + # --------------------------------------------------------------------------- + bench: + name: Bench / ${{ matrix.task_id }} / ${{ matrix.physics_backend }}${{ matrix.render_backend != '' && format('_{0}', matrix.render_backend) || '' }} + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} + runs-on: ${{ fromJSON(vars.PERF_GATE_RUNS_ON || '["linux-amd64-gpu-rtxpro6000-latest-1"]') }} + needs: [config] + continue-on-error: true + timeout-minutes: ${{ matrix.job_timeout_minutes }} + strategy: + fail-fast: false + matrix: + # Load the benchmark matrix from the canonical tasks.json task definition file. + include: ${{ fromJson(needs.config.outputs.bench_matrix) }} + + steps: + - name: Checkout Code + uses: actions/checkout@v6 + with: + fetch-depth: 1 + lfs: true + + # Pull the CI image built (or cached) by the build workflow for this commit. + # ecr-build-push-pull will build the image if it is not already in ECR. + - name: Pull CI image + uses: ./.github/actions/ecr-build-push-pull + with: + image-tag: ${{ env.CI_IMAGE_TAG }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + + - name: Run benchmark in Docker + id: bench_run + run: | + set -euo pipefail + TASK_ID="${{ matrix.task_id }}" + PHYSICS_BACKEND="${{ matrix.physics_backend }}" + RENDER_BACKEND="${{ matrix.render_backend }}" + + # Compute backend_key: "{physics}_{render}" when render is set, else "{physics}" + if [ -n "${RENDER_BACKEND}" ]; then + BACKEND_KEY="${PHYSICS_BACKEND}_${RENDER_BACKEND}" + else + BACKEND_KEY="${PHYSICS_BACKEND}" + fi + + ARTIFACT_DIR="$(pwd)/artifacts/${TASK_ID}/${BACKEND_KEY}" + + # Sanitize task_id and backend_key for Docker container name + SAFE_TASK_ID="${TASK_ID//[^a-zA-Z0-9]/-}" + SAFE_BACKEND="${BACKEND_KEY//[^a-zA-Z0-9]/-}" + CONTAINER_NAME="perf-bench-${SAFE_TASK_ID}-${SAFE_BACKEND}-${{ github.run_id }}" + LOG_FILE="${ARTIFACT_DIR}/benchmark.log" + + mkdir -p "${ARTIFACT_DIR}" + GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)" + if [[ -z "${GPU_MODEL}" ]]; then + GPU_MODEL="unknown-gpu" + fi + python3 tools/perf_regression_gate/write_launch_config.py \ + --task_id "${TASK_ID}" \ + --physics_backend "${PHYSICS_BACKEND}" \ + --render_backend "${RENDER_BACKEND}" \ + --gpu_model "${GPU_MODEL}" \ + --artifact_dir "${ARTIFACT_DIR}" + + # Remove any stale container from a previous run attempt + docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + + # Build optional render backend preset token for the CLI. + # benchmark_non_rl.py accepts Hydra-style preset tokens (no leading dashes). + # Physics is selected via the sim_backend token; render via renderer= token. + # TODO: Update these tokens once benchmark_non_rl.py CLI flags are finalised. + HYDRA_ARGS="${{ matrix.hydra_args }}" + + # Start the benchmark container in detached mode + docker run -d --name "${CONTAINER_NAME}" \ + --init --stop-timeout 10 \ + --entrypoint bash --gpus all --network=host \ + --security-opt=no-new-privileges:true \ + --ulimit nofile=65536:65536 \ + --ulimit nproc=4096:4096 \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e PYTHONUNBUFFERED=1 \ + -v "${ARTIFACT_DIR}:/tmp/bench_out" \ + "${{ env.CI_IMAGE_TAG }}" \ + -c " + set -e + cd /workspace/isaaclab + rm -f _isaac_sim + ln -s /isaac-sim _isaac_sim + ./isaaclab.sh -p scripts/benchmarks/benchmark_non_rl.py \ + --task '${TASK_ID}' \ + --num_envs ${{ matrix.num_envs }} \ + --num_frames ${{ matrix.num_frames }} \ + --benchmark_backend json \ + --output_path /tmp/bench_out \ + ${{ matrix.seed != '' && format('--seed {0}', matrix.seed) || '' }} \ + ${HYDRA_ARGS} + " + + START=$(date +%s) + + # Stream container logs to file and terminal while benchmark runs + docker logs -f "${CONTAINER_NAME}" 2>&1 | tee "${LOG_FILE}" & + LOGS_PID=$! + + # Wait for the container to exit, with a hard wall-clock timeout + BENCH_EXIT=1 + if docker_exit=$(timeout ${{ matrix.bench_timeout_s }} docker wait "${CONTAINER_NAME}" 2>/dev/null); then + BENCH_EXIT="${docker_exit:-1}" + else + echo "::warning::Benchmark for ${TASK_ID}/${BACKEND_KEY} timed out after ${{ matrix.bench_timeout_s }}s" + fi + + END=$(date +%s) + WALL_TIME_S=$((END - START)) + + kill "${LOGS_PID}" 2>/dev/null || true + wait "${LOGS_PID}" 2>/dev/null || true + + docker kill "${CONTAINER_NAME}" 2>/dev/null || true + docker rm "${CONTAINER_NAME}" 2>/dev/null || true + + echo "exit_code=${BENCH_EXIT}" >> "$GITHUB_OUTPUT" + echo "wall_time_s=${WALL_TIME_S}" >> "$GITHUB_OUTPUT" + echo "artifact_dir=${ARTIFACT_DIR}" >> "$GITHUB_OUTPUT" + echo "backend_key=${BACKEND_KEY}" >> "$GITHUB_OUTPUT" + + # Retry once on first-attempt failure before giving up. + # Runs only when bench_run reports a non-zero exit code. + - name: Retry benchmark on failure + id: bench_retry + if: always() && steps.bench_run.outputs.exit_code != '0' && steps.bench_run.outputs.exit_code != '' + run: | + set -uo pipefail + TASK_ID="${{ matrix.task_id }}" + PHYSICS_BACKEND="${{ matrix.physics_backend }}" + RENDER_BACKEND="${{ matrix.render_backend }}" + + if [ -n "${RENDER_BACKEND}" ]; then + BACKEND_KEY="${PHYSICS_BACKEND}_${RENDER_BACKEND}" + else + BACKEND_KEY="${PHYSICS_BACKEND}" + fi + + ARTIFACT_DIR="$(pwd)/artifacts/${TASK_ID}/${BACKEND_KEY}" + SAFE_TASK_ID="${TASK_ID//[^a-zA-Z0-9]/-}" + SAFE_BACKEND="${BACKEND_KEY//[^a-zA-Z0-9]/-}" + CONTAINER_NAME="perf-bench-${SAFE_TASK_ID}-${SAFE_BACKEND}-${{ github.run_id }}-retry" + LOG_FILE="${ARTIFACT_DIR}/benchmark.log" + + echo "::notice::First attempt failed (exit=${{ steps.bench_run.outputs.exit_code }}); retrying benchmark for ${TASK_ID}/${BACKEND_KEY}" + + # Remove any partial perf output from the failed first attempt so the retry's + # output is picked up cleanly by build_bench_result's normalize step. + rm -f "${ARTIFACT_DIR}/perf_regression_gate_info.json" + + docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + + HYDRA_ARGS="${{ matrix.hydra_args }}" + + docker run -d --name "${CONTAINER_NAME}" \ + --init --stop-timeout 10 \ + --entrypoint bash --gpus all --network=host \ + --security-opt=no-new-privileges:true \ + --ulimit nofile=65536:65536 \ + --ulimit nproc=4096:4096 \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e PYTHONUNBUFFERED=1 \ + -v "${ARTIFACT_DIR}:/tmp/bench_out" \ + "${{ env.CI_IMAGE_TAG }}" \ + -c " + set -e + cd /workspace/isaaclab + rm -f _isaac_sim + ln -s /isaac-sim _isaac_sim + ./isaaclab.sh -p scripts/benchmarks/benchmark_non_rl.py \ + --task '${TASK_ID}' \ + --num_envs ${{ matrix.num_envs }} \ + --num_frames ${{ matrix.num_frames }} \ + --benchmark_backend json \ + --output_path /tmp/bench_out \ + ${{ matrix.seed != '' && format('--seed {0}', matrix.seed) || '' }} \ + ${HYDRA_ARGS} + " + + START=$(date +%s) + + docker logs -f "${CONTAINER_NAME}" 2>&1 | tee "${LOG_FILE}" & + LOGS_PID=$! + + BENCH_EXIT=1 + if docker_exit=$(timeout ${{ matrix.bench_timeout_s }} docker wait "${CONTAINER_NAME}" 2>/dev/null); then + BENCH_EXIT="${docker_exit:-1}" + else + echo "::warning::Retry benchmark for ${TASK_ID}/${BACKEND_KEY} timed out after ${{ matrix.bench_timeout_s }}s" + fi + + END=$(date +%s) + WALL_TIME_S=$((END - START)) + + kill "${LOGS_PID}" 2>/dev/null || true + wait "${LOGS_PID}" 2>/dev/null || true + + docker kill "${CONTAINER_NAME}" 2>/dev/null || true + docker rm "${CONTAINER_NAME}" 2>/dev/null || true + + echo "exit_code=${BENCH_EXIT}" >> "$GITHUB_OUTPUT" + echo "wall_time_s=${WALL_TIME_S}" >> "$GITHUB_OUTPUT" + + # Build bench_result.json from the Docker run outputs. + # Runs even when bench_run/bench_retry fail so partial results are always uploaded. + # Uses the retry's exit code and wall time when a retry ran, and sets was_retried accordingly. + - name: Build bench_result + if: always() + run: | + TASK_ID="${{ matrix.task_id }}" + PHYSICS_BACKEND="${{ matrix.physics_backend }}" + RENDER_BACKEND="${{ matrix.render_backend }}" + if [ -n "${RENDER_BACKEND}" ]; then + BACKEND_KEY="${PHYSICS_BACKEND}_${RENDER_BACKEND}" + else + BACKEND_KEY="${PHYSICS_BACKEND}" + fi + ARTIFACT_DIR="$(pwd)/artifacts/${TASK_ID}/${BACKEND_KEY}" + + # If a retry ran, use its exit code/wall time and mark was_retried. + RETRY_EXIT="${{ steps.bench_retry.outputs.exit_code }}" + if [ -n "${RETRY_EXIT}" ]; then + FINAL_EXIT="${RETRY_EXIT}" + FINAL_WALL="${{ steps.bench_retry.outputs.wall_time_s || '0' }}" + EXTRA_FLAGS="--was_retried --attempt 2" + else + FINAL_EXIT="${{ steps.bench_run.outputs.exit_code || '1' }}" + FINAL_WALL="${{ steps.bench_run.outputs.wall_time_s || '0' }}" + EXTRA_FLAGS="" + fi + + python3 tools/perf_regression_gate/build_bench_result.py \ + --task_id "${TASK_ID}" \ + --physics_backend "${PHYSICS_BACKEND}" \ + --render_backend "${RENDER_BACKEND}" \ + --artifact_dir "${ARTIFACT_DIR}" \ + --exit_code "${FINAL_EXIT}" \ + --wall_time_s "${FINAL_WALL}" \ + --timeout_s "${{ matrix.bench_timeout_s }}" \ + --log_file "${ARTIFACT_DIR}/benchmark.log" \ + --launch_config "${ARTIFACT_DIR}/launch_config.json" \ + --gate_config tools/perf_regression_gate/gate_config.json \ + ${EXTRA_FLAGS} + + - name: Upload bench artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: bench-${{ matrix.task_id }}-${{ matrix.physics_backend }}${{ matrix.render_backend != '' && format('_{0}', matrix.render_backend) || '' }}-${{ github.run_id }} + path: artifacts/${{ matrix.task_id }}/${{ matrix.physics_backend }}${{ matrix.render_backend != '' && format('_{0}', matrix.render_backend) || '' }}/ + retention-days: 7 + if-no-files-found: warn + + # Force-clean the container if the job is cancelled mid-run + - name: Cleanup container on cancellation + if: cancelled() + run: | + TASK_ID="${{ matrix.task_id }}" + PHYSICS_BACKEND="${{ matrix.physics_backend }}" + RENDER_BACKEND="${{ matrix.render_backend }}" + if [ -n "${RENDER_BACKEND}" ]; then + BACKEND_KEY="${PHYSICS_BACKEND}_${RENDER_BACKEND}" + else + BACKEND_KEY="${PHYSICS_BACKEND}" + fi + SAFE_TASK_ID="${TASK_ID//[^a-zA-Z0-9]/-}" + SAFE_BACKEND="${BACKEND_KEY//[^a-zA-Z0-9]/-}" + CONTAINER_NAME="perf-bench-${SAFE_TASK_ID}-${SAFE_BACKEND}-${{ github.run_id }}" + docker kill "${CONTAINER_NAME}" 2>/dev/null || true + docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + + # --------------------------------------------------------------------------- + # Aggregate: oracle verdicts, baseline update, step summary + # --------------------------------------------------------------------------- + aggregate: + name: Aggregate + Verdict + runs-on: ${{ fromJSON(vars.PERF_GATE_RUNS_ON || '["linux-amd64-gpu-rtxpro6000-latest-1"]') }} + needs: [bench] + # Always run for real gate attempts, even when some bench jobs fail. + if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} + + steps: + - name: Checkout Code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + # Prime the remote-tracking baseline ref. aggregate.py refetches immediately + # before reading and again inside the transactional push retry loop. + - name: Fetch baselines branch + run: | + git fetch origin +refs/heads/angehu/perf-baselines:refs/remotes/origin/angehu/perf-baselines || \ + echo "::warning::Baseline branch not found; gate will run as a seed run (no baseline comparison)" + + - name: Download bench artifacts + uses: actions/download-artifact@v4 + with: + pattern: bench-*-${{ github.run_id }} + path: artifacts/ + merge-multiple: false + + - name: Resolve gate context + id: gate_context + env: + ALLOW_MIRROR_UPDATE: ${{ vars.PERF_GATE_ALLOW_MIRROR_BASELINE_UPDATE || 'false' }} + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 tools/perf_regression_gate/github_gate_context.py + + - name: Run aggregate oracle + id: aggregate + run: | + GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)" + if [[ -z "${GPU_MODEL}" ]]; then + GPU_MODEL="unknown-gpu" + fi + python3 tools/perf_regression_gate/aggregate.py \ + --artifacts_dir artifacts/ \ + --gpu_model "${GPU_MODEL}" \ + --gate_config tools/perf_regression_gate/gate_config.json \ + --baseline_branch angehu/perf-baselines \ + --baseline_remote origin \ + --baseline_push_retries 3 \ + --base_sha "${{ steps.gate_context.outputs.base_sha }}" \ + --target_branch "${{ steps.gate_context.outputs.target_branch }}" \ + --source_branch "${{ steps.gate_context.outputs.source_branch }}" \ + --allow_baseline_update "${{ steps.gate_context.outputs.allow_update }}" \ + --trusted_source "${{ steps.gate_context.outputs.trusted_source }}" \ + --summary_file "${GITHUB_STEP_SUMMARY}" diff --git a/tools/perf_regression_gate/.gitignore b/tools/perf_regression_gate/.gitignore new file mode 100644 index 000000000000..84ec617f93d3 --- /dev/null +++ b/tools/perf_regression_gate/.gitignore @@ -0,0 +1,6 @@ +# Generated at runtime — not source +artifacts/ +local_baselines/ +__pycache__/ +*.pyc +.pytest_cache/ diff --git a/tools/perf_regression_gate/aggregate.py b/tools/perf_regression_gate/aggregate.py new file mode 100644 index 000000000000..62de74f5e4de --- /dev/null +++ b/tools/perf_regression_gate/aggregate.py @@ -0,0 +1,348 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Aggregate benchmark artifacts, run the oracle, and update trusted baselines""" + +import argparse +import json +import os +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +_TOOLS_DIR = _MODULE_DIR.parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) +DEFAULT_BASELINE_BRANCH = "angehu/perf-baselines" # TODO: replace with actual dorphan branch + +from baseline_manager import ( # noqa: E402 + BaselineUpdateRecord, + load_baseline, + load_baseline_git, + make_sample_metadata, + match_context_from_bench_result, + refresh_baseline_branch, + update_baseline, + update_baselines_git, +) +from gate_config import BASELINE_PUSH_RETRIES, load_gate_config # noqa: E402 +from gpu_identity import canonical_gpu_model, gpu_model_config_keys # noqa: E402 +from gate_types import OracleVerdict # noqa: E402 +from oracle import compare # noqa: E402 +from task_config import get_task # noqa: E402 + + +def _parse_args(): + parser = argparse.ArgumentParser(description="Aggregate bench results and run oracle.") + parser.add_argument("--artifacts_dir", required=True, type=Path) + parser.add_argument("--gpu_model", default="L40S") + parser.add_argument("--gate_config", type=Path, default=_MODULE_DIR / "gate_config.json") + parser.add_argument("--baseline_branch", default=DEFAULT_BASELINE_BRANCH) + parser.add_argument("--baseline_remote", default="origin", help="Git remote that owns the baseline branch; empty = local only") + parser.add_argument("--baseline_push_retries", type=int, default=None) + parser.add_argument("--baselines_dir", type=Path, default=None, help="Flat-file baseline directory; bypasses git") + parser.add_argument("--allow_baseline_update", default="false") + parser.add_argument("--summary_file", default=None) + parser.add_argument("--base_sha", default=None, help="PR base SHA for ancestry-aware baseline matching") + parser.add_argument("--target_branch", default=None, help="Target protected branch, e.g. main/develop/release/x") + parser.add_argument("--source_branch", default=None, help="Branch that produced baseline updates") + parser.add_argument("--trusted_source", default="protected_branch", help="Audit label for baseline samples written by this run") + return parser.parse_args() + + +def _find_bench_results(artifacts_dir: Path) -> list[tuple[Path, dict]]: + found = [] + for path in sorted(artifacts_dir.rglob("perf_regression_gate_result.json")): + with path.open() as fh: + found.append((path.parent, json.load(fh))) + return found + + +def _excluded_frames(bench_result: dict) -> frozenset[int]: + launch_config = bench_result.get("launch_config") or {} + raw = launch_config.get("excluded_frames_raw") + if raw is None: + raw = (bench_result.get("task_config_snapshot") or {}).get("excluded_frames_raw", []) + indices: set[int] = set() + for entry in raw or []: + if isinstance(entry, list): + indices.update(range(int(entry[0]), int(entry[1]) + 1)) + else: + indices.add(int(entry)) + return frozenset(indices) + + +def _fmt(value, decimals: int = 1) -> str: + return f"{value:.{decimals}f}" if value is not None else "N/A" + + +def _short_sha(value: str | None) -> str: + return value[:12] if value else "none" + + +def _bench_gpu_model(bench_result: dict, fallback: str) -> str: + launch_config = bench_result.get("launch_config") or {} + gpu_model = canonical_gpu_model(launch_config.get("gpu_model") or launch_config.get("gpu_model_raw")) + return canonical_gpu_model(fallback) if gpu_model == "unknown_gpu" else gpu_model + + +def _hard_floor(bench_result: dict, gpu_model: str, backend: str) -> float: + launch_config = bench_result.get("launch_config") or {} + if launch_config.get("fps_mean_floor") is not None: + return float(launch_config.get("fps_mean_floor") or 0.0) + try: + task = get_task(bench_result["task_id"], backend) + for key in gpu_model_config_keys(gpu_model): + value = task.fps_mean_floor.get(key, {}).get(backend) + if value is not None: + return float(value) + return 0.0 + except Exception: + return 0.0 + + +def _build_summary_table(rows: list[tuple]) -> str: + lines = [ + "| Task | Backend | Verdict | FPS | Baseline | Samples | Regression% | Floor | Threshold | Phase | Retry | GPU | Runtime | Note |", + "|---|---|---|---:|---:|---:|---:|---:|---|---|---|---|---|---|", + ] + for result, bench_result in rows: + gpu_diag = bench_result.get("gpu_diag") or {} + launch_config = bench_result.get("launch_config") or {} + gpu_name = gpu_diag.get("gpu_name") or launch_config.get("gpu_model_raw") or launch_config.get("gpu_model", "") + provenance = bench_result.get("provenance") or {} + software = provenance.get("software") or {} + runtime = ", ".join( + part + for part in ( + f"cuda={gpu_diag.get('cuda_version')}" if gpu_diag.get("cuda_version") else "", + f"driver={gpu_diag.get('nvidia_driver_version')}" if gpu_diag.get("nvidia_driver_version") else "", + f"warp={software.get('warp')}" if software.get("warp") else "", + ) + if part + ) + note_parts = [part for part in (result.note, bench_result.get("config_mismatch")) if part] + if bench_result.get("p99_over_median") is not None: + note_parts.append(f"p99/med={bench_result['p99_over_median']}") + if bench_result.get("outlier_count") is not None: + note_parts.append(f"outliers={bench_result['outlier_count']}") + lines.append( + f"| {result.task_id} | {result.backend} | {result.verdict.value}" + f" | {_fmt(result.measured_fps)} | {_fmt(result.baseline_fps)} | {result.baseline_sample_count}" + f" | {_fmt(result.regression_pct, 2)} | {_fmt(result.hard_floor_fps)} | {result.threshold_source}" + f" | {result.failure_phase or ''} | {result.was_retried} | {gpu_name}" + f" | {runtime} | {'; '.join(note_parts)} |" + ) + return "\n".join(lines) + + +def _write_github_output(**values) -> None: + github_output = os.environ.get("GITHUB_OUTPUT", "") + if not github_output: + return + with open(github_output, "a") as fh: + for key, value in values.items(): + if value is not None: + fh.write(f"{key}={value}\n") + + +def main() -> int: + args = _parse_args() + use_flat = args.baselines_dir is not None + allow_update = args.allow_baseline_update.strip().lower() in ("true", "1", "yes") + baseline_remote = args.baseline_remote or None + + gate_config = load_gate_config(args.gate_config) + blocking = bool(gate_config.get("blocking", False)) + min_block_regression_pct = float(gate_config.get("min_block_regression_pct", 3.0)) + baseline_push_retries = int(args.baseline_push_retries or gate_config.get("baseline_push_retries", BASELINE_PUSH_RETRIES)) + + items = _find_bench_results(args.artifacts_dir) + if not items: + print(f"[aggregate] No perf_regression_gate_result.json files found under {args.artifacts_dir}") + return 1 + + baseline_read_sha = None + baseline_read_ref = None + if not use_flat: + try: + baseline_read_sha = refresh_baseline_branch(args.baseline_branch, remote=baseline_remote, allow_missing=True) + baseline_read_ref = baseline_read_sha + except Exception as exc: + print(f"::error::Failed to refresh baseline branch before reading: {exc}") + return 1 + if baseline_read_sha: + print(f"[aggregate] Baseline read snapshot: {args.baseline_branch}@{_short_sha(baseline_read_sha)}") + else: + print(f"[aggregate] Baseline branch {args.baseline_branch!r} not found; treating this as a seed run") + + rows = [] + has_block = False + has_hard_failure = False + baselines_updated = False + baseline_update_failed = False + pending_git_updates: list[BaselineUpdateRecord] = [] + + for artifact_dir, bench_result in items: + task_id = bench_result["task_id"] + backend = bench_result.get("backend_key") or bench_result.get("backend") + bench_gpu_model = _bench_gpu_model(bench_result, args.gpu_model) + match_context = match_context_from_bench_result( + bench_result, + gpu_model=bench_gpu_model, + base_sha=args.base_sha, + target_branch=args.target_branch, + ) + + baseline = None + try: + if use_flat: + baseline = load_baseline(args.baselines_dir, bench_gpu_model, task_id, backend, match_context=match_context) + elif baseline_read_ref: + baseline = load_baseline_git( + baseline_read_ref, + bench_gpu_model, + task_id, + backend, + None, + match_context, + ) + except Exception as exc: + print(f"[aggregate] Warning: baseline load failed for {task_id}/{backend}: {exc}") + + oracle_result = compare( + bench_result=bench_result, + baseline=baseline, + fps_mean_floor=_hard_floor(bench_result, bench_gpu_model, backend), + excluded_frames=_excluded_frames(bench_result), + artifact_dir=artifact_dir, + min_block_regression_pct=min_block_regression_pct, + ) + rows.append((oracle_result, bench_result)) + + print( + f"[aggregate] {task_id}/{backend}: {oracle_result.verdict.value}" + f" fps={_fmt(oracle_result.measured_fps)} baseline={_fmt(oracle_result.baseline_fps)}" + f" samples={oracle_result.baseline_sample_count} source={oracle_result.threshold_source}" + ) + + if oracle_result.verdict == OracleVerdict.BLOCK: + has_block = True + elif oracle_result.verdict == OracleVerdict.HARD_FAILURE: + has_hard_failure = True + + if ( + allow_update + and oracle_result.verdict in (OracleVerdict.PASS, OracleVerdict.WARN) + and oracle_result.measured_fps is not None + ): + sample_metadata = make_sample_metadata( + gpu_model=bench_gpu_model, + task_id=task_id, + backend=backend, + fps=oracle_result.measured_fps, + bench_result=bench_result, + target_branch=args.target_branch, + source_branch=args.source_branch, + trusted_source=args.trusted_source, + ) + if baseline_read_sha: + sample_metadata["baseline_read_sha"] = baseline_read_sha + + if use_flat: + try: + update_baseline( + args.baselines_dir, + bench_gpu_model, + task_id, + backend, + oracle_result.measured_fps, + sample_metadata=sample_metadata, + ) + baselines_updated = True + print(f"[aggregate] -> baseline updated locally: {oracle_result.measured_fps:.1f} FPS") + except Exception as exc: + baseline_update_failed = True + print(f"::error::Baseline update failed for {task_id}/{backend}: {exc}") + else: + pending_git_updates.append( + BaselineUpdateRecord( + gpu_model=bench_gpu_model, + task_id=task_id, + backend=backend, + fps=oracle_result.measured_fps, + sample_metadata=sample_metadata, + ) + ) + print(f"[aggregate] -> baseline update queued: {oracle_result.measured_fps:.1f} FPS") + + baseline_push_result = None + if pending_git_updates: + try: + baseline_push_result = update_baselines_git( + args.baseline_branch, + pending_git_updates, + remote=baseline_remote, + max_retries=baseline_push_retries, + ) + baselines_updated = baseline_push_result.pushed + if baseline_push_result.pushed: + print( + f"[aggregate] Baseline push succeeded: {args.baseline_branch}@" + f"{_short_sha(baseline_push_result.pushed_sha)} " + f"after {baseline_push_result.attempts} attempt(s)" + ) + else: + print("[aggregate] Baseline samples were already present; no push needed") + except Exception as exc: + baseline_update_failed = True + print(f"::error::Baseline push failed: {exc}") + + table = _build_summary_table(rows) + print("\n## Performance Gate Results\n") + print(table) + print() + + if args.summary_file: + with open(args.summary_file, "a") as fh: + fh.write("\n## Performance Gate Results\n\n") + if not use_flat: + fh.write(f"Baseline read SHA: `{_short_sha(baseline_read_sha)}`\n\n") + if baseline_push_result and baseline_push_result.pushed_sha: + fh.write( + f"Baseline pushed SHA: `{_short_sha(baseline_push_result.pushed_sha)}` " + f"after {baseline_push_result.attempts} attempt(s)\n\n" + ) + fh.write(table) + fh.write("\n") + + output_values = {"baseline_read_sha": baseline_read_sha} + if baseline_push_result: + output_values.update( + { + "baselines_updated": "true" if baseline_push_result.pushed else "false", + "baseline_pushed_sha": baseline_push_result.pushed_sha, + "baseline_push_attempts": baseline_push_result.attempts, + } + ) + elif baselines_updated: + output_values["baselines_updated"] = "true" + _write_github_output(**output_values) + + if baseline_update_failed: + return 1 + + if blocking: + if has_block: + return 1 + if has_hard_failure: + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_regression_gate/backend_identity.py b/tools/perf_regression_gate/backend_identity.py new file mode 100644 index 000000000000..7d4653415995 --- /dev/null +++ b/tools/perf_regression_gate/backend_identity.py @@ -0,0 +1,147 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Canonical backend identity helpers for the performance regression gate""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +_DEFAULT_PHYSICS_BACKEND = "physx" +_EMPTY_VALUES = {"", "none", "null"} +_DEFAULT_VALUES = _EMPTY_VALUES | {"default"} +_PHYSICS_PRESET_TO_BACKEND = { + "physx": "physx", + "newton": "newton", + "newton_mjwarp": "newton", +} +_RENDER_PRESET_TOKENS = frozenset( + { + "newton_renderer", + "ovrtx_renderer", + "warp_renderer", + "rtx_renderer", + } +) +_KNOWN_PHYSICS_BACKENDS = ("physx", "newton") + + +@dataclass(frozen=True) +class BackendIdentity: + physics_backend: str + render_backend: str | None = None + + @property + def backend_key(self) -> str: + return make_backend_key(self.physics_backend, self.render_backend) + + def to_dict(self) -> dict[str, str | None]: + return { + "physics_backend": self.physics_backend, + "render_backend": self.render_backend, + "backend_key": self.backend_key, + } + + +def _clean(value: Any) -> str | None: + if value is None: + return None + cleaned = str(value).strip() + return cleaned or None + + +def normalize_physics_backend(value: Any, *, default: str | None = None) -> str | None: + cleaned = _clean(value) + if cleaned is None: + return default + lowered = cleaned.lower() + if lowered in _DEFAULT_VALUES: + return default + return _PHYSICS_PRESET_TO_BACKEND.get(lowered, lowered) + + +def normalize_render_backend(value: Any) -> str | None: + cleaned = _clean(value) + if cleaned is None: + return None + lowered = cleaned.lower() + if lowered in _DEFAULT_VALUES: + return None + return lowered + + +def make_backend_key(physics_backend: str, render_backend: str | None = None) -> str: + physics = normalize_physics_backend(physics_backend) + if not physics: + raise ValueError("physics_backend is required to build backend_key") + render = normalize_render_backend(render_backend) + return f"{physics}_{render}" if render else physics + + +def identity_from_parts(physics_backend: Any, render_backend: Any = None) -> BackendIdentity | None: + physics = normalize_physics_backend(physics_backend) + if not physics: + return None + return BackendIdentity(physics, normalize_render_backend(render_backend)) + + +def split_backend_key(backend_key: Any) -> BackendIdentity | None: + key = _clean(backend_key) + if not key: + return None + for physics in _KNOWN_PHYSICS_BACKENDS: + if key == physics: + return BackendIdentity(physics, None) + prefix = f"{physics}_" + if key.startswith(prefix): + return BackendIdentity(physics, normalize_render_backend(key[len(prefix):])) + if "_" in key: + physics, render = key.split("_", 1) + return identity_from_parts(physics, render) + return BackendIdentity(key, None) + + +def preset_tokens(value: Any) -> frozenset[str]: + cleaned = _clean(value) + if not cleaned: + return frozenset() + tokens: set[str] = set() + for chunk in cleaned.replace(";", ",").split(","): + token = chunk.strip().lower() + if token: + tokens.add(token) + return frozenset(tokens) + + +def identity_from_presets(presets: Any) -> BackendIdentity | None: + tokens = preset_tokens(presets) + if not tokens: + return None + physics = "newton" if "newton_mjwarp" in tokens else _DEFAULT_PHYSICS_BACKEND + render = next((token for token in sorted(tokens) if token in _RENDER_PRESET_TOKENS), None) + return BackendIdentity(physics, render) + + +def backend_identity_from_launch_config(config: dict[str, Any]) -> BackendIdentity | None: + identity = identity_from_parts(config.get("physics_backend"), config.get("render_backend")) + if identity is not None: + return identity + return split_backend_key(config.get("backend_key") or config.get("backend")) + + +def backend_identity_from_benchmark_info(info: dict[str, Any]) -> BackendIdentity | None: + direct_key = info.get("backend_key") or info.get("backend") + if direct_key: + return split_backend_key(direct_key) + + identity = identity_from_parts( + info.get("physics_backend") or info.get("physics"), + info.get("render_backend") or info.get("render"), + ) + if identity is not None: + return identity + + return identity_from_presets(info.get("presets") or info.get("preset")) diff --git a/tools/perf_regression_gate/baseline_manager.py b/tools/perf_regression_gate/baseline_manager.py new file mode 100644 index 000000000000..600022a7b566 --- /dev/null +++ b/tools/perf_regression_gate/baseline_manager.py @@ -0,0 +1,565 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Baseline storage for the CI performance regression gate + +The baseline store keeps immutable structured samples in ``samples.ndjson``. +Threshold stats are calculated over the newest compatible samples instead of +truncating history on write. Git-backed updates are append-only transactions: +the manager refetches the remote branch, reapplies queued samples, and retries +the push to prevent races across CI runners +""" + +import contextlib +import hashlib +import json +import os +import shutil +import statistics +import subprocess +import tempfile +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +try: + from .gate_config import BASELINE_PUSH_RETRIES, DEFAULT_K_BLOCK, DEFAULT_K_WARN, MAX_BASELINE_SAMPLES + from .oracle import Baseline +except ImportError: # pragma: no cover - supports direct script imports + from gate_config import BASELINE_PUSH_RETRIES, DEFAULT_K_BLOCK, DEFAULT_K_WARN, MAX_BASELINE_SAMPLES + from oracle import Baseline + +SAMPLES_FILENAME = "samples.ndjson" +_REPO_DIR = Path(__file__).resolve().parent +_COMMIT_ENV_DEFAULTS = { + "GIT_AUTHOR_NAME": "perf-regression-gate", + "GIT_AUTHOR_EMAIL": "perf-regression-gate@localhost", + "GIT_COMMITTER_NAME": "perf-regression-gate", + "GIT_COMMITTER_EMAIL": "perf-regression-gate@localhost", +} + + +@dataclass(frozen=True) +class BaselineUpdateRecord: + gpu_model: str + task_id: str + backend: str + fps: float + fingerprint: str | None = None + sample_metadata: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class BaselinePushResult: + branch: str + remote: str | None + base_sha: str | None + pushed_sha: str | None + attempts: int + update_count: int + pushed: bool + + +def _bucket_dir(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> Path: + base = baselines_dir / gpu_model / task_id / backend + return base if fingerprint is None else base / fingerprint + + +def _samples_path(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> Path: + return _bucket_dir(baselines_dir, gpu_model, task_id, backend, fingerprint) / SAMPLES_FILENAME + + +def _baseline_from_values(values: list[float], *, source: str, total_sample_count: int | None = None) -> Baseline | None: + if not values: + return None + selected = values[-MAX_BASELINE_SAMPLES:] + median = statistics.median(selected) + deviations = [abs(v - median) for v in selected] + mad = statistics.median(deviations) if len(deviations) > 1 else 0.0 + return Baseline( + median_fps=median, + mad_fps=mad, + k_warn=DEFAULT_K_WARN, + k_block=DEFAULT_K_BLOCK, + sample_count=len(selected), + source=source, + total_sample_count=total_sample_count if total_sample_count is not None else len(values), + ) + + +def _load_sample_records_from_text(content: str) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for line in content.splitlines(): + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, dict) and isinstance(record.get("fps"), (int, float)): + records.append(record) + return records + + +def _commit_env() -> dict[str, str]: + env = os.environ.copy() + for key, value in _COMMIT_ENV_DEFAULTS.items(): + env.setdefault(key, value) + return env + + +def _git( + args: list[str], + *, + cwd: Path | None = None, + check: bool = False, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess: + result = subprocess.run( + ["git", *args], + cwd=str(cwd or _REPO_DIR), + capture_output=True, + text=True, + env=env, + ) + if check and result.returncode != 0: + raise subprocess.CalledProcessError(result.returncode, ["git", *args], result.stdout, result.stderr) + return result + + +def _git_error(result: subprocess.CompletedProcess) -> str: + return (result.stderr or result.stdout or "unknown git error").strip() + + +def _remote_ref_missing(result: subprocess.CompletedProcess) -> bool: + message = _git_error(result).lower() + return "couldn't find remote ref" in message or "could not find remote ref" in message + + +def _resolve_git_ref(ref: str, *, repo_dir: Path | None = None) -> str | None: + result = _git(["rev-parse", "--verify", ref], cwd=repo_dir) + if result.returncode != 0: + return None + return result.stdout.strip() + + +def refresh_baseline_branch( + branch: str, + *, + remote: str | None = "origin", + repo_dir: Path | None = None, + allow_missing: bool = True, +) -> str | None: + """Fetch and return the exact baseline branch SHA to read. + + Returning a SHA instead of the branch name makes aggregate comparisons traceable + and prevents a stale local branch from being used after the remote moved. + """ + if not remote: + return _resolve_git_ref(branch, repo_dir=repo_dir) + + remote_ref = f"refs/remotes/{remote}/{branch}" + refspec = f"+refs/heads/{branch}:{remote_ref}" + result = _git(["fetch", remote, refspec], cwd=repo_dir) + if result.returncode != 0: + if allow_missing and _remote_ref_missing(result): + return None + raise RuntimeError(f"Failed to fetch baseline branch {branch!r} from {remote!r}: {_git_error(result)}") + return _resolve_git_ref(remote_ref, repo_dir=repo_dir) + + +def _git_is_ancestor(commit_sha: str, base_sha: str, *, repo_dir: Path | None = None) -> bool: + result = _git(["merge-base", "--is-ancestor", commit_sha, base_sha], cwd=repo_dir) + return result.returncode == 0 + + +def _git_distance(commit_sha: str, base_sha: str, *, repo_dir: Path | None = None) -> int: + result = _git(["rev-list", "--count", f"{commit_sha}..{base_sha}"], cwd=repo_dir) + if result.returncode != 0: + return 10**9 + try: + return int(result.stdout.strip()) + except ValueError: + return 10**9 + + +def _sample_matches(record: dict[str, Any], context: dict[str, Any] | None) -> bool: + if context is None: + return True + exact_fields = ( + "gpu_model", + "task_id", + "backend_key", + "launch_config_hash", + "baseline_epoch", + "benchmark_contract_hash", + "runtime_contract_hash", + ) + for field in exact_fields: + expected = context.get(field) + if expected is not None and record.get(field) != expected: + return False + base_sha = context.get("base_sha") + commit_sha = record.get("commit_sha") + repo_dir = context.get("_repo_dir") + if base_sha: + if not commit_sha: + return False + if not _git_is_ancestor(str(commit_sha), str(base_sha), repo_dir=repo_dir): + return False + return True + + +def _select_records(records: list[dict[str, Any]], context: dict[str, Any] | None) -> list[dict[str, Any]]: + compatible = [r for r in records if _sample_matches(r, context)] + base_sha = context.get("base_sha") if context else None + if base_sha: + repo_dir = context.get("_repo_dir") if context else None + compatible.sort( + key=lambda r: (_git_distance(str(r.get("commit_sha", "")), str(base_sha), repo_dir=repo_dir), r.get("timestamp", "")) + ) + return compatible[:MAX_BASELINE_SAMPLES] + return compatible[-MAX_BASELINE_SAMPLES:] + + +def _load_baseline_from_contents( + samples_content: str | None, + *, + match_context: dict[str, Any] | None = None, +) -> Baseline | None: + if not samples_content: + return None + records = _load_sample_records_from_text(samples_content) + selected = _select_records(records, match_context) + return _baseline_from_values( + [float(r["fps"]) for r in selected], + source="samples", + total_sample_count=len(records), + ) + + +def load_baseline( + baselines_dir: Path, + gpu_model: str, + task_id: str, + backend: str, + fingerprint=None, + match_context: dict[str, Any] | None = None, +) -> Baseline | None: + """Load compatible baseline stats for a task/backend pair.""" + samples = _samples_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + return _load_baseline_from_contents( + samples.read_text() if samples.exists() else None, + match_context=match_context, + ) + + +def _stable_sample_id(metadata: dict[str, Any]) -> str: + keys = ( + "ci_run_id", + "ci_run_attempt", + "commit_sha", + "task_id", + "backend_key", + "launch_config_hash", + "benchmark_contract_hash", + "runtime_contract_hash", + "baseline_epoch", + "fps", + "attempt", + "was_retried", + ) + payload = {key: metadata.get(key) for key in keys if metadata.get(key) is not None} + if not payload.get("ci_run_id"): + payload["timestamp"] = metadata.get("timestamp") + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:24] + + +def _sample_id_exists(samples_path: Path, sample_id: str | None) -> bool: + if not sample_id or not samples_path.exists(): + return False + for record in _load_sample_records_from_text(samples_path.read_text()): + if record.get("sample_id") == sample_id: + return True + return False + + +def make_sample_metadata( + *, + gpu_model: str, + task_id: str, + backend: str, + fps: float, + bench_result: dict | None = None, + target_branch: str | None = None, + source_branch: str | None = None, + trusted_source: str = "protected_branch", +) -> dict[str, Any]: + bench_result = bench_result or {} + launch_config = bench_result.get("launch_config") or {} + provenance = bench_result.get("provenance") or {} + git_info = provenance.get("git") or {} + software = provenance.get("software") or {} + gpu_diag = bench_result.get("gpu_diag") or {} + metadata = { + "schema_version": 1, + "fps": float(fps), + "timestamp": datetime.now(timezone.utc).isoformat(), + "trusted_source": trusted_source, + "gpu_model": gpu_model, + "task_id": task_id, + "backend_key": backend, + "physics_backend": launch_config.get("physics_backend") or bench_result.get("physics_backend"), + "render_backend": launch_config.get("render_backend") or bench_result.get("render_backend"), + "commit_sha": git_info.get("commit_hash"), + "branch": git_info.get("branch") or source_branch, + "target_branch": target_branch, + "launch_config_hash": bench_result.get("launch_config_hash") or launch_config.get("launch_config_hash"), + "benchmark_contract_hash": bench_result.get("benchmark_contract_hash") + or launch_config.get("benchmark_contract_hash"), + "runtime_contract_hash": bench_result.get("runtime_contract_hash"), + "runtime_contract": bench_result.get("runtime_contract"), + "runtime_info": bench_result.get("runtime_info"), + "baseline_epoch": bench_result.get("baseline_epoch") or launch_config.get("baseline_epoch", 1), + "attempt": bench_result.get("attempt"), + "was_retried": bench_result.get("was_retried"), + "ci_run_id": os.environ.get("GITHUB_RUN_ID"), + "ci_run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"), + "ci_workflow": os.environ.get("GITHUB_WORKFLOW"), + "ci_job": os.environ.get("GITHUB_JOB"), + "launch_config": launch_config, + "runtime": { + "isaacsim": software.get("isaacsim"), + "warp": software.get("warp"), + "cuda": (gpu_diag or {}).get("cuda_version"), + "driver": (gpu_diag or {}).get("nvidia_driver_version"), + }, + } + metadata["sample_id"] = _stable_sample_id(metadata) + return metadata + + +def match_context_from_bench_result( + bench_result: dict, + *, + gpu_model: str, + base_sha: str | None = None, + target_branch: str | None = None, +) -> dict[str, Any]: + launch_config = bench_result.get("launch_config") or {} + return { + "gpu_model": gpu_model, + "task_id": bench_result.get("task_id"), + "backend_key": bench_result.get("backend_key") or bench_result.get("backend"), + "launch_config_hash": bench_result.get("launch_config_hash") or launch_config.get("launch_config_hash"), + "benchmark_contract_hash": bench_result.get("benchmark_contract_hash") + or launch_config.get("benchmark_contract_hash"), + "runtime_contract_hash": bench_result.get("runtime_contract_hash"), + "baseline_epoch": bench_result.get("baseline_epoch") or launch_config.get("baseline_epoch", 1), + "base_sha": base_sha, + "target_branch": target_branch, + } + + +def update_baseline( + baselines_dir: Path, + gpu_model: str, + task_id: str, + backend: str, + fps: float, + fingerprint=None, + sample_metadata: dict[str, Any] | None = None, +) -> bool: + """Append a structured baseline sample. Returns False when already present.""" + bucket = _bucket_dir(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + bucket.mkdir(parents=True, exist_ok=True) + samples = _samples_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + + metadata = dict(sample_metadata or {}) + metadata.setdefault("schema_version", 1) + metadata.setdefault("fps", float(fps)) + metadata.setdefault("timestamp", datetime.now(timezone.utc).isoformat()) + metadata.setdefault("gpu_model", gpu_model) + metadata.setdefault("task_id", task_id) + metadata.setdefault("backend_key", backend) + metadata.setdefault("sample_id", _stable_sample_id(metadata)) + + if _sample_id_exists(samples, metadata.get("sample_id")): + return False + + with samples.open("a") as fh: + fh.write(json.dumps(metadata, sort_keys=True) + "\n") + return True + + +def delete_baseline_files(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> None: + """Delete structured samples for a task/backend pair.""" + samples = _samples_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + if samples.exists(): + samples.unlink() + + +def seed_baseline_with_spread( + baselines_dir: Path, + gpu_model: str, + task_id: str, + backend: str, + center_fps: float, + noise_fps: float = 5.0, + n_samples: int = 10, + seed: int = 0, + fingerprint=None, +) -> None: + """Populate a deterministic structured baseline window for tests and demos.""" + import random as _random + + rng = _random.Random(seed) + delete_baseline_files(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + for _ in range(n_samples): + fps = max(1.0, rng.gauss(center_fps, noise_fps)) + update_baseline(baselines_dir, gpu_model, task_id, backend, fps, fingerprint=fingerprint) + + +def _git_show_file(ref: str, rel_path: str, *, repo_dir: Path | None = None) -> str | None: + result = _git(["show", f"{ref}:{rel_path}"], cwd=repo_dir) + return result.stdout if result.returncode == 0 else None + + +def load_baseline_git( + ref: str, + gpu_model: str, + task_id: str, + backend: str, + fingerprint: str | None, + match_context: dict[str, Any] | None = None, + *, + repo_dir: Path | None = None, +) -> Baseline | None: + """Load compatible baseline stats from an exact git ref or SHA.""" + samples = str(_samples_path(Path(""), gpu_model, task_id, backend, fingerprint)) + context = dict(match_context or {}) + if repo_dir is not None: + context["_repo_dir"] = repo_dir + return _load_baseline_from_contents( + _git_show_file(ref, samples, repo_dir=repo_dir), + match_context=context or None, + ) + + +@contextlib.contextmanager +def _baseline_update_worktree(base_ref: str | None, *, repo_dir: Path | None = None): + repo_dir = repo_dir or _REPO_DIR + tmpdir = tempfile.mkdtemp(prefix="perf-bl-wt-") + orphan_branch = None + try: + if base_ref: + _git(["worktree", "add", "--detach", tmpdir, base_ref], cwd=repo_dir, check=True) + else: + orphan_branch = f"perf-baseline-seed-{os.getpid()}-{Path(tmpdir).name}" + _git(["worktree", "add", "--detach", tmpdir, "HEAD"], cwd=repo_dir, check=True) + _git(["checkout", "--orphan", orphan_branch], cwd=Path(tmpdir), check=True) + rm_result = _git(["rm", "-rf", "."], cwd=Path(tmpdir)) + if rm_result.returncode not in (0, 128): + raise RuntimeError(f"Failed to clear orphan baseline worktree: {_git_error(rm_result)}") + yield Path(tmpdir) + finally: + _git(["worktree", "remove", "--force", tmpdir], cwd=repo_dir) + shutil.rmtree(tmpdir, ignore_errors=True) + if orphan_branch: + _git(["branch", "-D", orphan_branch], cwd=repo_dir) + + +def _commit_baseline_worktree(worktree: Path) -> str | None: + status = _git(["status", "--porcelain"], cwd=worktree, check=True) + if not status.stdout.strip(): + return None + _git(["add", "-A"], cwd=worktree, check=True) + _git( + ["commit", "-m", "[baseline_manager] Append baseline samples"], + cwd=worktree, + check=True, + env=_commit_env(), + ) + commit = _git(["rev-parse", "HEAD"], cwd=worktree, check=True) + return commit.stdout.strip() + + +def _apply_updates(root: Path, updates: list[BaselineUpdateRecord]) -> int: + appended = 0 + for update in updates: + if update_baseline( + root, + update.gpu_model, + update.task_id, + update.backend, + update.fps, + fingerprint=update.fingerprint, + sample_metadata=update.sample_metadata, + ): + appended += 1 + return appended + + +def update_baselines_git( + branch: str, + updates: list[BaselineUpdateRecord], + *, + remote: str | None = "origin", + max_retries: int = BASELINE_PUSH_RETRIES, + repo_dir: Path | None = None, +) -> BaselinePushResult: + """Append samples to a git-backed baseline branch and push safely. + + Each attempt starts from the latest remote branch SHA. If the push loses a + race, the next attempt refetches and reapplies the same sample IDs, making + retries idempotent. + """ + if not updates: + return BaselinePushResult(branch, remote, None, None, 0, 0, False) + if max_retries < 1: + raise ValueError("max_retries must be >= 1") + + last_error = "unknown push failure" + for attempt in range(1, max_retries + 1): + base_sha = refresh_baseline_branch(branch, remote=remote, repo_dir=repo_dir, allow_missing=True) + with _baseline_update_worktree(base_sha, repo_dir=repo_dir) as worktree: + appended = _apply_updates(worktree, updates) + commit_sha = _commit_baseline_worktree(worktree) + if commit_sha is None: + return BaselinePushResult(branch, remote, base_sha, base_sha, attempt, appended, False) + push_ref = f"HEAD:refs/heads/{branch}" + if remote: + push = _git(["push", remote, push_ref], cwd=worktree) + else: + push = _git(["branch", "--force", branch, "HEAD"], cwd=worktree) + if push.returncode == 0: + return BaselinePushResult(branch, remote, base_sha, commit_sha, attempt, appended, bool(remote)) + last_error = _git_error(push) + print( + f"[baseline_manager] baseline push attempt {attempt}/{max_retries} failed; " + "refetching and retrying" + ) + + raise RuntimeError(f"Failed to push baseline branch {branch!r} after {max_retries} attempts: {last_error}") + + +def update_baseline_git( + branch: str, + gpu_model: str, + task_id: str, + backend: str, + fps: float, + fingerprint: str | None, + sample_metadata: dict[str, Any] | None = None, + *, + remote: str | None = "origin", + max_retries: int = BASELINE_PUSH_RETRIES, + repo_dir: Path | None = None, +) -> BaselinePushResult: + update = BaselineUpdateRecord(gpu_model, task_id, backend, fps, fingerprint, sample_metadata) + return update_baselines_git(branch, [update], remote=remote, max_retries=max_retries, repo_dir=repo_dir) diff --git a/tools/perf_regression_gate/build_bench_result.py b/tools/perf_regression_gate/build_bench_result.py new file mode 100644 index 000000000000..99d38ab9a057 --- /dev/null +++ b/tools/perf_regression_gate/build_bench_result.py @@ -0,0 +1,550 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Post-benchmark script: normalizes benchmark output and writes perf_regression_gate_result.json. + +Locates the timestamped benchmark output file written by benchmark_non_rl.py, renames it +to the canonical ``perf_regression_gate_info.json``, classifies the failure phase from the +captured log, and writes ``perf_regression_gate_result.json`` for the aggregate job. + +Usage:: + + python3 tools/perf_regression_gate/build_bench_result.py \\ + --task_id Isaac-Cartpole-Direct-v0 \\ + --artifact_dir artifacts/Isaac-Cartpole-Direct-v0 \\ + --exit_code 0 \\ + --wall_time_s 48.3 \\ + --timeout_s 600 \\ + --log_file artifacts/Isaac-Cartpole-Direct-v0/benchmark.log +""" + +import argparse +import glob +import json +import shutil +import statistics +import subprocess +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +_TOOLS_DIR = _MODULE_DIR.parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + +from subprocess_runner import classify_failure_phase # noqa: E402 +from backend_identity import ( # noqa: E402 + backend_identity_from_benchmark_info, + backend_identity_from_launch_config, + identity_from_parts, + make_backend_key, + normalize_physics_backend, + normalize_render_backend, +) +from gate_config import load_gate_config # noqa: E402 +from gate_types import FailurePhase # noqa: E402 +from gpu_identity import normalize_gpu_fields # noqa: E402 +from launch_config import fallback_launch_config, load_launch_config # noqa: E402 +from runtime_contract import build_runtime_contract, build_runtime_publish_info # noqa: E402 +from task_config import get_task # noqa: E402 + + +def _percentile(sorted_data: list[float], p: float) -> float: + """Linear-interpolation percentile on a pre-sorted list""" + n = len(sorted_data) + if n == 1: + return sorted_data[0] + idx = p / 100.0 * (n - 1) + lo = int(idx) + hi = min(lo + 1, n - 1) + return sorted_data[lo] + (sorted_data[hi] - sorted_data[lo]) * (idx - lo) + + +def _strip_phase_prefix(name: str, phase_name: str) -> str: + """Strip the '{task_name} {phase_name} ' prefix added by JSONFileMetrics.finalize()""" + marker = f" {phase_name} " + idx = name.find(marker) + return name[idx + len(marker) :] if idx >= 0 else name + + +def _duration_to_seconds(value: float, unit: str | None) -> float: + normalized = str(unit or "s").strip().lower() + if normalized in {"ms", "millisecond", "milliseconds"}: + return value / 1000.0 + if normalized in {"us", "microsecond", "microseconds"}: + return value / 1_000_000.0 + return value + + +def _gpu_driver_version() -> str | None: + """Return the GPU driver version string from nvidia-smi, or None if unavailable""" + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader,nounits"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + version = result.stdout.strip().splitlines()[0].strip() + return version if version else None + except Exception: + pass + return None + + +def _extract_info_provenance(info_path: Path) -> dict: + """Parse ``perf_regression_gate_info.json`` and return provenance + FPS fields + + Extracts hardware metadata (GPU name/memory/CUDA, CPU, RAM), software versions, + git provenance, FPS distribution statistics, GPU memory used at runtime, and + startup time. All fields are best-effort; missing data is omitted. + + Returns: + Dict with a subset of the following keys: + + - ``raw_fps_{mean,std,min,max,median,p5,p95}``: float + - ``startup_time_s``: float + - ``gpu_diag``: dict with gpu_name, gpu_total_memory_gb, cuda_version, + nvidia_driver_version, gpu_mem_used_mb + - ``provenance``: dict with ``hardware``, ``software``, ``git`` sub-dicts + """ + try: + phases: list[dict] = json.loads(info_path.read_text()) + except Exception: + return {} + + hardware: dict = {} + software: dict = {} + git: dict = {} + fps_stats: dict = {} + gpu_mem_used_gb: float | None = None + startup_time_s: float | None = None + + for phase in phases: + pname: str = phase.get("phase_name", "") + metadata_map = { + _strip_phase_prefix(m["name"], pname): m["data"] + for m in phase.get("metadata", []) + if "name" in m and "data" in m + } + + if pname == "hardware_info": + hardware["cpu_name"] = metadata_map.get("cpu_name") + hardware["cpu_physical_cores"] = metadata_map.get("physical_cores") + hardware["total_ram_gb"] = metadata_map.get("total_ram_gb") + hardware["gpu_device_count"] = metadata_map.get("gpu_device_count") + hardware["cuda_version"] = metadata_map.get("cuda_version") + gpu_devices = metadata_map.get("gpu_devices") + if isinstance(gpu_devices, dict): + current = str(metadata_map.get("gpu_current_device", 0)) + dev = gpu_devices.get(current) or next(iter(gpu_devices.values()), {}) + hardware["gpu_name"] = dev.get("name") + hardware["gpu_total_memory_gb"] = dev.get("total_memory_gb") + hardware["gpu_compute_capability"] = dev.get("compute_capability") + hardware["gpu_multi_processor_count"] = dev.get("multi_processor_count") + + elif pname == "version_info": + dev_data: dict = metadata_map.pop("dev", {}) or {} + for k, v in metadata_map.items(): + # strip trailing "_version" suffix added by VersionInfoRecorder + key = k[: -len("_version")] if k.endswith("_version") else k + if v is not None: + software[key] = v + git = { + k: dev_data[k] + for k in ("commit_hash", "commit_hash_short", "branch", "commit_date", "dirty") + if k in dev_data + } + + elif pname == "runtime": + for m in phase.get("measurements", []): + name: str = m.get("name", "") + value = m.get("value") + # FPS series: DictMeasurement written by BenchmarkMonitor + if name.endswith("Step Frametimes") and isinstance(value, dict): + fps_series: list[float] = value.get("Environment step effective FPS", []) + if fps_series: + sorted_fps = sorted(fps_series) + n = len(sorted_fps) + fps_stats = { + "raw_fps_mean": statistics.mean(fps_series), + "raw_fps_std": statistics.stdev(fps_series) if n > 1 else 0.0, + "raw_fps_min": sorted_fps[0], + "raw_fps_max": sorted_fps[-1], + "raw_fps_median": _percentile(sorted_fps, 50.0), + "raw_fps_p5": _percentile(sorted_fps, 5.0), + "raw_fps_p95": _percentile(sorted_fps, 95.0), + } + # GPU memory used (mean over run, in GB): SingleMeasurement from GPUInfoRecorder + elif name.endswith("GPU Memory Used") and isinstance(value, (int, float)): + gpu_mem_used_gb = float(value) + + elif pname == "startup": + for m in phase.get("measurements", []): + if m.get("name", "").endswith("Total Start Time (Launch to Train)"): + val = m.get("value") + if isinstance(val, (int, float)): + startup_time_s = _duration_to_seconds(float(val), m.get("unit")) + break + + gpu_diag: dict = {k: v for k, v in { + "gpu_name": hardware.get("gpu_name"), + "gpu_total_memory_gb": hardware.get("gpu_total_memory_gb"), + "cuda_version": hardware.get("cuda_version"), + "nvidia_driver_version": _gpu_driver_version(), + "gpu_mem_used_mb": round(gpu_mem_used_gb * 1024, 2) if gpu_mem_used_gb is not None else None, + }.items() if v is not None} + + result: dict = {} + result.update(fps_stats) + if startup_time_s is not None: + result["startup_time_s"] = startup_time_s + if gpu_diag: + result["gpu_diag"] = gpu_diag + result["provenance"] = { + "hardware": {k: v for k, v in hardware.items() if v is not None}, + "software": software, + "git": git, + } + return result + + + +# --------------------------------------------------------------------------- +# Launch/run provenance guard + step-time debug KPIs +# --------------------------------------------------------------------------- + +_OUTLIER_FACTOR = 2.0 +_WARMUP_GUARD_FACTOR = 3.0 +_MAX_REPORTED_OUTLIERS = 8 + + +def _extract_benchmark_info(info_path: Path) -> dict: + """Return the run's self-reported benchmark_info metadata, if present.""" + try: + phases: list[dict] = json.loads(info_path.read_text()) + except Exception: + return {} + for phase in phases: + if phase.get("phase_name") != "benchmark_info": + continue + out = {} + for item in phase.get("metadata", []): + if "name" in item and "data" in item: + out[_strip_phase_prefix(item["name"], "benchmark_info")] = item["data"] + return out + return {} + + +def _coerce_int(value: object) -> int | None: + try: + return int(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return None + + +def _config_drift(benchmark_info: dict, launch_config: dict) -> str | None: + """Return a compact mismatch string when the actual run differs from launch intent.""" + if not benchmark_info: + return None + mismatches: list[str] = [] + + wanted_task = launch_config.get("task_id") + ran_task = benchmark_info.get("task") + if isinstance(ran_task, str) and ran_task and wanted_task and ran_task != wanted_task: + mismatches.append(f"task(ran={ran_task},want={wanted_task})") + + for field in ("num_envs", "seed"): + wanted = _coerce_int(launch_config.get(field)) + ran = _coerce_int(benchmark_info.get(field)) + if wanted is not None and ran is not None and ran != wanted: + mismatches.append(f"{field}(ran={ran},want={wanted})") + + wanted_frames = _coerce_int(launch_config.get("num_frames")) + ran_frames = _coerce_int(benchmark_info.get("num_frames")) + if wanted_frames is not None and ran_frames is not None and ran_frames < wanted_frames: + mismatches.append(f"num_frames(ran={ran_frames},want>={wanted_frames})") + + wanted_backend = backend_identity_from_launch_config(launch_config) + ran_backend = backend_identity_from_benchmark_info(benchmark_info) + if wanted_backend is not None and ran_backend is not None and wanted_backend.backend_key != ran_backend.backend_key: + mismatches.append(f"backend(ran={ran_backend.backend_key},want={wanted_backend.backend_key})") + + return " ".join(mismatches) if mismatches else None + + +def _expand_excluded_frames(raw: list) -> frozenset[int]: + indices: set[int] = set() + for entry in raw or []: + if isinstance(entry, list): + indices.update(range(int(entry[0]), int(entry[1]) + 1)) + else: + indices.add(int(entry)) + return frozenset(indices) + + +def _extract_debug_kpis(info_path: Path, excluded_frames: frozenset[int]) -> dict: + """Return post-warm-up step-time diagnostics for aggregate summaries.""" + try: + phases: list[dict] = json.loads(info_path.read_text()) + except Exception: + return {} + steps: list[float] = [] + for phase in phases: + if phase.get("phase_name") != "runtime": + continue + for measurement in phase.get("measurements", []): + value = measurement.get("value") + if measurement.get("name", "").endswith("Step Frametimes") and isinstance(value, dict): + raw = value.get("Environment step times", []) + steps = [float(v) for v in raw if isinstance(v, (int, float)) and not isinstance(v, bool)] + break + if not steps: + return {} + steady = [value for idx, value in enumerate(steps) if idx not in excluded_frames] + if len(steady) < 2: + return {} + ordered = sorted(steady) + n = len(ordered) + median = ordered[n // 2] if n % 2 else (ordered[n // 2 - 1] + ordered[n // 2]) / 2.0 + p99 = ordered[min(n - 1, int(round(0.99 * (n - 1))))] + out: dict = {"steady_frames": n} + if median > 0: + out["p99_over_median"] = round(p99 / median, 3) + outliers = [(idx, value) for idx, value in enumerate(steady) if value > _OUTLIER_FACTOR * median] + out["outlier_count"] = len(outliers) + if outliers: + out["outlier_idx"] = ",".join(str(idx) for idx, _ in outliers[:_MAX_REPORTED_OUTLIERS]) + out["outlier_mag_x"] = ",".join(f"{value / median:.2g}" for _, value in outliers[:_MAX_REPORTED_OUTLIERS]) + if steady[0] > _WARMUP_GUARD_FACTOR * median: + out["warmup_flag"] = f"first_kept_frame={steady[0] / median:.1f}x_median" + return out + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Build perf_regression_gate_result.json from a benchmark run") + p.add_argument("--task_id", required=True) + p.add_argument("--physics_backend", required=True, help="Physics backend used (e.g. physx, newton)") + p.add_argument("--render_backend", default="", help="Render backend used (e.g. rtx, warp, ovrtx); empty = none") + p.add_argument("--artifact_dir", required=True, type=Path) + p.add_argument("--exit_code", required=True, type=int) + p.add_argument("--wall_time_s", required=True, type=float) + p.add_argument("--timeout_s", required=True, type=float) + p.add_argument("--log_file", type=Path, default=None) + p.add_argument("--launch_config", type=Path, default=None, help="Path to launch_config.json (default: artifact_dir/launch_config.json)") + p.add_argument("--gate_config", type=Path, default=_MODULE_DIR / "gate_config.json") + p.add_argument("--attempt", type=int, default=1, help="Attempt number (1 = first run, 2 = after one retry)") + p.add_argument("--was_retried", action="store_true", help="Set when this result comes from a retry of a failed first attempt") + return p.parse_args() + + +def _normalize_benchmark_output(artifact_dir: Path, task_id: str) -> bool: + """Rename the timestamped benchmark JSON to ``perf_regression_gate_info.json`` + + benchmark_non_rl.py writes ``benchmark_non_rl_{task_id}_{timestamp}.json``. + The oracle reads ``perf_regression_gate_info.json``. This function bridges the gap. + + Returns True if perf_regression_gate_info.json exists after call + """ + perf_regression_gate_info = artifact_dir / "perf_regression_gate_info.json" + if perf_regression_gate_info.exists(): + return True + # Primary pattern: exact task_id match + matches = sorted(glob.glob(str(artifact_dir / f"benchmark_non_rl_{task_id}_*.json"))) + if not matches: + # Fallback: any benchmark_non_rl_*.json in the artifact dir + matches = sorted(glob.glob(str(artifact_dir / "benchmark_non_rl_*.json"))) + if not matches: + return False + shutil.copy(matches[-1], perf_regression_gate_info) + return True + + +def main() -> int: + args = _parse_args() + artifact_dir = args.artifact_dir + artifact_dir.mkdir(parents=True, exist_ok=True) + gate_config = load_gate_config(args.gate_config) + runtime_policy = gate_config.get("runtime_compatibility", {}) + + cli_physics_backend = normalize_physics_backend(args.physics_backend) + if cli_physics_backend is None: + raise ValueError("--physics_backend must name a concrete backend") + cli_render_backend = normalize_render_backend(args.render_backend) + cli_backend_key = make_backend_key(cli_physics_backend, cli_render_backend) + + launch_config = load_launch_config(artifact_dir, args.launch_config) + if launch_config is None: + try: + task = get_task(args.task_id, cli_backend_key) + except KeyError: + print( + f"[build_bench_result] Warning: ({args.task_id!r}, {cli_backend_key!r}) not found in tasks.json; using defaults" + ) + task = None + launch_config = fallback_launch_config( + task_id=args.task_id, + physics_backend=cli_physics_backend, + render_backend=cli_render_backend, + backend_key=cli_backend_key, + timeout_s=args.timeout_s, + task=task, + ) + launch_config = dict(launch_config) + + expected_backend = backend_identity_from_launch_config(launch_config) or identity_from_parts( + cli_physics_backend, cli_render_backend + ) + if expected_backend is None: + raise ValueError("launch_config must define a concrete backend identity") + task_id = str(launch_config.get("task_id") or args.task_id) + physics_backend = expected_backend.physics_backend + render_backend = expected_backend.render_backend + backend_key = expected_backend.backend_key + + phase2_mismatches: list[str] = [] + if args.task_id != task_id: + phase2_mismatches.append(f"phase2_task_arg(arg={args.task_id},want={task_id})") + if cli_backend_key != backend_key: + phase2_mismatches.append(f"phase2_backend_arg(arg={cli_backend_key},want={backend_key})") + phase2_arg_mismatch = " ".join(phase2_mismatches) if phase2_mismatches else None + + gpu_fields = normalize_gpu_fields(launch_config.get("gpu_model_raw") or launch_config.get("gpu_model")) + launch_config["task_id"] = task_id + launch_config["backend_key"] = backend_key + launch_config["backend"] = backend_key + launch_config["physics_backend"] = physics_backend + launch_config["render_backend"] = render_backend + launch_config["gpu_model"] = gpu_fields["gpu_model"] + launch_config["gpu_model_raw"] = gpu_fields["gpu_model_raw"] + + num_envs = launch_config.get("num_envs", 0) + num_frames = launch_config.get("num_frames", 0) + excluded_frames_raw = launch_config.get("excluded_frames_raw", []) + excluded_frames = _expand_excluded_frames(excluded_frames_raw) + timeout_minutes = launch_config.get("timeout_minutes", int(args.timeout_s / 60)) + preset = launch_config.get("preset", "default") + tags = launch_config.get("tags", ["always"]) + seed = launch_config.get("seed") + + # Read combined stdout/stderr log for failure classification + log_text = "" + if args.log_file and args.log_file.exists(): + log_text = args.log_file.read_text(errors="replace") + + perf_regression_gate_info_present = _normalize_benchmark_output(artifact_dir, task_id) + + failure_phase = classify_failure_phase( + stdout=log_text, + stderr="", + exit_code=args.exit_code, + wall_time_s=args.wall_time_s, + timeout_s=args.timeout_s, + ) + + # Extract FPS stats, startup time, GPU diag, run config, and full provenance. + info_provenance: dict = {} + benchmark_info: dict = {} + debug_kpis: dict = {} + config_mismatch: str | None = None + observed_backend = None + runtime_contract = None + runtime_contract_hash = None + runtime_info = None + if perf_regression_gate_info_present: + info_path = artifact_dir / "perf_regression_gate_info.json" + info_provenance = _extract_info_provenance(info_path) + benchmark_info = _extract_benchmark_info(info_path) + observed_backend = backend_identity_from_benchmark_info(benchmark_info) + debug_kpis = _extract_debug_kpis(info_path, excluded_frames) + runtime_contract, runtime_contract_hash = build_runtime_contract( + provenance=info_provenance.get("provenance"), + gpu_diag=info_provenance.get("gpu_diag"), + backend=expected_backend, + policy=runtime_policy, + ) + runtime_info = build_runtime_publish_info( + provenance=info_provenance.get("provenance"), + gpu_diag=info_provenance.get("gpu_diag"), + policy=runtime_policy, + ) + config_mismatch = _config_drift(benchmark_info, launch_config) + config_mismatch = " ".join(part for part in (phase2_arg_mismatch, config_mismatch) if part) or None + if config_mismatch and failure_phase is None: + failure_phase = FailurePhase.CONFIG_MISMATCH.value + + bench_result = { + "task_id": task_id, + "backend": backend_key, + "physics_backend": physics_backend, + "render_backend": render_backend, + "backend_key": backend_key, + "preset": preset, + "attempt": args.attempt, + "was_retried": args.was_retried, + "exit_code": args.exit_code, + "failure_phase": failure_phase, + "stdout_tail": log_text[-2000:] if len(log_text) > 2000 else log_text, + "wall_time_s": args.wall_time_s, + "startup_time_s": info_provenance.get("startup_time_s"), + "perf_regression_gate_info_present": perf_regression_gate_info_present, + "raw_fps_mean": info_provenance.get("raw_fps_mean"), + "raw_fps_std": info_provenance.get("raw_fps_std"), + "raw_fps_min": info_provenance.get("raw_fps_min"), + "raw_fps_max": info_provenance.get("raw_fps_max"), + "raw_fps_median": info_provenance.get("raw_fps_median"), + "raw_fps_p5": info_provenance.get("raw_fps_p5"), + "raw_fps_p95": info_provenance.get("raw_fps_p95"), + "p99_over_median": debug_kpis.get("p99_over_median"), + "outlier_count": debug_kpis.get("outlier_count"), + "debug_kpis": debug_kpis, + "benchmark_info": benchmark_info, + "observed_backend": observed_backend.to_dict() if observed_backend else None, + "config_mismatch": config_mismatch, + "runtime_contract": runtime_contract, + "runtime_contract_hash": runtime_contract_hash, + "runtime_info": runtime_info, + "gpu_diag": info_provenance.get("gpu_diag"), + "provenance": info_provenance.get("provenance"), + "launch_config": launch_config, + "launch_config_hash": launch_config.get("launch_config_hash"), + "benchmark_contract_hash": launch_config.get("benchmark_contract_hash"), + "baseline_epoch": launch_config.get("baseline_epoch", 1), + "task_config_snapshot": { + "task_id": task_id, + "backend": backend_key, + "physics_backend": physics_backend, + "render_backend": render_backend, + "backend_key": backend_key, + "preset": preset, + "num_envs": num_envs, + "num_frames": num_frames, + "excluded_frames_raw": excluded_frames_raw, + "timeout_minutes": timeout_minutes, + "tags": tags, + "seed": seed, + "launch_config_hash": launch_config.get("launch_config_hash"), + "benchmark_contract_hash": launch_config.get("benchmark_contract_hash"), + "runtime_contract_hash": runtime_contract_hash, + "baseline_epoch": launch_config.get("baseline_epoch", 1), + }, + } + + out = artifact_dir / "perf_regression_gate_result.json" + out.write_text(json.dumps(bench_result, indent=2)) + + status = ( + f"failure_phase={failure_phase!r}, perf_regression_gate_info_present={perf_regression_gate_info_present}, " + f"exit_code={args.exit_code}, config_mismatch={config_mismatch!r}" + ) + print(f"[build_bench_result] {task_id}: {status}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_regression_gate/dev/sim_regression.py b/tools/perf_regression_gate/dev/sim_regression.py new file mode 100644 index 000000000000..2e8ec959bdf9 --- /dev/null +++ b/tools/perf_regression_gate/dev/sim_regression.py @@ -0,0 +1,149 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Regression simulation helper that injects degraded FPS artifacts into a scratch artifacts directory. + +Used to demonstrate that the gate produces BLOCK verdicts when a real regression +is introduced, WITHOUT re-running the full benchmark suite. + + python3 tools/perf_regression_gate/dev/sim_regression.py --fps_scale 0.53 + +Then run aggregate.py against the output to see BLOCK verdicts: + + python3 tools/perf_regression_gate/aggregate.py \\ + --artifacts_dir /tmp/sim_artifacts \\ + --gpu_model L40S \\ + --baselines_dir tools/perf_regression_gate/local_baselines \\ + --allow_baseline_update false +""" + +import argparse +import json +import random +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +_GATE_DIR = _MODULE_DIR.parent # tools/perf_regression_gate/ + +sys.path.insert(0, str(_GATE_DIR)) +sys.path.insert(0, str(_GATE_DIR.parent)) + +from baseline_manager import load_baseline # noqa: E402 +from task_config import load_tasks # noqa: E402 + + +def _make_perf_info(task_id: str, fps_mean: float, num_frames: int) -> list: + rng = random.Random(42) + fps_series = [max(1.0, fps_mean + rng.gauss(0, fps_mean * 0.01)) for _ in range(num_frames)] + return [ + { + "phase_name": "runtime", + "measurements": [ + { + "name": f"{task_id} Step Frametimes", + "value": {"Environment step effective FPS": fps_series}, + } + ], + "metadata": [], + } + ] + + +def _make_bench_result(task, fps_mean: float) -> dict: + return { + "task_id": task.task_id, + "backend": task.backend_key, + "backend_key": task.backend_key, + "physics_backend": task.physics_backend, + "render_backend": task.render_backend, + "preset": task.preset, + "attempt": 1, + "was_retried": False, + "exit_code": 0, + "failure_phase": None, + "stdout_tail": "", + "wall_time_s": 60.0, + "startup_time_s": 10.0, + "perf_regression_gate_info_present": True, + "raw_fps_mean": fps_mean, + "raw_fps_std": fps_mean * 0.01, + "raw_fps_min": fps_mean * 0.95, + "raw_fps_max": fps_mean * 1.05, + "raw_fps_median": fps_mean, + "raw_fps_p5": fps_mean * 0.96, + "raw_fps_p95": fps_mean * 1.04, + "outlier_count": 0, + "gpu_diag": None, + "task_config_snapshot": { + "task_id": task.task_id, + "backend": task.backend_key, + "backend_key": task.backend_key, + "physics_backend": task.physics_backend, + "render_backend": task.render_backend, + "preset": task.preset, + "num_envs": task.num_envs, + "num_frames": task.num_frames, + "excluded_frames_raw": task.excluded_frames_raw, + "timeout_minutes": task.timeout_minutes, + "camera_resolution": task.camera_resolution, + "tags": task.tags, + }, + } + + +def main() -> int: + p = argparse.ArgumentParser(description="Inject simulated regression artifacts.") + p.add_argument("--fps_scale", type=float, default=0.53, + help="Multiply baseline FPS by this factor (0.53 = 47%% regression, default)") + p.add_argument("--tags", nargs="+", default=["always"], + help="Task tags to include (default: always)") + p.add_argument("--gpu_model", default="L40S") + p.add_argument("--baselines_dir", type=Path, + default=_GATE_DIR / "local_baselines") + p.add_argument("--out_dir", type=Path, default=Path("/tmp/sim_artifacts")) + args = p.parse_args() + + all_tasks = load_tasks() + tag_set = frozenset(args.tags) + tasks = [t for t in all_tasks if tag_set.intersection(frozenset(t.tags))] + + print(f"\n[sim_regression] fps_scale={args.fps_scale} ({(1-args.fps_scale)*100:.0f}% regression)") + print(f"[sim_regression] writing artifacts to {args.out_dir}\n") + + generated = 0 + for task in tasks: + baseline = load_baseline(args.baselines_dir, args.gpu_model, task.task_id, task.backend_key) + if baseline is None: + print(f" SKIP (no baseline): {task.task_id}/{task.backend_key}") + continue + + baseline_fps = baseline.median_fps + regressed_fps = baseline_fps * args.fps_scale + + art_dir = args.out_dir / task.task_id / task.backend_key + art_dir.mkdir(parents=True, exist_ok=True) + + perf_info = _make_perf_info(task.task_id, regressed_fps, task.num_frames) + (art_dir / "perf_regression_gate_info.json").write_text(json.dumps(perf_info)) + + bench_result = _make_bench_result(task, regressed_fps) + (art_dir / "perf_regression_gate_result.json").write_text(json.dumps(bench_result)) + + print(f" {task.task_id}/{task.backend_key}: baseline={baseline_fps:.1f} regressed={regressed_fps:.1f}") + generated += 1 + + print(f"\n[sim_regression] wrote {generated} artifact sets") + print(f"\nNow run aggregate.py:") + print(f" python3 {_GATE_DIR}/aggregate.py \\") + print(f" --artifacts_dir {args.out_dir} \\") + print(f" --gpu_model {args.gpu_model} \\") + print(f" --baselines_dir {args.baselines_dir} \\") + print(f" --allow_baseline_update false") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_regression_gate/dev/stub_benchmark.py b/tools/perf_regression_gate/dev/stub_benchmark.py new file mode 100644 index 000000000000..8eddf6520c2d --- /dev/null +++ b/tools/perf_regression_gate/dev/stub_benchmark.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +import argparse +import json +import math +import random +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).resolve().parent.parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) + +from backend_identity import split_backend_key # noqa: E402 +from launch_config import hydra_args_for_task # noqa: E402 +from task_config import TaskConfig # noqa: E402 + + +def _presets_for_backend(task_id: str, identity) -> str: + task = TaskConfig( + task_id=task_id, + physics_backend=identity.physics_backend, + render_backend=identity.render_backend, + preset="default", + num_envs=1, + num_frames=1, + excluded_frames_raw=[], + camera_resolution=None, + timeout_minutes=1, + fps_mean_floor={}, + caches=[], + ) + args = hydra_args_for_task(task) + if not args: + return "default" + return args[0].split("=", 1)[1] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--task_id") + parser.add_argument("--backend") + parser.add_argument("--num_envs", type=int, default=1) + parser.add_argument("--num_frames", type=int, default=200) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--out_dir", required=True) + parser.add_argument("--fps_mean", type=float, default=200.0) + parser.add_argument("--failure_phase", default="none") + args = parser.parse_args() + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + # Simulate failures + if args.failure_phase == "import": + # Emit a fake traceback and exit non-zero without writing perf file + print("Traceback (most recent call last):") + print(" File \"\", line 1, in ") + print("ImportError: simulated import failure") + sys.exit(1) + + if args.failure_phase == "init": + # Emit AppLauncher init message then exit non-zero (no perf file) + print("AppLauncher initialization complete") + sys.exit(2) + + # Prepare FPS series + n = int(args.num_frames) + rng = random.Random(0) + noise = 5.0 + fps_series = [max(0.0, rng.gauss(args.fps_mean, noise)) for _ in range(n)] + + # Write perf_regression_gate_info.json + identity = split_backend_key(args.backend) + if identity is None: + raise RuntimeError(f"Cannot parse backend identity from {args.backend!r}") + benchmark_info_phase = { + "phase_name": "benchmark_info", + "metadata": [ + {"name": "stub benchmark_info task", "data": args.task_id}, + {"name": "stub benchmark_info num_envs", "data": args.num_envs}, + {"name": "stub benchmark_info num_frames", "data": args.num_frames}, + {"name": "stub benchmark_info seed", "data": args.seed}, + {"name": "stub benchmark_info physics_backend", "data": identity.physics_backend}, + {"name": "stub benchmark_info render_backend", "data": identity.render_backend}, + {"name": "stub benchmark_info backend_key", "data": identity.backend_key}, + {"name": "stub benchmark_info presets", "data": _presets_for_backend(args.task_id, identity)}, + ], + } + + runtime_phase = { + "phase_name": "runtime", + "measurements": [ + { + "name": "Step Frametimes", + "value": { + "Environment step effective FPS": fps_series, + "Environment step times": [1000.0 / max(fps, 1.0) for fps in fps_series], + }, + } + ], + } + info_path = out_dir / "perf_regression_gate_info.json" + with info_path.open("w") as fh: + json.dump([benchmark_info_phase, runtime_phase], fh) + + # Print Step Frametimes marker so classify_failure_phase can see it + print("Step Frametimes") + + # For runtime failure emulate crash after printing frame times + if args.failure_phase == "runtime": + print("RuntimeError: simulated crash during runtime") + sys.exit(3) + + # Successful exit + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/tools/perf_regression_gate/docs/module-interfaces.md b/tools/perf_regression_gate/docs/module-interfaces.md new file mode 100644 index 000000000000..b142e0e03c81 --- /dev/null +++ b/tools/perf_regression_gate/docs/module-interfaces.md @@ -0,0 +1,731 @@ +# Performance Regression Gate — Module Interface Reference + +Full function signatures, CLI arguments, and JSON schemas for every module. +All commands run from the `IsaacLab/` repository root unless otherwise noted. + +--- + +## `backend_identity.py` + +Canonical backend identity helpers. `backend_key` is always derived as `physics_backend` for physics-only runs, or `{physics_backend}_{render_backend}` when a render backend is set. + +```python +def make_backend_key(physics_backend: str, render_backend: str | None = None) -> str +def backend_identity_from_launch_config(config: dict) -> BackendIdentity | None +def backend_identity_from_benchmark_info(info: dict) -> BackendIdentity | None +``` + +This is used by task loading, launch config generation, Phase 2 config-drift checks, aggregate baseline lookup, and baseline sample metadata. Render backends are workload identity, so `physx_newton_renderer`, `newton_newton_renderer`, and `newton_ovrtx_renderer` are separate baseline buckets. + +## `gpu_identity.py` + +Canonical GPU identity helpers. `gpu_model` is the baseline bucket key; `gpu_model_raw` is preserved for display/provenance. Existing `tasks.json` hard-floor maps can keep legacy keys such as `L40S` because floor lookup tries canonical, raw, and known legacy aliases. + +```python +def canonical_gpu_model(value: Any) -> str +def normalize_gpu_fields(value: Any) -> dict[str, str] +def gpu_model_config_keys(value: Any) -> list[str] +``` + +Examples: `NVIDIA L40S -> l40s`, `RTX6000 -> rtx_6000`, `NVIDIA GeForce RTX 5090 -> geforce_rtx_5090`. + +## `runtime_contract.py` + +Builds the runtime compatibility contract used for baseline matching. The matching code only sees `runtime_contract_hash`; package/version field selection stays in `gate_config.py`. + +```python +def build_runtime_contract(*, provenance: dict | None, gpu_diag: dict | None, backend: BackendIdentity, policy: Mapping[str, Any]) -> tuple[dict, str] +def build_runtime_publish_info(*, provenance: dict | None, gpu_diag: dict | None, policy: Mapping[str, Any]) -> dict +``` + +Default compatibility fields gate on top-level active-path packages: IsaacSim, IsaacLab, Torch, Warp, the active physics package (`isaaclab_physx` or `isaaclab_newton`/`newton`), and `isaaclab_ov` for renderer backends. CUDA version, NVIDIA driver, GPU memory, and compute capability are published for humans but do not affect the compatibility hash by default. + +## `github_gate_context.py` + +Resolves GitHub event metadata into the aggregate arguments used for baseline matching and publication policy. + +```python +def resolve_gate_context(env=None, event=None, fetch_pr=None) -> GateContext +``` + +Outputs: `base_sha`, `target_branch`, `source_branch`, `allow_update`, `trusted_source`, and `event_kind`. +Mirrored PR pushes under `pull-request/` use the GitHub PR API to recover the real PR base/source branches. They are read-only unless `PERF_GATE_ALLOW_MIRROR_BASELINE_UPDATE`/`ALLOW_MIRROR_UPDATE` is explicitly true. + +## `oracle.py` + +### `compare()` + +```python +def compare( + bench_result: dict, + baseline: Baseline | None, + fps_mean_floor: float, + excluded_frames: frozenset[int], + artifact_dir: Path, +) -> OracleResult +``` + +The central verdict function. Reads `perf_regression_gate_info.json` from `artifact_dir`, +applies `excluded_frames`, computes mean FPS, and returns an `OracleResult`. + +| Parameter | Type | Description | +|---|---|---| +| `bench_result` | `dict` | Loaded `perf_regression_gate_result.json` | +| `baseline` | `Baseline \| None` | Rolling baseline stats; `None` for seed run | +| `fps_mean_floor` | `float` | Hard minimum FPS; 0.0 = disabled | +| `excluded_frames` | `frozenset[int]` | 0-based frame indices to drop before computing mean | +| `artifact_dir` | `Path` | Directory containing `perf_regression_gate_info.json` | + +### `apply_excluded_frames()` + +```python +def apply_excluded_frames(fps_series: list[float], excluded_frames: frozenset[int]) -> list[float] +``` + +Returns `fps_series` with indices listed in `excluded_frames` removed. + +### `class Baseline` + +```python +@dataclass +class Baseline: + median_fps: float # Median FPS of the baseline window + mad_fps: float # Median absolute deviation of FPS in the window + k_warn: float = 2.5 # MAD multiplier for WARN threshold + k_block: float = 4.0 # MAD multiplier for BLOCK threshold + sample_count: int = 0 # Number of samples in the window +``` + +Thresholds: `warn_thresh = median - k_warn × MAD`, `block_thresh = median - k_block × MAD`. + +### `class OracleResult` + +```python +@dataclass +class OracleResult: + verdict: OracleVerdict # PASS / WARN / BLOCK / HARD_FAILURE + bisect_verdict: str # "GOOD" / "BAD" / "SKIP" + failure_phase: str | None # From bench_result; see failure phase table + measured_fps: float | None # Mean FPS post-filter; None on HARD_FAILURE + baseline_fps: float | None # baseline.median_fps; None if no baseline + regression_pct: float | None # ((measured - baseline) / baseline) × 100; None if no baseline + fps_median: float | None # Median of filtered series [informational] + fps_p5: float | None # 5th-percentile of filtered series [informational] + fps_p95: float | None # 95th-percentile of filtered series [informational] + gpu_mem_used_mb: float | None # From bench_result.gpu_diag [informational] + startup_time_s: float | None # From bench_result [informational] + wall_time_s: float | None # From bench_result [informational] + was_retried: bool # Whether Phase 1 succeeded only after a retry + task_id: str + backend: str +``` + +`fps_median`, `fps_p5`, `fps_p95` are informational — they do not affect the verdict. +`measured_fps` (mean of filtered series) is the blocking metric. + +### `class OracleVerdict` + +```python +class OracleVerdict(str, Enum): + PASS = "PASS" + WARN = "WARN" + BLOCK = "BLOCK" + HARD_FAILURE = "HARD_FAILURE" +``` + +--- + +## `task_config.py` + +### `load_tasks()` + +```python +def load_tasks(tasks_json_path: Path | str | None = None) -> list[TaskConfig] +``` + +Loads all benchmark tasks from `tasks.json`, expanding each task's `backends` array into +one `TaskConfig` per `(task_id, backend)` combination. Applies `defaults` block to each task. + +Default path: `tools/perf_regression_gate/tasks.json` (sibling of the module). + +### `get_task()` + +```python +def get_task(task_id: str, backend_key: str, tasks_json_path: Path | str | None = None) -> TaskConfig +``` + +Returns the `TaskConfig` for a specific `(task_id, backend_key)` pair. +Raises `KeyError` if not found. + +### `caches_for_backend()` + +```python +def caches_for_backend(backend: str) -> list[str] +``` + +Returns cache identifiers needed before benchmarking with a given physics backend. +Currently: `"newton"` → `["mjwarp_jit"]`; all others → `[]`. + +### `class TaskConfig` + +```python +@dataclass +class TaskConfig: + task_id: str + physics_backend: str # "physx" or "newton" + render_backend: str | None # "newton_renderer", "ovrtx_renderer", or None + preset: str # Hydra preset base (usually "default") + num_envs: int + num_frames: int + excluded_frames_raw: list[int | list[int]] # Raw JSON; use .excluded_frames + camera_resolution: tuple[int, int] | None + timeout_minutes: int + fps_mean_floor: dict # {"L40S": {"physx": 100.0, ...}} + caches: list[str] # Cache identifiers from caches_for_backend() + tags: list[str] # ["always"] or ["camera"] + task_type: str # "benchmark" + runs_on: str # "gpu-l40s" + seed: int | None # Random seed for benchmark (default 42) + + @property + def backend_key(self) -> str: + # "{physics}_{render}" if render_backend else "{physics}" + + @property + def excluded_frames(self) -> frozenset[int]: + # Expands excluded_frames_raw ranges to individual indices +``` + +`excluded_frames_raw` supports two entry types: +- `[start, end]` — inclusive range, expanded to `range(start, end+1)` +- `N` — single index + +Default value `[[0, 100]]` expands to indices 0–100 (101 frames excluded from 300 total). + +--- + +## `baseline_manager.py` + +### Flat-file operations (local / testing) + +```python +def load_baseline( + baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None +) -> Baseline | None +``` +Loads `samples.ndjson` for a task/backend pair and computes thresholds from compatible samples. Matching can require exact `gpu_model`, `task_id`, `backend_key`, `launch_config_hash`, `benchmark_contract_hash`, `runtime_contract_hash`, and `baseline_epoch`; with `base_sha`, samples must also come from ancestor commits. Returns `None` if no structured compatible samples exist. + +```python +def update_baseline( + baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fps: float, fingerprint=None +) -> None +``` +Appends one structured sample to `samples.ndjson`. Thresholds are computed at read time. +Only call for PASS/WARN results — aggregate.py enforces this policy. + +```python +def delete_baseline_files( + baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None +) -> None +``` +Removes `samples.ndjson` for a task/backend pair. Used in tests and resets. + +```python +def seed_baseline_with_spread( + baselines_dir: Path, gpu_model: str, task_id: str, backend: str, + center_fps: float, noise_fps: float = 5.0, n_samples: int = 10, + seed: int = 0, fingerprint=None +) -> None +``` +Populates a baseline window with `n_samples` Gaussian samples around `center_fps` ± `noise_fps`. +Used in tests to create deterministic baselines without running real benchmarks. + +### Git operations (production) + +```python +def refresh_baseline_branch(branch: str, remote: str | None = "origin") -> str | None +def load_baseline_git(ref: str, gpu_model: str, task_id: str, backend: str, fingerprint: str | None) -> Baseline | None +def update_baselines_git(branch: str, updates: list[BaselineUpdateRecord], remote: str | None = "origin") -> BaselinePushResult +def update_baseline_git(branch: str, gpu_model: str, task_id: str, backend: str, fps: float, fingerprint: str | None) -> BaselinePushResult +``` + +`refresh_baseline_branch()` fetches the remote baseline branch and returns the exact SHA +used for reads. When `base_sha` is supplied, baseline matching requires each selected sample +to have a `commit_sha` that is an ancestor of that base. `update_baselines_git()` is the +production writer: it refetches the branch, applies queued structured samples in a temporary +worktree, commits once, and retries on non-fast-forward push races. Sample IDs make retries +idempotent if a previous push outcome was ambiguous. + +### File paths (flat-file) + +``` +{baselines_dir}/{gpu_model}/{task_id}/{backend}/samples.ndjson + +With fingerprint: +{baselines_dir}/{gpu_model}/{task_id}/{backend}/{fingerprint}/samples.ndjson +``` + +--- + +## `subprocess_runner.py` + +### `classify_failure_phase()` + +```python +def classify_failure_phase( + stdout: str, stderr: str, exit_code: int, wall_time_s: float, timeout_s: float +) -> str | None +``` + +Classifies the failure phase of a benchmark run by scanning combined output. Priority order: + +| Priority | Phase | Trigger | +|---|---|---| +| 1 | `"oom"` | `exit_code == 137` OR `"oom-kill"` in `stderr` | +| 2 | `"hang"` | `wall_time_s >= timeout_s * 0.95` | +| 3 | `"import"` | `"Traceback"` in combined AND `"AppLauncher"` NOT in `stdout` | +| 4 | `"driver"` | `"CudaError"` OR `"CUDA_ERROR_"` in combined | +| 5 | `"init"` | `exit_code != 0` AND `"AppLauncher initialization complete"` in `stdout` AND `"Step Frametimes"` NOT in `stdout` | +| 6 | `"runtime"` | `exit_code != 0` AND `"Step Frametimes"` in `stdout` | +| 7 | `None` | `exit_code == 0` | + +### `run_benchmark()` + +```python +def run_benchmark(cmd: list, timeout_s: float) -> dict +``` + +Runs `cmd` with `capture_test_output_with_timeout()` and returns: + +```python +{ + "exit_code": int, + "stdout_tail": str, # last 2000 chars of combined stdout + "wall_time_s": float, + "startup_time_s": float, # stub: 0.0 + "failure_phase": str | None, +} +``` + +### `capture_test_output_with_timeout()` + +```python +def capture_test_output_with_timeout( + cmd, timeout, env, startup_deadline=0, report_file="" +) -> tuple[int, bytes, bytes, str, float, str] +``` + +Returns `(returncode, stdout_bytes, stderr_bytes, kill_reason, wall_time, pre_kill_diag)`. + +`kill_reason` values: `""` (normal), `"timeout"`, `"startup_hang"`, `"shutdown_hang"`. + +Uses `select()` + non-blocking I/O for real-time streaming. Kills the entire process group +(including Kit / Isaac Sim child processes) on timeout. + +--- + +## `tasks_to_ci_matrix.py` + +``` +python3 tools/perf_regression_gate/tasks_to_ci_matrix.py +``` + +No arguments. Reads `tasks.json` via `load_tasks()` and prints a JSON array to stdout, +one object per `(task_id, backend)` combination. Used by the `build_matrix` step in +`perf-regression-gate.yaml` to populate the GitHub Actions job matrix. + +Each object contains: `task_id`, `physics_backend`, `render_backend` (empty string if none), +`num_envs`, `num_frames`, `seed`, `hydra_args`, `bench_timeout_s`, and `job_timeout_minutes`. + +--- + +## `build_bench_result.py` CLI + +``` +python3 tools/perf_regression_gate/build_bench_result.py \ + --task_id task identifier (required) + --physics_backend "physx" or "newton" (required) + --render_backend render backend name or "" for none (default: "") + --artifact_dir directory for artifacts (required) + --exit_code Phase 1 process exit code (required) + --wall_time_s Phase 1 wall-clock time in seconds (required) + --timeout_s Phase 1 timeout in seconds (required) + --log_file combined stdout+stderr log from Phase 1 (default: none) + --launch_config launch_config.json from Phase 1 (default: artifact_dir/launch_config.json) + --gate_config gate_config.json for runtime compatibility policy + --attempt attempt number: 1 = first try, 2 = after retry (default: 1) + --was_retried flag: set when this result comes from a retry +``` + +**Output:** `{artifact_dir}/perf_regression_gate_result.json` (always written). + +**Side effect:** Renames `benchmark_non_rl_{task_id}_{timestamp}.json` to +`perf_regression_gate_info.json` if the canonical name does not already exist. + +When `perf_regression_gate_info.json` is present, `build_bench_result.py` also +calls `_extract_info_provenance()` to populate the FPS distribution, startup time, +GPU diagnostics, and full software/hardware/git provenance directly into the result JSON. +It also compares observed benchmark identity to `launch_config.json`, records `observed_backend`, +computes `runtime_contract_hash`, and publishes non-matching runtime diagnostics such as CUDA and driver version. +`nvidia-smi` is queried once at post-processing time to capture the driver version. + +--- + +## `aggregate.py` CLI + +``` +python3 tools/perf_regression_gate/aggregate.py \ + --artifacts_dir root directory containing per-task artifact subdirectories (required) + --gpu_model GPU model label for baseline lookup (default: L40S) + --gate_config path to gate_config.json (default: perf_regression_gate/gate_config.json) + --baseline_branch git branch for baseline storage (default: angehu/perf-baselines) + --baseline_remote git remote that owns the baseline branch (default: origin; empty = local only) + --baseline_push_retries max retry attempts for transactional baseline pushes (default: config) + --baselines_dir flat-file baseline directory; bypasses git (default: None = use git) + --allow_baseline_update "true"/"false": extend baseline window for PASS/WARN (default: false) + --summary_file append step-summary markdown to this path (default: none) + --base_sha PR/protected branch base SHA for ancestry-aware matching + --target_branch protected target branch name + --source_branch source branch name recorded in baseline metadata + --trusted_source audit label for written samples +``` + +**Exit codes:** +- `0` — gate is non-blocking, or all tasks PASS/WARN +- `1` — any BLOCK verdict and `gate_config.blocking == true` +- `2` — any HARD_FAILURE verdict and `gate_config.blocking == true` + +**Environment variable:** When `GITHUB_OUTPUT` is set, aggregate writes baseline trace +outputs such as `baseline_read_sha`, `baseline_pushed_sha`, `baseline_push_attempts`, and +`baselines_updated`. The push has already happened inside aggregate when these outputs are +written. + +--- + +## `local_runner.py` CLI + +``` +python3 tools/perf_regression_gate/local_runner.py \ + --tags task tags to run (default: always) + --gpu_model GPU model label for baselines (default: auto-detect with nvidia-smi) + --artifacts_dir root for per-task artifacts (default: perf_regression_gate/artifacts/) + --baselines_dir flat-file baseline dir (default: perf_regression_gate/local_baselines/) + --allow_baseline_update extend baseline window for PASS/WARN results + --dry_run print task matrix and exit without running anything + --skip_existing skip tasks whose perf_regression_gate_result.json already exists + --gate_config path to gate_config.json +``` + +Orchestrates Phase 1+2+3 sequentially. For each task: +1. Runs `./isaaclab.sh -p scripts/benchmarks/benchmark_non_rl.py` with the derived command +2. On non-zero exit, retries once (sets `was_retried=True`) +3. Calls `build_bench_result.py` (Phase 2) +4. After all tasks, calls `aggregate.py` (Phase 3) + +Returns aggregate.py's exit code. + +--- + +## `gate_config.py` + +```python +def load_gate_config(path: Path | str) -> dict +``` + +Loads `gate_config.json` when present and otherwise returns conservative defaults. +The dict contract includes `blocking`, `min_baseline_samples`, `max_baseline_samples`, +`min_block_regression_pct`, `baseline_push_retries`, and `runtime_compatibility`. +`runtime_compatibility` owns the policy for fields included in `runtime_contract_hash` vs +publish-only diagnostic fields. + +--- + +## `dev/stub_benchmark.py` CLI + +Simulates `benchmark_non_rl.py` for testing. Does not require IsaacSim. + +``` +python3 tools/perf_regression_gate/dev/stub_benchmark.py \ + --task_id + --backend + --num_envs (default: 1) + --num_frames (default: 200) + --out_dir (required) + --fps_mean (default: 200.0) + --failure_phase "none" | "import" | "init" | "runtime" (default: none) +``` + +On success: writes `perf_regression_gate_info.json` with a Gaussian FPS series centered on +`--fps_mean`, prints `"Step Frametimes"` to stdout, exits 0. + +On failure modes: +- `import`: prints traceback-like output, exits 1 (no perf file written) +- `init`: prints `"AppLauncher initialization complete"`, exits 2 (no perf file written) +- `runtime`: writes perf file, prints frametimes, then prints error and exits 3 + +--- + +## `dev/sim_regression.py` CLI + +Injects degraded FPS artifacts for demo/testing without re-running benchmarks. + +``` +python3 tools/perf_regression_gate/dev/sim_regression.py \ + --fps_scale multiply baseline FPS by this factor (default: 0.53 = 47% regression) + --tags task tags to include (default: always) + --gpu_model GPU model label (default: L40S) + --baselines_dir (default: perf_regression_gate/local_baselines) + --out_dir output artifacts directory (default: /tmp/sim_artifacts) +``` + +For each task with an existing baseline, loads `samples.ndjson`, computes +`regressed_fps = baseline.median_fps × fps_scale`, and writes: +- `{out_dir}/{task_id}/{backend_key}/perf_regression_gate_info.json` +- `{out_dir}/{task_id}/{backend_key}/perf_regression_gate_result.json` + +Skips tasks with no baseline (prints `SKIP (no baseline)`). + +--- + +## `tasks.json` Schema + +```json +{ + "defaults": { + "type": "benchmark", + "runs_on": "gpu-l40s", + "preset": "default", + "seed": 42, + "num_envs": 512, + "num_frames": 300, + "excluded_frames": [[0, 100]], + "camera_resolution": null, + "timeout_minutes": 10, + "tags": ["always"] + }, + "tasks": [ + { + "task_id": "", + "num_envs": , // overrides defaults + "timeout_minutes": , // overrides defaults + "tags": ["always" | "camera"], // overrides defaults + "backends": [ + {"physics": "physx"}, + {"physics": "newton"}, + {"physics": "physx", "render": "newton_renderer"}, + {"physics": "newton", "render": "ovrtx_renderer"} + ], + "fps_mean_floor": { + "": { + "": // 0.0 = disabled + } + } + } + ] +} +``` + +Each entry in `backends` becomes one `TaskConfig`. `backend_key = physics` when no +`render` field; `backend_key = {physics}_{render}` when `render` is present. + +--- + +## Artifact Schemas + +### `perf_regression_gate_result.json` (Phase 2 output) + +```json +{ + "task_id": "Isaac-Velocity-Flat-G1-v0", + "backend": "physx", + "backend_key": "physx", + "physics_backend": "physx", + "render_backend": null, + "preset": "default", + "attempt": 1, + "was_retried": false, + "exit_code": 0, + "failure_phase": null, + "stdout_tail": "", + "wall_time_s": 23.7, + "startup_time_s": 12.5, + "perf_regression_gate_info_present": true, + "raw_fps_mean": 1655000.0, + "raw_fps_std": 9800.0, + "raw_fps_min": 1632000.0, + "raw_fps_max": 1680000.0, + "raw_fps_median": 1655000.0, + "raw_fps_p5": 1638000.0, + "raw_fps_p95": 1672000.0, + "outlier_count": null, + "gpu_diag": { + "gpu_name": "NVIDIA L40S", + "gpu_total_memory_gb": 45.62, + "cuda_version": "12.1", + "nvidia_driver_version": "550.54.15", + "gpu_mem_used_mb": 18432.0 + }, + "provenance": { + "hardware": { + "cpu_name": "Intel Xeon Gold 6438Y+", + "cpu_physical_cores": 32, + "total_ram_gb": 251.5, + "gpu_device_count": 1, + "gpu_name": "NVIDIA L40S", + "gpu_total_memory_gb": 45.62, + "gpu_compute_capability": "8.9", + "gpu_multi_processor_count": 142, + "cuda_version": "12.1" + }, + "software": { + "isaaclab": "2.1.0", + "warp": "1.6.0", + "isaacsim": "4.5.0", + "torch": "2.3.0+cu121", + "numpy": "1.26.4", + "newton": "1.0.0", + "mujoco_warp": "0.3.0" + }, + "git": { + "commit_hash": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "commit_hash_short": "a1b2c3d4", + "branch": "develop", + "commit_date": "2024-06-10 09:00:00 +0000", + "dirty": false + } + }, + "launch_config": { + "gpu_model": "l40s", + "gpu_model_raw": "NVIDIA L40S", + "launch_config_hash": "...", + "benchmark_contract_hash": "..." + }, + "observed_backend": { + "physics_backend": "physx", + "render_backend": null, + "backend_key": "physx" + }, + "runtime_contract_hash": "...", + "runtime_contract": { + "runtime_contract_version": 1, + "fields": {"software.warp": "1.6.0"} + }, + "runtime_info": { + "software": {"warp": "1.6.0"}, + "publish_only": {"gpu_diag.cuda_version": "12.1"} + }, + "task_config_snapshot": { + "task_id": "Isaac-Velocity-Flat-G1-v0", + "backend": "physx", + "backend_key": "physx", + "physics_backend": "physx", + "render_backend": null, + "preset": "default", + "num_envs": 512, + "num_frames": 300, + "excluded_frames_raw": [[0, 100]], + "timeout_minutes": 12, + "tags": ["always"], + "seed": 42 + } +} +``` + +**Fields populated from `perf_regression_gate_info.json`** (all `null` when +`perf_regression_gate_info_present` is false): + +| Field | Source | Notes | +|---|---|---| +| `raw_fps_mean` … `raw_fps_p95` | `runtime` phase, `Step Frametimes` measurement | Full distribution before excluded-frame filtering | +| `startup_time_s` | `startup` phase, `Total Start Time (Launch to Train)` measurement | `null` if startup phase absent | +| `gpu_diag.gpu_mem_used_mb` | `runtime` phase, `GPU Memory Used` measurement | Converted from GB | +| `gpu_diag.gpu_name`, `.cuda_version`, `.gpu_total_memory_gb` | `hardware_info` phase | | +| `gpu_diag.nvidia_driver_version` | `nvidia-smi` subprocess at post-processing time | `null` if nvidia-smi unavailable | +| `provenance.hardware` | `hardware_info` phase | CPU, GPU, RAM identity | +| `provenance.software` | `version_info` phase | Package versions; `_version` suffix stripped | +| `provenance.git` | `version_info` phase, `dev` dict | commit, branch, date, dirty flag | + +Key fields the oracle reads: `perf_regression_gate_info_present`, `failure_phase`, +`was_retried`, `gpu_diag.gpu_mem_used_mb`, `startup_time_s`, `wall_time_s`. + +The `raw_fps_*` fields capture the full unfiltered distribution and are for audit/debug; +`oracle.compare()` recomputes mean FPS independently after applying `excluded_frames`. + +### `perf_regression_gate_info.json` (Phase 1 output, renamed from `benchmark_non_rl_*.json`) + +A list of `TestPhase` objects serialized by `JSONFileMetrics`. Measurement and metadata +names are prefixed with `"{task_id} {phase_name} "` by the serializer. + +```json +[ + { + "phase_name": "hardware_info", + "measurements": [], + "metadata": [ + {"name": " hardware_info cpu_name", "data": "Intel Xeon Gold 6438Y+", "type": "string"}, + {"name": " hardware_info physical_cores", "data": 32, "type": "int"}, + {"name": " hardware_info total_ram_gb", "data": 251.5, "type": "float"}, + {"name": " hardware_info gpu_device_count", "data": 1, "type": "int"}, + {"name": " hardware_info cuda_version", "data": "12.1","type": "string"}, + {"name": " hardware_info gpu_devices", "data": { + "0": {"name": "NVIDIA L40S", "total_memory_gb": 45.62, + "compute_capability": "8.9", "multi_processor_count": 142} + }, "type": "dict"} + ] + }, + { + "phase_name": "version_info", + "measurements": [], + "metadata": [ + {"name": " version_info isaaclab_version", "data": "2.1.0", "type": "string"}, + {"name": " version_info warp_version", "data": "1.6.0", "type": "string"}, + {"name": " version_info dev", "data": { + "commit_hash": "a1b2c3d4...", "commit_hash_short": "a1b2c3d4", + "branch": "develop", "commit_date": "2024-06-10 09:00:00 +0000", "dirty": false + }, "type": "dict"} + ] + }, + { + "phase_name": "runtime", + "measurements": [ + { + "name": " runtime Step Frametimes", + "value": {"Environment step effective FPS": [1644720.5, 1638400.0, ...]}, + "type": "dict" + }, + { + "name": " runtime GPU Memory Used", + "value": 18.0, + "unit": "GB", + "type": "single" + } + ], + "metadata": [] + }, + { + "phase_name": "startup", + "measurements": [ + { + "name": " startup Total Start Time (Launch to Train)", + "value": 12.5, + "unit": "s", + "type": "single" + } + ], + "metadata": [] + } +] +``` + +The oracle looks for `phase_name == "runtime"`, then the measurement whose `name` ends with +`"Step Frametimes"`, then extracts `value["Environment step effective FPS"]` as the raw series. +`build_bench_result.py` reads the remaining phases for provenance extraction. + +### `samples.ndjson` (baseline history) + +One JSON object per accepted baseline sample, append-only. The exact metadata can grow +without changing the file contract; the required fields for threshold calculation are: + +```json +{"fps": 1667482.3, "gpu_model": "l40s", "task_id": "Isaac-Cartpole-Direct-v0", "backend_key": "physx", "launch_config_hash": "...", "benchmark_contract_hash": "...", "runtime_contract_hash": "...", "baseline_epoch": 1} +``` diff --git a/tools/perf_regression_gate/docs/system-design.md b/tools/perf_regression_gate/docs/system-design.md new file mode 100644 index 000000000000..7e43d6af7ecc --- /dev/null +++ b/tools/perf_regression_gate/docs/system-design.md @@ -0,0 +1,460 @@ +# IsaacLab CI Performance Regression Gate — System Design +**Status:** POC / MVP running locally, pending productionization, deployment, deployment features +**Date:** 2026-06-15 +**Owners:** Angelina Hu, Neil Mehta + +--- + +## 1. Purpose and Use Cases + +The performance regression gate runs a fixed benchmark matrix on every PR and blocks merge +when throughput drops below an explicit hard floor or a MAD-derived threshold relative to compatible rolling baseline samples. + +**Use cases:** + +| When | What happens | +|---|---| +| Feature PR touches physics or RL code | 5 "always" benchmarks run automatically | +| PR touches camera/rendering paths | 5 additional Shadow-Vision camera benchmarks added | +| Any task regresses > k_block × MAD below baseline | Aggregate exits 1; GitHub required check fails | +| Gate is in advisory mode (`blocking: false`) | Verdicts print but PR is not blocked | +| Baseline does not yet exist | Seed run: WARN with `no_baseline` (transparent, non-blocking in advisory mode) | +| Protected branch (main/develop/release) merges | Baseline history extended with structured PASS/WARN samples | + +--- + +## 2. Design Principles + +1. **One source of truth** Task and backend parameters live in `tasks.json`. + Python, shell, and GitHub Actions YAML all read from it so there is no duplication of info. + +2. **Modularizable** Logic should be back-end and task agnostic; backend is a data dimension in + `tasks.json`, not a logic branch in `oracle.py`, `subprocess_runner.py`, or `task_config.py`. + Each pipeline stage should have proper separation of concerns. Individual components should + be agnostic to environment/call method as long as contract is maintained. + +3. **Minimal invasiveness** Bench jobs directly call `benchmark_non_rl.py --benchmark_backend json`. + They do not invoke `tools/conftest.py` at runtime, interfere with existing tests, or modify task code. + +4. **Traceability** Every stage leaves informative artifacts. Every bench job writes `perf_regression_gate_result.json` + regardless of success or failure so the aggregator always has a structured artifact to read. + +5. **Lightweight** Bench jobs are only run when necessary and with minimally sufficient configs. + Warmed caches are pulled when needed (Newton). + +--- + +## 3. System Diagram + +``` +PR opened (to main / release / develop) + │ + ▼ +┌─────────────────────────────────────────┐ +│ .github/workflows/perf-regression-gate.yaml │ +│ │ +│ 1. Expand task matrix from tasks.json │ +│ 2. Activate tags from changed files │ +│ 3. Fan out: one runner per task │ +└────────────────┬────────────────────────┘ + │ (parallel per task) + ┌──────────┴──────────┐ + ▼ ▼ +┌───────────┐ ┌───────────┐ +│ Phase 1 │ ... │ Phase 1 │ benchmark_non_rl.py +│ Cartpole │ │ G1/newton │ --benchmark_backend json +│ /physx │ │ │ writes benchmark_non_rl_*.json +└─────┬─────┘ └─────┬─────┘ + │ │ + ▼ ▼ +┌───────────┐ ┌───────────┐ +│ Phase 2 │ ... │ Phase 2 │ build_bench_result.py +│ │ │ │ renames → perf_regression_gate_info.json +│ │ │ │ classifies failure_phase +│ │ │ │ writes perf_regression_gate_result.json +└─────┬─────┘ └─────┬─────┘ + │ │ + └──────────┬──────────┘ + │ (artifacts dir) + ▼ + ┌────────────────┐ + │ Phase 3 │ aggregate.py + │ aggregate │ oracle.compare() per task + │ + oracle │ prints verdict table + │ │ updates baseline window (if allowed) + └───────┬────────┘ + │ + ┌───────┴────────┐ + │ exit 0 │ all PASS/WARN, or gate non-blocking + │ exit 1 │ any BLOCK + blocking=true + │ exit 2 │ any HARD_FAILURE + blocking=true + └────────────────┘ +``` + +--- + +## 4. Repository File Layout + +``` +IsaacLab/ +├── .github/ +│ └── workflows/ +│ ├── build.yaml MODIFIED: add image_tag output to build job +│ └── perf-regression-gate.yaml CI gate workflow +│ +└── tools/ + ├── conftest.py MODIFIED: one-line import change + ├── subprocess_runner.py run_benchmark(), classify_failure_phase() + │ capture_test_output_with_timeout() borrowed from + │ existing conftest CI infrastructure + │ + └── perf_regression_gate/ + ├── __init__.py + ├── tasks.json SINGLE SOURCE OF TRUTH — task/backend matrix + ├── task_config.py TaskConfig dataclass, load_tasks(), get_task() + ├── backend_identity.py canonical physics/render backend identity + ├── gpu_identity.py canonical GPU buckets + legacy floor aliases + ├── runtime_contract.py runtime compatibility contract/hash builder + ├── launch_config.py artifact-carried launch intent + hashes + ├── write_launch_config.py CI/local helper to write launch_config.json + ├── github_gate_context.py PR/merge/push context + baseline write policy + ├── gate_types.py verdict/failure/threshold enums + ├── tasks_to_ci_matrix.py Converts tasks.json → GitHub Actions matrix JSON + │ (called by perf-regression-gate.yaml build_matrix step) + ├── oracle.py compare() → OracleResult; PASS/WARN/BLOCK/HARD_FAILURE + ├── build_bench_result.py Phase 2: reads log + benchmark JSON, + │ extracts FPS stats + SW/HW/git provenance, + │ writes perf_regression_gate_result.json + ├── aggregate.py Phase 3: scans result JSONs, calls oracle, + │ prints table, updates baselines, exits 0/1/2 + ├── baseline_manager.py load/update baseline, flat-file + git variants + ├── gate_config.py policy constants + runtime compatibility defaults + ├── local_runner.py LOCAL END-TO-END RUNNER: orchestrates Phase 1+2+3 + | without Github/Docker/cloud platform dependencies + │ + ├── dev/ + │ ├── stub_benchmark.py Simulates benchmark_non_rl.py for unit tests + │ └── sim_regression.py Injects regressed FPS artifacts for demos + │ + ├── docs/ + │ ├── system-design.md High-level overview + │ ├── module-interfaces.md Full function/CLI interface reference + │ + └── tests/ Unit tests (no GPU) +``` + +**Baseline storage:** + +``` +Local (testing): +tools/perf_regression_gate/local_baselines/ + {gpu_model}/{task_id}/{backend_key}/ + samples.ndjson append-only structured baseline samples + +Production: +perf-baselines branch (git orphan) + {gpu_model}/{task_id}/{backend_key}/ + samples.ndjson append-only structured samples; compatibility fields live in each sample +``` + +--- + +## 5. Three-Phase Pipeline + +``` +Phase 1 — bench tasks.json → matrix → benchmark_non_rl.py (one process per task/backend) +Phase 2 — post-bench build_bench_result.py (reads log + benchmark JSON → writes result JSON) +Phase 3 — aggregate aggregate.py + oracle → verdict table → baseline update +``` + +### Phase 1: `benchmark_non_rl.py` + +Called via `./isaaclab.sh -p scripts/benchmarks/benchmark_non_rl.py`: + +```bash +./isaaclab.sh -p scripts/benchmarks/benchmark_non_rl.py \ + --task Isaac-Cartpole-Direct-v0 \ + --num_envs 4096 \ + --num_frames 300 \ + --benchmark_backend json \ + --output_path \ + [presets=newton_mjwarp] +``` + +Output: `benchmark_non_rl_{task_id}_{timestamp}.json` in `artifact_dir`. + +- Only step that depends on IsaacLab run-time. +- Need the `--benchmark_backend json` because the JSON backend preserves `DictMeasurement` +objects including the raw per-step FPS list but the OmniPerf backend drops these. + +### Phase 2: `build_bench_result.py` + +Runs once per task after Phase 1 completes: + +- Renames `benchmark_non_rl_*.json` → `perf_regression_gate_info.json` +- Classifies failure phase by scanning the benchmark log +- Parses the info artifact to extract FPS distribution statistics, startup time, GPU diagnostics, and full SW/HW/git provenance +- Computes `runtime_contract_hash` and publish-only runtime info +- Writes `perf_regression_gate_result.json` (always written, even on failure) + +### Phase 3: `aggregate.py` + +Plain Python. Scans `--artifacts_dir` recursively for `perf_regression_gate_result.json`, +calls `oracle.compare()` for each, prints the verdict table, optionally updates baselines. + +Exit codes: 0 = all clear or non-blocking; 1 = BLOCK + blocking mode; 2 = HARD_FAILURE + blocking mode. + +--- + +## 6. Run Modes and Tag System + +Each task entry in `tasks.json` has a `"tags"` array. CI activates tags from the PR's changed +file list; the benchmark matrix is filtered to tasks whose tags intersect the activated set. + +| Tag | Meaning | Tasks | +|---|---|---| +| `"always"` | Run on every PR | Cartpole ×2, Factory ×1, G1 ×2 (5 tasks) | +| `"camera"` | Run when camera/rendering paths change | Shadow-Vision ×5 | + +Shadow-Vision has `"camera"` rather than `"always"` because its FPS is dominated by rendering +cost, not physics and Factory already covers manipulation and high-contact behavior, so it is +only a signal when camera code changes to save test time cost. + +**Tag activation rules WIP (production):** +- Any changed file → `"always"` always activated +- Files matching `source/isaaclab/sensors/**` or `source/isaaclab/envs/**/*vision*` → `"camera"` also activated + +--- + +## 7. Full Task Matrix + +10 (task, backend) combinations. "Effective FPS" = per-env FPS × num_envs. + +| task_id | backend_key | num_envs | frames | timeout | tags | floor (L40S) | +|---|---|---|---|---|---|---| +| Isaac-Cartpole-Direct-v0 | physx | 4096 | 300 | 10 min | always | 100 | +| Isaac-Cartpole-Direct-v0 | newton | 4096 | 300 | 10 min | always | 0 | +| Isaac-Factory-GearMesh-Direct-v0 | physx | 512 | 300 | 15 min | always | 30 | +| Isaac-Velocity-Flat-G1-v0 | physx | 512 | 300 | 12 min | always | 40 | +| Isaac-Velocity-Flat-G1-v0 | newton | 512 | 300 | 12 min | always | 0 | +| Isaac-Repose-Cube-Shadow-Vision-Direct-v0 | physx | 512 | 300 | 20 min | camera | 20 | +| Isaac-Repose-Cube-Shadow-Vision-Direct-v0 | physx_newton_renderer | 512 | 300 | 20 min | camera | 0 | +| Isaac-Repose-Cube-Shadow-Vision-Direct-v0 | newton | 512 | 300 | 20 min | camera | 0 | +| Isaac-Repose-Cube-Shadow-Vision-Direct-v0 | newton_newton_renderer | 512 | 300 | 20 min | camera | 0 | +| Isaac-Repose-Cube-Shadow-Vision-Direct-v0 | newton_ovrtx_renderer | 512 | 300 | 20 min | camera | 0 | + +`backend_key` = `{physics}` or `{physics}_{render}`. Preset tokens are derived automatically +by `local_runner.py` and the CI workflow. + +Floor = 0 means "no hard floor"; baseline MAD thresholds apply only. + +--- + +## 8. Oracle Logic + +``` +compare(bench_result, baseline, fps_mean_floor, excluded_frames, artifact_dir) + → OracleResult +``` + +**Verdict decision tree:** + +``` +perf_regression_gate_info_present == False? + → HARD_FAILURE (file-based check skipped) + +Load perf_regression_gate_info.json, extract fps_series from runtime phase +Apply excluded_frames filter +filtered empty? + → HARD_FAILURE + +mean_fps = statistics.mean(filtered) + +mean_fps < fps_mean_floor? + → BLOCK + +baseline is None? + → WARN (no_baseline) +baseline.sample_count < MIN_BASELINE_SAMPLES? + → WARN (insufficient_baseline) + +mean_fps < baseline.median - 4.0 * baseline.mad AND regression_pct <= -MIN_BLOCK_REGRESSION_PCT? + → BLOCK +mean_fps < baseline.median - 2.5 * baseline.mad? + → WARN +else + → PASS + +verdict == PASS and was_retried? + → downgrade to WARN +``` + +**Bisect verdicts:** + +| Oracle verdict | Condition | Bisect | +|---|---|---| +| PASS | clean first attempt | GOOD | +| PASS | was_retried | SKIP | +| WARN | any | SKIP | +| BLOCK | any | BAD | +| HARD_FAILURE | failure_phase in {init, runtime} | BAD | +| HARD_FAILURE | failure_phase in {import, driver, oom, hang, None} | SKIP | + +--- + +## 9. Failure Phase Classification + +`classify_failure_phase()` in `subprocess_runner.py` scans combined stdout+stderr in priority +order. Classification is entirely string-pattern-based — no backend branching. + +| `failure_phase` | Trigger | Bisect | +|---|---|---| +| `"oom"` | exit 137 or `"oom-kill"` in stderr | SKIP | +| `"hang"` | wall_time ≥ timeout × 0.95 | SKIP | +| `"import"` | `"Traceback"` in output, no `"AppLauncher"` seen | SKIP | +| `"driver"` | `"CudaError"` or `"CUDA_ERROR_"` in output | SKIP | +| `"init"` | exit ≠ 0, AppLauncher present, no `"Step Frametimes"` | BAD | +| `"runtime"` | exit ≠ 0, `"Step Frametimes"` present (partial run) | BAD | +| `null` | exit 0, no error markers | (clean) | + +`import` → SKIP because import failures indicate environment mismatch, not code regression. +`init` and `runtime` → BAD because Isaac Sim started normally: the failure is the code's fault. + +--- + +## 10. Baseline Storage + +**Flat-file (local testing):** + +``` +local_baselines/{gpu_model}/{task_id}/{backend_key}/ + samples.ndjson append-only structured baseline samples +``` + +`baseline_manager.update_baseline()` appends one structured sample per accepted result. +The rolling median/MAD thresholds are computed from the newest compatible samples at read time. +BLOCK results are never written. + +**Git branch (production):** `perf-baselines` orphan branch. `aggregate.py` reads from a +freshly fetched baseline branch SHA. Accepted PASS/WARN samples are pushed through +`baseline_manager.update_baselines_git()`, which refetches before writing, commits +append-only `samples.ndjson` updates in a temporary worktree, and retries the push +if another runner updates the branch first. + +**Trigger/write policy:** Non-draft PRs and merge-queue candidates run the gate but do +not publish baselines. Protected-branch push events (main/develop/release/*, plus the +POC branch while enabled) publish accepted PASS/WARN samples through the transactional +git writer. Feature branch runs are read-only unless `--allow_baseline_update` is set +for local testing. + +--- + +## 11. Artifact Schema (Key Fields) + +`perf_regression_gate_result.json` (Phase 2 output): + +```json +{ + "task_id": "Isaac-Velocity-Flat-G1-v0", + "backend": "physx", + "failure_phase": null, + "wall_time_s": 23.7, + "startup_time_s": 12.5, + "perf_regression_gate_info_present": true, + "raw_fps_mean": 1655000.0, + "raw_fps_std": 9800.0, + "raw_fps_min": 1632000.0, + "raw_fps_max": 1680000.0, + "raw_fps_median": 1655000.0, + "raw_fps_p5": 1638000.0, + "raw_fps_p95": 1672000.0, + "gpu_diag": { + "gpu_name": "NVIDIA L40S", + "gpu_total_memory_gb": 45.62, + "cuda_version": "12.1", + "nvidia_driver_version": "550.54.15", + "gpu_mem_used_mb": 18432.0 + }, + "provenance": { + "hardware": { "cpu_name": "...", "gpu_name": "NVIDIA L40S", "cuda_version": "12.1", "..." : "..." }, + "software": { "isaaclab": "2.1.0", "warp": "1.6.0", "torch": "2.3.0+cu121", "...": "..." }, + "git": { "commit_hash": "a1b2c3d4...", "branch": "develop", "dirty": false } + }, + "task_config_snapshot": { "..." : "..." } +} +``` + +`raw_fps_*` fields capture the unfiltered per-step distribution for audit/debug; +`oracle.compare()` recomputes mean FPS independently after applying `excluded_frames`. +`provenance` enables cross-run comparison: when two baselines diverge, diff their +`provenance` blocks to identify driver, CUDA, or package version changes. + +See `module-interfaces.md` for the full schema with all fields. + +`perf_regression_gate_info.json` (Phase 1 output, renamed from `benchmark_non_rl_*.json`): + +A list of `TestPhase` objects. The oracle reads `"Environment step effective FPS"` from the +`"runtime"` phase's `"Step Frametimes"` measurement. `build_bench_result.py` also reads +the `"hardware_info"`, `"version_info"`, and `"startup"` phases for provenance extraction. + +--- + +## 12. Environment Requirements (Local Testing) + +Validated local smoke configuration as of 2026-06-15: + +| Component | Version | +|---|---| +| IsaacSim | 6.0.0.1 (NOT 6.0.0.0 — `omni.physics.tensors.api` moved in 6.0.0.1) | +| isaacsim-extscache-physics | 6.0.0.1 | +| warp-lang | 1.12.0 (NOT 1.13.0 — `warp.context` removed, breaks omni.replicator.core) | +| mujoco-warp | 3.8.1 | +| Python | 3.12 | +| GPU | RTX 5090 (local) / RTX PRO 6000 target runners / L40S historical reference | + +**Installation after fresh `./isaaclab.sh -i --extra rl`:** + +```bash +pip install isaacsim==6.0.0.1 isaacsim-extscache-physics==6.0.0.1 +pip install warp-lang==1.12.0 +``` + +--- + +## 13. Alignment with OVPLC Testing Principles + +### 13.1 PR-gated runs vs. nightly-authoritative runs + +**Principle:** Authoritative runs SHOULD execute on a nightly schedule. +**Deviation:** We run on non-draft PRs and merge-queue candidates, and publish baselines +from protected-branch pushes. +**Defense:** The gate is intended to catch merge-time regressions before they land. Baseline +publication remains restricted to trusted protected-branch states, and matching prefers +compatible nearest-ancestor samples when a base SHA is available. + +### 13.2 Single FPS sample per run vs. N=10 iterations + +**Principle:** Minimum N=10 iterations per benchmark. +**Deviation:** Each CI run produces one FPS value (mean of 200 post-warmup frames) +**Defense:** Simulation steps ARE independent samples and this gate is meant to be light-weight. +The baseline window accumulates samples across runs, building MAD statistics over +the true run-to-run variance distribution. + +### 13.3 Memory tracking scope + +**Current state:** `gpu_diag.gpu_mem_used_mb` is captured and surfaced in `OracleResult` +as an informational field. No memory regression threshold yet. + +--- + +## 14. Bisect Engine (Future / TBD) + +`oracle.py` already populates `bisect_verdict` (`GOOD`/`BAD`/`SKIP`) on every run. +A bisect engine would consume it directly and execute a similar workflow to local_runner: + +``` +tools/bisect/ + env_resolver.py resolve_env(commit) → ResolvedEnv { isaaclab_root, python_launcher, … } + bisect_runner.py run_at_commit(env, task_id, backend) → (Path|None, SubprocessResult) + bisect_engine.py git bisect loop using OracleResult.bisect_verdict +``` diff --git a/tools/perf_regression_gate/gate_config.py b/tools/perf_regression_gate/gate_config.py new file mode 100644 index 000000000000..30f91cbe1a52 --- /dev/null +++ b/tools/perf_regression_gate/gate_config.py @@ -0,0 +1,84 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import json +from pathlib import Path + + +DEFAULT_K_WARN = 2.5 +DEFAULT_K_BLOCK = 4.0 +MIN_BASELINE_SAMPLES = 5 +MAX_BASELINE_SAMPLES = 20 +MIN_BLOCK_REGRESSION_PCT = 3.0 +BASELINE_PUSH_RETRIES = 3 +DEFAULT_RUNTIME_COMPATIBILITY = { + "contract_version": 1, + "always": [ + "software.isaacsim", + "software.isaaclab", + "software.torch", + "software.warp", + ], + "by_physics_backend": { + "physx": [ + "software.isaaclab_physx", + ], + "newton": [ + "software.isaaclab_newton", + "software.newton", + ], + }, + "by_render_backend": { + "newton_renderer": [ + "software.isaaclab_ov", + ], + "ovrtx_renderer": [ + "software.isaaclab_ov", + ], + "warp_renderer": [ + "software.isaaclab_ov", + ], + "rtx_renderer": [ + "software.isaaclab_ov", + ], + }, + "publish_only": [ + "hardware.gpu_compute_capability", + "hardware.gpu_total_memory_gb", + "gpu_diag.cuda_version", + "gpu_diag.nvidia_driver_version", + ], +} + + +def _merge_runtime_compatibility(raw: dict | None) -> dict: + policy = json.loads(json.dumps(DEFAULT_RUNTIME_COMPATIBILITY)) + if not raw: + return policy + for key, value in raw.items(): + if isinstance(value, dict) and isinstance(policy.get(key), dict): + policy[key].update(value) + else: + policy[key] = value + return policy + + +def load_gate_config(path: Path | str) -> dict: + config = { + "blocking": False, + "min_baseline_samples": MIN_BASELINE_SAMPLES, + "max_baseline_samples": MAX_BASELINE_SAMPLES, + "min_block_regression_pct": MIN_BLOCK_REGRESSION_PCT, + "baseline_push_retries": BASELINE_PUSH_RETRIES, + "runtime_compatibility": _merge_runtime_compatibility(None), + } + p = Path(path) + if p.exists(): + with p.open() as fh: + loaded = json.load(fh) + runtime_policy = _merge_runtime_compatibility(loaded.pop("runtime_compatibility", None)) + config.update(loaded) + config["runtime_compatibility"] = runtime_policy + return config diff --git a/tools/perf_regression_gate/gate_types.py b/tools/perf_regression_gate/gate_types.py new file mode 100644 index 000000000000..ed72b0b61046 --- /dev/null +++ b/tools/perf_regression_gate/gate_types.py @@ -0,0 +1,39 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared model states for the performance regression gate""" + +from enum import Enum + + +class OracleVerdict(str, Enum): + PASS = "PASS" + WARN = "WARN" + BLOCK = "BLOCK" + HARD_FAILURE = "HARD_FAILURE" + + +class BisectVerdict(str, Enum): + GOOD = "GOOD" + BAD = "BAD" + SKIP = "SKIP" + + +class FailurePhase(str, Enum): + IMPORT = "import" + INIT = "init" + RUNTIME = "runtime" + OOM = "oom" + HANG = "hang" + DRIVER = "driver" + CONFIG_MISMATCH = "config_mismatch" + + +class ThresholdSource(str, Enum): + NO_BASELINE = "no_baseline" + INSUFFICIENT_WINDOW = "insufficient_window" + ROLLING_WINDOW = "rolling_window" + HARD_FLOOR = "hard_floor" + NOT_APPLICABLE = "n/a" diff --git a/tools/perf_regression_gate/github_gate_context.py b/tools/perf_regression_gate/github_gate_context.py new file mode 100644 index 000000000000..e418cd121cb4 --- /dev/null +++ b/tools/perf_regression_gate/github_gate_context.py @@ -0,0 +1,196 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Resolve GitHub event context for the performance regression gate""" + +from __future__ import annotations + +import json +import os +import re +import urllib.request +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +_MIRRORED_PR_BRANCH_RE = re.compile(r"^pull-request/(\d+)$") +_BASELINE_PUBLISH_BRANCHES = frozenset({"main", "develop", "angehu/perf-gate-poc"}) +_BASELINE_PUBLISH_PREFIXES = ("release/",) + + +@dataclass(frozen=True) +class GateContext: + base_sha: str + target_branch: str + source_branch: str + allow_update: bool + trusted_source: str + event_kind: str + + def outputs(self) -> dict[str, str]: + return { + "base_sha": self.base_sha, + "target_branch": self.target_branch, + "source_branch": self.source_branch, + "allow_update": "true" if self.allow_update else "false", + "trusted_source": self.trusted_source, + "event_kind": self.event_kind, + } + + +def _strip_heads_ref(value: str) -> str: + prefix = "refs/heads/" + return value[len(prefix):] if value.startswith(prefix) else value + + +def _truthy(value: str | None) -> bool: + return str(value or "").strip().lower() in {"1", "true", "yes"} + + +def _event_value(env: Mapping[str, str], key: str) -> str: + try: + return env[key] + except KeyError as exc: + raise RuntimeError(f"Missing required GitHub environment variable: {key}") from exc + + +def _load_event(event_path: str | None) -> dict[str, Any]: + if not event_path: + return {} + with Path(event_path).open(encoding="utf-8") as fh: + event = json.load(fh) + return event if isinstance(event, dict) else {} + + +def _baseline_publish_branch(ref: str) -> bool: + branch = _strip_heads_ref(ref) + return branch in _BASELINE_PUBLISH_BRANCHES or branch.startswith(_BASELINE_PUBLISH_PREFIXES) + + +def fetch_pr_from_github(repo: str, pr_number: str, env: Mapping[str, str]) -> dict[str, Any]: + token = env.get("GITHUB_TOKEN") + if not token: + raise RuntimeError("GITHUB_TOKEN is required to resolve mirrored PR context") + api_url = env.get("GITHUB_API_URL", "https://api.github.com").rstrip("/") + request = urllib.request.Request( + f"{api_url}/repos/{repo}/pulls/{pr_number}", + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + with urllib.request.urlopen(request, timeout=30) as response: + pr = json.load(response) + if not isinstance(pr, dict): + raise RuntimeError(f"GitHub API returned non-object PR payload for PR {pr_number}") + return pr + + +def _pr_context(pr: Mapping[str, Any], *, allow_update: bool, trusted_source: str, event_kind: str) -> GateContext: + base = pr["base"] + head = pr["head"] + return GateContext( + base_sha=str(base["sha"]), + target_branch=str(base["ref"]), + source_branch=str(head["ref"]), + allow_update=allow_update, + trusted_source=trusted_source, + event_kind=event_kind, + ) + + +def resolve_gate_context( + env: Mapping[str, str] | None = None, + event: Mapping[str, Any] | None = None, + fetch_pr: Callable[[str, str, Mapping[str, str]], Mapping[str, Any]] | None = None, +) -> GateContext: + env = env or os.environ + event = event if event is not None else _load_event(env.get("GITHUB_EVENT_PATH")) + fetch_pr = fetch_pr or fetch_pr_from_github + + event_name = _event_value(env, "GITHUB_EVENT_NAME") + github_ref = _event_value(env, "GITHUB_REF") + ref_name = _event_value(env, "GITHUB_REF_NAME") + github_sha = _event_value(env, "GITHUB_SHA") + + if event_name == "pull_request": + return _pr_context( + event["pull_request"], + allow_update=False, + trusted_source="read_only", + event_kind="pull_request", + ) + + if event_name == "merge_group": + merge_group = event.get("merge_group") or {} + return GateContext( + base_sha=str(merge_group.get("base_sha") or github_sha), + target_branch=_strip_heads_ref(str(merge_group.get("base_ref") or ref_name)), + source_branch=_strip_heads_ref(str(merge_group.get("head_ref") or ref_name)), + allow_update=False, + trusted_source="read_only", + event_kind="merge_group", + ) + + if event_name == "push" and github_ref.startswith("refs/heads/pull-request/"): + match = _MIRRORED_PR_BRANCH_RE.fullmatch(ref_name) + if not match: + raise RuntimeError(f"Cannot parse mirrored PR number from GITHUB_REF_NAME={ref_name!r}") + repo = _event_value(env, "GITHUB_REPOSITORY") + pr = fetch_pr(repo, match.group(1), env) + allow_update = _truthy(env.get("ALLOW_MIRROR_UPDATE") or env.get("PERF_GATE_ALLOW_MIRROR_BASELINE_UPDATE")) + return _pr_context( + pr, + allow_update=allow_update, + trusted_source="mirrored_pr_push" if allow_update else "read_only", + event_kind="mirrored_pr_push", + ) + + if event_name == "push": + allow_update = _baseline_publish_branch(github_ref) + return GateContext( + base_sha=github_sha, + target_branch=ref_name, + source_branch=ref_name, + allow_update=allow_update, + trusted_source="protected_branch" if allow_update else "read_only", + event_kind="protected_push" if allow_update else "push", + ) + + return GateContext( + base_sha=github_sha, + target_branch=ref_name, + source_branch=ref_name, + allow_update=False, + trusted_source="read_only", + event_kind=event_name, + ) + + +def write_github_outputs(outputs: Mapping[str, str], output_path: str | None) -> None: + if not output_path: + return + with Path(output_path).open("a", encoding="utf-8") as fh: + for key, value in outputs.items(): + fh.write(f"{key}={value}\n") + + +def main() -> int: + context = resolve_gate_context() + outputs = context.outputs() + write_github_outputs(outputs, os.environ.get("GITHUB_OUTPUT")) + print( + "gate_context " + f"event={outputs['event_kind']} base={outputs['base_sha'][:12]} " + f"target={outputs['target_branch']} source={outputs['source_branch']} " + f"allow_update={outputs['allow_update']} trusted_source={outputs['trusted_source']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/perf_regression_gate/gpu_identity.py b/tools/perf_regression_gate/gpu_identity.py new file mode 100644 index 000000000000..786929e33999 --- /dev/null +++ b/tools/perf_regression_gate/gpu_identity.py @@ -0,0 +1,87 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Canonical GPU identity helpers for baseline bucket selection""" + +from __future__ import annotations + +import re +from typing import Any + +_UNKNOWN_GPU = "unknown_gpu" + + +def _clean(value: Any) -> str: + return str(value or "").strip() + + +def _slug(value: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_") + return slug or _UNKNOWN_GPU + + +def canonical_gpu_model(value: Any) -> str: + """Return the canonical baseline bucket key for a raw GPU model string.""" + raw = _clean(value) + if not raw: + return _UNKNOWN_GPU + normalized = re.sub(r"\s+", " ", raw.lower()).strip() + compact = re.sub(r"[^a-z0-9]+", "", normalized) + + if "l40s" in compact: + return "l40s" + if compact.endswith("l40") or compact == "nvidial40" or "teslal40" in compact: + return "l40" + if "rtxpro6000" in compact and "blackwell" in compact: + return "rtx_pro_6000_blackwell" + if "rtxpro6000" in compact: + return "rtx_pro_6000" + if "rtx6000adageneration" in compact or ("rtx6000" in compact and "ada" in compact): + return "rtx_6000_ada" + if compact in {"rtx6000", "nvidiartx6000"} or "rtx6000" in compact: + return "rtx_6000" + if "rtxa6000" in compact or "a6000" in compact: + return "rtx_a6000" + if "geforcertx5090" in compact: + return "geforce_rtx_5090" + if "geforcertx4090" in compact: + return "geforce_rtx_4090" + return _slug(raw) + + +def gpu_model_config_keys(value: Any) -> list[str]: + """Return candidate keys for reading existing GPU-keyed config dictionaries. + + `gpu_model` is canonical for new artifacts, but existing task floor configs may + still use legacy display keys such as `L40S`. + """ + raw = _clean(value) + canonical = canonical_gpu_model(raw) + keys: list[str] = [] + for key in (canonical, raw): + if key and key not in keys: + keys.append(key) + + legacy = { + "l40s": ["L40S"], + "l40": ["L40"], + "rtx_pro_6000_blackwell": ["RTX6000", "RTX PRO 6000", "RTX PRO 6000 Blackwell"], + "rtx_pro_6000": ["RTX6000", "RTX PRO 6000"], + "rtx_6000_ada": ["RTX6000", "RTX 6000 Ada"], + "rtx_6000": ["RTX6000", "RTX 6000"], + "rtx_a6000": ["RTXA6000", "RTX A6000"], + } + for key in legacy.get(canonical, []): + if key not in keys: + keys.append(key) + return keys + + +def normalize_gpu_fields(value: Any) -> dict[str, str]: + raw = _clean(value) + return { + "gpu_model": canonical_gpu_model(raw), + "gpu_model_raw": raw or _UNKNOWN_GPU, + } diff --git a/tools/perf_regression_gate/launch_config.py b/tools/perf_regression_gate/launch_config.py new file mode 100644 index 000000000000..4d640701f5c0 --- /dev/null +++ b/tools/perf_regression_gate/launch_config.py @@ -0,0 +1,181 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Launch configuration artifact helpers for the performance regression gate + +``launch_config.json`` is the durable contract between the matrix builder, the +benchmark runner, post-processing, and aggregate. Phase 2 treats this artifact +as the run intent instead of re-reading ``tasks.json``, so it can catch workflow +command bugs and self-hosted-runner handoff issues. +""" + +import hashlib +import json +from pathlib import Path +from typing import Any + +try: + from .backend_identity import make_backend_key, normalize_render_backend + from .gpu_identity import normalize_gpu_fields + from .task_config import TaskConfig +except ImportError: # pragma: no cover (for direct scripting execution/import) + from backend_identity import make_backend_key, normalize_render_backend + from gpu_identity import normalize_gpu_fields + from task_config import TaskConfig + +LAUNCH_CONFIG_SCHEMA_VERSION = 1 +BENCHMARK_CONTRACT_VERSION = 1 +DEFAULT_BASELINE_EPOCH = 1 +LAUNCH_CONFIG_FILENAME = "launch_config.json" + + +def _canonical_json(data: dict[str, Any]) -> str: + return json.dumps(data, sort_keys=True, separators=(",", ":")) + + +def stable_hash(data: dict[str, Any]) -> str: + return hashlib.sha256(_canonical_json(data).encode("utf-8")).hexdigest()[:16] + + +def workload_contract(config: dict[str, Any]) -> dict[str, Any]: + """Return the workload-defining subset used for launch_config_hash.""" + keys = ( + "task_id", + "backend_key", + "physics_backend", + "render_backend", + "preset", + "num_envs", + "num_frames", + "seed", + "excluded_frames_raw", + "camera_resolution", + "benchmark_backend", + "hydra_args", + ) + return {key: config.get(key) for key in keys} + + +def hydra_args_for_task(task: TaskConfig) -> list[str]: + """Return Hydra args used by the current local/CI benchmark launch path.""" + presets: list[str] = [] + if task.physics_backend == "newton": + presets.append("newton_mjwarp") + if task.render_backend: + presets.append(task.render_backend) + if task.render_backend == "newton_renderer": + presets.append("rgb") + return [f"presets={','.join(presets)}"] if presets else [] + + +def task_to_launch_config( + task: TaskConfig, + *, + fps_mean_floor: float, + gpu_model: str | None = None, + hydra_args: list[str] | None = None, + benchmark_backend: str = "json", +) -> dict[str, Any]: + """Build a serializable launch config for one task/backend job""" + gpu_fields = normalize_gpu_fields(gpu_model) + config: dict[str, Any] = { + "schema_version": LAUNCH_CONFIG_SCHEMA_VERSION, + "task_id": task.task_id, + "backend_key": make_backend_key(task.physics_backend, task.render_backend), + "physics_backend": task.physics_backend, + "render_backend": normalize_render_backend(task.render_backend), + "preset": task.preset, + "num_envs": task.num_envs, + "num_frames": task.num_frames, + "seed": task.seed, + "excluded_frames_raw": task.excluded_frames_raw, + "camera_resolution": list(task.camera_resolution) if task.camera_resolution else None, + "timeout_minutes": task.timeout_minutes, + "tags": list(task.tags), + "gpu_model": gpu_fields["gpu_model"], + "gpu_model_raw": gpu_fields["gpu_model_raw"], + "benchmark_backend": benchmark_backend, + "hydra_args": list(hydra_args or []), + "fps_mean_floor": float(fps_mean_floor), + "baseline_epoch": int(getattr(task, "baseline_epoch", DEFAULT_BASELINE_EPOCH)), + "benchmark_contract_version": BENCHMARK_CONTRACT_VERSION, + } + config["launch_config_hash"] = stable_hash(workload_contract(config)) + config["benchmark_contract_hash"] = stable_hash( + { + "benchmark_contract_version": config["benchmark_contract_version"], + "excluded_frames_raw": config["excluded_frames_raw"], + "benchmark_backend": config["benchmark_backend"], + } + ) + return config + + +def fallback_launch_config( + *, + task_id: str, + physics_backend: str, + render_backend: str | None, + backend_key: str, + timeout_s: float, + task: TaskConfig | None = None, +) -> dict[str, Any]: + """Build a launch config for legacy/manual Phase 2 calls without an artifact""" + normalized_backend_key = make_backend_key(physics_backend, render_backend) + gpu_fields = normalize_gpu_fields(None) + + if task is None: + config: dict[str, Any] = { + "schema_version": LAUNCH_CONFIG_SCHEMA_VERSION, + "task_id": task_id, + "backend_key": normalized_backend_key, + "physics_backend": physics_backend, + "render_backend": normalize_render_backend(render_backend), + "preset": "default", + "num_envs": 0, + "num_frames": 0, + "seed": None, + "excluded_frames_raw": [], + "camera_resolution": None, + "timeout_minutes": int(timeout_s / 60), + "tags": ["always"], + "gpu_model": gpu_fields["gpu_model"], + "gpu_model_raw": gpu_fields["gpu_model_raw"], + "benchmark_backend": "json", + "hydra_args": [], + "fps_mean_floor": 0.0, + "baseline_epoch": DEFAULT_BASELINE_EPOCH, + "benchmark_contract_version": BENCHMARK_CONTRACT_VERSION, + } + config["launch_config_hash"] = stable_hash(workload_contract(config)) + config["benchmark_contract_hash"] = stable_hash( + { + "benchmark_contract_version": config["benchmark_contract_version"], + "excluded_frames_raw": config["excluded_frames_raw"], + "benchmark_backend": config["benchmark_backend"], + } + ) + return config + + return task_to_launch_config( + task, + fps_mean_floor=(task.fps_mean_floor.get("L40S", {}) or {}).get(task.backend_key, 0.0), + hydra_args=[], + ) + + +def load_launch_config(artifact_dir: Path, explicit_path: Path | None = None) -> dict[str, Any] | None: + path = explicit_path or artifact_dir / LAUNCH_CONFIG_FILENAME + if not path.exists(): + return None + with path.open() as fh: + return json.load(fh) + + +def write_launch_config(artifact_dir: Path, config: dict[str, Any]) -> Path: + artifact_dir.mkdir(parents=True, exist_ok=True) + path = artifact_dir / LAUNCH_CONFIG_FILENAME + path.write_text(json.dumps(config, indent=2, sort_keys=True)) + return path diff --git a/tools/perf_regression_gate/local_runner.py b/tools/perf_regression_gate/local_runner.py new file mode 100644 index 000000000000..94a68d1ef3be --- /dev/null +++ b/tools/perf_regression_gate/local_runner.py @@ -0,0 +1,369 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Local end-to-end benchmark runner without Docker/Github actions + +Mirrors the structure of the three-phase CI workflow: + + Phase 1 bench tasks.json → matrix → benchmark_non_rl.py (per task) + Phase 2 post-bench build_bench_result.py (per task, plain Python) + Phase 3 aggregate aggregate.py + oracle → verdict table + +Usage:: + + # Preview the full task matrix without running anything + python3 tools/perf_regression_gate/local_runner.py --dry_run + + # Seed/build baselines (run a few times on stable main) + python3 tools/perf_regression_gate/local_runner.py --allow_baseline_update + + # Check a branch against baselines (no update) + python3 tools/perf_regression_gate/local_runner.py + + # Run only "always"-tagged tasks on L40S label + python3 tools/perf_regression_gate/local_runner.py --tags always --gpu_model L40S + +Preconditions: + IsaacSim must be accessible via ./isaaclab.sh -p. + Activate venv before running +""" + +import argparse +import subprocess +import sys +import time +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +_TOOLS_DIR = _MODULE_DIR.parent +_REPO_ROOT = _TOOLS_DIR.parent + +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + +from gpu_identity import canonical_gpu_model, gpu_model_config_keys # noqa: E402 +from launch_config import hydra_args_for_task, task_to_launch_config, write_launch_config # noqa: E402 +from task_config import TaskConfig, load_tasks # noqa: E402 + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Local end-to-end benchmark runner (no Docker required).", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument( + "--tags", + nargs="+", + default=["always"], + help="Run only tasks whose tag list overlaps this set (default: always)", + ) + p.add_argument( + "--gpu_model", + default=None, + help="GPU model label for baseline bucket (default: auto-detect with nvidia-smi)", + ) + p.add_argument( + "--artifacts_dir", + type=Path, + default=_MODULE_DIR / "artifacts", + help="Root directory for per-task benchmark artifacts (default: tools/perf_regression_gate/artifacts/)", + ) + p.add_argument( + "--baselines_dir", + type=Path, + default=_MODULE_DIR / "local_baselines", + help="Flat-file baseline directory (default: tools/perf_regression_gate/local_baselines/)", + ) + p.add_argument( + "--allow_baseline_update", + action="store_true", + help="Append this run's FPS to the baseline window (use when building baselines on stable main)", + ) + p.add_argument( + "--dry_run", + action="store_true", + help="Print the expanded task matrix and exit without running anything", + ) + p.add_argument( + "--skip_existing", + action="store_true", + help="Skip tasks whose perf_regression_gate_result.json already exists in --artifacts_dir", + ) + p.add_argument( + "--gate_config", + type=Path, + default=_MODULE_DIR / "gate_config.json", + help="Path to gate_config.json (controls advisory vs blocking mode)", + ) + return p.parse_args() + + +# --------------------------------------------------------------------------- +# Phase 1 helpers: matrix expansion and benchmark execution +# --------------------------------------------------------------------------- + + +def _detect_gpu_model() -> str: + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"], + capture_output=True, + text=True, + check=True, + ) + except Exception: + return "unknown-gpu" + for line in result.stdout.splitlines(): + gpu_model = line.strip() + if gpu_model: + return gpu_model + return "unknown-gpu" + + +def _print_matrix(tasks: list[TaskConfig], tags: list[str]) -> None: + print(f"\n{'=' * 68}") + print(f" Phase 1 — Task Matrix ({len(tasks)} entries, tags={tags})") + print(f"{'=' * 68}") + col = "{:<42} {:<14} {:>8} {:>8} {:>5}" + print(col.format("task_id", "backend_key", "num_envs", "frames", "tmin")) + print("-" * 68) + for t in tasks: + print(col.format(t.task_id, t.backend_key, t.num_envs, t.num_frames, t.timeout_minutes)) + print() + + +def _hydra_args(task: TaskConfig) -> list[str]: + return hydra_args_for_task(task) + + +def _fps_mean_floor(task: TaskConfig, gpu_model: str) -> float: + for key in gpu_model_config_keys(gpu_model): + value = task.fps_mean_floor.get(key, {}).get(task.backend_key) + if value is not None: + return float(value) + return 0.0 + + +def _isaaclab_cmd(bench_script: Path, task: TaskConfig, artifact_dir: Path) -> list[str]: + """Build the ./isaaclab.sh -p benchmark_non_rl.py command for one task/backend.""" + cmd = [ + str(_REPO_ROOT / "isaaclab.sh"), + "-p", + str(bench_script), + "--task", task.task_id, + "--num_envs", str(task.num_envs), + "--num_frames", str(task.num_frames), + "--benchmark_backend", "json", + "--output_path", str(artifact_dir), + ] + if task.seed is not None: + cmd.extend(["--seed", str(task.seed)]) + cmd.extend(_hydra_args(task)) + return cmd + + +def _run_benchmark(task: TaskConfig, artifact_dir: Path, bench_script: Path) -> tuple[int, float]: + """Run benchmark_non_rl.py for one task, streaming output live + writing to log. + + Returns (exit_code, wall_time_s). + """ + log_file = artifact_dir / "benchmark.log" + cmd = _isaaclab_cmd(bench_script, task, artifact_dir) + + print(f" cmd: {' '.join(cmd)}") + print(f" log: {log_file}") + print() + + start = time.monotonic() + with open(log_file, "w") as log_fh: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + assert proc.stdout is not None + for raw_line in proc.stdout: + line = raw_line.decode(errors="replace") + sys.stdout.write(line) + sys.stdout.flush() + log_fh.write(line) + proc.wait() + wall_time = time.monotonic() - start + return proc.returncode, wall_time + + +# --------------------------------------------------------------------------- +# Phase 2 helper: build_bench_result.py +# --------------------------------------------------------------------------- + + +def _run_build_bench_result( + task: TaskConfig, + artifact_dir: Path, + exit_code: int, + wall_time: float, + *, + was_retried: bool = False, + attempt: int = 1, + gate_config: Path | None = None, +) -> None: + """Run build_bench_result.py for one task (plain Python, no IsaacSim needed)""" + build_script = _MODULE_DIR / "build_bench_result.py" + cmd = [ + sys.executable, + str(build_script), + "--task_id", task.task_id, + "--physics_backend", task.physics_backend, + "--render_backend", task.render_backend or "", + "--artifact_dir", str(artifact_dir), + "--exit_code", str(exit_code), + "--wall_time_s", f"{wall_time:.1f}", + "--timeout_s", str(task.timeout_minutes * 60), + "--log_file", str(artifact_dir / "benchmark.log"), + "--launch_config", str(artifact_dir / "launch_config.json"), + "--attempt", str(attempt), + ] + if gate_config is not None: + cmd.extend(["--gate_config", str(gate_config)]) + if was_retried: + cmd.append("--was_retried") + sys.stdout.flush() + subprocess.run(cmd, check=True) + + +# --------------------------------------------------------------------------- +# Phase 3 helper: aggregate.py +# --------------------------------------------------------------------------- + + +def _run_aggregate(artifacts_dir: Path, baselines_dir: Path, gpu_model: str, + gate_config: Path, allow_baseline_update: bool) -> int: + """Run aggregate.py over all artifacts and return its exit code.""" + agg_script = _MODULE_DIR / "aggregate.py" + cmd = [ + sys.executable, + str(agg_script), + "--artifacts_dir", str(artifacts_dir), + "--gpu_model", gpu_model, + "--baselines_dir", str(baselines_dir), + "--gate_config", str(gate_config), + "--allow_baseline_update", "true" if allow_baseline_update else "false", + ] + sys.stdout.flush() + result = subprocess.run(cmd) + return result.returncode + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main() -> int: + args = _parse_args() + gpu_model_raw = args.gpu_model or _detect_gpu_model() + gpu_model = canonical_gpu_model(gpu_model_raw) + + # Expand matrix from tasks.json + all_tasks = load_tasks() + tag_set = frozenset(args.tags) + tasks = [t for t in all_tasks if tag_set.intersection(frozenset(t.tags))] + + if not tasks: + print(f"[local_runner] No tasks match tags {args.tags}.") + return 1 + + _print_matrix(tasks, args.tags) + + print(f"[local_runner] GPU model bucket: {gpu_model} (raw={gpu_model_raw})") + + if args.dry_run: + print("[local_runner] --dry_run: exiting without running benchmarks.") + return 0 + + bench_script = _REPO_ROOT / "scripts" / "benchmarks" / "benchmark_non_rl.py" + if not bench_script.exists(): + print(f"[local_runner] ERROR: benchmark script not found at {bench_script}") + return 1 + + # ----------------------------------------------------------------------- + # Phase 1 + 2: benchmark → build_bench_result, one task at a time + # ----------------------------------------------------------------------- + print(f"{'=' * 68}") + print(" Phase 1+2 — Benchmark + Post-process") + print(f"{'=' * 68}\n") + + for i, task in enumerate(tasks, 1): + artifact_dir = args.artifacts_dir / task.task_id / task.backend_key + artifact_dir.mkdir(parents=True, exist_ok=True) + + bench_result_path = artifact_dir / "perf_regression_gate_result.json" + if args.skip_existing and bench_result_path.exists(): + print(f"[{i}/{len(tasks)}] SKIP (perf_regression_gate_result.json exists): {task.task_id} / {task.backend_key}") + continue + + launch_config = task_to_launch_config( + task, + fps_mean_floor=_fps_mean_floor(task, gpu_model), + gpu_model=gpu_model_raw, + hydra_args=_hydra_args(task), + ) + write_launch_config(artifact_dir, launch_config) + + print(f"[{i}/{len(tasks)}] {task.task_id} / {task.backend_key}") + print(f" envs={task.num_envs} frames={task.num_frames} timeout={task.timeout_minutes}m") + + exit_code, wall_time = _run_benchmark(task, artifact_dir, bench_script) + + was_retried = False + if exit_code != 0: + print(f"\n[{i}/{len(tasks)}] first attempt failed (exit={exit_code}), retrying once...") + # Remove any partial perf output from the failed attempt so the retry is clean + stale = artifact_dir / "perf_regression_gate_info.json" + if stale.exists(): + stale.unlink() + exit_code, wall_time = _run_benchmark(task, artifact_dir, bench_script) + was_retried = True + + attempt = 2 if was_retried else 1 + print(f"\n[{i}/{len(tasks)}] exit={exit_code} wall={wall_time:.0f}s attempt={attempt} → build_bench_result") + _run_build_bench_result( + task, + artifact_dir, + exit_code, + wall_time, + was_retried=was_retried, + attempt=attempt, + gate_config=args.gate_config, + ) + print() + + # ----------------------------------------------------------------------- + # Phase 3: aggregate + oracle + # ----------------------------------------------------------------------- + print(f"\n{'=' * 68}") + print(" Phase 3 — Aggregate + Oracle Verdict") + print(f"{'=' * 68}\n") + + if args.allow_baseline_update: + print("[local_runner] --allow_baseline_update: PASS/WARN results will extend the baseline window.\n") + else: + print("[local_runner] Read-only baseline run (pass --allow_baseline_update to extend window).\n") + + rc = _run_aggregate( + args.artifacts_dir, + args.baselines_dir, + gpu_model, + args.gate_config, + args.allow_baseline_update, + ) + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_regression_gate/oracle.py b/tools/perf_regression_gate/oracle.py new file mode 100644 index 000000000000..5a92a27af111 --- /dev/null +++ b/tools/perf_regression_gate/oracle.py @@ -0,0 +1,250 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Oracle layer for the CI performance regression gate.""" + +import json +import statistics +from dataclasses import dataclass +from pathlib import Path + +try: + from .gate_config import DEFAULT_K_BLOCK, DEFAULT_K_WARN, MIN_BASELINE_SAMPLES, MIN_BLOCK_REGRESSION_PCT + from .gate_types import BisectVerdict, FailurePhase, OracleVerdict, ThresholdSource +except ImportError: # pragma: no cover (for direct scripting execution/import) + from gate_config import DEFAULT_K_BLOCK, DEFAULT_K_WARN, MIN_BASELINE_SAMPLES, MIN_BLOCK_REGRESSION_PCT + from gate_types import BisectVerdict, FailurePhase, OracleVerdict, ThresholdSource + + +@dataclass +class Baseline: + """Rolling-window statistics for one compatible benchmark history""" + + median_fps: float + mad_fps: float + k_warn: float = DEFAULT_K_WARN + k_block: float = DEFAULT_K_BLOCK + sample_count: int = 0 + source: str = "unknown" + total_sample_count: int | None = None + + +@dataclass +class OracleResult: + """Full verdict record produced by :func:`compare`""" + + verdict: OracleVerdict + bisect_verdict: str + failure_phase: str | None + measured_fps: float | None + baseline_fps: float | None + regression_pct: float | None + fps_median: float | None + fps_p5: float | None + fps_p95: float | None + gpu_mem_used_mb: float | None + startup_time_s: float | None + wall_time_s: float | None + was_retried: bool + task_id: str + backend: str + baseline_sample_count: int = 0 + baseline_source: str = "none" + threshold_source: str = ThresholdSource.NO_BASELINE.value + warn_threshold_fps: float | None = None + block_threshold_fps: float | None = None + hard_floor_fps: float | None = None + min_block_regression_pct: float = MIN_BLOCK_REGRESSION_PCT + note: str | None = None + + +_BISECT_BAD_PHASES: frozenset[str] = frozenset({FailurePhase.INIT.value, FailurePhase.RUNTIME.value}) + + +def _bisect_verdict(verdict: OracleVerdict, was_retried: bool, failure_phase: str | None) -> str: + """Compute the bisect-friendly label for a given verdict.""" + if verdict == OracleVerdict.PASS: + return BisectVerdict.SKIP.value if was_retried else BisectVerdict.GOOD.value + if verdict == OracleVerdict.WARN: + return BisectVerdict.SKIP.value + if verdict == OracleVerdict.BLOCK: + return BisectVerdict.BAD.value + if failure_phase in _BISECT_BAD_PHASES: + return BisectVerdict.BAD.value + return BisectVerdict.SKIP.value + + +def _percentile(sorted_data: list[float], p: float) -> float: + """Linear-interpolation percentile on a pre-sorted non-empty list.""" + n = len(sorted_data) + if n == 1: + return sorted_data[0] + idx = p / 100.0 * (n - 1) + lo = int(idx) + hi = min(lo + 1, n - 1) + return sorted_data[lo] + (sorted_data[hi] - sorted_data[lo]) * (idx - lo) + + +def apply_excluded_frames(fps_series: list[float], excluded_frames: frozenset[int]) -> list[float]: + """Return fps_series with frames at 0-based indices listed in excluded_frames removed""" + if not excluded_frames: + return list(fps_series) + return [fps for idx, fps in enumerate(fps_series) if idx not in excluded_frames] + + +def _hard_failure( + bench_result: dict, + failure_phase: str | None, + was_retried: bool, + gpu_mem_used_mb: float | None, + *, + note: str | None = None, +) -> OracleResult: + verdict = OracleVerdict.HARD_FAILURE + return OracleResult( + verdict=verdict, + bisect_verdict=_bisect_verdict(verdict, was_retried, failure_phase), + failure_phase=failure_phase, + measured_fps=None, + baseline_fps=None, + regression_pct=None, + fps_median=None, + fps_p5=None, + fps_p95=None, + gpu_mem_used_mb=gpu_mem_used_mb, + startup_time_s=bench_result.get("startup_time_s"), + wall_time_s=bench_result.get("wall_time_s"), + was_retried=was_retried, + task_id=bench_result["task_id"], + backend=bench_result.get("backend_key") or bench_result["backend"], + threshold_source=ThresholdSource.NOT_APPLICABLE.value, + note=note, + ) + + +def _extract_fps_series(perf_regression_gate_info: list[dict]) -> list[float]: + for phase in perf_regression_gate_info: + if phase.get("phase_name") == "runtime": + for measurement in phase.get("measurements", []): + if measurement.get("name", "").endswith("Step Frametimes"): + value = measurement.get("value", {}) + return list(value.get("Environment step effective FPS", [])) + break + return [] + + +def compare( + bench_result: dict, + baseline: "Baseline | None", + fps_mean_floor: float, + excluded_frames: "frozenset[int]", + artifact_dir: "Path", + *, + min_block_regression_pct: float = MIN_BLOCK_REGRESSION_PCT, +) -> OracleResult: + """Compare a benchmark result against its baseline and return an OracleResult.""" + task_id: str = bench_result["task_id"] + backend: str = bench_result.get("backend_key") or bench_result["backend"] + failure_phase: str | None = bench_result.get("failure_phase") + was_retried: bool = bool(bench_result.get("was_retried")) + startup_time_s: float | None = bench_result.get("startup_time_s") + wall_time_s: float | None = bench_result.get("wall_time_s") + gpu_mem_used_mb: float | None = (bench_result.get("gpu_diag") or {}).get("gpu_mem_used_mb") + + config_mismatch = bench_result.get("config_mismatch") + if config_mismatch or failure_phase == FailurePhase.CONFIG_MISMATCH.value: + return _hard_failure( + bench_result, + FailurePhase.CONFIG_MISMATCH.value, + was_retried, + gpu_mem_used_mb, + note=str(config_mismatch or "config_mismatch"), + ) + + if not bench_result.get("perf_regression_gate_info_present", False): + return _hard_failure(bench_result, failure_phase, was_retried, gpu_mem_used_mb) + + perf_info_path = Path(artifact_dir) / "perf_regression_gate_info.json" + with perf_info_path.open() as fh: + perf_info = json.load(fh) + + filtered = apply_excluded_frames(_extract_fps_series(perf_info), excluded_frames) + if not filtered: + return _hard_failure(bench_result, failure_phase, was_retried, gpu_mem_used_mb, note="empty_fps_series") + + mean_fps = statistics.mean(filtered) + sorted_filtered = sorted(filtered) + fps_median = _percentile(sorted_filtered, 50.0) + fps_p5 = _percentile(sorted_filtered, 5.0) + fps_p95 = _percentile(sorted_filtered, 95.0) + + baseline_fps = baseline.median_fps if baseline is not None else None + baseline_sample_count = baseline.sample_count if baseline is not None else 0 + baseline_source = baseline.source if baseline is not None else "none" + regression_pct = None + if baseline_fps: + regression_pct = ((mean_fps - baseline_fps) / baseline_fps) * 100.0 + + hard_floor_fps = fps_mean_floor if fps_mean_floor > 0 else None + threshold_source = ThresholdSource.NO_BASELINE.value + warn_threshold = None + block_threshold = None + note = None + + if fps_mean_floor > 0 and mean_fps < fps_mean_floor: + verdict = OracleVerdict.BLOCK + threshold_source = ThresholdSource.HARD_FLOOR.value + note = "below_hard_floor" + elif baseline is None: + verdict = OracleVerdict.WARN + note = "no_baseline" + elif baseline.sample_count < MIN_BASELINE_SAMPLES: + verdict = OracleVerdict.WARN + threshold_source = ThresholdSource.INSUFFICIENT_WINDOW.value + note = f"insufficient_baseline(n={baseline.sample_count},min={MIN_BASELINE_SAMPLES})" + else: + threshold_source = ThresholdSource.ROLLING_WINDOW.value + block_threshold = baseline.median_fps - baseline.k_block * baseline.mad_fps + warn_threshold = baseline.median_fps - baseline.k_warn * baseline.mad_fps + if mean_fps < block_threshold: + if regression_pct is None or regression_pct <= -float(min_block_regression_pct): + verdict = OracleVerdict.BLOCK + else: + verdict = OracleVerdict.WARN + note = "below_mad_block_but_inside_regression_floor" + elif mean_fps < warn_threshold: + verdict = OracleVerdict.WARN + else: + verdict = OracleVerdict.PASS + + if verdict == OracleVerdict.PASS and was_retried: + verdict = OracleVerdict.WARN + note = note or "was_retried" + + return OracleResult( + verdict=verdict, + bisect_verdict=_bisect_verdict(verdict, was_retried, failure_phase), + failure_phase=failure_phase, + measured_fps=mean_fps, + baseline_fps=baseline_fps, + regression_pct=regression_pct, + fps_median=fps_median, + fps_p5=fps_p5, + fps_p95=fps_p95, + gpu_mem_used_mb=gpu_mem_used_mb, + startup_time_s=startup_time_s, + wall_time_s=wall_time_s, + was_retried=was_retried, + task_id=task_id, + backend=backend, + baseline_sample_count=baseline_sample_count, + baseline_source=baseline_source, + threshold_source=threshold_source, + warn_threshold_fps=warn_threshold, + block_threshold_fps=block_threshold, + hard_floor_fps=hard_floor_fps, + min_block_regression_pct=float(min_block_regression_pct), + note=note, + ) diff --git a/tools/perf_regression_gate/runtime_contract.py b/tools/perf_regression_gate/runtime_contract.py new file mode 100644 index 000000000000..5c2e02884a85 --- /dev/null +++ b/tools/perf_regression_gate/runtime_contract.py @@ -0,0 +1,99 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Runtime compatibility contract construction for baseline matching""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from typing import Any + +try: + from .backend_identity import BackendIdentity +except ImportError: # pragma: no cover - supports direct script imports + from backend_identity import BackendIdentity + + +def _canonical_json(data: dict[str, Any]) -> str: + return json.dumps(data, sort_keys=True, separators=(",", ":")) + + +def stable_hash(data: dict[str, Any]) -> str: + return hashlib.sha256(_canonical_json(data).encode("utf-8")).hexdigest()[:16] + + +def _get_path(data: Mapping[str, Any], dotted_path: str) -> Any: + cur: Any = data + for part in dotted_path.split("."): + if not isinstance(cur, Mapping) or part not in cur: + return None + cur = cur[part] + return cur + + +def _field_list(policy: Mapping[str, Any], backend: BackendIdentity) -> list[str]: + fields: list[str] = [] + + def add_many(values: Any) -> None: + for value in values or []: + field = str(value) + if field not in fields: + fields.append(field) + + add_many(policy.get("always")) + by_physics = policy.get("by_physics_backend") or {} + if isinstance(by_physics, Mapping): + add_many(by_physics.get(backend.physics_backend)) + by_render = policy.get("by_render_backend") or {} + if isinstance(by_render, Mapping) and backend.render_backend: + add_many(by_render.get(backend.render_backend)) + return fields + + +def runtime_source(provenance: dict[str, Any] | None, gpu_diag: dict[str, Any] | None) -> dict[str, Any]: + provenance = provenance or {} + return { + "software": provenance.get("software") or {}, + "hardware": provenance.get("hardware") or {}, + "gpu_diag": gpu_diag or {}, + } + + +def build_runtime_contract( + *, + provenance: dict[str, Any] | None, + gpu_diag: dict[str, Any] | None, + backend: BackendIdentity, + policy: Mapping[str, Any], +) -> tuple[dict[str, Any], str]: + """Build the compatibility contract and hash used for baseline matching.""" + source = runtime_source(provenance, gpu_diag) + selected = {field: _get_path(source, field) for field in _field_list(policy, backend)} + contract = { + "runtime_contract_version": int(policy.get("contract_version", 1)), + "fields": selected, + } + return contract, stable_hash(contract) + + +def build_runtime_publish_info( + *, + provenance: dict[str, Any] | None, + gpu_diag: dict[str, Any] | None, + policy: Mapping[str, Any], +) -> dict[str, Any]: + """Return debug/runtime fields published for humans but not used for matching.""" + source = runtime_source(provenance, gpu_diag) + publish_only = { + str(field): _get_path(source, str(field)) + for field in policy.get("publish_only", []) + } + publish_only = {key: value for key, value in publish_only.items() if value is not None} + return { + "software": source["software"], + "publish_only": publish_only, + } diff --git a/tools/perf_regression_gate/task_config.py b/tools/perf_regression_gate/task_config.py new file mode 100644 index 000000000000..35b616c0d93f --- /dev/null +++ b/tools/perf_regression_gate/task_config.py @@ -0,0 +1,187 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import json +from dataclasses import dataclass, field +from pathlib import Path + +try: + from .backend_identity import make_backend_key, normalize_physics_backend, normalize_render_backend +except ImportError: # pragma: no cover - supports direct script imports + from backend_identity import make_backend_key, normalize_physics_backend, normalize_render_backend + +_DEFAULT_TASKS_JSON = Path(__file__).parent / "tasks.json" + +# Maps backend name to list of cache identifiers that CI needs to pull; absent key defaults to no caches +_BACKEND_CACHES: dict[str, list[str]] = { + "newton": ["mjwarp_jit"], +} + + +def caches_for_backend(backend: str) -> list[str]: + """Return cache identifiers required before benchmarking with a given physics backend. + + The returned identifiers are consumed by the CI cache-pull step to locate and + restore named cache artifacts by name or pattern. An empty list means no + pre-run cache restoration is needed. + + Currently defined identifiers: + ``"mjwarp_jit"``: Newton MJWarp JIT compilation cache. + + Args: + backend: Backend name (e.g. ``"newton"``, ``"physx"``). + + Returns: + List of cache identifier strings. + """ + return list(_BACKEND_CACHES.get(backend, [])) + + +@dataclass +class TaskConfig: + """Configuration for a single benchmark task and backend combination.""" + + task_id: str + physics_backend: str + render_backend: str | None + preset: str + num_envs: int + num_frames: int + excluded_frames_raw: list[int | list[int]] + camera_resolution: tuple[int, int] | None + timeout_minutes: int + fps_mean_floor: dict + caches: list[str] + tags: list[str] = field(default_factory=lambda: ["always"]) + task_type: str = "benchmark" + runs_on: str = "gpu-l40s" + seed: int | None = None + baseline_epoch: int = 1 + + @property + def backend_key(self) -> str: + """Composite key identifying the backend combination. + + Returns f"{physics_backend}_{render_backend}" when render_backend is set, + otherwise returns physics_backend. + """ + return make_backend_key(self.physics_backend, self.render_backend) + + @property + def excluded_frames(self) -> frozenset[int]: + """Expand raw excluded_frames entries (single index or inclusive range) to a + frozenset of integer frame indices. + """ + indices: set[int] = set() + for entry in self.excluded_frames_raw: + if isinstance(entry, list): + if len(entry) != 2: + raise ValueError(f"excluded_frames range entry must have exactly 2 elements, got {entry!r}") + start, end = entry[0], entry[1] + if start > end: + raise ValueError(f"excluded_frames range start must be <= end, got [{start}, {end}]") + indices.update(range(start, end + 1)) + else: + indices.add(int(entry)) + return frozenset(indices) + + +def _load_tasks_json(path: Path) -> tuple[dict, list[dict]]: + with open(path) as f: + raw_data = json.load(f) + + if isinstance(raw_data, dict): + defaults = raw_data.get("defaults", {}) + raw_list = raw_data.get("tasks", []) + if not isinstance(raw_list, list): + raise TypeError(f"'tasks' field in {path} must be a list") + elif isinstance(raw_data, list): + defaults = {} + raw_list = raw_data + else: + raise TypeError(f"{path} must contain a JSON list or an object with a top-level 'tasks' list") + + if not isinstance(defaults, dict): + raise TypeError(f"'defaults' field in {path} must be an object") + + return defaults, raw_list + + +def load_tasks(tasks_json_path: Path | str | None = None) -> list[TaskConfig]: + """Load all benchmark tasks from tasks.json, producing a TaskConfig for each backend combination. + + Args: + tasks_json_path: Path to tasks.json. Defaults to the tasks.json next to this module. + + Returns: + List of TaskConfig objects, one per (task_id, backend) combination. + """ + path = Path(tasks_json_path) if tasks_json_path is not None else _DEFAULT_TASKS_JSON + defaults, raw_list = _load_tasks_json(path) + + tasks: list[TaskConfig] = [] + for raw in raw_list: + if not isinstance(raw, dict): + raise TypeError(f"task entry in {path} must be an object") + merged = {**defaults, **raw} + + camera_raw = merged.get("camera_resolution") + camera_resolution: tuple[int, int] | None = ( + tuple(camera_raw) if camera_raw is not None else None # type: ignore[assignment] + ) + fps_mean_floor: dict = merged.get("fps_mean_floor", {}) + backends: list[dict] = merged.get("backends", []) + + for backend_entry in backends: + physics = normalize_physics_backend(backend_entry["physics"]) + if physics is None: + raise ValueError(f"backend entry in {path} must define a non-default physics backend") + render = normalize_render_backend(backend_entry.get("render")) + tasks.append( + TaskConfig( + task_id=merged["task_id"], + physics_backend=physics, + render_backend=render, + preset=merged["preset"], + num_envs=merged["num_envs"], + num_frames=merged["num_frames"], + excluded_frames_raw=merged["excluded_frames"], + camera_resolution=camera_resolution, + timeout_minutes=int(merged["timeout_minutes"]), + fps_mean_floor=fps_mean_floor, + caches=caches_for_backend(physics), + tags=merged["tags"], + task_type=merged["type"], + runs_on=merged["runs_on"], + seed=merged.get("seed"), + baseline_epoch=int(merged.get("baseline_epoch", 1)), + ) + ) + return tasks + + +def get_task( + task_id: str, + backend_key: str, + tasks_json_path: Path | str | None = None, +) -> TaskConfig: + """Return the TaskConfig for the given task_id and backend_key combination. + + Args: + task_id: The task identifier to look up. + backend_key: The backend key (e.g. "physx", "newton", "physx_rtx"). + tasks_json_path: Optional path to tasks.json. + + Returns: + The matching TaskConfig. + + Raises: + KeyError: If no task with the given (task_id, backend_key) exists. + """ + tasks = load_tasks(tasks_json_path) + for task in tasks: + if task.task_id == task_id and task.backend_key == backend_key: + return task + raise KeyError(f"Task not found: task_id={task_id!r} backend_key={backend_key!r}") diff --git a/tools/perf_regression_gate/tasks.json b/tools/perf_regression_gate/tasks.json new file mode 100644 index 000000000000..6a4e1a8776fe --- /dev/null +++ b/tools/perf_regression_gate/tasks.json @@ -0,0 +1,77 @@ +{ + "defaults": { + "type": "benchmark", + "runs_on": "gpu-l40s", + "preset": "default", + "seed": 42, + "num_envs": 512, + "num_frames": 300, + "excluded_frames": [[0, 100]], + "camera_resolution": null, + "timeout_minutes": 10, + "tags": ["always"] + }, + "tasks": [ + { + "task_id": "Isaac-Cartpole-Direct-v0", + "num_envs": 4096, + "backends": [ + {"physics": "physx"}, + {"physics": "newton"} + ], + "fps_mean_floor": { + "L40S": { + "physx": 100.0, + "newton": 0.0 + } + } + }, + { + "task_id": "Isaac-Factory-GearMesh-Direct-v0", + "timeout_minutes": 15, + "backends": [ + {"physics": "physx"} + ], + "fps_mean_floor": { + "L40S": { + "physx": 30.0 + } + } + }, + { + "task_id": "Isaac-Repose-Cube-Shadow-Vision-Direct-v0", + "timeout_minutes": 20, + "tags": ["camera"], + "backends": [ + {"physics": "physx"}, + {"physics": "physx", "render": "newton_renderer"}, + {"physics": "newton"}, + {"physics": "newton", "render": "newton_renderer"}, + {"physics": "newton", "render": "ovrtx_renderer"} + ], + "fps_mean_floor": { + "L40S": { + "physx": 20.0, + "physx_newton_renderer": 0.0, + "newton": 0.0, + "newton_newton_renderer": 0.0, + "newton_ovrtx_renderer": 0.0 + } + } + }, + { + "task_id": "Isaac-Velocity-Flat-G1-v0", + "timeout_minutes": 12, + "backends": [ + {"physics": "physx"}, + {"physics": "newton"} + ], + "fps_mean_floor": { + "L40S": { + "physx": 40.0, + "newton": 0.0 + } + } + } + ] +} diff --git a/tools/perf_regression_gate/tasks_to_ci_matrix.py b/tools/perf_regression_gate/tasks_to_ci_matrix.py new file mode 100644 index 000000000000..dab6390ccfdd --- /dev/null +++ b/tools/perf_regression_gate/tasks_to_ci_matrix.py @@ -0,0 +1,40 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Convert tasks.json into the GitHub Actions bench matrix JSON + +Prints a JSON array to stdout, one object per (task_id, backend) combination, +containing the fields consumed by the ``bench`` job matrix in perf-regression-gate.yaml. + +Usage:: + + python3 tools/perf_regression_gate/tasks_to_ci_matrix.py +""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from launch_config import hydra_args_for_task # noqa: E402 +from task_config import load_tasks # noqa: E402 + +tasks = load_tasks() +rows = [] +for task in tasks: + rows.append({ + "task_id": task.task_id, + "physics_backend": task.physics_backend, + "render_backend": task.render_backend or "", + "num_envs": task.num_envs, + "num_frames": task.num_frames, + "seed": task.seed if task.seed is not None else "", + "hydra_args": " ".join(hydra_args_for_task(task)), + "bench_timeout_s": task.timeout_minutes * 60, + "job_timeout_minutes": max(30, task.timeout_minutes + 15), + }) + +print(json.dumps(rows)) diff --git a/tools/perf_regression_gate/write_launch_config.py b/tools/perf_regression_gate/write_launch_config.py new file mode 100644 index 000000000000..a996b9ce9779 --- /dev/null +++ b/tools/perf_regression_gate/write_launch_config.py @@ -0,0 +1,55 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Write launch_config.json for a perf-gate task/backend job""" + +import argparse +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) + +from backend_identity import make_backend_key, normalize_render_backend +from gpu_identity import gpu_model_config_keys +from launch_config import hydra_args_for_task, task_to_launch_config, write_launch_config +from task_config import get_task + + +def _parse_args(): + parser = argparse.ArgumentParser(description="Write launch_config.json for a perf-gate benchmark job") + parser.add_argument("--task_id", required=True) + parser.add_argument("--physics_backend", required=True) + parser.add_argument("--render_backend", default="") + parser.add_argument("--gpu_model", default="L40S") + parser.add_argument("--artifact_dir", required=True, type=Path) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + render_backend = normalize_render_backend(args.render_backend) + backend_key = make_backend_key(args.physics_backend, render_backend) + task = get_task(args.task_id, backend_key) + fps_mean_floor = 0.0 + for gpu_key in gpu_model_config_keys(args.gpu_model): + configured_floor = task.fps_mean_floor.get(gpu_key, {}).get(task.backend_key) + if configured_floor is not None: + fps_mean_floor = float(configured_floor) + break + config = task_to_launch_config( + task, + fps_mean_floor=fps_mean_floor, + gpu_model=args.gpu_model, + hydra_args=hydra_args_for_task(task), + ) + path = write_launch_config(args.artifact_dir, config) + print(path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/subprocess_runner.py b/tools/subprocess_runner.py new file mode 100644 index 000000000000..6075eff42844 --- /dev/null +++ b/tools/subprocess_runner.py @@ -0,0 +1,326 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import contextlib +import os +import select +import signal +import subprocess +import sys +import time + +from perf_regression_gate.gate_types import FailurePhase + + +def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, report_file=""): + """Run a command with timeout and capture all output while streaming in real-time. + + Args: + cmd: Command to execute. + timeout: Maximum wall-clock seconds before the process is killed. + env: Environment variables for the subprocess. + startup_deadline: If > 0, the process is killed early when neither + ``AppLauncher initialization complete`` (stderr) nor ``collected`` + (stdout) appears within this many seconds. + report_file: Path to the JUnit XML report file. When set, the process + is given only :data:`SHUTDOWN_GRACE_PERIOD` seconds to exit after + the file appears on disk. + + Returns: + Tuple of ``(returncode, stdout_bytes, stderr_bytes, kill_reason, + wall_time, pre_kill_diag)``. *kill_reason* is ``""`` for normal exits, + ``"timeout"`` for hard timeouts, ``"startup_hang"`` when the process + did not reach pytest collection in time, or ``"shutdown_hang"`` when + the test completed but the process hung during shutdown. + """ + # Import here to avoid circular dependency; SHUTDOWN_GRACE_PERIOD is defined in conftest. + # We define a local default that matches conftest's constant. + _SHUTDOWN_GRACE_PERIOD = 30 + + stdout_data = b"" + stderr_data = b"" + process = None + + try: + # Each test gets its own session so orphaned Kit/Isaac Sim child + # processes cannot send SIGHUP to the next test's process group. + process = subprocess.Popen( + cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + universal_newlines=False, + start_new_session=True, + ) + pgid = os.getpgid(process.pid) + + stdout_fd = process.stdout.fileno() + stderr_fd = process.stderr.fileno() + + try: + import fcntl + + for fd in [stdout_fd, stderr_fd]: + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + except ImportError: + pass + + start_time = time.time() + startup_done = startup_deadline <= 0 + shutdown_deadline = 0.0 + + while process.poll() is None: + elapsed = time.time() - start_time + + if not startup_done: + if b"AppLauncher initialization complete" in stderr_data or b"collected " in stdout_data: + startup_done = True + + if report_file and not shutdown_deadline and os.path.exists(report_file): + shutdown_deadline = time.time() + _SHUTDOWN_GRACE_PERIOD + + kill_reason = None + if not startup_done and elapsed > startup_deadline: + kill_reason = "startup_hang" + elif shutdown_deadline and time.time() > shutdown_deadline: + kill_reason = "shutdown_hang" + elif elapsed > timeout: + kill_reason = "timeout" + + if kill_reason: + pre_kill_diag = _capture_system_diagnostics() + + # Kill the entire process group (test + any Kit children). + try: + os.killpg(pgid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, OSError): + process.kill() + try: + remaining_stdout, remaining_stderr = process.communicate(timeout=5) + stdout_data += remaining_stdout + stderr_data += remaining_stderr + except subprocess.TimeoutExpired: + pass + wall_time = time.time() - start_time + return -1, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag + + try: + ready_fds, _, _ = select.select([stdout_fd, stderr_fd], [], [], 0.1) + + for fd in ready_fds: + with contextlib.suppress(OSError): + if fd == stdout_fd: + chunk = process.stdout.read(1024) + if chunk: + stdout_data += chunk + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + elif fd == stderr_fd: + chunk = process.stderr.read(1024) + if chunk: + stderr_data += chunk + sys.stderr.buffer.write(chunk) + sys.stderr.buffer.flush() + except OSError: + time.sleep(0.1) + continue + + # Drain any output the process wrote before or just after exiting. + try: + remaining_stdout, remaining_stderr = process.communicate(timeout=10) + stdout_data += remaining_stdout + stderr_data += remaining_stderr + except Exception: + pass + + # Kill any orphaned child processes (Kit, Isaac Sim) left by the test. + try: + os.killpg(pgid, signal.SIGKILL) + time.sleep(1) + except (ProcessLookupError, PermissionError, OSError): + pass + + wall_time = time.time() - start_time + return process.returncode, stdout_data, stderr_data, "", wall_time, "" + + except Exception as e: + if process is not None and process.poll() is None: + process.kill() + with contextlib.suppress(Exception): + rem_out, rem_err = process.communicate(timeout=5) + stdout_data += rem_out + stderr_data += rem_err + stdout_data += f"\n[capture error: {e}]\n".encode() + return -1, stdout_data, stderr_data, "", 0.0, "" + + +def _capture_system_diagnostics(): + """Capture system diagnostics (GPU, memory, processes) for crash investigation. + + All errors are caught and reported inline so this never raises. + """ + sections = [] + + try: + r = subprocess.run(["nvidia-smi"], capture_output=True, text=True, timeout=10) + if r.stdout: + sections.append(f"--- nvidia-smi ---\n{r.stdout.strip()}") + except Exception as e: + sections.append(f"--- nvidia-smi --- FAILED: {e}") + + try: + with open("/proc/meminfo") as f: + lines = f.readlines() + keys = ("MemTotal", "MemFree", "MemAvailable", "Committed_AS", "SwapTotal", "SwapFree") + relevant = [line.strip() for line in lines if any(line.startswith(k) for k in keys)] + if relevant: + sections.append("--- /proc/meminfo ---\n" + "\n".join(relevant)) + except Exception as e: + sections.append(f"--- /proc/meminfo --- FAILED: {e}") + + cgroup_lines = [] + for path in ( + "/sys/fs/cgroup/memory.current", + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory.events", + "/sys/fs/cgroup/memory/memory.usage_in_bytes", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + "/sys/fs/cgroup/memory/memory.oom_control", + ): + try: + with open(path) as f: + cgroup_lines.append(f"{path}: {f.read().strip()}") + except FileNotFoundError: + pass + except Exception as e: + cgroup_lines.append(f"{path}: FAILED ({e})") + if cgroup_lines: + sections.append("--- cgroup memory ---\n" + "\n".join(cgroup_lines)) + + try: + r = subprocess.run(["ps", "auxf"], capture_output=True, text=True, timeout=5) + if r.stdout: + sections.append(f"--- process tree (ps auxf) ---\n{r.stdout.strip()}") + except Exception as e: + sections.append(f"--- process tree --- FAILED: {e}") + + try: + r = subprocess.run(["dmesg", "-T"], capture_output=True, text=True, timeout=5) + if r.stdout: + lines = r.stdout.strip().split("\n") + sections.append("--- dmesg (last 30 lines) ---\n" + "\n".join(lines[-30:])) + except Exception: + pass + + return "\n\n".join(sections) + + +def classify_failure_phase( + stdout: str, stderr: str, exit_code: int, wall_time_s: float, timeout_s: float +) -> str | None: + """Classify the failure phase of a benchmark run. + + Priority order (highest to lowest): + 1. oom: exit_code == 137 or "oom-kill" in stderr + 2. hang: wall_time_s >= timeout_s * 0.95 + 3. import: "Traceback" in stdout/stderr AND no "AppLauncher" in stdout + 4. driver: "CudaError" or "CUDA_ERROR_" in stdout/stderr (case-sensitive) + 5. init: "AppLauncher initialization complete" in stdout but no "Step Frametimes" in stdout + 6. runtime: exit_code != 0 and "Step Frametimes" in stdout + 7. null: exit_code == 0 + + Args: + stdout: Captured standard output as a string. + stderr: Captured standard error as a string. + exit_code: Process exit code. + wall_time_s: Measured wall-clock time in seconds. + timeout_s: Configured timeout in seconds. + + Returns: + A failure phase string or None for a clean exit. + """ + combined = stdout + stderr + + # 1. OOM + if exit_code == 137 or "oom-kill" in stderr: + return FailurePhase.OOM.value + + # 2. Hang + if wall_time_s >= timeout_s * 0.95: + return FailurePhase.HANG.value + + # 3. Import error + if "Traceback" in combined and "AppLauncher" not in stdout: + return FailurePhase.IMPORT.value + + # 4. Driver error + if "CudaError" in combined or "CUDA_ERROR_" in combined: + return FailurePhase.DRIVER.value + + # 5. Init failure + if exit_code != 0 and "AppLauncher initialization complete" in stdout and "Step Frametimes" not in stdout: + return FailurePhase.INIT.value + + # 6. Runtime failure (partial run then crash) + if exit_code != 0 and "Step Frametimes" in stdout: + return FailurePhase.RUNTIME.value + + # 7. Clean exit + return None + + +def run_benchmark(cmd: list, timeout_s: float) -> dict: + """Run a benchmark command and return structured result. + + Args: + cmd: Command list to execute. + timeout_s: Hard timeout in seconds. + + Returns: + Dict with keys: + - exit_code (int): Process exit code. + - stdout_tail (str): Last 2000 characters of combined stdout. + - wall_time_s (float): Measured wall-clock time in seconds. + - startup_time_s (float): Startup time in seconds (0.0 stub for POC). + - failure_phase (str | None): Classified failure phase. + """ + env = os.environ.copy() + + returncode, stdout_bytes, stderr_bytes, kill_reason, wall_time, _ = capture_test_output_with_timeout( + cmd, + timeout=timeout_s, + env=env, + startup_deadline=0, + report_file="", + ) + + stdout_str = stdout_bytes.decode("utf-8", errors="replace") + stderr_str = stderr_bytes.decode("utf-8", errors="replace") + + # Use kill_reason to override exit_code for hang detection + effective_exit_code = returncode + if kill_reason in ("timeout", "startup_hang"): + effective_exit_code = -1 + + failure_phase = classify_failure_phase( + stdout=stdout_str, + stderr=stderr_str, + exit_code=effective_exit_code, + wall_time_s=wall_time, + timeout_s=timeout_s, + ) + + combined_output = stdout_str + stdout_tail = combined_output[-2000:] if len(combined_output) > 2000 else combined_output + + return { + "exit_code": returncode, + "stdout_tail": stdout_tail, + "wall_time_s": wall_time, + "startup_time_s": 0.0, + "failure_phase": failure_phase, + }