From 4ca1bad064ec6ecd29224cce8b1ce32b3014f5df Mon Sep 17 00:00:00 2001 From: Angelina Hu Date: Thu, 16 Jul 2026 14:30:07 -0700 Subject: [PATCH 1/4] Validate physics and render backend names in perf-smoke tasks Reject an unknown or misspelled physics/render backend in tasks.json at load time instead of silently producing a backend_key that no threshold targets (which disables the FPS gate) or emitting a bad Hydra presets= token that crashes the GPU benchmark job at launch. --- tools/perf_smoke_test/backend_identity.py | 46 +++++++++++++++++++++++ tools/perf_smoke_test/task_config.py | 19 ++++++++-- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/tools/perf_smoke_test/backend_identity.py b/tools/perf_smoke_test/backend_identity.py index c09b2ef4d8ef..814dffeaa22f 100644 --- a/tools/perf_smoke_test/backend_identity.py +++ b/tools/perf_smoke_test/backend_identity.py @@ -73,6 +73,52 @@ def normalize_render_backend(value: Any) -> str | None: return lowered +def validate_physics_backend(value: str, *, source: str | None = None) -> str: + """Return ``value`` when it is a known physics backend, else raise. + + Guards against a typo'd or unsupported ``physics`` in ``tasks.json`` (e.g. + ``"phsyx"``) silently flowing through :func:`normalize_physics_backend` into a + ``backend_key`` that no threshold ever targets, which would disable the FPS gate + while still writing a stray baseline bucket. + + Args: + value: A normalized physics backend name (see :func:`normalize_physics_backend`). + source: Optional origin (e.g. a file path) included in the error message. + + Raises: + ValueError: If ``value`` is not one of the known physics backends. + """ + if value not in _KNOWN_PHYSICS_BACKENDS: + where = f" in {source}" if source else "" + raise ValueError( + f"unknown physics backend {value!r}{where}; expected one of {list(_KNOWN_PHYSICS_BACKENDS)}" + ) + return value + + +def validate_render_backend(value: str | None, *, source: str | None = None) -> str | None: + """Return ``value`` when it is unset or a known render backend, else raise. + + Guards against a typo'd ``render`` in ``tasks.json`` (e.g. ``"newton_rendrer"``) + that would otherwise be emitted as a Hydra ``presets=`` token and crash the + multi-minute GPU benchmark job at launch. + + Args: + value: A normalized render backend name, or ``None`` when unset (see + :func:`normalize_render_backend`). + source: Optional origin (e.g. a file path) included in the error message. + + Raises: + ValueError: If ``value`` is a non-empty name that is not a known render backend. + """ + if value is not None and value not in _RENDER_PRESET_TOKENS: + where = f" in {source}" if source else "" + raise ValueError( + f"unknown render backend {value!r}{where}; expected one of {sorted(_RENDER_PRESET_TOKENS)}" + ) + return value + + def make_backend_key(physics_backend: str, render_backend: str | None = None) -> str: physics = normalize_physics_backend(physics_backend) if not physics: diff --git a/tools/perf_smoke_test/task_config.py b/tools/perf_smoke_test/task_config.py index ad661be7bf36..6a0b55b9d49f 100644 --- a/tools/perf_smoke_test/task_config.py +++ b/tools/perf_smoke_test/task_config.py @@ -8,11 +8,23 @@ from pathlib import Path try: - from .backend_identity import make_backend_key, normalize_physics_backend, normalize_render_backend + from .backend_identity import ( + make_backend_key, + normalize_physics_backend, + normalize_render_backend, + validate_physics_backend, + validate_render_backend, + ) from .gate_types import FpsMeanThreshold from .gpu_identity import gpu_model_config_keys except ImportError: # pragma: no cover - supports direct script imports - from backend_identity import make_backend_key, normalize_physics_backend, normalize_render_backend + from backend_identity import ( + make_backend_key, + normalize_physics_backend, + normalize_render_backend, + validate_physics_backend, + validate_render_backend, + ) from gate_types import FpsMeanThreshold from gpu_identity import gpu_model_config_keys @@ -172,7 +184,8 @@ def load_tasks(tasks_json_path: Path | str | None = None) -> list[TaskConfig]: physics = normalize_physics_backend(backend_entry["physics"]) if physics is None: raise ValueError(f"backend entry in {path} must define a non-default physics backend") - render = normalize_render_backend(backend_entry.get("render")) + physics = validate_physics_backend(physics, source=str(path)) + render = validate_render_backend(normalize_render_backend(backend_entry.get("render")), source=str(path)) tasks.append( TaskConfig( task_id=merged["task_id"], From cd9cf02bc581d58c03bc561d949e25a71db519e9 Mon Sep 17 00:00:00 2001 From: Angelina Hu Date: Tue, 28 Jul 2026 09:59:55 -0700 Subject: [PATCH 2/4] Add golden correctness gate as an optional bench second stage Runs a frozen policy checkpoint per (task, backend) after the perf smoke benchmark, on the same warm runner via the shared isaac-rollout composite action, and scores behavioural KPIs (reward / episode length / success rate) against hard-set thresholds in golden_tasks.json. Standalone from perf (no baselines); advisory via a separate gate_config.golden_blocking toggle. Golden piggybacks perf cells so it needs no extra image pull; a separate ubuntu-latest golden_aggregate emits its own omni-github artifact (test_tool_id=golden-policy). --- .github/actions/isaac-rollout/action.yml | 168 ++++++++++ .github/workflows/perf-smoke-test.yaml | 242 +++++++++++++- tools/perf_smoke_test/build_golden_result.py | 224 +++++++++++++ tools/perf_smoke_test/dev/stub_golden.py | 142 ++++++++ tools/perf_smoke_test/gate_config.py | 4 + tools/perf_smoke_test/golden_aggregate.py | 167 ++++++++++ tools/perf_smoke_test/golden_config.py | 253 ++++++++++++++ tools/perf_smoke_test/golden_contracts.py | 189 +++++++++++ tools/perf_smoke_test/golden_kpi.py | 172 ++++++++++ tools/perf_smoke_test/golden_omni_github.py | 126 +++++++ tools/perf_smoke_test/golden_oracle.py | 196 +++++++++++ tools/perf_smoke_test/golden_probes.py | 69 ++++ .../perf_smoke_test/golden_result_adapter.py | 96 ++++++ tools/perf_smoke_test/golden_runtime.py | 315 ++++++++++++++++++ tools/perf_smoke_test/golden_tasks.json | 38 +++ tools/perf_smoke_test/tasks_to_ci_matrix.py | 90 ++++- 16 files changed, 2473 insertions(+), 18 deletions(-) create mode 100644 .github/actions/isaac-rollout/action.yml create mode 100644 tools/perf_smoke_test/build_golden_result.py create mode 100644 tools/perf_smoke_test/dev/stub_golden.py create mode 100644 tools/perf_smoke_test/golden_aggregate.py create mode 100644 tools/perf_smoke_test/golden_config.py create mode 100644 tools/perf_smoke_test/golden_contracts.py create mode 100644 tools/perf_smoke_test/golden_kpi.py create mode 100644 tools/perf_smoke_test/golden_omni_github.py create mode 100644 tools/perf_smoke_test/golden_oracle.py create mode 100644 tools/perf_smoke_test/golden_probes.py create mode 100644 tools/perf_smoke_test/golden_result_adapter.py create mode 100644 tools/perf_smoke_test/golden_runtime.py create mode 100644 tools/perf_smoke_test/golden_tasks.json diff --git a/.github/actions/isaac-rollout/action.yml b/.github/actions/isaac-rollout/action.yml new file mode 100644 index 000000000000..8b71c0aac535 --- /dev/null +++ b/.github/actions/isaac-rollout/action.yml @@ -0,0 +1,168 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +name: 'Isaac rollout in Docker' +description: > + Run one Isaac Sim rollout (perf benchmark or golden correctness) inside the + prebuilt CI container and return its exit code + wall time. Encapsulates the + container lifecycle shared by the perf-smoke bench job and the golden job: + writable cache/artifact dirs, optional PR-source overlay, the hardened + `docker run`, a hard wall-clock timeout via `docker wait`, log capture, and + cleanup. The driver-specific command is supplied by the caller as + `container-command`; matrix values flow through the environment (never + interpolated into shell source) and are forwarded into the container with + `docker run -e`, to prevent command injection. + +inputs: + container-command: + description: 'Bash body run inside the container (references forwarded env vars like "$TASK_ID").' + required: true + container-name: + description: 'Unique, pre-sanitized container name.' + required: true + image-tag: + description: 'CI image tag to run.' + required: true + image-was-pulled: + description: 'When "true", overlay the PR checkout over the image source (mounts workspace).' + required: false + default: 'false' + workspace: + description: 'github.workspace: host root for the jit/kit caches and the optional source overlay.' + required: true + host-artifact-dir: + description: 'Host directory mounted to /tmp/bench_out for this rollout''s outputs.' + required: true + timeout-s: + description: 'Hard wall-clock timeout for the rollout, in seconds.' + required: true + # --- run shape forwarded into the container (superset of perf + golden needs) --- + task-id: + required: true + description: 'Gym task id.' + num-envs: + required: true + description: 'Number of parallel environments.' + seed: + required: false + default: '' + description: 'Environment seed (empty = driver default).' + hydra-args: + required: false + default: '' + description: 'Hydra preset tokens (e.g. "presets=newton_mjwarp").' + num-frames: + required: false + default: '' + description: 'Perf: total benchmark steps.' + warmup-frames: + required: false + default: '' + description: 'Perf: leading steps to discard.' + eval-steps: + required: false + default: '' + description: 'Golden: rollout steps.' + checkpoint-path: + required: false + default: '' + description: 'Golden: local checkpoint path (never fetched over the network here).' + +outputs: + exit-code: + description: 'Container exit code (1 on timeout/failure to start).' + value: ${{ steps.run.outputs.exit-code }} + wall-time-s: + description: 'Measured wall-clock time of the rollout, in seconds.' + value: ${{ steps.run.outputs.wall-time-s }} + +runs: + using: 'composite' + steps: + - id: run + shell: bash + env: + CONTAINER_COMMAND: ${{ inputs.container-command }} + CONTAINER_NAME: ${{ inputs.container-name }} + CI_IMAGE_TAG: ${{ inputs.image-tag }} + IMAGE_WAS_PULLED: ${{ inputs.image-was-pulled }} + WORKSPACE: ${{ inputs.workspace }} + ARTIFACT_DIR: ${{ inputs.host-artifact-dir }} + TIMEOUT_S: ${{ inputs.timeout-s }} + # Run-shape values are forwarded into the container by name below; the + # container-command references them as "$TASK_ID" etc. Kept out of the + # shell source here so a crafted value cannot inject. + TASK_ID: ${{ inputs.task-id }} + NUM_ENVS: ${{ inputs.num-envs }} + SEED: ${{ inputs.seed }} + HYDRA_ARGS: ${{ inputs.hydra-args }} + NUM_FRAMES: ${{ inputs.num-frames }} + WARMUP_FRAMES: ${{ inputs.warmup-frames }} + EVAL_STEPS: ${{ inputs.eval-steps }} + CHECKPOINT_PATH: ${{ inputs.checkpoint-path }} + run: | + set -uo pipefail + + mkdir -p "${ARTIFACT_DIR}" "${WORKSPACE}/jit-cache/warp" "${WORKSPACE}/jit-cache/nv" "${WORKSPACE}/kit-cache" + # World-writable bind mounts: the CI image runs as non-root uid 1000 but these + # host dirs are created by the runner user, so the in-container user must be + # able to write the Warp/CUDA JIT cache and the Kit/RTX shader cache. + chmod -R 0777 "${ARTIFACT_DIR}" "${WORKSPACE}/jit-cache" "${WORKSPACE}/kit-cache" + + docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + + # Overlay the PR checkout when running a pulled (prebuilt) env image, so the + # rollout runs the PR's code (the editable install resolves to the mounted tree). + SRC_MOUNT="" + if [ "${IMAGE_WAS_PULLED}" = "true" ]; then + chmod -R a+rwX "${WORKSPACE}" 2>/dev/null || true + SRC_MOUNT="-v ${WORKSPACE}:/workspace/isaaclab" + fi + + docker run -d --name "${CONTAINER_NAME}" \ + --init --stop-timeout 10 \ + --entrypoint bash --gpus all --network=host \ + --security-opt=no-new-privileges:true \ + --ulimit nofile=65536:65536 \ + --ulimit nproc=4096:4096 \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e PYTHONUNBUFFERED=1 \ + -e PYTHONDONTWRITEBYTECODE=1 \ + -e WARP_CACHE_PATH=/tmp/jit-cache/warp \ + -e CUDA_CACHE_PATH=/tmp/jit-cache/nv \ + -e TASK_ID -e NUM_ENVS -e NUM_FRAMES -e WARMUP_FRAMES -e EVAL_STEPS -e SEED -e HYDRA_ARGS -e CHECKPOINT_PATH \ + -v "${ARTIFACT_DIR}:/tmp/bench_out" \ + -v "${WORKSPACE}/jit-cache:/tmp/jit-cache" \ + -v "${WORKSPACE}/kit-cache:/isaac-sim/kit/cache" \ + ${SRC_MOUNT} \ + "${CI_IMAGE_TAG}" \ + -c "${CONTAINER_COMMAND}" + + START=$(date +%s) + + # Stream container logs to file and terminal while the rollout runs. + docker logs -f "${CONTAINER_NAME}" 2>&1 | tee "${ARTIFACT_DIR}/benchmark.log" & + LOGS_PID=$! + + # Wait for the container to exit, with a hard wall-clock timeout. + ROLLOUT_EXIT=1 + if docker_exit=$(timeout "${TIMEOUT_S}" docker wait "${CONTAINER_NAME}" 2>/dev/null); then + ROLLOUT_EXIT="${docker_exit:-1}" + else + echo "::warning::Isaac rollout ${CONTAINER_NAME} timed out after ${TIMEOUT_S}s" + fi + + END=$(date +%s) + + kill "${LOGS_PID}" 2>/dev/null || true + wait "${LOGS_PID}" 2>/dev/null || true + docker kill "${CONTAINER_NAME}" 2>/dev/null || true + docker rm "${CONTAINER_NAME}" 2>/dev/null || true + + echo "exit-code=${ROLLOUT_EXIT}" >> "${GITHUB_OUTPUT}" + echo "wall-time-s=$((END - START))" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/perf-smoke-test.yaml b/.github/workflows/perf-smoke-test.yaml index e7ca3cb0631d..aa32a2955927 100644 --- a/.github/workflows/perf-smoke-test.yaml +++ b/.github/workflows/perf-smoke-test.yaml @@ -609,6 +609,163 @@ jobs: retention-days: 7 if-no-files-found: warn + # ========================================================================= + # Golden correctness — OPTIONAL second stage on the SAME warm runner. + # + # Runs only when this (task,backend) has a golden config (matrix.golden_present, + # from tasks_to_ci_matrix enrichment) AND a baked checkpoint is present. It + # reuses the image perf already pulled (env.CI_IMAGE_TAG) via the shared + # ./.github/actions/isaac-rollout action, so it pays NO extra image pull -- the + # whole reason it piggybacks here on an ephemeral fleet. + # + # Isolation: every step is placed AFTER perf has fully committed (result banked, + # per-task status posted, bench artifact uploaded) and is continue-on-error, so a + # golden crash/hang can never change perf's measurement, status, or artifacts. + # Golden self-limits via its own rollout timeout (matrix.golden_timeout_s), which + # the job timeout budgets for; a fresh container + GPU-clean pre-check keep it off + # a dirty GPU. Golden scoring/reporting happens in the separate golden_aggregate + # job (advisory, ubuntu-latest). + # ========================================================================= + - name: Golden — resolve checkpoint + GPU clean + id: golden_gpu + if: ${{ always() && matrix.golden_present == 'true' }} + continue-on-error: true + env: + GOLDEN_ROOT: ${{ vars.PERF_SMOKE_GOLDEN_ROOT }} + CHECKPOINT_RELPATH: ${{ matrix.golden_checkpoint_relpath }} + run: | + set -uo pipefail + echo "run=false" >> "$GITHUB_OUTPUT" + # Checkpoint is baked to GOLDEN_ROOT ahead of the runner stage; the rollout + # reads a LOCAL path only (no network). FIXME: wire baking in publish-image. + CKPT="${GOLDEN_ROOT%/}/${CHECKPOINT_RELPATH}" + if [ -z "${GOLDEN_ROOT}" ] || [ ! -f "${CKPT}" ]; then + echo "::notice::No golden checkpoint at '${GOLDEN_ROOT:-}/${CHECKPOINT_RELPATH}'; skipping golden (advisory)." + exit 0 + fi + # Never run golden on a dirty GPU (perf's container is already reaped). If + # perf left the device busy, skip rather than contend/contaminate. + USED_MB="$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -dc '0-9')" + if [ -n "${USED_MB}" ] && [ "${USED_MB}" -gt 2000 ]; then + echo "::warning::GPU not clean (${USED_MB} MB used) after bench; skipping golden (advisory)." + exit 0 + fi + BACKEND_KEY="${{ matrix.physics_backend }}${{ matrix.render_backend != '' && format('_{0}', matrix.render_backend) || '' }}" + ARTIFACT_DIR="$(pwd)/golden-artifacts/${{ matrix.task_id }}/${BACKEND_KEY}" + SAFE="${{ matrix.task_id }}-${{ matrix.physics_backend }}" + mkdir -p "${ARTIFACT_DIR}" + echo "run=true" >> "$GITHUB_OUTPUT" + echo "path=${CKPT}" >> "$GITHUB_OUTPUT" + echo "backend_key=${BACKEND_KEY}" >> "$GITHUB_OUTPUT" + echo "artifact_dir=${ARTIFACT_DIR}" >> "$GITHUB_OUTPUT" + echo "container=perf-golden-${SAFE//[^a-zA-Z0-9]/-}-${{ github.run_id }}" >> "$GITHUB_OUTPUT" + + - name: Golden — run rollout + id: golden_run + if: ${{ always() && steps.golden_gpu.outputs.run == 'true' }} + continue-on-error: true + uses: ./.github/actions/isaac-rollout + with: + container-name: ${{ steps.golden_gpu.outputs.container }} + image-tag: ${{ env.CI_IMAGE_TAG }} + image-was-pulled: ${{ steps.pull_image.outcome == 'success' }} + workspace: ${{ github.workspace }} + host-artifact-dir: ${{ steps.golden_gpu.outputs.artifact_dir }} + timeout-s: ${{ matrix.golden_timeout_s }} + task-id: ${{ matrix.task_id }} + num-envs: ${{ matrix.golden_num_envs }} + seed: ${{ matrix.golden_seed }} + hydra-args: ${{ matrix.golden_hydra_args }} + eval-steps: ${{ matrix.golden_eval_steps }} + checkpoint-path: ${{ steps.golden_gpu.outputs.path }} + container-command: | + set -e + cd /workspace/isaaclab + rm -f _isaac_sim + ln -s /isaac-sim _isaac_sim + ./isaaclab.sh -p tools/perf_smoke_test/golden_runtime.py \ + --task "$TASK_ID" \ + --num_envs "$NUM_ENVS" \ + --eval_steps "$EVAL_STEPS" \ + --checkpoint "$CHECKPOINT_PATH" \ + --output_path /tmp/bench_out \ + ${SEED:+--seed "$SEED"} \ + $HYDRA_ARGS + + - name: Golden — retry once on failure + id: golden_retry + if: ${{ always() && steps.golden_gpu.outputs.run == 'true' && steps.golden_run.outputs.exit-code != '0' && steps.golden_run.outputs.exit-code != '' }} + continue-on-error: true + uses: ./.github/actions/isaac-rollout + with: + container-name: ${{ steps.golden_gpu.outputs.container }}-retry + image-tag: ${{ env.CI_IMAGE_TAG }} + image-was-pulled: ${{ steps.pull_image.outcome == 'success' }} + workspace: ${{ github.workspace }} + host-artifact-dir: ${{ steps.golden_gpu.outputs.artifact_dir }} + timeout-s: ${{ matrix.golden_timeout_s }} + task-id: ${{ matrix.task_id }} + num-envs: ${{ matrix.golden_num_envs }} + seed: ${{ matrix.golden_seed }} + hydra-args: ${{ matrix.golden_hydra_args }} + eval-steps: ${{ matrix.golden_eval_steps }} + checkpoint-path: ${{ steps.golden_gpu.outputs.path }} + container-command: | + set -e + cd /workspace/isaaclab + rm -f _isaac_sim + ln -s /isaac-sim _isaac_sim + ./isaaclab.sh -p tools/perf_smoke_test/golden_runtime.py \ + --task "$TASK_ID" \ + --num_envs "$NUM_ENVS" \ + --eval_steps "$EVAL_STEPS" \ + --checkpoint "$CHECKPOINT_PATH" \ + --output_path /tmp/bench_out \ + ${SEED:+--seed "$SEED"} \ + $HYDRA_ARGS + + - name: Golden — build result + if: ${{ always() && steps.golden_gpu.outputs.run == 'true' }} + continue-on-error: true + env: + TASK_ID: ${{ matrix.task_id }} + PHYSICS_BACKEND: ${{ matrix.physics_backend }} + RENDER_BACKEND: ${{ matrix.render_backend }} + ARTIFACT_DIR: ${{ steps.golden_gpu.outputs.artifact_dir }} + run: | + RETRY_EXIT="${{ steps.golden_retry.outputs.exit-code }}" + if [ -n "${RETRY_EXIT}" ]; then + FINAL_EXIT="${RETRY_EXIT}" + FINAL_WALL="${{ steps.golden_retry.outputs.wall-time-s || '0' }}" + EXTRA_FLAGS="--was_retried --attempt 2" + else + FINAL_EXIT="${{ steps.golden_run.outputs.exit-code || '1' }}" + FINAL_WALL="${{ steps.golden_run.outputs.wall-time-s || '0' }}" + EXTRA_FLAGS="" + fi + python3 tools/perf_smoke_test/build_golden_result.py \ + --task_id "${TASK_ID}" \ + --physics_backend "${PHYSICS_BACKEND}" \ + --render_backend "${RENDER_BACKEND}" \ + --artifact_dir "${ARTIFACT_DIR}" \ + --exit_code "${FINAL_EXIT}" \ + --wall_time_s "${FINAL_WALL}" \ + --timeout_s "${{ matrix.golden_timeout_s }}" \ + --log_file "${ARTIFACT_DIR}/benchmark.log" \ + --golden_tasks tools/perf_smoke_test/golden_tasks.json \ + --checkpoint_id "${{ matrix.golden_checkpoint_id }}" \ + ${EXTRA_FLAGS} + + - name: Golden — upload artifact + if: ${{ always() && steps.golden_gpu.outputs.run == 'true' }} + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: golden-${{ matrix.task_id }}-${{ matrix.physics_backend }}${{ matrix.render_backend != '' && format('_{0}', matrix.render_backend) || '' }}-${{ github.run_id }} + path: golden-artifacts/${{ matrix.task_id }}/${{ steps.golden_gpu.outputs.backend_key }}/ + retention-days: 7 + if-no-files-found: warn + # Force-clean the container if the job is cancelled mid-run - name: Cleanup container on cancellation if: cancelled() @@ -624,9 +781,15 @@ jobs: fi SAFE_TASK_ID="${TASK_ID//[^a-zA-Z0-9]/-}" SAFE_BACKEND="${BACKEND_KEY//[^a-zA-Z0-9]/-}" - CONTAINER_NAME="perf-bench-${SAFE_TASK_ID}-${SAFE_BACKEND}-${{ github.run_id }}" - docker kill "${CONTAINER_NAME}" 2>/dev/null || true - docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + SAFE_GOLDEN="${TASK_ID}-${PHYSICS_BACKEND}" + SAFE_GOLDEN="${SAFE_GOLDEN//[^a-zA-Z0-9]/-}" + for CONTAINER_NAME in \ + "perf-bench-${SAFE_TASK_ID}-${SAFE_BACKEND}-${{ github.run_id }}" \ + "perf-golden-${SAFE_GOLDEN}-${{ github.run_id }}" \ + "perf-golden-${SAFE_GOLDEN}-${{ github.run_id }}-retry"; do + docker kill "${CONTAINER_NAME}" 2>/dev/null || true + docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + done # --------------------------------------------------------------------------- # Aggregate: oracle verdicts, baseline update, step summary @@ -932,3 +1095,76 @@ jobs: target_branch: develop strict_ancestry: true dry_run: false + + # --------------------------------------------------------------------------- + # Golden aggregate (ADVISORY, standalone). Scores every golden_result.json from + # the bench-job golden stage against golden_tasks.json, writes a step-summary + # table, and emits a SEPARATE omni-github artifact (test_tool_id=golden-policy). + # Pure-Python -> ubuntu-latest (no GPU, no image pull). Least privilege; never + # touches perf's baselines / comment / status. No PR comment (keeps golden off + # the slow network and out of perf's authoritative reporting path). + # --------------------------------------------------------------------------- + golden_aggregate: + name: Golden Aggregate + runs-on: ubuntu-latest + needs: [config, bench] + if: >- + ${{ always() + && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} + permissions: + contents: read + steps: + - name: Checkout Code + uses: actions/checkout@v6 + with: + fetch-depth: 1 + + - name: Download golden artifacts + uses: actions/download-artifact@v4 + continue-on-error: true + with: + pattern: golden-*-${{ github.run_id }} + path: golden-artifacts/ + merge-multiple: false + + - name: Run golden aggregate + id: golden_agg + run: | + case "${RUNNER_OS:-Linux}-${RUNNER_ARCH:-X64}" in + Linux-X64) APP_PLATFORM="linux-x86_64" ;; + Linux-ARM64) APP_PLATFORM="linux-aarch64" ;; + *) APP_PLATFORM="$(printf '%s-%s' "${RUNNER_OS:-linux}" "${RUNNER_ARCH:-x64}" | tr '[:upper:]' '[:lower:]')" ;; + esac + python3 tools/perf_smoke_test/golden_aggregate.py \ + --artifacts_dir golden-artifacts/ \ + --golden_tasks tools/perf_smoke_test/golden_tasks.json \ + --gate_config tools/perf_smoke_test/gate_config.json \ + --summary_file "${GITHUB_STEP_SUMMARY}" \ + --omni_github_dir "${{ github.workspace }}/golden-omni-github-artifact" \ + --omni_platform "${APP_PLATFORM}" + + - name: Validate golden omni-github artifact + id: golden_validate + if: always() + run: | + result_json="${{ github.workspace }}/golden-omni-github-artifact/_testoutput/golden_test_results.json" + echo "upload=false" >> "${GITHUB_OUTPUT}" + if [ ! -f "${result_json}" ]; then + echo "::notice::No golden omni-github result produced; skipping upload" + exit 0 + fi + python3 -c "import json,sys; d=json.load(open(sys.argv[1])); assert d.get('test_tool_id') and d.get('app') and isinstance(d.get('tests'),list) and all(r.get('test_id') and isinstance(r.get('passed'),bool) and isinstance(r.get('duration'),(int,float)) and r.get('custom') for r in d['tests']); print('golden omni-github result OK:', len(d['tests']), 'rows')" "${result_json}" + echo "upload=true" >> "${GITHUB_OUTPUT}" + + # Upload under the omni-github name contract + # (--v1----) so omni-github + # ingests it as its own tool (test_tool_id=golden-policy), separate from perf. + - name: Upload golden results to omni-github + if: always() && steps.golden_validate.outputs.upload == 'true' + uses: actions/upload-artifact@v7 + with: + name: golden-policy-results--v1-${{ github.repository_id }}-${{ github.run_id }}-${{ github.run_attempt }}-${{ job.check_run_id }} + path: ${{ github.workspace }}/golden-omni-github-artifact + if-no-files-found: error + retention-days: 7 + compression-level: 9 diff --git a/tools/perf_smoke_test/build_golden_result.py b/tools/perf_smoke_test/build_golden_result.py new file mode 100644 index 000000000000..d3976a9d447b --- /dev/null +++ b/tools/perf_smoke_test/build_golden_result.py @@ -0,0 +1,224 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Post-rollout script: normalize golden output and write ``golden_result.json``. + +The golden analogue of :mod:`build_bench_result`. Locates the timestamped +``PlayBundle`` written by ``golden_runtime.py``, copies it to the canonical +``golden_info.json``, classifies the failure phase from the captured log, +detects run-integrity drift (did the rollout run the requested task / num_envs / +seed / backend?), and writes ``golden_result.json`` for the golden aggregate. + +Unlike ``build_bench_result``, the intended run shape is taken from CLI args (the +golden gate has no separate ``launch_config.json`` yet), and there is no baseline +or contract hashing -- the golden gate is standalone. + +Usage:: + + python3 tools/perf_smoke_test/build_golden_result.py \\ + --task_id Isaac-Cartpole-Direct --physics_backend physx \\ + --artifact_dir artifacts/Isaac-Cartpole-Direct/physx \\ + --exit_code 0 --wall_time_s 42.0 --timeout_s 600 \\ + --num_envs 64 --seed 42 --checkpoint_id cartpole-physx-v1 \\ + --log_file artifacts/Isaac-Cartpole-Direct/physx/golden.log +""" + +import argparse +import glob +import json +import shutil +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +_TOOLS_DIR = _MODULE_DIR.parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + +from backend_identity import ( # noqa: E402 + identity_from_parts, + make_backend_key, + normalize_physics_backend, + normalize_render_backend, +) +from golden_config import get_golden_task # noqa: E402 +from golden_contracts import GoldenResult, GoldenSample # noqa: E402 +from golden_result_adapter import project_play # noqa: E402 +from gate_types import FailurePhase # noqa: E402 +from subprocess_runner import classify_failure_phase # noqa: E402 + + +def _config_drift(sample: GoldenSample, *, task_id: str, num_envs: int | None, seed: int | None, backend_key: str): + """Return a compact run-integrity mismatch string, or ``None`` when the run matched intent.""" + mismatches: list[str] = [] + if sample.task and sample.task != task_id: + mismatches.append(f"task(ran={sample.task},want={task_id})") + if sample.num_envs is not None and num_envs is not None and sample.num_envs != num_envs: + mismatches.append(f"num_envs(ran={sample.num_envs},want={num_envs})") + if sample.seed is not None and seed is not None and sample.seed != seed: + mismatches.append(f"seed(ran={sample.seed},want={seed})") + ran_backend = identity_from_parts(sample.physics_backend, sample.render_backend) + if ran_backend is not None and ran_backend.backend_key != backend_key: + mismatches.append(f"backend(ran={ran_backend.backend_key},want={backend_key})") + return " ".join(mismatches) if mismatches else None + + +def _normalize_golden_output(artifact_dir: Path, task_id: str) -> bool: + """Copy the timestamped play bundle to ``golden_info.json``; return whether it now exists.""" + golden_info = artifact_dir / "golden_info.json" + if golden_info.exists(): + return True + matches = sorted(glob.glob(str(artifact_dir / f"benchmark_play_{task_id}_*.json"))) + if not matches: + matches = sorted(glob.glob(str(artifact_dir / "benchmark_play_*.json"))) + if not matches: + return False + shutil.copy(matches[-1], golden_info) + return True + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Build golden_result.json from a golden rollout") + p.add_argument("--task_id", required=True) + p.add_argument("--physics_backend", required=True, help="Physics backend used (e.g. physx, newton)") + p.add_argument("--render_backend", default="", help="Render backend used (e.g. newton_renderer); empty = none") + p.add_argument("--preset", default="default") + p.add_argument("--artifact_dir", required=True, type=Path) + p.add_argument("--exit_code", required=True, type=int) + p.add_argument("--wall_time_s", required=True, type=float) + p.add_argument("--timeout_s", required=True, type=float) + p.add_argument("--golden_tasks", type=Path, default=_MODULE_DIR / "golden_tasks.json") + p.add_argument("--num_envs", type=int, default=None, help="Fallback eval env count if not in golden_tasks.json") + p.add_argument("--seed", type=int, default=None, help="Fallback seed if not in golden_tasks.json") + p.add_argument("--eval_steps", type=int, default=None, help="Fallback rollout steps if not in golden_tasks.json") + p.add_argument("--checkpoint_id", default=None, help="Fallback logical golden checkpoint id") + p.add_argument("--checkpoint_path", default=None, help="Local checkpoint path that was requested") + p.add_argument("--log_file", type=Path, default=None) + p.add_argument("--attempt", type=int, default=1) + p.add_argument("--was_retried", action="store_true") + return p.parse_args() + + +def main() -> int: + args = _parse_args() + artifact_dir = args.artifact_dir + artifact_dir.mkdir(parents=True, exist_ok=True) + + physics_backend = normalize_physics_backend(args.physics_backend) + if physics_backend is None: + raise ValueError("--physics_backend must name a concrete backend") + render_backend = normalize_render_backend(args.render_backend) + backend_key = make_backend_key(physics_backend, render_backend) + + # Single source of truth for run intent: prefer golden_tasks.json (the same + # config that drives the rollout) over CLI args, so the driver and this builder + # cannot disagree on the requested shape and spuriously flag drift. + try: + golden_task = get_golden_task(args.task_id, backend_key, args.golden_tasks) + except Exception: + golden_task = None + want_num_envs = golden_task.num_envs if golden_task else args.num_envs + want_seed = golden_task.seed if golden_task else args.seed + want_eval_steps = golden_task.eval_steps if golden_task else args.eval_steps + checkpoint_id = golden_task.checkpoint_id if golden_task else args.checkpoint_id + + log_text = "" + if args.log_file and args.log_file.exists(): + log_text = args.log_file.read_text(errors="replace") + + golden_info_present = _normalize_golden_output(artifact_dir, args.task_id) + + failure_phase = classify_failure_phase( + stdout=log_text, + stderr="", + exit_code=args.exit_code, + wall_time_s=args.wall_time_s, + timeout_s=args.timeout_s, + ) + + sample: GoldenSample | None = None + config_mismatch: str | None = None + if golden_info_present: + info_path = artifact_dir / "golden_info.json" + try: + bundle = json.loads(info_path.read_text()) + except Exception: + bundle = None + sample = project_play(bundle) if isinstance(bundle, dict) else None + if sample is None: + # File exists but is not a valid schema-v1 play bundle (corrupt/truncated). + golden_info_present = False + else: + config_mismatch = _config_drift( + sample, task_id=args.task_id, num_envs=want_num_envs, seed=want_seed, backend_key=backend_key + ) + if config_mismatch and failure_phase is None: + failure_phase = FailurePhase.CONFIG_MISMATCH.value + + result = GoldenResult( + task_id=args.task_id, + backend=backend_key, + physics_backend=physics_backend, + render_backend=render_backend, + backend_key=backend_key, + preset=args.preset, + checkpoint_id=checkpoint_id, + checkpoint_path=(sample.checkpoint_path if sample and sample.checkpoint_path else args.checkpoint_path), + attempt=args.attempt, + was_retried=args.was_retried, + exit_code=args.exit_code, + failure_phase=failure_phase, + stdout_tail=log_text[-2000:] if len(log_text) > 2000 else log_text, + wall_time_s=args.wall_time_s, + startup_time_s=sample.startup_time_s if sample else None, + golden_info_present=golden_info_present, + reward_mean=sample.reward_mean if sample else None, + reward_std=sample.reward_std if sample else None, + ep_length_mean=sample.ep_length_mean if sample else None, + ep_length_std=sample.ep_length_std if sample else None, + success_rate=sample.success_rate if sample else None, + num_episodes=sample.num_episodes if sample else None, + benchmark_info=sample.benchmark_info() if sample else {}, + config_mismatch=config_mismatch, + runtime_resources=(sample.runtime_resources or None) if sample else None, + provenance=sample.provenance if sample else None, + launch_config={ + "task_id": args.task_id, + "backend_key": backend_key, + "physics_backend": physics_backend, + "render_backend": render_backend, + "num_envs": want_num_envs, + "seed": want_seed, + "eval_steps": want_eval_steps, + "checkpoint_id": checkpoint_id, + }, + task_config_snapshot={ + "task_id": args.task_id, + "backend": backend_key, + "physics_backend": physics_backend, + "render_backend": render_backend, + "backend_key": backend_key, + "preset": args.preset, + "num_envs": want_num_envs, + "seed": want_seed, + "eval_steps": want_eval_steps, + "checkpoint_id": checkpoint_id, + }, + ) + + out = artifact_dir / "golden_result.json" + out.write_text(json.dumps(result.to_dict(), indent=2)) + print( + f"[build_golden_result] {args.task_id}/{backend_key}: " + f"failure_phase={failure_phase!r}, golden_info_present={golden_info_present}, " + f"exit_code={args.exit_code}, config_mismatch={config_mismatch!r}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_smoke_test/dev/stub_golden.py b/tools/perf_smoke_test/dev/stub_golden.py new file mode 100644 index 000000000000..e9a10018d3b5 --- /dev/null +++ b/tools/perf_smoke_test/dev/stub_golden.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Local-testing stub that fakes a ``golden_runtime.py`` rollout without a GPU/sim. + +Emits a schema-v1 :class:`~isaaclab.test.benchmark.schema.PlayBundle` +(``benchmark_play_{task}_{stamp}.json``) identical in shape to what the real +golden driver writes, so ``build_golden_result.py`` -> ``golden_result_adapter`` +-> ``golden_oracle`` can be exercised end-to-end offline. Uses the real +``isaaclab.test.benchmark`` builders/serialize (pure-Python, no GPU) and the +shared stub fixtures from :mod:`stub_benchmark`, so it stays in lockstep with the +schema; run it with the Isaac Lab Python env. + +The behavioural aggregates are fully configurable so a single stub can drive +every oracle path (healthy PASS, a reward-floor BLOCK, a pole-ceiling probe +BLOCK, a zero-episode WARN, and the import/init/runtime failure phases). +""" + +import argparse +import sys +from datetime import datetime, timezone +from pathlib import Path + +_DEV_DIR = Path(__file__).resolve().parent +_MODULE_DIR = _DEV_DIR.parent +for _p in (str(_MODULE_DIR), str(_DEV_DIR)): + if _p not in sys.path: + sys.path.insert(0, _p) + +from backend_identity import split_backend_key # noqa: E402 +from stub_benchmark import _stub_hardware, _stub_resources, _stub_versions # noqa: E402 + +from isaaclab.test.benchmark import builders, serialize # noqa: E402 +from isaaclab.test.benchmark.schema import MeanStd, StartupTime # noqa: E402 + + +def _mean_std(mean: float | None, std: float) -> MeanStd | None: + """Build a :class:`MeanStd`, or ``None`` to simulate an aggregate over zero episodes.""" + return None if mean is None else MeanStd(mean=mean, std=std) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Emit a stub golden PlayBundle for offline pipeline testing.") + parser.add_argument("--task_id", required=True) + parser.add_argument("--backend", required=True, help="Backend key, e.g. physx / newton / physx_newton_renderer") + parser.add_argument("--num_envs", type=int, default=64) + parser.add_argument("--eval_steps", type=int, default=200) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--out_dir", required=True) + parser.add_argument("--reward_mean", type=float, default=480.0, help="Set negative to omit (simulate no episode).") + parser.add_argument("--reward_std", type=float, default=5.0) + parser.add_argument("--ep_length_mean", type=float, default=500.0, help="Set negative to omit.") + parser.add_argument("--ep_length_std", type=float, default=0.0) + parser.add_argument("--success_rate", type=float, default=None) + parser.add_argument( + "--num_episodes", + type=int, + default=None, + help="Completed-episode count to record; omit to match the real driver, which does not report one.", + ) + parser.add_argument("--checkpoint_path", default="/opt/golden/stub/policy.pt") + parser.add_argument("--failure_phase", default="none", choices=["none", "import", "init", "runtime"]) + args = parser.parse_args() + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + # --- simulated pre-output failures (no bundle written) --- + if args.failure_phase == "import": + print("Traceback (most recent call last):") + print("ImportError: simulated import failure") + sys.exit(1) + print("AppLauncher initialization complete", flush=True) + if args.failure_phase == "init": + sys.exit(2) + + identity = split_backend_key(args.backend) + if identity is None: + raise RuntimeError(f"Cannot parse backend identity from {args.backend!r}") + + # A play bundle still carries a Runtime section; synthesize a trivial steady series. + startup = StartupTime(app_launch=2.0, env_creation=1.0, first_step=0.3, python_imports=1.5) + step_times = [args.num_envs / 200.0 for _ in range(max(1, args.eval_steps))] + fps = [200.0 for _ in step_times] + runtime = builders.build_runtime( + startup_time_s=startup, + iteration_times_s=step_times, + collection_fps=fps, + total_fps=fps, + steps_per_iteration=args.num_envs, + ) + cfg = builders.build_run_config( + physics_backend=identity.physics_backend, + rendering_backend=identity.render_backend or "none", + presets=[], + ) + start_utc = datetime.now(timezone.utc).isoformat() + run = builders.build_run_identity( + run_id=f"stub-golden-{args.task_id}-{identity.backend_key}", + framework="rsl_rl", + config=cfg, + task=args.task_id, + seed=args.seed, + start_utc=start_utc, + end_utc=start_utc, + num_envs=args.num_envs, + ) + extra: dict = {"stub": True} + if args.num_episodes is not None: + extra["num_episodes"] = args.num_episodes + bundle = builders.build_play_bundle( + run=run, + versions=_stub_versions(), + hardware=_stub_hardware(), + runtime=runtime, + resources=_stub_resources(), + success_rate=args.success_rate, + reward=_mean_std(None if args.reward_mean < 0 else args.reward_mean, args.reward_std), + ep_length=_mean_std(None if args.ep_length_mean < 0 else args.ep_length_mean, args.ep_length_std), + checkpoint_path=args.checkpoint_path, + extra=extra, + ) + + stamp = datetime.now(timezone.utc).strftime("%Y-%m-%d_%H-%M-%S") + out_path = out_dir / f"benchmark_play_{args.task_id}_{stamp}.json" + serialize.write_bundle_file(bundle, str(out_path)) + + # Progress marker consumed by subprocess_runner.classify_failure_phase (matches the + # perf driver's stdout contract) so a runtime crash is not misread as an init failure. + print("Step Frametimes", flush=True) + + if args.failure_phase == "runtime": + print("RuntimeError: simulated crash during rollout") + sys.exit(3) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/tools/perf_smoke_test/gate_config.py b/tools/perf_smoke_test/gate_config.py index d62d2b303a65..558b991bdddc 100644 --- a/tools/perf_smoke_test/gate_config.py +++ b/tools/perf_smoke_test/gate_config.py @@ -73,6 +73,10 @@ def _merge_runtime_compatibility(raw: dict | None) -> dict: def load_gate_config(path: Path | str) -> dict: config = { "blocking": False, + # Separate advisory toggle for the golden correctness gate, independent of the + # performance ``blocking`` flag: golden verdicts never affect the PR outcome + # until this is explicitly flipped (see golden_aggregate). + "golden_blocking": False, "min_baseline_samples": MIN_BASELINE_SAMPLES, "max_baseline_samples": MAX_BASELINE_SAMPLES, "min_block_regression_pct": MIN_BLOCK_REGRESSION_PCT, diff --git a/tools/perf_smoke_test/golden_aggregate.py b/tools/perf_smoke_test/golden_aggregate.py new file mode 100644 index 000000000000..00070e0373a4 --- /dev/null +++ b/tools/perf_smoke_test/golden_aggregate.py @@ -0,0 +1,167 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Aggregate golden results, run the golden oracle, and report. + +The golden analogue of :mod:`aggregate`, but standalone: it scores each +``golden_result.json`` against the hard-set KPI thresholds in ``golden_tasks.json`` +(no baselines, no git, no compatibility hashing), renders a *separate* golden +results table, and optionally emits a *separate* omni-github artifact +(``test_tool_id=golden-policy``). The performance aggregate is untouched. + +The gate is advisory by default: golden verdicts affect the exit code only when +``gate_config.golden_blocking`` is explicitly enabled (independent of the perf +``blocking`` flag). +""" + +import argparse +import json +import os +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +_TOOLS_DIR = _MODULE_DIR.parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + +from gate_config import load_gate_config # noqa: E402 +from gate_types import OracleVerdict # noqa: E402 +from golden_config import get_golden_task # noqa: E402 +from golden_contracts import GoldenResult # noqa: E402 +from golden_omni_github import write_artifact as write_golden_omni_github # noqa: E402 +from golden_oracle import evaluate # noqa: E402 + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Aggregate golden results and run the golden oracle.") + p.add_argument("--artifacts_dir", required=True, type=Path) + p.add_argument("--golden_tasks", type=Path, default=_MODULE_DIR / "golden_tasks.json") + p.add_argument("--gate_config", type=Path, default=_MODULE_DIR / "gate_config.json") + p.add_argument("--summary_file", default=None) + p.add_argument("--omni_github_dir", type=Path, default=None, help="Write the golden omni-github artifact here") + p.add_argument("--omni_platform", default="linux-x86_64") + p.add_argument("--omni_app_config", default="golden-policy") + return p.parse_args() + + +def _find_golden_results(artifacts_dir: Path) -> list[tuple[Path, GoldenResult]]: + found = [] + for path in sorted(artifacts_dir.rglob("golden_result.json")): + with path.open() as fh: + found.append((path.parent, GoldenResult.from_dict(json.load(fh)))) + return found + + +def _kpis_for(result: GoldenResult, golden_tasks_path: Path) -> dict: + """Resolve the configured KPI thresholds for a result's (task, backend), or empty if none.""" + try: + task = get_golden_task(result.task_id, result.backend_key or result.backend, golden_tasks_path) + except KeyError: + return {} + return task.kpis + + +def _fmt(value, decimals: int = 2) -> str: + return f"{value:.{decimals}f}" if isinstance(value, (int, float)) else "N/A" + + +def _crossed_tags(kpi_results: list[dict]) -> str: + tags = [] + for kr in kpi_results: + for ct in kr.get("crossed_thresholds", []): + verdict = ct.get("threshold_verdict") or "report" + tags.append(f"{kr['kpi']}:{ct.get('threshold_name')}({verdict})") + return "; ".join(tags) + + +def _build_table(rows: list[tuple]) -> str: + lines = [ + "| Task | Backend | Verdict | Checkpoint | Reward | EpLen | Success | Episodes | Crossed | Phase | Note |", + "|---|---|---|---|---:|---:|---:|---:|---|---|---|", + ] + for result, _golden in rows: + lines.append( + f"| {result.task_id} | {result.backend} | {result.verdict.value}" + f" | {result.checkpoint_id or ''} | {_fmt(result.reward_mean)} | {_fmt(result.ep_length_mean)}" + f" | {_fmt(result.success_rate)} | {result.num_episodes if result.num_episodes is not None else 'N/A'}" + f" | {_crossed_tags(result.kpi_results)} | {result.failure_phase or ''} | {result.note or ''} |" + ) + return "\n".join(lines) + + +def _write_github_output(**values) -> None: + github_output = os.environ.get("GITHUB_OUTPUT", "") + if not github_output: + return + with open(github_output, "a") as fh: + for key, value in values.items(): + if value is not None: + fh.write(f"{key}={value}\n") + + +def main() -> int: + args = _parse_args() + gate_config = load_gate_config(args.gate_config) + golden_blocking = bool(gate_config.get("golden_blocking", False)) + + items = _find_golden_results(args.artifacts_dir) + if not items: + print(f"[golden_aggregate] No golden_result.json files found under {args.artifacts_dir}") + # Advisory add-on: an empty/missing golden artifact set must not break the + # build unless golden gating is explicitly enabled. + return 1 if golden_blocking else 0 + + rows = [] + has_block = False + has_hard_failure = False + for _artifact_dir, result in items: + oracle_result = evaluate(result, _kpis_for(result, args.golden_tasks)) + rows.append((oracle_result, result)) + print( + f"[golden_aggregate] {oracle_result.task_id}/{oracle_result.backend}: {oracle_result.verdict.value}" + f" reward={_fmt(oracle_result.reward_mean)} ep_length={_fmt(oracle_result.ep_length_mean)}" + f" episodes={oracle_result.num_episodes}" + + (f" note={oracle_result.note}" if oracle_result.note else "") + ) + if oracle_result.verdict == OracleVerdict.BLOCK: + has_block = True + elif oracle_result.verdict == OracleVerdict.HARD_FAILURE: + has_hard_failure = True + + table = _build_table(rows) + print("\n## Golden Correctness Results\n") + print(table) + print() + + if args.summary_file: + with open(args.summary_file, "a") as fh: + fh.write("\n## Golden Correctness Results\n\n") + fh.write(table) + fh.write("\n") + + if args.omni_github_dir: + write_golden_omni_github( + rows, args.omni_github_dir, platform=args.omni_platform, app_config=args.omni_app_config + ) + + _write_github_output( + golden_has_block="true" if has_block else "false", + golden_has_hard_failure="true" if has_hard_failure else "false", + golden_blocking="true" if golden_blocking else "false", + ) + + if golden_blocking: + if has_block: + return 1 + if has_hard_failure: + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_smoke_test/golden_config.py b/tools/perf_smoke_test/golden_config.py new file mode 100644 index 000000000000..f22a084ed85b --- /dev/null +++ b/tools/perf_smoke_test/golden_config.py @@ -0,0 +1,253 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Golden correctness task configuration. + +Loads ``golden_tasks.json`` -- the sibling of ``tasks.json`` for the golden gate +-- into one :class:`GoldenTaskConfig` per (task, backend) combination. It is kept +as a separate file (rather than extra fields on ``tasks.json``) so the two gates +stay fully decoupled: the performance suite parses and runs with no awareness of +golden config, and a task can carry a performance entry, a golden entry, or both. + +A golden entry names a frozen policy **checkpoint** (a logical id plus a path +relative to a baked checkpoint root -- never a URL, so the runner needs no +network) and the hard-set behavioural **KPI thresholds** to judge its rollout +against (see :class:`~golden_kpi.KpiThreshold`). +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path + +try: + from .backend_identity import make_backend_key, normalize_physics_backend, normalize_render_backend + from .golden_contracts import UNIVERSAL_KPIS + from .golden_kpi import KpiThreshold + from .golden_probes import get_probe +except ImportError: # pragma: no cover - supports direct script imports + from backend_identity import make_backend_key, normalize_physics_backend, normalize_render_backend + from golden_contracts import UNIVERSAL_KPIS + from golden_kpi import KpiThreshold + from golden_probes import get_probe + +_DEFAULT_GOLDEN_TASKS_JSON = Path(__file__).parent / "golden_tasks.json" + + +def parse_kpi_thresholds(raw) -> dict[str, list[KpiThreshold]]: + """Parse the ``{kpi_name: [threshold entries]}`` map for one golden task. + + Validation (mandatory names, gating-verdict enum, direction enum, value-less + skips) happens in :meth:`~golden_kpi.KpiThreshold.from_list`, so a malformed + ``golden_tasks.json`` fails fast at load time. + + Args: + raw: The raw ``kpis`` value from a task entry (may be empty/absent). + + Returns: + Mapping of KPI name to its parsed threshold list. + """ + if not raw: + return {} + if not isinstance(raw, dict): + raise TypeError("kpis must be an object keyed by KPI name") + return {kpi_name: KpiThreshold.from_list(entries, context=kpi_name) for kpi_name, entries in raw.items()} + + +@dataclass +class GoldenTaskConfig: + """Golden correctness configuration for a single task and backend combination.""" + + task_id: str + physics_backend: str + render_backend: str | None + preset: str + rl_library: str + agent: str + checkpoint_id: str + checkpoint_relpath: str + num_envs: int + eval_steps: int + seed: int | None + deterministic: bool + kpis: dict[str, list[KpiThreshold]] + timeout_minutes: int + tags: list[str] = field(default_factory=lambda: ["always"]) + task_type: str = "golden" + runs_on: str = "gpu-l40s" + + @property + def backend_key(self) -> str: + """Composite key identifying the backend combination (see :func:`~backend_identity.make_backend_key`).""" + return make_backend_key(self.physics_backend, self.render_backend) + + def resolve_checkpoint_path(self, golden_root: Path | str) -> Path: + """Return the absolute local checkpoint path under a baked ``golden_root``. + + The checkpoint is resolved to a local path only; it is baked into the + runner image at publish time (network-capable), so the runner itself + never fetches it. :attr:`checkpoint_relpath` is already backend-expanded + at load time. + + Args: + golden_root: Directory the golden checkpoints were baked into. + + Returns: + Absolute path to this task/backend's checkpoint file. + """ + return Path(golden_root) / self.checkpoint_relpath + + +def _load_golden_tasks_json(path: Path) -> tuple[dict, list[dict]]: + with open(path) as f: + raw_data = json.load(f) + + if isinstance(raw_data, dict): + defaults = raw_data.get("defaults", {}) + raw_list = raw_data.get("tasks", []) + if not isinstance(raw_list, list): + raise TypeError(f"'tasks' field in {path} must be a list") + elif isinstance(raw_data, list): + defaults = {} + raw_list = raw_data + else: + raise TypeError(f"{path} must contain a JSON list or an object with a top-level 'tasks' list") + + if not isinstance(defaults, dict): + raise TypeError(f"'defaults' field in {path} must be an object") + + return defaults, raw_list + + +def _require_positive(name: str, value: int, task_id: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"golden task {task_id!r}: {name} must be a positive integer, got {value!r}") + return value + + +def _validate_kpi_names(task_id: str, kpis: dict[str, list[KpiThreshold]]) -> None: + """Ensure every configured KPI name is measurable for this task. + + A KPI is measurable if it is one of the universal, episode-derived KPIs + (:data:`~golden_contracts.UNIVERSAL_KPIS`) or the task has a registered + sim-state probe (see :mod:`golden_probes`) that can produce it. Without this + check a typo (e.g. ``"success"`` for ``"success_rate"``) or a probe KPI + configured before its probe is registered would parse fine but silently never + gate, since the oracle treats an unmeasured KPI as non-gating. + + Raises: + ValueError: If a configured KPI name is neither universal nor backed by a + registered probe for ``task_id``. + """ + if not kpis: + return + has_probe = get_probe(task_id) is not None + for kpi_name in kpis: + if kpi_name not in UNIVERSAL_KPIS and not has_probe: + raise ValueError( + f"golden task {task_id!r} configures KPI {kpi_name!r}, which is neither a universal KPI " + f"({sorted(UNIVERSAL_KPIS)}) nor produced by a registered sim-state probe" + ) + + +def _expand(template: object, fmt: dict, *, task_id: str, field_name: str) -> str: + """Expand ``{task_id}``/``{backend_key}`` placeholders, with task context on error.""" + try: + return str(template).format(**fmt) + except (KeyError, IndexError, ValueError) as exc: + raise ValueError( + f"golden task {task_id!r}: bad placeholder in checkpoint {field_name} {template!r} ({exc})" + ) from exc + + +def load_golden_tasks(golden_tasks_json_path: Path | str | None = None) -> list[GoldenTaskConfig]: + """Load all golden tasks, producing one :class:`GoldenTaskConfig` per backend combination. + + Args: + golden_tasks_json_path: Path to ``golden_tasks.json``. Defaults to the one + next to this module. + + Returns: + List of :class:`GoldenTaskConfig`, one per (task_id, backend) combination. + """ + path = Path(golden_tasks_json_path) if golden_tasks_json_path is not None else _DEFAULT_GOLDEN_TASKS_JSON + defaults, raw_list = _load_golden_tasks_json(path) + + tasks: list[GoldenTaskConfig] = [] + for raw in raw_list: + if not isinstance(raw, dict): + raise TypeError(f"golden task entry in {path} must be an object") + merged = {**defaults, **raw} + + task_id = merged["task_id"] + checkpoint = merged.get("checkpoint") + if not isinstance(checkpoint, dict) or not checkpoint.get("id") or not checkpoint.get("path"): + raise ValueError( + f"golden task {task_id!r} must define a 'checkpoint' object with non-empty 'id' and 'path'" + ) + + kpis = parse_kpi_thresholds(merged.get("kpis", {})) + _validate_kpi_names(task_id, kpis) + num_envs = _require_positive("num_envs", int(merged["num_envs"]), task_id) + eval_steps = _require_positive("eval_steps", int(merged["eval_steps"]), task_id) + backends: list[dict] = merged.get("backends", []) + if not backends: + raise ValueError(f"golden task {task_id!r} must define at least one backend") + + for backend_entry in backends: + physics = normalize_physics_backend(backend_entry["physics"]) + if physics is None: + raise ValueError(f"golden backend entry in {path} must define a non-default physics backend") + render = normalize_render_backend(backend_entry.get("render")) + backend_key = make_backend_key(physics, render) + # Expand path/id placeholders once per backend so downstream code sees a concrete reference. + fmt = {"task_id": task_id, "backend_key": backend_key} + tasks.append( + GoldenTaskConfig( + task_id=task_id, + physics_backend=physics, + render_backend=render, + preset=merged["preset"], + rl_library=merged["rl_library"], + agent=merged["agent"], + checkpoint_id=_expand(checkpoint["id"], fmt, task_id=task_id, field_name="id"), + checkpoint_relpath=_expand(checkpoint["path"], fmt, task_id=task_id, field_name="path"), + num_envs=num_envs, + eval_steps=eval_steps, + seed=int(merged["seed"]) if merged.get("seed") is not None else None, + deterministic=bool(merged.get("deterministic", True)), + kpis=kpis, + timeout_minutes=int(merged["timeout_minutes"]), + tags=merged.get("tags", ["always"]), + task_type=merged.get("type", "golden"), + runs_on=merged.get("runs_on", "gpu-l40s"), + ) + ) + return tasks + + +def get_golden_task( + task_id: str, + backend_key: str, + golden_tasks_json_path: Path | str | None = None, +) -> GoldenTaskConfig: + """Return the :class:`GoldenTaskConfig` for a (task_id, backend_key) pair. + + Args: + task_id: Task identifier to look up. + backend_key: Backend key (e.g. ``"physx"``, ``"newton"``). + golden_tasks_json_path: Optional path to ``golden_tasks.json``. + + Returns: + The matching :class:`GoldenTaskConfig`. + + Raises: + KeyError: If no golden task with the given (task_id, backend_key) exists. + """ + for task in load_golden_tasks(golden_tasks_json_path): + if task.task_id == task_id and task.backend_key == backend_key: + return task + raise KeyError(f"Golden task not found: task_id={task_id!r} backend_key={backend_key!r}") diff --git a/tools/perf_smoke_test/golden_contracts.py b/tools/perf_smoke_test/golden_contracts.py new file mode 100644 index 000000000000..f42a62fe4935 --- /dev/null +++ b/tools/perf_smoke_test/golden_contracts.py @@ -0,0 +1,189 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Typed contract for the golden gate's own artifact (schema v1). + +:class:`GoldenResult` is the golden analogue of :class:`~contracts.BenchResult`: +the typed shape of ``golden_result.json``, the per-(task, backend) artifact a +golden rollout produces and the golden oracle consumes. It follows the same +"typed gate-critical fields + open pass-through dicts" pattern -- the measured +behavioural KPIs and run identity are typed attributes, while provenance and +launch/config payloads stay as ``dict`` fields carried through untouched. + +The behavioural KPIs come straight from the merged benchmark core's play +rollout (:func:`~isaaclab.test.benchmark.stepping.run_play_loop` -> a +:class:`~isaaclab.test.benchmark.schema.PlayBundle`): ``reward`` and +``ep_length`` as scalar mean/std aggregates over completed episodes, plus a +``success_rate``. :attr:`probe_kpis` carries any additional sim-state signals +produced by an optional per-task probe (see :mod:`golden_probes`); it is empty +unless a probe is registered. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field, fields +from typing import Any + +CONTRACT_SCHEMA_VERSION = "1.0" + +# The KPI names backed by typed fields on :class:`GoldenResult` rather than by an +# optional sim-state probe. All three are derived from *completed episodes*, so a +# run that completed no episode leaves them unmeasured (see +# :meth:`GoldenResult.has_completed_episodes`). Any other configured KPI name must +# be produced by a registered probe (see :mod:`golden_probes`). +UNIVERSAL_KPIS = frozenset({"reward", "ep_length", "success_rate"}) + + +@dataclass(frozen=True) +class GoldenSample: + """Adapter projection of a schema-v1 ``PlayBundle`` into the golden gate's fields. + + The behavioural aggregates and run identity are typed; ``provenance`` and + ``runtime_resources`` stay dicts (open provenance payloads carried through). + Mirrors :class:`~contracts.RuntimeSample` on the performance side. + """ + + reward_mean: float | None = None + reward_std: float | None = None + ep_length_mean: float | None = None + ep_length_std: float | None = None + success_rate: float | None = None + num_episodes: int | None = None + checkpoint_path: str | None = None + startup_time_s: float | None = None + task: str | None = None + num_envs: int | None = None + seed: int | None = None + physics_backend: str | None = None + render_backend: str | None = None + presets: list[str] = field(default_factory=list) + provenance: dict[str, Any] = field(default_factory=dict) + runtime_resources: dict[str, Any] = field(default_factory=dict) + + def benchmark_info(self) -> dict[str, Any]: + """Return the run's self-reported identity as a dict (None-valued keys omitted).""" + info = { + "task": self.task, + "num_envs": self.num_envs, + "seed": self.seed, + "physics_backend": self.physics_backend, + "render_backend": self.render_backend, + "presets": self.presets or None, + } + return {k: v for k, v in info.items() if v is not None} + + +@dataclass(frozen=True) +class GoldenResult: + """Typed shape of ``golden_result.json`` (schema v1). + + The leading identity fields are required (a build that forgets one fails at + construction); the rest default so a HARD_FAILURE result -- e.g. a missing + checkpoint or a crashed rollout, where no KPI was measured -- still + constructs cleanly. + """ + + # --- identity (required) --- + task_id: str + backend: str + physics_backend: str + render_backend: str | None + backend_key: str + preset: str + # --- golden reference identity --- + checkpoint_id: str | None = None + checkpoint_path: str | None = None + # --- run status --- + attempt: int = 1 + was_retried: bool = False + exit_code: int = 0 + failure_phase: str | None = None + stdout_tail: str = "" + wall_time_s: float | None = None + startup_time_s: float | None = None + golden_info_present: bool = False + # --- behavioural KPIs (aggregates over completed episodes) --- + reward_mean: float | None = None + reward_std: float | None = None + ep_length_mean: float | None = None + ep_length_std: float | None = None + success_rate: float | None = None + num_episodes: int | None = None + # --- optional sim-state probe signals (name -> scalar); empty unless a probe is registered --- + probe_kpis: dict[str, float] = field(default_factory=dict) + # --- open pass-through payloads --- + benchmark_info: dict[str, Any] = field(default_factory=dict) + config_mismatch: str | None = None + runtime_resources: dict[str, Any] | None = None + provenance: dict[str, Any] | None = None + launch_config: dict[str, Any] = field(default_factory=dict) + task_config_snapshot: dict[str, Any] = field(default_factory=dict) + schema_version: str = CONTRACT_SCHEMA_VERSION + + def has_completed_episodes(self) -> bool: + """Return whether the rollout completed at least one episode. + + Episode-derived KPIs (:data:`UNIVERSAL_KPIS`) are only meaningful when an + episode finished. ``num_episodes`` is trusted when reported; an unset + (``None``) count is treated as "unknown, don't force-unmeasure" so a run + that reports a value but omits the count is not silently discarded. Only an + explicit non-positive count marks the episode KPIs as unmeasured -- which + closes the hole where a runner reports e.g. ``success_rate = 0.0`` over + zero episodes. + """ + return self.num_episodes is None or self.num_episodes > 0 + + def kpi_value(self, kpi_name: str) -> float | None: + """Return the measured value for a configured KPI name, or ``None`` if unmeasured. + + The three universal KPIs map to typed fields; any other name is looked up + in :attr:`probe_kpis` (populated only when a task registers a sim-state + probe). A ``None`` return means "configured but not measured this run" + (e.g. ``reward`` when no episode completed) and is handled by the oracle. + + Args: + kpi_name: KPI key as configured in ``golden_tasks.json``. + + Returns: + The measured scalar, or ``None`` when the KPI was not produced. + """ + universal = { + "reward": self.reward_mean, + "ep_length": self.ep_length_mean, + "success_rate": self.success_rate, + } + if kpi_name in universal: + return universal[kpi_name] + return self.probe_kpis.get(kpi_name) + + def measured_kpi(self, kpi_name: str) -> float | None: + """Return the value for a KPI treating unfinished-episode aggregates as unmeasured. + + Like :meth:`kpi_value`, but an episode-derived universal KPI is reported as + ``None`` when no episode completed (see :meth:`has_completed_episodes`), + regardless of any placeholder value the runner may have emitted. + + Args: + kpi_name: KPI key as configured in ``golden_tasks.json``. + + Returns: + The measured scalar, or ``None`` when the KPI was not produced. + """ + if kpi_name in UNIVERSAL_KPIS and not self.has_completed_episodes(): + return None + return self.kpi_value(kpi_name) + + def to_dict(self) -> dict[str, Any]: + """Serialize to the on-disk JSON shape (plain dicts, wire-stable order).""" + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> GoldenResult: + """Reconstruct from a parsed ``golden_result.json``. + + Unknown keys are dropped; missing required identity fields raise. + """ + known = {f.name for f in fields(cls)} + return cls(**{k: v for k, v in data.items() if k in known}) diff --git a/tools/perf_smoke_test/golden_kpi.py b/tools/perf_smoke_test/golden_kpi.py new file mode 100644 index 000000000000..583f2c1a33d5 --- /dev/null +++ b/tools/perf_smoke_test/golden_kpi.py @@ -0,0 +1,172 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Hard-set KPI thresholds for golden correctness checks. + +The golden gate is *standalone*: unlike the performance gate (which compares a +measured FPS against a rolling baseline window), a golden KPI is judged against a +fixed, blessed target that ships in ``golden_tasks.json``. :class:`KpiThreshold` +is the golden analogue of :class:`~gate_types.FpsMeanThreshold`, with one added +degree of freedom: a :class:`KpiDirection`, because behavioural KPIs cross in +both directions (a *reward* below its floor is bad; a *pole angle* above its +ceiling is bad), whereas an FPS threshold is always a floor. + +Verdict primitives (:class:`~gate_types.OracleVerdict`, the gating-verdict set) +are shared with the performance gate; only the threshold shape differs. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from enum import Enum + +try: + from .gate_types import THRESHOLD_VERDICTS, OracleVerdict +except ImportError: # pragma: no cover - supports direct script imports + from gate_types import THRESHOLD_VERDICTS, OracleVerdict + + +class KpiDirection(str, Enum): + """Direction in which a KPI value crosses (violates) its threshold. + + ``FLOOR`` covers "more is better" KPIs (reward, success rate, survival + length): a value *below* the threshold is a violation. ``CEILING`` covers + "less is better" KPIs (e.g. a pole-angle bound, a drift metric): a value + *above* the threshold is a violation. + """ + + FLOOR = "floor" + CEILING = "ceiling" + + +@dataclass(frozen=True) +class KpiThreshold: + """A single hard-set threshold for one behavioural KPI of a golden task. + + A threshold is *crossed* when the measured KPI violates it in its + :attr:`direction`. A crossed threshold whose :attr:`verdict` is set + contributes that verdict to the golden outcome; a threshold with + :attr:`verdict` = ``None`` is *reporting-only* -- surfaced in outputs without + ever changing the verdict (useful for shipping a placeholder target before it + is calibrated and blessed). + + Args: + name: Informative label for the threshold (e.g. ``"reward-floor"``). + value: The threshold value, in the KPI's own unit. + direction: Whether a violation is below (:attr:`KpiDirection.FLOOR`) or + above (:attr:`KpiDirection.CEILING`) :attr:`value`. + verdict: Gating verdict to raise when crossed, or ``None`` for + reporting-only. + """ + + name: str + value: float + direction: KpiDirection + verdict: OracleVerdict | None + + @property + def is_gating(self) -> bool: + """Whether crossing this threshold can change the golden verdict.""" + return self.verdict is not None + + def crosses(self, measured: float) -> bool: + """Return whether ``measured`` violates this threshold in its direction.""" + if self.direction is KpiDirection.CEILING: + return measured > self.value + return measured < self.value + + def to_dict(self) -> dict: + """Serialize to the ``golden_tasks.json`` entry shape.""" + return { + "threshold_verdict": self.verdict.value if self.verdict is not None else None, + "threshold_name": self.name, + "threshold": self.value, + "direction": self.direction.value, + } + + @classmethod + def from_dict(cls, raw: dict, *, context: str = "") -> KpiThreshold | None: + """Parse and validate one raw threshold entry. + + Returns ``None`` when the entry has no ``threshold`` value (skipped), so a + KPI can be declared with an empty/placeholder entry list without gating. + + Args: + raw: One raw threshold object from ``golden_tasks.json``. + context: Optional location hint (e.g. ``"cartpole/physx/reward"``) + included in error messages. + + Raises: + TypeError: If ``raw`` is not an object. + ValueError: If ``threshold_name`` is missing/empty, ``threshold`` is + non-numeric, ``direction`` is not a known :class:`KpiDirection`, + or ``threshold_verdict`` is not a gating verdict. + """ + where = f" ({context})" if context else "" + if not isinstance(raw, dict): + raise TypeError(f"kpi threshold entry{where} must be an object, got {type(raw).__name__}") + + name = raw.get("threshold_name") + if not isinstance(name, str) or not name.strip(): + raise ValueError(f"kpi threshold entry{where} must define a non-empty 'threshold_name'") + name = name.strip() + + value = raw.get("threshold") + if value is None: + return None + try: + threshold = float(value) + except (TypeError, ValueError): + raise ValueError(f"kpi threshold entry {name!r} has non-numeric 'threshold': {value!r}") + # Reject NaN/Infinity (which ``json.load`` and ``float`` both accept): a NaN + # threshold never crosses and an infinite one is unreachable -- either would + # silently disable the gate for this KPI. + if not math.isfinite(threshold): + raise ValueError(f"kpi threshold entry {name!r} has non-finite 'threshold': {value!r}") + + # Direction defaults to FLOOR (the common "more is better" case), so only + # ceiling-style KPIs (e.g. a pole-angle bound) must declare it explicitly. + direction_raw = raw.get("direction", KpiDirection.FLOOR.value) + try: + direction = KpiDirection(direction_raw) + except ValueError: + allowed_dirs = sorted(d.value for d in KpiDirection) + raise ValueError( + f"kpi threshold entry {name!r} has invalid 'direction' {direction_raw!r}; must be one of {allowed_dirs}" + ) + + verdict_raw = raw.get("threshold_verdict") + if verdict_raw is None: + verdict: OracleVerdict | None = None + else: + allowed = sorted(v.value for v in THRESHOLD_VERDICTS) + try: + verdict = OracleVerdict(verdict_raw) + except ValueError: + raise ValueError( + f"kpi threshold entry {name!r} has invalid 'threshold_verdict' {verdict_raw!r}; " + f"must be one of {allowed} (or omitted for reporting-only)" + ) + if verdict not in THRESHOLD_VERDICTS: + raise ValueError( + f"kpi threshold entry {name!r} has non-gating 'threshold_verdict' {verdict_raw!r}; " + f"must be one of {allowed} (or omitted for reporting-only)" + ) + return cls(name=name, value=threshold, direction=direction, verdict=verdict) + + @classmethod + def from_list(cls, raw_list, *, context: str = "") -> list[KpiThreshold]: + """Parse a leaf list of raw threshold entries, dropping value-less entries.""" + if raw_list is None: + return [] + if not isinstance(raw_list, list): + raise TypeError(f"kpi threshold list ({context}) must be a list, got {type(raw_list).__name__}") + parsed: list[KpiThreshold] = [] + for i, entry in enumerate(raw_list): + threshold = cls.from_dict(entry, context=f"{context}[{i}]" if context else f"[{i}]") + if threshold is not None: + parsed.append(threshold) + return parsed diff --git a/tools/perf_smoke_test/golden_omni_github.py b/tools/perf_smoke_test/golden_omni_github.py new file mode 100644 index 000000000000..45d304e761c5 --- /dev/null +++ b/tools/perf_smoke_test/golden_omni_github.py @@ -0,0 +1,126 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Emit the omni-github test-result artifact for the golden correctness gate. + +The golden analogue of :mod:`omni_github`. Writes a manifest plus a result JSON +from the golden aggregate's scored rows, with a distinct ``test_tool_id`` and a +``custom.golden_policy.*`` namespace so omni-github ingests it as its own tool -- +keeping the performance gate's artifact byte-for-byte unchanged (the golden gate +is an add-on, not a modification of the perf reporting path). +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import Any + +try: + from .gate_types import OracleVerdict +except ImportError: # pragma: no cover - executed as a script, not a package + from gate_types import OracleVerdict + +MANIFEST_NAME = "omni-github-test-results-upload.json" +RESULT_REL_PATH = "_testoutput/golden_test_results.json" +RESULT_SCHEMA_VERSION = 1 +MANIFEST_SCHEMA_VERSION = 1 +CUSTOM_NAMESPACE = "golden_policy" +TEST_TOOL_ID = "golden-policy" + +_PASSING_VERDICTS = (OracleVerdict.PASS, OracleVerdict.WARN) + + +def _number(value: Any) -> float | int | None: + """Return a finite number for storage, or ``None`` so the field is dropped (bools rejected).""" + if value is None or isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return value if math.isfinite(value) else None + return None + + +def _drop_none(fields: dict[str, Any]) -> dict[str, Any]: + """Drop keys whose value is ``None`` (omni-github stores omitted, not null).""" + return {key: value for key, value in fields.items() if value is not None} + + +def _custom_fields(result: Any, golden_result: Any) -> dict[str, Any]: + """Project the golden verdict and result into ``custom.golden_policy`` fields.""" + return _drop_none( + { + "task_id": result.task_id, + "physics_backend": getattr(golden_result, "physics_backend", None), + "render_backend": getattr(golden_result, "render_backend", None), + "checkpoint_id": result.checkpoint_id, + "verdict": result.verdict.value, + "failure_phase": result.failure_phase, + "wall_time_s": _number(getattr(golden_result, "wall_time_s", None)), + "reward_mean": _number(result.reward_mean), + "ep_length_mean": _number(result.ep_length_mean), + "success_rate": _number(result.success_rate), + "num_episodes": _number(result.num_episodes), + } + ) + + +def _message(result: Any) -> str: + parts = [result.verdict.value] + if result.reward_mean is not None: + parts.append(f"reward={result.reward_mean:.2f}") + if result.failure_phase: + parts.append(f"phase={result.failure_phase}") + if result.note: + parts.append(result.note) + return " ".join(parts) + + +def _row(result: Any, golden_result: Any) -> dict[str, Any]: + duration = max(float(getattr(golden_result, "wall_time_s", None) or 0.0), 0.0) + passed = result.verdict in _PASSING_VERDICTS + row: dict[str, Any] = { + "test_id": f"golden.{result.task_id}::{result.backend}", + "test_name": result.backend, + "test_type": "correctness", + "passed": passed, + "duration": duration, + "custom": {CUSTOM_NAMESPACE: _custom_fields(result, golden_result)}, + } + if not passed: + row["message"] = _message(result) + return row + + +def build_result(rows, *, platform: str, app_config: str, test_tool_id: str = TEST_TOOL_ID) -> dict[str, Any]: + """Build the omni-github result payload from ``(golden_oracle_result, golden_result)`` rows.""" + return { + "result_schema_version": RESULT_SCHEMA_VERSION, + "test_tool_id": test_tool_id, + "app": {"platform": platform, "config": app_config}, + "tests": [_row(result, golden_result) for result, golden_result in rows], + } + + +def write_artifact(rows, output_dir, *, platform: str, app_config: str, test_tool_id: str = TEST_TOOL_ID) -> Path: + """Write the manifest and result JSON omni-github ingests; returns the artifact root. + + ``output_dir`` MUST be separate from the performance gate's artifact directory: + the omni-github manifest filename is fixed by the ingestion contract, so a + shared directory would clobber one gate's manifest and only one would ingest. + Upload the golden artifact as its own GitHub Actions artifact. + """ + output_dir = Path(output_dir) + result_path = output_dir / RESULT_REL_PATH + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text( + json.dumps(build_result(rows, platform=platform, app_config=app_config, test_tool_id=test_tool_id)), + encoding="utf-8", + ) + manifest = {"schema_version": MANIFEST_SCHEMA_VERSION, "result_paths": [RESULT_REL_PATH]} + (output_dir / MANIFEST_NAME).write_text(json.dumps(manifest), encoding="utf-8") + return output_dir diff --git a/tools/perf_smoke_test/golden_oracle.py b/tools/perf_smoke_test/golden_oracle.py new file mode 100644 index 000000000000..0e382ac77c91 --- /dev/null +++ b/tools/perf_smoke_test/golden_oracle.py @@ -0,0 +1,196 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Oracle layer for the golden correctness gate. + +Unlike the performance oracle (:mod:`oracle`), the golden oracle is +**standalone**: it judges a rollout's behavioural KPIs against the hard-set, +blessed thresholds shipped in ``golden_tasks.json`` -- there is no rolling +baseline window, no cross-run pooling, and therefore no compatibility hashing. +A configured KPI that is measured is compared to its thresholds; the overall +verdict is the most severe verdict any gating threshold raises. + +Verdict primitives and the bisect mapping are shared with the performance gate. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field + +try: + from .gate_types import ( + BisectVerdict, + FailurePhase, + OracleVerdict, + worst_verdict, + ) + from .golden_contracts import GoldenResult + from .golden_kpi import KpiThreshold +except ImportError: # pragma: no cover - supports direct script imports + from gate_types import ( + BisectVerdict, + FailurePhase, + OracleVerdict, + worst_verdict, + ) + from golden_contracts import GoldenResult + from golden_kpi import KpiThreshold + + +@dataclass +class GoldenOracleResult: + """Full verdict record produced by :func:`evaluate`.""" + + verdict: OracleVerdict + bisect_verdict: str + failure_phase: str | None + task_id: str + backend: str + checkpoint_id: str | None + reward_mean: float | None + ep_length_mean: float | None + success_rate: float | None + num_episodes: int | None + wall_time_s: float | None + was_retried: bool + kpi_results: list[dict] = field(default_factory=list) + note: str | None = None + + +_BISECT_BAD_PHASES: frozenset[str] = frozenset({FailurePhase.INIT.value, FailurePhase.RUNTIME.value}) + + +def _bisect_verdict(verdict: OracleVerdict, was_retried: bool, failure_phase: str | None) -> str: + """Compute the bisect-friendly label for a golden verdict. + + A golden ``BLOCK`` is a clean behavioural good->bad signal, so it maps to + ``BAD``. A HARD_FAILURE maps to ``BAD`` only when it looks like the code's + fault (an init/runtime crash); setup failures such as a missing checkpoint + map to ``SKIP`` so bisection does not blame an unrelated commit. + """ + if verdict == OracleVerdict.PASS: + return BisectVerdict.SKIP.value if was_retried else BisectVerdict.GOOD.value + if verdict == OracleVerdict.WARN: + return BisectVerdict.SKIP.value + if verdict == OracleVerdict.BLOCK: + return BisectVerdict.BAD.value + if failure_phase in _BISECT_BAD_PHASES: + return BisectVerdict.BAD.value + return BisectVerdict.SKIP.value + + +def _hard_failure(result: GoldenResult, *, note: str | None, phase: str | None = None) -> GoldenOracleResult: + verdict = OracleVerdict.HARD_FAILURE + was_retried = bool(result.was_retried) + # A config mismatch is normalized to CONFIG_MISMATCH so the bisect label is + # SKIP (a setup problem), not BAD -- mirroring the perf oracle. + failure_phase = phase if phase is not None else result.failure_phase + return GoldenOracleResult( + verdict=verdict, + bisect_verdict=_bisect_verdict(verdict, was_retried, failure_phase), + failure_phase=failure_phase, + task_id=result.task_id, + backend=result.backend_key or result.backend, + checkpoint_id=result.checkpoint_id, + reward_mean=result.reward_mean, + ep_length_mean=result.ep_length_mean, + success_rate=result.success_rate, + num_episodes=result.num_episodes, + wall_time_s=result.wall_time_s, + was_retried=was_retried, + note=note, + ) + + +def evaluate(result: GoldenResult, kpis: dict[str, list[KpiThreshold]]) -> GoldenOracleResult: + """Judge a golden rollout's KPIs against their hard-set thresholds. + + Args: + result: The rollout artifact for one (task, backend). + kpis: The task's configured KPI thresholds, keyed by KPI name. + + Returns: + A :class:`GoldenOracleResult` whose verdict is the most severe raised by + any gating threshold. A run that produced no rollout output is a + HARD_FAILURE; a configured KPI that was not measured this run is noted + but does not gate on its own (a good stability policy may legitimately + complete no episode, leaving ``reward`` unmeasured). If nothing at all was + measured, the verdict is WARN. + """ + was_retried = bool(result.was_retried) + + if result.config_mismatch or result.failure_phase == FailurePhase.CONFIG_MISMATCH.value: + note = str(result.config_mismatch) if result.config_mismatch else "config_mismatch" + return _hard_failure(result, note=note, phase=FailurePhase.CONFIG_MISMATCH.value) + if not result.golden_info_present: + return _hard_failure(result, note="no_golden_output") + + notes: list[str] = [] + kpi_results: list[dict] = [] + verdicts: list[OracleVerdict] = [] + measured_count = 0 + + for kpi_name, thresholds in kpis.items(): + measured = result.measured_kpi(kpi_name) + if measured is None: + notes.append(f"unmeasured:{kpi_name}") + kpi_results.append({"kpi": kpi_name, "measured": None, "verdict": None, "crossed_thresholds": []}) + continue + # A non-finite KPI (NaN/inf) means a diverged/garbage rollout: it crosses no + # floor or ceiling, so without this guard it would silently PASS. Fail the + # whole run, mirroring the perf oracle's non-numeric/<=0 dead-run handling. + if not math.isfinite(measured): + return _hard_failure(result, note=f"non_finite:{kpi_name}={measured}") + measured_count += 1 + + crossed = [t for t in thresholds if t.crosses(measured)] + kpi_verdict = OracleVerdict.PASS + for t in crossed: + if t.is_gating: + kpi_verdict = worst_verdict(kpi_verdict, t.verdict) + verdicts.append(kpi_verdict) + kpi_results.append( + { + "kpi": kpi_name, + "measured": measured, + "verdict": kpi_verdict.value, + "crossed_thresholds": [ + { + "threshold_name": t.name, + "threshold": t.value, + "direction": t.direction.value, + "threshold_verdict": t.verdict.value if t.verdict is not None else None, + "gating": t.is_gating, + } + for t in crossed + ], + } + ) + + verdict = worst_verdict(*verdicts) if verdicts else OracleVerdict.PASS + if measured_count == 0: + verdict = worst_verdict(verdict, OracleVerdict.WARN) + notes.append("no_kpi_measured") + if verdict == OracleVerdict.PASS and was_retried: + verdict = OracleVerdict.WARN + notes.append("was_retried") + + return GoldenOracleResult( + verdict=verdict, + bisect_verdict=_bisect_verdict(verdict, was_retried, result.failure_phase), + failure_phase=result.failure_phase, + task_id=result.task_id, + backend=result.backend_key or result.backend, + checkpoint_id=result.checkpoint_id, + reward_mean=result.reward_mean, + ep_length_mean=result.ep_length_mean, + success_rate=result.success_rate, + num_episodes=result.num_episodes, + wall_time_s=result.wall_time_s, + was_retried=was_retried, + kpi_results=kpi_results, + note="; ".join(notes) if notes else None, + ) diff --git a/tools/perf_smoke_test/golden_probes.py b/tools/perf_smoke_test/golden_probes.py new file mode 100644 index 000000000000..f194cb9a1936 --- /dev/null +++ b/tools/perf_smoke_test/golden_probes.py @@ -0,0 +1,69 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Optional per-task simulation-state probes for golden correctness checks. + +A *probe* reads named scalar signals directly from the live environment's +simulation state -- e.g. a cartpole's pole angle -- so a golden check can assert +on *physical* correctness beyond the reward / episode-length / success-rate +signals that the rollout already produces on its own. + +Probes are deliberately **optional**. A task with no registered probe is +evaluated on reward / ep_length / success_rate alone (which require no task +specific code, since the environment reports them). The probe registry is the +single, contained escape hatch for the open-ended "query arbitrary sim state" +case, so that adding one is a few lines of typed code rather than a new pipeline. + +Probe contract:: + + probe_sim_state(env) -> dict[str, float] + +The callable receives the *unwrapped-compatible* Gym environment and returns a +mapping of signal name to a scalar reduced over the parallel environments (e.g. +the mean pole angle across all envs at the current step). Each returned name +must match a ``kpis`` key configured for the task in ``golden_tasks.json`` so the +oracle can threshold it. Reading simulation state forces a GPU->CPU sync, so a +probe should read only what the configured KPIs need and stay cheap. + +v1 status: the seam is plumbed end-to-end -- the driver calls the probe when one +is registered, and its values flow through :attr:`GoldenResult.probe_kpis +` into the oracle -- but the registry ships **empty**; no task +registers a probe yet. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +# A probe maps a live environment to named scalar sim-state signals for one step. +ProbeFn = Callable[[Any], dict[str, float]] + +# Intentionally empty in v1: the pipeline supports probes, but none are defined. +# Keyed by gym task id (namespace/version-insensitive callers should normalize +# before lookup if needed; v1 matches on the exact configured task id). +_PROBES: dict[str, ProbeFn] = {} + + +def register_probe(task_id: str, probe: ProbeFn) -> None: + """Register a sim-state probe for a task id. + + Args: + task_id: Gym task id the probe applies to. + probe: Callable implementing the :data:`ProbeFn` contract. + + Raises: + ValueError: If a probe is already registered for ``task_id`` (probes are + unique per task; re-registration is a configuration error rather than + a silent override). + """ + if task_id in _PROBES: + raise ValueError(f"a sim-state probe is already registered for {task_id!r}") + _PROBES[task_id] = probe + + +def get_probe(task_id: str) -> ProbeFn | None: + """Return the registered sim-state probe for ``task_id``, or ``None`` if unset.""" + return _PROBES.get(task_id) diff --git a/tools/perf_smoke_test/golden_result_adapter.py b/tools/perf_smoke_test/golden_result_adapter.py new file mode 100644 index 000000000000..7f6187251324 --- /dev/null +++ b/tools/perf_smoke_test/golden_result_adapter.py @@ -0,0 +1,96 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Adapter: schema-v1 ``PlayBundle`` JSON -> golden-gate normalized fields. + +``golden_runtime.py`` emits a :class:`~isaaclab.test.benchmark.schema.PlayBundle` +(a checkpoint-driven inference rollout) serialized by +:func:`~isaaclab.test.benchmark.serialize.write_bundle_file`. This module is the +single point that reads that JSON and projects it into the typed +:class:`~golden_contracts.GoldenSample` the golden :mod:`build_golden_result` and +:mod:`golden_oracle` consume -- the analogue of +:mod:`benchmark_result_adapter` on the performance side. + +The generic bundle sections (provenance, resource utilisation, startup, render +backend) are shared between the runtime and play bundles, so this module reuses +those projections from :mod:`benchmark_result_adapter` and only adds the +play-specific behavioural aggregates (reward / episode length / success rate / +episode count / checkpoint path). +""" + +from __future__ import annotations + +from typing import Any + +from benchmark_result_adapter import provenance, render_backend, runtime_resources, startup_seconds +from golden_contracts import GoldenSample + + +def _as_dict(value: Any) -> dict: + """Return ``value`` if it is a dict, else an empty dict (graceful on malformed bundles).""" + return value if isinstance(value, dict) else {} + + +def _num(value: Any) -> float | None: + """Return ``value`` as a float, or ``None`` if not a real number (bools rejected).""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) + + +def _int(value: Any) -> int | None: + """Return ``value`` as an int, or ``None`` if not coercible.""" + try: + return int(value) + except (TypeError, ValueError): + return None + + +def is_play_bundle(data: Any) -> bool: + """Return True when ``data`` looks like a schema-v1 play bundle. + + Distinguishes a play bundle from a runtime bundle by the presence of the + inference-evaluation fields (``success_rate``/``reward``/``ep_length``), which + a runtime bundle never serializes. + """ + return ( + isinstance(data, dict) + and isinstance(data.get("run"), dict) + and "schema_version" in data + and all(key in data for key in ("success_rate", "reward", "ep_length")) + ) + + +def project_play(bundle: dict) -> GoldenSample | None: + """Project a play bundle into a typed :class:`~golden_contracts.GoldenSample`. + + Returns ``None`` when ``bundle`` is not a valid schema-v1 play bundle, so the + caller can degrade to a HARD_FAILURE (missing/invalid golden output). + """ + if not is_play_bundle(bundle): + return None + run = _as_dict(bundle.get("run")) + config = _as_dict(run.get("config")) + extra = _as_dict(bundle.get("extra")) + reward = _as_dict(bundle.get("reward")) + ep_length = _as_dict(bundle.get("ep_length")) + return GoldenSample( + reward_mean=_num(reward.get("mean")), + reward_std=_num(reward.get("std")), + ep_length_mean=_num(ep_length.get("mean")), + ep_length_std=_num(ep_length.get("std")), + success_rate=_num(bundle.get("success_rate")), + num_episodes=_int(extra.get("num_episodes")), + checkpoint_path=bundle.get("checkpoint_path"), + startup_time_s=startup_seconds(bundle), + task=run.get("task"), + num_envs=_int(run.get("num_envs")), + seed=_int(run.get("seed")), + physics_backend=config.get("physics_backend"), + render_backend=render_backend(bundle), + presets=config.get("presets") or [], + provenance=provenance(bundle), + runtime_resources=runtime_resources(bundle), + ) diff --git a/tools/perf_smoke_test/golden_runtime.py b/tools/perf_smoke_test/golden_runtime.py new file mode 100644 index 000000000000..0d90f1b53f53 --- /dev/null +++ b/tools/perf_smoke_test/golden_runtime.py @@ -0,0 +1,315 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Golden correctness rollout driver (checkpoint-driven, deterministic). + +The golden analogue of :mod:`perf_runtime`. Rolls out a frozen policy on a task +for a fixed number of deterministic steps and emits a schema-v1 +:class:`~isaaclab.test.benchmark.schema.PlayBundle` (behavioural aggregates: +reward, episode length, success rate), which ``build_golden_result.py`` then +normalizes into ``golden_result.json``. + +Like :mod:`perf_runtime`, it imports only the *merged* Part-1 benchmark building +blocks (:mod:`~isaaclab.test.benchmark.stepping`/``builders``/``capture``) and +reuses :func:`~isaaclab.test.benchmark.stepping.run_play_loop`; it does **not** +depend on the still-unmerged ``scripts/benchmarks`` play scripts. + +Two policy sources: + +* ``--checkpoint ``: load a trained RSL-RL policy from a **local** + file (baked into the runner image at publish time). The upstream Nucleus + fallback is intentionally *not* used, so the runner needs no network. +* ``--dummy_policy``: a zero-action policy that needs no checkpoint. This is a + development aid to validate the rollout -> bundle -> result pipeline on a GPU + without a blessed checkpoint; it is not a correctness reference. + +Usage example:: + + ./isaaclab.sh -p tools/perf_smoke_test/golden_runtime.py \\ + --task Isaac-Cartpole-Direct --num_envs 64 --eval_steps 200 \\ + --dummy_policy --output_path /tmp/golden_out \\ + presets=newton_mjwarp --headless +""" + +"""Launch Isaac Sim Simulator first.""" + +import argparse +import sys +import time +from pathlib import Path + +from isaaclab.app import AppLauncher + +from isaaclab_tasks.utils import setup_preset_cli + +# --- argument parsing ------------------------------------------------------- +parser = argparse.ArgumentParser(description="Golden correctness rollout (checkpoint-driven, deterministic).") +parser.add_argument("--task", type=str, required=True, help="Gym task id to roll out.") +parser.add_argument("--num_envs", type=int, default=None, help="Number of parallel environments.") +parser.add_argument("--eval_steps", type=int, default=200, help="Number of environment steps to roll out.") +parser.add_argument("--seed", type=int, default=None, help="Environment seed (determinism).") +parser.add_argument( + "--checkpoint", + type=str, + default=None, + help="Local checkpoint path to roll out (never fetched over the network). Omit with --dummy_policy.", +) +parser.add_argument( + "--dummy_policy", + action="store_true", + help="Dev only: use a deterministic dummy policy instead of a checkpoint (validates the pipeline, not correctness).", +) +parser.add_argument( + "--dummy_mode", + type=str, + default="zero", + choices=["zero", "stabilizer"], + help="Dummy policy behaviour (with --dummy_policy): 'zero' (failure-like) or 'stabilizer' (recognizable signal).", +) +parser.add_argument("--dummy_kp", type=float, default=8.0, help="Stabilizer proportional gain on obs[:, 0].") +parser.add_argument("--dummy_kd", type=float, default=1.0, help="Stabilizer derivative gain on obs[:, 1].") +parser.add_argument("--rl_library", type=str, default="rsl_rl", help="RL library the checkpoint was trained with.") +parser.add_argument("--agent", type=str, default="rsl_rl_cfg_entry_point", help="RL agent config entry point.") +parser.add_argument("--output_path", type=str, default=".", help="Directory to write the output JSON.") +parser.add_argument( + "--benchmark_formatter", + type=str, + default="schema", + help="Output format(s): comma-separated 'schema' (default), 'omniperf', 'osmo', 'json', 'summary'.", +) + +# append AppLauncher cli args and resolve Hydra preset tokens +AppLauncher.add_app_launcher_args(parser) +args_cli, hydra_args = setup_preset_cli(parser) +sys.argv = [sys.argv[0]] + hydra_args + +if not args_cli.dummy_policy and not args_cli.checkpoint: + parser.error("either --checkpoint or --dummy_policy is required") +if args_cli.dummy_policy and args_cli.checkpoint: + print( + "[golden_runtime] WARNING: --dummy_policy given; ignoring --checkpoint " + "(zero-action policy is a pipeline aid, not a correctness reference).", + flush=True, + ) + +# --- heavy imports (after CLI parse, before app launch is measured) --------- +imports_time_begin = time.perf_counter_ns() + +import contextlib + +import gymnasium as gym + +from isaaclab.app import launch_simulation +from isaaclab.test.benchmark import BaseIsaacLabBenchmark, BenchmarkMonitor, builders, capture, stepping +from isaaclab.test.benchmark.schema import StartupTime + +import isaaclab_tasks # noqa: F401 +from isaaclab_tasks.utils import resolve_task_config + +# PLACEHOLDER: Extension template (do not remove this comment) +with contextlib.suppress(ImportError): + import isaaclab_tasks_experimental # noqa: F401 + +imports_time_end = time.perf_counter_ns() + + +def _dummy_policy(env, mode: str, kp: float, kd: float): + """Return a deterministic dummy policy callable for pipeline validation (no checkpoint). + + Two modes: + + * ``zero``: a task-agnostic zero-action baseline (Cartpole pole falls almost + immediately -> very short episodes, ~0 reward). Deliberately *failure-like*; + used to exercise the low-signal / forced-failure path. + * ``stabilizer``: a fixed proportional-derivative feedback on the first two + observation components (for Cartpole, pole angle + angular velocity), + ``action = clamp(-(kp*obs[:,0] + kd*obs[:,1]), -1, 1)``. It *partially* balances + the pole, giving a RECOGNIZABLE non-trivial signal (clearly-positive reward, + mid-range episodes, non-zero success) that cannot be conflated with an error or + a genuine no-reward result. Not a correctness reference -- a pipeline probe. + + Args: + env: The (unwrapped-compatible) environment. + mode: ``"zero"`` or ``"stabilizer"``. + kp: Proportional gain on ``obs[:, 0]`` (stabilizer mode). + kd: Derivative gain on ``obs[:, 1]`` (stabilizer mode). + """ + import torch # noqa: PLC0415 + + u = env.unwrapped + num_envs, device = u.num_envs, u.device + action_dim = int(env.action_space.shape[-1]) + + if mode == "zero": + zeros = torch.zeros((num_envs, action_dim), device=device) + return lambda obs: zeros + + def stabilizer(obs): + x = obs[0] if isinstance(obs, tuple) else obs + if isinstance(x, dict): + x = x.get("policy", next(iter(x.values()))) + signal = -(kp * x[:, 0:1] + kd * x[:, 1:2]) + return torch.clamp(signal, -1.0, 1.0).expand(num_envs, action_dim) + + return stabilizer + + +def _load_rsl_rl_policy(env, agent_cfg, checkpoint_path: str): + """Load a trained RSL-RL inference policy from a **local** checkpoint file. + + Mirrors ``scripts/benchmarks/rsl_rl/benchmark_rsl_rl_play.py`` but resolves the + checkpoint as a plain local path -- the upstream Nucleus/``retrieve_file_path`` + fallback is deliberately avoided so the runner performs no network access. + """ + import importlib.metadata as metadata # noqa: PLC0415 + + from rsl_rl.runners import DistillationRunner, OnPolicyRunner # noqa: PLC0415 + + from isaaclab_rl.rsl_rl import RslRlVecEnvWrapper, handle_deprecated_rsl_rl_cfg # noqa: PLC0415 + + if not Path(checkpoint_path).is_file(): + raise FileNotFoundError(f"golden checkpoint not found at local path: {checkpoint_path!r}") + + agent_cfg = handle_deprecated_rsl_rl_cfg(agent_cfg, metadata.version("rsl-rl-lib")) + env = RslRlVecEnvWrapper(env, clip_actions=agent_cfg.clip_actions) + if agent_cfg.class_name == "OnPolicyRunner": + runner = OnPolicyRunner(env, agent_cfg.to_dict(), log_dir=None, device=agent_cfg.device) + elif agent_cfg.class_name == "DistillationRunner": + runner = DistillationRunner(env, agent_cfg.to_dict(), log_dir=None, device=agent_cfg.device) + else: + raise ValueError(f"Unsupported RSL-RL runner class: {agent_cfg.class_name}") + runner.load(checkpoint_path) + return env, runner.get_inference_policy(device=env.unwrapped.device) + + +def main(env_cfg, agent_cfg, app_start_time_begin: int, app_start_time_end: int) -> None: + """Run the golden rollout and write the selected formatter outputs. + + Args: + env_cfg: Resolved environment configuration for :attr:`args_cli.task`. + agent_cfg: Resolved RL agent configuration, or ``None`` in ``--dummy_policy`` mode. + app_start_time_begin: ``perf_counter_ns`` sampled just before the app launch. + app_start_time_end: ``perf_counter_ns`` sampled just after the app launch. + """ + if args_cli.num_envs is not None: + env_cfg.scene.num_envs = args_cli.num_envs + if args_cli.device is not None: + env_cfg.sim.device = args_cli.device + if args_cli.seed is not None: + env_cfg.seed = args_cli.seed + + cfg = capture.run_config_from_presets(hydra_args, env_cfg=env_cfg) + start_utc = capture.now_utc_iso() + + benchmark = BaseIsaacLabBenchmark( + benchmark_name="benchmark_play", + formatter_type=args_cli.benchmark_formatter, + output_path=args_cli.output_path, + use_recorders=True, + frametime_recorders=False, + output_prefix=f"benchmark_play_{args_cli.task}", + workflow_metadata={ + "metadata": [ + {"name": "task", "data": args_cli.task}, + {"name": "num_envs", "data": args_cli.num_envs}, + {"name": "eval_steps", "data": args_cli.eval_steps}, + {"name": "presets", "data": ",".join(cfg.presets)}, + ] + }, + ) + + env_t0 = time.perf_counter_ns() + with contextlib.closing(gym.make(args_cli.task, cfg=env_cfg)) as base_env: + env_t1 = time.perf_counter_ns() + + if args_cli.dummy_policy: + env = base_env + policy = _dummy_policy(env, args_cli.dummy_mode, args_cli.dummy_kp, args_cli.dummy_kd) + checkpoint_path = None + else: + env, policy = _load_rsl_rl_policy(base_env, agent_cfg, args_cli.checkpoint) + checkpoint_path = args_cli.checkpoint + + num_envs = env.unwrapped.num_envs + + with BenchmarkMonitor(benchmark, interval=1.0): + step_times, reward, ep_length, success_rate = stepping.run_play_loop(env, policy, args_cli.eval_steps) + + # Progress marker consumed by subprocess_runner.classify_failure_phase (shared + # stdout contract) so a runtime-phase crash is not misread as an init failure. + print("Step Frametimes", flush=True) + + benchmark.update_manual_recorders() + + startup = StartupTime( + app_launch=(app_start_time_end - app_start_time_begin) / 1e9, + env_creation=(env_t1 - env_t0) / 1e9, + first_step=(step_times[0] if step_times else 0.0), + python_imports=(imports_time_end - imports_time_begin) / 1e9, + ) + fps = [num_envs / t for t in step_times if t > 0] + runtime = builders.build_runtime( + startup_time_s=startup, + iteration_times_s=step_times, + collection_fps=fps, + total_fps=fps, + steps_per_iteration=num_envs, + ) + + versions = capture.capture_versions(benchmark) + hardware = capture.capture_hardware(benchmark) + resources = capture.capture_resources(benchmark) + + end_utc = capture.now_utc_iso() + stamp = end_utc.translate(str.maketrans("", "", ":-"))[:15] + seed = args_cli.seed if args_cli.seed is not None else 0 + run_id = capture.synth_run_id(args_cli.rl_library, cfg.physics_backend, args_cli.task, seed, stamp) + + run = builders.build_run_identity( + run_id=run_id, + framework=args_cli.rl_library, + config=cfg, + task=args_cli.task, + seed=seed, + start_utc=start_utc, + end_utc=end_utc, + num_envs=num_envs, + ) + + # ``run_play_loop`` returns only the aggregates, not a completed-episode + # count, so ``num_episodes`` is intentionally not recorded here. A true + # zero-episode run yields reward/ep_length/success_rate = None, which the + # oracle already treats as unmeasured -- that None contract, not an episode + # count, is what protects the golden verdict from an unfinished rollout. + bundle = builders.build_play_bundle( + run=run, + versions=versions, + hardware=hardware, + runtime=runtime, + resources=resources, + success_rate=success_rate, + reward=reward, + ep_length=ep_length, + checkpoint_path=checkpoint_path, + extra={ + "eval_steps": args_cli.eval_steps, + "dummy_policy": bool(args_cli.dummy_policy), + "dummy_mode": (args_cli.dummy_mode if args_cli.dummy_policy else "checkpoint"), + }, + ) + + benchmark.attach_bundle(bundle) + benchmark._finalize_impl() + + +if __name__ == "__main__": + # Dummy mode needs no agent config; real mode resolves it for the checkpoint runner. + agent_ref = None if args_cli.dummy_policy else args_cli.agent + env_cfg, agent_cfg = resolve_task_config(args_cli.task, agent_ref) + + app_start_time_begin = time.perf_counter_ns() + with launch_simulation(env_cfg, args_cli): + app_start_time_end = time.perf_counter_ns() + main(env_cfg, agent_cfg, app_start_time_begin, app_start_time_end) diff --git a/tools/perf_smoke_test/golden_tasks.json b/tools/perf_smoke_test/golden_tasks.json new file mode 100644 index 000000000000..8488d539babc --- /dev/null +++ b/tools/perf_smoke_test/golden_tasks.json @@ -0,0 +1,38 @@ +{ + "_comment": "Golden correctness tasks (sibling of tasks.json). KPI thresholds ship as reporting-only (threshold_verdict=null) until calibrated via the offline sweep and blessed by CODEOWNERS; the golden gate is advisory (gate_config.golden_blocking=false) regardless. checkpoint.path is relative to a baked golden-checkpoint root; the orchestration resolves it to an absolute local path via golden_config.resolve_checkpoint_path(golden_root) and passes it to golden_runtime.py --checkpoint (never fetched over the network on the runner). {task_id} and {backend_key} in checkpoint id/path are expanded per backend at load time.", + "defaults": { + "type": "golden", + "runs_on": "gpu-l40s", + "preset": "default", + "rl_library": "rsl_rl", + "agent": "rsl_rl_cfg_entry_point", + "num_envs": 64, + "eval_steps": 500, + "seed": 42, + "deterministic": true, + "timeout_minutes": 10, + "tags": ["always"] + }, + "tasks": [ + { + "task_id": "Isaac-Cartpole-Direct", + "checkpoint": { + "id": "cartpole-{backend_key}-v1", + "path": "Isaac-Cartpole-Direct/{backend_key}/policy.pt" + }, + "backends": [ + {"physics": "physx"}, + {"physics": "newton"} + ], + "kpis": { + "reward": [ + {"threshold_name": "reward-floor", "threshold": 0.0, "threshold_verdict": null} + ], + "ep_length": [ + {"threshold_name": "survival-floor", "threshold": 0.0, "threshold_verdict": null} + ], + "success_rate": [] + } + } + ] +} diff --git a/tools/perf_smoke_test/tasks_to_ci_matrix.py b/tools/perf_smoke_test/tasks_to_ci_matrix.py index 6e0f5661c277..6a2b706874d2 100644 --- a/tools/perf_smoke_test/tasks_to_ci_matrix.py +++ b/tools/perf_smoke_test/tasks_to_ci_matrix.py @@ -3,11 +3,27 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Convert tasks.json into the GitHub Actions bench matrix JSON +"""Convert tasks.json into the GitHub Actions bench matrix JSON. Prints a JSON array to stdout, one object per (task_id, backend) combination, containing the fields consumed by the ``bench`` job matrix in perf-smoke-test.yaml. +Each row is optionally *enriched* with golden-correctness fields when a matching +(task_id, backend) exists in ``golden_tasks.json``: the golden rollout runs as an +optional second stage inside the same bench job (same warm runner, no extra image +pull), so its run-shape rides on the same matrix cell and ``job_timeout_minutes`` +auto-includes the golden budget. Golden is fully optional -- absent/parse-failing +golden config simply leaves ``golden_present`` false and the perf budget unchanged. + +CAVEAT (golden requires a perf pair): because golden piggybacks the perf ``bench`` +cells, a golden task runs ONLY for a (task_id, backend) that ALSO has a perf entry +in ``tasks.json``. A golden config with no matching perf entry is never placed in +the matrix and simply does not run -- there is deliberately no standalone golden +job (that would force a guaranteed cold image pull on the ephemeral runner fleet, +the exact cost the piggyback design eliminates). If a golden-only task is ever +needed, add a (cheap) perf entry for it, or extend the matrix to perf-union-golden +with perf-optional cells. + Usage:: python3 tools/perf_smoke_test/tasks_to_ci_matrix.py @@ -22,22 +38,66 @@ from launch_config import hydra_args_for_task # noqa: E402 from task_config import load_tasks # noqa: E402 +# Golden enrichment is best-effort: a broken/absent golden config must never break +# perf matrix generation (golden is an optional add-on). +try: + from golden_config import get_golden_task + + _GOLDEN_AVAILABLE = True +except Exception: # pragma: no cover - golden is optional + _GOLDEN_AVAILABLE = False + +# Buffer minutes added on top of a rollout's hard timeout for retry + overhead, +# mirroring the perf gate's existing "+15" job-timeout headroom. +_PERF_JOB_BUFFER_MIN = 15 +_GOLDEN_JOB_BUFFER_MIN = 5 + + +def _golden_for(task): + """Return the matching GoldenTaskConfig for a perf task, or None (best-effort).""" + if not _GOLDEN_AVAILABLE: + return None + try: + return get_golden_task(task.task_id, task.backend_key) + except Exception: + return None + + tasks = load_tasks() rows = [] for task in tasks: - rows.append( - { - "task_id": task.task_id, - "physics_backend": task.physics_backend, - "render_backend": task.render_backend or "", - "num_envs": task.num_envs, - "num_frames": task.num_frames, - "warmup_frames": task.warmup_frames, - "seed": task.seed if task.seed is not None else "", - "hydra_args": " ".join(hydra_args_for_task(task)), - "bench_timeout_s": task.timeout_minutes * 60, - "job_timeout_minutes": max(30, task.timeout_minutes + 15), - } - ) + row = { + "task_id": task.task_id, + "physics_backend": task.physics_backend, + "render_backend": task.render_backend or "", + "num_envs": task.num_envs, + "num_frames": task.num_frames, + "warmup_frames": task.warmup_frames, + "seed": task.seed if task.seed is not None else "", + "hydra_args": " ".join(hydra_args_for_task(task)), + "bench_timeout_s": task.timeout_minutes * 60, + } + + # job_timeout_minutes auto-calculates from the perf budget plus the golden + # budget (0 when this cell has no golden config), so the shared bench job has + # headroom for perf(run+retry) + optional golden(run+retry). + golden = _golden_for(task) + golden_budget_min = 0 + if golden is not None: + row["golden_present"] = "true" + row["golden_num_envs"] = golden.num_envs + row["golden_eval_steps"] = golden.eval_steps + row["golden_seed"] = golden.seed if golden.seed is not None else "" + row["golden_hydra_args"] = " ".join(hydra_args_for_task(golden)) + row["golden_checkpoint_id"] = golden.checkpoint_id + row["golden_checkpoint_relpath"] = golden.checkpoint_relpath + row["golden_timeout_s"] = golden.timeout_minutes * 60 + golden_budget_min = golden.timeout_minutes + _GOLDEN_JOB_BUFFER_MIN + else: + row["golden_present"] = "false" + + perf_budget_min = task.timeout_minutes + _PERF_JOB_BUFFER_MIN + row["job_timeout_minutes"] = max(30, perf_budget_min + golden_budget_min) + rows.append(row) print(json.dumps(rows)) From 405f857285228a63880f446e09cae90305b469bd Mon Sep 17 00:00:00 2001 From: Angelina Hu Date: Tue, 28 Jul 2026 10:02:13 -0700 Subject: [PATCH 3/4] TEST: golden CI dry-run on RTX 6000 (Cartpole/physx + G1/physx skip) Throwaway branch (do not merge). Trims tasks.json to Cartpole/physx (golden pair) + Velocity-Flat-G1/physx (no golden -> must skip); golden_tasks.json to Cartpole/physx. Adds a TEST-only step that pulls a real rsl_rl checkpoint from the fork release 'angehu-test-golden-ckpt' into a workspace GOLDEN_ROOT and times it, so the real --checkpoint pull+load path is exercised on RTX 6000. --- .github/workflows/perf-smoke-test.yaml | 28 ++++++++++- tools/perf_smoke_test/golden_tasks.json | 9 ++-- tools/perf_smoke_test/tasks.json | 62 ++----------------------- 3 files changed, 34 insertions(+), 65 deletions(-) diff --git a/.github/workflows/perf-smoke-test.yaml b/.github/workflows/perf-smoke-test.yaml index aa32a2955927..ec560d50ba9a 100644 --- a/.github/workflows/perf-smoke-test.yaml +++ b/.github/workflows/perf-smoke-test.yaml @@ -626,12 +626,38 @@ jobs: # a dirty GPU. Golden scoring/reporting happens in the separate golden_aggregate # job (advisory, ubuntu-latest). # ========================================================================= + # TEST-ONLY (angehu-test/golden-ci-dryrun): pull the dummy golden checkpoint (a + # real rsl_rl policy) from a fork release asset into the workspace GOLDEN_ROOT, so + # the real --checkpoint resolve+load path is exercised and TIMED. Production bakes + # the checkpoint into the image instead (this step + the GOLDEN_ROOT override below + # do not merge). + - name: Golden — pull checkpoint (TEST) + id: golden_pull + if: ${{ always() && matrix.golden_present == 'true' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + GOLDEN_ROOT: ${{ github.workspace }}/golden-checkpoints + RELPATH: ${{ matrix.golden_checkpoint_relpath }} + RELEASE_TAG: angehu-test-golden-ckpt + ASSET: cartpole-physx-policy.pt + run: | + set -uo pipefail + dest="${GOLDEN_ROOT%/}/${RELPATH}" + mkdir -p "$(dirname "${dest}")" + start_ns=$(date +%s%N) + gh release download "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" --pattern "${ASSET}" --output "${dest}" --clobber + pull_ms=$(( ($(date +%s%N) - start_ns) / 1000000 )) + bytes=$(stat -c%s "${dest}" 2>/dev/null || echo 0) + echo "pull_ms=${pull_ms}" >> "${GITHUB_OUTPUT}" + echo "::notice::Golden checkpoint pull: ${pull_ms} ms, ${bytes} bytes -> ${dest}" + - name: Golden — resolve checkpoint + GPU clean id: golden_gpu if: ${{ always() && matrix.golden_present == 'true' }} continue-on-error: true env: - GOLDEN_ROOT: ${{ vars.PERF_SMOKE_GOLDEN_ROOT }} + GOLDEN_ROOT: ${{ github.workspace }}/golden-checkpoints CHECKPOINT_RELPATH: ${{ matrix.golden_checkpoint_relpath }} run: | set -uo pipefail diff --git a/tools/perf_smoke_test/golden_tasks.json b/tools/perf_smoke_test/golden_tasks.json index 8488d539babc..ac2a3e11282c 100644 --- a/tools/perf_smoke_test/golden_tasks.json +++ b/tools/perf_smoke_test/golden_tasks.json @@ -1,12 +1,12 @@ { - "_comment": "Golden correctness tasks (sibling of tasks.json). KPI thresholds ship as reporting-only (threshold_verdict=null) until calibrated via the offline sweep and blessed by CODEOWNERS; the golden gate is advisory (gate_config.golden_blocking=false) regardless. checkpoint.path is relative to a baked golden-checkpoint root; the orchestration resolves it to an absolute local path via golden_config.resolve_checkpoint_path(golden_root) and passes it to golden_runtime.py --checkpoint (never fetched over the network on the runner). {task_id} and {backend_key} in checkpoint id/path are expanded per backend at load time.", + "_comment": "TEST-BRANCH TRIM (angehu-test/golden-ci-dryrun): Cartpole/physx only, paired with the perf Cartpole/physx cell. The checkpoint is a real rsl_rl policy pulled from a fork release asset (see the 'Golden - pull checkpoint (TEST)' workflow step) and loaded via the real --checkpoint path. KPI thresholds stay reporting-only (advisory). Not for merge.", "defaults": { "type": "golden", "runs_on": "gpu-l40s", "preset": "default", "rl_library": "rsl_rl", "agent": "rsl_rl_cfg_entry_point", - "num_envs": 64, + "num_envs": 128, "eval_steps": 500, "seed": 42, "deterministic": true, @@ -17,12 +17,11 @@ { "task_id": "Isaac-Cartpole-Direct", "checkpoint": { - "id": "cartpole-{backend_key}-v1", + "id": "cartpole-{backend_key}-testdummy", "path": "Isaac-Cartpole-Direct/{backend_key}/policy.pt" }, "backends": [ - {"physics": "physx"}, - {"physics": "newton"} + {"physics": "physx"} ], "kpis": { "reward": [ diff --git a/tools/perf_smoke_test/tasks.json b/tools/perf_smoke_test/tasks.json index 030b635c194e..a4ca6448f6d3 100644 --- a/tools/perf_smoke_test/tasks.json +++ b/tools/perf_smoke_test/tasks.json @@ -1,4 +1,5 @@ { + "_comment": "TEST-BRANCH TRIM (angehu-test/golden-ci-dryrun): Cartpole/physx (has a golden pair -> golden runs) + Velocity-Flat-G1/physx (no golden pair -> golden must SKIP). Not for merge.", "defaults": { "type": "benchmark", "runs_on": "gpu-l40s", @@ -15,58 +16,13 @@ { "task_id": "Isaac-Cartpole-Direct", "num_envs": 4096, - "backends": [ - {"physics": "physx"}, - {"physics": "newton"} - ], - "fps_mean_thresholds": { - "l40s": { - "physx": [ - {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 100.0} - ], - "newton": [ - {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 0.0} - ] - } - } - }, - { - "task_id": "IsaacContrib-Factory-GearMesh-Direct", - "timeout_minutes": 15, "backends": [ {"physics": "physx"} ], "fps_mean_thresholds": { "l40s": { "physx": [ - {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 30.0} - ] - } - } - }, - { - "task_id": "Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct", - "timeout_minutes": 20, - "tags": ["camera"], - "backends": [ - {"physics": "physx"}, - {"physics": "physx", "render": "newton_renderer"}, - {"physics": "newton"}, - {"physics": "newton", "render": "newton_renderer"} - ], - "fps_mean_thresholds": { - "l40s": { - "physx": [ - {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 20.0} - ], - "physx_newton_renderer": [ - {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 0.0} - ], - "newton": [ - {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 0.0} - ], - "newton_newton_renderer": [ - {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 0.0} + {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 100.0} ] } } @@ -75,26 +31,14 @@ "task_id": "Isaac-Velocity-Flat-G1", "timeout_minutes": 12, "backends": [ - {"physics": "physx"}, - {"physics": "newton"} + {"physics": "physx"} ], "fps_mean_thresholds": { "l40s": { "physx": [ {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 40.0} - ], - "newton": [ - {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 0.0} ] } - }, - "noise_floor_pct": { - "rtx_pro_6000_blackwell": { - "newton": 2.52 - }, - "l40s": { - "newton": 2.06 - } } } ] From 693f2d937dff12e4f9bf28b5167393862d16f66e Mon Sep 17 00:00:00 2001 From: Angelina Hu Date: Tue, 28 Jul 2026 18:11:33 -0700 Subject: [PATCH 4/4] Use 'Golden Policy Correctness' as the user-facing title --- .github/workflows/perf-smoke-test.yaml | 2 +- tools/perf_smoke_test/golden_aggregate.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/perf-smoke-test.yaml b/.github/workflows/perf-smoke-test.yaml index ec560d50ba9a..3ab8c7fcf868 100644 --- a/.github/workflows/perf-smoke-test.yaml +++ b/.github/workflows/perf-smoke-test.yaml @@ -1131,7 +1131,7 @@ jobs: # the slow network and out of perf's authoritative reporting path). # --------------------------------------------------------------------------- golden_aggregate: - name: Golden Aggregate + name: Golden Policy Correctness Aggregate runs-on: ubuntu-latest needs: [config, bench] if: >- diff --git a/tools/perf_smoke_test/golden_aggregate.py b/tools/perf_smoke_test/golden_aggregate.py index 00070e0373a4..85b1522fa081 100644 --- a/tools/perf_smoke_test/golden_aggregate.py +++ b/tools/perf_smoke_test/golden_aggregate.py @@ -134,13 +134,13 @@ def main() -> int: has_hard_failure = True table = _build_table(rows) - print("\n## Golden Correctness Results\n") + print("\n## Golden Policy Correctness Results\n") print(table) print() if args.summary_file: with open(args.summary_file, "a") as fh: - fh.write("\n## Golden Correctness Results\n\n") + fh.write("\n## Golden Policy Correctness Results\n\n") fh.write(table) fh.write("\n")