diff --git a/.github/workflows/perf-smoke-seed-baselines.yaml b/.github/workflows/perf-smoke-seed-baselines.yaml new file mode 100644 index 000000000000..56f9eadcdafa --- /dev/null +++ b/.github/workflows/perf-smoke-seed-baselines.yaml @@ -0,0 +1,345 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Seed the perf-baselines branch from real commit history. +# +# This is the deployment-simulation counterpart to the live gate: instead of +# benchmarking one PR head, it walks a slice of a protected branch's history, +# re-runs each commit's own benchmark inside the CI image (source-mounted so the +# container git-tags the real commit), and appends the results to perf-baselines. +# The samples are keyed by their real commit SHA, so the gate's merge-base / +# ancestry isolation has a populated, branch-correct baseline to compare against. +# A preflight drops any seed commit that is not an ancestor of the target branch +# tip (the gate would silently ignore it), and a post-publish step replays the +# gate's per-bucket match logic to confirm every bucket has MIN_BASELINE_SAMPLES +# usable samples -- i.e. that the seeded baselines will actually be used. +# +# Resolves the CI image exactly like the gate: the era-pinned immutable image for +# the seeded commit's container era when recorded in the manifest on perf-baselines, +# otherwise the moving prebuilt tag. Seeding through the same resolver guarantees +# baselines and later PR runs share an identical environment. Set repo variable +# PERF_SMOKE_CI_IMAGE only to override that default, e.g. to pin an immutable tag. +# +# Manual dispatch defaults to dry-run so maintainers can validate the L40S +# runner/image path before explicitly choosing dry_run=false to publish samples. +# There is no push trigger: seeding is expensive, so it runs only when a +# maintainer dispatches it or when the gate calls it to refill an under-filled +# bucket. + +name: Performance Smoke - Seed Baselines + +on: + workflow_dispatch: + inputs: + branches: + description: "Branches to seed in one run (comma/space separated). Use 'branch:target' to override the stamp. Empty = use commit_branch/commits. NOTE: only seed branches whose code matches the prebuilt CI image's era." + required: false + default: "develop" + commits: + description: "Explicit commit SHAs/refs (space/comma separated). Used only when branches is empty." + required: false + default: "" + commit_branch: + description: "Single branch to seed when branches and commits are empty." + required: false + default: "develop" + commit_count: + description: "Number of recent commits to seed from commit_branch." + required: false + default: "5" + samples_per_commit: + description: "Benchmark repetitions per commit/backend." + required: false + default: "5" + tasks: + description: "Comma-separated task_id allowlist (empty = all tasks.json tasks)." + required: false + default: "Isaac-Cartpole-Direct" + backends: + description: "Comma-separated backend_key allowlist (empty = all backends)." + required: false + default: "" + target_branch: + description: "Protected branch stamped onto each seeded sample." + required: false + default: "develop" + strict_ancestry: + description: "Abort if any seed commit is not an ancestor of the target branch tip (default: skip+warn). Enable when you require every seeded sample to be gate-usable." + type: boolean + required: false + default: false + dry_run: + description: "Run benchmarks + build samples but DO NOT push to perf-baselines." + type: boolean + required: false + default: true + # Reusable: the gate's reseed job calls this to fill under-filled buckets. + # Callers provide the NGC credential explicitly or via `secrets: inherit`. + # The booleans are declared explicitly so a missing value can never coerce to + # "publish". + workflow_call: + inputs: + branches: + description: "Branches/refs to seed ('ref:target' overrides the stamp)." + required: false + default: "develop" + type: string + commits: + description: "Explicit commit SHAs/refs (space/comma separated). Used only when branches is empty." + required: false + default: "" + type: string + commit_branch: + description: "Single branch to seed when branches and commits are empty." + required: false + default: "develop" + type: string + commit_count: + description: "Number of recent commits to seed per branch/ref." + required: false + default: "5" + type: string + samples_per_commit: + description: "Benchmark repetitions per commit/backend." + required: false + default: "5" + type: string + tasks: + description: "Comma-separated task_id allowlist (empty = all tasks.json tasks)." + required: false + default: "Isaac-Cartpole-Direct" + type: string + backends: + description: "Comma-separated backend_key allowlist (empty = all backends)." + required: false + default: "" + type: string + target_branch: + description: "Protected branch stamped onto each seeded sample." + required: false + default: "develop" + type: string + strict_ancestry: + description: "Abort if any seed commit is not an ancestor of the target branch tip." + type: boolean + required: false + default: false + dry_run: + description: "Run benchmarks + build samples but DO NOT push to perf-baselines." + type: boolean + required: false + default: true + secrets: + NGC_API_KEY: + description: "Optional credential for pulling private NGC images." + required: false + +concurrency: + # Serialize seeding so concurrent runs can't race the append-only baseline branch. + group: perf-smoke-seed + cancel-in-progress: false + +permissions: + contents: write # push appended samples to the perf-baselines branch + +env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + CI_IMAGE_TAG: isaac-lab-ci:seed-${{ github.run_id }} + +jobs: + config: + name: Load Config + runs-on: ubuntu-latest + outputs: + isaaclab_image_ref: ${{ steps.load.outputs.isaaclab_image_ref }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + sparse-checkout: .github/workflows/config.yaml + sparse-checkout-cone-mode: false + - id: load + run: | + set -euo pipefail + f=.github/workflows/config.yaml + ISAACLAB_IMAGE_NAME="$(yq -r .isaaclab_image_name "$f")" + echo "isaaclab_image_ref=${ISAACLAB_IMAGE_NAME}:latest-develop" >> "$GITHUB_OUTPUT" + + seed: + name: Seed baselines (${{ inputs.commit_branch || github.ref_name || 'develop' }} x${{ inputs.commit_count || '1' }}) + runs-on: ${{ fromJSON(vars.PERF_SMOKE_RUNS_ON || '["self-hosted","gpu"]') }} + needs: [config] + timeout-minutes: 600 + + steps: + - name: Checkout Code + uses: actions/checkout@v6 + with: + # Full history so historical commits are resolvable and ancestry checks work. + fetch-depth: 0 + lfs: true + + # Make the seed/target branch history and the baseline branch available to the + # orchestrator. The repo only auto-fetches the checked-out ref, so pull these + # explicitly; the clone the orchestrator makes draws its objects from here. + - name: Fetch seed + baseline refs + env: + SEED_BRANCHES: ${{ inputs.branches }} + SEED_COMMIT_BRANCH: ${{ inputs.commit_branch || 'develop' }} + run: | + set -euo pipefail + # Collect every branch we might seed from: the multi-branch list (stripping + # any ':target' suffix) plus the single-branch fallback. Fetch each so the + # orchestrator's clone can resolve their commits offline. + BRANCHES="${SEED_BRANCHES//,/ } ${SEED_COMMIT_BRANCH}" + for entry in ${BRANCHES}; do + BRANCH="${entry%%:*}" + [ -z "${BRANCH}" ] && continue + git fetch --no-tags origin "+refs/heads/${BRANCH}:refs/remotes/origin/${BRANCH}" || \ + echo "::warning::Could not fetch ${BRANCH}; skipping (explicit commits may still work)" + done + git fetch --no-tags origin "+refs/heads/perf-baselines:refs/remotes/origin/perf-baselines" || \ + echo "::notice::perf-baselines not found yet; first seed run will create it" + + # Resolve the era-pinned immutable image so seeded baselines share the exact + # image the gate will later pin for this era (no seed-vs-gate environment + # offset). An explicit PERF_SMOKE_CI_IMAGE wins outright; otherwise resolve the + # container era (docker/.env.base) against the manifest on perf-baselines and + # fall back to the config job's moving tag on a miss. + - name: Resolve era-pinned CI image + id: era + env: + FALLBACK_REF: ${{ needs.config.outputs.isaaclab_image_ref }} + EXPLICIT_IMAGE: ${{ vars.PERF_SMOKE_CI_IMAGE }} + run: | + set -euo pipefail + if [ -n "${EXPLICIT_IMAGE}" ]; then + echo "๐Ÿ”ต Using explicit PERF_SMOKE_CI_IMAGE=${EXPLICIT_IMAGE}" + echo "image_ref=${EXPLICIT_IMAGE}" >> "$GITHUB_OUTPUT" + exit 0 + fi + ERA_JSON="$(python3 tools/perf_smoke_test/image_era.py \ + --source_root . \ + --manifest_from_git \ + --branch perf-baselines \ + --remote origin \ + --fallback_image "${FALLBACK_REF}")" + echo "${ERA_JSON}" + IMAGE_REF="$(echo "${ERA_JSON}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["image"])')" + echo "image_ref=${IMAGE_REF}" >> "$GITHUB_OUTPUT" + + # Pull the published CI image and retag it locally (the exact image the gate pulls, + # so seeded baselines and PR runs share one environment). Unlike the gate there is no + # build fallback: seeding is a deliberate, baseline-writing op, so a missing image + # should fail fast. + - name: Pull prebuilt CI image + env: + CI_IMAGE_REF: ${{ steps.era.outputs.image_ref }} + run: | + set -euo pipefail + + # The runner's docker credential store backend is broken ("not implemented"), + # so disable credsStore before any login (same trick as ecr-build-push-pull). + # With credsStore disabled, docker writes the NGC key to config.json in clear + # text, so wipe the temp dir on ANY exit -- a failed docker pull must not leave + # the long-lived credential on the self-hosted runner's filesystem. + DOCKER_CONFIG_DIR="$(mktemp -d)" + trap 'rm -rf "${DOCKER_CONFIG_DIR}"' EXIT + echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json" + export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}" + + REGISTRY="${CI_IMAGE_REF%%/*}" + case "${REGISTRY}" in + nvcr.io) + if [ -n "${NGC_API_KEY:-}" ]; then + echo "๐Ÿ”ต Logging into nvcr.io..." + echo "${NGC_API_KEY}" | docker login nvcr.io -u '$oauthtoken' --password-stdin + else + echo "::warning::NGC_API_KEY not set; attempting anonymous pull from nvcr.io" + fi + ;; + *) + echo "::notice::No known login for registry '${REGISTRY}'; attempting anonymous pull" + ;; + esac + + echo "๐Ÿ”ต Pulling prebuilt image ${CI_IMAGE_REF}..." + docker pull "${CI_IMAGE_REF}" || { + echo "::error::Failed to pull ${CI_IMAGE_REF}. Publish the main IsaacLab CI image or set PERF_SMOKE_CI_IMAGE." + exit 1 + } + docker tag "${CI_IMAGE_REF}" "${{ env.CI_IMAGE_TAG }}" + echo "๐ŸŸข Tagged ${CI_IMAGE_REF} as ${{ env.CI_IMAGE_TAG }}" + + - name: Seed baselines from commit history + env: + SEED_BRANCHES: ${{ inputs.branches }} + SEED_COMMITS: ${{ inputs.commits || '' }} + SEED_COMMIT_BRANCH: ${{ inputs.commit_branch || 'develop' }} + SEED_COMMIT_COUNT: ${{ inputs.commit_count || '1' }} + SEED_SAMPLES_PER_COMMIT: ${{ inputs.samples_per_commit || '3' }} + SEED_TASKS: ${{ inputs.tasks }} + SEED_BACKENDS: ${{ inputs.backends || '' }} + SEED_TARGET_BRANCH: ${{ inputs.target_branch || 'develop' }} + SEED_STRICT_ANCESTRY: ${{ inputs.strict_ancestry == true && 'true' || 'false' }} + # Fail safe: anything other than an explicit dry_run=false stays a dry run, + # so a missing or malformed input can never publish to perf-baselines. + SEED_DRY_RUN: ${{ inputs.dry_run == false && 'false' || 'true' }} + PERF_SMOKE_RUNNER_NAME: ${{ runner.name }} + run: | + set -euo pipefail + GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)" + python3 tools/perf_smoke_test/seed_baselines.py \ + --branches "${SEED_BRANCHES}" \ + --commits "${SEED_COMMITS}" \ + --commit_branch "${SEED_COMMIT_BRANCH}" \ + --commit_count "${SEED_COMMIT_COUNT}" \ + --samples_per_commit "${SEED_SAMPLES_PER_COMMIT}" \ + --tasks "${SEED_TASKS}" \ + --backends "${SEED_BACKENDS}" \ + --image "${{ env.CI_IMAGE_TAG }}" \ + --gpu_model "${GPU_MODEL}" \ + --target_branch "${SEED_TARGET_BRANCH}" \ + --strict_ancestry "${SEED_STRICT_ANCESTRY}" \ + --baseline_branch perf-baselines \ + --baseline_remote origin \ + --baseline_push_retries 3 \ + --workdir "${{ github.workspace }}" \ + --artifacts_root seed-artifacts \ + --source_mount true \ + --dry_run "${SEED_DRY_RUN}" + + # Prove the just-pushed samples will actually be used by the gate: replay the + # gate's per-bucket match logic (fingerprint + commit ancestry) against the + # target branch tip and require MIN_BASELINE_SAMPLES usable samples per bucket. + # Only meaningful on a real publish (dry-run pushes nothing to verify). + - name: Verify seeded baselines are gate-usable + if: ${{ inputs.dry_run != true }} + env: + SEED_TASKS: ${{ inputs.tasks }} + SEED_BACKENDS: ${{ inputs.backends || '' }} + SEED_TARGET_BRANCH: ${{ inputs.target_branch || 'develop' }} + run: | + set -euo pipefail + GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)" + python3 tools/perf_smoke_test/verify_baselines.py \ + --baseline_branch perf-baselines \ + --baseline_remote origin \ + --gpu_model "${GPU_MODEL}" \ + --target_branch "${SEED_TARGET_BRANCH}" \ + --expected_records seed-artifacts/seed_records.json \ + --tasks "${SEED_TASKS}" \ + --backends "${SEED_BACKENDS}" \ + --repo_dir "${{ github.workspace }}" \ + --require + + - name: Upload seed artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: seed-baselines-${{ github.run_id }} + path: seed-artifacts/ + retention-days: 7 + if-no-files-found: warn diff --git a/.github/workflows/perf-smoke-test.yaml b/.github/workflows/perf-smoke-test.yaml new file mode 100644 index 000000000000..3930eafc3da8 --- /dev/null +++ b/.github/workflows/perf-smoke-test.yaml @@ -0,0 +1,1022 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Performance Smoke Test +# +# GPU jobs default to the shared IsaacLab L40S runner pool. Override with repo +# variable PERF_SMOKE_RUNS_ON, e.g. a JSON array of runner labels. +# +# Runs a matrix of benchmark tasks on self-hosted runners, compares +# results to the rolling baseline in the perf-baselines branch, and posts a verdict +# table to the job summary. Non-blocking by default (gate_config.json). +# +# Architecture: +# config โ†’ load isaacsim version from config.yaml (ubuntu-latest) +# bench โ†’ per-task/backend matrix, each pulling the CI image and running +# perf_runtime.py inside Docker (self-hosted, gpu) +# aggregate โ†’ download all bench artifacts, run oracle, update baselines +# (self-hosted, gpu) +# +# Baseline updates are published from protected-branch push events +# (main/develop/release). Pull requests and merge queue candidates are read-only +# and post verdicts without updating the rolling baselines. +# +# backend_key: "{physics_backend}_{render_backend}" when render_backend is +# set, otherwise just physics_backend. Matches TaskConfig.backend_key. + +name: Performance Smoke Test + +on: + pull_request: + branches: [main, develop, 'release/**'] + # ready_for_review is not a default type. Without it, a PR opened as a draft + # (whose jobs are skipped below) would never be benchmarked when it is marked + # ready, and would silently wait for an unrelated push to trigger the gate. + types: [opened, synchronize, reopened, ready_for_review] + merge_group: + branches: [main, develop, 'release/**'] + push: + branches: [main, develop, 'release/**'] + workflow_dispatch: + +# Supersede an outdated run for the same pull request, but never cancel a +# protected-branch push: that run is the only thing that appends to +# perf-baselines, and develop lands ~10 commits a day with a median gap of about +# 45 minutes. Sharing one group across pushes cancelled the majority of baseline +# runs before they could publish, so the window could never reach +# MIN_BASELINE_SAMPLES and every bucket stayed stuck on NO_BASELINE / +# INSUFFICIENT_WINDOW. Keying pushes by SHA gives each commit its own group. +concurrency: + group: perf-smoke-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +# Read-only by default. A same-repo PR runs its own checked-out Python on the +# host, so write scopes are granted per job and only where a job actually calls +# the API: `bench` posts per-task statuses and `aggregate` posts the verdict +# comment. `config` and `validate` also run PR code and need no write scope at +# all, so a modified tasks_to_ci_matrix.py or validate_tasks.py holds a +# read-only token. +permissions: + contents: read + +env: + # NGC_API_KEY is intentionally NOT set workflow-wide: a same-repo PR runs + # checked-out PR code on the host (e.g. tasks_to_ci_matrix.py, aggregate.py), + # and a workflow-level secret would be in that code's environment. It is scoped + # per-step to only the nvcr.io login/pull (and the fallback build) instead. + CI_IMAGE_TAG: >- + isaac-lab-ci:${{ + github.event_name == 'pull_request' + && format('pr-{0}', github.event.pull_request.number) + || 'sha' + }}-${{ github.sha }} + +jobs: + # --------------------------------------------------------------------------- + # Load shared image config (matches build.yaml pattern) + # --------------------------------------------------------------------------- + config: + name: Load Config + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} + runs-on: ubuntu-latest + outputs: + isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }} + isaacsim_image_tag: ${{ steps.load.outputs.isaacsim_image_tag }} + isaaclab_image_name: ${{ steps.load.outputs.isaaclab_image_name }} + isaaclab_image_ref: ${{ steps.load.outputs.isaaclab_image_ref }} + bench_matrix: ${{ steps.build_matrix.outputs.bench_matrix }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + - id: load + env: + # The branch whose image this run should benchmark against. On a + # pull_request GITHUB_REF_NAME is the synthetic "/merge" ref and on a + # merge_group it is a gh-readonly-queue ref, so neither names the target + # branch; both would fall through to latest-develop and make a PR into + # main or release/** measure develop's image. Its samples would then carry + # a runtime_contract_hash that no baseline on those branches can match. + TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref_name }} + run: | + set -euo pipefail + f=.github/workflows/config.yaml + ISAACSIM_IMAGE_NAME="$(yq -r .isaacsim_image_name "$f")" + ISAACSIM_IMAGE_TAG="$(yq -r .isaacsim_image_tag "$f")" + ISAACLAB_IMAGE_NAME="$(yq -r .isaaclab_image_name "$f")" + + # merge_group supplies a fully qualified ref; the others are bare names. + TARGET_BRANCH="${TARGET_BRANCH#refs/heads/}" + + # Match the image tags published by .github/workflows/publish-images.yaml. + case "${TARGET_BRANCH}" in + main) + ISAACLAB_IMAGE_TAG="latest" + ;; + release/*) + RELEASE_SUFFIX="$(echo "${TARGET_BRANCH#release/}" | sed 's/[^a-zA-Z0-9._-]/-/g')" + ISAACLAB_IMAGE_TAG="latest-release-${RELEASE_SUFFIX}" + ;; + *) + ISAACLAB_IMAGE_TAG="latest-develop" + ;; + esac + echo "Resolving the CI image for target branch '${TARGET_BRANCH}' -> ${ISAACLAB_IMAGE_TAG}" + + FALLBACK_REF="${ISAACLAB_IMAGE_NAME}:${ISAACLAB_IMAGE_TAG}" + + # Resolve the era-pinned immutable image for this commit's container era + # (hashed from docker/.env.base) against the manifest on perf-baselines. + # On a manifest miss -- a new era whose image has not been published yet -- + # this degrades to FALLBACK_REF (the moving tag used before eras existed), + # so the gate never hard-fails on a new era. A hit pins an immutable image + # so a nightly rebuild of latest-develop can't shift the baseline under us. + ERA_JSON="$(python3 tools/perf_smoke_test/image_era.py \ + --source_root . \ + --manifest_from_git \ + --branch perf-baselines \ + --remote origin \ + --fallback_image "${FALLBACK_REF}")" + echo "${ERA_JSON}" + ISAACLAB_IMAGE_REF="$(echo "${ERA_JSON}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["image"])')" + + echo "isaacsim_image_name=${ISAACSIM_IMAGE_NAME}" >> "$GITHUB_OUTPUT" + echo "isaacsim_image_tag=${ISAACSIM_IMAGE_TAG}" >> "$GITHUB_OUTPUT" + echo "isaaclab_image_name=${ISAACLAB_IMAGE_NAME}" >> "$GITHUB_OUTPUT" + echo "isaaclab_image_ref=${ISAACLAB_IMAGE_REF}" >> "$GITHUB_OUTPUT" + - id: build_matrix + run: | + set -euo pipefail + MATRIX=$(python3 tools/perf_smoke_test/tasks_to_ci_matrix.py) + echo "bench_matrix=$MATRIX" >> "$GITHUB_OUTPUT" + echo "Matrix: $MATRIX" + + # --------------------------------------------------------------------------- + # Static pre-flight: catch stale task_ids / bad task config before burning a + # GPU runner (no Isaac Sim needed). Fails fast with a clear message. + # --------------------------------------------------------------------------- + validate: + name: Validate tasks.json + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + - run: python3 tools/perf_smoke_test/validate_tasks.py + + # --------------------------------------------------------------------------- + # Per-task/backend benchmark jobs + # --------------------------------------------------------------------------- + bench: + name: Bench / ${{ matrix.task_id }} / ${{ matrix.physics_backend }}${{ matrix.render_backend != '' && format('_{0}', matrix.render_backend) || '' }} + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} + runs-on: ${{ fromJSON(vars.PERF_SMOKE_RUNS_ON || '["self-hosted","gpu"]') }} + needs: [config, validate] + continue-on-error: true + timeout-minutes: ${{ matrix.job_timeout_minutes }} + permissions: + contents: read + statuses: write # post per-task benchmark commit statuses on the PR head + strategy: + fail-fast: false + matrix: + # Load the benchmark matrix from the canonical tasks.json task definition file. + include: ${{ fromJson(needs.config.outputs.bench_matrix) }} + + steps: + - name: Checkout Code + uses: actions/checkout@v6 + with: + fetch-depth: 1 + lfs: true + + # Warm JIT-cache sidecar: restore the Warp/CUDA compile cache for this + # task/backend so repeat runs skip cold Newton/Warp compilation. Saved + # automatically on job end; purely a speed optimization (never affects the verdict). + # + # The primary key embeds the run id so it is unique per run: actions/cache only + # saves when the primary key did NOT already exist, so a static key would freeze + # the cache after its first save and never pick up newly compiled kernels. The + # restore-keys fall back to the newest prior cache by prefix, giving a rolling + # cache that refills every run. + # Derive the Warp version from the pinned dependency so the JIT cache key + # tracks the toolchain automatically: bumping the pin invalidates stale + # kernels instead of silently reusing a cache compiled by a different Warp. + - name: Resolve Warp version for JIT cache key + id: warp_version + run: | + # warp-lang is pinned in the repo-root pyproject.toml, not the isaaclab + # package one. Reading the wrong file silently yielded "unpinned", so the + # key never varied with Warp and a cache built by another version could + # restore as a hit. Fail loudly rather than degrade quietly again. + version="$(sed -n 's/.*warp-lang[^0-9]*\([0-9][0-9A-Za-z.\-]*\).*/\1/p' pyproject.toml | head -n1)" + if [ -z "$version" ]; then + echo "::error::Could not resolve the warp-lang pin from pyproject.toml; the JIT cache key would stop tracking the Warp version" + exit 1 + fi + echo "value=$version" >> "$GITHUB_OUTPUT" + echo "JIT cache keyed to Warp $version" + + - name: Restore warm JIT cache + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/jit-cache + key: perf-jit-warp${{ steps.warp_version.outputs.value }}-${{ matrix.task_id }}-${{ matrix.physics_backend }}-${{ hashFiles('tools/perf_smoke_test/tasks.json') }}-${{ github.run_id }} + restore-keys: | + perf-jit-warp${{ steps.warp_version.outputs.value }}-${{ matrix.task_id }}-${{ matrix.physics_backend }}-${{ hashFiles('tools/perf_smoke_test/tasks.json') }}- + perf-jit-warp${{ steps.warp_version.outputs.value }}-${{ matrix.task_id }}-${{ matrix.physics_backend }}- + perf-jit-warp${{ steps.warp_version.outputs.value }}-${{ matrix.task_id }}- + + # Image source โ€” pull the published image (primary), build only as a fallback: + # + # Primary: `docker pull` the ref resolved by the config job. That ref is the + # era-pinned immutable image for this commit's container era when the era is + # recorded in the manifest on perf-baselines; otherwise it degrades to the + # moving tag (latest-develop/latest/latest-release-*). Pinning stops a nightly + # rebuild of latest-develop from shifting the environment (and the baseline) + # under a PR. Set the optional PERF_SMOKE_CI_IMAGE repo variable to force a + # specific image. The seeder resolves the same era ref, so baselines and PR + # runs share an identical environment (same runtime_contract_hash, no seed-vs- + # gate offset). The pulled image is retagged to the workflow-local + # CI_IMAGE_TAG so nothing downstream changes. + # + # Fallback: if the pull fails (e.g. a new era before the image is republished), build + # from source via ecr-build-push-pull, matching the main IsaacLab CI build path. + # This makes a missing image self-heal (slowly) instead of hard-failing the gate. + - name: Pull prebuilt CI image + id: pull_image + continue-on-error: true + env: + # Scoped here (not workflow-level) so PR-checked-out code in other steps + # never sees the registry secret. + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + CI_IMAGE_REF: ${{ vars.PERF_SMOKE_CI_IMAGE || needs.config.outputs.isaaclab_image_ref }} + run: | + set -euo pipefail + + # The runner's docker credential store backend is broken ("not implemented"), + # so disable credsStore before any login (same trick as ecr-build-push-pull). + # With credsStore disabled, docker writes the NGC key to config.json in clear + # text, so wipe the temp dir on ANY exit -- a failed docker pull must not leave + # the long-lived credential on the self-hosted runner's filesystem. + DOCKER_CONFIG_DIR="$(mktemp -d)" + trap 'rm -rf "${DOCKER_CONFIG_DIR}"' EXIT + echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json" + export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}" + + REGISTRY="${CI_IMAGE_REF%%/*}" + case "${REGISTRY}" in + nvcr.io) + if [ -n "${NGC_API_KEY:-}" ]; then + echo "๐Ÿ”ต Logging into nvcr.io..." + echo "${NGC_API_KEY}" | docker login nvcr.io -u '$oauthtoken' --password-stdin + else + echo "::warning::NGC_API_KEY not set; attempting anonymous pull from nvcr.io" + fi + ;; + *) + echo "::notice::No known login for registry '${REGISTRY}'; attempting anonymous pull" + ;; + esac + + echo "๐Ÿ”ต Pulling prebuilt image ${CI_IMAGE_REF}..." + docker pull "${CI_IMAGE_REF}" + docker tag "${CI_IMAGE_REF}" "${{ env.CI_IMAGE_TAG }}" + echo "๐ŸŸข Tagged ${CI_IMAGE_REF} as ${{ env.CI_IMAGE_TAG }}" + + - name: Build CI image (fallback) + if: ${{ steps.pull_image.outcome == 'failure' }} + # Scoped here (not workflow-level) so PR-checked-out code in other steps + # never sees the registry secret; the action logs into nvcr.io to pull the base. + env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + uses: ./.github/actions/ecr-build-push-pull + with: + image-tag: ${{ env.CI_IMAGE_TAG }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + # Keep a dedicated GHA cache scope so perf-smoke fallback builds do not collide + # with the main Docker test workflow's cache entries. + gha-cache-scope: perf-smoke-base + + - name: Run benchmark in Docker + id: bench_run + env: + # When the env image was pulled prebuilt, overlay the PR checkout over the image's + # baked source so the benchmark runs the PR's code (the editable install resolves to + # the mounted tree), not the image's era source. When the image was built from source + # the PR code is already baked in, so no mount (preserves the proven build path). + IMAGE_WAS_PULLED: ${{ steps.pull_image.outcome == 'success' }} + # Matrix values from tasks.json flow through the environment, never interpolated into + # into shell source, to prevent injection. They are referenced as + # "$VAR" below and forwarded into the container with `docker run -e`. + TASK_ID: ${{ matrix.task_id }} + PHYSICS_BACKEND: ${{ matrix.physics_backend }} + RENDER_BACKEND: ${{ matrix.render_backend }} + NUM_ENVS: ${{ matrix.num_envs }} + NUM_FRAMES: ${{ matrix.num_frames }} + WARMUP_FRAMES: ${{ matrix.warmup_frames }} + SEED: ${{ matrix.seed }} + HYDRA_ARGS: ${{ matrix.hydra_args }} + run: | + set -euo pipefail + + # Compute backend_key: "{physics}_{render}" when render is set, else "{physics}" + if [ -n "${RENDER_BACKEND}" ]; then + BACKEND_KEY="${PHYSICS_BACKEND}_${RENDER_BACKEND}" + else + BACKEND_KEY="${PHYSICS_BACKEND}" + fi + + ARTIFACT_DIR="$(pwd)/artifacts/${TASK_ID}/${BACKEND_KEY}" + + # Sanitize task_id and backend_key for Docker container name + SAFE_TASK_ID="${TASK_ID//[^a-zA-Z0-9]/-}" + SAFE_BACKEND="${BACKEND_KEY//[^a-zA-Z0-9]/-}" + CONTAINER_NAME="perf-bench-${SAFE_TASK_ID}-${SAFE_BACKEND}-${{ github.run_id }}" + LOG_FILE="${ARTIFACT_DIR}/benchmark.log" + + mkdir -p "${ARTIFACT_DIR}" + mkdir -p "${{ github.workspace }}/jit-cache/warp" "${{ github.workspace }}/jit-cache/nv" + mkdir -p "${{ github.workspace }}/kit-cache" + + # The CI image runs as non-root user `isaaclab` (uid 1000), but these host + # dirs are created by the runner user (root on the NVIDIA fleet), so the + # in-container user can't write them by default. That manifests as a + # PermissionError in wp.init() creating WARP_CACHE_PATH/, which the + # lazy task loader then masks as a misleading `mdp.` AttributeError. + # + # kit-cache covers the Omniverse Kit / RTX shader cache (mounted over + # /isaac-sim/kit/cache, which is root-owned 755 in the image). Without a + # writable location the RTX renderer cannot create its shader DB + # ("Failed to initialize rtx::shaderdb::ContextManager"), leaving the + # renderer broken and crashing later with cudaErrorIllegalAddress. Only + # the RTX-renderer camera configs need it; the Warp renderer does not. + # World-writable bind mounts fix all of these, independent of runner uid. + chmod -R 0777 "${ARTIFACT_DIR}" "${{ github.workspace }}/jit-cache" "${{ github.workspace }}/kit-cache" + + GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)" + if [[ -z "${GPU_MODEL}" ]]; then + GPU_MODEL="unknown-gpu" + fi + python3 tools/perf_smoke_test/write_launch_config.py \ + --task_id "${TASK_ID}" \ + --physics_backend "${PHYSICS_BACKEND}" \ + --render_backend "${RENDER_BACKEND}" \ + --gpu_model "${GPU_MODEL}" \ + --artifact_dir "${ARTIFACT_DIR}" + + # Remove any stale container from a previous run attempt + docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + + # Overlay the PR checkout when running a pulled (prebuilt) env image, so the + # benchmark runs the PR's code instead of the image's baked era source. The + # in-container user (uid 1000) must be able to write _isaac_sim into the mounted + # root, so open up perms on the runner-owned checkout (mirrors the seeder). + SRC_MOUNT="" + if [ "${IMAGE_WAS_PULLED}" = "true" ]; then + chmod -R a+rwX "${{ github.workspace }}" 2>/dev/null || true + SRC_MOUNT="-v ${{ github.workspace }}:/workspace/isaaclab" + fi + + # Start the benchmark container in detached mode + docker run -d --name "${CONTAINER_NAME}" \ + --init --stop-timeout 10 \ + --entrypoint bash --gpus all --network=host \ + --security-opt=no-new-privileges:true \ + --ulimit nofile=65536:65536 \ + --ulimit nproc=4096:4096 \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e PYTHONUNBUFFERED=1 \ + -e PYTHONDONTWRITEBYTECODE=1 \ + -e WARP_CACHE_PATH=/tmp/jit-cache/warp \ + -e CUDA_CACHE_PATH=/tmp/jit-cache/nv \ + -e TASK_ID -e NUM_ENVS -e NUM_FRAMES -e WARMUP_FRAMES -e SEED -e HYDRA_ARGS \ + -v "${ARTIFACT_DIR}:/tmp/bench_out" \ + -v "${{ github.workspace }}/jit-cache:/tmp/jit-cache" \ + -v "${{ github.workspace }}/kit-cache:/isaac-sim/kit/cache" \ + ${SRC_MOUNT} \ + "${{ env.CI_IMAGE_TAG }}" \ + -c ' + set -e + cd /workspace/isaaclab + rm -f _isaac_sim + ln -s /isaac-sim _isaac_sim + ./isaaclab.sh -p tools/perf_smoke_test/perf_runtime.py \ + --task "$TASK_ID" \ + --num_envs "$NUM_ENVS" \ + --num_frames "$NUM_FRAMES" \ + --warmup_frames "$WARMUP_FRAMES" \ + --benchmark_formatter schema \ + --output_path /tmp/bench_out \ + ${SEED:+--seed "$SEED"} \ + $HYDRA_ARGS + ' + + START=$(date +%s) + + # Stream container logs to file and terminal while benchmark runs + docker logs -f "${CONTAINER_NAME}" 2>&1 | tee "${LOG_FILE}" & + LOGS_PID=$! + + # Wait for the container to exit, with a hard wall-clock timeout + BENCH_EXIT=1 + if docker_exit=$(timeout ${{ matrix.bench_timeout_s }} docker wait "${CONTAINER_NAME}" 2>/dev/null); then + BENCH_EXIT="${docker_exit:-1}" + else + echo "::warning::Benchmark for ${TASK_ID}/${BACKEND_KEY} timed out after ${{ matrix.bench_timeout_s }}s" + fi + + END=$(date +%s) + WALL_TIME_S=$((END - START)) + + kill "${LOGS_PID}" 2>/dev/null || true + wait "${LOGS_PID}" 2>/dev/null || true + + docker kill "${CONTAINER_NAME}" 2>/dev/null || true + docker rm "${CONTAINER_NAME}" 2>/dev/null || true + + echo "exit_code=${BENCH_EXIT}" >> "$GITHUB_OUTPUT" + echo "wall_time_s=${WALL_TIME_S}" >> "$GITHUB_OUTPUT" + echo "artifact_dir=${ARTIFACT_DIR}" >> "$GITHUB_OUTPUT" + echo "backend_key=${BACKEND_KEY}" >> "$GITHUB_OUTPUT" + + # Retry once on first-attempt failure before giving up. + # Runs only when bench_run reports a non-zero exit code. + - name: Retry benchmark on failure + id: bench_retry + if: always() && steps.bench_run.outputs.exit_code != '0' && steps.bench_run.outputs.exit_code != '' + env: + # Same source-overlay rule as the first attempt (see "Run benchmark in Docker"). + IMAGE_WAS_PULLED: ${{ steps.pull_image.outcome == 'success' }} + TASK_ID: ${{ matrix.task_id }} + PHYSICS_BACKEND: ${{ matrix.physics_backend }} + RENDER_BACKEND: ${{ matrix.render_backend }} + NUM_ENVS: ${{ matrix.num_envs }} + NUM_FRAMES: ${{ matrix.num_frames }} + WARMUP_FRAMES: ${{ matrix.warmup_frames }} + SEED: ${{ matrix.seed }} + HYDRA_ARGS: ${{ matrix.hydra_args }} + run: | + set -uo pipefail + + if [ -n "${RENDER_BACKEND}" ]; then + BACKEND_KEY="${PHYSICS_BACKEND}_${RENDER_BACKEND}" + else + BACKEND_KEY="${PHYSICS_BACKEND}" + fi + + ARTIFACT_DIR="$(pwd)/artifacts/${TASK_ID}/${BACKEND_KEY}" + SAFE_TASK_ID="${TASK_ID//[^a-zA-Z0-9]/-}" + SAFE_BACKEND="${BACKEND_KEY//[^a-zA-Z0-9]/-}" + CONTAINER_NAME="perf-bench-${SAFE_TASK_ID}-${SAFE_BACKEND}-${{ github.run_id }}-retry" + LOG_FILE="${ARTIFACT_DIR}/benchmark.log" + + echo "::notice::First attempt failed (exit=${{ steps.bench_run.outputs.exit_code }}); retrying benchmark for ${TASK_ID}/${BACKEND_KEY}" + + # Drop the first attempt's runtime bundle. build_bench_result globs + # benchmark_runtime_*.json, so leaving it behind lets a retry that dies + # before writing its own output report the failed attempt's FPS under the + # retry's exit code -- and that sample would be eligible for the baseline. + # perf_smoke_test_info.json is written later by build_bench_result itself + # and so cannot exist yet. + rm -f "${ARTIFACT_DIR}"/benchmark_runtime_*.json + + docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + + # Overlay the PR checkout when running a pulled (prebuilt) env image (see the + # first attempt for rationale). + SRC_MOUNT="" + if [ "${IMAGE_WAS_PULLED}" = "true" ]; then + chmod -R a+rwX "${{ github.workspace }}" 2>/dev/null || true + SRC_MOUNT="-v ${{ github.workspace }}:/workspace/isaaclab" + fi + + docker run -d --name "${CONTAINER_NAME}" \ + --init --stop-timeout 10 \ + --entrypoint bash --gpus all --network=host \ + --security-opt=no-new-privileges:true \ + --ulimit nofile=65536:65536 \ + --ulimit nproc=4096:4096 \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e PYTHONUNBUFFERED=1 \ + -e PYTHONDONTWRITEBYTECODE=1 \ + -e WARP_CACHE_PATH=/tmp/jit-cache/warp \ + -e CUDA_CACHE_PATH=/tmp/jit-cache/nv \ + -e TASK_ID -e NUM_ENVS -e NUM_FRAMES -e WARMUP_FRAMES -e SEED -e HYDRA_ARGS \ + -v "${ARTIFACT_DIR}:/tmp/bench_out" \ + -v "${{ github.workspace }}/jit-cache:/tmp/jit-cache" \ + -v "${{ github.workspace }}/kit-cache:/isaac-sim/kit/cache" \ + ${SRC_MOUNT} \ + "${{ env.CI_IMAGE_TAG }}" \ + -c ' + set -e + cd /workspace/isaaclab + rm -f _isaac_sim + ln -s /isaac-sim _isaac_sim + ./isaaclab.sh -p tools/perf_smoke_test/perf_runtime.py \ + --task "$TASK_ID" \ + --num_envs "$NUM_ENVS" \ + --num_frames "$NUM_FRAMES" \ + --warmup_frames "$WARMUP_FRAMES" \ + --benchmark_formatter schema \ + --output_path /tmp/bench_out \ + ${SEED:+--seed "$SEED"} \ + $HYDRA_ARGS + ' + + START=$(date +%s) + + docker logs -f "${CONTAINER_NAME}" 2>&1 | tee "${LOG_FILE}" & + LOGS_PID=$! + + BENCH_EXIT=1 + if docker_exit=$(timeout ${{ matrix.bench_timeout_s }} docker wait "${CONTAINER_NAME}" 2>/dev/null); then + BENCH_EXIT="${docker_exit:-1}" + else + echo "::warning::Retry benchmark for ${TASK_ID}/${BACKEND_KEY} timed out after ${{ matrix.bench_timeout_s }}s" + fi + + END=$(date +%s) + WALL_TIME_S=$((END - START)) + + kill "${LOGS_PID}" 2>/dev/null || true + wait "${LOGS_PID}" 2>/dev/null || true + + docker kill "${CONTAINER_NAME}" 2>/dev/null || true + docker rm "${CONTAINER_NAME}" 2>/dev/null || true + + echo "exit_code=${BENCH_EXIT}" >> "$GITHUB_OUTPUT" + echo "wall_time_s=${WALL_TIME_S}" >> "$GITHUB_OUTPUT" + + # Build bench_result.json from the Docker run outputs. + # Runs even when bench_run/bench_retry fail so partial results are always uploaded. + # Uses the retry's exit code and wall time when a retry ran, and sets was_retried accordingly. + - name: Build bench_result + if: always() + env: + # Matrix values via env, never interpolated into shell (see "Run benchmark in Docker"). + TASK_ID: ${{ matrix.task_id }} + PHYSICS_BACKEND: ${{ matrix.physics_backend }} + RENDER_BACKEND: ${{ matrix.render_backend }} + run: | + if [ -n "${RENDER_BACKEND}" ]; then + BACKEND_KEY="${PHYSICS_BACKEND}_${RENDER_BACKEND}" + else + BACKEND_KEY="${PHYSICS_BACKEND}" + fi + ARTIFACT_DIR="$(pwd)/artifacts/${TASK_ID}/${BACKEND_KEY}" + + # If a retry ran, use its exit code/wall time and mark was_retried. + RETRY_EXIT="${{ steps.bench_retry.outputs.exit_code }}" + if [ -n "${RETRY_EXIT}" ]; then + FINAL_EXIT="${RETRY_EXIT}" + FINAL_WALL="${{ steps.bench_retry.outputs.wall_time_s || '0' }}" + EXTRA_FLAGS="--was_retried --attempt 2" + else + FINAL_EXIT="${{ steps.bench_run.outputs.exit_code || '1' }}" + FINAL_WALL="${{ steps.bench_run.outputs.wall_time_s || '0' }}" + EXTRA_FLAGS="" + fi + + python3 tools/perf_smoke_test/build_bench_result.py \ + --task_id "${TASK_ID}" \ + --physics_backend "${PHYSICS_BACKEND}" \ + --render_backend "${RENDER_BACKEND}" \ + --artifact_dir "${ARTIFACT_DIR}" \ + --exit_code "${FINAL_EXIT}" \ + --wall_time_s "${FINAL_WALL}" \ + --timeout_s "${{ matrix.bench_timeout_s }}" \ + --log_file "${ARTIFACT_DIR}/benchmark.log" \ + --launch_config "${ARTIFACT_DIR}/launch_config.json" \ + --gate_config tools/perf_smoke_test/gate_config.json \ + ${EXTRA_FLAGS} + + # Surface per-task benchmark execution health as an independent commit status + # on the PR head. This reads build_bench_result's structured result so a green + # job (bench is continue-on-error) can no longer hide a benchmark that failed. + # Note: this reports whether the benchmark *ran*, not the regression verdict โ€” + # that is the separate `perf-smoke-test` status posted by aggregate. + # A pull_request run from a fork gets a read-only GITHUB_TOKEN whatever the + # permissions block says, so writing a status would 403 and fail the job for + # an external contributor. Skip the API write there; the verdict is still in + # the job summary and the artifacts. + - name: Report per-task status + if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const taskId = ${{ toJSON(matrix.task_id) }}; + const physics = ${{ toJSON(matrix.physics_backend) }}; + const render = ${{ toJSON(matrix.render_backend) }}; + const backendKey = render ? `${physics}_${render}` : physics; + const resultPath = `artifacts/${taskId}/${backendKey}/perf_smoke_test_result.json`; + let ok = false; + let description = 'no benchmark result produced'; + try { + const r = JSON.parse(fs.readFileSync(resultPath, 'utf8')); + ok = r.exit_code === 0 && !r.failure_phase && r.perf_smoke_test_info_present === true; + description = ok + ? 'benchmark completed' + : `benchmark failed (${r.failure_phase || 'exit ' + r.exit_code})`; + } catch (e) { + description = 'benchmark result missing โ€” job crashed before build_bench_result'; + } + const statusSha = context.payload.pull_request?.head?.sha || context.sha; + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: statusSha, + state: ok ? 'success' : 'failure', + context: `perf-smoke (${taskId}/${backendKey})`, + description: description.slice(0, 140), + }); + + - name: Upload bench artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: bench-${{ matrix.task_id }}-${{ matrix.physics_backend }}${{ matrix.render_backend != '' && format('_{0}', matrix.render_backend) || '' }}-${{ github.run_id }} + path: artifacts/${{ matrix.task_id }}/${{ matrix.physics_backend }}${{ matrix.render_backend != '' && format('_{0}', matrix.render_backend) || '' }}/ + retention-days: 7 + if-no-files-found: warn + + # Force-clean the container if the job is cancelled mid-run + - name: Cleanup container on cancellation + if: cancelled() + env: + TASK_ID: ${{ matrix.task_id }} + PHYSICS_BACKEND: ${{ matrix.physics_backend }} + RENDER_BACKEND: ${{ matrix.render_backend }} + run: | + if [ -n "${RENDER_BACKEND}" ]; then + BACKEND_KEY="${PHYSICS_BACKEND}_${RENDER_BACKEND}" + else + BACKEND_KEY="${PHYSICS_BACKEND}" + fi + SAFE_TASK_ID="${TASK_ID//[^a-zA-Z0-9]/-}" + SAFE_BACKEND="${BACKEND_KEY//[^a-zA-Z0-9]/-}" + CONTAINER_NAME="perf-bench-${SAFE_TASK_ID}-${SAFE_BACKEND}-${{ github.run_id }}" + docker kill "${CONTAINER_NAME}" 2>/dev/null || true + docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + + # --------------------------------------------------------------------------- + # Aggregate: oracle verdicts, baseline update, step summary + # --------------------------------------------------------------------------- + aggregate: + name: Aggregate + Verdict + runs-on: ${{ fromJSON(vars.PERF_SMOKE_RUNS_ON || '["self-hosted","gpu"]') }} + needs: [config, bench] + # Always run for real gate attempts, even when some bench jobs fail. + if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} + outputs: + # Tasks whose current runtime_contract_hash bucket is under-filled (a valid + # measurement but < MIN_BASELINE_SAMPLES matching samples). Drives the reseed + # job below on protected develop pushes. Empty string when nothing to reseed. + reseed_tasks: ${{ steps.aggregate.outputs.reseed_tasks }} + reseed_min_samples: ${{ steps.aggregate.outputs.reseed_min_samples }} + # Read-only on purpose: this job runs checked-out PR code (aggregate.py) on + # same-repo PRs, so it must not hold a write-capable token. Baseline writes + # happen only in the trusted-push-gated baseline_update job below. + permissions: + contents: read + issues: write # post/update the sticky verdict comment on the PR + pull-requests: read + statuses: write + + steps: + - name: Checkout Code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + # Prime the remote-tracking baseline ref. aggregate.py refetches immediately + # before reading and again inside the transactional push retry loop. + - name: Fetch baselines branch + run: | + git fetch origin +refs/heads/perf-baselines:refs/remotes/origin/perf-baselines || \ + echo "::warning::Baseline branch not found; gate will run as a seed run (no baseline comparison)" + + - name: Download bench artifacts + uses: actions/download-artifact@v4 + with: + pattern: bench-*-${{ github.run_id }} + path: artifacts/ + merge-multiple: false + + - name: Resolve gate context + id: gate_context + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 tools/perf_smoke_test/github_gate_context.py + + - name: Run aggregate oracle + id: aggregate + run: | + GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)" + if [[ -z "${GPU_MODEL}" ]]; then + GPU_MODEL="unknown-gpu" + fi + # omni-github app.platform mirrors the shared upload action's derivation so + # dashboard rows are grouped consistently with the rest of IsaacLab CI. + case "${RUNNER_OS:-Linux}-${RUNNER_ARCH:-X64}" in + Linux-X64) APP_PLATFORM="linux-x86_64" ;; + Linux-ARM64) APP_PLATFORM="linux-aarch64" ;; + Windows-X64) APP_PLATFORM="windows-x86_64" ;; + *) APP_PLATFORM="$(printf '%s-%s' "${RUNNER_OS:-linux}" "${RUNNER_ARCH:-x64}" | tr '[:upper:]' '[:lower:]')" ;; + esac + # Write the verdict table to a dedicated file so it can be reused for the + # sticky PR comment, then mirror it into the run's step summary. The + # omni-github artifact carries the same verdicts plus per-task diagnostics + # (FPS, regression, hardware, software, contract hashes) as custom.perf_smoke.* fields. + SUMMARY_FILE="${{ github.workspace }}/verdict_summary.md" + # Never let a nonzero aggregate exit abort this step before the diagnostics + # are published: the step runs under `bash -e`, so without this the job + # summary (and, historically, everything after it) was skipped on exactly + # the runs that most needed explaining. Capture the status, publish, then + # re-raise it as the step's own exit code. + set +e + python3 tools/perf_smoke_test/aggregate.py \ + --artifacts_dir artifacts/ \ + --gpu_model "${GPU_MODEL}" \ + --gate_config tools/perf_smoke_test/gate_config.json \ + --baseline_branch perf-baselines \ + --baseline_remote origin \ + --baseline_push_retries 3 \ + --base_sha "${{ steps.gate_context.outputs.base_sha }}" \ + --target_branch "${{ steps.gate_context.outputs.target_branch }}" \ + --source_branch "${{ steps.gate_context.outputs.source_branch }}" \ + --allow_baseline_update false \ + --trusted_source "${{ steps.gate_context.outputs.trusted_source }}" \ + --summary_file "${SUMMARY_FILE}" \ + --omni_github_dir "${{ github.workspace }}/omni-github-artifact" \ + --omni_platform "${APP_PLATFORM}" \ + --omni_app_config "${GITHUB_JOB}" + AGGREGATE_STATUS=$? + set -e + if [[ -s "${SUMMARY_FILE}" ]]; then + cat "${SUMMARY_FILE}" >> "${GITHUB_STEP_SUMMARY}" + else + { + echo "## Performance smoke test" + echo + echo "The aggregate step exited ${AGGREGATE_STATUS} without writing a verdict summary." + echo "This means the gate itself failed to produce a verdict -- check the step log above." + } >> "${GITHUB_STEP_SUMMARY}" + fi + exit "${AGGREGATE_STATUS}" + + # Sanity-check the emitted omni-github result before upload so a malformed + # payload is caught here (with a clear log) rather than silently dropped by + # omni-github ingestion. Kept dependency-free (stdlib json only): the shared + # result-json.schema.json is a stricter lint whose customValue oneOf rejects + # integral numbers, so it cannot gate a numeric-diagnostics payload. Additive: + # a missing result (e.g. aggregate crashed before writing) is skipped with a + # warning, never fails the job. + - name: Validate omni-github artifact + id: omni_validate + if: always() + run: | + set -euo pipefail + result_json="${{ github.workspace }}/omni-github-artifact/_testoutput/test_results.json" + if [ ! -f "${result_json}" ]; then + echo "::warning::No omni-github result produced; skipping upload" + echo "upload=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + python3 - "${result_json}" <<'PY' + import json, sys + + result = json.load(open(sys.argv[1])) + assert result.get("test_tool_id"), "missing test_tool_id" + assert result.get("app", {}).get("platform"), "missing app.platform" + tests = result.get("tests") + assert isinstance(tests, list) and tests, "tests must be a non-empty list" + for row in tests: + assert row.get("test_id"), "row missing test_id" + assert isinstance(row.get("passed"), bool), "row.passed must be bool" + assert isinstance(row.get("duration"), (int, float)), "row.duration must be numeric" + namespaces = row.get("custom", {}) + assert namespaces, "row.custom must be a non-empty namespace map" + for name, fields in namespaces.items(): + assert fields, f"custom namespace {name!r} must be non-empty" + print(f"omni-github result OK: {len(tests)} rows") + PY + echo "upload=true" >> "${GITHUB_OUTPUT}" + + # Upload the artifact under the omni-github name contract + # (--v1----) so omni-github + # ingests it for the registered NVIDIA-Omniverse/IsaacLab repository. + - name: Upload perf results to omni-github + if: always() && steps.omni_validate.outputs.upload == 'true' + uses: actions/upload-artifact@v7 + with: + name: perf-smoke-results--v1-${{ github.repository_id }}-${{ github.run_id }}-${{ github.run_attempt }}-${{ job.check_run_id }} + path: ${{ github.workspace }}/omni-github-artifact + if-no-files-found: error + retention-days: 7 + compression-level: 9 + + # Aggregate gate signal as a single stable commit status on the PR head. This + # is the status maintainers can mark required when the gate graduates to + # blocking. + # + # It carries the VERDICT, independently of whether the aggregate job passed: + # in advisory mode a BLOCK/HARD_FAILURE paints this status red while the job + # stays green, so the signal is visible without an unexplained red check on an + # unrelated pull request. If the aggregate step could not produce a verdict at + # all, the status reports that rather than silently claiming success. + # Distinct from the per-task `perf-smoke (...)` statuses, which only report + # whether each benchmark ran. + # Skipped for fork pull requests; see the per-task status step for why. + - name: Report aggregate status + if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} + uses: actions/github-script@v7 + env: + AGGREGATE_OUTCOME: ${{ steps.aggregate.outcome }} + STATUS_STATE: ${{ steps.aggregate.outputs.status_state }} + STATUS_DESCRIPTION: ${{ steps.aggregate.outputs.status_description }} + with: + script: | + // No verdict emitted => the gate malfunctioned; say so instead of + // inferring a pass from a missing output. + const state = process.env.STATUS_STATE + || (process.env.AGGREGATE_OUTCOME === 'success' ? 'success' : 'failure'); + const description = process.env.STATUS_DESCRIPTION + || 'perf-smoke gate did not produce a verdict'; + const statusSha = context.payload.pull_request?.head?.sha || context.sha; + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: statusSha, + state, + context: 'perf-smoke-test', + description: description.slice(0, 140), + }); + + # Post the verdict table as a single sticky comment on the PR (updated in place + # each run), so reviewers see regressions without opening the Actions tab. + # Skipped on manual dispatch / protected-branch pushes where there is no PR. + # Skipped for fork pull requests; see the per-task status step for why. + - name: Post verdict PR comment + if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const issue_number = context.issue && context.issue.number; + if (!issue_number) { + core.info('no pull request issue number in context; skipping PR comment'); + return; + } + let table = '_No verdict summary was produced._'; + try { + table = fs.readFileSync(`${process.env.GITHUB_WORKSPACE}/verdict_summary.md`, 'utf8'); + } catch (e) { + core.info('verdict_summary.md not found; posting placeholder'); + } + const marker = ''; + const body = `${marker}\n${table}\n\n_Updated for ${context.sha.slice(0, 8)} ยท [run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})_`; + const { owner, repo } = context.repo; + const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number }); + const existing = comments.find(c => c.body && c.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number, body }); + } + + # Fork pull requests get a read-only GITHUB_TOKEN, so the two reporting steps + # above cannot post. Without this, an external contributor sees a perf-smoke + # job with no verdict anywhere. The job summary is always written (see the + # aggregate step), so point at it explicitly instead of leaving them guessing. + - name: Explain skipped reporting (fork pull request) + if: ${{ always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + run: | + echo "::notice::perf-smoke ran, but this is a fork pull request so GITHUB_TOKEN is read-only:" \ + "the verdict comment and commit statuses cannot be posted." \ + "The full verdict table is in this job's summary, and in the uploaded perf-smoke-results artifact." + + # --------------------------------------------------------------------------- + # Baseline update (trusted writes only) + # + # Appends this run's healthy samples to the perf-baselines branch. Split out of + # the aggregate job so the write-capable token is only ever granted on a trusted + # protected-branch push -- never while the aggregate job runs checked-out PR code. + # --------------------------------------------------------------------------- + baseline_update: + name: Update baselines (protected) + runs-on: ${{ fromJSON(vars.PERF_SMOKE_RUNS_ON || '["self-hosted","gpu"]') }} + needs: [config, bench] + permissions: + contents: write # push protected-branch baseline updates to perf-baselines + # Only trusted protected-branch pushes (main/develop/release) publish baselines -- + # exactly where github_gate_context resolves allow_update=true and the checked-out + # code is already-merged trusted code, not PR code. + if: >- + ${{ github.event_name == 'push' + && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' + || startsWith(github.ref, 'refs/heads/release/')) }} + steps: + - name: Checkout Code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Fetch baselines branch + run: | + git fetch origin +refs/heads/perf-baselines:refs/remotes/origin/perf-baselines || \ + echo "::warning::Baseline branch not found; first push will create it" + + - name: Download bench artifacts + uses: actions/download-artifact@v4 + with: + pattern: bench-*-${{ github.run_id }} + path: artifacts/ + merge-multiple: false + + - name: Resolve gate context + id: gate_context + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 tools/perf_smoke_test/github_gate_context.py + + - name: Append trusted baselines + run: | + GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)" + if [[ -z "${GPU_MODEL}" ]]; then + GPU_MODEL="unknown-gpu" + fi + # Push-only: no summary/omni/comment side effects (the aggregate job already + # produced those). allow_baseline_update is re-checked in-tool via gate_context + # (true only on protected pushes). + python3 tools/perf_smoke_test/aggregate.py \ + --artifacts_dir artifacts/ \ + --gpu_model "${GPU_MODEL}" \ + --gate_config tools/perf_smoke_test/gate_config.json \ + --baseline_branch perf-baselines \ + --baseline_remote origin \ + --baseline_push_retries 3 \ + --base_sha "${{ steps.gate_context.outputs.base_sha }}" \ + --target_branch "${{ steps.gate_context.outputs.target_branch }}" \ + --source_branch "${{ steps.gate_context.outputs.source_branch }}" \ + --allow_baseline_update "${{ steps.gate_context.outputs.allow_update }}" \ + --trusted_source "${{ steps.gate_context.outputs.trusted_source }}" + + # --------------------------------------------------------------------------- + # Reseed: post-gate under-filled-bucket detector. + # + # When a protected develop push observes a valid measurement whose current + # runtime_contract_hash bucket has < MIN_BASELINE_SAMPLES samples (a new bucket + # opened by, e.g., a Warp/Newton/IsaacLab bump), the self-seed-forward step only + # adds one sample -- the bucket would stay advisory (NO_BASELINE / + # INSUFFICIENT_WINDOW) for several more pushes. This fills it in one shot by + # reseeding exactly the flagged tasks for the pushed commit through the same + # era-image resolver the gate used, so the samples share the gate's environment. + # + # Self-limiting: once filled the bucket is no longer under-filled, so the next + # push emits no reseed_tasks. No loop risk -- seeding writes to perf-baselines, + # which triggers neither this gate nor the seed workflow. + # --------------------------------------------------------------------------- + reseed: + name: Reseed under-filled buckets + needs: [aggregate] + # The called seed workflow pushes to perf-baselines (contents: write). A reusable + # workflow can only downgrade the caller job's permissions, so grant write here; + # the gate's top-level contents:read would otherwise fail the call at startup. + permissions: + contents: write + # Protected develop push is the production trigger. workflow_dispatch on develop + # is also allowed: dispatch is write-access-only (trusted) and lets the reseed + # chain be exercised manually, including on forks whose default branch is develop. + if: >- + ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + && github.ref == 'refs/heads/develop' + && needs.aggregate.outputs.reseed_tasks != '' }} + uses: ./.github/workflows/perf-smoke-seed-baselines.yaml + # Pass only the credential the seed workflow declares. `secrets: inherit` + # would hand it every secret this repository holds. + secrets: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + with: + branches: "${{ github.sha }}:develop" + commit_count: "1" + samples_per_commit: ${{ needs.aggregate.outputs.reseed_min_samples }} + tasks: ${{ needs.aggregate.outputs.reseed_tasks }} + target_branch: develop + strict_ancestry: true + dry_run: false diff --git a/.github/workflows/perf-smoke-unit-tests.yaml b/.github/workflows/perf-smoke-unit-tests.yaml new file mode 100644 index 000000000000..816dcd5973e2 --- /dev/null +++ b/.github/workflows/perf-smoke-unit-tests.yaml @@ -0,0 +1,59 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Unit tests for the perf-smoke gate's own logic. +# +# The gate decides whether a PR's benchmark result is a regression, so a silent +# break in its oracle, baseline matching, or contract hashing would either fail +# innocent PRs or wave real regressions through. These tests pin that logic. +# +# They are pure Python -- no GPU, no simulator, no Isaac Lab install -- because +# tools/perf_smoke_test/pyproject.toml makes pytest root there and skips the +# isaaclab-importing tools/conftest.py above it. That keeps this check on a +# free ubuntu-latest runner rather than the L40S pool the benchmarks need. + +name: Performance Smoke - Unit Tests + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - "tools/perf_smoke_test/**" + - "tools/subprocess_runner.py" + - ".github/workflows/perf-smoke-*.yaml" + push: + branches: [main, develop, 'release/**'] + paths: + - "tools/perf_smoke_test/**" + - "tools/subprocess_runner.py" + - ".github/workflows/perf-smoke-*.yaml" + workflow_dispatch: + +concurrency: + group: perf-smoke-unit-tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + unit-tests: + name: Gate logic unit tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install test dependencies + # pyyaml is needed by test_workflow_contracts.py, which parses the gate's + # own workflow files. + run: python3 -m pip install pytest pyyaml + + - name: Test perf-smoke gate logic + run: python3 -m pytest tools/perf_smoke_test/ -q diff --git a/tools/perf_smoke_test/.gitignore b/tools/perf_smoke_test/.gitignore new file mode 100644 index 000000000000..84ec617f93d3 --- /dev/null +++ b/tools/perf_smoke_test/.gitignore @@ -0,0 +1,6 @@ +# Generated at runtime โ€” not source +artifacts/ +local_baselines/ +__pycache__/ +*.pyc +.pytest_cache/ diff --git a/tools/perf_smoke_test/aggregate.py b/tools/perf_smoke_test/aggregate.py new file mode 100644 index 000000000000..4e2d24e40d12 --- /dev/null +++ b/tools/perf_smoke_test/aggregate.py @@ -0,0 +1,831 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Aggregate benchmark artifacts, run the oracle, and update trusted baselines""" + +import argparse +import json +import os +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +_TOOLS_DIR = _MODULE_DIR.parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) +DEFAULT_BASELINE_BRANCH = "perf-baselines" + +from baseline_manager import ( # noqa: E402 + BaselineUpdateRecord, + load_baseline, + load_baseline_git, + make_sample_metadata, + match_context_from_bench_result, + refresh_baseline_branch, + update_baseline, + update_baselines_git, +) +from contracts import BenchResult # noqa: E402 +from gate_config import BASELINE_PUSH_RETRIES, MIN_BASELINE_SAMPLES, load_gate_config # noqa: E402 +from gate_types import FpsMeanThreshold, OracleVerdict # noqa: E402 +from gpu_identity import canonical_gpu_model, gpu_model_config_keys # noqa: E402 +from omni_github import write_artifact as write_omni_github_artifact # noqa: E402 +from oracle import compare # noqa: E402 +from task_config import get_task, load_tasks # noqa: E402 + + +def _parse_args(): + parser = argparse.ArgumentParser(description="Aggregate bench results and run oracle.") + parser.add_argument("--artifacts_dir", required=True, type=Path) + parser.add_argument("--gpu_model", default="L40S") + parser.add_argument("--gate_config", type=Path, default=_MODULE_DIR / "gate_config.json") + parser.add_argument("--baseline_branch", default=DEFAULT_BASELINE_BRANCH) + parser.add_argument( + "--baseline_remote", default="origin", help="Git remote that owns the baseline branch; empty = local only" + ) + parser.add_argument("--baseline_push_retries", type=int, default=None) + parser.add_argument("--baselines_dir", type=Path, default=None, help="Flat-file baseline directory; bypasses git") + parser.add_argument("--allow_baseline_update", default="false") + parser.add_argument("--summary_file", default=None) + parser.add_argument( + "--omni_github_dir", + type=Path, + default=None, + help="Write the omni-github test-result artifact (manifest + result JSON) to this directory", + ) + parser.add_argument( + "--omni_platform", default="linux-x86_64", help="omni-github app.platform for the emitted artifact" + ) + parser.add_argument( + "--omni_app_config", default="perf-smoke", help="omni-github app.config for the emitted artifact" + ) + parser.add_argument("--base_sha", default=None, help="PR base SHA for ancestry-aware baseline matching") + parser.add_argument("--target_branch", default=None, help="Target protected branch, e.g. main/develop/release/x") + parser.add_argument("--source_branch", default=None, help="Branch that produced baseline updates") + parser.add_argument( + "--trusted_source", default="protected_branch", help="Audit label for baseline samples written by this run" + ) + return parser.parse_args() + + +def _find_bench_results(artifacts_dir: Path) -> list[tuple[Path, BenchResult]]: + found = [] + for path in sorted(artifacts_dir.rglob("perf_smoke_test_result.json")): + with path.open() as fh: + found.append((path.parent, BenchResult.from_dict(json.load(fh)))) + return found + + +def _fmt(value, decimals: int = 1) -> str: + return f"{value:.{decimals}f}" if value is not None else "N/A" + + +def _short_sha(value: str | None) -> str: + return value[:12] if value else "none" + + +def _bench_gpu_model(bench_result: BenchResult, fallback: str) -> str: + launch_config = bench_result.launch_config or {} + gpu_model = canonical_gpu_model(launch_config.get("gpu_model") or launch_config.get("gpu_model_raw")) + return canonical_gpu_model(fallback) if gpu_model == "unknown_gpu" else gpu_model + + +def _thresholds(bench_result: BenchResult, gpu_model: str, backend: str) -> list[FpsMeanThreshold]: + """Resolve configured FPS thresholds, preferring the run's launch_config artifact.""" + launch_config = bench_result.launch_config or {} + raw = launch_config.get("fps_mean_thresholds") + if raw is not None: + return FpsMeanThreshold.from_list(raw, context=f"{bench_result.task_id}/{backend}") + try: + task = get_task(bench_result.task_id, backend) + return task.thresholds_for(gpu_model) + except Exception: + return [] + + +def _noise_floor_pct(bench_result: BenchResult, gpu_model: str, backend: str) -> float: + """Resolve the per-task/GPU/backend noise floor % (0.0 when unconfigured).""" + launch_config = bench_result.launch_config or {} + if launch_config.get("noise_floor_pct") is not None: + return float(launch_config.get("noise_floor_pct") or 0.0) + try: + task = get_task(bench_result.task_id, backend) + for key in gpu_model_config_keys(gpu_model): + value = (task.noise_floor_pct or {}).get(key, {}).get(backend) + if value is not None: + return float(value) + except Exception: + pass + return 0.0 + + +def _render_crossed(crossed: list[dict]) -> list[str]: + """Render crossed thresholds as compact ``name(verdict)@value`` tags for reporting.""" + parts = [] + for rec in crossed: + tag = rec.get("threshold_verdict") or "report" + parts.append(f"crossed:{rec.get('threshold_name')}({tag})@{_fmt(rec.get('threshold'))}") + return parts + + +def _fmt_change(value: float | None) -> str: + """Format a signed percent change where positive means faster.""" + return f"{value:+.2f}%" if value is not None else "N/A" + + +def _runtime_context(bench_result: BenchResult) -> tuple[str, str]: + """Return concise GPU and runtime labels for one benchmark result.""" + runtime_resources = bench_result.runtime_resources or {} + launch_config = bench_result.launch_config or {} + gpu_name = ( + runtime_resources.get("gpu_name") or launch_config.get("gpu_model_raw") or launch_config.get("gpu_model", "") + ) + provenance = bench_result.provenance or {} + software = provenance.get("software") or {} + runtime = ", ".join( + part + for part in ( + f"cuda={runtime_resources.get('cuda_version')}" if runtime_resources.get("cuda_version") else "", + f"driver={runtime_resources.get('nvidia_driver_version')}" + if runtime_resources.get("nvidia_driver_version") + else "", + f"warp={software.get('warp')}" if software.get("warp") else "", + ) + if part + ) + return str(gpu_name or "N/A"), runtime or "N/A" + + +# --- Source-versus-image dependency skew ------------------------------------- +# +# The gate bind-mounts Isaac Lab source over a prebuilt CI image, but the image +# supplies the installed third-party packages (Newton, Warp, Isaac Sim). Between +# a dependency-pin change landing on develop and the next image publish, a PR's +# source can reference a symbol the installed package does not have yet, which +# crashes every affected task before any FPS is measured. That crash says nothing +# about the PR's performance, so it is reported as a stale image and left +# advisory rather than read as a performance failure. + +# Packages installed into the CI image rather than bind-mounted from the PR. A +# missing symbol in Isaac Lab's own source is a genuine defect in the change +# under test and still fails. +IMAGE_PROVIDED_PACKAGES: frozenset[str] = frozenset( + { + "carb", + "isaacsim", + "mujoco", + "mujoco_warp", + "newton", + "omni", + "pxr", + "warp", + } +) + +# Python spells "this name is not in the installed package" three ways. +_MISSING_NAME_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile(r"ImportError: cannot import name ['\"](?P\w+)['\"] from ['\"](?P[\w.]+)['\"]"), + re.compile(r"ModuleNotFoundError: No module named ['\"](?P[\w.]+)['\"]"), + re.compile(r"AttributeError: module ['\"](?P[\w.]+)['\"] has no attribute ['\"](?P\w+)['\"]"), +) + + +@dataclass(frozen=True) +class DependencySkew: + """One detected mismatch between the PR's source and the image's packages.""" + + package: str + module: str + symbol: str | None + + def describe(self) -> str: + """Return a one-line reviewer-facing description of the mismatch.""" + if self.symbol: + return f"`{self.module}` in the CI image has no `{self.symbol}`" + return f"`{self.module}` is not installed in the CI image" + + +def detect_dependency_skew(log_text: str | None) -> DependencySkew | None: + """Return the dependency skew a benchmark log indicates, if any. + + Args: + log_text: Captured benchmark output, typically ``BenchResult.stdout_tail``. + + Returns: + The detected mismatch, or ``None`` when the log shows no missing symbol + from an image-provided package. + """ + if not log_text: + return None + for pattern in _MISSING_NAME_PATTERNS: + match = pattern.search(log_text) + if match is None: + continue + module = match.group("module") + package = module.split(".", 1)[0] + if package not in IMAGE_PROVIDED_PACKAGES: + continue + return DependencySkew(package=package, module=module, symbol=match.groupdict().get("symbol")) + return None + + +def _skewed_rows(rows: list[tuple]) -> list[tuple[str, str, DependencySkew]]: + """Return ``(task_id, backend, skew)`` for failures caused by a stale CI image.""" + skewed = [] + for result, bench_result in rows: + if result.verdict != OracleVerdict.HARD_FAILURE: + continue + skew = detect_dependency_skew(bench_result.stdout_tail) + if skew is not None: + skewed.append((result.task_id, result.backend, skew)) + return skewed + + +def _row_explanation(result) -> str: + """Explain one verdict in reviewer-facing language.""" + if result.verdict == OracleVerdict.HARD_FAILURE: + explanation = ( + f"Benchmark failed during {result.failure_phase}" + if result.failure_phase + else "Benchmark produced no usable FPS" + ) + elif result.threshold_source == "no_baseline": + explanation = "No compatible baseline yet" + elif result.threshold_source == "insufficient_window": + explanation = f"Baseline warming up ({result.baseline_sample_count}/{MIN_BASELINE_SAMPLES} samples)" + elif result.verdict == OracleVerdict.BLOCK: + explanation = "Blocking-level slowdown detected" + elif result.verdict == OracleVerdict.WARN: + explanation = "Possible slowdown; review recommended" + else: + explanation = "No meaningful slowdown" + if result.was_retried: + explanation += "; retry also failed" if result.verdict == OracleVerdict.HARD_FAILURE else "; result was retried" + return explanation + + +def _build_reviewer_table(rows: list[tuple]) -> str: + """Build the compact table reviewers see first.""" + verdict_labels = { + OracleVerdict.PASS: "โœ… PASS", + OracleVerdict.WARN: "โš ๏ธ WARN", + OracleVerdict.BLOCK: "๐Ÿšซ BLOCK", + OracleVerdict.HARD_FAILURE: "โŒ HARD FAILURE", + } + lines = [ + "| Task | Backend | Result | FPS | Baseline | Change | Samples | What it means |", + "|---|---|---|---:|---:|---:|---:|---|", + ] + for result, _ in rows: + lines.append( + f"| {result.task_id} | {result.backend} | {verdict_labels[result.verdict]}" + f" | {_fmt(result.measured_fps)} | {_fmt(result.baseline_fps)} | {_fmt_change(result.regression_pct)}" + f" | {result.baseline_sample_count} | {_row_explanation(result)} |" + ) + return "\n".join(lines) + + +def _build_technical_table(rows: list[tuple]) -> str: + """Build the complete diagnostic table shown on demand.""" + lines = [ + "| Task | Backend | Verdict | FPS | Baseline | Samples | Regression% | Floor | Threshold | Phase | " + "Retry | GPU | Runtime | Note |", + "|---|---|---|---:|---:|---:|---:|---:|---|---|---|---|---|---|", + ] + for result, bench_result in rows: + gpu_name, runtime = _runtime_context(bench_result) + # result.note already carries the config_mismatch string on the + # config-mismatch HARD_FAILURE path; dedupe so it is not shown twice. + note_parts = list(dict.fromkeys(part for part in (result.note, bench_result.config_mismatch) if part)) + note_parts.extend(_render_crossed(result.crossed_thresholds)) + lines.append( + f"| {result.task_id} | {result.backend} | {result.verdict.value}" + f" | {_fmt(result.measured_fps)} | {_fmt(result.baseline_fps)} | {result.baseline_sample_count}" + f" | {_fmt(result.regression_pct, 2)} | {_fmt(result.hard_floor_fps)} | {result.threshold_source}" + f" | {result.failure_phase or ''} | {'yes' if result.was_retried else 'no'} | {gpu_name}" + f" | {runtime} | {'; '.join(note_parts)} |" + ) + return "\n".join(lines) + + +def _build_stale_image_section(skewed: list[tuple[str, str, DependencySkew]]) -> str: + """Explain that a stale CI image, not the PR, caused these failures.""" + packages = sorted({skew.package for _, _, skew in skewed}) + affected = "\n".join(f"- `{task_id}` ({backend}): {skew.describe()}" for task_id, backend, skew in skewed) + return ( + "### Stale CI image\n\n" + f"The prebuilt CI image does not match this PR's pinned {' and '.join(packages)} version, so the " + "tasks below crashed before producing any FPS. This reflects the image, not the change under review, " + "so these results are advisory and do not fail the check.\n\n" + f"{affected}\n\n" + "This resolves itself once the CI image is rebuilt for the current dependency pins. Re-run the gate " + "after the next image publish to get real numbers for these tasks." + ) + + +def _coverage(rows: list[tuple]) -> tuple[list[str], int]: + """Return ``(missing_labels, expected_total)`` for the configured matrix. + + A bucket whose job died before ``build_bench_result`` uploads nothing, so it + contributes no row. Grading only the rows that arrived would report an + all-clear over a bucket that was never measured, which is exactly the class + of silent green this gate exists to catch. Returns ``([], 0)`` when the + matrix cannot be read, so this can never invent a failure. + + Counts are over distinct buckets, not rows, so a duplicated artifact cannot + inflate the total. + """ + try: + expected = {(task.task_id, task.backend_key) for task in load_tasks()} + except Exception: + return [], 0 + reported = {(result.task_id, result.backend) for result, _ in rows} + missing = sorted(f"{task_id}/{backend}" for task_id, backend in expected - reported) + return missing, len(expected) + + +def _build_summary_markdown( + rows: list[tuple], *, blocking: bool, missing: list[str] | None = None, expected_total: int = 0 +) -> str: + """Build reviewer-first Markdown for the sticky PR comment. + + Args: + rows: ``(OracleResult, BenchResult)`` pairs for every scored bucket. + blocking: Whether the gate is in blocking mode, for the mode banner. + missing: ``task/backend`` labels that produced no result at all. Kept in + the headline so the comment cannot disagree with the commit status. + expected_total: Size of the configured matrix, for the "N of M" phrasing. + """ + missing = missing or [] + counts = {verdict: 0 for verdict in OracleVerdict} + for result, _ in rows: + counts[result.verdict] += 1 + + skewed = _skewed_rows(rows) + skewed_keys = {(task_id, backend) for task_id, backend, _ in skewed} + unexplained_failures = sum( + 1 + for result, _ in rows + if result.verdict == OracleVerdict.HARD_FAILURE and (result.task_id, result.backend) not in skewed_keys + ) + + if not rows: + overall = "โŒ No benchmark results were produced" + elif unexplained_failures: + overall = "โŒ One or more benchmarks failed before producing usable performance data" + elif missing: + # Ranked above skew/BLOCK/WARN: the rows that did arrive may all be clean, + # but the change is not covered, so no all-clear may be printed. + reported_n = expected_total - len(missing) + overall = f"โŒ Only {reported_n} of {expected_total} benchmark buckets reported โ€” coverage is incomplete" + elif skewed: + overall = "โš ๏ธ The CI image is stale for this PR, so some tasks could not be measured" + elif counts[OracleVerdict.BLOCK]: + overall = "๐Ÿšซ One or more blocking-level performance regressions were detected" + elif counts[OracleVerdict.WARN]: + overall = "โš ๏ธ No confirmed regression, but one or more results need attention" + else: + overall = "โœ… No meaningful performance regressions detected" + + warnings_label = "warning" if counts[OracleVerdict.WARN] == 1 else "warnings" + blocks_label = "blocking signal" if counts[OracleVerdict.BLOCK] == 1 else "blocking signals" + failures_label = "benchmark failure" if counts[OracleVerdict.HARD_FAILURE] == 1 else "benchmark failures" + count_parts = [ + f"โœ… {counts[OracleVerdict.PASS]} passed", + f"โš ๏ธ {counts[OracleVerdict.WARN]} {warnings_label}", + f"๐Ÿšซ {counts[OracleVerdict.BLOCK]} {blocks_label}", + f"โŒ {counts[OracleVerdict.HARD_FAILURE]} {failures_label}", + ] + if missing: + # Name them: "8 of 9" alone leaves the reviewer guessing which task is + # unmeasured, and a missing bucket is the one a regression can hide in. + count_parts.append(f"๐Ÿšซ {len(missing)} did not report ({', '.join(missing)})") + count_summary = " ยท ".join(count_parts) + mode = ( + "**Blocking:** BLOCK and HARD FAILURE results fail the check." + if blocking + else ( + "**Advisory:** this job stays green whatever the verdict. Anything that is not a clean pass --" + " a BLOCK, a HARD FAILURE, or a bucket that never reported -- still shows up as a red" + " `perf-smoke-test` commit status, and the reason is in the overall result above." + " It just does not fail the PR." + ) + ) + + contexts = {_runtime_context(bench_result) for _, bench_result in rows} + if len(contexts) == 1: + gpu_name, runtime = next(iter(contexts)) + elif contexts: + gpu_name, runtime = "Multiple; see technical details", "Multiple; see technical details" + else: + gpu_name, runtime = "N/A", "N/A" + + sections = [ + f"### Overall result\n\n**{overall}**\n\n{count_summary}\n\n{mode}", + f"### Run context\n\n- **GPU:** {gpu_name}\n- **Runtime:** {runtime}", + ] + if skewed: + sections.append(_build_stale_image_section(skewed)) + sections.extend( + ( + "### How to read this\n\n" + "Start with **BLOCK** and **HARD FAILURE**, then review any **WARN** rows.\n\n" + "- **โœ… PASS:** no meaningful slowdown was detected.\n" + "- **โš ๏ธ WARN:** the result is uncertainโ€”for example, the baseline is still warming up, the run " + "needed a retry, or performance is in the warning band.\n" + "- **๐Ÿšซ BLOCK:** performance crossed a blocking threshold. This fails the check only when the gate " + "is in blocking mode.\n" + "- **โŒ HARD FAILURE:** the benchmark did not produce usable FPS, usually because it failed during " + "import, initialization, or runtime.\n" + "- **FPS:** current throughput; higher is better. **Baseline:** the median of compatible historical " + "runs. **Change:** `+` is faster and `-` is slower.\n" + f"- **Samples:** compatible historical runs. At least {MIN_BASELINE_SAMPLES} are required before " + "the rolling baseline can make a confident decision.", + _build_reviewer_table(rows), + "
\nTechnical details\n\n" + "Threshold values, failure phases, retries, hardware, runtime versions, and diagnostic notes:\n\n" + f"{_build_technical_table(rows)}\n\n
", + ) + ) + return "\n\n".join(sections) + + +def _write_github_output(**values) -> None: + github_output = os.environ.get("GITHUB_OUTPUT", "") + if not github_output: + return + with open(github_output, "a") as fh: + for key, value in values.items(): + if value is not None: + fh.write(f"{key}={value}\n") + + +def _verdict_outputs( + rows: list[tuple], + *, + has_block: bool, + has_hard_failure: bool, + blocking: bool, + missing: list[str] | None = None, + expected_total: int = 0, +) -> dict[str, str]: + """Derive the reported verdict and the `perf-smoke-test` commit status from the rows. + + The verdict is reported through the commit status, the sticky PR comment and + the job summary -- never through the process exit code. Emitting it as a step + output is what lets the workflow paint the status red on a real regression + while the aggregate job itself stays green in advisory mode. + + Args: + rows: ``(OracleResult, BenchResult)`` pairs for every scored bucket. + has_block: Whether any bucket crossed a blocking threshold. + has_hard_failure: Whether any bucket failed to produce a usable measurement, + excluding failures already excused as a stale CI image. + blocking: The gate's ``blocking`` setting, used only to label the status. + missing: ``task/backend`` labels that produced no result. Shared with + :func:`_build_summary_markdown` so the commit status and the PR + comment cannot disagree about coverage. + + Returns: + The ``overall_verdict`` / ``status_state`` / ``status_description`` / + ``blocking`` step outputs. + """ + # Derive from the rows, never from the booleans alone. `has_hard_failure` is + # cleared for crashes excused as CI-image skew, and `main` only bails when + # there are *zero* artifacts -- so a run where every bucket crashed, or where + # most bench jobs never uploaded, would otherwise fall through to an + # affirmative "no regression detected" over measurements that never happened. + unmeasured = [result for result, _ in rows if result.verdict == OracleVerdict.HARD_FAILURE] + missing = missing or [] + + if has_hard_failure: + verdict = OracleVerdict.HARD_FAILURE + description = "perf-smoke: a benchmark failed to produce a usable measurement" + elif unmeasured: + # Excused as a stale CI image: not the change's fault, but nothing was + # measured either, so the gate must not claim the change is clean. + verdict = OracleVerdict.HARD_FAILURE + description = f"perf-smoke: no usable measurement for {len(unmeasured)} bucket(s); CI image looks stale" + elif missing or not rows: + verdict = OracleVerdict.HARD_FAILURE + description = f"perf-smoke: only {expected_total - len(missing)} of {expected_total} buckets reported a result" + elif has_block: + verdict = OracleVerdict.BLOCK + description = "perf-smoke: blocking-level performance regression detected" + elif any(result.verdict == OracleVerdict.WARN for result, _ in rows): + verdict = OracleVerdict.WARN + description = "perf-smoke: results need attention (see the verdict comment)" + else: + verdict = OracleVerdict.PASS + description = f"perf-smoke: no meaningful regression across {len(rows)} buckets" + + # A run can be several things at once -- a stale image on one bucket and a + # genuine regression on another. The verdict takes the most severe, but the + # description must not misattribute: reporting a real BLOCK as "CI image + # looks stale" is the same misattribution the skew excuse already risks. + if has_block and verdict != OracleVerdict.BLOCK: + description += "; a blocking-level regression was also detected" + + state = "success" if verdict in (OracleVerdict.PASS, OracleVerdict.WARN) else "failure" + if not blocking and state == "failure": + description += " (advisory)" + + return { + "overall_verdict": verdict.value, + "status_state": state, + "status_description": description, + "blocking": "true" if blocking else "false", + } + + +def _exit_code(*, baseline_update_failed: bool, has_block: bool, has_hard_failure: bool, blocking: bool) -> int: + """Decide the process exit code. + + The exit code answers "did the gate run?", never "what did the gate + conclude?". The conclusion travels through the ``perf-smoke-test`` commit + status, the sticky PR comment, the job summary and the omni-github artifact. + + In advisory mode (``blocking: false``) every verdict, including + HARD_FAILURE, exits 0. A crashed benchmark is still reported loudly -- the + commit status goes red and the summary names the bucket that died. What + advisory mode buys is that a registry outage or image drift unrelated to the + change under test does not paint an unexplained red check on somebody else's + pull request. + + Flipping ``blocking: true`` is the deliberate rollout step: HARD_FAILURE + then exits 2 and BLOCK exits 1, failing the aggregate job itself. + + Gate malfunctions are the exception and stay fatal in both modes: a failed + baseline push here, and (in :func:`main`) no bench artifacts at all or an + unreadable baseline branch. Those mean no trustworthy verdict was produced. + + Returns: + ``0`` to pass, ``1`` for a gate malfunction or a blocking regression, + ``2`` for a blocking execution failure. + """ + if baseline_update_failed: + return 1 + if not blocking: + return 0 + if has_hard_failure: + return 2 + if has_block: + return 1 + return 0 + + +def main() -> int: + args = _parse_args() + use_flat = args.baselines_dir is not None + allow_update = args.allow_baseline_update.strip().lower() in ("true", "1", "yes") + baseline_remote = args.baseline_remote or None + + gate_config = load_gate_config(args.gate_config) + blocking = bool(gate_config.get("blocking", False)) + min_block_regression_pct = float(gate_config.get("min_block_regression_pct", 3.0)) + baseline_push_retries = int( + args.baseline_push_retries or gate_config.get("baseline_push_retries", BASELINE_PUSH_RETRIES) + ) + + items = _find_bench_results(args.artifacts_dir) + if not items: + print(f"[aggregate] No perf_smoke_test_result.json files found under {args.artifacts_dir}") + return 1 + + baseline_read_sha = None + baseline_read_ref = None + if not use_flat: + try: + baseline_read_sha = refresh_baseline_branch( + args.baseline_branch, remote=baseline_remote, allow_missing=True + ) + baseline_read_ref = baseline_read_sha + except Exception as exc: + print(f"::error::Failed to refresh baseline branch before reading: {exc}") + return 1 + if baseline_read_sha: + print(f"[aggregate] Baseline read snapshot: {args.baseline_branch}@{_short_sha(baseline_read_sha)}") + else: + print(f"[aggregate] Baseline branch {args.baseline_branch!r} not found; treating this as a seed run") + + rows = [] + has_block = False + has_hard_failure = False + baselines_updated = False + baseline_update_failed = False + pending_git_updates: list[BaselineUpdateRecord] = [] + # Tasks whose current runtime_contract_hash bucket is under-filled despite a + # valid measurement this run. On a protected develop push these drive a targeted + # reseed so a freshly opened bucket (new deps -> new runtime_contract_hash) is + # filled at once instead of over ~MIN_BASELINE_SAMPLES self-seed pushes. + underfilled_tasks: set[str] = set() + + for artifact_dir, bench_result in items: + task_id = bench_result.task_id + backend = bench_result.backend_key or bench_result.backend + bench_gpu_model = _bench_gpu_model(bench_result, args.gpu_model) + # The baseline_manager storage layer works with the serialized (dict) form. + match_context = match_context_from_bench_result( + bench_result.to_dict(), + gpu_model=bench_gpu_model, + base_sha=args.base_sha, + target_branch=args.target_branch, + ) + + baseline = None + try: + if use_flat: + baseline = load_baseline( + args.baselines_dir, bench_gpu_model, task_id, backend, match_context=match_context + ) + elif baseline_read_ref: + baseline = load_baseline_git( + baseline_read_ref, + bench_gpu_model, + task_id, + backend, + None, + match_context, + ) + except Exception as exc: + print(f"[aggregate] Warning: baseline load failed for {task_id}/{backend}: {exc}") + + oracle_result = compare( + bench_result=bench_result, + baseline=baseline, + fps_mean_thresholds=_thresholds(bench_result, bench_gpu_model, backend), + min_block_regression_pct=min_block_regression_pct, + noise_floor_pct=_noise_floor_pct(bench_result, bench_gpu_model, backend), + ) + rows.append((oracle_result, bench_result)) + + # A valid run whose matching baseline window is short of MIN_BASELINE_SAMPLES + # is an under-filled bucket for this exact runtime_contract_hash (the oracle + # already scoped the load to it). Measured-fps guard avoids reseeding a task + # that crashed this run (nothing trustworthy to base a bucket on). + if oracle_result.measured_fps is not None and oracle_result.baseline_sample_count < MIN_BASELINE_SAMPLES: + underfilled_tasks.add(task_id) + + crossed_summary = _render_crossed(oracle_result.crossed_thresholds) + print( + f"[aggregate] {task_id}/{backend}: {oracle_result.verdict.value}" + f" fps={_fmt(oracle_result.measured_fps)} baseline={_fmt(oracle_result.baseline_fps)}" + f" samples={oracle_result.baseline_sample_count} source={oracle_result.threshold_source}" + + (f" {' '.join(crossed_summary)}" if crossed_summary else "") + ) + + if oracle_result.verdict == OracleVerdict.BLOCK: + has_block = True + elif oracle_result.verdict == OracleVerdict.HARD_FAILURE: + # A crash caused by the image lacking a symbol this source pins is a + # property of the image, not of the change under test, so it is + # reported loudly but never fails the PR. + skew = detect_dependency_skew(bench_result.stdout_tail) + if skew is None: + has_hard_failure = True + else: + print(f"[aggregate] {task_id}/{backend}: stale CI image; {skew.describe()}") + + if ( + allow_update + and oracle_result.verdict in (OracleVerdict.PASS, OracleVerdict.WARN) + and oracle_result.measured_fps is not None + ): + sample_metadata = make_sample_metadata( + gpu_model=bench_gpu_model, + task_id=task_id, + backend=backend, + fps=oracle_result.measured_fps, + bench_result=bench_result.to_dict(), + target_branch=args.target_branch, + source_branch=args.source_branch, + trusted_source=args.trusted_source, + ) + if baseline_read_sha: + sample_metadata["baseline_read_sha"] = baseline_read_sha + + if use_flat: + try: + update_baseline( + args.baselines_dir, + bench_gpu_model, + task_id, + backend, + oracle_result.measured_fps, + sample_metadata=sample_metadata, + ) + baselines_updated = True + print(f"[aggregate] -> baseline updated locally: {oracle_result.measured_fps:.1f} FPS") + except Exception as exc: + baseline_update_failed = True + print(f"::error::Baseline update failed for {task_id}/{backend}: {exc}") + else: + pending_git_updates.append( + BaselineUpdateRecord( + gpu_model=bench_gpu_model, + task_id=task_id, + backend=backend, + fps=oracle_result.measured_fps, + sample_metadata=sample_metadata, + ) + ) + print(f"[aggregate] -> baseline update queued: {oracle_result.measured_fps:.1f} FPS") + + baseline_push_result = None + if pending_git_updates: + try: + baseline_push_result = update_baselines_git( + args.baseline_branch, + pending_git_updates, + remote=baseline_remote, + max_retries=baseline_push_retries, + ) + baselines_updated = baseline_push_result.pushed + if baseline_push_result.pushed: + print( + f"[aggregate] Baseline push succeeded: {args.baseline_branch}@" + f"{_short_sha(baseline_push_result.pushed_sha)} " + f"after {baseline_push_result.attempts} attempt(s)" + ) + else: + print("[aggregate] Baseline samples were already present; no push needed") + except Exception as exc: + baseline_update_failed = True + print(f"::error::Baseline push failed: {exc}") + + # One source of truth for coverage, shared by the summary/comment and the + # commit status so the two surfaces cannot contradict each other. + missing_buckets, expected_total = _coverage(rows) + summary = _build_summary_markdown(rows, blocking=blocking, missing=missing_buckets, expected_total=expected_total) + print("\n## Performance Smoke Results\n") + print(summary) + print() + + if args.summary_file: + with open(args.summary_file, "a") as fh: + fh.write("\n## Performance Smoke Results\n\n") + if not use_flat: + fh.write(f"Baseline read SHA: `{_short_sha(baseline_read_sha)}`\n\n") + if baseline_push_result and baseline_push_result.pushed_sha: + fh.write( + f"Baseline pushed SHA: `{_short_sha(baseline_push_result.pushed_sha)}` " + f"after {baseline_push_result.attempts} attempt(s)\n\n" + ) + fh.write(summary) + fh.write("\n") + + if args.omni_github_dir: + write_omni_github_artifact( + rows, + args.omni_github_dir, + platform=args.omni_platform, + app_config=args.omni_app_config, + ) + + output_values = { + "baseline_read_sha": baseline_read_sha, + **_verdict_outputs( + rows, + has_block=has_block, + has_hard_failure=has_hard_failure, + blocking=blocking, + missing=missing_buckets, + expected_total=expected_total, + ), + } + if baseline_push_result: + output_values.update( + { + "baselines_updated": "true" if baseline_push_result.pushed else "false", + "baseline_pushed_sha": baseline_push_result.pushed_sha, + "baseline_push_attempts": baseline_push_result.attempts, + } + ) + elif baselines_updated: + output_values["baselines_updated"] = "true" + if underfilled_tasks: + # Consumed by the gate's reseed job (protected develop push only) to seed + # exactly these tasks for HEAD. Emitted everywhere for visibility; the + # workflow gates the actual reseed on the trusted push context. + reseed_tasks = ",".join(sorted(underfilled_tasks)) + output_values["reseed_tasks"] = reseed_tasks + output_values["reseed_min_samples"] = MIN_BASELINE_SAMPLES + print(f"[aggregate] Under-filled buckets flagged for reseed: {reseed_tasks}") + _write_github_output(**output_values) + + return _exit_code( + baseline_update_failed=baseline_update_failed, + has_block=has_block, + has_hard_failure=has_hard_failure, + blocking=blocking, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_smoke_test/backend_identity.py b/tools/perf_smoke_test/backend_identity.py new file mode 100644 index 000000000000..1fbe07b6eb56 --- /dev/null +++ b/tools/perf_smoke_test/backend_identity.py @@ -0,0 +1,152 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Canonical backend identity helpers for the performance smoke test""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +_DEFAULT_PHYSICS_BACKEND = "physx" +_EMPTY_VALUES = {"", "none", "null"} +_DEFAULT_VALUES = _EMPTY_VALUES | {"default"} +_PHYSICS_PRESET_TO_BACKEND = { + "physx": "physx", + "newton": "newton", + "newton_mjwarp": "newton", +} +_RENDER_PRESET_TO_BACKEND = { + "isaacsim_rtx": "rtx_renderer", + "newton_renderer": "newton_renderer", + "ovrtx_renderer": "ovrtx_renderer", + "rtx_renderer": "rtx_renderer", + "warp_renderer": "warp_renderer", +} +_KNOWN_PHYSICS_BACKENDS = ("physx", "newton") + + +@dataclass(frozen=True) +class BackendIdentity: + physics_backend: str + render_backend: str | None = None + + @property + def backend_key(self) -> str: + return make_backend_key(self.physics_backend, self.render_backend) + + def to_dict(self) -> dict[str, str | None]: + return { + "physics_backend": self.physics_backend, + "render_backend": self.render_backend, + "backend_key": self.backend_key, + } + + +def _clean(value: Any) -> str | None: + if value is None: + return None + cleaned = str(value).strip() + return cleaned or None + + +def normalize_physics_backend(value: Any, *, default: str | None = None) -> str | None: + cleaned = _clean(value) + if cleaned is None: + return default + lowered = cleaned.lower() + if lowered in _DEFAULT_VALUES: + return default + return _PHYSICS_PRESET_TO_BACKEND.get(lowered, lowered) + + +def normalize_render_backend(value: Any) -> str | None: + cleaned = _clean(value) + if cleaned is None: + return None + lowered = cleaned.lower() + if lowered in _DEFAULT_VALUES: + return None + return lowered + + +def make_backend_key(physics_backend: str, render_backend: str | None = None) -> str: + physics = normalize_physics_backend(physics_backend) + if not physics: + raise ValueError("physics_backend is required to build backend_key") + render = normalize_render_backend(render_backend) + return f"{physics}_{render}" if render else physics + + +def identity_from_parts(physics_backend: Any, render_backend: Any = None) -> BackendIdentity | None: + physics = normalize_physics_backend(physics_backend) + if not physics: + return None + return BackendIdentity(physics, normalize_render_backend(render_backend)) + + +def split_backend_key(backend_key: Any) -> BackendIdentity | None: + key = _clean(backend_key) + if not key: + return None + for physics in _KNOWN_PHYSICS_BACKENDS: + if key == physics: + return BackendIdentity(physics, None) + prefix = f"{physics}_" + if key.startswith(prefix): + return BackendIdentity(physics, normalize_render_backend(key[len(prefix) :])) + if "_" in key: + physics, render = key.split("_", 1) + return identity_from_parts(physics, render) + return BackendIdentity(key, None) + + +def preset_tokens(value: Any) -> frozenset[str]: + cleaned = _clean(value) + if not cleaned: + return frozenset() + tokens: set[str] = set() + for chunk in cleaned.replace(";", ",").split(","): + token = chunk.strip().lower() + if token: + tokens.add(token) + return frozenset(tokens) + + +def identity_from_presets(presets: Any) -> BackendIdentity | None: + tokens = preset_tokens(presets) + if not tokens: + return None + physics = next( + (_PHYSICS_PRESET_TO_BACKEND[token] for token in sorted(tokens) if token in _PHYSICS_PRESET_TO_BACKEND), + _DEFAULT_PHYSICS_BACKEND, + ) + render = next( + (_RENDER_PRESET_TO_BACKEND[token] for token in sorted(tokens) if token in _RENDER_PRESET_TO_BACKEND), + None, + ) + return BackendIdentity(physics, render) + + +def backend_identity_from_launch_config(config: dict[str, Any]) -> BackendIdentity | None: + identity = identity_from_parts(config.get("physics_backend"), config.get("render_backend")) + if identity is not None: + return identity + return split_backend_key(config.get("backend_key") or config.get("backend")) + + +def backend_identity_from_benchmark_info(info: dict[str, Any]) -> BackendIdentity | None: + direct_key = info.get("backend_key") or info.get("backend") + if direct_key: + return split_backend_key(direct_key) + + identity = identity_from_parts( + info.get("physics_backend") or info.get("physics"), + info.get("render_backend") or info.get("render"), + ) + if identity is not None: + return identity + + return identity_from_presets(info.get("presets") or info.get("preset")) diff --git a/tools/perf_smoke_test/baseline_manager.py b/tools/perf_smoke_test/baseline_manager.py new file mode 100644 index 000000000000..cf01e8b28718 --- /dev/null +++ b/tools/perf_smoke_test/baseline_manager.py @@ -0,0 +1,533 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Baseline storage for the CI performance smoke test + +The baseline store keeps immutable structured samples in ``samples.ndjson``. +Threshold stats are calculated over the newest compatible samples instead of +truncating history on write. Git-backed updates are append-only transactions: +the manager refetches the remote branch, reapplies queued samples, and retries +the push to prevent races across CI runners +""" + +import contextlib +import hashlib +import json +import os +import shutil +import statistics +import subprocess +import tempfile +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +try: + from .gate_config import BASELINE_PUSH_RETRIES, DEFAULT_K_BLOCK, DEFAULT_K_WARN, MAX_BASELINE_SAMPLES + from .oracle import Baseline +except ImportError: # pragma: no cover - supports direct script imports + from gate_config import BASELINE_PUSH_RETRIES, DEFAULT_K_BLOCK, DEFAULT_K_WARN, MAX_BASELINE_SAMPLES + from oracle import Baseline + +SAMPLES_FILENAME = "samples.ndjson" +_REPO_DIR = Path(__file__).resolve().parent +_COMMIT_ENV_DEFAULTS = { + "GIT_AUTHOR_NAME": "perf-smoke-test", + "GIT_AUTHOR_EMAIL": "perf-smoke-test@localhost", + "GIT_COMMITTER_NAME": "perf-smoke-test", + "GIT_COMMITTER_EMAIL": "perf-smoke-test@localhost", +} + + +@dataclass(frozen=True) +class BaselineUpdateRecord: + gpu_model: str + task_id: str + backend: str + fps: float + fingerprint: str | None = None + sample_metadata: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class BaselinePushResult: + branch: str + remote: str | None + base_sha: str | None + pushed_sha: str | None + attempts: int + update_count: int + pushed: bool + + +def _bucket_dir(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> Path: + base = baselines_dir / gpu_model / task_id / backend + return base if fingerprint is None else base / fingerprint + + +def _samples_path(baselines_dir: Path, gpu_model: str, task_id: str, backend: str, fingerprint=None) -> Path: + return _bucket_dir(baselines_dir, gpu_model, task_id, backend, fingerprint) / SAMPLES_FILENAME + + +def _baseline_from_values( + values: list[float], *, source: str, total_sample_count: int | None = None +) -> Baseline | None: + if not values: + return None + selected = values[-MAX_BASELINE_SAMPLES:] + median = statistics.median(selected) + deviations = [abs(v - median) for v in selected] + mad = statistics.median(deviations) if len(deviations) > 1 else 0.0 + return Baseline( + median_fps=median, + mad_fps=mad, + k_warn=DEFAULT_K_WARN, + k_block=DEFAULT_K_BLOCK, + sample_count=len(selected), + source=source, + total_sample_count=total_sample_count if total_sample_count is not None else len(values), + ) + + +def _load_sample_records_from_text(content: str) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for line in content.splitlines(): + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, dict) and isinstance(record.get("fps"), (int, float)): + records.append(record) + return records + + +def _commit_env() -> dict[str, str]: + env = os.environ.copy() + for key, value in _COMMIT_ENV_DEFAULTS.items(): + env.setdefault(key, value) + return env + + +def _git( + args: list[str], + *, + cwd: Path | None = None, + check: bool = False, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess: + result = subprocess.run( + ["git", *args], + cwd=str(cwd or _REPO_DIR), + capture_output=True, + text=True, + env=env, + ) + if check and result.returncode != 0: + raise subprocess.CalledProcessError(result.returncode, ["git", *args], result.stdout, result.stderr) + return result + + +def _git_error(result: subprocess.CompletedProcess) -> str: + return (result.stderr or result.stdout or "unknown git error").strip() + + +def _remote_ref_missing(result: subprocess.CompletedProcess) -> bool: + message = _git_error(result).lower() + return "couldn't find remote ref" in message or "could not find remote ref" in message + + +def _resolve_git_ref(ref: str, *, repo_dir: Path | None = None) -> str | None: + result = _git(["rev-parse", "--verify", ref], cwd=repo_dir) + if result.returncode != 0: + return None + return result.stdout.strip() + + +def refresh_baseline_branch( + branch: str, + *, + remote: str | None = "origin", + repo_dir: Path | None = None, + allow_missing: bool = True, +) -> str | None: + """Fetch and return the exact baseline branch SHA to read. + + Returning a SHA instead of the branch name makes aggregate comparisons traceable + and prevents a stale local branch from being used after the remote moved. + """ + if not remote: + return _resolve_git_ref(branch, repo_dir=repo_dir) + + remote_ref = f"refs/remotes/{remote}/{branch}" + refspec = f"+refs/heads/{branch}:{remote_ref}" + result = _git(["fetch", remote, refspec], cwd=repo_dir) + if result.returncode != 0: + if allow_missing and _remote_ref_missing(result): + return None + raise RuntimeError(f"Failed to fetch baseline branch {branch!r} from {remote!r}: {_git_error(result)}") + return _resolve_git_ref(remote_ref, repo_dir=repo_dir) + + +def _git_is_ancestor(commit_sha: str, base_sha: str, *, repo_dir: Path | None = None) -> bool: + result = _git(["merge-base", "--is-ancestor", commit_sha, base_sha], cwd=repo_dir) + return result.returncode == 0 + + +def _git_distance(commit_sha: str, base_sha: str, *, repo_dir: Path | None = None) -> int: + result = _git(["rev-list", "--count", f"{commit_sha}..{base_sha}"], cwd=repo_dir) + if result.returncode != 0: + return 10**9 + try: + return int(result.stdout.strip()) + except ValueError: + return 10**9 + + +def _sample_matches(record: dict[str, Any], context: dict[str, Any] | None) -> bool: + if context is None: + return True + exact_fields = ( + "gpu_model", + "task_id", + "backend_key", + "target_branch", + "launch_config_hash", + "baseline_epoch", + "benchmark_contract_hash", + "runtime_contract_hash", + ) + for field in exact_fields: + expected = context.get(field) + if expected is not None and record.get(field) != expected: + return False + base_sha = context.get("base_sha") + commit_sha = record.get("commit_sha") + repo_dir = context.get("_repo_dir") + if base_sha: + if not commit_sha: + return False + if not _git_is_ancestor(str(commit_sha), str(base_sha), repo_dir=repo_dir): + return False + return True + + +def _select_records(records: list[dict[str, Any]], context: dict[str, Any] | None) -> list[dict[str, Any]]: + compatible = [r for r in records if _sample_matches(r, context)] + base_sha = context.get("base_sha") if context else None + if base_sha: + repo_dir = context.get("_repo_dir") if context else None + compatible.sort( + key=lambda r: ( + _git_distance(str(r.get("commit_sha", "")), str(base_sha), repo_dir=repo_dir), + r.get("timestamp", ""), + ) + ) + return compatible[:MAX_BASELINE_SAMPLES] + return compatible[-MAX_BASELINE_SAMPLES:] + + +def _load_baseline_from_contents( + samples_content: str | None, + *, + match_context: dict[str, Any] | None = None, +) -> Baseline | None: + if not samples_content: + return None + records = _load_sample_records_from_text(samples_content) + selected = _select_records(records, match_context) + return _baseline_from_values( + [float(r["fps"]) for r in selected], + source="samples", + total_sample_count=len(records), + ) + + +def load_baseline( + baselines_dir: Path, + gpu_model: str, + task_id: str, + backend: str, + fingerprint=None, + match_context: dict[str, Any] | None = None, +) -> Baseline | None: + """Load compatible baseline stats for a task/backend pair.""" + samples = _samples_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + return _load_baseline_from_contents( + samples.read_text() if samples.exists() else None, + match_context=match_context, + ) + + +def _stable_sample_id(metadata: dict[str, Any]) -> str: + keys = ( + "ci_run_id", + "ci_run_attempt", + "ci_run_label", + "commit_sha", + "task_id", + "backend_key", + "launch_config_hash", + "benchmark_contract_hash", + "runtime_contract_hash", + "baseline_epoch", + "fps", + "attempt", + "was_retried", + "sample_index", + ) + payload = {key: metadata.get(key) for key in keys if metadata.get(key) is not None} + if not payload.get("ci_run_id"): + payload["timestamp"] = metadata.get("timestamp") + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:24] + + +def _sample_id_exists(samples_path: Path, sample_id: str | None) -> bool: + if not sample_id or not samples_path.exists(): + return False + for record in _load_sample_records_from_text(samples_path.read_text()): + if record.get("sample_id") == sample_id: + return True + return False + + +def make_sample_metadata( + *, + gpu_model: str, + task_id: str, + backend: str, + fps: float, + bench_result: dict | None = None, + target_branch: str | None = None, + source_branch: str | None = None, + trusted_source: str = "protected_branch", + commit_sha: str | None = None, + sample_index: int | None = None, +) -> dict[str, Any]: + bench_result = bench_result or {} + launch_config = bench_result.get("launch_config") or {} + provenance = bench_result.get("provenance") or {} + git_info = provenance.get("git") or {} + software = provenance.get("software") or {} + runtime_resources = bench_result.get("runtime_resources") or {} + metadata = { + "schema_version": 1, + "fps": float(fps), + "timestamp": datetime.now(timezone.utc).isoformat(), + "trusted_source": trusted_source, + "gpu_model": gpu_model, + "task_id": task_id, + "backend_key": backend, + "physics_backend": launch_config.get("physics_backend") or bench_result.get("physics_backend"), + "render_backend": launch_config.get("render_backend") or bench_result.get("render_backend"), + # An explicit ``commit_sha`` (e.g. from the seeder, which checks out each + # commit by SHA) takes precedence over benchmark-captured provenance, which + # can be empty when git cannot read the source tree inside the container. + "commit_sha": commit_sha or git_info.get("commit_hash"), + "branch": git_info.get("branch") or source_branch, + "target_branch": target_branch, + "launch_config_hash": bench_result.get("launch_config_hash") or launch_config.get("launch_config_hash"), + "benchmark_contract_hash": bench_result.get("benchmark_contract_hash") + or launch_config.get("benchmark_contract_hash"), + "runtime_contract_hash": bench_result.get("runtime_contract_hash"), + "runtime_contract": bench_result.get("runtime_contract"), + "runtime_info": bench_result.get("runtime_info"), + "baseline_epoch": bench_result.get("baseline_epoch") or launch_config.get("baseline_epoch", 1), + "attempt": bench_result.get("attempt"), + "was_retried": bench_result.get("was_retried"), + "sample_index": sample_index, + "ci_run_id": os.environ.get("GITHUB_RUN_ID"), + "ci_run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"), + "ci_workflow": os.environ.get("GITHUB_WORKFLOW"), + "ci_job": os.environ.get("GITHUB_JOB"), + "ci_run_label": os.environ.get("PERF_SMOKE_RUN_LABEL"), + "ci_runner_name": os.environ.get("PERF_SMOKE_RUNNER_NAME"), + "launch_config": launch_config, + "runtime": { + "isaacsim": software.get("isaacsim"), + "warp": software.get("warp"), + "cuda": (runtime_resources or {}).get("cuda_version"), + "driver": (runtime_resources or {}).get("nvidia_driver_version"), + }, + } + metadata["sample_id"] = _stable_sample_id(metadata) + return metadata + + +def match_context_from_bench_result( + bench_result: dict, + *, + gpu_model: str, + base_sha: str | None = None, + target_branch: str | None = None, +) -> dict[str, Any]: + launch_config = bench_result.get("launch_config") or {} + return { + "gpu_model": gpu_model, + "task_id": bench_result.get("task_id"), + "backend_key": bench_result.get("backend_key") or bench_result.get("backend"), + "launch_config_hash": bench_result.get("launch_config_hash") or launch_config.get("launch_config_hash"), + "benchmark_contract_hash": bench_result.get("benchmark_contract_hash") + or launch_config.get("benchmark_contract_hash"), + "runtime_contract_hash": bench_result.get("runtime_contract_hash"), + "baseline_epoch": bench_result.get("baseline_epoch") or launch_config.get("baseline_epoch", 1), + "base_sha": base_sha, + "target_branch": target_branch, + } + + +def update_baseline( + baselines_dir: Path, + gpu_model: str, + task_id: str, + backend: str, + fps: float, + fingerprint=None, + sample_metadata: dict[str, Any] | None = None, +) -> bool: + """Append a structured baseline sample. Returns False when already present.""" + bucket = _bucket_dir(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + bucket.mkdir(parents=True, exist_ok=True) + samples = _samples_path(baselines_dir, gpu_model, task_id, backend, fingerprint=fingerprint) + + metadata = dict(sample_metadata or {}) + metadata.setdefault("schema_version", 1) + metadata.setdefault("fps", float(fps)) + metadata.setdefault("timestamp", datetime.now(timezone.utc).isoformat()) + metadata.setdefault("gpu_model", gpu_model) + metadata.setdefault("task_id", task_id) + metadata.setdefault("backend_key", backend) + metadata.setdefault("sample_id", _stable_sample_id(metadata)) + + if _sample_id_exists(samples, metadata.get("sample_id")): + return False + + with samples.open("a") as fh: + fh.write(json.dumps(metadata, sort_keys=True) + "\n") + return True + + +def _git_show_file(ref: str, rel_path: str, *, repo_dir: Path | None = None) -> str | None: + result = _git(["show", f"{ref}:{rel_path}"], cwd=repo_dir) + return result.stdout if result.returncode == 0 else None + + +def load_baseline_git( + ref: str, + gpu_model: str, + task_id: str, + backend: str, + fingerprint: str | None, + match_context: dict[str, Any] | None = None, + *, + repo_dir: Path | None = None, +) -> Baseline | None: + """Load compatible baseline stats from an exact git ref or SHA.""" + samples = str(_samples_path(Path(""), gpu_model, task_id, backend, fingerprint)) + context = dict(match_context or {}) + if repo_dir is not None: + context["_repo_dir"] = repo_dir + return _load_baseline_from_contents( + _git_show_file(ref, samples, repo_dir=repo_dir), + match_context=context or None, + ) + + +@contextlib.contextmanager +def _baseline_update_worktree(base_ref: str | None, *, repo_dir: Path | None = None): + repo_dir = repo_dir or _REPO_DIR + tmpdir = tempfile.mkdtemp(prefix="perf-bl-wt-") + orphan_branch = None + try: + if base_ref: + _git(["worktree", "add", "--detach", tmpdir, base_ref], cwd=repo_dir, check=True) + else: + orphan_branch = f"perf-baseline-seed-{os.getpid()}-{Path(tmpdir).name}" + _git(["worktree", "add", "--detach", tmpdir, "HEAD"], cwd=repo_dir, check=True) + _git(["checkout", "--orphan", orphan_branch], cwd=Path(tmpdir), check=True) + rm_result = _git(["rm", "-rf", "."], cwd=Path(tmpdir)) + if rm_result.returncode not in (0, 128): + raise RuntimeError(f"Failed to clear orphan baseline worktree: {_git_error(rm_result)}") + yield Path(tmpdir) + finally: + _git(["worktree", "remove", "--force", tmpdir], cwd=repo_dir) + shutil.rmtree(tmpdir, ignore_errors=True) + if orphan_branch: + _git(["branch", "-D", orphan_branch], cwd=repo_dir) + + +def _commit_baseline_worktree(worktree: Path) -> str | None: + status = _git(["status", "--porcelain"], cwd=worktree, check=True) + if not status.stdout.strip(): + return None + _git(["add", "-A"], cwd=worktree, check=True) + _git( + ["commit", "-m", "[baseline_manager] Append baseline samples"], + cwd=worktree, + check=True, + env=_commit_env(), + ) + commit = _git(["rev-parse", "HEAD"], cwd=worktree, check=True) + return commit.stdout.strip() + + +def _apply_updates(root: Path, updates: list[BaselineUpdateRecord]) -> int: + appended = 0 + for update in updates: + if update_baseline( + root, + update.gpu_model, + update.task_id, + update.backend, + update.fps, + fingerprint=update.fingerprint, + sample_metadata=update.sample_metadata, + ): + appended += 1 + return appended + + +def update_baselines_git( + branch: str, + updates: list[BaselineUpdateRecord], + *, + remote: str | None = "origin", + max_retries: int = BASELINE_PUSH_RETRIES, + repo_dir: Path | None = None, +) -> BaselinePushResult: + """Append samples to a git-backed baseline branch and push safely. + + Each attempt starts from the latest remote branch SHA. If the push loses a + race, the next attempt refetches and reapplies the same sample IDs, making + retries idempotent. + """ + if not updates: + return BaselinePushResult(branch, remote, None, None, 0, 0, False) + if max_retries < 1: + raise ValueError("max_retries must be >= 1") + + last_error = "unknown push failure" + for attempt in range(1, max_retries + 1): + base_sha = refresh_baseline_branch(branch, remote=remote, repo_dir=repo_dir, allow_missing=True) + with _baseline_update_worktree(base_sha, repo_dir=repo_dir) as worktree: + appended = _apply_updates(worktree, updates) + commit_sha = _commit_baseline_worktree(worktree) + if commit_sha is None: + return BaselinePushResult(branch, remote, base_sha, base_sha, attempt, appended, False) + push_ref = f"HEAD:refs/heads/{branch}" + if remote: + push = _git(["push", remote, push_ref], cwd=worktree) + else: + push = _git(["branch", "--force", branch, "HEAD"], cwd=worktree) + if push.returncode == 0: + return BaselinePushResult(branch, remote, base_sha, commit_sha, attempt, appended, bool(remote)) + last_error = _git_error(push) + print(f"[baseline_manager] baseline push attempt {attempt}/{max_retries} failed; refetching and retrying") + + raise RuntimeError(f"Failed to push baseline branch {branch!r} after {max_retries} attempts: {last_error}") diff --git a/tools/perf_smoke_test/benchmark_result_adapter.py b/tools/perf_smoke_test/benchmark_result_adapter.py new file mode 100644 index 000000000000..7d73f0f15ca7 --- /dev/null +++ b/tools/perf_smoke_test/benchmark_result_adapter.py @@ -0,0 +1,301 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Adapter: schema-v1 ``RuntimeBundle`` JSON -> perf-gate normalized fields. + +``perf_runtime.py`` emits a +:class:`~isaaclab.benchmark.schema.RuntimeBundle` (Isaac Lab benchmark +refactor Part 1, PR #6197) serialized by +:func:`~isaaclab.benchmark.serialize.write_bundle_file`. This module is the +single point that reads that JSON and projects it into the flat +``provenance`` / ``runtime_resources`` / fps / ``benchmark_info`` shapes the gate's +:mod:`oracle` and :mod:`build_bench_result` consume, replacing the legacy +phase-array parsing. + +``raw_fps_min`` is *recovered* from the slowest steady-state step +(``num_envs / iteration_time_s.peak``); ``raw_fps_{mean,std,max}`` come straight +from ``runtime.total_fps.{mean,std,peak}``. Warmup exclusion is applied at the +source by ``perf_runtime.py`` (``--warmup_frames``), so ``total_fps.mean`` is +already steady-state. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +from contracts import RuntimeSample + +# Map the schema's rendering-backend vocabulary to the gate's render-preset +# tokens used in tasks.json / backend_identity. ``"none"`` (headless, no camera) +# maps to ``None``. +_SCHEMA_RENDER_TO_GATE: dict[str | None, str | None] = { + None: None, + "": None, + "none": None, + "newton": "newton_renderer", + "ovrtx": "ovrtx_renderer", + "isaacsim_rtx": "rtx_renderer", +} + + +def _as_dict(value: Any) -> dict: + """Return ``value`` if it is a dict, else an empty dict. + + Bundle sections are always dicts in a valid schema-v1 file; this keeps a + malformed/hand-edited bundle (e.g. a list where a dict is expected) from + crashing the projection so the gate degrades gracefully instead of the + result builder aborting with no output. + """ + return value if isinstance(value, dict) else {} + + +def steady_state_slice(step_times: list[float], warmup_frames: int) -> tuple[list[float], int]: + """Drop the leading ``warmup_frames`` cold-start steps, keeping >=1 frame. + + Producer-side helper used by ``perf_runtime.py`` to exclude warmup at the + source before aggregation. If ``warmup_frames`` would leave nothing, it is + clamped to ``len(step_times) - 1`` so the aggregate never silently falls back + to the full, cold-start-inclusive series (which would misreport non-steady + numbers as steady-state). + + Args: + step_times: Per-step wall times [s], in order. + warmup_frames: Requested number of leading steps to discard. + + Returns: + ``(measured_step_times, warmup_applied)`` where ``warmup_applied`` is the + number of leading steps actually discarded (may be clamped below the + request). + """ + n = len(step_times) + if n == 0: + return [], 0 + warmup = max(0, min(warmup_frames, n - 1)) + return list(step_times[warmup:]), warmup + + +def load_info(info_path: Path) -> dict | None: + """Load the benchmark info JSON, or ``None`` if it is missing/unreadable.""" + try: + data = json.loads(Path(info_path).read_text()) + except Exception: + return None + return data if isinstance(data, dict) else None + + +def is_runtime_bundle(data: Any) -> bool: + """Return True when ``data`` looks like a schema-v1 runtime/training bundle.""" + return ( + isinstance(data, dict) + and isinstance(data.get("run"), dict) + and isinstance(data.get("runtime"), dict) + and "schema_version" in data + ) + + +def gpu_driver_version() -> str | None: + """Return the GPU driver version from nvidia-smi, or ``None`` if unavailable.""" + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader,nounits"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + version = result.stdout.strip().splitlines()[0].strip() + return version or None + except Exception: + pass + return None + + +def _current_gpu(bundle: dict) -> dict: + """Return the first GPU device dict from the hardware snapshot (or ``{}``).""" + devices = _as_dict(bundle.get("hardware")).get("gpu_devices") or [] + return devices[0] if devices and isinstance(devices[0], dict) else {} + + +def render_backend(bundle: dict) -> str | None: + """Return the gate render token for the bundle's rendering backend.""" + schema_render = _as_dict(_as_dict(bundle.get("run")).get("config")).get("rendering_backend") + key = schema_render.lower() if isinstance(schema_render, str) else schema_render + return _SCHEMA_RENDER_TO_GATE.get(key, key or None) + + +def fps_stats(bundle: dict) -> dict: + """Return ``raw_fps_{mean,std,min,max}`` from the bundle's runtime aggregates. + + ``mean``/``std``/``max`` come from ``total_fps``; ``min`` (worst steady-state + frame) is recovered from the slowest step time. Percentile fields are not + available in the schema and are intentionally omitted. + """ + runtime = _as_dict(bundle.get("runtime")) + total_fps = _as_dict(runtime.get("total_fps")) + iter_time = _as_dict(runtime.get("iteration_time_s")) + steps_per_iter = runtime.get("steps_per_iteration") or _as_dict(bundle.get("run")).get("num_envs") + + out: dict = {} + if isinstance(total_fps.get("mean"), (int, float)): + out["raw_fps_mean"] = float(total_fps["mean"]) + if isinstance(total_fps.get("std"), (int, float)): + out["raw_fps_std"] = float(total_fps["std"]) + if isinstance(total_fps.get("peak"), (int, float)): + out["raw_fps_max"] = float(total_fps["peak"]) + peak_step = iter_time.get("peak") + if isinstance(peak_step, (int, float)) and peak_step > 0 and steps_per_iter: + out["raw_fps_min"] = float(steps_per_iter) / float(peak_step) + return out + + +def startup_seconds(bundle: dict) -> float | None: + """Return total launch-to-first-step wall time [s] (sum of startup phases).""" + startup = _as_dict(_as_dict(bundle.get("runtime")).get("startup_time_s")) + values = [v for v in startup.values() if isinstance(v, (int, float))] + return float(sum(values)) if values else None + + +def provenance(bundle: dict) -> dict: + """Return ``{hardware, software, git}`` for the runtime-compatibility contract. + + ``software`` is the bundle's typed ``versions`` map verbatim (its field names + โ€” ``isaaclab``/``isaacsim``/``torch``/``warp``/``isaaclab_physx``/ + ``isaaclab_newton``/``newton``/``isaaclab_ov`` โ€” match the contract policy + paths, so the ``runtime_contract_hash`` is preserved across the migration). + """ + versions = _as_dict(bundle.get("versions")) + hardware_snapshot = _as_dict(bundle.get("hardware")) + gpu = _current_gpu(bundle) + + software = {k: v for k, v in versions.items() if v is not None and not k.startswith("git_")} + hardware = { + k: v + for k, v in { + "cpu_name": hardware_snapshot.get("cpu_name"), + "cpu_physical_cores": hardware_snapshot.get("cpu_count"), + "total_ram_gb": hardware_snapshot.get("ram_gb"), + "gpu_device_count": len(hardware_snapshot.get("gpu_devices") or []) or None, + "gpu_name": gpu.get("name"), + "gpu_total_memory_gb": gpu.get("mem_gb"), + "gpu_compute_capability": gpu.get("compute_cap"), + }.items() + if v is not None + } + git = { + gate_key: versions[schema_key] + for gate_key, schema_key in ( + ("commit_hash", "git_commit"), + ("branch", "git_branch"), + ("dirty", "git_dirty"), + ) + if versions.get(schema_key) is not None + } + return {"hardware": hardware, "software": software, "git": git} + + +def _gb_to_mb(value: Any) -> float | None: + """Return ``value`` GB converted to MB (2 dp), or ``None`` if not numeric.""" + return round(float(value) * 1024, 2) if isinstance(value, (int, float)) else None + + +def _pct(value: Any) -> float | None: + """Return ``value`` as a utilisation percent (2 dp), or ``None`` if not numeric.""" + return round(float(value), 2) if isinstance(value, (int, float)) else None + + +def runtime_resources(bundle: dict) -> dict: + """Return the GPU-diagnostics + resource-utilisation block (non-gating publish info). + + Combines GPU identity (name / total memory / CUDA / driver) with the run's + measured resource utilisation from the bundle's ``resources`` section: VRAM + (mean + peak), system RAM (mean + peak), and GPU/CPU utilisation (mean). Every + field here is informational โ€” it is published for humans but never feeds the + gate verdict or the ``runtime_contract_hash``. + + Memory is reported in MB (the schema stores GB); utilisation in percent. Two + semantic caveats worth remembering when reading these values: + + * ``gpu_mem_*`` is **device-wide** VRAM (``nvidia-smi memory.used`` includes + any other process on the GPU) โ€” accurate on a 1-benchmark-per-GPU runner. + * ``system_ram_*`` is the benchmark **process** resident set size (psutil + ``memory_info().rss``), not whole-host RAM and excluding child processes. + + Absent/malformed sub-sections drop their fields (``None`` filtered out) so a + partial bundle degrades gracefully instead of crashing the projection. + """ + gpu = _current_gpu(bundle) + resources = _as_dict(bundle.get("resources")) + gpu_mem = _as_dict(resources.get("gpu_mem_gb")) + ram = _as_dict(resources.get("ram_gb")) + gpu_util = _as_dict(resources.get("gpu_util_pct")) + cpu_util = _as_dict(resources.get("cpu_util_pct")) + # schema-v1 Hardware has no CUDA-runtime field; use the CUDA bindings version + # (Versions.cuda_bindings) as the closest available proxy for display. + cuda_version = _as_dict(bundle.get("versions")).get("cuda_bindings") + diag = { + "gpu_name": gpu.get("name"), + "gpu_total_memory_gb": gpu.get("mem_gb"), + "cuda_version": cuda_version, + "nvidia_driver_version": gpu_driver_version(), + "gpu_mem_used_mb": _gb_to_mb(gpu_mem.get("mean")), + "gpu_mem_peak_mb": _gb_to_mb(gpu_mem.get("peak")), + "gpu_util_pct": _pct(gpu_util.get("mean")), + "system_ram_used_mb": _gb_to_mb(ram.get("mean")), + "system_ram_peak_mb": _gb_to_mb(ram.get("peak")), + "cpu_util_pct": _pct(cpu_util.get("mean")), + } + return {k: v for k, v in diag.items() if v is not None} + + +def benchmark_info(bundle: dict) -> dict: + """Return the run's self-reported identity for launch/run drift checks.""" + run = _as_dict(bundle.get("run")) + config = _as_dict(run.get("config")) + extra = _as_dict(bundle.get("extra")) + info = { + "task": run.get("task"), + "num_envs": run.get("num_envs"), + "seed": run.get("seed"), + "num_frames": extra.get("num_frames"), + "warmup_frames": extra.get("warmup_frames"), + "status": run.get("status"), + "physics_backend": config.get("physics_backend"), + "render_backend": render_backend(bundle), + "presets": config.get("presets") or [], + } + return {k: v for k, v in info.items() if v is not None} + + +def project_runtime(bundle: dict) -> RuntimeSample | None: + """Project a runtime bundle into a typed :class:`~contracts.RuntimeSample`. + + Returns ``None`` when ``bundle`` is not a valid schema-v1 runtime bundle, so the + caller can degrade to a HARD_FAILURE (missing benchmark output). + """ + if not is_runtime_bundle(bundle): + return None + stats = fps_stats(bundle) + info = benchmark_info(bundle) + return RuntimeSample( + fps_mean=stats.get("raw_fps_mean"), + fps_std=stats.get("raw_fps_std"), + fps_min=stats.get("raw_fps_min"), + fps_max=stats.get("raw_fps_max"), + startup_time_s=startup_seconds(bundle), + task=info.get("task"), + num_envs=info.get("num_envs"), + seed=info.get("seed"), + num_frames=info.get("num_frames"), + warmup_frames=info.get("warmup_frames"), + status=info.get("status"), + physics_backend=info.get("physics_backend"), + render_backend=info.get("render_backend"), + presets=info.get("presets") or [], + provenance=provenance(bundle), + runtime_resources=runtime_resources(bundle), + ) diff --git a/tools/perf_smoke_test/build_bench_result.py b/tools/perf_smoke_test/build_bench_result.py new file mode 100644 index 000000000000..84be016610fe --- /dev/null +++ b/tools/perf_smoke_test/build_bench_result.py @@ -0,0 +1,319 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Post-benchmark script: normalizes benchmark output and writes perf_smoke_test_result.json. + +Locates the timestamped runtime bundle written by perf_runtime.py, copies it +to the canonical ``perf_smoke_test_info.json``, classifies the failure phase from the +captured log, and writes ``perf_smoke_test_result.json`` for the aggregate job. + +Usage:: + + python3 tools/perf_smoke_test/build_bench_result.py \\ + --task_id Isaac-Cartpole-Direct-v0 \\ + --artifact_dir artifacts/Isaac-Cartpole-Direct-v0 \\ + --exit_code 0 \\ + --wall_time_s 48.3 \\ + --timeout_s 600 \\ + --log_file artifacts/Isaac-Cartpole-Direct-v0/benchmark.log +""" + +import argparse +import glob +import json +import shutil +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +_TOOLS_DIR = _MODULE_DIR.parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + +from backend_identity import ( # noqa: E402 + backend_identity_from_benchmark_info, + backend_identity_from_launch_config, + identity_from_parts, + make_backend_key, + normalize_physics_backend, + normalize_render_backend, +) +from benchmark_result_adapter import load_info, project_runtime # noqa: E402 +from contracts import BenchResult # noqa: E402 +from gate_config import load_gate_config # noqa: E402 +from gate_types import FailurePhase # noqa: E402 +from gpu_identity import normalize_gpu_fields # noqa: E402 +from launch_config import fallback_launch_config, load_launch_config # noqa: E402 +from runtime_contract import build_runtime_contract, build_runtime_publish_info # noqa: E402 +from subprocess_runner import classify_failure_phase # noqa: E402 +from task_config import get_task # noqa: E402 + + +def _coerce_int(value: object) -> int | None: + try: + return int(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return None + + +def _config_drift(benchmark_info: dict, launch_config: dict) -> str | None: + """Return a compact mismatch string when the actual run differs from launch intent.""" + if not benchmark_info: + return None + mismatches: list[str] = [] + + wanted_task = launch_config.get("task_id") + ran_task = benchmark_info.get("task") + if isinstance(ran_task, str) and ran_task and wanted_task and ran_task != wanted_task: + mismatches.append(f"task(ran={ran_task},want={wanted_task})") + + for field in ("num_envs", "seed"): + wanted = _coerce_int(launch_config.get(field)) + ran = _coerce_int(benchmark_info.get(field)) + if wanted is not None and ran is not None and ran != wanted: + mismatches.append(f"{field}(ran={ran},want={wanted})") + + wanted_frames = _coerce_int(launch_config.get("num_frames")) + ran_frames = _coerce_int(benchmark_info.get("num_frames")) + if wanted_frames is not None and ran_frames is not None and ran_frames < wanted_frames: + mismatches.append(f"num_frames(ran={ran_frames},want>={wanted_frames})") + + wanted_backend = backend_identity_from_launch_config(launch_config) + ran_backend = backend_identity_from_benchmark_info(benchmark_info) + if wanted_backend is not None and ran_backend is not None and wanted_backend.backend_key != ran_backend.backend_key: + mismatches.append(f"backend(ran={ran_backend.backend_key},want={wanted_backend.backend_key})") + + return " ".join(mismatches) if mismatches else None + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Build perf_smoke_test_result.json from a benchmark run") + p.add_argument("--task_id", required=True) + p.add_argument("--physics_backend", required=True, help="Physics backend used (e.g. physx, newton)") + p.add_argument("--render_backend", default="", help="Render backend used (e.g. rtx, warp, ovrtx); empty = none") + p.add_argument("--artifact_dir", required=True, type=Path) + p.add_argument("--exit_code", required=True, type=int) + p.add_argument("--wall_time_s", required=True, type=float) + p.add_argument("--timeout_s", required=True, type=float) + p.add_argument("--log_file", type=Path, default=None) + p.add_argument( + "--launch_config", + type=Path, + default=None, + help="Path to launch_config.json (default: artifact_dir/launch_config.json)", + ) + p.add_argument("--gate_config", type=Path, default=_MODULE_DIR / "gate_config.json") + p.add_argument("--attempt", type=int, default=1, help="Attempt number (1 = first run, 2 = after one retry)") + p.add_argument( + "--was_retried", action="store_true", help="Set when this result comes from a retry of a failed first attempt" + ) + return p.parse_args() + + +def _normalize_benchmark_output(artifact_dir: Path, task_id: str) -> bool: + """Copy the timestamped runtime bundle to ``perf_smoke_test_info.json``. + + ``perf_runtime.py`` writes ``benchmark_runtime_{task_id}_{timestamp}.json`` + (a schema-v1 :class:`~isaaclab.benchmark.schema.RuntimeBundle`). The gate + reads the canonical ``perf_smoke_test_info.json``; this bridges the gap. + + Returns True if perf_smoke_test_info.json exists after call + """ + perf_smoke_test_info = artifact_dir / "perf_smoke_test_info.json" + if perf_smoke_test_info.exists(): + return True + # Primary pattern: exact task_id match + matches = sorted(glob.glob(str(artifact_dir / f"benchmark_runtime_{task_id}_*.json"))) + if not matches: + # Fallback: any benchmark_runtime_*.json in the artifact dir + matches = sorted(glob.glob(str(artifact_dir / "benchmark_runtime_*.json"))) + if not matches: + return False + shutil.copy(matches[-1], perf_smoke_test_info) + return True + + +def main() -> int: + args = _parse_args() + artifact_dir = args.artifact_dir + artifact_dir.mkdir(parents=True, exist_ok=True) + gate_config = load_gate_config(args.gate_config) + runtime_policy = gate_config.get("runtime_compatibility", {}) + + cli_physics_backend = normalize_physics_backend(args.physics_backend) + if cli_physics_backend is None: + raise ValueError("--physics_backend must name a concrete backend") + cli_render_backend = normalize_render_backend(args.render_backend) + cli_backend_key = make_backend_key(cli_physics_backend, cli_render_backend) + + launch_config = load_launch_config(artifact_dir, args.launch_config) + if launch_config is None: + try: + task = get_task(args.task_id, cli_backend_key) + except KeyError: + print( + f"[build_bench_result] Warning: ({args.task_id!r}, {cli_backend_key!r}) not found in " + "tasks.json; using defaults" + ) + task = None + launch_config = fallback_launch_config( + task_id=args.task_id, + physics_backend=cli_physics_backend, + render_backend=cli_render_backend, + backend_key=cli_backend_key, + timeout_s=args.timeout_s, + task=task, + ) + launch_config = dict(launch_config) + + expected_backend = backend_identity_from_launch_config(launch_config) or identity_from_parts( + cli_physics_backend, cli_render_backend + ) + if expected_backend is None: + raise ValueError("launch_config must define a concrete backend identity") + task_id = str(launch_config.get("task_id") or args.task_id) + physics_backend = expected_backend.physics_backend + render_backend = expected_backend.render_backend + backend_key = expected_backend.backend_key + + phase2_mismatches: list[str] = [] + if args.task_id != task_id: + phase2_mismatches.append(f"phase2_task_arg(arg={args.task_id},want={task_id})") + if cli_backend_key != backend_key: + phase2_mismatches.append(f"phase2_backend_arg(arg={cli_backend_key},want={backend_key})") + phase2_arg_mismatch = " ".join(phase2_mismatches) if phase2_mismatches else None + + gpu_fields = normalize_gpu_fields(launch_config.get("gpu_model_raw") or launch_config.get("gpu_model")) + launch_config["task_id"] = task_id + launch_config["backend_key"] = backend_key + launch_config["backend"] = backend_key + launch_config["physics_backend"] = physics_backend + launch_config["render_backend"] = render_backend + launch_config["gpu_model"] = gpu_fields["gpu_model"] + launch_config["gpu_model_raw"] = gpu_fields["gpu_model_raw"] + + num_envs = launch_config.get("num_envs", 0) + num_frames = launch_config.get("num_frames", 0) + warmup_frames = launch_config.get("warmup_frames", 0) + timeout_minutes = launch_config.get("timeout_minutes", int(args.timeout_s / 60)) + preset = launch_config.get("preset", "default") + tags = launch_config.get("tags", ["always"]) + seed = launch_config.get("seed") + + # Read combined stdout/stderr log for failure classification + log_text = "" + if args.log_file and args.log_file.exists(): + log_text = args.log_file.read_text(errors="replace") + + perf_smoke_test_info_present = _normalize_benchmark_output(artifact_dir, task_id) + + failure_phase = classify_failure_phase( + stdout=log_text, + stderr="", + exit_code=args.exit_code, + wall_time_s=args.wall_time_s, + timeout_s=args.timeout_s, + ) + + # Read the runtime bundle (schema v1) and project it into a typed RuntimeSample. + sample = None + benchmark_info: dict = {} + config_mismatch: str | None = None + observed_backend = None + runtime_contract = None + runtime_contract_hash = None + runtime_info = None + if perf_smoke_test_info_present: + info_path = artifact_dir / "perf_smoke_test_info.json" + bundle = load_info(info_path) + sample = project_runtime(bundle) if bundle is not None else None + if sample is not None: + benchmark_info = sample.benchmark_info() + observed_backend = backend_identity_from_benchmark_info(benchmark_info) + runtime_contract, runtime_contract_hash = build_runtime_contract( + provenance=sample.provenance, + runtime_resources=sample.runtime_resources, + backend=expected_backend, + policy=runtime_policy, + ) + runtime_info = build_runtime_publish_info( + provenance=sample.provenance, + runtime_resources=sample.runtime_resources, + policy=runtime_policy, + ) + config_mismatch = _config_drift(benchmark_info, launch_config) + else: + # File exists but is not a valid schema-v1 bundle (corrupt/truncated). + perf_smoke_test_info_present = False + config_mismatch = " ".join(part for part in (phase2_arg_mismatch, config_mismatch) if part) or None + if config_mismatch and failure_phase is None: + failure_phase = FailurePhase.CONFIG_MISMATCH.value + + bench_result = BenchResult( + task_id=task_id, + backend=backend_key, + physics_backend=physics_backend, + render_backend=render_backend, + backend_key=backend_key, + preset=preset, + attempt=args.attempt, + was_retried=args.was_retried, + exit_code=args.exit_code, + failure_phase=failure_phase, + stdout_tail=log_text[-2000:] if len(log_text) > 2000 else log_text, + wall_time_s=args.wall_time_s, + startup_time_s=sample.startup_time_s if sample else None, + perf_smoke_test_info_present=perf_smoke_test_info_present, + raw_fps_mean=sample.fps_mean if sample else None, + raw_fps_std=sample.fps_std if sample else None, + raw_fps_min=sample.fps_min if sample else None, + raw_fps_max=sample.fps_max if sample else None, + benchmark_info=benchmark_info, + observed_backend=observed_backend.to_dict() if observed_backend else None, + config_mismatch=config_mismatch, + runtime_contract=runtime_contract, + runtime_contract_hash=runtime_contract_hash, + runtime_info=runtime_info, + runtime_resources=(sample.runtime_resources or None) if sample else None, + provenance=sample.provenance if sample else None, + launch_config=launch_config, + launch_config_hash=launch_config.get("launch_config_hash"), + benchmark_contract_hash=launch_config.get("benchmark_contract_hash"), + baseline_epoch=launch_config.get("baseline_epoch", 1), + task_config_snapshot={ + "task_id": task_id, + "backend": backend_key, + "physics_backend": physics_backend, + "render_backend": render_backend, + "backend_key": backend_key, + "preset": preset, + "num_envs": num_envs, + "num_frames": num_frames, + "warmup_frames": warmup_frames, + "timeout_minutes": timeout_minutes, + "tags": tags, + "seed": seed, + "launch_config_hash": launch_config.get("launch_config_hash"), + "benchmark_contract_hash": launch_config.get("benchmark_contract_hash"), + "runtime_contract_hash": runtime_contract_hash, + "baseline_epoch": launch_config.get("baseline_epoch", 1), + }, + ) + + out = artifact_dir / "perf_smoke_test_result.json" + out.write_text(json.dumps(bench_result.to_dict(), indent=2)) + + status = ( + f"failure_phase={failure_phase!r}, perf_smoke_test_info_present={perf_smoke_test_info_present}, " + f"exit_code={args.exit_code}, config_mismatch={config_mismatch!r}" + ) + print(f"[build_bench_result] {task_id}: {status}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_smoke_test/contracts.py b/tools/perf_smoke_test/contracts.py new file mode 100644 index 000000000000..f74a957be168 --- /dev/null +++ b/tools/perf_smoke_test/contracts.py @@ -0,0 +1,132 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Typed contracts for the perf-gate's own artifacts (schema v1). + +The gate's analogue of :mod:`isaaclab.benchmark.schema`: the fields the gate +computes on are typed dataclass attributes, while open pass-through sub-structures +(``benchmark_info``, ``provenance``, ``launch_config``, the compatibility contracts) +stay as ``dict`` fields โ€” mirroring ``RuntimeBundle``'s typed fields + ``extra: dict``. + +:class:`RuntimeSample` is the adapter's projection of a benchmark ``RuntimeBundle``. +:class:`BenchResult` is the typed shape of ``perf_smoke_test_result.json``; it +serializes (:meth:`BenchResult.to_dict`) to the same wire format the gate has always +written (plus an additive ``schema_version``), and reconstructs strictly via +:meth:`BenchResult.from_dict`. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field, fields +from typing import Any + +CONTRACT_SCHEMA_VERSION = "1.0" + + +@dataclass(frozen=True) +class RuntimeSample: + """Adapter projection of a schema-v1 ``RuntimeBundle`` into the gate's fields. + + Aggregates (fps/startup) and run identity are typed; ``provenance`` and + ``runtime_resources`` remain dicts (open provenance payloads the gate carries through). + """ + + fps_mean: float | None = None + fps_std: float | None = None + fps_min: float | None = None + fps_max: float | None = None + startup_time_s: float | None = None + task: str | None = None + num_envs: int | None = None + seed: int | None = None + num_frames: int | None = None + warmup_frames: int | None = None + status: str | None = None + physics_backend: str | None = None + render_backend: str | None = None + presets: list[str] = field(default_factory=list) + provenance: dict[str, Any] = field(default_factory=dict) + runtime_resources: dict[str, Any] = field(default_factory=dict) + + def benchmark_info(self) -> dict[str, Any]: + """Return the run's self-reported identity as a dict. + + Feeds the dict-based drift / backend-identity helpers; keys whose value is + ``None`` are omitted (matching the historical ``benchmark_info`` shape). + """ + info = { + "task": self.task, + "num_envs": self.num_envs, + "seed": self.seed, + "num_frames": self.num_frames, + "warmup_frames": self.warmup_frames, + "status": self.status, + "physics_backend": self.physics_backend, + "render_backend": self.render_backend, + "presets": self.presets or None, + } + return {k: v for k, v in info.items() if v is not None} + + +@dataclass(frozen=True) +class BenchResult: + """Typed shape of ``perf_smoke_test_result.json`` (schema v1). + + Field order matches the historical JSON so serialization is wire-stable; the + only addition is a trailing ``schema_version``. The leading identity fields are + required (a build that forgets one fails at construction); the rest default so a + HARD_FAILURE result (no benchmark output) still constructs cleanly. + """ + + # --- identity (required) --- + task_id: str + backend: str + physics_backend: str + render_backend: str | None + backend_key: str + preset: str + # --- run status --- + attempt: int = 1 + was_retried: bool = False + exit_code: int = 0 + failure_phase: str | None = None + stdout_tail: str = "" + wall_time_s: float | None = None + startup_time_s: float | None = None + perf_smoke_test_info_present: bool = False + # --- perf (steady-state aggregates) --- + raw_fps_mean: float | None = None + raw_fps_std: float | None = None + raw_fps_min: float | None = None + raw_fps_max: float | None = None + # --- open pass-through payloads + matching/contracts --- + benchmark_info: dict[str, Any] = field(default_factory=dict) + observed_backend: dict[str, Any] | None = None + config_mismatch: str | None = None + runtime_contract: dict[str, Any] | None = None + runtime_contract_hash: str | None = None + runtime_info: dict[str, Any] | None = None + runtime_resources: dict[str, Any] | None = None + provenance: dict[str, Any] | None = None + launch_config: dict[str, Any] = field(default_factory=dict) + launch_config_hash: str | None = None + benchmark_contract_hash: str | None = None + baseline_epoch: int = 1 + task_config_snapshot: dict[str, Any] = field(default_factory=dict) + schema_version: str = CONTRACT_SCHEMA_VERSION + + def to_dict(self) -> dict[str, Any]: + """Serialize to the on-disk JSON shape (plain dicts, wire-stable order).""" + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> BenchResult: + """Reconstruct from a parsed ``perf_smoke_test_result.json``. + + Strict on this version's own artifacts: unknown keys are dropped, and + missing required identity fields raise (there is no legacy-tolerance layer). + """ + known = {f.name for f in fields(cls)} + return cls(**{k: v for k, v in data.items() if k in known}) diff --git a/tools/perf_smoke_test/gate_config.json b/tools/perf_smoke_test/gate_config.json new file mode 100644 index 000000000000..d67f3caa6341 --- /dev/null +++ b/tools/perf_smoke_test/gate_config.json @@ -0,0 +1,6 @@ +{ + "_comment": "Perf-smoke gate configuration. 'blocking' is intentionally false: in advisory mode aggregate.py exits 0 for every verdict, including HARD_FAILURE, so the gate never fails a pull request on its own. The verdict is still reported -- a BLOCK or HARD FAILURE paints the 'perf-smoke-test' commit status red and is spelled out in the sticky PR comment and the job summary. Gate malfunctions (no bench artifacts at all, unreadable baseline branch, failed baseline push) exit nonzero in both modes, because those mean no trustworthy verdict was produced. Flipping 'blocking' to true is a deliberate, separate rollout: HARD_FAILURE then exits 2 and BLOCK exits 1, failing the aggregate job itself, and the 'perf-smoke-test' status becomes suitable to mark as a required check. Only keys the gate actually reads belong here; the baseline window sizes (MIN_BASELINE_SAMPLES, MAX_BASELINE_SAMPLES) are imported directly by the oracle and the baseline loader and cannot be overridden from this file. runtime_compatibility is intentionally omitted so it is sourced from gate_config.DEFAULT_RUNTIME_COMPATIBILITY (see gate_config.py); add a key only to override the code default.", + "blocking": false, + "min_block_regression_pct": 3.0, + "baseline_push_retries": 3 +} diff --git a/tools/perf_smoke_test/gate_config.py b/tools/perf_smoke_test/gate_config.py new file mode 100644 index 000000000000..c59ff6cb804a --- /dev/null +++ b/tools/perf_smoke_test/gate_config.py @@ -0,0 +1,90 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import json +from pathlib import Path + +DEFAULT_K_WARN = 2.5 +DEFAULT_K_BLOCK = 4.0 +MIN_BASELINE_SAMPLES = 5 +MAX_BASELINE_SAMPLES = 20 +MIN_BLOCK_REGRESSION_PCT = 3.0 +BASELINE_PUSH_RETRIES = 3 +DEFAULT_RUNTIME_COMPATIBILITY = { + "contract_version": 2, + "always": [ + "software.isaacsim", + "software.isaaclab", + "software.torch", + "software.warp", + # CPU model is part of the hardware identity: FPS on CPU-sensitive tasks + # (e.g. PhysX stepping) varies by CPU, so baselines must not pool samples + # across heterogeneous CPUs. GPU model is already enforced separately via + # the per-GPU baseline partition key (see gpu_identity), so it is not + # repeated here. + "hardware.cpu_name", + ], + "by_physics_backend": { + "physx": [ + "software.isaaclab_physx", + ], + "newton": [ + "software.isaaclab_newton", + "software.newton", + ], + }, + "by_render_backend": { + "newton_renderer": [ + "software.isaaclab_ov", + ], + "ovrtx_renderer": [ + "software.isaaclab_ov", + ], + "warp_renderer": [ + "software.isaaclab_ov", + ], + "rtx_renderer": [ + "software.isaaclab_ov", + ], + }, + "publish_only": [ + "hardware.gpu_compute_capability", + "hardware.gpu_total_memory_gb", + "runtime_resources.cuda_version", + "runtime_resources.nvidia_driver_version", + ], +} + + +def _merge_runtime_compatibility(raw: dict | None) -> dict: + policy = json.loads(json.dumps(DEFAULT_RUNTIME_COMPATIBILITY)) + if not raw: + return policy + for key, value in raw.items(): + if isinstance(value, dict) and isinstance(policy.get(key), dict): + policy[key].update(value) + else: + policy[key] = value + return policy + + +def load_gate_config(path: Path | str) -> dict: + # The baseline window sizes are deliberately absent: MIN_BASELINE_SAMPLES and + # MAX_BASELINE_SAMPLES are imported directly by the oracle and the baseline + # loader, so surfacing them here would offer a knob that changes nothing. + config = { + "blocking": False, + "min_block_regression_pct": MIN_BLOCK_REGRESSION_PCT, + "baseline_push_retries": BASELINE_PUSH_RETRIES, + "runtime_compatibility": _merge_runtime_compatibility(None), + } + p = Path(path) + if p.exists(): + with p.open() as fh: + loaded = json.load(fh) + runtime_policy = _merge_runtime_compatibility(loaded.pop("runtime_compatibility", None)) + config.update(loaded) + config["runtime_compatibility"] = runtime_policy + return config diff --git a/tools/perf_smoke_test/gate_types.py b/tools/perf_smoke_test/gate_types.py new file mode 100644 index 000000000000..2ef94b2af694 --- /dev/null +++ b/tools/perf_smoke_test/gate_types.py @@ -0,0 +1,162 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared model states for the performance smoke test""" + +from dataclasses import dataclass +from enum import Enum + + +class OracleVerdict(str, Enum): + PASS = "PASS" + WARN = "WARN" + BLOCK = "BLOCK" + HARD_FAILURE = "HARD_FAILURE" + + +# Severity ordering used to combine independent verdict signals; higher is worse. +_VERDICT_SEVERITY: dict[OracleVerdict, int] = { + OracleVerdict.PASS: 0, + OracleVerdict.WARN: 1, + OracleVerdict.BLOCK: 2, + OracleVerdict.HARD_FAILURE: 3, +} + +# Verdicts a configured FPS threshold is allowed to declare (gating verdicts). +# ``PASS``/``HARD_FAILURE`` are not thresholdable: PASS gates nothing and +# HARD_FAILURE is reserved for benchmark-level failures, not FPS floors. +THRESHOLD_VERDICTS: frozenset[OracleVerdict] = frozenset({OracleVerdict.WARN, OracleVerdict.BLOCK}) + + +def verdict_severity(verdict: OracleVerdict) -> int: + """Return the severity rank of a verdict; higher means worse.""" + return _VERDICT_SEVERITY[verdict] + + +def worst_verdict(*verdicts: OracleVerdict) -> OracleVerdict: + """Return the most severe verdict among the arguments.""" + return max(verdicts, key=verdict_severity) + + +class BisectVerdict(str, Enum): + GOOD = "GOOD" + BAD = "BAD" + SKIP = "SKIP" + + +class FailurePhase(str, Enum): + IMPORT = "import" + INIT = "init" + RUNTIME = "runtime" + OOM = "oom" + HANG = "hang" + DRIVER = "driver" + CONFIG_MISMATCH = "config_mismatch" + + +class ThresholdSource(str, Enum): + NO_BASELINE = "no_baseline" + INSUFFICIENT_WINDOW = "insufficient_window" + ROLLING_WINDOW = "rolling_window" + THRESHOLD = "threshold" + NOT_APPLICABLE = "n/a" + + +@dataclass(frozen=True) +class FpsMeanThreshold: + """A single configured mean-FPS threshold for one task/backend/GPU. + + A threshold is *crossed* when the measured mean FPS falls below :attr:`value`. + A crossed threshold whose :attr:`verdict` is set contributes that verdict to + the gate outcome; a threshold with :attr:`verdict` = ``None`` is *reporting-only* + and is surfaced in outputs without ever changing the verdict. + + Args: + name: Informative label for the threshold (e.g. ``"IsaacLab-2.0"``). + value: Mean-FPS floor [FPS]. ``0.0`` is a valid, effectively non-gating floor. + verdict: Gating verdict to raise when crossed, or ``None`` for reporting-only. + """ + + name: str + value: float + verdict: OracleVerdict | None + + @property + def is_gating(self) -> bool: + """Whether crossing this threshold can change the gate verdict.""" + return self.verdict is not None + + def crosses(self, mean_fps: float) -> bool: + """Return whether ``mean_fps`` [FPS] falls below this threshold.""" + return mean_fps < self.value + + def to_dict(self) -> dict: + """Serialize to the ``launch_config.json`` / ``tasks.json`` entry shape.""" + return { + "threshold_verdict": self.verdict.value if self.verdict is not None else None, + "threshold_name": self.name, + "threshold": self.value, + } + + @classmethod + def from_dict(cls, raw: dict, *, context: str = "") -> "FpsMeanThreshold | None": + """Parse and validate one raw threshold entry. + + Returns ``None`` when the entry has no ``threshold`` value (skipped). + + Raises: + TypeError: If ``raw`` is not an object. + ValueError: If ``threshold_name`` is missing/empty, ``threshold`` is + non-numeric, or ``threshold_verdict`` is not a gating verdict. + """ + where = f" ({context})" if context else "" + if not isinstance(raw, dict): + raise TypeError(f"fps_mean_thresholds entry{where} must be an object, got {type(raw).__name__}") + + name = raw.get("threshold_name") + if not isinstance(name, str) or not name.strip(): + raise ValueError(f"fps_mean_thresholds entry{where} must define a non-empty 'threshold_name'") + name = name.strip() + + value = raw.get("threshold") + if value is None: + return None + try: + threshold = float(value) + except (TypeError, ValueError): + raise ValueError(f"fps_mean_thresholds entry {name!r} has non-numeric 'threshold': {value!r}") + + verdict_raw = raw.get("threshold_verdict") + if verdict_raw is None: + verdict: OracleVerdict | None = None + else: + allowed = sorted(v.value for v in THRESHOLD_VERDICTS) + try: + verdict = OracleVerdict(verdict_raw) + except ValueError: + raise ValueError( + f"fps_mean_thresholds entry {name!r} has invalid 'threshold_verdict' {verdict_raw!r}; " + f"must be one of {allowed} (or omitted for reporting-only)" + ) + if verdict not in THRESHOLD_VERDICTS: + raise ValueError( + f"fps_mean_thresholds entry {name!r} has non-gating 'threshold_verdict' {verdict_raw!r}; " + f"must be one of {allowed} (or omitted for reporting-only)" + ) + return cls(name=name, value=threshold, verdict=verdict) + + @classmethod + def from_list(cls, raw_list, *, context: str = "") -> "list[FpsMeanThreshold]": + """Parse a leaf list of raw threshold entries, dropping value-less entries.""" + if raw_list is None: + return [] + if not isinstance(raw_list, list): + raise TypeError(f"fps_mean_thresholds leaf ({context}) must be a list, got {type(raw_list).__name__}") + parsed: list[FpsMeanThreshold] = [] + for i, entry in enumerate(raw_list): + threshold = cls.from_dict(entry, context=f"{context}[{i}]" if context else f"[{i}]") + if threshold is not None: + parsed.append(threshold) + return parsed diff --git a/tools/perf_smoke_test/github_gate_context.py b/tools/perf_smoke_test/github_gate_context.py new file mode 100644 index 000000000000..2b4bf1c69bd1 --- /dev/null +++ b/tools/perf_smoke_test/github_gate_context.py @@ -0,0 +1,157 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Resolve GitHub event context for the performance smoke test""" + +from __future__ import annotations + +import json +import os +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +_BASELINE_PUBLISH_BRANCHES = frozenset({"main", "develop"}) +_BASELINE_PUBLISH_PREFIXES = ("release/",) + + +@dataclass(frozen=True) +class GateContext: + base_sha: str + target_branch: str + source_branch: str + allow_update: bool + trusted_source: str + event_kind: str + + def outputs(self) -> dict[str, str]: + return { + "base_sha": self.base_sha, + "target_branch": self.target_branch, + "source_branch": self.source_branch, + "allow_update": "true" if self.allow_update else "false", + "trusted_source": self.trusted_source, + "event_kind": self.event_kind, + } + + +def _strip_heads_ref(value: str) -> str: + prefix = "refs/heads/" + return value[len(prefix) :] if value.startswith(prefix) else value + + +def _truthy(value: str | None) -> bool: + return str(value or "").strip().lower() in {"1", "true", "yes"} + + +def _event_value(env: Mapping[str, str], key: str) -> str: + try: + return env[key] + except KeyError as exc: + raise RuntimeError(f"Missing required GitHub environment variable: {key}") from exc + + +def _load_event(event_path: str | None) -> dict[str, Any]: + if not event_path: + return {} + with Path(event_path).open(encoding="utf-8") as fh: + event = json.load(fh) + return event if isinstance(event, dict) else {} + + +def _baseline_publish_branch(ref: str) -> bool: + branch = _strip_heads_ref(ref) + return branch in _BASELINE_PUBLISH_BRANCHES or branch.startswith(_BASELINE_PUBLISH_PREFIXES) + + +def _pr_context(pr: Mapping[str, Any], *, allow_update: bool, trusted_source: str, event_kind: str) -> GateContext: + base = pr["base"] + head = pr["head"] + return GateContext( + base_sha=str(base["sha"]), + target_branch=str(base["ref"]), + source_branch=str(head["ref"]), + allow_update=allow_update, + trusted_source=trusted_source, + event_kind=event_kind, + ) + + +def resolve_gate_context( + env: Mapping[str, str] | None = None, + event: Mapping[str, Any] | None = None, +) -> GateContext: + env = env or os.environ + event = event if event is not None else _load_event(env.get("GITHUB_EVENT_PATH")) + + event_name = _event_value(env, "GITHUB_EVENT_NAME") + github_ref = _event_value(env, "GITHUB_REF") + ref_name = _event_value(env, "GITHUB_REF_NAME") + github_sha = _event_value(env, "GITHUB_SHA") + + if event_name == "pull_request": + return _pr_context( + event["pull_request"], + allow_update=False, + trusted_source="read_only", + event_kind="pull_request", + ) + + if event_name == "merge_group": + merge_group = event.get("merge_group") or {} + return GateContext( + base_sha=str(merge_group.get("base_sha") or github_sha), + target_branch=_strip_heads_ref(str(merge_group.get("base_ref") or ref_name)), + source_branch=_strip_heads_ref(str(merge_group.get("head_ref") or ref_name)), + allow_update=False, + trusted_source="read_only", + event_kind="merge_group", + ) + + if event_name == "push": + allow_update = _baseline_publish_branch(github_ref) + return GateContext( + base_sha=github_sha, + target_branch=ref_name, + source_branch=ref_name, + allow_update=allow_update, + trusted_source="protected_branch" if allow_update else "read_only", + event_kind="protected_push" if allow_update else "push", + ) + + return GateContext( + base_sha=github_sha, + target_branch=ref_name, + source_branch=ref_name, + allow_update=False, + trusted_source="read_only", + event_kind=event_name, + ) + + +def write_github_outputs(outputs: Mapping[str, str], output_path: str | None) -> None: + if not output_path: + return + with Path(output_path).open("a", encoding="utf-8") as fh: + for key, value in outputs.items(): + fh.write(f"{key}={value}\n") + + +def main() -> int: + context = resolve_gate_context() + outputs = context.outputs() + write_github_outputs(outputs, os.environ.get("GITHUB_OUTPUT")) + print( + "gate_context " + f"event={outputs['event_kind']} base={outputs['base_sha'][:12]} " + f"target={outputs['target_branch']} source={outputs['source_branch']} " + f"allow_update={outputs['allow_update']} trusted_source={outputs['trusted_source']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/perf_smoke_test/gpu_identity.py b/tools/perf_smoke_test/gpu_identity.py new file mode 100644 index 000000000000..08e8c51ca517 --- /dev/null +++ b/tools/perf_smoke_test/gpu_identity.py @@ -0,0 +1,102 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Canonical GPU identity helpers for baseline bucket selection""" + +from __future__ import annotations + +import re +import subprocess +from typing import Any + +_UNKNOWN_GPU = "unknown_gpu" + + +def _clean(value: Any) -> str: + return str(value or "").strip() + + +def _slug(value: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_") + return slug or _UNKNOWN_GPU + + +def canonical_gpu_model(value: Any) -> str: + """Return the canonical baseline bucket key for a raw GPU model string.""" + raw = _clean(value) + if not raw: + return _UNKNOWN_GPU + normalized = re.sub(r"\s+", " ", raw.lower()).strip() + compact = re.sub(r"[^a-z0-9]+", "", normalized) + + if "l40s" in compact: + return "l40s" + if compact.endswith("l40") or compact == "nvidial40" or "teslal40" in compact: + return "l40" + if "rtxpro6000" in compact and "blackwell" in compact: + return "rtx_pro_6000_blackwell" + if "rtxpro6000" in compact: + return "rtx_pro_6000" + if "rtx6000adageneration" in compact or ("rtx6000" in compact and "ada" in compact): + return "rtx_6000_ada" + if compact in {"rtx6000", "nvidiartx6000"} or "rtx6000" in compact: + return "rtx_6000" + if "rtxa6000" in compact or "a6000" in compact: + return "rtx_a6000" + if "geforcertx5090" in compact: + return "geforce_rtx_5090" + if "geforcertx4090" in compact: + return "geforce_rtx_4090" + return _slug(raw) + + +def gpu_model_config_keys(value: Any) -> list[str]: + """Return candidate keys for reading GPU-keyed config dictionaries. + + Config (``tasks.json`` thresholds) and baseline buckets are keyed by the + canonical slug (e.g. ``l40s``); the raw string is kept as a defensive + fallback for a config hand-keyed to the exact detected name. + """ + raw = _clean(value) + canonical = canonical_gpu_model(raw) + keys: list[str] = [] + for key in (canonical, raw): + if key and key not in keys: + keys.append(key) + return keys + + +def normalize_gpu_fields(value: Any) -> dict[str, str]: + raw = _clean(value) + return { + "gpu_model": canonical_gpu_model(raw), + "gpu_model_raw": raw or _UNKNOWN_GPU, + } + + +def detect_gpu_model(explicit: str = "") -> str: + """Return the raw GPU model label, preferring ``explicit`` else nvidia-smi. + + Falls back to ``"unknown-gpu"`` when nvidia-smi is unavailable or reports + nothing. Shared by the local runner and the baseline seeder. + """ + explicit = _clean(explicit) + if explicit: + return explicit + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + for line in result.stdout.splitlines(): + name = line.strip() + if name: + return name + except Exception: + pass + return "unknown-gpu" diff --git a/tools/perf_smoke_test/hashing.py b/tools/perf_smoke_test/hashing.py new file mode 100644 index 000000000000..0c9349febd27 --- /dev/null +++ b/tools/perf_smoke_test/hashing.py @@ -0,0 +1,29 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared canonical-JSON hashing used by every perf-smoke contract. + +All contract hashes (``runtime_contract_hash``, ``launch_config_hash``, +``benchmark_contract_hash``, ``era_key``) must agree byte-for-byte across the +producer and consumer sides of the gate, so the hashing primitive lives in one +place. Changing :func:`canonical_json` or :func:`stable_hash` shifts every +stored hash and invalidates the baseline branch, so treat this module as frozen. +""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + + +def canonical_json(data: dict[str, Any]) -> str: + """Return a deterministic JSON encoding with sorted keys and no whitespace.""" + return json.dumps(data, sort_keys=True, separators=(",", ":")) + + +def stable_hash(data: dict[str, Any]) -> str: + """Return the first 16 hex chars of the SHA-256 over the canonical JSON.""" + return hashlib.sha256(canonical_json(data).encode("utf-8")).hexdigest()[:16] diff --git a/tools/perf_smoke_test/image_era.py b/tools/perf_smoke_test/image_era.py new file mode 100644 index 000000000000..7ac2db48a745 --- /dev/null +++ b/tools/perf_smoke_test/image_era.py @@ -0,0 +1,349 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Source-derivable image-era key and manifest resolution for the perf smoke test. + +The gate pulls a prebuilt CI image, but older commits may expect a different +dependency era than the moving ``latest-perf`` tag provides. The era key is a +hash computed *statically* from a commit's container-defining source (today: +``docker/.env.base``), so the gate can pick the right immutable image +(``sha-``) *before* running anything -- unlike ``runtime_contract_hash``, +which is only known after a benchmark runs inside the container. + +This module is the single authority for that key: both the publish side (which +records ``era_key -> image`` into the manifest) and the gate/seed/bisect side +(which reads it) MUST compute the key here so their values agree byte-for-byte. +A divergence would silently miss every lookup and fall back forever. + +The era key deliberately captures only the *base/container* layer, not +IsaacLab's own source dependencies: the PR code is bind-mounted over the +container, so those deps are what we are testing, not part of the image era. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +try: + from .hashing import stable_hash +except ImportError: # pragma: no cover - supports direct script imports + from hashing import stable_hash + +# Bump when the selected field set changes; old keys stay valid under their version. +IMAGE_ERA_VERSION = 1 + +# Container-defining source file, relative to the repo root. +ENV_BASE_RELPATH = "docker/.env.base" + +# v1 field set: the Isaac Sim base image + version are the dominant era drivers +# and the only inputs that change the base layer the image is built FROM. +ERA_ENV_FIELDS: tuple[str, ...] = ("ISAACSIM_BASE_IMAGE", "ISAACSIM_VERSION") + +MANIFEST_SCHEMA_VERSION = 1 +DEFAULT_FALLBACK_IMAGE = "nvcr.io/nvidian/isaac-lab:latest-perf" + +# The manifest is machine-managed mutable state, so it lives on the same +# ``perf-baselines`` branch as the rolling baselines (not in the source tree): +# the gate already fetches this branch, and CI already has ``contents: write`` +# for it. Keeping era -> image mappings beside the baselines they pin means one +# branch carries all of the gate's out-of-tree state. +MANIFEST_BRANCH = "perf-baselines" +MANIFEST_RELPATH = "image_era_manifest.json" + + +def parse_env_file(text: str) -> dict[str, str]: + """Parse a ``KEY=VALUE`` env file, ignoring comments and surrounding quotes.""" + env: dict[str, str] = {} + for raw_line in text.splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + key = key.strip() + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}: + value = value[1:-1] + if key: + env[key] = value + return env + + +def read_env_base_from_tree(source_root: str | Path) -> str: + """Return ``docker/.env.base`` contents from a checked-out working tree.""" + path = Path(source_root) / ENV_BASE_RELPATH + if not path.exists(): + raise FileNotFoundError(f"{ENV_BASE_RELPATH} not found under {source_root}") + return path.read_text(encoding="utf-8") + + +def read_env_base_from_commit(commit: str, repo_root: str | Path = ".") -> str: + """Return ``docker/.env.base`` contents at a git commit via ``git show``. + + Lets the gate compute an older commit's era key without checking it out. + """ + try: + return subprocess.check_output( + ["git", "show", f"{commit}:{ENV_BASE_RELPATH}"], + cwd=str(repo_root), + text=True, + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as exc: # pragma: no cover - exercised via integration + raise FileNotFoundError(f"{ENV_BASE_RELPATH} not found at commit {commit}: {exc.stderr.strip()}") from exc + + +def build_era_contract(env: Mapping[str, str]) -> dict[str, Any]: + """Return the versioned ``{version, fields}`` contract that the key hashes.""" + fields = {field: env.get(field) for field in ERA_ENV_FIELDS} + return {"image_era_version": IMAGE_ERA_VERSION, "fields": fields} + + +def compute_era_key(env: Mapping[str, str]) -> str: + """Return the era key for a parsed ``.env.base`` mapping.""" + return stable_hash(build_era_contract(env)) + + +def era_key_from_tree(source_root: str | Path) -> str: + """Convenience: compute the era key from a working tree.""" + return compute_era_key(parse_env_file(read_env_base_from_tree(source_root))) + + +def era_key_from_commit(commit: str, repo_root: str | Path = ".") -> str: + """Convenience: compute the era key for a git commit.""" + return compute_era_key(parse_env_file(read_env_base_from_commit(commit, repo_root))) + + +def resolve_image( + era_key: str, + manifest: Mapping[str, Any] | None, + *, + fallback_image: str | None = None, +) -> tuple[str, bool]: + """Resolve an image reference for an era key. + + Returns ``(image_ref, matched)``. ``matched`` is ``True`` when the manifest + has an entry for ``era_key``; otherwise the fallback is returned so the gate + degrades to ``latest-perf`` instead of failing. + """ + eras = (manifest or {}).get("eras") or {} + entry = eras.get(era_key) + if isinstance(entry, Mapping) and entry.get("image"): + return str(entry["image"]), True + fallback = fallback_image or (manifest or {}).get("fallback_image") or DEFAULT_FALLBACK_IMAGE + return str(fallback), False + + +def empty_manifest() -> dict[str, Any]: + """Return a fresh, empty manifest with the current schema and default fallback.""" + return {"schema_version": MANIFEST_SCHEMA_VERSION, "fallback_image": DEFAULT_FALLBACK_IMAGE, "eras": {}} + + +def load_manifest(path: str | Path) -> dict[str, Any]: + """Load an image-era manifest from disk; return an empty manifest if absent.""" + manifest_path = Path(path) + if not manifest_path.exists(): + return empty_manifest() + with manifest_path.open(encoding="utf-8") as fh: + return json.load(fh) + + +def manifest_upsert( + manifest: Mapping[str, Any] | None, + era_key: str, + image: str, + *, + extra: Mapping[str, Any] | None = None, +) -> tuple[dict[str, Any], bool]: + """Return ``(new_manifest, changed)`` with ``era_key`` mapped to ``image``. + + ``changed`` is ``False`` when the era already resolves to ``image`` so callers + can skip an empty commit/push (the mapping is the source of truth; identical + remaps are no-ops even if ``extra`` metadata differs). + + Args: + manifest: Existing manifest to update, or ``None`` to start from empty. + era_key: The era key to map. + image: The immutable image reference to record for the era. + extra: Optional metadata to store alongside the image (e.g. the Isaac Sim + version or publishing commit); ``None`` values are dropped. + """ + result = json.loads(json.dumps(dict(manifest))) if manifest else empty_manifest() + result.setdefault("schema_version", MANIFEST_SCHEMA_VERSION) + result.setdefault("fallback_image", DEFAULT_FALLBACK_IMAGE) + eras = result.setdefault("eras", {}) + + prior = eras.get(era_key) + if isinstance(prior, Mapping) and prior.get("image") == image: + return result, False + + entry: dict[str, Any] = {"image": image} + if extra: + entry.update({key: value for key, value in extra.items() if value is not None}) + eras[era_key] = entry + return result, True + + +def load_manifest_from_git( + *, + branch: str = MANIFEST_BRANCH, + remote: str | None = "origin", + repo_dir: str | Path | None = None, +) -> dict[str, Any]: + """Load the manifest from ``branch`` in git; empty manifest if branch/file absent. + + A missing branch or file degrades to :func:`empty_manifest` so a first run + resolves to the fallback image instead of failing. + """ + # Lazy import: keeps the pure key/resolve path free of the baseline git deps. + from baseline_manager import _git_show_file, refresh_baseline_branch + + repo_path = Path(repo_dir) if repo_dir is not None else None + sha = refresh_baseline_branch(branch, remote=remote, repo_dir=repo_path, allow_missing=True) + if not sha: + return empty_manifest() + content = _git_show_file(sha, MANIFEST_RELPATH, repo_dir=repo_path) + if not content: + return empty_manifest() + try: + loaded = json.loads(content) + except json.JSONDecodeError: + return empty_manifest() + return loaded if isinstance(loaded, dict) else empty_manifest() + + +def record_era_image( + era_key: str, + image: str, + *, + extra: Mapping[str, Any] | None = None, + branch: str = MANIFEST_BRANCH, + remote: str | None = "origin", + repo_dir: str | Path | None = None, + max_retries: int = 3, +) -> bool: + """Record ``era_key -> image`` in the manifest on ``branch`` and push. + + Returns ``True`` when a new/updated entry was pushed, ``False`` when the era + already resolved to ``image`` (idempotent no-op). Mirrors the baseline + manager's refetch-reapply-retry transaction so concurrent publishes cannot + clobber each other's entries. + """ + # Lazy import: reuse the baseline manager's git transaction machinery without + # coupling the pure key/resolve path (its only importer today) to it. + from baseline_manager import ( + _baseline_update_worktree, + _commit_env, + _git, + _git_error, + refresh_baseline_branch, + ) + + if max_retries < 1: + raise ValueError("max_retries must be >= 1") + + repo_path = Path(repo_dir) if repo_dir is not None else None + last_error = "unknown push failure" + for attempt in range(1, max_retries + 1): + base_sha = refresh_baseline_branch(branch, remote=remote, repo_dir=repo_path, allow_missing=True) + with _baseline_update_worktree(base_sha, repo_dir=repo_path) as worktree: + manifest_path = worktree / MANIFEST_RELPATH + updated, changed = manifest_upsert(load_manifest(manifest_path), era_key, image, extra=extra) + if not changed: + return False + manifest_path.write_text(json.dumps(updated, indent=2, sort_keys=True) + "\n", encoding="utf-8") + _git(["add", MANIFEST_RELPATH], cwd=worktree, check=True) + _git( + ["commit", "-m", f"[image_era] Record {era_key[:12]} -> {image}"], + cwd=worktree, + check=True, + env=_commit_env(), + ) + push_ref = f"HEAD:refs/heads/{branch}" + if remote: + push = _git(["push", remote, push_ref], cwd=worktree) + else: + push = _git(["branch", "--force", branch, "HEAD"], cwd=worktree) + if push.returncode == 0: + return True + last_error = _git_error(push) + print(f"[image_era] manifest push attempt {attempt}/{max_retries} failed; refetching and retrying") + + raise RuntimeError(f"Failed to push era manifest to {branch!r} after {max_retries} attempts: {last_error}") + + +def _parse_extra(pairs: list[str]) -> dict[str, str]: + extra: dict[str, str] = {} + for pair in pairs or []: + key, sep, value = pair.partition("=") + if not sep: + raise SystemExit(f"--extra expects KEY=VALUE, got {pair!r}") + extra[key.strip()] = value.strip() + return extra + + +def _main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Compute the perf-smoke image-era key / resolve or record an image.") + source = parser.add_mutually_exclusive_group() + source.add_argument("--source_root", help="Working tree root to read docker/.env.base from.") + source.add_argument("--commit", help="Git commit to read docker/.env.base from (via git show).") + parser.add_argument("--repo_root", default=".", help="Repo root for --commit lookups and git manifest ops.") + resolve_group = parser.add_mutually_exclusive_group() + resolve_group.add_argument("--manifest", help="Resolve the key against this on-disk manifest file.") + resolve_group.add_argument( + "--manifest_from_git", + action="store_true", + help="Resolve against the manifest on the era branch (fetched from --remote).", + ) + parser.add_argument("--fallback_image", help="Fallback image when the era is not in the manifest.") + parser.add_argument("--branch", default=MANIFEST_BRANCH, help="Branch holding the era manifest.") + parser.add_argument("--remote", default="origin", help="Git remote for --manifest_from_git / --record_image.") + parser.add_argument("--record_image", help="Record era_key -> this image into the manifest on --branch and push.") + parser.add_argument( + "--extra", + action="append", + default=[], + metavar="KEY=VALUE", + help="Extra metadata stored on the recorded era entry (repeatable).", + ) + args = parser.parse_args(argv) + + if args.commit: + era_key = era_key_from_commit(args.commit, args.repo_root) + else: + era_key = era_key_from_tree(args.source_root or args.repo_root) + + result: dict[str, Any] = {"era_key": era_key, "image_era_version": IMAGE_ERA_VERSION} + if args.record_image: + recorded = record_era_image( + era_key, + args.record_image, + extra=_parse_extra(args.extra), + branch=args.branch, + remote=args.remote, + repo_dir=args.repo_root, + ) + result["recorded_image"] = args.record_image + result["recorded"] = recorded + elif args.manifest_from_git: + manifest = load_manifest_from_git(branch=args.branch, remote=args.remote, repo_dir=args.repo_root) + image, matched = resolve_image(era_key, manifest, fallback_image=args.fallback_image) + result["image"] = image + result["matched"] = matched + elif args.manifest: + image, matched = resolve_image(era_key, load_manifest(args.manifest), fallback_image=args.fallback_image) + result["image"] = image + result["matched"] = matched + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(_main()) diff --git a/tools/perf_smoke_test/launch_config.py b/tools/perf_smoke_test/launch_config.py new file mode 100644 index 000000000000..31d9ac394593 --- /dev/null +++ b/tools/perf_smoke_test/launch_config.py @@ -0,0 +1,179 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Launch configuration artifact helpers for the performance smoke test + +``launch_config.json`` is the durable contract between the matrix builder, the +benchmark runner, post-processing, and aggregate. Phase 2 treats this artifact +as the run intent instead of re-reading ``tasks.json``, so it can catch workflow +command bugs and self-hosted-runner handoff issues. +""" + +import json +from pathlib import Path +from typing import Any + +try: + from .backend_identity import make_backend_key, normalize_render_backend + from .gate_types import FpsMeanThreshold + from .gpu_identity import normalize_gpu_fields + from .hashing import stable_hash + from .task_config import TaskConfig +except ImportError: # pragma: no cover (for direct scripting execution/import) + from backend_identity import make_backend_key, normalize_render_backend + from gate_types import FpsMeanThreshold + from gpu_identity import normalize_gpu_fields + from hashing import stable_hash + from task_config import TaskConfig + +LAUNCH_CONFIG_SCHEMA_VERSION = 1 +BENCHMARK_CONTRACT_VERSION = 1 +DEFAULT_BASELINE_EPOCH = 1 +LAUNCH_CONFIG_FILENAME = "launch_config.json" +_HYDRA_RENDER_PRESETS = { + "rtx_renderer": "isaacsim_rtx", +} + + +def workload_contract(config: dict[str, Any]) -> dict[str, Any]: + """Return the workload-defining subset used for launch_config_hash.""" + keys = ( + "task_id", + "backend_key", + "physics_backend", + "render_backend", + "preset", + "num_envs", + "num_frames", + "seed", + "warmup_frames", + "camera_resolution", + "benchmark_formatter", + "hydra_args", + ) + return {key: config.get(key) for key in keys} + + +def hydra_args_for_task(task: TaskConfig) -> list[str]: + """Return Hydra args used by the current local/CI benchmark launch path.""" + presets: list[str] = [] + if task.physics_backend == "newton": + presets.append("newton_mjwarp") + if task.render_backend: + presets.append(_HYDRA_RENDER_PRESETS.get(task.render_backend, task.render_backend)) + if task.render_backend == "newton_renderer": + presets.append("rgb") + return [f"presets={','.join(presets)}"] if presets else [] + + +def task_to_launch_config( + task: TaskConfig, + *, + fps_mean_thresholds: list[FpsMeanThreshold], + gpu_model: str | None = None, + hydra_args: list[str] | None = None, + benchmark_formatter: str = "schema", +) -> dict[str, Any]: + """Build a serializable launch config for one task/backend job""" + gpu_fields = normalize_gpu_fields(gpu_model) + config: dict[str, Any] = { + "schema_version": LAUNCH_CONFIG_SCHEMA_VERSION, + "task_id": task.task_id, + "backend_key": make_backend_key(task.physics_backend, task.render_backend), + "physics_backend": task.physics_backend, + "render_backend": normalize_render_backend(task.render_backend), + "preset": task.preset, + "num_envs": task.num_envs, + "num_frames": task.num_frames, + "seed": task.seed, + "warmup_frames": task.warmup_frames, + "camera_resolution": list(task.camera_resolution) if task.camera_resolution else None, + "timeout_minutes": task.timeout_minutes, + "tags": list(task.tags), + "gpu_model": gpu_fields["gpu_model"], + "gpu_model_raw": gpu_fields["gpu_model_raw"], + "benchmark_formatter": benchmark_formatter, + "hydra_args": list(hydra_args or []), + "fps_mean_thresholds": [t.to_dict() for t in fps_mean_thresholds], + "baseline_epoch": int(getattr(task, "baseline_epoch", DEFAULT_BASELINE_EPOCH)), + "benchmark_contract_version": BENCHMARK_CONTRACT_VERSION, + } + config["launch_config_hash"] = stable_hash(workload_contract(config)) + config["benchmark_contract_hash"] = stable_hash( + { + "benchmark_contract_version": config["benchmark_contract_version"], + "warmup_frames": config["warmup_frames"], + "benchmark_formatter": config["benchmark_formatter"], + } + ) + return config + + +def fallback_launch_config( + *, + task_id: str, + physics_backend: str, + render_backend: str | None, + backend_key: str, + timeout_s: float, + task: TaskConfig | None = None, +) -> dict[str, Any]: + """Build a launch config for legacy/manual Phase 2 calls without an artifact""" + normalized_backend_key = make_backend_key(physics_backend, render_backend) + gpu_fields = normalize_gpu_fields(None) + + if task is None: + config: dict[str, Any] = { + "schema_version": LAUNCH_CONFIG_SCHEMA_VERSION, + "task_id": task_id, + "backend_key": normalized_backend_key, + "physics_backend": physics_backend, + "render_backend": normalize_render_backend(render_backend), + "preset": "default", + "num_envs": 0, + "num_frames": 0, + "seed": None, + "warmup_frames": 0, + "camera_resolution": None, + "timeout_minutes": int(timeout_s / 60), + "tags": ["always"], + "gpu_model": gpu_fields["gpu_model"], + "gpu_model_raw": gpu_fields["gpu_model_raw"], + "benchmark_formatter": "schema", + "hydra_args": [], + "fps_mean_thresholds": [], + "baseline_epoch": DEFAULT_BASELINE_EPOCH, + "benchmark_contract_version": BENCHMARK_CONTRACT_VERSION, + } + config["launch_config_hash"] = stable_hash(workload_contract(config)) + config["benchmark_contract_hash"] = stable_hash( + { + "benchmark_contract_version": config["benchmark_contract_version"], + "warmup_frames": config["warmup_frames"], + "benchmark_formatter": config["benchmark_formatter"], + } + ) + return config + + return task_to_launch_config( + task, + fps_mean_thresholds=task.thresholds_for("L40S"), + hydra_args=[], + ) + + +def load_launch_config(artifact_dir: Path, explicit_path: Path | None = None) -> dict[str, Any] | None: + path = explicit_path or artifact_dir / LAUNCH_CONFIG_FILENAME + if not path.exists(): + return None + with path.open() as fh: + return json.load(fh) + + +def write_launch_config(artifact_dir: Path, config: dict[str, Any]) -> Path: + artifact_dir.mkdir(parents=True, exist_ok=True) + path = artifact_dir / LAUNCH_CONFIG_FILENAME + path.write_text(json.dumps(config, indent=2, sort_keys=True)) + return path diff --git a/tools/perf_smoke_test/omni_github.py b/tools/perf_smoke_test/omni_github.py new file mode 100644 index 000000000000..94bf08f7bd2a --- /dev/null +++ b/tools/perf_smoke_test/omni_github.py @@ -0,0 +1,147 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Emit the omni-github test-result artifact for the perf-smoke gate. + +Writes the artifact directory omni-github ingests (a manifest plus a result JSON) +directly from the aggregate's scored rows, so per-task perf diagnostics -- FPS, +regression, hardware, software, and contract hashes -- are indexed as +``custom.perf_smoke.*`` fields instead of being flattened into a lossy pass/fail +message. This is the perf gate's analogue of the shared JUnit upload action; it is +kept separate because that action's JUnit contract cannot carry these structured +custom fields. + +See ``.github/actions/upload-omni-github-test-results/result-json.schema.json`` for +the schema the emitted result JSON validates against. +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import Any + +try: + from .gate_types import OracleVerdict +except ImportError: # executed as a script, not a package + from gate_types import OracleVerdict + +MANIFEST_NAME = "omni-github-test-results-upload.json" +RESULT_REL_PATH = "_testoutput/test_results.json" +RESULT_SCHEMA_VERSION = 1 +MANIFEST_SCHEMA_VERSION = 1 +CUSTOM_NAMESPACE = "perf_smoke" +TEST_TOOL_ID = "perf-smoke" + +_PASSING_VERDICTS = (OracleVerdict.PASS, OracleVerdict.WARN) + + +def _number(value: Any) -> float | int | None: + """Return a finite number for storage, or ``None`` so the field is dropped. + + Booleans are rejected here because they are handled as booleans elsewhere. + """ + if value is None or isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return value if math.isfinite(value) else None + return None + + +def _drop_none(fields: dict[str, Any]) -> dict[str, Any]: + """Drop keys whose value is ``None`` (omni-github stores omitted, not null).""" + return {key: value for key, value in fields.items() if value is not None} + + +def _launch_value(bench_result: Any, key: str) -> Any: + """Read a run-shape value, preferring the requested launch_config over the ran info.""" + launch_config = getattr(bench_result, "launch_config", None) or {} + if launch_config.get(key) is not None: + return launch_config.get(key) + benchmark_info = getattr(bench_result, "benchmark_info", None) or {} + return benchmark_info.get(key) + + +def _custom_fields(result: Any, bench_result: Any) -> dict[str, Any]: + """Project the oracle verdict and bench result into ``custom.perf_smoke`` fields. + + Intentionally a compact "dashboard core": identity, run shape, verdict, timing, + the headline FPS/regression, and GPU/host memory. Deeper diagnostics (software + versions, contract hashes, raw fps spread, threshold provenance) stay in the + uploaded bench artifacts and can be promoted here later if a dashboard needs them. + """ + runtime_resources = getattr(bench_result, "runtime_resources", None) or {} + fps_mean = result.measured_fps if result.measured_fps is not None else getattr(bench_result, "raw_fps_mean", None) + return _drop_none( + { + "task_id": result.task_id, + "physics_backend": getattr(bench_result, "physics_backend", None), + "render_backend": getattr(bench_result, "render_backend", None), + "num_envs": _number(_launch_value(bench_result, "num_envs")), + "num_frames": _number(_launch_value(bench_result, "num_frames")), + "verdict": result.verdict.value, + "failure_phase": result.failure_phase, + "wall_time_s": _number(getattr(bench_result, "wall_time_s", None)), + "startup_time_s": _number(getattr(bench_result, "startup_time_s", None)), + "fps_mean": _number(fps_mean), + "regression_pct": _number(result.regression_pct), + "vram_mb": _number(runtime_resources.get("gpu_mem_used_mb")), + "sysram_mb": _number(runtime_resources.get("system_ram_used_mb")), + } + ) + + +def _message(result: Any) -> str: + parts = [result.verdict.value] + if result.measured_fps is not None: + parts.append(f"fps={result.measured_fps:.1f}") + if result.regression_pct is not None: + parts.append(f"regression={result.regression_pct:.2f}%") + if result.failure_phase: + parts.append(f"phase={result.failure_phase}") + return " ".join(parts) + + +def _row(result: Any, bench_result: Any) -> dict[str, Any]: + duration = max(float(getattr(bench_result, "wall_time_s", None) or 0.0), 0.0) + passed = result.verdict in _PASSING_VERDICTS + row: dict[str, Any] = { + "test_id": f"perf-smoke.{result.task_id}::{result.backend}", + "test_name": result.backend, + "test_type": "performance", + "passed": passed, + "duration": duration, + "custom": {CUSTOM_NAMESPACE: _custom_fields(result, bench_result)}, + } + if not passed: + row["message"] = _message(result) + return row + + +def build_result(rows, *, platform: str, app_config: str, test_tool_id: str = TEST_TOOL_ID) -> dict[str, Any]: + """Build the omni-github result payload from ``(oracle_result, bench_result)`` rows.""" + return { + "result_schema_version": RESULT_SCHEMA_VERSION, + "test_tool_id": test_tool_id, + "app": {"platform": platform, "config": app_config}, + "tests": [_row(result, bench_result) for result, bench_result in rows], + } + + +def write_artifact(rows, output_dir, *, platform: str, app_config: str, test_tool_id: str = TEST_TOOL_ID) -> Path: + """Write the manifest and result JSON omni-github ingests; returns the artifact root.""" + output_dir = Path(output_dir) + result_path = output_dir / RESULT_REL_PATH + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text( + json.dumps(build_result(rows, platform=platform, app_config=app_config, test_tool_id=test_tool_id)), + encoding="utf-8", + ) + manifest = {"schema_version": MANIFEST_SCHEMA_VERSION, "result_paths": [RESULT_REL_PATH]} + (output_dir / MANIFEST_NAME).write_text(json.dumps(manifest), encoding="utf-8") + return output_dir diff --git a/tools/perf_smoke_test/oracle.py b/tools/perf_smoke_test/oracle.py new file mode 100644 index 000000000000..8145b5abca3b --- /dev/null +++ b/tools/perf_smoke_test/oracle.py @@ -0,0 +1,289 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Oracle layer for the CI performance smoke test""" + +from dataclasses import dataclass, field + +try: + from .contracts import BenchResult + from .gate_config import DEFAULT_K_BLOCK, DEFAULT_K_WARN, MIN_BASELINE_SAMPLES, MIN_BLOCK_REGRESSION_PCT + from .gate_types import ( + BisectVerdict, + FailurePhase, + FpsMeanThreshold, + OracleVerdict, + ThresholdSource, + verdict_severity, + worst_verdict, + ) +except ImportError: # pragma: no cover (for direct scripting execution/import) + from contracts import BenchResult + from gate_config import DEFAULT_K_BLOCK, DEFAULT_K_WARN, MIN_BASELINE_SAMPLES, MIN_BLOCK_REGRESSION_PCT + from gate_types import ( + BisectVerdict, + FailurePhase, + FpsMeanThreshold, + OracleVerdict, + ThresholdSource, + verdict_severity, + worst_verdict, + ) + + +@dataclass +class Baseline: + """Rolling-window statistics for one compatible benchmark history""" + + median_fps: float + mad_fps: float + k_warn: float = DEFAULT_K_WARN + k_block: float = DEFAULT_K_BLOCK + sample_count: int = 0 + source: str = "unknown" + total_sample_count: int | None = None + + +@dataclass +class OracleResult: + """Full verdict record produced by :func:`compare`""" + + verdict: OracleVerdict + bisect_verdict: str + failure_phase: str | None + measured_fps: float | None + baseline_fps: float | None + regression_pct: float | None + gpu_mem_used_mb: float | None + startup_time_s: float | None + wall_time_s: float | None + was_retried: bool + task_id: str + backend: str + baseline_sample_count: int = 0 + baseline_source: str = "none" + threshold_source: str = ThresholdSource.NO_BASELINE.value + warn_threshold_fps: float | None = None + block_threshold_fps: float | None = None + hard_floor_fps: float | None = None + noise_floor_pct: float | None = None + min_block_regression_pct: float = MIN_BLOCK_REGRESSION_PCT + note: str | None = None + crossed_thresholds: list[dict] = field(default_factory=list) + + +_BISECT_BAD_PHASES: frozenset[str] = frozenset({FailurePhase.INIT.value, FailurePhase.RUNTIME.value}) + + +def _bisect_verdict(verdict: OracleVerdict, was_retried: bool, failure_phase: str | None) -> str: + """Compute the bisect-friendly label for a given verdict.""" + if verdict == OracleVerdict.PASS: + return BisectVerdict.SKIP.value if was_retried else BisectVerdict.GOOD.value + if verdict == OracleVerdict.WARN: + return BisectVerdict.SKIP.value + if verdict == OracleVerdict.BLOCK: + return BisectVerdict.BAD.value + if failure_phase in _BISECT_BAD_PHASES: + return BisectVerdict.BAD.value + return BisectVerdict.SKIP.value + + +def _hard_failure( + bench_result: BenchResult, + failure_phase: str | None, + was_retried: bool, + gpu_mem_used_mb: float | None, + *, + note: str | None = None, +) -> OracleResult: + verdict = OracleVerdict.HARD_FAILURE + return OracleResult( + verdict=verdict, + bisect_verdict=_bisect_verdict(verdict, was_retried, failure_phase), + failure_phase=failure_phase, + measured_fps=None, + baseline_fps=None, + regression_pct=None, + gpu_mem_used_mb=gpu_mem_used_mb, + startup_time_s=bench_result.startup_time_s, + wall_time_s=bench_result.wall_time_s, + was_retried=was_retried, + task_id=bench_result.task_id, + backend=bench_result.backend_key or bench_result.backend, + threshold_source=ThresholdSource.NOT_APPLICABLE.value, + note=note, + ) + + +def compare( + bench_result: BenchResult, + baseline: "Baseline | None", + fps_mean_thresholds: "list[FpsMeanThreshold]", + *, + min_block_regression_pct: float = MIN_BLOCK_REGRESSION_PCT, + noise_floor_pct: float = 0.0, +) -> OracleResult: + """Compare a benchmark result against its baseline and return an OracleResult. + + The steady-state mean FPS is taken from ``bench_result.raw_fps_mean``, which + :mod:`build_bench_result` derives from the runtime bundle's ``total_fps.mean`` + (warmup already excluded at the source by ``perf_runtime.py``); the oracle no + longer re-parses the benchmark output. + + The final verdict is the most severe of the rolling-window/baseline verdict and + the verdict from any crossed gating threshold (see :class:`~gate_types.FpsMeanThreshold`). + Reporting-only thresholds that cross are recorded in ``crossed_thresholds`` without + changing the verdict. + """ + task_id: str = bench_result.task_id + backend: str = bench_result.backend_key or bench_result.backend + failure_phase: str | None = bench_result.failure_phase + was_retried: bool = bool(bench_result.was_retried) + startup_time_s: float | None = bench_result.startup_time_s + wall_time_s: float | None = bench_result.wall_time_s + gpu_mem_used_mb: float | None = (bench_result.runtime_resources or {}).get("gpu_mem_used_mb") + + config_mismatch = bench_result.config_mismatch + if config_mismatch or failure_phase == FailurePhase.CONFIG_MISMATCH.value: + return _hard_failure( + bench_result, + FailurePhase.CONFIG_MISMATCH.value, + was_retried, + gpu_mem_used_mb, + note=str(config_mismatch or "config_mismatch"), + ) + + # Execution health is checked before any measurement is considered. A run can + # write a complete bundle and then die -- crashing on teardown, being OOM-killed, + # or running so close to its timeout that it is classified as a hang. Judging + # that bundle on FPS alone would let a crash score PASS and be appended to the + # baseline as if it were a clean sample. This mirrors the per-task commit status, + # which already treats a nonzero exit or any failure phase as unhealthy. + if bench_result.exit_code != 0 or failure_phase: + return _hard_failure( + bench_result, + failure_phase, + was_retried, + gpu_mem_used_mb, + note=f"unhealthy_run(exit={bench_result.exit_code},phase={failure_phase or 'none'})", + ) + + if not bench_result.perf_smoke_test_info_present: + return _hard_failure(bench_result, failure_phase, was_retried, gpu_mem_used_mb) + + mean_fps = bench_result.raw_fps_mean + if not isinstance(mean_fps, (int, float)) or isinstance(mean_fps, bool): + return _hard_failure(bench_result, failure_phase, was_retried, gpu_mem_used_mb, note="missing_fps_mean") + if mean_fps <= 0: + # A run reporting no forward progress is a dead run, not a regression; + # fail it regardless of baseline availability. + return _hard_failure(bench_result, failure_phase, was_retried, gpu_mem_used_mb, note="zero_fps") + + baseline_fps = baseline.median_fps if baseline is not None else None + baseline_sample_count = baseline.sample_count if baseline is not None else 0 + baseline_source = baseline.source if baseline is not None else "none" + regression_pct = None + if baseline_fps: + regression_pct = ((mean_fps - baseline_fps) / baseline_fps) * 100.0 + + # --- Configured FPS thresholds (hard floors and named reference points) --- + crossed = [t for t in fps_mean_thresholds if t.crosses(mean_fps)] + crossed_thresholds = [ + { + "threshold_name": t.name, + "threshold": t.value, + "threshold_verdict": t.verdict.value if t.verdict is not None else None, + "gating": t.is_gating, + } + for t in crossed + ] + threshold_verdict = OracleVerdict.PASS + for t in crossed: + if t.is_gating: + threshold_verdict = worst_verdict(threshold_verdict, t.verdict) + block_values = [t.value for t in fps_mean_thresholds if t.verdict == OracleVerdict.BLOCK] + hard_floor_fps = max(block_values) if block_values else None + + # --- Rolling-window / baseline verdict --- + warn_threshold = None + block_threshold = None + baseline_threshold_source = ThresholdSource.NO_BASELINE.value + notes: list[str] = [] + + # Noise floor: when a baseline is unusually tight (tiny MAD), floor the effective + # noise at ``noise_floor_pct`` % of the baseline median so the rolling-window + # thresholds don't become hypersensitive and over-flag sub-noise dips. + effective_noise_fps: float | None = None + applied_noise_floor_pct: float | None = None + if baseline is not None and baseline.median_fps > 0.0: + noise_floor_fps = max(0.0, float(noise_floor_pct)) / 100.0 * baseline.median_fps + effective_noise_fps = max(baseline.mad_fps, noise_floor_fps) + if noise_floor_fps > baseline.mad_fps: + applied_noise_floor_pct = float(noise_floor_pct) + + if baseline is None: + baseline_verdict = OracleVerdict.WARN + notes.append("no_baseline") + elif baseline.sample_count < MIN_BASELINE_SAMPLES: + baseline_verdict = OracleVerdict.WARN + baseline_threshold_source = ThresholdSource.INSUFFICIENT_WINDOW.value + notes.append(f"insufficient_baseline(n={baseline.sample_count},min={MIN_BASELINE_SAMPLES})") + else: + baseline_threshold_source = ThresholdSource.ROLLING_WINDOW.value + threshold_noise = effective_noise_fps if effective_noise_fps is not None else baseline.mad_fps + block_threshold = baseline.median_fps - baseline.k_block * threshold_noise + warn_threshold = baseline.median_fps - baseline.k_warn * threshold_noise + if applied_noise_floor_pct is not None: + notes.append(f"noise_floor={applied_noise_floor_pct:.2f}%") + if mean_fps < block_threshold: + if regression_pct is None or regression_pct <= -float(min_block_regression_pct): + baseline_verdict = OracleVerdict.BLOCK + else: + baseline_verdict = OracleVerdict.WARN + notes.append("below_mad_block_but_inside_regression_floor") + elif mean_fps < warn_threshold: + baseline_verdict = OracleVerdict.WARN + else: + baseline_verdict = OracleVerdict.PASS + + # --- Combine: the most severe of the threshold and baseline verdicts wins --- + verdict = worst_verdict(threshold_verdict, baseline_verdict) + if threshold_verdict != OracleVerdict.PASS and verdict_severity(threshold_verdict) >= verdict_severity( + baseline_verdict + ): + threshold_source = ThresholdSource.THRESHOLD.value + else: + threshold_source = baseline_threshold_source + + if verdict == OracleVerdict.PASS and was_retried: + verdict = OracleVerdict.WARN + notes.append("was_retried") + + note = "; ".join(notes) if notes else None + + return OracleResult( + verdict=verdict, + bisect_verdict=_bisect_verdict(verdict, was_retried, failure_phase), + failure_phase=failure_phase, + measured_fps=mean_fps, + baseline_fps=baseline_fps, + regression_pct=regression_pct, + gpu_mem_used_mb=gpu_mem_used_mb, + startup_time_s=startup_time_s, + wall_time_s=wall_time_s, + was_retried=was_retried, + task_id=task_id, + backend=backend, + baseline_sample_count=baseline_sample_count, + baseline_source=baseline_source, + threshold_source=threshold_source, + warn_threshold_fps=warn_threshold, + block_threshold_fps=block_threshold, + hard_floor_fps=hard_floor_fps, + noise_floor_pct=applied_noise_floor_pct, + min_block_regression_pct=float(min_block_regression_pct), + note=note, + crossed_thresholds=crossed_thresholds, + ) diff --git a/tools/perf_smoke_test/perf_runtime.py b/tools/perf_smoke_test/perf_runtime.py new file mode 100644 index 000000000000..2dd006969b87 --- /dev/null +++ b/tools/perf_smoke_test/perf_runtime.py @@ -0,0 +1,236 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Perf-smoke-test runtime benchmark driver (random actions, no policy). + +Thin wrapper over the public Isaac Lab benchmark library that produces a +schema-v1 :class:`~isaaclab.benchmark.schema.RuntimeBundle`. It imports only +the stable building blocks (:mod:`~isaaclab.benchmark.stepping`, +:mod:`~isaaclab.benchmark.builders`, :mod:`~isaaclab.benchmark.capture`) rather +than driving ``isaaclab benchmark runtime``, so the gate controls its own +warmup handling and output layout. + +The benchmark framework was promoted to the public ``isaaclab.benchmark`` +namespace in #6564; the previous internal namespace no longer exists. See +``docs/source/migration/migrating_to_isaaclab_3-0.rst``. The import guards in +``test/test_framework_imports.py`` keep this file honest against the checkout. + +Difference from the upstream runtime script: the perf gate discards a +configurable number of leading **warmup** steps *before* aggregation, so the +reported ``total_fps`` is steady-state. This replaces the gate's previous +post-hoc ``excluded_frames`` mechanism (which required the raw per-frame series +the bundle schema deliberately drops); handling warmup at the source keeps the +aggregate directly comparable to the pre-migration steady-state mean without +serialising the raw series. + +Usage example:: + + ./isaaclab.sh -p tools/perf_smoke_test/perf_runtime.py \ + --task Isaac-Cartpole-Direct \ + --num_envs 4096 --num_frames 300 --warmup_frames 100 \ + --benchmark_formatter schema \ + --output_path /tmp/bench_out \ + presets=newton_mjwarp --headless +""" + +"""Launch Isaac Sim Simulator first.""" + +import argparse +import sys +import time + +from benchmark_result_adapter import steady_state_slice + +from isaaclab.app import AppLauncher + +from isaaclab_tasks.utils import setup_preset_cli + +# --- argument parsing ------------------------------------------------------- +parser = argparse.ArgumentParser(description="Benchmark environment runtime (random actions, no policy).") +parser.add_argument("--task", type=str, required=True, help="Gym task id to benchmark.") +parser.add_argument("--num_envs", type=int, default=None, help="Number of parallel environments.") +parser.add_argument( + "--num_frames", + type=int, + default=300, + help="Total number of environment steps to run (including warmup).", +) +parser.add_argument( + "--warmup_frames", + type=int, + default=0, + help="Number of leading steps to discard before aggregation (steady-state warmup exclusion).", +) +parser.add_argument("--seed", type=int, default=None, help="Environment seed.") +parser.add_argument("--output_path", type=str, default=".", help="Directory to write the output JSON.") +parser.add_argument( + "--benchmark_formatter", + type=str, + default="schema", + help=( + "Output format(s): comma-separated list of 'schema' (default, the typed benchmark bundle)," + " 'omniperf', 'osmo', 'json', 'summary'. Example: 'schema,omniperf'." + ), +) + +# append AppLauncher cli args and resolve Hydra preset tokens +AppLauncher.add_app_launcher_args(parser) +args_cli, hydra_args = setup_preset_cli(parser) +sys.argv = [sys.argv[0]] + hydra_args + +# --- heavy imports (after CLI parse, before app launch is measured) --------- +imports_time_begin = time.perf_counter_ns() + +import contextlib + +import gymnasium as gym + +from isaaclab.app import launch_simulation +from isaaclab.benchmark import BaseIsaacLabBenchmark, BenchmarkMonitor, builders, capture, stepping +from isaaclab.benchmark.schema import StartupTime + +import isaaclab_tasks # noqa: F401 +from isaaclab_tasks.utils import resolve_task_config + +# PLACEHOLDER: Extension template (do not remove this comment) +with contextlib.suppress(ImportError): + import isaaclab_tasks_experimental # noqa: F401 + +imports_time_end = time.perf_counter_ns() + + +def main(env_cfg, app_start_time_begin: int, app_start_time_end: int) -> None: + """Run the runtime benchmark and write the selected formatter outputs. + + Args: + env_cfg: Resolved environment configuration for :attr:`args_cli.task`. + app_start_time_begin: ``time.perf_counter_ns()`` sampled just before the + simulation app launch. + app_start_time_end: ``time.perf_counter_ns()`` sampled just after the + simulation app launch. + """ + if args_cli.num_envs is not None: + env_cfg.scene.num_envs = args_cli.num_envs + if args_cli.device is not None: + env_cfg.sim.device = args_cli.device + if args_cli.seed is not None: + env_cfg.seed = args_cli.seed + + formatter_types = [value.strip() for value in args_cli.benchmark_formatter.split(",") if value.strip()] + formatter_types = formatter_types or ["schema"] + + # RunConfig (physics/rendering/presets) is derived from the Hydra preset + # tokens and the resolved env cfg, matching the upstream runtime script. + cfg = capture.run_config_from_presets(hydra_args, env_cfg=env_cfg) + + start_utc = capture.now_utc_iso() + + benchmark = BaseIsaacLabBenchmark( + benchmark_name="benchmark_runtime", + formatter_type=args_cli.benchmark_formatter, + output_path=args_cli.output_path, + use_recorders=True, + frametime_recorders=any(t in ("summary", "omniperf") for t in formatter_types), + output_prefix=f"benchmark_runtime_{args_cli.task}", + workflow_metadata={ + "metadata": [ + {"name": "task", "data": args_cli.task}, + {"name": "num_envs", "data": args_cli.num_envs}, + {"name": "num_frames", "data": args_cli.num_frames}, + {"name": "warmup_frames", "data": args_cli.warmup_frames}, + {"name": "presets", "data": ",".join(cfg.presets)}, + ] + }, + ) + + # --- create env ------------------------------------------------------- + env_t0 = time.perf_counter_ns() + with contextlib.closing(gym.make(args_cli.task, cfg=env_cfg)) as env: + env_t1 = time.perf_counter_ns() + + num_envs = env.unwrapped.num_envs + + # --- step (warmup + measured) with resource monitoring ------------ + with BenchmarkMonitor(benchmark, interval=1.0): + step_times_s = stepping.run_runtime_loop(env, args_cli.num_frames) + + # Progress marker consumed by subprocess_runner.classify_failure_phase to + # tell an init-phase failure from a runtime-phase failure (a later crash + # with this marker present is attributed to runtime). Matches the legacy + # driver's stdout contract. + print("Step Frametimes", flush=True) + + benchmark.update_manual_recorders() + + # --- warmup exclusion at source: aggregate only steady-state frames. + measured_step_times, warmup = steady_state_slice(step_times_s, args_cli.warmup_frames) + if warmup != args_cli.warmup_frames: + print( + f"[perf_runtime] WARNING: warmup_frames={args_cli.warmup_frames} leaves no measured" + f" frames out of {len(step_times_s)}; clamped to {warmup} to keep >=1 steady-state frame." + ) + fps = [num_envs / t for t in measured_step_times if t > 0] + + # ``first_step`` is the (cold) first observed step, kept as a startup + # signal even though it is excluded from the steady-state fps. + startup = StartupTime( + app_launch=(app_start_time_end - app_start_time_begin) / 1e9, + env_creation=(env_t1 - env_t0) / 1e9, + first_step=(step_times_s[0] if step_times_s else 0.0), + python_imports=(imports_time_end - imports_time_begin) / 1e9, + ) + + runtime = builders.build_runtime( + startup_time_s=startup, + iteration_times_s=measured_step_times, + collection_fps=fps, + total_fps=fps, + steps_per_iteration=num_envs, + ) + + versions = capture.capture_versions(benchmark) + hardware = capture.capture_hardware(benchmark) + resources = capture.capture_resources(benchmark) + + end_utc = capture.now_utc_iso() + stamp = end_utc.translate(str.maketrans("", "", ":-"))[:15] + seed = args_cli.seed if args_cli.seed is not None else 0 + run_id = capture.synth_run_id(None, cfg.physics_backend, args_cli.task, seed, stamp) + + run = builders.build_run_identity( + run_id=run_id, + framework=None, + config=cfg, + task=args_cli.task, + seed=seed, + start_utc=start_utc, + end_utc=end_utc, + num_envs=num_envs, + ) + + # ``extra`` carries producer-specific scalars the perf gate needs but + # that are not part of the stable schema contract (num_frames is not a + # RunIdentity field; warmup_frames is gate-specific). The gate's + # benchmark_result_adapter reads these; other consumers may ignore them. + bundle = builders.build_runtime_bundle( + run=run, + versions=versions, + hardware=hardware, + runtime=runtime, + resources=resources, + extra={"num_frames": args_cli.num_frames, "warmup_frames": warmup}, + ) + + benchmark.attach_bundle(bundle) + benchmark._finalize_impl() + + +if __name__ == "__main__": + env_cfg, _agent_cfg = resolve_task_config(args_cli.task, None) + + app_start_time_begin = time.perf_counter_ns() + with launch_simulation(env_cfg, args_cli): + app_start_time_end = time.perf_counter_ns() + main(env_cfg, app_start_time_begin, app_start_time_end) diff --git a/tools/perf_smoke_test/pyproject.toml b/tools/perf_smoke_test/pyproject.toml new file mode 100644 index 000000000000..ca1c675fc83f --- /dev/null +++ b/tools/perf_smoke_test/pyproject.toml @@ -0,0 +1,12 @@ +# Scopes pytest to this self-contained tool: the ``[tool.pytest.ini_options]`` +# section makes pytest treat ``tools/perf_smoke_test/`` as its rootdir, so: +# +# 1. ``pythonpath = ["."]`` adds ``tools/perf_smoke_test/`` to ``sys.path``, +# making ``import oracle`` work from the test files without any shim. +# 2. ``tools/conftest.py`` sits above rootdir and is therefore not loaded, +# keeping these tests runnable without an Isaac Lab / Isaac Sim install. +# +# Run with: ``uv run python -m pytest tools/perf_smoke_test/`` +[tool.pytest.ini_options] +pythonpath = ["."] +testpaths = ["test"] diff --git a/tools/perf_smoke_test/runtime_contract.py b/tools/perf_smoke_test/runtime_contract.py new file mode 100644 index 000000000000..7aba9732b5bc --- /dev/null +++ b/tools/perf_smoke_test/runtime_contract.py @@ -0,0 +1,88 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Runtime compatibility contract construction for baseline matching""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +try: + from .backend_identity import BackendIdentity + from .hashing import stable_hash +except ImportError: # pragma: no cover - supports direct script imports + from backend_identity import BackendIdentity + from hashing import stable_hash + + +def _get_path(data: Mapping[str, Any], dotted_path: str) -> Any: + cur: Any = data + for part in dotted_path.split("."): + if not isinstance(cur, Mapping) or part not in cur: + return None + cur = cur[part] + return cur + + +def _field_list(policy: Mapping[str, Any], backend: BackendIdentity) -> list[str]: + fields: list[str] = [] + + def add_many(values: Any) -> None: + for value in values or []: + field = str(value) + if field not in fields: + fields.append(field) + + add_many(policy.get("always")) + by_physics = policy.get("by_physics_backend") or {} + if isinstance(by_physics, Mapping): + add_many(by_physics.get(backend.physics_backend)) + by_render = policy.get("by_render_backend") or {} + if isinstance(by_render, Mapping) and backend.render_backend: + add_many(by_render.get(backend.render_backend)) + return fields + + +def runtime_source(provenance: dict[str, Any] | None, runtime_resources: dict[str, Any] | None) -> dict[str, Any]: + provenance = provenance or {} + return { + "software": provenance.get("software") or {}, + "hardware": provenance.get("hardware") or {}, + "runtime_resources": runtime_resources or {}, + } + + +def build_runtime_contract( + *, + provenance: dict[str, Any] | None, + runtime_resources: dict[str, Any] | None, + backend: BackendIdentity, + policy: Mapping[str, Any], +) -> tuple[dict[str, Any], str]: + """Build the compatibility contract and hash used for baseline matching.""" + source = runtime_source(provenance, runtime_resources) + selected = {field: _get_path(source, field) for field in _field_list(policy, backend)} + contract = { + "runtime_contract_version": int(policy.get("contract_version", 1)), + "fields": selected, + } + return contract, stable_hash(contract) + + +def build_runtime_publish_info( + *, + provenance: dict[str, Any] | None, + runtime_resources: dict[str, Any] | None, + policy: Mapping[str, Any], +) -> dict[str, Any]: + """Return debug/runtime fields published for humans but not used for matching.""" + source = runtime_source(provenance, runtime_resources) + publish_only = {str(field): _get_path(source, str(field)) for field in policy.get("publish_only", [])} + publish_only = {key: value for key, value in publish_only.items() if value is not None} + return { + "software": source["software"], + "publish_only": publish_only, + } diff --git a/tools/perf_smoke_test/seed_baselines.py b/tools/perf_smoke_test/seed_baselines.py new file mode 100755 index 000000000000..d13160bd04ff --- /dev/null +++ b/tools/perf_smoke_test/seed_baselines.py @@ -0,0 +1,852 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Seed the perf-baselines branch from real commit history. + +This orchestrator simulates "the smoke test ran when each commit landed": for every +seed commit it checks that commit out into a throwaway clone, bind-mounts that +clone over the CI image's IsaacLab source, and runs the benchmark inside the +container. Because the container's ``.git`` then resolves to the seed commit, +:class:`VersionInfoRecorder` tags each ``perf_smoke_test_info.json`` with +that commit's hash, and the resulting baseline sample is keyed by the real SHA. +That is what lets the smoke test's merge-base/ancestry isolation work against a +populated baseline. + +Why a separate clone (not an in-place ``git checkout``): + +* Checking out an old commit in the main workspace would also revert this + orchestrator and the rest of ``tools/perf_smoke_test``. We run the + *current* tooling against *historical* source, so the historical source lives + in an isolated clone that is bind-mounted into the container only. +* A ``git worktree`` cannot be used because its ``.git`` is a pointer file into + the parent repo's ``.git/worktrees/...``; that path is not visible inside the + container, so ``git rev-parse HEAD`` would fail and the commit tag would be + lost. A real clone has a self-contained ``.git`` directory. + +Faithful re-runs require the seed commit's Python dependencies to match the CI +image (the image installs IsaacLab editable, so the bind mount swaps the running +code without a reinstall). Commits outside that dependency window will fail to +import and are skipped (their samples never reach the baseline). + +The FPS for each sample is computed via :func:`oracle.compare` with no baseline +and no hard floor, so it is byte-for-byte the same statistic the live smoke test +records. Samples are pushed with :func:`baseline_manager.update_baselines_git` +(append-only, idempotent), exactly like :mod:`aggregate`. +""" + +from __future__ import annotations + +import argparse +import atexit +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from aggregate import _bench_gpu_model # noqa: E402 +from baseline_manager import BaselineUpdateRecord, make_sample_metadata, update_baselines_git # noqa: E402 +from contracts import BenchResult # noqa: E402 +from gpu_identity import detect_gpu_model # noqa: E402 +from launch_config import hydra_args_for_task # noqa: E402 +from oracle import compare # noqa: E402 +from task_config import TaskConfig, load_tasks # noqa: E402 + +_TOOL_DIR = Path(__file__).resolve().parent +_CONTAINER_LOCAL_JIT_CACHE_BUCKETS = frozenset( + { + ("Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct", "physx_rtx_renderer"), + ("Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct", "newton_rtx_renderer"), + } +) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Seed perf-baselines from real commit history.") + parser.add_argument( + "--branches", + default="", + help=( + "Comma/space-separated branches to seed in one run (each stamped with its own target_branch). " + "Use 'branch:target' to stamp a different target_branch, e.g. 'develop, release/2.1'. " + "Takes precedence over --commits / --commit_branch." + ), + ) + parser.add_argument( + "--commits", + default="", + help="Explicit commit refs/SHAs (space/comma/newline separated). Overrides --commit_branch.", + ) + parser.add_argument( + "--commit_branch", + default="develop", + help="Branch whose recent history is sampled when --branches and --commits are empty.", + ) + parser.add_argument("--commit_count", type=int, default=5, help="Number of recent commits to seed from the branch.") + parser.add_argument("--samples_per_commit", type=int, default=5, help="Benchmark repetitions per commit/backend.") + parser.add_argument("--tasks", default="", help="Comma-separated task_id allowlist (empty = all tasks.json tasks).") + parser.add_argument("--backends", default="", help="Comma-separated backend_key allowlist (empty = all backends).") + parser.add_argument("--image", required=True, help="Local docker image tag to run (already pulled/tagged).") + parser.add_argument("--gpu_model", default="", help="GPU model label; auto-detected via nvidia-smi when empty.") + parser.add_argument("--target_branch", default="develop", help="Protected branch stamped onto each sample.") + parser.add_argument( + "--target_sha", + default="", + help=( + "Tip commit to check seed-commit ancestry against (for single-target runs). Empty resolves " + "each target branch's tip (origin/). Seed commits that are not ancestors of this tip " + "would be dropped by the gate's ancestry filter, so they are skipped (see --strict_ancestry)." + ), + ) + parser.add_argument( + "--strict_ancestry", + default="false", + help="When 'true', abort if any seed commit is not an ancestor of its target tip (default: skip+warn).", + ) + parser.add_argument("--baseline_branch", default="perf-baselines", help="Git branch storing baseline samples.") + parser.add_argument("--baseline_remote", default="origin", help="Remote to push baselines to (empty = local only).") + parser.add_argument("--baseline_push_retries", type=int, default=3) + parser.add_argument("--workdir", default=".", type=Path, help="Repo root (source of historical commits).") + parser.add_argument("--artifacts_root", default="seed-artifacts", type=Path, help="Where per-run outputs land.") + parser.add_argument( + "--seed_src_dir", + default="", + help="Throwaway clone path bind-mounted into the container (default: a temp dir).", + ) + parser.add_argument( + "--source_mount", + default="true", + help="Mount each commit's source into the container (true) or run the baked image source (false).", + ) + parser.add_argument( + "--dry_run", + default="true", + help="Run benchmarks and build records but skip the baseline push (default: true; pass false to publish).", + ) + return parser.parse_args() + + +def _as_bool(value: str) -> bool: + return str(value).strip().lower() in ("true", "1", "yes", "on") + + +def _run( + cmd: list[str], *, cwd: Path | None = None, check: bool = True, env: dict[str, str] | None = None +) -> subprocess.CompletedProcess: + print(f"[seed] $ {' '.join(cmd)}", flush=True) + result = subprocess.run(cmd, cwd=str(cwd) if cwd else None, text=True, env=env) + if check and result.returncode != 0: + raise RuntimeError(f"command failed (exit {result.returncode}): {' '.join(cmd)}") + return result + + +def _capture(cmd: list[str], *, cwd: Path | None = None) -> str: + result = subprocess.run(cmd, cwd=str(cwd) if cwd else None, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"command failed (exit {result.returncode}): {' '.join(cmd)}\n{result.stderr.strip()}") + return result.stdout.strip() + + +def _resolve_sha(token: str, workdir: Path) -> str: + return _capture(["git", "rev-parse", "--verify", f"{token}^{{commit}}"], cwd=workdir) + + +def _branch_commits(branch: str, count: int, workdir: Path) -> list[str]: + """Newest-first slice of a branch's history. + + Prefer the remote-tracking ref so a stale local branch cannot silently shadow + what the runner fetched. + """ + branch = branch.strip() + rev_source = None + for ref in (f"origin/{branch}", branch): + check = subprocess.run( + ["git", "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"], + cwd=str(workdir), + capture_output=True, + text=True, + ) + if check.returncode == 0: + rev_source = ref + break + if rev_source is None: + raise RuntimeError(f"Could not resolve branch {branch!r} (tried origin/{branch}, {branch}).") + out = _capture(["git", "rev-list", f"--max-count={count}", rev_source], cwd=workdir) + return [line for line in out.splitlines() if line.strip()] + + +def _resolve_branch_tip(branch: str, workdir: Path) -> str | None: + """Resolve a branch to its tip commit SHA, preferring the remote-tracking ref.""" + for ref in (f"origin/{branch}", branch): + check = subprocess.run( + ["git", "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"], + cwd=str(workdir), + capture_output=True, + text=True, + ) + if check.returncode == 0: + return check.stdout.strip() + return None + + +def _is_ancestor(commit: str, tip: str, workdir: Path) -> bool: + """Return True when ``commit`` is an ancestor of (reachable from) ``tip``.""" + result = subprocess.run( + ["git", "merge-base", "--is-ancestor", commit, tip], + cwd=str(workdir), + capture_output=True, + text=True, + ) + return result.returncode == 0 + + +def _filter_plan_by_ancestry( + plan: list[tuple[str, str]], + workdir: Path, + *, + target_sha: str = "", + strict: bool = False, +) -> list[tuple[str, str]]: + """Drop seed commits that are not ancestors of their target branch tip. + + The gate reads baselines with ``base_sha`` set to the target branch HEAD and + keeps only samples whose ``commit_sha`` is an ancestor of that HEAD (see + :func:`baseline_manager._sample_matches`). Seeding a commit that is not on the + target branch's history therefore produces samples the gate silently ignores, + which is the most common reason freshly seeded baselines "don't work". This + preflight refuses to seed them. + + Args: + plan: Ordered ``(target_branch, commit_sha)`` pairs to seed. + workdir: Repo root used for git lookups. + target_sha: Explicit tip to check ancestry against for every entry (for + single-target runs); empty resolves each target branch's tip. + strict: Raise instead of skipping when a non-ancestor (or unresolvable + tip) is found. + """ + kept: list[tuple[str, str]] = [] + dropped: list[str] = [] + tip_cache: dict[str, str | None] = {} + for target, commit in plan: + if target_sha: + tip: str | None = target_sha + else: + if target not in tip_cache: + tip_cache[target] = _resolve_branch_tip(target, workdir) + tip = tip_cache[target] + if not tip: + msg = f"cannot resolve tip of target branch {target!r} to verify ancestry of {commit[:8]}" + if strict: + raise RuntimeError(msg) + print(f"::warning::[seed] {msg}; seeding it anyway (ancestry unverified)") + kept.append((target, commit)) + continue + if _is_ancestor(commit, tip, workdir): + kept.append((target, commit)) + else: + dropped.append(f"{target}/{commit[:8]}") + if dropped: + detail = ", ".join(dropped) + msg = ( + f"{len(dropped)} seed commit(s) are NOT ancestors of their target branch tip and would be " + f"ignored by the gate's ancestry filter: {detail}" + ) + if strict: + raise RuntimeError(msg) + print(f"::warning::[seed] {msg}") + return kept + + +def _build_seed_plan(args: argparse.Namespace) -> list[tuple[str, str]]: + """Resolve what to seed as an ordered list of ``(target_branch, commit_sha)`` pairs. + + Precedence: ``--branches`` (multi-branch) > ``--commits`` (explicit SHAs) > + ``--commit_branch`` (single branch). Each pair is de-duplicated so the same + commit is not seeded twice under the same target branch. + """ + workdir = args.workdir + plan: list[tuple[str, str]] = [] + + if args.branches.strip(): + for entry in args.branches.replace(",", " ").split(): + entry = entry.strip() + if not entry: + continue + # "branch" stamps target=branch; "branch:target" overrides the stamp. + branch, _, target = entry.partition(":") + target = target.strip() or branch.strip() + for sha in _branch_commits(branch.strip(), args.commit_count, workdir): + plan.append((target, sha)) + elif args.commits.strip(): + target = args.target_branch.strip() + for token in args.commits.replace(",", " ").split(): + if token.strip(): + plan.append((target, _resolve_sha(token.strip(), workdir))) + else: + target = args.target_branch.strip() + for sha in _branch_commits(args.commit_branch.strip(), args.commit_count, workdir): + plan.append((target, sha)) + + deduped: list[tuple[str, str]] = [] + seen: set[tuple[str, str]] = set() + for target, sha in plan: + key = (target, sha) + if key not in seen: + seen.add(key) + deduped.append(key) + if not deduped: + raise RuntimeError("No commits resolved to seed.") + return deduped + + +def _select_tasks(args: argparse.Namespace) -> list[TaskConfig]: + tasks = load_tasks() + task_filter = {t.strip() for t in args.tasks.split(",") if t.strip()} + backend_filter = {b.strip() for b in args.backends.split(",") if b.strip()} + selected = [ + task + for task in tasks + if (not task_filter or task.task_id in task_filter) + and (not backend_filter or task.backend_key in backend_filter) + ] + if not selected: + raise RuntimeError(f"No tasks matched filters tasks={sorted(task_filter)} backends={sorted(backend_filter)}.") + return selected + + +def _prepare_seed_source(workdir: Path, seed_src_dir: Path, sha: str) -> None: + """Materialize ``sha`` into a self-contained clone at ``seed_src_dir``. + + The clone is created once and reused across commits; each commit is fetched + from the (full-history) workspace by SHA and hard-checked-out so leftover + files from a previous commit cannot leak in. + """ + # Skip Git LFS smudge for every git op below. The benchmark only needs the + # Python source, never the LFS-tracked binaries (e.g. rendering golden-image + # test fixtures). A branch whose tip references a missing/orphaned LFS object + # (common after a force-update) would otherwise abort ``checkout`` with a + # smudge-filter failure and sink the whole seed run. With smudge skipped, LFS + # files materialize as harmless pointer files instead. + git_env = {**os.environ, "GIT_LFS_SKIP_SMUDGE": "1"} + if not (seed_src_dir / ".git").exists(): + seed_src_dir.parent.mkdir(parents=True, exist_ok=True) + _run(["git", "clone", "--no-checkout", str(workdir.resolve()), str(seed_src_dir)], env=git_env) + _run(["git", "fetch", "--no-tags", str(workdir.resolve()), sha], cwd=seed_src_dir, env=git_env) + _run(["git", "checkout", "-f", "--detach", sha], cwd=seed_src_dir, env=git_env) + # Best-effort: a prior commit's container can leave behind files the runner + # user cannot delete (e.g. root-owned ``__pycache__`` written into the + # bind-mount). ``checkout -f`` already restored every tracked file, so leftover + # untracked residue is harmless to the benchmark; never let cleanup abort the + # whole seed run. Open up permissions first so files the runner *can* chmod + # become removable. + subprocess.run(["chmod", "-R", "a+rwX", str(seed_src_dir)], check=False) + clean = _run(["git", "clean", "-fdx"], cwd=seed_src_dir, check=False) + if clean.returncode != 0: + print(f"[seed] warning: git clean left undeletable residue (exit {clean.returncode}); continuing", flush=True) + # The in-container user is uid 1000 (isaaclab); the clone is created by the + # runner user. Open it up so the benchmark can read the source and write the + # throwaway _isaac_sim symlink, mirroring the smoke test's bind-mount chmod. + subprocess.run(["chmod", "-R", "a+rwX", str(seed_src_dir)], check=False) + + +def _create_seed_source_dir() -> Path: + """Create a unique disposable source-clone directory.""" + return Path(tempfile.mkdtemp(prefix="perf-seed-src-")) + + +def _prepare_jit_cache(cache_dir: Path) -> Path: + """Create a writable JIT cache directory for one task/backend bucket.""" + for subdir in (cache_dir / "warp", cache_dir / "nv"): + subdir.mkdir(parents=True, exist_ok=True) + # Containers can write cache files as a different uid than the runner. + _run(["chmod", "-R", "0777", str(cache_dir)], check=False) + return cache_dir + + +def _uses_container_local_jit_cache(task: TaskConfig) -> bool: + """Return whether a task/backend must avoid the host-mounted JIT cache.""" + return (task.task_id, task.backend_key) in _CONTAINER_LOCAL_JIT_CACHE_BUCKETS + + +def _prepare_kit_cache(cache_dir: Path) -> Path: + """Create a writable Kit cache directory for one task/backend bucket.""" + cache_dir.mkdir(parents=True, exist_ok=True) + # Kit writes shader and generated-node caches from the in-container user. + _run(["chmod", "-R", "0777", str(cache_dir)], check=False) + return cache_dir + + +def _cache_bucket_path(cache_root: Path, target_branch: str, commit: str, task_id: str, backend_key: str) -> Path: + """Return the cache shared by repeated samples of one task/backend bucket.""" + return ( + cache_root + / _safe_path_component(target_branch) + / commit[:8] + / _safe_path_component(task_id) + / _safe_path_component(backend_key) + ) + + +def _cache_run_name() -> str: + """Return an allocation-isolated cache directory name for this invocation.""" + run_id = os.environ.get("GITHUB_RUN_ID", f"local-{os.getpid()}") + run_attempt = os.environ.get("GITHUB_RUN_ATTEMPT", "0") + run_label = os.environ.get("PERF_SMOKE_RUN_LABEL", "").strip() + name = f"run-{run_id}-attempt-{run_attempt}" + if run_label: + name += f"-{_safe_path_component(run_label)}" + return name + + +def _cleanup_run_dir(run_dir: Path) -> None: + """Remove a disposable directory for one seeder invocation.""" + if not run_dir.exists(): + return + subprocess.run(["chmod", "-R", "a+rwX", str(run_dir)], check=False, capture_output=True, text=True) + shutil.rmtree(run_dir, ignore_errors=True) + if run_dir.exists(): + print(f"::warning::[seed] Could not fully remove run directory {run_dir}") + + +def _docker_run_benchmark( + *, + image: str, + task: TaskConfig, + artifact_dir: Path, + jit_cache: Path | None, + kit_cache: Path, + seed_src_dir: Path | None, + container_name: str, +) -> int: + """Run one benchmark in the container, mirroring the smoke test's invocation.""" + hydra_args = " ".join(hydra_args_for_task(task)) + seed_token = f"--seed {task.seed}" if task.seed is not None else "" + inner = ( + "set -e\n" + # Warp and Kit create nested cache directories at runtime. A permissive + # umask keeps them writable for successive containers. + "umask 000\n" + "mkdir -p /tmp/bench_out /tmp/jit-cache/warp /tmp/jit-cache/nv\n" + "cd /workspace/isaaclab\n" + "rm -f _isaac_sim\n" + "ln -s /isaac-sim _isaac_sim\n" + "./isaaclab.sh -p tools/perf_smoke_test/perf_runtime.py " + f"--task '{task.task_id}' " + f"--num_envs {task.num_envs} " + f"--num_frames {task.num_frames} " + f"--warmup_frames {task.warmup_frames} " + "--benchmark_formatter schema " + "--output_path /tmp/bench_out " + f"{seed_token} {hydra_args}\n" + ) + + cmd = [ + "docker", + "run", + "--name", + container_name, + "--init", + "--stop-timeout", + "10", + "--entrypoint", + "bash", + "--gpus", + "all", + "--network=host", + "--security-opt=no-new-privileges:true", + "--ulimit", + "nofile=65536:65536", + "--ulimit", + "nproc=4096:4096", + "-e", + "OMNI_KIT_ACCEPT_EULA=yes", + "-e", + "ACCEPT_EULA=Y", + "-e", + "OMNI_KIT_DISABLE_CUP=1", + "-e", + "ISAAC_SIM_HEADLESS=1", + "-e", + "PYTHONUNBUFFERED=1", + # Keep the bind-mounted source clean: writing __pycache__ here leaves + # root-owned files the runner user cannot remove on the next git clean. + "-e", + "PYTHONDONTWRITEBYTECODE=1", + "-e", + "WARP_CACHE_PATH=/tmp/jit-cache/warp", + "-e", + "CUDA_CACHE_PATH=/tmp/jit-cache/nv", + ] + if jit_cache is not None: + cmd += ["-v", f"{jit_cache}:/tmp/jit-cache"] + cmd += ["-v", f"{kit_cache}:/isaac-sim/kit/cache"] + if seed_src_dir is not None: + # Bind-mount the historical source over the image's IsaacLab tree so the + # container runs (and git-tags) the seed commit, not the baked source. + cmd += ["-v", f"{seed_src_dir}:/workspace/isaaclab"] + cmd += [image, "-c", inner] + + result = subprocess.run(cmd, text=True) + permissions_returncode = 0 + try: + copy_result = subprocess.run( + ["docker", "cp", f"{container_name}:/tmp/bench_out/.", str(artifact_dir)], + text=True, + ) + if copy_result.returncode == 0: + permissions_returncode = subprocess.run( + ["chmod", "-R", "a+rwX", str(artifact_dir)], + text=True, + ).returncode + finally: + subprocess.run(["docker", "rm", "-f", container_name], capture_output=True, text=True) + return result.returncode or copy_result.returncode or permissions_returncode + + +def _build_bench_result(task: TaskConfig, artifact_dir: Path, exit_code: int, wall_time_s: int) -> None: + _run( + [ + "python3", + str(_TOOL_DIR / "build_bench_result.py"), + "--task_id", + task.task_id, + "--physics_backend", + task.physics_backend, + "--render_backend", + task.render_backend or "", + "--artifact_dir", + str(artifact_dir), + "--exit_code", + str(exit_code), + "--wall_time_s", + str(wall_time_s), + "--timeout_s", + str(task.timeout_minutes * 60), + "--log_file", + str(artifact_dir / "benchmark.log"), + "--launch_config", + str(artifact_dir / "launch_config.json"), + "--gate_config", + str(_TOOL_DIR / "gate_config.json"), + ], + check=False, + ) + + +def _write_launch_config(task: TaskConfig, artifact_dir: Path, gpu_model: str) -> None: + _run( + [ + "python3", + str(_TOOL_DIR / "write_launch_config.py"), + "--task_id", + task.task_id, + "--physics_backend", + task.physics_backend, + "--render_backend", + task.render_backend or "", + "--gpu_model", + gpu_model, + "--artifact_dir", + str(artifact_dir), + ] + ) + + +def _record_from_result( + artifact_dir: Path, + gpu_model: str, + target_branch: str, + sample_index: int, + known_commit: str | None = None, +) -> BaselineUpdateRecord | None: + """Turn one built ``perf_smoke_test_result.json`` into a baseline sample. + + Returns ``None`` when the run was unhealthy (no benchmark info / no FPS) so the + caller can skip it. The FPS is computed via :func:`oracle.compare` with no + baseline and no hard floor, matching the live smoke test's statistic exactly. + """ + result_path = artifact_dir / "perf_smoke_test_result.json" + if not result_path.exists(): + print(f"[seed] skip @ {artifact_dir}: no result json (job crashed before build_bench_result)") + return None + bench_result = BenchResult.from_dict(json.loads(result_path.read_text())) + task_id = bench_result.task_id + backend = bench_result.backend_key or bench_result.backend + bench_gpu_model = _bench_gpu_model(bench_result, gpu_model) + # The seeder checks out each commit by SHA, so it authoritatively knows the + # commit even when the in-container benchmark cannot capture git provenance + # (detached HEAD on a runner-owned bind mount). Prefer the known SHA so the + # sample stays ancestry-selectable by the smoke test. + commit_sha = known_commit or ((bench_result.provenance or {}).get("git") or {}).get("commit_hash") + + if not bench_result.perf_smoke_test_info_present: + print(f"[seed] skip {task_id}/{backend} @ {artifact_dir.name}: no benchmark info (run failed)") + return None + oracle_result = compare( + bench_result=bench_result, + baseline=None, + fps_mean_thresholds=[], + min_block_regression_pct=0.0, + ) + if oracle_result.measured_fps is None: + print(f"[seed] skip {task_id}/{backend} @ {artifact_dir.name}: no measurable FPS") + return None + if not commit_sha: + print( + f"[seed] WARNING {task_id}/{backend} @ {artifact_dir.name}: no commit_sha in provenance; " + "sample will not be ancestry-selectable" + ) + sample_metadata = make_sample_metadata( + gpu_model=bench_gpu_model, + task_id=task_id, + backend=backend, + fps=oracle_result.measured_fps, + bench_result=bench_result.to_dict(), + target_branch=target_branch, + trusted_source="seed", + commit_sha=commit_sha, + sample_index=sample_index, + ) + short = (commit_sha or "????????")[:8] + print( + f"[seed] sample {task_id}/{backend} commit={short} target={target_branch} fps={oracle_result.measured_fps:.1f}" + ) + return BaselineUpdateRecord( + gpu_model=bench_gpu_model, + task_id=task_id, + backend=backend, + fps=oracle_result.measured_fps, + sample_metadata=sample_metadata, + ) + + +def _safe_path_component(name: str) -> str: + """Make a branch name safe to use as a single path segment (no nested dirs).""" + return "".join(ch if ch.isalnum() or ch in "-_." else "-" for ch in name) + + +# Log fragments that mean "this commit is incompatible with the baked container" +# rather than a genuine performance failure. A seeded historical commit can ship a +# kit experience / Python that references Isaac Sim extensions or APIs absent from +# the era-fixed image; in an offline image that surfaces as a dependency-solver +# failure at startup. We treat these as skips, not hard failures, so a divergent or +# pre-migration commit cannot sink a seed run that also has compatible commits. +_INCOMPATIBLE_ENV_SIGNATURES = ( + "registry cache path is not set", + "because of dependency solver failure", + "Can't pull extension", + "Failed to resolve extension", +) + + +def _detect_incompatible_env(artifact_dir: Path) -> str | None: + """Return a short reason if the benchmark log shows the run was incompatible with + the container (an Isaac Sim extension/API the image does not contain), else ``None``. + """ + try: + text = (artifact_dir / "benchmark.log").read_text(errors="replace") + except OSError: + return None + for sig in _INCOMPATIBLE_ENV_SIGNATURES: + if sig in text: + return sig + return None + + +def main() -> int: + import time + + args = _parse_args() + source_mount = _as_bool(args.source_mount) + dry_run = _as_bool(args.dry_run) + gpu_model = detect_gpu_model(args.gpu_model) + workdir = args.workdir.resolve() + artifacts_root = (workdir / args.artifacts_root).resolve() + artifacts_root.mkdir(parents=True, exist_ok=True) + + if args.seed_src_dir: + seed_src_dir = Path(args.seed_src_dir).resolve() + else: + seed_src_dir = _create_seed_source_dir() + atexit.register(_cleanup_run_dir, seed_src_dir) + cache_run = _cache_run_name() + jit_cache_root = workdir / "jit-cache" / "seed" / cache_run + kit_cache_root = workdir / "kit-cache" / "seed" / cache_run + atexit.register(_cleanup_run_dir, jit_cache_root) + atexit.register(_cleanup_run_dir, kit_cache_root) + + plan = _build_seed_plan(args) + plan = _filter_plan_by_ancestry( + plan, workdir, target_sha=args.target_sha.strip(), strict=_as_bool(args.strict_ancestry) + ) + if not plan: + raise RuntimeError( + "No seed commits remain after the ancestry preflight; every candidate is off the target " + "branch's history and would be ignored by the gate. Seed commits that live on the target " + "branch (e.g. run against origin/)." + ) + tasks = _select_tasks(args) + + plan_by_target: dict[str, list[str]] = {} + for target, sha in plan: + plan_by_target.setdefault(target, []).append(sha[:8]) + + print("[seed] ----------------------------------------------------------------") + for target, shas in plan_by_target.items(): + print(f"[seed] target={target}: commits {shas}") + print(f"[seed] tasks ({len(tasks)}): {[f'{t.task_id}/{t.backend_key}' for t in tasks]}") + print(f"[seed] samples_per_commit={args.samples_per_commit} source_mount={source_mount} dry_run={dry_run}") + print(f"[seed] baseline_branch={args.baseline_branch} gpu_model={gpu_model}") + print("[seed] ----------------------------------------------------------------") + + records: list[BaselineUpdateRecord] = [] + incompatible_skips: list[str] = [] + prep_skips: list[str] = [] + other_failures = 0 + total = 0 + for target_branch, commit in plan: + short = commit[:8] + target_dir = _safe_path_component(target_branch) + if source_mount: + try: + _prepare_seed_source(workdir, seed_src_dir, commit) + except (RuntimeError, OSError) as exc: + # Don't let one un-checkout-able commit (e.g. a broken LFS object or + # rewritten history) abort commits that would otherwise seed cleanly. + print(f"::warning::[seed] skip commit {short} on {target_branch}: source prep failed: {exc}") + prep_skips.append(f"{target_branch}/{short}") + continue + for task in tasks: + if _uses_container_local_jit_cache(task): + task_jit_cache = None + print(f"[seed] using container-local JIT cache for {task.task_id}/{task.backend_key}") + else: + task_jit_cache = _prepare_jit_cache( + _cache_bucket_path(jit_cache_root, target_branch, commit, task.task_id, task.backend_key) + ) + task_kit_cache = _prepare_kit_cache( + _cache_bucket_path(kit_cache_root, target_branch, commit, task.task_id, task.backend_key) + ) + for sample_idx in range(args.samples_per_commit): + total += 1 + artifact_dir = ( + artifacts_root / target_dir / short / task.task_id / task.backend_key / f"sample{sample_idx}" + ) + artifact_dir.mkdir(parents=True, exist_ok=True) + _run(["chmod", "-R", "0777", str(artifact_dir)], check=False) + container = f"perf-seed-{target_dir}-{short}-{task.task_id}-{task.backend_key}-{sample_idx}" + container = "".join(ch if ch.isalnum() or ch in "-_" else "-" for ch in container) + print( + f"[seed] >>> target={target_branch} {short} {task.task_id}/{task.backend_key} " + f"sample {sample_idx + 1}/{args.samples_per_commit}" + ) + + _write_launch_config(task, artifact_dir, gpu_model) + subprocess.run(["docker", "rm", "-f", container], capture_output=True, text=True) + start = time.time() + exit_code = _docker_run_benchmark( + image=args.image, + task=task, + artifact_dir=artifact_dir, + jit_cache=task_jit_cache, + kit_cache=task_kit_cache, + seed_src_dir=seed_src_dir if source_mount else None, + container_name=container, + ) + wall_time_s = int(time.time() - start) + _build_bench_result(task, artifact_dir, exit_code, wall_time_s) + + record = _record_from_result( + artifact_dir, + gpu_model, + target_branch, + sample_index=sample_idx, + known_commit=commit, + ) + if record is not None: + records.append(record) + continue + label = f"{target_branch}/{short} {task.task_id}/{task.backend_key}" + reason = _detect_incompatible_env(artifact_dir) + if reason is not None: + print(f"[seed] skip {label}: incompatible with container ({reason})") + incompatible_skips.append(label) + else: + other_failures += 1 + + print( + f"[seed] collected {len(records)} baseline sample(s) from {total} benchmark run(s); " + f"skipped {len(incompatible_skips)} incompatible, {len(prep_skips)} prep-failed, " + f"{other_failures} other failure(s)" + ) + + summary_path = artifacts_root / "seed_records.json" + summary_path.write_text( + json.dumps( + [ + { + "gpu_model": r.gpu_model, + "task_id": r.task_id, + "backend": r.backend, + "fps": r.fps, + "commit_sha": (r.sample_metadata or {}).get("commit_sha"), + "target_branch": (r.sample_metadata or {}).get("target_branch"), + "launch_config_hash": (r.sample_metadata or {}).get("launch_config_hash"), + "benchmark_contract_hash": (r.sample_metadata or {}).get("benchmark_contract_hash"), + "runtime_contract_hash": (r.sample_metadata or {}).get("runtime_contract_hash"), + "baseline_epoch": (r.sample_metadata or {}).get("baseline_epoch"), + "sample_index": (r.sample_metadata or {}).get("sample_index"), + "sample_id": (r.sample_metadata or {}).get("sample_id"), + "timestamp": (r.sample_metadata or {}).get("timestamp"), + "ci_run_id": (r.sample_metadata or {}).get("ci_run_id"), + "ci_run_attempt": (r.sample_metadata or {}).get("ci_run_attempt"), + "ci_job": (r.sample_metadata or {}).get("ci_job"), + "ci_run_label": (r.sample_metadata or {}).get("ci_run_label"), + "ci_runner_name": (r.sample_metadata or {}).get("ci_runner_name"), + } + for r in records + ], + indent=2, + ) + ) + print(f"[seed] wrote sample summary -> {summary_path}") + + if not records: + if incompatible_skips and not other_failures: + print( + "::warning::No samples seeded: every commit was incompatible with this container. " + "A baseline run only works against commits from the same simulator era as the image " + "(one container = one era). Seed the branch the image was built from (e.g. develop), " + "not a divergent or pre-migration branch such as main." + ) + else: + print("::warning::No healthy samples collected; nothing to push.") + return 1 + if dry_run: + print("[seed] dry_run=true; skipping baseline push.") + return 0 + + push = update_baselines_git( + args.baseline_branch, + records, + remote=args.baseline_remote or None, + max_retries=args.baseline_push_retries, + repo_dir=workdir, + ) + print( + f"[seed] baseline push: pushed={push.pushed} branch={push.branch} " + f"base={(push.base_sha or '-')[:8]} new={(push.pushed_sha or '-')[:8]} " + f"appended={push.update_count} attempts={push.attempts}" + ) + return 0 if push.pushed or args.baseline_remote == "" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/perf_smoke_test/task_config.py b/tools/perf_smoke_test/task_config.py new file mode 100644 index 000000000000..ad661be7bf36 --- /dev/null +++ b/tools/perf_smoke_test/task_config.py @@ -0,0 +1,222 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import json +from dataclasses import dataclass, field +from pathlib import Path + +try: + from .backend_identity import make_backend_key, normalize_physics_backend, normalize_render_backend + from .gate_types import FpsMeanThreshold + from .gpu_identity import gpu_model_config_keys +except ImportError: # pragma: no cover - supports direct script imports + from backend_identity import make_backend_key, normalize_physics_backend, normalize_render_backend + from gate_types import FpsMeanThreshold + from gpu_identity import gpu_model_config_keys + +_DEFAULT_TASKS_JSON = Path(__file__).parent / "tasks.json" + +# Maps backend name to list of cache identifiers that CI needs to pull; absent key defaults to no caches +_BACKEND_CACHES: dict[str, list[str]] = { + "newton": ["mjwarp_jit"], +} + + +def parse_fps_mean_thresholds(raw) -> dict[str, dict[str, list[FpsMeanThreshold]]]: + """Parse the nested ``{gpu_model: {backend_key: [entries]}}`` threshold config. + + Validation (mandatory names, gating-verdict enum, value-less skips) happens + here via :meth:`FpsMeanThreshold.from_list`, so malformed ``tasks.json`` fails + fast at load time. + + Args: + raw: The raw ``fps_mean_thresholds`` value from ``tasks.json`` (may be empty). + + Returns: + Nested mapping of GPU model to backend key to parsed thresholds. + """ + if not raw: + return {} + if not isinstance(raw, dict): + raise TypeError("fps_mean_thresholds must be an object keyed by GPU model") + parsed: dict[str, dict[str, list[FpsMeanThreshold]]] = {} + for gpu_key, backends in raw.items(): + if not isinstance(backends, dict): + raise TypeError(f"fps_mean_thresholds[{gpu_key!r}] must be an object keyed by backend") + parsed[gpu_key] = { + backend_key: FpsMeanThreshold.from_list(entries, context=f"{gpu_key}/{backend_key}") + for backend_key, entries in backends.items() + } + return parsed + + +def caches_for_backend(backend: str) -> list[str]: + """Return cache identifiers required before benchmarking with a given physics backend. + + The returned identifiers are consumed by the CI cache-pull step to locate and + restore named cache artifacts by name or pattern. An empty list means no + pre-run cache restoration is needed. + + Currently defined identifiers: + ``"mjwarp_jit"``: Newton MJWarp JIT compilation cache. + + Args: + backend: Backend name (e.g. ``"newton"``, ``"physx"``). + + Returns: + List of cache identifier strings. + """ + return list(_BACKEND_CACHES.get(backend, [])) + + +@dataclass +class TaskConfig: + """Configuration for a single benchmark task and backend combination.""" + + task_id: str + physics_backend: str + render_backend: str | None + preset: str + num_envs: int + num_frames: int + warmup_frames: int + camera_resolution: tuple[int, int] | None + timeout_minutes: int + fps_mean_thresholds: dict[str, dict[str, list[FpsMeanThreshold]]] + caches: list[str] + tags: list[str] = field(default_factory=lambda: ["always"]) + task_type: str = "benchmark" + runs_on: str = "gpu-l40s" + seed: int | None = None + baseline_epoch: int = 1 + noise_floor_pct: dict = field(default_factory=dict) + + @property + def backend_key(self) -> str: + """Composite key identifying the backend combination. + + Returns f"{physics_backend}_{render_backend}" when render_backend is set, + otherwise returns physics_backend. + """ + return make_backend_key(self.physics_backend, self.render_backend) + + def thresholds_for(self, gpu_model: str) -> list[FpsMeanThreshold]: + """Return configured FPS thresholds for this task/backend on ``gpu_model``. + + Normalizes ``gpu_model`` to its canonical key (e.g. ``l40s``) and returns + the first matching backend entry, or an empty list if none apply. + + Args: + gpu_model: GPU model string (canonical slug, display name, or raw name). + + Returns: + List of :class:`~gate_types.FpsMeanThreshold` (possibly empty). + """ + for key in gpu_model_config_keys(gpu_model): + backends = self.fps_mean_thresholds.get(key) + if backends and self.backend_key in backends: + return backends[self.backend_key] + return [] + + +def _load_tasks_json(path: Path) -> tuple[dict, list[dict]]: + with open(path) as f: + raw_data = json.load(f) + + if isinstance(raw_data, dict): + defaults = raw_data.get("defaults", {}) + raw_list = raw_data.get("tasks", []) + if not isinstance(raw_list, list): + raise TypeError(f"'tasks' field in {path} must be a list") + elif isinstance(raw_data, list): + defaults = {} + raw_list = raw_data + else: + raise TypeError(f"{path} must contain a JSON list or an object with a top-level 'tasks' list") + + if not isinstance(defaults, dict): + raise TypeError(f"'defaults' field in {path} must be an object") + + return defaults, raw_list + + +def load_tasks(tasks_json_path: Path | str | None = None) -> list[TaskConfig]: + """Load all benchmark tasks from tasks.json, producing a TaskConfig for each backend combination. + + Args: + tasks_json_path: Path to tasks.json. Defaults to the tasks.json next to this module. + + Returns: + List of TaskConfig objects, one per (task_id, backend) combination. + """ + path = Path(tasks_json_path) if tasks_json_path is not None else _DEFAULT_TASKS_JSON + defaults, raw_list = _load_tasks_json(path) + + tasks: list[TaskConfig] = [] + for raw in raw_list: + if not isinstance(raw, dict): + raise TypeError(f"task entry in {path} must be an object") + merged = {**defaults, **raw} + + camera_raw = merged.get("camera_resolution") + camera_resolution: tuple[int, int] | None = ( + tuple(camera_raw) if camera_raw is not None else None # type: ignore[assignment] + ) + fps_mean_thresholds = parse_fps_mean_thresholds(merged.get("fps_mean_thresholds", {})) + noise_floor_pct: dict = merged.get("noise_floor_pct", {}) + backends: list[dict] = merged.get("backends", []) + + for backend_entry in backends: + physics = normalize_physics_backend(backend_entry["physics"]) + if physics is None: + raise ValueError(f"backend entry in {path} must define a non-default physics backend") + render = normalize_render_backend(backend_entry.get("render")) + tasks.append( + TaskConfig( + task_id=merged["task_id"], + physics_backend=physics, + render_backend=render, + preset=merged["preset"], + num_envs=int(merged["num_envs"]), + num_frames=int(merged["num_frames"]), + warmup_frames=int(merged.get("warmup_frames", 0)), + camera_resolution=camera_resolution, + timeout_minutes=int(merged["timeout_minutes"]), + fps_mean_thresholds=fps_mean_thresholds, + noise_floor_pct=noise_floor_pct, + caches=caches_for_backend(physics), + tags=merged["tags"], + task_type=merged["type"], + runs_on=merged["runs_on"], + seed=int(merged["seed"]) if merged.get("seed") is not None else None, + baseline_epoch=int(merged.get("baseline_epoch", 1)), + ) + ) + return tasks + + +def get_task( + task_id: str, + backend_key: str, + tasks_json_path: Path | str | None = None, +) -> TaskConfig: + """Return the TaskConfig for the given task_id and backend_key combination. + + Args: + task_id: The task identifier to look up. + backend_key: The backend key (e.g. "physx", "newton", "physx_rtx"). + tasks_json_path: Optional path to tasks.json. + + Returns: + The matching TaskConfig. + + Raises: + KeyError: If no task with the given (task_id, backend_key) exists. + """ + tasks = load_tasks(tasks_json_path) + for task in tasks: + if task.task_id == task_id and task.backend_key == backend_key: + return task + raise KeyError(f"Task not found: task_id={task_id!r} backend_key={backend_key!r}") diff --git a/tools/perf_smoke_test/tasks.json b/tools/perf_smoke_test/tasks.json new file mode 100644 index 000000000000..4a2c140dbb08 --- /dev/null +++ b/tools/perf_smoke_test/tasks.json @@ -0,0 +1,101 @@ +{ + "defaults": { + "type": "benchmark", + "runs_on": "gpu-l40s", + "preset": "default", + "seed": 42, + "num_envs": 512, + "num_frames": 300, + "warmup_frames": 100, + "camera_resolution": null, + "timeout_minutes": 10, + "tags": ["always"] + }, + "tasks": [ + { + "task_id": "Isaac-Cartpole-Direct", + "num_envs": 4096, + "backends": [ + {"physics": "physx"}, + {"physics": "newton"} + ], + "fps_mean_thresholds": { + "l40s": { + "physx": [ + {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 100.0} + ], + "newton": [ + {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 0.0} + ] + } + } + }, + { + "task_id": "IsaacContrib-Factory-GearMesh-Direct", + "timeout_minutes": 15, + "backends": [ + {"physics": "physx"} + ], + "fps_mean_thresholds": { + "l40s": { + "physx": [ + {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 30.0} + ] + } + } + }, + { + "task_id": "Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct", + "timeout_minutes": 20, + "tags": ["camera"], + "backends": [ + {"physics": "physx", "render": "rtx_renderer"}, + {"physics": "physx", "render": "newton_renderer"}, + {"physics": "newton", "render": "rtx_renderer"}, + {"physics": "newton", "render": "newton_renderer"} + ], + "fps_mean_thresholds": { + "l40s": { + "physx_rtx_renderer": [ + {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 20.0} + ], + "physx_newton_renderer": [ + {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 0.0} + ], + "newton_rtx_renderer": [ + {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 0.0} + ], + "newton_newton_renderer": [ + {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 0.0} + ] + } + } + }, + { + "task_id": "Isaac-Velocity-Flat-G1", + "timeout_minutes": 12, + "backends": [ + {"physics": "physx"}, + {"physics": "newton"} + ], + "fps_mean_thresholds": { + "l40s": { + "physx": [ + {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 40.0} + ], + "newton": [ + {"threshold_verdict": "BLOCK", "threshold_name": "hard-floor", "threshold": 0.0} + ] + } + }, + "noise_floor_pct": { + "rtx_pro_6000_blackwell": { + "newton": 2.52 + }, + "l40s": { + "newton": 2.06 + } + } + } + ] +} diff --git a/tools/perf_smoke_test/tasks_to_ci_matrix.py b/tools/perf_smoke_test/tasks_to_ci_matrix.py new file mode 100644 index 000000000000..6e0f5661c277 --- /dev/null +++ b/tools/perf_smoke_test/tasks_to_ci_matrix.py @@ -0,0 +1,43 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Convert tasks.json into the GitHub Actions bench matrix JSON + +Prints a JSON array to stdout, one object per (task_id, backend) combination, +containing the fields consumed by the ``bench`` job matrix in perf-smoke-test.yaml. + +Usage:: + + python3 tools/perf_smoke_test/tasks_to_ci_matrix.py +""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from launch_config import hydra_args_for_task # noqa: E402 +from task_config import load_tasks # noqa: E402 + +tasks = load_tasks() +rows = [] +for task in tasks: + rows.append( + { + "task_id": task.task_id, + "physics_backend": task.physics_backend, + "render_backend": task.render_backend or "", + "num_envs": task.num_envs, + "num_frames": task.num_frames, + "warmup_frames": task.warmup_frames, + "seed": task.seed if task.seed is not None else "", + "hydra_args": " ".join(hydra_args_for_task(task)), + "bench_timeout_s": task.timeout_minutes * 60, + "job_timeout_minutes": max(30, task.timeout_minutes + 15), + } + ) + +print(json.dumps(rows)) diff --git a/tools/perf_smoke_test/test/test_advisory_exit.py b/tools/perf_smoke_test/test/test_advisory_exit.py new file mode 100644 index 000000000000..e226f9c4701e --- /dev/null +++ b/tools/perf_smoke_test/test/test_advisory_exit.py @@ -0,0 +1,411 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""The advisory/blocking exit contract, and diagnostics after a failed benchmark. + +``gate_config.blocking`` decides whether the aggregate job's exit code carries +the verdict. In advisory mode it never does: every verdict, including +HARD_FAILURE, exits 0, and the verdict travels through the ``perf-smoke-test`` +commit status, the sticky PR comment and the job summary instead. That is what +keeps an unrelated pull request from getting a red check because a registry +outage killed the benchmark. + +What advisory mode must *not* do is go quiet. These tests pin both halves: exit +0, and a verdict that still says HARD_FAILURE out loud, with a summary a +developer can read. + +Gate malfunctions are the exception and stay fatal in both modes -- if aggregate +found no bench artifacts at all it produced no verdict, and its owners need to +see that. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +_GATE_DIR = Path(__file__).resolve().parents[1] +if str(_GATE_DIR) not in sys.path: + sys.path.insert(0, str(_GATE_DIR)) + +import aggregate # noqa: E402 +from contracts import BenchResult # noqa: E402 + +_RUNTIME_HASH = "runtime-a" + + +def _bench_result(*, fps: float | None, info_present: bool, exit_code: int = 0, failure_phase: str | None = None): + launch_config = { + "task_id": "Isaac-Cartpole-Direct", + "backend": "physx", + "backend_key": "physx", + "physics_backend": "physx", + "render_backend": None, + "gpu_model": "l40s", + "launch_config_hash": "launch-a", + "benchmark_contract_hash": "bench-a", + "baseline_epoch": 1, + } + return BenchResult( + task_id="Isaac-Cartpole-Direct", + backend="physx", + physics_backend="physx", + render_backend=None, + backend_key="physx", + preset="default", + was_retried=False, + stdout_tail="", + perf_smoke_test_info_present=info_present, + raw_fps_mean=fps, + exit_code=exit_code, + failure_phase=failure_phase, + runtime_contract_hash=_RUNTIME_HASH, + runtime_resources={"gpu_name": "NVIDIA L40S"}, + provenance={"software": {"isaaclab": "3.0.0"}}, + launch_config=launch_config, + launch_config_hash="launch-a", + benchmark_contract_hash="bench-a", + baseline_epoch=1, + ) + + +def _run(tmp_path: Path, monkeypatch, *, blocking: bool, bench_result=None, write_artifact: bool = True): + """Drive aggregate.main() in flat-file mode; return (exit_code, outputs, summary).""" + artifacts_dir = tmp_path / "artifacts" + baselines_dir = tmp_path / "baselines" + baselines_dir.mkdir(parents=True, exist_ok=True) + artifacts_dir.mkdir(parents=True, exist_ok=True) + summary_file = tmp_path / "verdict_summary.md" + output_file = tmp_path / "gh_output.txt" + + gate_config = tmp_path / "gate_config.json" + gate_config.write_text(json.dumps({"blocking": blocking}), encoding="utf-8") + + # These cases exercise one bucket, so declare a one-bucket matrix. Otherwise + # the completeness check correctly reports 1-of-9 and masks what is under + # test here. The check itself is covered directly further down. + monkeypatch.setattr(aggregate, "load_tasks", lambda *a, **k: [object()]) + + if write_artifact: + task_dir = artifacts_dir / "bench-Isaac-Cartpole-Direct-physx" + task_dir.mkdir(parents=True, exist_ok=True) + (task_dir / "perf_smoke_test_result.json").write_text(json.dumps(bench_result.to_dict())) + + monkeypatch.setenv("GITHUB_OUTPUT", str(output_file)) + monkeypatch.setattr( + sys, + "argv", + [ + "aggregate.py", + "--artifacts_dir", + str(artifacts_dir), + "--gpu_model", + "L40S", + "--baselines_dir", + str(baselines_dir), + "--allow_baseline_update", + "false", + "--gate_config", + str(gate_config), + "--summary_file", + str(summary_file), + ], + ) + exit_code = aggregate.main() + + outputs: dict[str, str] = {} + if output_file.exists(): + for line in output_file.read_text().splitlines(): + if "=" in line: + key, _, value = line.partition("=") + outputs[key] = value + summary = summary_file.read_text(encoding="utf-8") if summary_file.exists() else "" + return exit_code, outputs, summary + + +# --- advisory mode: never fails the job, always reports -------------------- + + +@pytest.mark.parametrize( + "bench_kwargs", + [ + pytest.param({"fps": None, "info_present": False}, id="no-measurement"), + pytest.param({"fps": 0.0, "info_present": True}, id="zero-fps"), + pytest.param( + {"fps": 100.0, "info_present": True, "exit_code": 1, "failure_phase": "import"}, + id="crashed-after-writing-a-bundle", + ), + ], +) +def test_advisory_mode_exits_zero_on_hard_failure(tmp_path, monkeypatch, bench_kwargs) -> None: + """A crashed benchmark must not fail the check while the gate is advisory.""" + exit_code, outputs, _ = _run(tmp_path, monkeypatch, blocking=False, bench_result=_bench_result(**bench_kwargs)) + + assert exit_code == 0, "advisory mode must not fail the job on a benchmark failure" + assert outputs.get("overall_verdict") == "HARD_FAILURE" + + +def test_advisory_mode_still_reports_the_failure(tmp_path, monkeypatch) -> None: + """Advisory must mean 'does not fail the PR', never 'says nothing'.""" + _, outputs, summary = _run( + tmp_path, monkeypatch, blocking=False, bench_result=_bench_result(fps=None, info_present=False) + ) + + # The commit status is the signal that survives a green job. + assert outputs.get("status_state") == "failure" + assert outputs.get("blocking") == "false" + assert "advisory" in outputs.get("status_description", "").lower() + + # And the summary a developer actually reads must name the failure. + assert summary, "a summary must be written even when the benchmark failed" + assert "HARD FAILURE" in summary or "HARD_FAILURE" in summary + assert "Isaac-Cartpole-Direct" in summary + + +def test_advisory_mode_is_green_and_quiet_on_a_clean_run(tmp_path, monkeypatch) -> None: + """A healthy run reports success, so the red status stays meaningful.""" + exit_code, outputs, summary = _run( + tmp_path, monkeypatch, blocking=False, bench_result=_bench_result(fps=100.0, info_present=True) + ) + + assert exit_code == 0 + assert outputs.get("status_state") == "success" + # No baseline yet, so the honest verdict is WARN (not a silent PASS). + assert outputs.get("overall_verdict") == "WARN" + assert summary + + +# --- blocking mode: the exit code carries the verdict ---------------------- + + +def test_blocking_mode_fails_on_hard_failure(tmp_path, monkeypatch) -> None: + """Flipping blocking:true is what makes a crashed benchmark fail the job.""" + exit_code, outputs, _ = _run( + tmp_path, monkeypatch, blocking=True, bench_result=_bench_result(fps=None, info_present=False) + ) + + assert exit_code == 2 + assert outputs.get("overall_verdict") == "HARD_FAILURE" + assert outputs.get("blocking") == "true" + + +def test_blocking_mode_passes_a_healthy_run(tmp_path, monkeypatch) -> None: + exit_code, outputs, _ = _run( + tmp_path, monkeypatch, blocking=True, bench_result=_bench_result(fps=100.0, info_present=True) + ) + + assert exit_code == 0 + assert outputs.get("status_state") == "success" + + +# --- gate malfunctions stay fatal in both modes ---------------------------- + + +@pytest.mark.parametrize("blocking", [False, True], ids=["advisory", "blocking"]) +def test_missing_artifacts_fail_in_both_modes(tmp_path, monkeypatch, blocking: bool) -> None: + """No bench artifacts at all means no verdict was produced -- that is a gate fault.""" + exit_code, _outputs, _summary = _run(tmp_path, monkeypatch, blocking=blocking, write_artifact=False) + + assert exit_code == 1, "a gate that produced no verdict must fail regardless of advisory mode" + + +# --- the verdict must come from the rows, never from the flags alone ------- +# +# has_hard_failure is cleared for crashes excused as CI-image skew, and main() +# only bails when there are ZERO artifacts. Deriving the reported verdict from +# those two booleans therefore produced an affirmative "no meaningful +# performance regression detected" for a run in which nine buckets crashed and +# nothing was measured -- the exact silent-green this gate exists to prevent. + +from gate_types import OracleVerdict # noqa: E402 + + +def _rows(*verdicts): + return [(SimpleNamespace(verdict=v), None) for v in verdicts] + + +def test_skew_excused_crashes_do_not_report_pass() -> None: + """Nine crashed-but-excused buckets must not read as 'no regression'.""" + out = aggregate._verdict_outputs( + _rows(*([OracleVerdict.HARD_FAILURE] * 9)), + has_block=False, + has_hard_failure=False, # cleared by detect_dependency_skew + blocking=False, + missing=[], + expected_total=9, + ) + + assert out["overall_verdict"] == "HARD_FAILURE" + assert out["status_state"] == "failure" + assert "no meaningful" not in out["status_description"] + assert "stale" in out["status_description"] + + +def test_missing_buckets_are_not_graded_on_the_survivors() -> None: + """One passing bucket out of nine is not a pass.""" + out = aggregate._verdict_outputs( + _rows(OracleVerdict.PASS), + has_block=False, + has_hard_failure=False, + blocking=False, + missing=[f"Task-{i}/physx" for i in range(8)], + expected_total=9, + ) + + assert out["overall_verdict"] == "HARD_FAILURE" + assert out["status_state"] == "failure" + assert "only 1 of 9" in out["status_description"] + + +def test_a_complete_clean_run_still_passes() -> None: + """The guard must not swallow a genuine pass.""" + out = aggregate._verdict_outputs( + _rows(*([OracleVerdict.PASS] * 9)), + has_block=False, + has_hard_failure=False, + blocking=False, + missing=[], + expected_total=9, + ) + + assert out["overall_verdict"] == "PASS" + assert out["status_state"] == "success" + assert "9 buckets" in out["status_description"] + + +def test_unreadable_matrix_disables_the_completeness_check() -> None: + """An unreadable tasks.json must never invent a failure.""" + out = aggregate._verdict_outputs( + _rows(*([OracleVerdict.PASS] * 3)), + has_block=False, + has_hard_failure=False, + blocking=False, + missing=[], + expected_total=0, + ) + + assert out["overall_verdict"] == "PASS" + + +# --- the comment and the commit status must never disagree ---------------- +# +# They are produced by different code paths: the sticky comment and job summary +# come from _build_summary_markdown, the status from _verdict_outputs. An +# earlier fix taught only the latter about missing buckets, so a run with 8 of 9 +# buckets reported showed a red status reading "only 8 of 9 buckets reported" +# directly above a comment headlined "No meaningful performance regressions +# detected -- 0 benchmark failures". These drive the real aggregate.main() over +# the real tasks.json and compare both surfaces. + +import dataclasses # noqa: E402 + +from task_config import load_tasks # noqa: E402 + + +def _run_real_matrix(tmp_path: Path, monkeypatch, *, reported: int): + """Run aggregate over `reported` of the 9 real matrix buckets.""" + buckets = [(t.task_id, t.backend_key) for t in load_tasks()] + artifacts_dir = tmp_path / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + (tmp_path / "baselines").mkdir(parents=True, exist_ok=True) + summary_file = tmp_path / "verdict_summary.md" + output_file = tmp_path / "gh_output.txt" + gate_config = tmp_path / "gate_config.json" + gate_config.write_text(json.dumps({"blocking": False}), encoding="utf-8") + + for i, (task_id, backend_key) in enumerate(buckets[:reported]): + bench = dataclasses.replace( + _bench_result(fps=100.0, info_present=True), task_id=task_id, backend=backend_key, backend_key=backend_key + ) + task_dir = artifacts_dir / f"bench-{i}" + task_dir.mkdir(parents=True, exist_ok=True) + (task_dir / "perf_smoke_test_result.json").write_text(json.dumps(bench.to_dict())) + + monkeypatch.setenv("GITHUB_OUTPUT", str(output_file)) + monkeypatch.setattr( + sys, + "argv", + [ + "aggregate.py", + "--artifacts_dir", str(artifacts_dir), + "--gpu_model", "L40S", + "--baselines_dir", str(tmp_path / "baselines"), + "--allow_baseline_update", "false", + "--gate_config", str(gate_config), + "--summary_file", str(summary_file), + ], + ) # fmt: skip + aggregate.main() + outputs = dict(line.split("=", 1) for line in output_file.read_text().splitlines() if "=" in line) + return outputs, summary_file.read_text(encoding="utf-8"), len(buckets) + + +def test_partial_matrix_is_flagged_on_both_surfaces(tmp_path, monkeypatch) -> None: + """A red status must never sit above an all-clear comment.""" + total_reported = 8 + outputs, summary, total = _run_real_matrix(tmp_path, monkeypatch, reported=total_reported) + + assert outputs["status_state"] == "failure" + assert f"only {total_reported} of {total}" in outputs["status_description"] + # The comment is what a reviewer reads, so it must carry the same message. + assert f"Only {total_reported} of {total}" in summary + assert "No meaningful performance regressions detected" not in summary + # And it must name the bucket that vanished, not just count it. + assert "did not report" in summary + + +def test_complete_matrix_is_clean_on_both_surfaces(tmp_path, monkeypatch) -> None: + """The coverage guard must not fire when every bucket reported.""" + outputs, summary, total = _run_real_matrix(tmp_path, monkeypatch, reported=9) + + assert outputs["status_state"] == "success" + assert "did not report" not in summary + assert "coverage is incomplete" not in summary + + +def test_a_real_block_is_not_misreported_as_a_stale_image() -> None: + """A BLOCK alongside a skew-excused crash must still be named.""" + rows = _rows(OracleVerdict.HARD_FAILURE, OracleVerdict.BLOCK) + out = aggregate._verdict_outputs( + rows, + has_block=True, + has_hard_failure=False, # cleared by detect_dependency_skew + blocking=False, + missing=[], + expected_total=2, + ) + + assert out["status_state"] == "failure" + assert "stale" in out["status_description"] + assert "blocking-level regression was also detected" in out["status_description"] + + +def test_a_real_block_is_not_hidden_behind_a_bucket_shortfall() -> None: + """A BLOCK alongside missing buckets must still be named.""" + out = aggregate._verdict_outputs( + _rows(OracleVerdict.BLOCK), + has_block=True, + has_hard_failure=False, + blocking=False, + missing=["Task-X/physx"], + expected_total=2, + ) + + assert "only 1 of 2" in out["status_description"] + assert "blocking-level regression was also detected" in out["status_description"] + + +def test_a_lone_block_is_not_double_reported() -> None: + """The additive note must not duplicate when BLOCK is already the verdict.""" + out = aggregate._verdict_outputs( + _rows(OracleVerdict.BLOCK), has_block=True, has_hard_failure=False, blocking=False, missing=[], expected_total=1 + ) + + assert out["overall_verdict"] == "BLOCK" + assert out["status_description"].count("blocking-level") == 1 diff --git a/tools/perf_smoke_test/test/test_aggregate_reseed.py b/tools/perf_smoke_test/test/test_aggregate_reseed.py new file mode 100644 index 000000000000..5841ed7d0d4d --- /dev/null +++ b/tools/perf_smoke_test/test/test_aggregate_reseed.py @@ -0,0 +1,357 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""GPU-free checks for aggregate's post-gate reporting decisions. + +Two concerns are covered. The under-filled-bucket detector flags a task for +targeted reseeding when this run produced a valid measurement but the matching +baseline window for its exact runtime_contract_hash has fewer than +MIN_BASELINE_SAMPLES samples; those tests drive aggregate.main() in flat-file +mode (no git) and assert the reseed_tasks GITHUB_OUTPUT signal the gate's reseed +job consumes. The skew detector decides whether a crash is blamed on the change +under test or on an out-of-date CI image. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +_GATE_DIR = Path(__file__).resolve().parents[1] +if str(_GATE_DIR) not in sys.path: + sys.path.insert(0, str(_GATE_DIR)) + +import aggregate # noqa: E402 +from baseline_manager import update_baseline # noqa: E402 +from contracts import BenchResult # noqa: E402 +from gate_config import MIN_BASELINE_SAMPLES # noqa: E402 +from gate_types import OracleVerdict # noqa: E402 + +_RUNTIME_HASH = "runtime-a" + + +def _bench_result(*, fps: float | None = 100.0, info_present: bool = True, stdout_tail: str = "") -> BenchResult: + launch_config = { + "task_id": "Isaac-Cartpole-Direct", + "backend": "physx", + "backend_key": "physx", + "physics_backend": "physx", + "render_backend": None, + "gpu_model": "l40s", + "launch_config_hash": "launch-a", + "benchmark_contract_hash": "bench-a", + "baseline_epoch": 1, + } + return BenchResult( + task_id="Isaac-Cartpole-Direct", + backend="physx", + physics_backend="physx", + render_backend=None, + backend_key="physx", + preset="default", + was_retried=False, + stdout_tail=stdout_tail, + perf_smoke_test_info_present=info_present, + raw_fps_mean=fps, + raw_fps_std=1.0 if fps is not None else None, + raw_fps_min=(fps - 1.0) if fps is not None else None, + raw_fps_max=(fps + 1.0) if fps is not None else None, + runtime_contract_hash=_RUNTIME_HASH, + runtime_resources={"gpu_name": "NVIDIA L40S"}, + provenance={"software": {"isaaclab": "3.0.0", "warp": "1.13.0"}}, + launch_config=launch_config, + launch_config_hash="launch-a", + benchmark_contract_hash="bench-a", + baseline_epoch=1, + ) + + +def _write_artifact(artifacts_dir: Path, bench_result: BenchResult) -> None: + task_dir = artifacts_dir / "bench-Isaac-Cartpole-Direct-physx" + task_dir.mkdir(parents=True, exist_ok=True) + (task_dir / "perf_smoke_test_result.json").write_text(json.dumps(bench_result.to_dict())) + + +def _seed_flat_baseline(baselines_dir: Path, count: int) -> None: + for i in range(count): + update_baseline( + baselines_dir, + "l40s", + "Isaac-Cartpole-Direct", + "physx", + 100.0, + sample_metadata={ + "gpu_model": "l40s", + "task_id": "Isaac-Cartpole-Direct", + "backend_key": "physx", + "launch_config_hash": "launch-a", + "benchmark_contract_hash": "bench-a", + "runtime_contract_hash": _RUNTIME_HASH, + "baseline_epoch": 1, + "fps": 100.0, + "sample_id": f"seed-{i}", + }, + ) + + +def _run_aggregate( + tmp_path: Path, monkeypatch, bench_result: BenchResult, baseline_count: int +) -> tuple[int, dict[str, str]]: + artifacts_dir = tmp_path / "artifacts" + baselines_dir = tmp_path / "baselines" + output_file = tmp_path / "gh_output.txt" + gate_config = tmp_path / "gate_config.json" + gate_config.write_text('{"blocking": false}', encoding="utf-8") + _write_artifact(artifacts_dir, bench_result) + if baseline_count: + _seed_flat_baseline(baselines_dir, baseline_count) + else: + baselines_dir.mkdir(parents=True, exist_ok=True) + + monkeypatch.setenv("GITHUB_OUTPUT", str(output_file)) + monkeypatch.setattr( + sys, + "argv", + [ + "aggregate.py", + "--artifacts_dir", + str(artifacts_dir), + "--gpu_model", + "L40S", + "--baselines_dir", + str(baselines_dir), + "--allow_baseline_update", + "false", + "--gate_config", + str(gate_config), + ], + ) + exit_code = aggregate.main() + + outputs: dict[str, str] = {} + if output_file.exists(): + for line in output_file.read_text().splitlines(): + if "=" in line: + key, _, value = line.partition("=") + outputs[key] = value + return exit_code, outputs + + +def test_reseed_flagged_when_no_baseline(tmp_path, monkeypatch) -> None: + """A valid measurement with zero matching samples opens a bucket that needs reseeding.""" + exit_code, outputs = _run_aggregate(tmp_path, monkeypatch, _bench_result(fps=100.0), baseline_count=0) + + assert exit_code == 0 + assert outputs.get("reseed_tasks") == "Isaac-Cartpole-Direct" + assert outputs.get("reseed_min_samples") == str(MIN_BASELINE_SAMPLES) + + +def test_reseed_flagged_when_window_insufficient(tmp_path, monkeypatch) -> None: + """Fewer than MIN_BASELINE_SAMPLES matching samples still counts as under-filled.""" + exit_code, outputs = _run_aggregate( + tmp_path, monkeypatch, _bench_result(fps=100.0), baseline_count=MIN_BASELINE_SAMPLES - 1 + ) + + assert exit_code == 0 + assert outputs.get("reseed_tasks") == "Isaac-Cartpole-Direct" + + +def test_reseed_not_flagged_when_bucket_full(tmp_path, monkeypatch) -> None: + """A fully populated bucket must not trigger a reseed.""" + exit_code, outputs = _run_aggregate( + tmp_path, monkeypatch, _bench_result(fps=100.0), baseline_count=MIN_BASELINE_SAMPLES + ) + + assert exit_code == 0 + assert "reseed_tasks" not in outputs + + +def test_reseed_not_flagged_without_valid_measurement(tmp_path, monkeypatch) -> None: + """A crashed run (no measurement) must not reseed even with an empty bucket.""" + exit_code, outputs = _run_aggregate( + tmp_path, monkeypatch, _bench_result(fps=None, info_present=False), baseline_count=0 + ) + + # Advisory mode (the shipped default): the crash is reported through the + # verdict outputs, not through the exit code. See test_advisory_exit.py. + assert exit_code == 0 + assert outputs.get("overall_verdict") == "HARD_FAILURE" + assert outputs.get("status_state") == "failure" + assert "reseed_tasks" not in outputs + + +# Verbatim from the 2026-07-28 staging run, where source pinning a newer Newton +# met an image built before SolverNotifyFlags was replaced by ModelFlags. +_STALE_IMAGE_LOG = """ +Traceback (most recent call last): + File "/workspace/isaaclab/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py", line 21, in + from newton.solvers import SolverNotifyFlags +ImportError: cannot import name 'SolverNotifyFlags' from 'newton.solvers' +""" + + +def test_missing_newton_symbol_is_reported_as_stale_image() -> None: + """The crash that broke the last qualification run must be recognized.""" + skew = aggregate.detect_dependency_skew(_STALE_IMAGE_LOG) + + assert skew is not None + assert skew.package == "newton" + assert skew.symbol == "SolverNotifyFlags" + assert "no `SolverNotifyFlags`" in skew.describe() + + +def test_missing_image_module_is_reported_as_stale_image() -> None: + """A package the image should provide but does not is also image skew.""" + skew = aggregate.detect_dependency_skew("ModuleNotFoundError: No module named 'newton.solvers.kamino'") + + assert skew is not None + assert skew.package == "newton" + assert skew.symbol is None + assert "is not installed" in skew.describe() + + +def test_missing_module_attribute_is_reported_as_stale_image() -> None: + """Warp exposing no such attribute means the installed Warp predates the pin.""" + skew = aggregate.detect_dependency_skew("AttributeError: module 'warp' has no attribute 'sparse_matmul'") + + assert skew is not None + assert skew.package == "warp" + assert skew.symbol == "sparse_matmul" + + +def test_missing_isaaclab_symbol_is_not_excused() -> None: + """Isaac Lab source is mounted from the PR, so its own broken import is a real defect.""" + log = "ImportError: cannot import name 'ArticulationCfg' from 'isaaclab.assets'" + + assert aggregate.detect_dependency_skew(log) is None + + +def test_clean_log_reports_no_skew() -> None: + """A run that never raised an import error is not skew.""" + assert aggregate.detect_dependency_skew("Step Frametimes: 3.1 3.0 3.2") is None + assert aggregate.detect_dependency_skew(None) is None + + +def test_crash_from_stale_image_does_not_fail_the_gate(tmp_path, monkeypatch) -> None: + """An image missing a symbol this source pins is not the PR's fault, so it stays advisory.""" + bench_result = _bench_result(fps=None, info_present=False, stdout_tail=_STALE_IMAGE_LOG) + + exit_code, outputs = _run_aggregate(tmp_path, monkeypatch, bench_result, baseline_count=0) + + assert exit_code == 0 + assert "reseed_tasks" not in outputs + + +def test_stale_image_is_explained_in_the_sticky_comment() -> None: + """Reviewers are told the image is stale rather than left reading a bare crash.""" + result = SimpleNamespace( + task_id="Isaac-Cartpole-Direct", + backend="newton", + verdict=OracleVerdict.HARD_FAILURE, + measured_fps=None, + baseline_fps=None, + regression_pct=None, + baseline_sample_count=0, + threshold_source="no_baseline", + hard_floor_fps=None, + failure_phase="import", + was_retried=False, + note=None, + crossed_thresholds=[], + ) + bench_result = _bench_result(fps=None, info_present=False, stdout_tail=_STALE_IMAGE_LOG) + + summary = aggregate._build_summary_markdown([(result, bench_result)], blocking=True) + + assert "### Stale CI image" in summary + assert "The CI image is stale for this PR" in summary + assert "advisory and do not fail the check" in summary + assert "no `SolverNotifyFlags`" in summary + # The failure is surfaced, just not attributed to the change under review. + assert "failed before producing usable performance data" not in summary + + +def test_genuine_crash_still_fails_and_is_not_excused() -> None: + """A crash with no image-skew signature keeps its hard-failure framing.""" + result = SimpleNamespace( + task_id="Isaac-Cartpole-Direct", + backend="newton", + verdict=OracleVerdict.HARD_FAILURE, + measured_fps=None, + baseline_fps=None, + regression_pct=None, + baseline_sample_count=0, + threshold_source="no_baseline", + hard_floor_fps=None, + failure_phase="runtime", + was_retried=False, + note=None, + crossed_thresholds=[], + ) + bench_result = _bench_result(fps=None, info_present=False, stdout_tail="RuntimeError: CUDA out of memory") + + summary = aggregate._build_summary_markdown([(result, bench_result)], blocking=True) + + assert "### Stale CI image" not in summary + assert "failed before producing usable performance data" in summary + + +def test_sticky_summary_explains_results_in_reviewer_language() -> None: + """The sticky comment leads with actionable guidance and keeps diagnostics available.""" + result = SimpleNamespace( + task_id="Isaac-Cartpole-Direct", + backend="physx", + verdict=OracleVerdict.BLOCK, + measured_fps=90.0, + baseline_fps=100.0, + regression_pct=-10.0, + baseline_sample_count=5, + threshold_source="rolling_window", + hard_floor_fps=None, + failure_phase=None, + was_retried=True, + note="block_confirmed(n=1)", + crossed_thresholds=[], + ) + + summary = aggregate._build_summary_markdown([(result, _bench_result(fps=90.0))], blocking=False) + + assert "### Overall result" in summary + assert "blocking-level performance regressions were detected" in summary + assert "Advisory" in summary + assert "### How to read this" in summary + assert "Start with **BLOCK** and **HARD FAILURE**" in summary + assert "| Isaac-Cartpole-Direct | physx | ๐Ÿšซ BLOCK | 90.0 | 100.0 | -10.00% | 5 |" in summary + assert "Blocking-level slowdown detected; result was retried" in summary + assert "passed only after retry" not in summary + assert "Technical details" in summary + assert "block_confirmed(n=1)" in summary + + +def test_sticky_summary_identifies_baseline_warmup() -> None: + """WARN rows clearly distinguish baseline collection from a regression.""" + result = SimpleNamespace( + task_id="Isaac-Cartpole-Direct", + backend="newton", + verdict=OracleVerdict.WARN, + measured_fps=100.0, + baseline_fps=None, + regression_pct=None, + baseline_sample_count=2, + threshold_source="insufficient_window", + hard_floor_fps=None, + failure_phase=None, + was_retried=False, + note="insufficient baseline samples", + crossed_thresholds=[], + ) + + summary = aggregate._build_summary_markdown([(result, _bench_result())], blocking=True) + + assert "No confirmed regression" in summary + assert "Baseline warming up (2/5 samples)" in summary + assert "**Blocking:** BLOCK and HARD FAILURE results fail the check." in summary diff --git a/tools/perf_smoke_test/test/test_backend_identity.py b/tools/perf_smoke_test/test/test_backend_identity.py new file mode 100644 index 000000000000..377c10f1f9f5 --- /dev/null +++ b/tools/perf_smoke_test/test/test_backend_identity.py @@ -0,0 +1,64 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""GPU-free unit tests for backend identity normalization.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_GATE_DIR = Path(__file__).resolve().parents[1] +if str(_GATE_DIR) not in sys.path: + sys.path.insert(0, str(_GATE_DIR)) + +from backend_identity import identity_from_presets # noqa: E402 +from launch_config import hydra_args_for_task # noqa: E402 +from task_config import load_tasks # noqa: E402 + + +def test_identity_from_presets_treats_newton_as_newton() -> None: + """The 6.0 prerelease task presets report ``newton`` instead of ``newton_mjwarp``.""" + identity = identity_from_presets("newton") + + assert identity is not None + assert identity.backend_key == "newton" + + +def test_identity_from_presets_keeps_newton_mjwarp_compatibility() -> None: + """The newer preset spelling remains supported.""" + identity = identity_from_presets("cube,newton_mjwarp") + + assert identity is not None + assert identity.backend_key == "newton" + + +def test_camera_tasks_explicitly_identify_their_rtx_renderer() -> None: + """Camera buckets match the RTX renderer that the task activates by default.""" + camera_task = "Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct" + backend_keys = {task.backend_key for task in load_tasks() if task.task_id == camera_task} + + assert backend_keys == { + "physx_rtx_renderer", + "physx_newton_renderer", + "newton_rtx_renderer", + "newton_newton_renderer", + } + + +def test_rtx_camera_tasks_use_the_supported_isaac_sim_preset() -> None: + """Canonical RTX identity maps to the Hydra preset exposed by Isaac Lab.""" + camera_task = "Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct" + rtx_tasks = [task for task in load_tasks() if task.task_id == camera_task and task.render_backend == "rtx_renderer"] + + assert {tuple(hydra_args_for_task(task)) for task in rtx_tasks} == { + ("presets=isaacsim_rtx",), + ("presets=newton_mjwarp,isaacsim_rtx",), + } + for task in rtx_tasks: + preset_value = hydra_args_for_task(task)[0].split("=", 1)[1] + identity = identity_from_presets(preset_value) + assert identity is not None + assert identity.backend_key == task.backend_key diff --git a/tools/perf_smoke_test/test/test_contract_oracle_baseline.py b/tools/perf_smoke_test/test/test_contract_oracle_baseline.py new file mode 100644 index 000000000000..36158a1766d2 --- /dev/null +++ b/tools/perf_smoke_test/test/test_contract_oracle_baseline.py @@ -0,0 +1,287 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""GPU-free checks for the typed perf-smoke result and oracle path.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +_GATE_DIR = Path(__file__).resolve().parents[1] +if str(_GATE_DIR) not in sys.path: + sys.path.insert(0, str(_GATE_DIR)) + +from baseline_manager import ( # noqa: E402 + load_baseline, + make_sample_metadata, + match_context_from_bench_result, + update_baseline, +) +from contracts import CONTRACT_SCHEMA_VERSION, BenchResult # noqa: E402 +from gate_types import FpsMeanThreshold, OracleVerdict, ThresholdSource # noqa: E402 +from hashing import stable_hash # noqa: E402 +from oracle import Baseline, compare # noqa: E402 + + +def _bench_result( + *, + fps: float | None = 100.0, + info_present: bool = True, + was_retried: bool = False, + exit_code: int = 0, + failure_phase: str | None = None, +) -> BenchResult: + """Build the minimal typed result shape used by oracle and baseline tests.""" + launch_config = { + "task_id": "Isaac-Cartpole-Direct", + "backend": "physx", + "backend_key": "physx", + "physics_backend": "physx", + "render_backend": None, + "gpu_model": "l40s", + "launch_config_hash": "launch-a", + "benchmark_contract_hash": "bench-a", + "baseline_epoch": 1, + } + return BenchResult( + task_id="Isaac-Cartpole-Direct", + backend="physx", + physics_backend="physx", + render_backend=None, + backend_key="physx", + preset="default", + was_retried=was_retried, + perf_smoke_test_info_present=info_present, + raw_fps_mean=fps, + raw_fps_std=1.0 if fps is not None else None, + raw_fps_min=(fps - 1.0) if fps is not None else None, + raw_fps_max=(fps + 1.0) if fps is not None else None, + runtime_contract_hash="runtime-a", + runtime_resources={"gpu_name": "NVIDIA L40S", "gpu_mem_used_mb": 1024.0}, + provenance={"software": {"isaaclab": "3.0.0", "warp": "1.13.0"}}, + launch_config=launch_config, + launch_config_hash="launch-a", + benchmark_contract_hash="bench-a", + baseline_epoch=1, + exit_code=exit_code, + failure_phase=failure_phase, + ) + + +def test_bench_result_round_trips_with_schema_version() -> None: + """Typed results serialize to the stable wire shape and reconstruct strictly.""" + result = _bench_result(fps=123.0) + payload = result.to_dict() + payload["ignored_future_field"] = "allowed" + + restored = BenchResult.from_dict(payload) + + assert restored.schema_version == CONTRACT_SCHEMA_VERSION + assert restored.raw_fps_mean == 123.0 + assert not hasattr(restored, "ignored_future_field") + + +def test_bench_result_requires_identity_fields() -> None: + """The typed schema catches producer bugs that omit required identity fields.""" + payload = _bench_result().to_dict() + payload.pop("task_id") + + with pytest.raises(TypeError): + BenchResult.from_dict(payload) + + +def test_stable_hash_is_key_order_independent() -> None: + """Contract hashing must not depend on dict insertion order.""" + left = {"runtime": {"warp": "1.13.0", "torch": "2.9.0"}, "backend": "physx"} + right = {"backend": "physx", "runtime": {"torch": "2.9.0", "warp": "1.13.0"}} + + assert stable_hash(left) == stable_hash(right) + + +def test_seed_sample_index_prevents_identical_fps_deduplication(monkeypatch) -> None: + """Repeated samples remain unique even when they report identical FPS.""" + monkeypatch.setenv("GITHUB_RUN_ID", "123") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") + bench_result = _bench_result(fps=100.0).to_dict() + + first = make_sample_metadata( + gpu_model="l40s", + task_id="Isaac-Cartpole-Direct", + backend="physx", + fps=100.0, + bench_result=bench_result, + target_branch="develop", + commit_sha="abc123", + sample_index=0, + ) + second = make_sample_metadata( + gpu_model="l40s", + task_id="Isaac-Cartpole-Direct", + backend="physx", + fps=100.0, + bench_result=bench_result, + target_branch="develop", + commit_sha="abc123", + sample_index=1, + ) + + assert first["sample_id"] != second["sample_id"] + + +def test_sample_metadata_records_runner_allocation_identity(monkeypatch) -> None: + """Stability evidence identifies both the matrix allocation and runner registration.""" + monkeypatch.setenv("GITHUB_RUN_ID", "123") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") + monkeypatch.setenv("PERF_SMOKE_RUN_LABEL", "runner-allocation-3") + monkeypatch.setenv("PERF_SMOKE_RUNNER_NAME", "l40s-runner-03") + + first = make_sample_metadata( + gpu_model="l40s", + task_id="Isaac-Cartpole-Direct", + backend="physx", + fps=100.0, + bench_result=_bench_result(fps=100.0).to_dict(), + target_branch="develop", + commit_sha="abc123", + sample_index=0, + ) + monkeypatch.setenv("PERF_SMOKE_RUN_LABEL", "runner-allocation-4") + second = make_sample_metadata( + gpu_model="l40s", + task_id="Isaac-Cartpole-Direct", + backend="physx", + fps=100.0, + bench_result=_bench_result(fps=100.0).to_dict(), + target_branch="develop", + commit_sha="abc123", + sample_index=0, + ) + + assert first["ci_run_label"] == "runner-allocation-3" + assert first["ci_runner_name"] == "l40s-runner-03" + assert second["ci_run_label"] == "runner-allocation-4" + assert first["sample_id"] != second["sample_id"] + + +def test_oracle_pass_warn_and_block_from_typed_result() -> None: + """The post-migration oracle still scores typed BenchResult objects correctly.""" + baseline = Baseline(median_fps=100.0, mad_fps=2.0, sample_count=5) + + pass_result = compare(_bench_result(fps=100.0), baseline, []) + warn_result = compare(_bench_result(fps=94.0), baseline, []) + block_result = compare(_bench_result(fps=88.0), baseline, []) + + assert pass_result.verdict == OracleVerdict.PASS + assert warn_result.verdict == OracleVerdict.WARN + assert warn_result.threshold_source == ThresholdSource.ROLLING_WINDOW.value + assert block_result.verdict == OracleVerdict.BLOCK + assert block_result.regression_pct == pytest.approx(-12.0) + + +def test_oracle_threshold_floor_can_block_without_baseline() -> None: + """Configured hard floors remain gating even before rolling baselines exist.""" + threshold = FpsMeanThreshold(name="hard-floor", value=50.0, verdict=OracleVerdict.BLOCK) + + result = compare(_bench_result(fps=40.0), baseline=None, fps_mean_thresholds=[threshold]) + + assert result.verdict == OracleVerdict.BLOCK + assert result.threshold_source == ThresholdSource.THRESHOLD.value + assert result.hard_floor_fps == 50.0 + + +def test_oracle_hard_fails_missing_benchmark_info() -> None: + """A missing runtime bundle stays a hard failure instead of looking like no baseline.""" + result = compare(_bench_result(fps=None, info_present=False), baseline=None, fps_mean_thresholds=[]) + + assert result.verdict == OracleVerdict.HARD_FAILURE + assert result.measured_fps is None + + +def test_oracle_hard_fails_a_run_that_died_after_writing_its_bundle() -> None: + """A healthy FPS number does not redeem a run that exited nonzero. + + A benchmark can write a complete bundle and then crash on teardown. Scoring + that on FPS alone would return PASS and make the sample eligible for the + baseline, so the gate would learn its window from a run that failed. + """ + baseline = Baseline(median_fps=100.0, mad_fps=2.0, sample_count=5) + + result = compare(_bench_result(fps=100.0, exit_code=1, failure_phase="runtime"), baseline, []) + + assert result.verdict == OracleVerdict.HARD_FAILURE + + +def test_oracle_hard_fails_a_phase_classified_run_that_exited_clean() -> None: + """An OOM kill or near-timeout hang is unhealthy even when the exit code is zero.""" + baseline = Baseline(median_fps=100.0, mad_fps=2.0, sample_count=5) + + result = compare(_bench_result(fps=100.0, exit_code=0, failure_phase="hang"), baseline, []) + + assert result.verdict == OracleVerdict.HARD_FAILURE + + +def test_oracle_still_passes_a_clean_run() -> None: + """The health check must not make every ordinary run a failure.""" + baseline = Baseline(median_fps=100.0, mad_fps=2.0, sample_count=5) + + result = compare(_bench_result(fps=100.0), baseline, []) + + assert result.verdict == OracleVerdict.PASS + + +def test_baseline_selection_filters_incompatible_contract_hashes(tmp_path: Path) -> None: + """Seeder/gate matching only selects samples with compatible launch and runtime contracts.""" + baselines_dir = tmp_path / "baselines" + compatible = _bench_result(fps=100.0).to_dict() + incompatible = _bench_result(fps=50.0).to_dict() + incompatible["runtime_contract_hash"] = "runtime-old" + + update_baseline( + baselines_dir, + "l40s", + "Isaac-Cartpole-Direct", + "physx", + 100.0, + sample_metadata={ + "gpu_model": "l40s", + "task_id": "Isaac-Cartpole-Direct", + "backend_key": "physx", + "launch_config_hash": compatible["launch_config_hash"], + "benchmark_contract_hash": compatible["benchmark_contract_hash"], + "runtime_contract_hash": compatible["runtime_contract_hash"], + "baseline_epoch": compatible["baseline_epoch"], + "fps": 100.0, + "sample_id": "compatible", + }, + ) + update_baseline( + baselines_dir, + "l40s", + "Isaac-Cartpole-Direct", + "physx", + 50.0, + sample_metadata={ + "gpu_model": "l40s", + "task_id": "Isaac-Cartpole-Direct", + "backend_key": "physx", + "launch_config_hash": incompatible["launch_config_hash"], + "benchmark_contract_hash": incompatible["benchmark_contract_hash"], + "runtime_contract_hash": incompatible["runtime_contract_hash"], + "baseline_epoch": incompatible["baseline_epoch"], + "fps": 50.0, + "sample_id": "incompatible-runtime", + }, + ) + + context = match_context_from_bench_result(compatible, gpu_model="l40s") + baseline = load_baseline(baselines_dir, "l40s", "Isaac-Cartpole-Direct", "physx", match_context=context) + + assert baseline is not None + assert baseline.median_fps == 100.0 + assert baseline.sample_count == 1 + assert baseline.total_sample_count == 2 diff --git a/tools/perf_smoke_test/test/test_framework_imports.py b/tools/perf_smoke_test/test/test_framework_imports.py new file mode 100644 index 000000000000..c02d3224cd40 --- /dev/null +++ b/tools/perf_smoke_test/test/test_framework_imports.py @@ -0,0 +1,165 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Guard the gate's Isaac Lab imports against the checked-out source tree. + +``perf_runtime.py`` is the only gate module that imports Isaac Lab, and it runs +exclusively inside the CI container -- so a stale import path is invisible to +every other test in this suite and to every local check. That is not +hypothetical: the gate was authored against ``isaaclab.test.benchmark``, #6564 +moved the package to the public ``isaaclab.benchmark``, and the mismatch turned +all nine buckets into ``HARD_FAILURE(phase=import)`` while the bench jobs still +reported success. + +These tests resolve each framework import statically against ``source/`` in the +current checkout, so the suite fails on the machine that rebases rather than on +the GPU runner an hour later. Static resolution is deliberate: importing +``isaaclab`` for real needs Isaac Sim, which this suite is specifically built +not to require. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +_GATE_DIR = Path(__file__).resolve().parents[1] +_REPO_ROOT = _GATE_DIR.parents[1] +_SOURCE = _REPO_ROOT / "source" + +# Gate modules that import the Isaac Lab framework. Everything else in +# tools/perf_smoke_test/ is deliberately framework-free so it can be unit tested +# without Isaac Sim; if that ever changes, add the module here. +_FRAMEWORK_IMPORTERS = ("perf_runtime.py",) + + +def _is_framework_root(name: str) -> bool: + """Whether ``name`` is an Isaac Lab package laid out under ``source/``. + + Derived from the tree rather than hard-coded: an explicit tuple silently + stopped checking ``isaaclab_tasks`` (it is not ``isaaclab``), which is where + ``setup_preset_cli`` and ``resolve_task_config`` come from. + """ + return (_SOURCE / name / name).is_dir() + + +def _module_to_path(dotted: str) -> Path | None: + """Resolve ``a.b.c`` to its file or package directory under ``source/``. + + Isaac Lab lays every package out as ``source///...``. + """ + parts = dotted.split(".") + pkg_root = _SOURCE / parts[0] / parts[0] + if not pkg_root.is_dir(): + return None + candidate = pkg_root.joinpath(*parts[1:]) if len(parts) > 1 else pkg_root + if candidate.is_dir(): + return candidate + py = candidate.with_suffix(".py") + return py if py.is_file() else None + + +def _defines(path: Path, name: str) -> bool: + """Whether ``path`` (a module or package) exposes ``name``.""" + if path.is_dir(): + # A submodule satisfies `from pkg import name` via the import system's + # fromlist fallback, even when the package uses a lazy __getattr__. + if (path / name).is_dir() or (path / f"{name}.py").is_file(): + return True + # Otherwise it must be declared in the package's stub or __init__. + sources = [path / "__init__.pyi", path / "__init__.py"] + else: + sources = [path] + + for src in sources: + if not src.is_file(): + continue + try: + tree = ast.parse(src.read_text(encoding="utf-8")) + except SyntaxError: + continue + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node.name == name: + return True + if isinstance(node, ast.ImportFrom): + if any(alias.asname == name or alias.name == name for alias in node.names): + return True + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == name: + return True + # `__all__ = [...]` entries count as declared exports. + if isinstance(target, ast.Name) and target.id == "__all__" and isinstance(node.value, ast.List): + for elt in node.value.elts: + if isinstance(elt, ast.Constant) and elt.value == name: + return True + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) and node.target.id == name: + return True + return False + + +def _framework_imports(module: str) -> list[tuple[str, tuple[str, ...], int]]: + """Return ``(module, names, lineno)`` for framework imports in a gate module.""" + tree = ast.parse((_GATE_DIR / module).read_text(encoding="utf-8")) + found = [] + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + if _is_framework_root(node.module.split(".")[0]): + found.append((node.module, tuple(a.name for a in node.names), node.lineno)) + elif isinstance(node, ast.Import): + for alias in node.names: + if _is_framework_root(alias.name.split(".")[0]): + found.append((alias.name, (), node.lineno)) + return found + + +@pytest.mark.parametrize("module", _FRAMEWORK_IMPORTERS) +def test_framework_modules_exist_in_checkout(module: str) -> None: + """Every Isaac Lab module the gate imports must exist in this checkout.""" + missing = [ + f"{module}:{lineno} imports '{dotted}', which does not exist under source/" + for dotted, _names, lineno in _framework_imports(module) + if _module_to_path(dotted) is None + ] + assert not missing, "stale framework import path(s):\n " + "\n ".join(missing) + + +@pytest.mark.parametrize("module", _FRAMEWORK_IMPORTERS) +def test_framework_symbols_exist_in_checkout(module: str) -> None: + """Every name imported from an Isaac Lab module must be resolvable there.""" + missing = [] + for dotted, names, lineno in _framework_imports(module): + target = _module_to_path(dotted) + if target is None: + continue # reported by the module-level test + for name in names: + if not _defines(target, name): + missing.append(f"{module}:{lineno} imports '{name}' from '{dotted}', which does not provide it") + assert not missing, "stale framework symbol(s):\n " + "\n ".join(missing) + + +def test_retired_benchmark_namespace_is_not_referenced() -> None: + """``isaaclab.test.benchmark`` was removed in #6564 and must not come back. + + Covers comments and docstrings too: the stale path survived in six of them + after the import statements themselves were first written. + """ + offenders = [] + for path in sorted(_GATE_DIR.rglob("*.py")): + if path.name == Path(__file__).name: + continue + for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if "isaaclab.test.benchmark" in line: + offenders.append(f"{path.relative_to(_REPO_ROOT)}:{lineno}: {line.strip()}") + assert not offenders, "retired isaaclab.test.benchmark namespace referenced:\n " + "\n ".join(offenders) + + +def test_source_tree_is_present() -> None: + """Fail loudly if the resolver is silently checking nothing.""" + assert (_SOURCE / "isaaclab" / "isaaclab" / "benchmark").is_dir(), ( + f"expected the Isaac Lab benchmark package under {_SOURCE}; the import guards above are vacuous without it" + ) diff --git a/tools/perf_smoke_test/test/test_image_era.py b/tools/perf_smoke_test/test/test_image_era.py new file mode 100644 index 000000000000..bf59d0e9cb01 --- /dev/null +++ b/tools/perf_smoke_test/test/test_image_era.py @@ -0,0 +1,275 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""GPU-free checks for the source-derivable image-era key and manifest resolver.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +_GATE_DIR = Path(__file__).resolve().parents[1] +_REPO_ROOT = _GATE_DIR.parents[1] +if str(_GATE_DIR) not in sys.path: + sys.path.insert(0, str(_GATE_DIR)) + +import image_era # noqa: E402 + +_ENV_BASE = """ +# General settings +ACCEPT_EULA=Y +ISAACSIM_BASE_IMAGE=nvcr.io/nvidia/isaac-sim +# version comment +ISAACSIM_VERSION=6.0.0-dev2 +DOCKER_USER_HOME=/root +DOCKER_NAME_SUFFIX="" +""" + + +def test_parse_env_file_strips_comments_and_quotes() -> None: + """Comments, blank lines, and surrounding quotes must not leak into values.""" + env = image_era.parse_env_file(_ENV_BASE) + + assert env["ISAACSIM_BASE_IMAGE"] == "nvcr.io/nvidia/isaac-sim" + assert env["ISAACSIM_VERSION"] == "6.0.0-dev2" + assert env["DOCKER_NAME_SUFFIX"] == "" + assert "# General settings" not in env + + +def test_era_key_is_deterministic() -> None: + """The same parsed env must always hash to the same era key.""" + env = image_era.parse_env_file(_ENV_BASE) + + assert image_era.compute_era_key(env) == image_era.compute_era_key(env) + + +def test_era_key_changes_when_isaacsim_version_changes() -> None: + """A different Isaac Sim version is a different era (under-sensitivity guard).""" + old = image_era.compute_era_key(image_era.parse_env_file(_ENV_BASE)) + new = image_era.compute_era_key(image_era.parse_env_file(_ENV_BASE.replace("6.0.0-dev2", "5.0.0"))) + + assert old != new + + +def test_era_key_ignores_cosmetic_and_non_era_fields() -> None: + """Comment edits, reordering, and non-era fields must not split the era (over-sensitivity guard).""" + base_key = image_era.compute_era_key(image_era.parse_env_file(_ENV_BASE)) + cosmetic = """ +# a totally different comment +ISAACSIM_VERSION=6.0.0-dev2 +ISAACSIM_BASE_IMAGE=nvcr.io/nvidia/isaac-sim +DOCKER_USER_HOME=/home/somethingelse +ACCEPT_EULA=N +""" + + assert image_era.compute_era_key(image_era.parse_env_file(cosmetic)) == base_key + + +def test_resolve_image_hit_returns_pinned_image() -> None: + """A known era resolves to its immutable image and reports a match.""" + manifest = { + "schema_version": 1, + "fallback_image": "nvcr.io/nvidian/isaac-lab:latest-perf", + "eras": {"era-abc": {"image": "nvcr.io/nvidian/isaac-lab:sha-deadbee"}}, + } + + image, matched = image_era.resolve_image("era-abc", manifest) + + assert matched is True + assert image == "nvcr.io/nvidian/isaac-lab:sha-deadbee" + + +def test_resolve_image_miss_falls_back_to_latest_perf() -> None: + """An unknown era degrades to the manifest fallback instead of failing.""" + manifest = { + "schema_version": 1, + "fallback_image": "nvcr.io/nvidian/isaac-lab:latest-perf", + "eras": {}, + } + + image, matched = image_era.resolve_image("era-unknown", manifest) + + assert matched is False + assert image == "nvcr.io/nvidian/isaac-lab:latest-perf" + + +def test_resolve_image_explicit_fallback_overrides_manifest() -> None: + """An explicit fallback takes precedence over the manifest default.""" + image, matched = image_era.resolve_image( + "era-unknown", {"eras": {}}, fallback_image="nvcr.io/nvidian/isaac-lab:sha-override" + ) + + assert matched is False + assert image == "nvcr.io/nvidian/isaac-lab:sha-override" + + +def test_resolve_image_empty_manifest_uses_default_fallback() -> None: + """A missing/empty manifest still yields the built-in default image.""" + image, matched = image_era.resolve_image("era-unknown", None) + + assert matched is False + assert image == image_era.DEFAULT_FALLBACK_IMAGE + + +def test_era_key_from_commit_matches_tree_at_head() -> None: + """The git-show path and the working-tree path must agree for the same source.""" + tree_key = image_era.era_key_from_tree(_REPO_ROOT) + commit_key = image_era.era_key_from_commit("HEAD", _REPO_ROOT) + + assert tree_key == commit_key + + +# -------------------------------------------------------------------------------------- +# manifest_upsert (pure) +# -------------------------------------------------------------------------------------- + + +def test_manifest_upsert_adds_new_entry_with_metadata() -> None: + """A new era records its image plus any provided metadata and reports a change.""" + updated, changed = image_era.manifest_upsert( + image_era.empty_manifest(), "era-1", "repo:sha-a", extra={"isaacsim_version": "6.0.0", "dropped": None} + ) + + assert changed is True + assert updated["eras"]["era-1"]["image"] == "repo:sha-a" + assert updated["eras"]["era-1"]["isaacsim_version"] == "6.0.0" + assert "dropped" not in updated["eras"]["era-1"] # None-valued metadata is not stored + + +def test_manifest_upsert_same_image_is_noop() -> None: + """Re-recording the identical era -> image mapping reports no change.""" + manifest, _ = image_era.manifest_upsert(None, "era-1", "repo:sha-a") + _, changed = image_era.manifest_upsert(manifest, "era-1", "repo:sha-a") + + assert changed is False + + +def test_manifest_upsert_remap_changes_image_and_preserves_other_eras() -> None: + """Remapping one era to a new image must not disturb sibling eras.""" + manifest, _ = image_era.manifest_upsert(None, "era-1", "repo:sha-a") + manifest, _ = image_era.manifest_upsert(manifest, "era-2", "repo:sha-b") + updated, changed = image_era.manifest_upsert(manifest, "era-1", "repo:sha-c") + + assert changed is True + assert updated["eras"]["era-1"]["image"] == "repo:sha-c" + assert updated["eras"]["era-2"]["image"] == "repo:sha-b" + + +def test_manifest_upsert_does_not_mutate_input() -> None: + """Upsert returns a new manifest and leaves the caller's manifest untouched.""" + manifest = image_era.empty_manifest() + image_era.manifest_upsert(manifest, "era-1", "repo:sha-a") + + assert manifest["eras"] == {} + + +# -------------------------------------------------------------------------------------- +# git-backed manifest record / load round trips +# -------------------------------------------------------------------------------------- + + +def _git(args: list[str], cwd: Path) -> None: + subprocess.run(["git", *args], cwd=str(cwd), check=True, capture_output=True, text=True) + + +def _init_repo_with_remote(tmp_path: Path) -> Path: + """Create a bare ``origin`` and a working clone with one commit; return the clone.""" + remote = tmp_path / "remote.git" + subprocess.run(["git", "init", "--bare", str(remote)], check=True, capture_output=True, text=True) + + work = tmp_path / "work" + work.mkdir() + _git(["init"], work) + _git(["remote", "add", "origin", str(remote)], work) + _git(["config", "user.email", "test@example.com"], work) + _git(["config", "user.name", "test"], work) + (work / "README.md").write_text("seed\n", encoding="utf-8") + _git(["add", "README.md"], work) + _git(["commit", "-m", "init"], work) + return work + + +def test_record_and_load_manifest_round_trips_over_git(tmp_path) -> None: + """A recorded era resolves to its immutable image when read back from git.""" + work = _init_repo_with_remote(tmp_path) + + pushed = image_era.record_era_image( + "era-1", "repo:sha-a", extra={"isaacsim_version": "6.0.0"}, remote="origin", repo_dir=work + ) + assert pushed is True + + manifest = image_era.load_manifest_from_git(remote="origin", repo_dir=work) + image, matched = image_era.resolve_image("era-1", manifest) + + assert matched is True + assert image == "repo:sha-a" + assert manifest["eras"]["era-1"]["isaacsim_version"] == "6.0.0" + + +def test_record_era_image_is_idempotent_and_supports_remap(tmp_path) -> None: + """Re-recording the same mapping is a no-op; a remap pushes the new image.""" + work = _init_repo_with_remote(tmp_path) + + assert image_era.record_era_image("era-1", "repo:sha-a", remote="origin", repo_dir=work) is True + assert image_era.record_era_image("era-1", "repo:sha-a", remote="origin", repo_dir=work) is False + assert image_era.record_era_image("era-1", "repo:sha-b", remote="origin", repo_dir=work) is True + + manifest = image_era.load_manifest_from_git(remote="origin", repo_dir=work) + assert manifest["eras"]["era-1"]["image"] == "repo:sha-b" + + +def test_record_era_image_preserves_existing_eras(tmp_path) -> None: + """Recording a second era appends without clobbering the first.""" + work = _init_repo_with_remote(tmp_path) + + image_era.record_era_image("era-1", "repo:sha-a", remote="origin", repo_dir=work) + image_era.record_era_image("era-2", "repo:sha-b", remote="origin", repo_dir=work) + + manifest = image_era.load_manifest_from_git(remote="origin", repo_dir=work) + assert manifest["eras"]["era-1"]["image"] == "repo:sha-a" + assert manifest["eras"]["era-2"]["image"] == "repo:sha-b" + + +def test_load_manifest_from_git_missing_branch_is_empty(tmp_path) -> None: + """A repo without the era branch resolves to an empty manifest (fallback path).""" + work = _init_repo_with_remote(tmp_path) + + manifest = image_era.load_manifest_from_git(remote="origin", repo_dir=work) + + assert manifest["eras"] == {} + + +def _resolve_matched(work: Path) -> bool: + """Reproduce the gate's image-resolution step: is HEAD's era already recorded?""" + manifest = image_era.load_manifest_from_git(remote="origin", repo_dir=work) + _, matched = image_era.resolve_image(image_era.era_key_from_tree(work), manifest) + return matched + + +def test_era_resolution_pins_recorded_eras_and_falls_back_otherwise(tmp_path) -> None: + """Resolution across a full environment-change cycle: miss, then pin, then miss again. + + ``matched`` is what decides whether the gate pins an immutable image or falls + back to the floating tag, so this asserts that signal end to end. + """ + work = _init_repo_with_remote(tmp_path) + (work / "docker").mkdir() + (work / "docker" / ".env.base").write_text(_ENV_BASE, encoding="utf-8") + + # New era: nothing recorded -> should_roll. + assert _resolve_matched(work) is False + + # Record this era's image -> subsequent pushes on the same era skip. + era_key = image_era.era_key_from_tree(work) + assert image_era.record_era_image(era_key, "repo:sha-a", remote="origin", repo_dir=work) is True + assert _resolve_matched(work) is True + + # A change to the era inputs is a new era again -> should_roll, and the old era + # stays recorded (pinnable) rather than being clobbered. + (work / "docker" / ".env.base").write_text(_ENV_BASE.replace("6.0.0-dev2", "7.0.0"), encoding="utf-8") + assert image_era.era_key_from_tree(work) != era_key + assert _resolve_matched(work) is False + assert image_era.load_manifest_from_git(remote="origin", repo_dir=work)["eras"][era_key]["image"] == "repo:sha-a" diff --git a/tools/perf_smoke_test/test/test_omni_github.py b/tools/perf_smoke_test/test/test_omni_github.py new file mode 100644 index 000000000000..07582c7d7c8d --- /dev/null +++ b/tools/perf_smoke_test/test/test_omni_github.py @@ -0,0 +1,155 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""GPU-free checks for the omni-github artifact emitted by the perf-smoke gate.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +_GATE_DIR = Path(__file__).resolve().parents[1] +_REPO_ROOT = _GATE_DIR.parents[1] +if str(_GATE_DIR) not in sys.path: + sys.path.insert(0, str(_GATE_DIR)) + +import omni_github # noqa: E402 +from contracts import BenchResult # noqa: E402 +from gate_types import OracleVerdict # noqa: E402 +from oracle import Baseline, compare # noqa: E402 + +_SCHEMA_PATH = _REPO_ROOT / ".github/actions/upload-omni-github-test-results/result-json.schema.json" + + +def _bench_result(*, task_id: str = "Isaac-Cartpole-Direct", fps: float | None = 100.0) -> BenchResult: + launch_config = { + "task_id": task_id, + "backend": "physx", + "backend_key": "physx", + "physics_backend": "physx", + "render_backend": None, + "gpu_model": "l40s", + "num_envs": 4096, + "num_frames": 600, + } + return BenchResult( + task_id=task_id, + backend="physx", + physics_backend="physx", + render_backend=None, + backend_key="physx", + preset="default", + was_retried=False, + perf_smoke_test_info_present=fps is not None, + raw_fps_mean=fps, + raw_fps_std=1.0 if fps is not None else None, + raw_fps_min=(fps - 1.0) if fps is not None else None, + raw_fps_max=(fps + 1.0) if fps is not None else None, + wall_time_s=42.5, + startup_time_s=12.3, + runtime_contract_hash="runtime-a", + runtime_resources={ + "gpu_name": "NVIDIA L40S", + "gpu_mem_used_mb": 1024.0, + "system_ram_used_mb": 2048.0, + "cuda_version": "12.4", + "nvidia_driver_version": "550.90", + }, + provenance={"software": {"isaacsim": "5.0.0", "isaaclab": "3.0.0", "warp": "1.13.0", "torch": "2.9.0"}}, + launch_config=launch_config, + launch_config_hash="launch-a", + benchmark_contract_hash="bench-a", + baseline_epoch=1, + ) + + +def _rows(): + baseline = Baseline(median_fps=100.0, mad_fps=2.0, sample_count=5) + pass_bench = _bench_result(fps=100.0) + block_bench = _bench_result(task_id="Isaac-Ant-Direct", fps=80.0) + return [ + (compare(pass_bench, baseline, []), pass_bench), + (compare(block_bench, baseline, []), block_bench), + ] + + +def test_build_result_shape_and_custom_diagnostics() -> None: + """Every row carries the verdict plus hardware/software diagnostics under custom.perf_smoke.""" + result = omni_github.build_result(_rows(), platform="linux-x86_64", app_config="aggregate") + + assert result["test_tool_id"] == omni_github.TEST_TOOL_ID + assert result["app"] == {"platform": "linux-x86_64", "config": "aggregate"} + assert len(result["tests"]) == 2 + + passed, blocked = result["tests"] + assert passed["passed"] is True and "message" not in passed + assert blocked["passed"] is False and blocked["message"].startswith("BLOCK") + + custom = passed["custom"]["perf_smoke"] + assert custom["task_id"] == "Isaac-Cartpole-Direct" + assert custom["verdict"] == "PASS" + assert custom["physics_backend"] == "physx" + assert custom["num_envs"] == 4096 + assert custom["num_frames"] == 600 + assert custom["wall_time_s"] == pytest.approx(42.5) + assert custom["startup_time_s"] == pytest.approx(12.3) + assert custom["fps_mean"] == pytest.approx(100.0) + assert custom["regression_pct"] == pytest.approx(0.0) + assert custom["vram_mb"] == pytest.approx(1024.0) + assert custom["sysram_mb"] == pytest.approx(2048.0) + # Trimmed to the dashboard core: deep diagnostics stay out of the artifact. + assert "runtime_contract_hash" not in custom + assert "warp_version" not in custom + # None-valued diagnostics are dropped, never emitted as null. + assert all(value is not None for value in custom.values()) + + +def _relax_custom_value_oneof(schema: dict) -> dict: + """Return the shared schema with ``customValue.oneOf`` rewritten to ``anyOf``. + + The committed schema defines custom values as a ``oneOf`` union that lists both + ``integer`` and ``number``. Under JSON Schema draft 2020-12 an integral value + (e.g. ``1`` or ``100.0``) matches *both* branches, so ``oneOf`` rejects every + integral number even though omni-github ingests them as ``l_*``/``d_*`` leaves. + We keep numeric diagnostics and validate against the intended union semantics. + """ + custom_value = schema["$defs"]["customValue"] + custom_value["anyOf"] = custom_value.pop("oneOf") + for branch in custom_value["anyOf"]: + items = branch.get("items") + if isinstance(items, dict) and "oneOf" in items: + items["anyOf"] = items.pop("oneOf") + return schema + + +def test_write_artifact_matches_shared_schema(tmp_path) -> None: + """The emitted result JSON validates against omni-github's committed schema.""" + jsonschema = pytest.importorskip("jsonschema") + + out = omni_github.write_artifact(_rows(), tmp_path, platform="linux-x86_64", app_config="aggregate") + + manifest = json.loads((out / omni_github.MANIFEST_NAME).read_text()) + assert manifest == {"schema_version": 1, "result_paths": [omni_github.RESULT_REL_PATH]} + + result = json.loads((out / omni_github.RESULT_REL_PATH).read_text()) + schema = _relax_custom_value_oneof(json.loads(_SCHEMA_PATH.read_text())) + jsonschema.validate(instance=result, schema=schema) + + +def test_hard_failure_row_has_no_measured_fps() -> None: + """A crashed benchmark (no FPS) still emits a valid failing row without null customs.""" + bench = _bench_result(fps=None) + result_row = compare(bench, baseline=None, fps_mean_thresholds=[]) + assert result_row.verdict == OracleVerdict.HARD_FAILURE + + payload = omni_github.build_result([(result_row, bench)], platform="linux-x86_64", app_config="aggregate") + row = payload["tests"][0] + + assert row["passed"] is False + assert "fps_mean" not in row["custom"]["perf_smoke"] + assert row["custom"]["perf_smoke"]["verdict"] == "HARD_FAILURE" diff --git a/tools/perf_smoke_test/test/test_seed_ancestry.py b/tools/perf_smoke_test/test/test_seed_ancestry.py new file mode 100644 index 000000000000..858f33961569 --- /dev/null +++ b/tools/perf_smoke_test/test/test_seed_ancestry.py @@ -0,0 +1,324 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""GPU-free checks for the seed-time ancestry preflight. + +The gate drops baseline samples whose ``commit_sha`` is not an ancestor of the +run's ``base_sha`` (the target branch HEAD). The seeder's preflight refuses to +seed such commits so they cannot silently produce unusable baselines. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +_GATE_DIR = Path(__file__).resolve().parents[1] +if str(_GATE_DIR) not in sys.path: + sys.path.insert(0, str(_GATE_DIR)) + +import seed_baselines # noqa: E402 + + +def _git(args: list[str], cwd: Path) -> str: + result = subprocess.run(["git", *args], cwd=str(cwd), check=True, capture_output=True, text=True) + return result.stdout.strip() + + +def _commit(work: Path, name: str) -> str: + (work / name).write_text(f"{name}\n", encoding="utf-8") + _git(["add", name], work) + _git(["commit", "-m", name], work) + return _git(["rev-parse", "HEAD"], work) + + +@pytest.fixture() +def repo(tmp_path: Path) -> dict[str, object]: + """A repo where ``develop`` (c0->c1->c2) and ``feature`` (c0->f1) diverge.""" + work = tmp_path / "work" + work.mkdir() + _git(["init", "-b", "develop"], work) + _git(["config", "user.email", "test@example.com"], work) + _git(["config", "user.name", "test"], work) + c0 = _commit(work, "c0") + c1 = _commit(work, "c1") + c2 = _commit(work, "c2") + + _git(["checkout", "-b", "feature", c0], work) + f1 = _commit(work, "f1") + _git(["checkout", "develop"], work) + return {"work": work, "c0": c0, "c1": c1, "c2": c2, "f1": f1} + + +def test_is_ancestor_tracks_reachability(repo) -> None: + """``_is_ancestor`` is true only for commits reachable from the tip.""" + work = repo["work"] + assert seed_baselines._is_ancestor(repo["c1"], repo["c2"], work) is True + assert seed_baselines._is_ancestor(repo["f1"], repo["c2"], work) is False + + +def test_resolve_branch_tip_prefers_local_when_no_remote(repo) -> None: + """With no ``origin`` the resolver still finds the local branch tip.""" + work = repo["work"] + assert seed_baselines._resolve_branch_tip("develop", work) == repo["c2"] + assert seed_baselines._resolve_branch_tip("nope", work) is None + + +def test_filter_drops_non_ancestor_commits(repo) -> None: + """A commit off the target branch is skipped; on-branch commits are kept.""" + work = repo["work"] + plan = [("develop", repo["c1"]), ("develop", repo["c2"]), ("develop", repo["f1"])] + + kept = seed_baselines._filter_plan_by_ancestry(plan, work, target_sha="") + + assert kept == [("develop", repo["c1"]), ("develop", repo["c2"])] + + +def test_filter_strict_raises_on_non_ancestor(repo) -> None: + """Strict mode turns an off-branch commit into a hard error.""" + work = repo["work"] + plan = [("develop", repo["c1"]), ("develop", repo["f1"])] + + with pytest.raises(RuntimeError, match="NOT ancestors"): + seed_baselines._filter_plan_by_ancestry(plan, work, target_sha="", strict=True) + + +def test_build_seed_plan_branches_ref_with_target_seeds_single_commit(repo) -> None: + """A pinned invocation (':develop', count 1) seeds exactly that commit. + + Seeding a fresh environment starts from one boundary commit via + ``branches: ":develop"``; this locks that the branches path resolves a + raw SHA and stamps the overridden target. + """ + work = repo["work"] + args = argparse.Namespace( + branches=f"{repo['c1']}:develop", + commits="", + commit_branch="develop", + commit_count=1, + target_branch="develop", + workdir=work, + ) + + plan = seed_baselines._build_seed_plan(args) + + assert plan == [("develop", repo["c1"])] + + +def test_filter_uses_explicit_target_sha(repo) -> None: + """An explicit target_sha is used verbatim for every plan entry.""" + work = repo["work"] + plan = [("develop", repo["c1"]), ("develop", repo["f1"])] + + kept = seed_baselines._filter_plan_by_ancestry(plan, work, target_sha=repo["c2"]) + + assert kept == [("develop", repo["c1"])] + + +def test_filter_unresolvable_tip_keeps_when_lenient_raises_when_strict(repo) -> None: + """An unresolvable target tip warns+keeps by default but aborts under strict.""" + work = repo["work"] + plan = [("ghost", repo["c1"])] + + kept = seed_baselines._filter_plan_by_ancestry(plan, work, target_sha="") + assert kept == plan + + with pytest.raises(RuntimeError, match="cannot resolve tip"): + seed_baselines._filter_plan_by_ancestry(plan, work, target_sha="", strict=True) + + +def test_prepare_jit_cache_opens_task_cache(tmp_path: Path) -> None: + """Each task/backend bucket gets writable Warp/CUDA cache roots.""" + cache_dir = tmp_path / "jit-cache" / "seed" / "task" / "newton" + + prepared = seed_baselines._prepare_jit_cache(cache_dir) + + assert prepared == cache_dir + assert (cache_dir / "warp").is_dir() + assert (cache_dir / "nv").is_dir() + assert cache_dir.stat().st_mode & 0o777 == 0o777 + + +def test_cache_run_name_isolates_stability_allocations(monkeypatch) -> None: + """Parallel allocations must not share writable JIT and Kit cache trees.""" + monkeypatch.setenv("GITHUB_RUN_ID", "123") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "2") + monkeypatch.setenv("PERF_SMOKE_RUN_LABEL", "runner-allocation-4") + + assert seed_baselines._cache_run_name() == "run-123-attempt-2-runner-allocation-4" + + +def test_only_failing_camera_backends_use_container_local_jit_cache() -> None: + """Only camera backends that failed on the host mount bypass cache reuse.""" + camera_task = "Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct" + + assert seed_baselines._uses_container_local_jit_cache( + SimpleNamespace(task_id=camera_task, backend_key="physx_rtx_renderer") + ) + assert seed_baselines._uses_container_local_jit_cache( + SimpleNamespace(task_id=camera_task, backend_key="newton_rtx_renderer") + ) + assert not seed_baselines._uses_container_local_jit_cache( + SimpleNamespace(task_id=camera_task, backend_key="physx_newton_renderer") + ) + assert not seed_baselines._uses_container_local_jit_cache( + SimpleNamespace(task_id="Isaac-Cartpole-Direct", backend_key="newton") + ) + + +def test_seed_source_directories_are_unique_and_disposable() -> None: + """Independent seeder invocations do not reuse source-clone residue.""" + first = seed_baselines._create_seed_source_dir() + second = seed_baselines._create_seed_source_dir() + try: + assert first != second + assert first.is_dir() + assert second.is_dir() + finally: + seed_baselines._cleanup_run_dir(first) + seed_baselines._cleanup_run_dir(second) + + +def test_jit_cache_reuses_samples_and_isolates_backends(tmp_path: Path) -> None: + """Repeated samples share compiled kernels, but different backends do not.""" + cache_root = tmp_path / "jit-cache" / "seed" / "run-1-attempt-1" + newton_cache = seed_baselines._cache_bucket_path( + cache_root, "release/2.0", "abcdef123456", "Isaac-Cartpole-Direct", "newton" + ) + repeated_newton_cache = seed_baselines._cache_bucket_path( + cache_root, "release/2.0", "abcdef123456", "Isaac-Cartpole-Direct", "newton" + ) + physx_cache = seed_baselines._cache_bucket_path( + cache_root, "release/2.0", "abcdef123456", "Isaac-Cartpole-Direct", "physx" + ) + + _ = seed_baselines._prepare_jit_cache(newton_cache) + compiled_kernel = newton_cache / "warp" / "compiled-kernel.so" + compiled_kernel.write_bytes(b"compiled") + _ = seed_baselines._prepare_jit_cache(repeated_newton_cache) + + assert repeated_newton_cache == newton_cache + assert compiled_kernel.read_bytes() == b"compiled" + assert physx_cache != newton_cache + assert not physx_cache.exists() + + +def test_kit_cache_reuses_samples_and_isolates_backends(tmp_path: Path) -> None: + """Repeated samples share Kit data, but different backends do not.""" + cache_root = tmp_path / "kit-cache" / "seed" / "run-1-attempt-1" + newton_cache = seed_baselines._cache_bucket_path( + cache_root, "release/2.0", "abcdef123456", "Isaac-Camera-Direct", "newton" + ) + repeated_newton_cache = seed_baselines._cache_bucket_path( + cache_root, "release/2.0", "abcdef123456", "Isaac-Camera-Direct", "newton" + ) + physx_cache = seed_baselines._cache_bucket_path( + cache_root, "release/2.0", "abcdef123456", "Isaac-Camera-Direct", "physx" + ) + + _ = seed_baselines._prepare_kit_cache(newton_cache) + shader_cache = newton_cache / "shader-cache.bin" + shader_cache.write_bytes(b"compiled") + _ = seed_baselines._prepare_kit_cache(repeated_newton_cache) + + assert repeated_newton_cache == newton_cache + assert shader_cache.read_bytes() == b"compiled" + assert newton_cache.stat().st_mode & 0o777 == 0o777 + assert physx_cache != newton_cache + assert not physx_cache.exists() + + +def test_cleanup_run_dir_removes_run_tree(tmp_path: Path) -> None: + """Seeder exit cleanup removes generated files from its run directory.""" + cache_dir = tmp_path / "jit-cache" / "seed" / "run-1-attempt-1" + compiled_dir = cache_dir / "warp" / "version" / "module" + compiled_dir.mkdir(parents=True) + (compiled_dir / "kernel.so").write_bytes(b"compiled") + compiled_dir.chmod(0o500) + + seed_baselines._cleanup_run_dir(cache_dir) + + assert not cache_dir.exists() + + +def test_docker_benchmark_copies_internal_output_after_exit(tmp_path: Path, monkeypatch) -> None: + """Benchmark output is copied from the container instead of bind-mounted.""" + calls: list[list[str]] = [] + + def _run(cmd: list[str], **_kwargs) -> SimpleNamespace: + calls.append(cmd) + return SimpleNamespace(returncode=0) + + task = SimpleNamespace( + task_id="Isaac-Cartpole-Direct", + num_envs=16, + num_frames=10, + warmup_frames=2, + seed=42, + ) + artifact_dir = tmp_path / "artifacts" + monkeypatch.setattr(seed_baselines, "hydra_args_for_task", lambda _task: []) + monkeypatch.setattr(seed_baselines.subprocess, "run", _run) + + exit_code = seed_baselines._docker_run_benchmark( + image="isaac-lab:test", + task=task, + artifact_dir=artifact_dir, + jit_cache=tmp_path / "jit-cache", + kit_cache=tmp_path / "kit-cache", + seed_src_dir=tmp_path / "source", + container_name="perf-seed-test", + ) + + docker_run = calls[0] + assert exit_code == 0 + assert "--rm" not in docker_run + assert f"{artifact_dir}:/tmp/bench_out" not in docker_run + assert f"{tmp_path / 'jit-cache'}:/tmp/jit-cache" in docker_run + assert f"{tmp_path / 'kit-cache'}:/isaac-sim/kit/cache" in docker_run + assert "umask 000" in docker_run[-1] + assert "mkdir -p /tmp/bench_out" in docker_run[-1] + assert calls[1] == ["docker", "cp", "perf-seed-test:/tmp/bench_out/.", str(artifact_dir)] + assert calls[2] == ["chmod", "-R", "a+rwX", str(artifact_dir)] + assert calls[3] == ["docker", "rm", "-f", "perf-seed-test"] + + +def test_docker_benchmark_can_use_container_local_jit_cache(tmp_path: Path, monkeypatch) -> None: + """A container-local cache creates JIT roots without a host bind mount.""" + calls: list[list[str]] = [] + + def _run(cmd: list[str], **_kwargs) -> SimpleNamespace: + calls.append(cmd) + return SimpleNamespace(returncode=0) + + task = SimpleNamespace( + task_id="Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct", + num_envs=16, + num_frames=10, + warmup_frames=2, + seed=42, + ) + monkeypatch.setattr(seed_baselines, "hydra_args_for_task", lambda _task: []) + monkeypatch.setattr(seed_baselines.subprocess, "run", _run) + + exit_code = seed_baselines._docker_run_benchmark( + image="isaac-lab:test", + task=task, + artifact_dir=tmp_path / "artifacts", + jit_cache=None, + kit_cache=tmp_path / "kit-cache", + seed_src_dir=tmp_path / "source", + container_name="perf-seed-test", + ) + + docker_run = calls[0] + assert exit_code == 0 + assert not any(arg.endswith(":/tmp/jit-cache") for arg in docker_run) + assert "mkdir -p /tmp/bench_out /tmp/jit-cache/warp /tmp/jit-cache/nv" in docker_run[-1] diff --git a/tools/perf_smoke_test/test/test_verify_baselines.py b/tools/perf_smoke_test/test/test_verify_baselines.py new file mode 100644 index 000000000000..55e79b74926f --- /dev/null +++ b/tools/perf_smoke_test/test/test_verify_baselines.py @@ -0,0 +1,229 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""GPU-free checks for the seeded-baseline usability verifier. + +Mirrors how the gate selects samples: a sample counts only if it shares a single +fingerprint bucket, targets the evaluated branch, and its ``commit_sha`` is an +ancestor of the target tip. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +_GATE_DIR = Path(__file__).resolve().parents[1] +if str(_GATE_DIR) not in sys.path: + sys.path.insert(0, str(_GATE_DIR)) + +import verify_baselines # noqa: E402 + + +def _sample( + fps: float, + commit: str, + runtime_hash: str = "rt-a", + epoch: int = 1, + target_branch: str = "develop", + sample_id: str | None = None, +) -> dict: + return { + "fps": fps, + "commit_sha": commit, + "target_branch": target_branch, + "launch_config_hash": "lc-a", + "benchmark_contract_hash": "bc-a", + "runtime_contract_hash": runtime_hash, + "baseline_epoch": epoch, + "sample_id": sample_id or f"{commit}-{fps}", + } + + +# -------------------------------------------------------------------------------------- +# usable_sample_count (pure) +# -------------------------------------------------------------------------------------- + + +def test_usable_count_single_fingerprint_no_ancestry() -> None: + """Uniform samples all count when ancestry is not enforced.""" + records = [_sample(100.0 + i, f"c{i}") for i in range(5)] + + usable, total, fingerprints = verify_baselines.usable_sample_count(records) + + assert (usable, total, fingerprints) == (5, 5, 1) + + +def test_usable_count_reports_largest_fingerprint_group() -> None: + """Mixed environments never pool; the largest single group is what the gate sees.""" + records = [_sample(100.0, f"a{i}", runtime_hash="rt-a") for i in range(3)] + records += [_sample(100.0, f"b{i}", runtime_hash="rt-b") for i in range(2)] + + usable, total, fingerprints = verify_baselines.usable_sample_count(records) + + assert usable == 3 + assert total == 5 + assert fingerprints == 2 + + +def test_usable_count_applies_ancestry_predicate() -> None: + """Only samples whose commit passes the ancestry predicate are eligible.""" + records = [_sample(100.0, "in1"), _sample(100.0, "in2"), _sample(100.0, "out1")] + ancestors = {"in1", "in2"} + + usable, total, _ = verify_baselines.usable_sample_count(records, lambda c: c in ancestors) + + assert usable == 2 + assert total == 3 + + +def test_usable_count_excludes_samples_missing_commit_sha_under_ancestry() -> None: + """A sample without a commit_sha cannot satisfy the gate's ancestry filter.""" + records = [_sample(100.0, "in1"), {"fps": 1.0, "runtime_contract_hash": "rt-a"}] + + usable, _, _ = verify_baselines.usable_sample_count(records, lambda c: True) + + assert usable == 1 + + +def test_usable_count_requires_target_branch_match() -> None: + """Samples stamped for another branch cannot satisfy a develop verification.""" + records = [ + _sample(100.0, "develop", target_branch="develop"), + _sample(100.0, "release", target_branch="release/2.0"), + ] + + usable, total, _ = verify_baselines.usable_sample_count(records, target_branch="develop") + + assert usable == 1 + assert total == 2 + + +def test_usable_count_requires_current_seeder_sample_ids() -> None: + """Old matching fingerprints cannot hide missing samples from the current run.""" + records = [ + _sample(100.0, "old1", sample_id="old-1"), + _sample(100.0, "old2", sample_id="old-2"), + _sample(100.0, "new1", sample_id="new-1"), + ] + + usable, total, _ = verify_baselines.usable_sample_count( + records, + target_branch="develop", + expected_sample_ids={"new-1", "new-2"}, + ) + + assert usable == 1 + assert total == 3 + + +def test_expected_sample_ids_are_scoped_to_gpu_and_target_branch(tmp_path: Path) -> None: + """The current-run verification ignores records for other gate contexts.""" + summary = tmp_path / "seed_records.json" + summary.write_text( + json.dumps( + [ + { + "gpu_model": "l40s", + "task_id": "task-a", + "backend": "physx", + "target_branch": "develop", + "sample_id": "expected", + }, + { + "gpu_model": "l40s", + "task_id": "task-a", + "backend": "physx", + "target_branch": "release/2.0", + "sample_id": "wrong-branch", + }, + { + "gpu_model": "rtx6000", + "task_id": "task-a", + "backend": "physx", + "target_branch": "develop", + "sample_id": "wrong-gpu", + }, + ] + ) + ) + + expected = verify_baselines._load_expected_sample_ids(summary, "l40s", "develop") + + assert expected == {("task-a", "physx"): {"expected"}} + + +# -------------------------------------------------------------------------------------- +# verify_bucket (git-backed) +# -------------------------------------------------------------------------------------- + + +def _git(args: list[str], cwd: Path) -> str: + result = subprocess.run(["git", *args], cwd=str(cwd), check=True, capture_output=True, text=True) + return result.stdout.strip() + + +def test_verify_bucket_counts_ancestor_samples_from_git(tmp_path: Path) -> None: + """verify_bucket reads samples.ndjson from a git ref and honors ancestry.""" + work = tmp_path / "work" + work.mkdir() + _git(["init", "-b", "develop"], work) + _git(["config", "user.email", "test@example.com"], work) + _git(["config", "user.name", "test"], work) + + # Build a short develop history and remember each commit SHA. + shas: list[str] = [] + for i in range(5): + (work / f"f{i}").write_text(f"{i}\n", encoding="utf-8") + _git(["add", f"f{i}"], work) + _git(["commit", "-m", f"c{i}"], work) + shas.append(_git(["rev-parse", "HEAD"], work)) + develop_tip = shas[-1] + + # A diverged commit that is NOT on develop. + _git(["checkout", "-b", "feature", shas[0]], work) + (work / "ff").write_text("x\n", encoding="utf-8") + _git(["add", "ff"], work) + _git(["commit", "-m", "feature"], work) + feature_sha = _git(["rev-parse", "HEAD"], work) + _git(["checkout", "develop"], work) + + # Write the bucket samples: 5 on develop history + 1 off-branch. + rel = str(verify_baselines._samples_path(Path(""), "l40s", "Isaac-Cartpole-Direct", "newton", None)) + sample_path = work / rel + sample_path.parent.mkdir(parents=True, exist_ok=True) + lines = [json.dumps(_sample(100.0 + i, shas[i])) for i in range(5)] + lines.append(json.dumps(_sample(9.0, feature_sha))) + sample_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + _git(["add", rel], work) + _git(["commit", "-m", "baselines"], work) + ref = _git(["rev-parse", "HEAD"], work) + + usable, total, fingerprints = verify_baselines.verify_bucket( + ref, "l40s", "Isaac-Cartpole-Direct", "newton", develop_tip, repo_dir=work + ) + + assert total == 6 # all samples present in the file + assert usable == 5 # the off-branch sample is excluded by ancestry + assert fingerprints == 1 + + +def test_verify_bucket_missing_bucket_is_empty(tmp_path: Path) -> None: + """A bucket with no samples.ndjson reports zero without raising.""" + work = tmp_path / "work" + work.mkdir() + _git(["init", "-b", "develop"], work) + _git(["config", "user.email", "test@example.com"], work) + _git(["config", "user.name", "test"], work) + (work / "README").write_text("x\n", encoding="utf-8") + _git(["add", "README"], work) + _git(["commit", "-m", "init"], work) + ref = _git(["rev-parse", "HEAD"], work) + + usable, total, fingerprints = verify_baselines.verify_bucket(ref, "l40s", "Nope", "newton", None, repo_dir=work) + + assert (usable, total, fingerprints) == (0, 0, 0) diff --git a/tools/perf_smoke_test/test/test_workflow_contracts.py b/tools/perf_smoke_test/test/test_workflow_contracts.py new file mode 100644 index 000000000000..e87f5cb61100 --- /dev/null +++ b/tools/perf_smoke_test/test/test_workflow_contracts.py @@ -0,0 +1,212 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Checks on workflow wiring that the Python tests cannot reach. + +Several gate defects have lived in the YAML rather than the code: a fork's +read-only token failing a reporting step, an input default that made the +documented empty value unreachable, a cleanup line naming a file that does not +exist yet. Each is invisible to a unit test of the modules those workflows call, +so the invariants are pinned here. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +_WORKFLOWS = Path(__file__).resolve().parents[3] / ".github" / "workflows" +_GATE = _WORKFLOWS / "perf-smoke-test.yaml" +_SEED = _WORKFLOWS / "perf-smoke-seed-baselines.yaml" + +# Steps that call the GitHub API and would 403 under a fork's read-only token. +_REPORTING_STEPS = ("Report per-task status", "Report aggregate status", "Post verdict PR comment") + + +def _load(path: Path) -> dict: + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def _steps(workflow: dict, job: str) -> list[dict]: + return workflow["jobs"][job].get("steps") or [] + + +def _step(workflow: dict, job: str, name: str) -> dict: + for step in _steps(workflow, job): + if step.get("name") == name: + return step + raise AssertionError(f"step {name!r} not found in job {job!r}") + + +@pytest.mark.parametrize("step_name", _REPORTING_STEPS) +def test_api_reporting_is_skipped_for_fork_pull_requests(step_name: str) -> None: + """A fork PR must not fail the gate just because it cannot write statuses. + + ``pull_request`` from a fork gets a read-only token no matter what the + permissions block requests, so an unguarded createCommitStatus or + createComment turns the job red for every external contributor. + """ + gate = _load(_GATE) + job = "bench" if step_name == "Report per-task status" else "aggregate" + + condition = str(_step(gate, job, step_name).get("if")) + + assert "github.event.pull_request.head.repo.full_name == github.repository" in condition + assert "always()" in condition, "the step must still run for failed runs on same-repo PRs" + + +def test_write_scopes_are_not_granted_workflow_wide() -> None: + """Jobs that run PR-authored code must not hold a token that can forge reports.""" + gate = _load(_GATE) + + assert gate["permissions"] == {"contents": "read"} + for job in ("config", "validate"): + assert gate["jobs"][job].get("permissions") is None, f"{job} should inherit read-only" + assert gate["jobs"]["bench"]["permissions"] == {"contents": "read", "statuses": "write"} + + +def test_image_tag_follows_the_target_branch_not_the_merge_ref() -> None: + """A PR into main or release must benchmark that branch's image, not develop's. + + On ``pull_request`` the ref name is the synthetic ``/merge``, so keying the + tag off it silently selects latest-develop and produces samples whose + runtime_contract_hash no baseline on the target branch can match. + """ + gate = _load(_GATE) + + load_step = next(step for step in _steps(gate, "config") if step.get("id") == "load") + target = load_step["env"]["TARGET_BRANCH"] + + assert "github.base_ref" in target + assert "github.event.merge_group.base_ref" in target + assert "GITHUB_REF_NAME" not in load_step["run"], "the case must switch on the resolved target branch" + + +def test_retry_clears_the_runtime_bundle_of_the_failed_attempt() -> None: + """Otherwise a retry that dies early reports the first attempt's FPS as its own.""" + gate = _load(_GATE) + + retry = _step(gate, "bench", "Retry benchmark on failure")["run"] + + assert "benchmark_runtime_*.json" in retry + assert "rm -f" in retry + + +@pytest.mark.parametrize("input_name", ["tasks", "branches"]) +def test_empty_seed_inputs_reach_the_seeder(input_name: str) -> None: + """``empty = all tasks`` and ``empty = use commits`` are documented, so they must work. + + ``${{ inputs.x || 'default' }}`` treats an explicitly empty value as unset and + substitutes the default, which makes those documented modes unreachable. + """ + seed_source = _SEED.read_text(encoding="utf-8") + + assert f"${{{{ inputs.{input_name} }}}}" in seed_source + assert f"inputs.{input_name} ||" not in seed_source, "a || default defeats an explicitly empty input" + + +def test_seed_workflow_declares_the_documented_empty_behavior() -> None: + """The input description and the expression must agree about what empty means.""" + seed = _load(_SEED) + call_inputs = seed[True]["workflow_call"]["inputs"] + + assert "empty = all" in call_inputs["tasks"]["description"].lower() + + +def test_reseed_passes_only_the_credential_the_seeder_declares() -> None: + """`secrets: inherit` would hand the seeding workflow every repository secret.""" + gate = _load(_GATE) + seed = _load(_SEED) + + passed = gate["jobs"]["reseed"]["secrets"] + + assert set(passed) == set(seed[True]["workflow_call"]["secrets"]) + + +# --- diagnostics must survive a nonzero aggregate exit --------------------- +# +# The aggregate step runs under `bash -e`. Before this was fixed, a nonzero +# aggregate.py exit aborted the step at the python call, so the job summary was +# never written -- on precisely the runs that needed explaining. On a fork PR, +# where the reporting steps are also skipped, that left a red check with no +# verdict anywhere: not in a comment, not in a status, not in the summary. + + +def _aggregate_run_block() -> str: + return _step(_load(_GATE), "aggregate", "Run aggregate oracle")["run"] + + +def test_aggregate_step_does_not_abort_before_publishing_diagnostics() -> None: + """`set +e` must wrap the aggregate call so the summary is still written.""" + run = _aggregate_run_block() + call = run.index("aggregate.py") + assert "set +e" in run[:call], "aggregate.py must be invoked with errexit disabled" + assert "AGGREGATE_STATUS=$?" in run, "the aggregate exit code must be captured, not swallowed" + + +def test_aggregate_step_still_reports_its_exit_code() -> None: + """Disabling errexit must not silently turn every aggregate run green.""" + run = _aggregate_run_block() + assert 'exit "${AGGREGATE_STATUS}"' in run, "the captured aggregate exit code must be re-raised" + assert run.index("AGGREGATE_STATUS=$?") < run.index('exit "${AGGREGATE_STATUS}"') + + +def test_summary_is_written_on_every_path() -> None: + """Both branches (summary produced, or not) must append to the step summary.""" + run = _aggregate_run_block() + assert run.count("GITHUB_STEP_SUMMARY") >= 2, ( + "the step must write to the job summary whether or not aggregate produced a verdict table" + ) + status_write = run.index("GITHUB_STEP_SUMMARY") + assert status_write < run.index('exit "${AGGREGATE_STATUS}"'), "diagnostics must be published before exiting" + + +def test_aggregate_status_reports_the_verdict_not_the_step_outcome() -> None: + """The commit status must carry the verdict, so advisory mode still signals.""" + step = _step(_load(_GATE), "aggregate", "Report aggregate status") + env = step.get("env") or {} + assert "steps.aggregate.outputs.status_state" in env.get("STATUS_STATE", ""), ( + "the aggregate commit status must be driven by the emitted verdict" + ) + script = step["with"]["script"] + assert "STATUS_STATE" in script and "STATUS_DESCRIPTION" in script + # A missing verdict must not be read as success. + assert "did not produce a verdict" in script + + +def test_fork_pull_requests_are_told_where_the_verdict_is() -> None: + """Fork PRs cannot get a comment or a status, so point them at the summary.""" + gate = _load(_GATE) + step = _step(gate, "aggregate", "Explain skipped reporting (fork pull request)") + assert "head.repo.full_name != github.repository" in step["if"] + assert "summary" in step["run"].lower() + + +# --- protected-branch runs must not cancel each other ---------------------- +# +# The push run is the only thing that appends to perf-baselines. develop lands +# ~10 commits a day (median gap ~45 min) against a perf run that takes over an +# hour, so a shared concurrency group cancelled the majority of baseline runs +# and the window could never reach MIN_BASELINE_SAMPLES. + + +def test_push_runs_are_not_cancelled_by_the_next_push() -> None: + concurrency = _load(_GATE)["concurrency"] + assert "github.sha" in concurrency["group"], ( + "protected-branch pushes must get a per-commit concurrency group, " + "otherwise the next merge cancels the run that publishes baselines" + ) + assert "github.ref" not in concurrency["group"] + + +def test_pull_request_runs_still_supersede_each_other() -> None: + """Cancelling a stale PR run is the useful half of cancel-in-progress.""" + concurrency = _load(_GATE)["concurrency"] + cancel = str(concurrency["cancel-in-progress"]) + assert "pull_request" in cancel, "cancel-in-progress must remain enabled for pull requests" + assert cancel.strip() != "true", "cancel-in-progress must not apply unconditionally to pushes" + assert "github.event.pull_request.number" in concurrency["group"] diff --git a/tools/perf_smoke_test/validate_tasks.py b/tools/perf_smoke_test/validate_tasks.py new file mode 100644 index 000000000000..63fb41027550 --- /dev/null +++ b/tools/perf_smoke_test/validate_tasks.py @@ -0,0 +1,113 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Static pre-flight validation for ``tasks.json``. + +Catches the class of bug where a ``task_id`` in ``tasks.json`` no longer matches +a registered Gymnasium environment (e.g. a renamed/version-bumped task), which +otherwise only surfaces after a multi-minute GPU benchmark job fails at task +lookup. Runs without Isaac Sim: it loads the task matrix via the normal loader +(schema check) and statically scans the repository's ``gym.register(id=...)`` +calls (registry check), so it is safe on a plain CPU runner. + +Usage:: + + python3 tools/perf_smoke_test/validate_tasks.py +""" + +import re +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +_TOOLS_DIR = _MODULE_DIR.parent +_REPO_ROOT = _TOOLS_DIR.parent + +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) +if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + +from task_config import load_tasks # noqa: E402 + +_ID_RE = re.compile(r"""id\s*=\s*["']([^"']+)["']""") + + +def registered_task_ids(source_root: Path) -> set[str]: + """Collect Gymnasium environment ids registered under ``source_root``. + + Scans ``__init__.py`` files that call ``gym.register(...)`` and extracts the + ``id="..."`` arguments. This is a static text scan (no imports), so it works + without Isaac Sim installed. + """ + ids: set[str] = set() + for init_file in source_root.rglob("__init__.py"): + try: + text = init_file.read_text(errors="replace") + except OSError: + continue + if "register(" not in text: + continue + for block in text.split("register(")[1:]: + match = _ID_RE.search(block[:200]) + if match: + ids.add(match.group(1)) + return ids + + +def validate(source_root: Path | None = None) -> list[str]: + """Return a list of human-readable problems; empty list means valid.""" + source_root = source_root or (_REPO_ROOT / "source") + problems: list[str] = [] + + try: + tasks = load_tasks() + except Exception as exc: # noqa: BLE001 - surface any schema/parse error verbatim + return [f"tasks.json failed to load: {exc}"] + + # Every task must discard at least the first 2 cold-start steps as warmup + # (excluded at the source by perf_runtime.py) and still leave measured steps. + _MIN_WARMUP_FRAMES = 2 + for task in tasks: + if task.warmup_frames < _MIN_WARMUP_FRAMES: + problems.append( + f"task {task.task_id!r}/{task.backend_key}: warmup_frames={task.warmup_frames} must be >= " + f"{_MIN_WARMUP_FRAMES} (at least the first {_MIN_WARMUP_FRAMES} steps are discarded as warmup)" + ) + if task.warmup_frames >= task.num_frames: + problems.append( + f"task {task.task_id!r}/{task.backend_key}: warmup_frames={task.warmup_frames} must be < " + f"num_frames={task.num_frames} to leave measured steps" + ) + + registered = registered_task_ids(source_root) + if not registered: + # Don't hard-fail when the source tree isn't present (e.g. partial checkout); + # the schema check above still ran. + problems.append(f"warning: no gym.register ids found under {source_root}; skipping registry check") + return problems + + seen_task_ids = {task.task_id for task in tasks} + for task_id in sorted(seen_task_ids): + if task_id not in registered: + problems.append(f"task_id {task_id!r} is not a registered Gymnasium environment") + return problems + + +def main() -> int: + problems = validate() + hard_errors = [p for p in problems if not p.startswith("warning:")] + for problem in problems: + prefix = "WARN " if problem.startswith("warning:") else "ERROR" + print(f"[validate_tasks] {prefix}: {problem}") + if hard_errors: + print(f"[validate_tasks] FAILED with {len(hard_errors)} problem(s).") + return 1 + print("[validate_tasks] OK: all task_ids resolve to registered environments.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_smoke_test/verify_baselines.py b/tools/perf_smoke_test/verify_baselines.py new file mode 100644 index 000000000000..3665c3a7c5f0 --- /dev/null +++ b/tools/perf_smoke_test/verify_baselines.py @@ -0,0 +1,289 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Verify that seeded baselines will actually be used by the gate. + +The gate keeps a baseline sample only when both hold: + +1. Its fingerprint -- ``launch_config_hash``, ``benchmark_contract_hash``, + ``runtime_contract_hash``, ``baseline_epoch`` (within a + ``gpu_model``/``task_id``/``backend_key`` bucket) -- matches the run. +2. Its ``target_branch`` matches the branch the gate will evaluate. +3. Its ``commit_sha`` is an ancestor of the run's ``base_sha`` (the target + branch HEAD). See :func:`baseline_manager._sample_matches`. + +This tool replays that selection against the ``perf-baselines`` branch for a +given target tip and reports, per bucket, how many samples would survive and +whether that clears ``MIN_BASELINE_SAMPLES`` (gate blocks) or leaves the gate +advisory (``< MIN_BASELINE_SAMPLES``). It answers the operational question +"did seeding actually populate baselines the gate will use?" without running a +benchmark. + +With ``--require`` the process exits non-zero when any expected bucket has fewer +than ``MIN_BASELINE_SAMPLES`` usable samples, so a seed workflow can gate on it. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from collections import defaultdict +from collections.abc import Callable, Iterable +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from baseline_manager import _git_is_ancestor, _git_show_file, _samples_path, refresh_baseline_branch # noqa: E402 +from gate_config import MIN_BASELINE_SAMPLES # noqa: E402 +from gpu_identity import canonical_gpu_model, detect_gpu_model # noqa: E402 +from task_config import load_tasks # noqa: E402 + +# Fields that (with the bucket key) form the gate's compatibility fingerprint. +_FINGERPRINT_FIELDS = ( + "launch_config_hash", + "benchmark_contract_hash", + "runtime_contract_hash", + "baseline_epoch", +) + + +def _fingerprint(record: dict[str, Any]) -> tuple: + return tuple(record.get(field) for field in _FINGERPRINT_FIELDS) + + +def usable_sample_count( + records: Iterable[dict[str, Any]], + is_ancestor: Callable[[str], bool] | None = None, + *, + target_branch: str | None = None, + expected_sample_ids: set[str] | None = None, +) -> tuple[int, int, int]: + """Return ``(usable, total, num_fingerprints)`` for a bucket's samples. + + ``usable`` is the size of the largest single-fingerprint group among the + ancestry-eligible samples -- the count the gate would actually compare + against, since it never pools samples across fingerprints. ``num_fingerprints`` + is the number of distinct fingerprints among eligible samples (``> 1`` means + the bucket mixes environments/configs and no single group may be large enough). + + Args: + records: Sample dicts (as stored in ``samples.ndjson``). + is_ancestor: Predicate ``commit_sha -> bool``; when provided, only samples + with a ``commit_sha`` that satisfies it are eligible (mirrors the + gate's ancestry filter). ``None`` disables the ancestry filter. + target_branch: Exact target branch required by the gate. + expected_sample_ids: When provided, only samples produced by the current + seeder invocation are eligible. + """ + records = list(records) + eligible = records + if target_branch is not None: + eligible = [r for r in eligible if r.get("target_branch") == target_branch] + if expected_sample_ids is not None: + eligible = [r for r in eligible if r.get("sample_id") in expected_sample_ids] + if is_ancestor is not None: + eligible = [r for r in eligible if r.get("commit_sha") and is_ancestor(str(r["commit_sha"]))] + by_fingerprint: dict[tuple, int] = defaultdict(int) + for record in eligible: + by_fingerprint[_fingerprint(record)] += 1 + return max(by_fingerprint.values(), default=0), len(records), len(by_fingerprint) + + +def _load_samples(ref: str, gpu_model: str, task_id: str, backend: str, *, repo_dir: Path | None = None) -> list[dict]: + rel_path = str(_samples_path(Path(""), gpu_model, task_id, backend, None)) + content = _git_show_file(ref, rel_path, repo_dir=repo_dir) + if not content: + return [] + records: list[dict] = [] + for line in content.splitlines(): + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, dict): + records.append(record) + return records + + +def _load_expected_sample_ids( + path: Path, + gpu_model: str, + target_branch: str, +) -> dict[tuple[str, str], set[str]]: + """Load current-run sample IDs grouped by task/backend bucket.""" + payload = json.loads(path.read_text()) + if not isinstance(payload, list): + raise ValueError("expected records must be a JSON list") + by_bucket: dict[tuple[str, str], set[str]] = defaultdict(set) + for record in payload: + if not isinstance(record, dict): + continue + if canonical_gpu_model(str(record.get("gpu_model", ""))) != gpu_model: + continue + if record.get("target_branch") != target_branch: + continue + task_id = record.get("task_id") + backend = record.get("backend") + sample_id = record.get("sample_id") + if task_id and backend and sample_id: + by_bucket[(str(task_id), str(backend))].add(str(sample_id)) + return dict(by_bucket) + + +def verify_bucket( + ref: str, + gpu_model: str, + task_id: str, + backend: str, + base_sha: str | None, + *, + target_branch: str | None = None, + expected_sample_ids: set[str] | None = None, + repo_dir: Path | None = None, +) -> tuple[int, int, int]: + """Load a bucket from git and report ``(usable, total, num_fingerprints)``.""" + records = _load_samples(ref, gpu_model, task_id, backend, repo_dir=repo_dir) + predicate: Callable[[str], bool] | None = None + if base_sha: + + def predicate(commit_sha: str) -> bool: + return _git_is_ancestor(commit_sha, str(base_sha), repo_dir=repo_dir) + + return usable_sample_count( + records, + predicate, + target_branch=target_branch, + expected_sample_ids=expected_sample_ids, + ) + + +def _resolve_tip(ref: str, repo_dir: Path | None = None) -> str | None: + for candidate in (f"origin/{ref}", ref): + result = subprocess.run( + ["git", "rev-parse", "--verify", "--quiet", f"{candidate}^{{commit}}"], + cwd=str(repo_dir) if repo_dir else None, + capture_output=True, + text=True, + ) + if result.returncode == 0: + return result.stdout.strip() + return None + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Verify seeded baselines are usable by the gate for a target tip.") + parser.add_argument("--baseline_branch", default="perf-baselines", help="Branch storing baseline samples.") + parser.add_argument("--baseline_remote", default="origin", help="Remote owning the baseline branch (empty=local).") + parser.add_argument("--gpu_model", default="", help="GPU model label; auto-detected via nvidia-smi when empty.") + parser.add_argument( + "--base_sha", + default="", + help="Tip to check ancestry against. Empty resolves --target_branch (origin/).", + ) + parser.add_argument("--target_branch", default="develop", help="Branch whose tip is used when --base_sha is empty.") + parser.add_argument("--tasks", default="", help="Comma-separated task_id allowlist (empty = all tasks.json tasks).") + parser.add_argument("--backends", default="", help="Comma-separated backend_key allowlist (empty = all backends).") + parser.add_argument( + "--expected_records", + default=None, + type=Path, + help="Seeder summary whose sample IDs must exist on the baseline branch.", + ) + parser.add_argument("--repo_dir", default=None, help="Repo root for git lookups (default: cwd).") + parser.add_argument( + "--require", + action="store_true", + help="Exit non-zero when any expected bucket has fewer than MIN_BASELINE_SAMPLES usable samples.", + ) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + repo_dir = Path(args.repo_dir) if args.repo_dir else None + gpu_model = canonical_gpu_model(detect_gpu_model(args.gpu_model)) + remote = args.baseline_remote or None + + ref = refresh_baseline_branch(args.baseline_branch, remote=remote, repo_dir=repo_dir, allow_missing=True) + if not ref: + print(f"::error::[verify] baseline branch {args.baseline_branch!r} not found; nothing seeded yet") + return 1 if args.require else 0 + + base_sha = args.base_sha.strip() or _resolve_tip(args.target_branch, repo_dir) + if not base_sha: + print(f"::warning::[verify] could not resolve target tip for {args.target_branch!r}; ancestry not checked") + + expected_ids_by_bucket: dict[tuple[str, str], set[str]] | None = None + if args.expected_records: + try: + expected_ids_by_bucket = _load_expected_sample_ids( + args.expected_records, + gpu_model, + args.target_branch, + ) + except (OSError, ValueError, json.JSONDecodeError) as exc: + print(f"::error::[verify] could not load expected records {args.expected_records}: {exc}") + return 1 + + task_filter = {t.strip() for t in args.tasks.split(",") if t.strip()} + backend_filter = {b.strip() for b in args.backends.split(",") if b.strip()} + tasks = [ + t + for t in load_tasks() + if (not task_filter or t.task_id in task_filter) and (not backend_filter or t.backend_key in backend_filter) + ] + + print(f"[verify] baseline={args.baseline_branch}@{ref[:12]} gpu={gpu_model} base_sha={(base_sha or 'n/a')[:12]}") + print(f"[verify] MIN_BASELINE_SAMPLES={MIN_BASELINE_SAMPLES}") + print("[verify] | task | backend | usable | total | fingerprints | verdict |") + + results: list[dict[str, Any]] = [] + all_ok = True + for task in tasks: + expected_sample_ids = ( + expected_ids_by_bucket.get((task.task_id, task.backend_key), set()) + if expected_ids_by_bucket is not None + else None + ) + usable, total, fingerprints = verify_bucket( + ref, + gpu_model, + task.task_id, + task.backend_key, + base_sha, + target_branch=args.target_branch, + expected_sample_ids=expected_sample_ids, + repo_dir=repo_dir, + ) + ok = usable >= MIN_BASELINE_SAMPLES + all_ok = all_ok and ok + verdict = "BLOCKING" if ok else ("WARMING" if usable > 0 else "EMPTY") + results.append( + { + "task_id": task.task_id, + "backend_key": task.backend_key, + "usable": usable, + "total": total, + "fingerprints": fingerprints, + "verdict": verdict, + } + ) + print(f"[verify] | {task.task_id} | {task.backend_key} | {usable} | {total} | {fingerprints} | {verdict} |") + + print("[verify] summary: " + json.dumps({"gpu_model": gpu_model, "buckets": results}, sort_keys=True)) + if args.require and not all_ok: + print("::error::[verify] one or more buckets have fewer usable samples than MIN_BASELINE_SAMPLES") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/perf_smoke_test/write_launch_config.py b/tools/perf_smoke_test/write_launch_config.py new file mode 100644 index 000000000000..b87d7cd29050 --- /dev/null +++ b/tools/perf_smoke_test/write_launch_config.py @@ -0,0 +1,48 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Write launch_config.json for a perf-smoke task/backend job""" + +import argparse +import sys +from pathlib import Path + +_MODULE_DIR = Path(__file__).parent +if str(_MODULE_DIR) not in sys.path: + sys.path.insert(0, str(_MODULE_DIR)) + +from backend_identity import make_backend_key, normalize_render_backend +from launch_config import hydra_args_for_task, task_to_launch_config, write_launch_config +from task_config import get_task + + +def _parse_args(): + parser = argparse.ArgumentParser(description="Write launch_config.json for a perf-smoke benchmark job") + parser.add_argument("--task_id", required=True) + parser.add_argument("--physics_backend", required=True) + parser.add_argument("--render_backend", default="") + parser.add_argument("--gpu_model", default="L40S") + parser.add_argument("--artifact_dir", required=True, type=Path) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + render_backend = normalize_render_backend(args.render_backend) + backend_key = make_backend_key(args.physics_backend, render_backend) + task = get_task(args.task_id, backend_key) + config = task_to_launch_config( + task, + fps_mean_thresholds=task.thresholds_for(args.gpu_model), + gpu_model=args.gpu_model, + hydra_args=hydra_args_for_task(task), + ) + path = write_launch_config(args.artifact_dir, config) + print(path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/subprocess_runner.py b/tools/subprocess_runner.py new file mode 100644 index 000000000000..57a750202647 --- /dev/null +++ b/tools/subprocess_runner.py @@ -0,0 +1,324 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import contextlib +import os +import select +import signal +import subprocess +import sys +import time + + +def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, report_file=""): + """Run a command with timeout and capture all output while streaming in real-time. + + Args: + cmd: Command to execute. + timeout: Maximum wall-clock seconds before the process is killed. + env: Environment variables for the subprocess. + startup_deadline: If > 0, the process is killed early when neither + ``AppLauncher initialization complete`` (stderr) nor ``collected`` + (stdout) appears within this many seconds. + report_file: Path to the JUnit XML report file. When set, the process + is given only :data:`SHUTDOWN_GRACE_PERIOD` seconds to exit after + the file appears on disk. + + Returns: + Tuple of ``(returncode, stdout_bytes, stderr_bytes, kill_reason, + wall_time, pre_kill_diag)``. *kill_reason* is ``""`` for normal exits, + ``"timeout"`` for hard timeouts, ``"startup_hang"`` when the process + did not reach pytest collection in time, or ``"shutdown_hang"`` when + the test completed but the process hung during shutdown. + """ + # Import here to avoid circular dependency; SHUTDOWN_GRACE_PERIOD is defined in conftest. + # We define a local default that matches conftest's constant. + _SHUTDOWN_GRACE_PERIOD = 30 + + stdout_data = b"" + stderr_data = b"" + process = None + + try: + # Each test gets its own session so orphaned Kit/Isaac Sim child + # processes cannot send SIGHUP to the next test's process group. + process = subprocess.Popen( + cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + universal_newlines=False, + start_new_session=True, + ) + pgid = os.getpgid(process.pid) + + stdout_fd = process.stdout.fileno() + stderr_fd = process.stderr.fileno() + + try: + import fcntl + + for fd in [stdout_fd, stderr_fd]: + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + except ImportError: + pass + + start_time = time.time() + startup_done = startup_deadline <= 0 + shutdown_deadline = 0.0 + + while process.poll() is None: + elapsed = time.time() - start_time + + if not startup_done: + if b"AppLauncher initialization complete" in stderr_data or b"collected " in stdout_data: + startup_done = True + + if report_file and not shutdown_deadline and os.path.exists(report_file): + shutdown_deadline = time.time() + _SHUTDOWN_GRACE_PERIOD + + kill_reason = None + if not startup_done and elapsed > startup_deadline: + kill_reason = "startup_hang" + elif shutdown_deadline and time.time() > shutdown_deadline: + kill_reason = "shutdown_hang" + elif elapsed > timeout: + kill_reason = "timeout" + + if kill_reason: + pre_kill_diag = _capture_system_diagnostics() + + # Kill the entire process group (test + any Kit children). + try: + os.killpg(pgid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, OSError): + process.kill() + try: + remaining_stdout, remaining_stderr = process.communicate(timeout=5) + stdout_data += remaining_stdout + stderr_data += remaining_stderr + except subprocess.TimeoutExpired: + pass + wall_time = time.time() - start_time + return -1, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag + + try: + ready_fds, _, _ = select.select([stdout_fd, stderr_fd], [], [], 0.1) + + for fd in ready_fds: + with contextlib.suppress(OSError): + if fd == stdout_fd: + chunk = process.stdout.read(1024) + if chunk: + stdout_data += chunk + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + elif fd == stderr_fd: + chunk = process.stderr.read(1024) + if chunk: + stderr_data += chunk + sys.stderr.buffer.write(chunk) + sys.stderr.buffer.flush() + except OSError: + time.sleep(0.1) + continue + + # Drain any output the process wrote before or just after exiting. + try: + remaining_stdout, remaining_stderr = process.communicate(timeout=10) + stdout_data += remaining_stdout + stderr_data += remaining_stderr + except Exception: + pass + + # Kill any orphaned child processes (Kit, Isaac Sim) left by the test. + try: + os.killpg(pgid, signal.SIGKILL) + time.sleep(1) + except (ProcessLookupError, PermissionError, OSError): + pass + + wall_time = time.time() - start_time + return process.returncode, stdout_data, stderr_data, "", wall_time, "" + + except Exception as e: + if process is not None and process.poll() is None: + process.kill() + with contextlib.suppress(Exception): + rem_out, rem_err = process.communicate(timeout=5) + stdout_data += rem_out + stderr_data += rem_err + stdout_data += f"\n[capture error: {e}]\n".encode() + return -1, stdout_data, stderr_data, "", 0.0, "" + + +def _capture_system_diagnostics(): + """Capture system diagnostics (GPU, memory, processes) for crash investigation. + + All errors are caught and reported inline so this never raises. + """ + sections = [] + + try: + r = subprocess.run(["nvidia-smi"], capture_output=True, text=True, timeout=10) + if r.stdout: + sections.append(f"--- nvidia-smi ---\n{r.stdout.strip()}") + except Exception as e: + sections.append(f"--- nvidia-smi --- FAILED: {e}") + + try: + with open("/proc/meminfo") as f: + lines = f.readlines() + keys = ("MemTotal", "MemFree", "MemAvailable", "Committed_AS", "SwapTotal", "SwapFree") + relevant = [line.strip() for line in lines if any(line.startswith(k) for k in keys)] + if relevant: + sections.append("--- /proc/meminfo ---\n" + "\n".join(relevant)) + except Exception as e: + sections.append(f"--- /proc/meminfo --- FAILED: {e}") + + cgroup_lines = [] + for path in ( + "/sys/fs/cgroup/memory.current", + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory.events", + "/sys/fs/cgroup/memory/memory.usage_in_bytes", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + "/sys/fs/cgroup/memory/memory.oom_control", + ): + try: + with open(path) as f: + cgroup_lines.append(f"{path}: {f.read().strip()}") + except FileNotFoundError: + pass + except Exception as e: + cgroup_lines.append(f"{path}: FAILED ({e})") + if cgroup_lines: + sections.append("--- cgroup memory ---\n" + "\n".join(cgroup_lines)) + + try: + r = subprocess.run(["ps", "auxf"], capture_output=True, text=True, timeout=5) + if r.stdout: + sections.append(f"--- process tree (ps auxf) ---\n{r.stdout.strip()}") + except Exception as e: + sections.append(f"--- process tree --- FAILED: {e}") + + try: + r = subprocess.run(["dmesg", "-T"], capture_output=True, text=True, timeout=5) + if r.stdout: + lines = r.stdout.strip().split("\n") + sections.append("--- dmesg (last 30 lines) ---\n" + "\n".join(lines[-30:])) + except Exception: + pass + + return "\n\n".join(sections) + + +def classify_failure_phase( + stdout: str, stderr: str, exit_code: int, wall_time_s: float, timeout_s: float +) -> str | None: + """Classify the failure phase of a benchmark run. + + Priority order (highest to lowest): + 1. oom: exit_code == 137 or "oom-kill" in stderr + 2. hang: wall_time_s >= timeout_s * 0.95 + 3. import: "Traceback" in stdout/stderr AND no "AppLauncher" in stdout + 4. driver: "CudaError" or "CUDA_ERROR_" in stdout/stderr (case-sensitive) + 5. init: "AppLauncher initialization complete" in stdout but no "Step Frametimes" in stdout + 6. runtime: exit_code != 0 and "Step Frametimes" in stdout + 7. null: exit_code == 0 + + Args: + stdout: Captured standard output as a string. + stderr: Captured standard error as a string. + exit_code: Process exit code. + wall_time_s: Measured wall-clock time in seconds. + timeout_s: Configured timeout in seconds. + + Returns: + A failure phase string or None for a clean exit. + """ + combined = stdout + stderr + + # 1. OOM + if exit_code == 137 or "oom-kill" in stderr: + return "oom" + + # 2. Hang + if wall_time_s >= timeout_s * 0.95: + return "hang" + + # 3. Import error + if "Traceback" in combined and "AppLauncher" not in stdout: + return "import" + + # 4. Driver error + if "CudaError" in combined or "CUDA_ERROR_" in combined: + return "driver" + + # 5. Init failure + if exit_code != 0 and "AppLauncher initialization complete" in stdout and "Step Frametimes" not in stdout: + return "init" + + # 6. Runtime failure (partial run then crash) + if exit_code != 0 and "Step Frametimes" in stdout: + return "runtime" + + # 7. Clean exit + return None + + +def run_benchmark(cmd: list, timeout_s: float) -> dict: + """Run a benchmark command and return structured result. + + Args: + cmd: Command list to execute. + timeout_s: Hard timeout in seconds. + + Returns: + Dict with keys: + - exit_code (int): Process exit code. + - stdout_tail (str): Last 2000 characters of combined stdout. + - wall_time_s (float): Measured wall-clock time in seconds. + - startup_time_s (float): Startup time in seconds (0.0 stub for POC). + - failure_phase (str | None): Classified failure phase. + """ + env = os.environ.copy() + + returncode, stdout_bytes, stderr_bytes, kill_reason, wall_time, _ = capture_test_output_with_timeout( + cmd, + timeout=timeout_s, + env=env, + startup_deadline=0, + report_file="", + ) + + stdout_str = stdout_bytes.decode("utf-8", errors="replace") + stderr_str = stderr_bytes.decode("utf-8", errors="replace") + + # Use kill_reason to override exit_code for hang detection + effective_exit_code = returncode + if kill_reason in ("timeout", "startup_hang"): + effective_exit_code = -1 + + failure_phase = classify_failure_phase( + stdout=stdout_str, + stderr=stderr_str, + exit_code=effective_exit_code, + wall_time_s=wall_time, + timeout_s=timeout_s, + ) + + combined_output = stdout_str + stdout_tail = combined_output[-2000:] if len(combined_output) > 2000 else combined_output + + return { + "exit_code": returncode, + "stdout_tail": stdout_tail, + "wall_time_s": wall_time, + "startup_time_s": 0.0, + "failure_phase": failure_phase, + }