From fc84b589e6f3068342bc4d264b1d4ee32d59eb91 Mon Sep 17 00:00:00 2001 From: Neil4561 Date: Thu, 25 Jun 2026 18:17:16 +0000 Subject: [PATCH 1/6] [DEMO] Scope perf-smoke matrix to Cartpole + Velocity-Flat-G1 Trim the benchmark matrix to the two well-baselined demo tasks (physx + newton each) for a fast, clean four-cell verdict table. Demo only; not for merge. --- tools/perf_smoke_test/tasks.json | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/tools/perf_smoke_test/tasks.json b/tools/perf_smoke_test/tasks.json index 21aeb76423ea..542d57122f43 100644 --- a/tools/perf_smoke_test/tasks.json +++ b/tools/perf_smoke_test/tasks.json @@ -26,37 +26,6 @@ } } }, - { - "task_id": "Isaac-Factory-GearMesh-Direct-v0", - "timeout_minutes": 15, - "backends": [ - {"physics": "physx"} - ], - "fps_mean_floor": { - "L40S": { - "physx": 30.0 - } - } - }, - { - "task_id": "Isaac-Repose-Cube-Shadow-Vision-Direct-v0", - "timeout_minutes": 20, - "tags": ["camera"], - "backends": [ - {"physics": "physx"}, - {"physics": "physx", "render": "newton_renderer"}, - {"physics": "newton"}, - {"physics": "newton", "render": "newton_renderer"} - ], - "fps_mean_floor": { - "L40S": { - "physx": 20.0, - "physx_newton_renderer": 0.0, - "newton": 0.0, - "newton_newton_renderer": 0.0 - } - } - }, { "task_id": "Isaac-Velocity-Flat-G1-v0", "timeout_minutes": 12, From f59321b7089c7935d93a5e2629624fb5140527a5 Mon Sep 17 00:00:00 2001 From: Neil4561 Date: Thu, 25 Jun 2026 18:17:16 +0000 Subject: [PATCH 2/6] Clarify benchmark_non_rl module docstring Expand the one-line module docstring to describe what the benchmark measures (per-step frametimes -> effective environment FPS). Documentation only; no behavior change. Serves as the GREEN/normal-PR control in the perf smoke-test demo. --- scripts/benchmarks/benchmark_non_rl.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/benchmarks/benchmark_non_rl.py b/scripts/benchmarks/benchmark_non_rl.py index 7d3b49a7395d..bd174079ca1c 100644 --- a/scripts/benchmarks/benchmark_non_rl.py +++ b/scripts/benchmarks/benchmark_non_rl.py @@ -3,7 +3,11 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Script to benchmark non-RL environment.""" +"""Script to benchmark the step throughput of a non-RL environment. + +Launches a task, steps it for a fixed number of frames, and records per-step +frametimes so downstream tooling can compute effective environment FPS. +""" """Launch Isaac Sim Simulator first.""" From 754083553c6df63a137119beebb486d25c5b13d6 Mon Sep 17 00:00:00 2001 From: Neil4561 Date: Thu, 25 Jun 2026 18:54:07 +0000 Subject: [PATCH 3/6] [DEMO] Add Factory-GearMesh (physx) to demo matrix Include the newly-seeded Factory-GearMesh task so the verdict table shows a third passing task alongside Cartpole and G1. Demo only; not for merge. --- tools/perf_smoke_test/tasks.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/perf_smoke_test/tasks.json b/tools/perf_smoke_test/tasks.json index 542d57122f43..337fe5a1f340 100644 --- a/tools/perf_smoke_test/tasks.json +++ b/tools/perf_smoke_test/tasks.json @@ -26,6 +26,18 @@ } } }, + { + "task_id": "Isaac-Factory-GearMesh-Direct-v0", + "timeout_minutes": 15, + "backends": [ + {"physics": "physx"} + ], + "fps_mean_floor": { + "L40S": { + "physx": 30.0 + } + } + }, { "task_id": "Isaac-Velocity-Flat-G1-v0", "timeout_minutes": 12, From 2d52c225b24bbda1c3b04e15337c222ef542ae6e Mon Sep 17 00:00:00 2001 From: Neil4561 Date: Fri, 26 Jun 2026 08:19:28 +0000 Subject: [PATCH 4/6] Confirm perf smoke blocks before final verdict Add a demo-branch confirmation pass that reruns only cells that initially block and evaluates the median attempt before posting the final sticky summary. This keeps the clean demo PR from relying on a single noisy benchmark draw while preserving the old behavior if no block appears. --- .github/workflows/perf-smoke-test.yaml | 66 ++++- tools/perf_smoke_test/aggregate.py | 174 +++++++++++-- tools/perf_smoke_test/confirm_block_reruns.py | 230 ++++++++++++++++++ tools/perf_smoke_test/oracle.py | 20 ++ 4 files changed, 462 insertions(+), 28 deletions(-) create mode 100644 tools/perf_smoke_test/confirm_block_reruns.py diff --git a/.github/workflows/perf-smoke-test.yaml b/.github/workflows/perf-smoke-test.yaml index e6cd1e5a167d..1bf5f9787f20 100644 --- a/.github/workflows/perf-smoke-test.yaml +++ b/.github/workflows/perf-smoke-test.yaml @@ -605,7 +605,69 @@ jobs: run: | python3 tools/perf_smoke_test/github_gate_context.py - - name: Run aggregate oracle + - name: Run initial aggregate oracle + id: initial_aggregate + run: | + GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)" + if [[ -z "${GPU_MODEL}" ]]; then + GPU_MODEL="unknown-gpu" + fi + INITIAL_VERDICTS_FILE="${{ github.workspace }}/initial_verdicts.json" + python3 tools/perf_smoke_test/aggregate.py \ + --artifacts_dir artifacts/ \ + --gpu_model "${GPU_MODEL}" \ + --gate_config tools/perf_smoke_test/gate_config.json \ + --baseline_branch perf-baselines \ + --baseline_remote origin \ + --baseline_push_retries 3 \ + --base_sha "${{ steps.gate_context.outputs.base_sha }}" \ + --target_branch "${{ steps.gate_context.outputs.target_branch }}" \ + --source_branch "${{ steps.gate_context.outputs.source_branch }}" \ + --allow_baseline_update "false" \ + --trusted_source "${{ steps.gate_context.outputs.trusted_source }}" \ + --verdicts_file "${INITIAL_VERDICTS_FILE}" + + - name: Confirm BLOCK cells + run: | + set -euo pipefail + INITIAL_VERDICTS_FILE="${{ github.workspace }}/initial_verdicts.json" + BLOCK_COUNT="$(python3 - <<'PY' + import json + from pathlib import Path + records = json.loads(Path("initial_verdicts.json").read_text()) + print(sum(1 for record in records if record.get("verdict") == "BLOCK")) + PY + )" + if [ "${BLOCK_COUNT}" = "0" ]; then + echo "::notice::No BLOCK cells found; skipping confirmation reruns" + exit 0 + fi + echo "::notice::Confirming ${BLOCK_COUNT} BLOCK cell(s) with two additional reruns each" + + DOCKER_CONFIG_DIR="$(mktemp -d)" + echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json" + export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}" + + CI_IMAGE_REF="${{ vars.PERF_SMOKE_CI_IMAGE || 'nvcr.io/nvidian/isaac-lab:latest-perf' }}" + REGISTRY="${CI_IMAGE_REF%%/*}" + if [ "${REGISTRY}" = "nvcr.io" ] && [ -n "${NGC_API_KEY:-}" ]; then + echo "${NGC_API_KEY}" | docker login nvcr.io -u '$oauthtoken' --password-stdin + fi + docker pull "${CI_IMAGE_REF}" + docker tag "${CI_IMAGE_REF}" "${{ env.CI_IMAGE_TAG }}" + rm -rf "${DOCKER_CONFIG_DIR}" + + mkdir -p "${{ github.workspace }}/jit-cache/warp" "${{ github.workspace }}/jit-cache/nv" + mkdir -p "${{ github.workspace }}/kit-cache" + chmod -R a+rwX "${{ github.workspace }}" 2>/dev/null || true + + python3 tools/perf_smoke_test/confirm_block_reruns.py \ + --verdicts_file "${INITIAL_VERDICTS_FILE}" \ + --workspace "${{ github.workspace }}" \ + --ci_image_tag "${{ env.CI_IMAGE_TAG }}" \ + --reruns 2 + + - name: Run final aggregate oracle id: aggregate run: | GPU_MODEL="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs)" @@ -654,7 +716,7 @@ jobs: core.info('verdict_summary.md not found; posting placeholder'); } const marker = ''; - const body = `${marker}\n## Performance Smoke Test\n\n${table}\n\n_Updated for ${context.sha.slice(0, 8)} · [run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})_`; + const body = `${marker}\n${table}\n\n_Updated for ${context.sha.slice(0, 8)} · [run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})_`; const { owner, repo } = context.repo; const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number }); const existing = comments.find(c => c.body && c.body.includes(marker)); diff --git a/tools/perf_smoke_test/aggregate.py b/tools/perf_smoke_test/aggregate.py index 95f4a9dae3e4..567bea8e9d44 100644 --- a/tools/perf_smoke_test/aggregate.py +++ b/tools/perf_smoke_test/aggregate.py @@ -49,6 +49,7 @@ def _parse_args(): parser.add_argument("--baselines_dir", type=Path, default=None, help="Flat-file baseline directory; bypasses git") parser.add_argument("--allow_baseline_update", default="false") parser.add_argument("--summary_file", default=None) + parser.add_argument("--verdicts_file", default=None) parser.add_argument("--base_sha", default=None, help="PR base SHA for ancestry-aware baseline matching") parser.add_argument("--target_branch", default=None, help="Target protected branch, e.g. main/develop/release/x") parser.add_argument("--source_branch", default=None, help="Branch that produced baseline updates") @@ -84,6 +85,14 @@ def _fmt(value, decimals: int = 1) -> str: return f"{value:.{decimals}f}" if value is not None else "N/A" +def _fmt_pct(value, decimals: int = 2) -> str: + return f"{value:.{decimals}f}%" if value is not None else "N/A" + + +def _fmt_signed_pct(value, decimals: int = 2) -> str: + return f"{value:+.{decimals}f}%" if value is not None else "N/A" + + def _short_sha(value: str | None) -> str: return value[:12] if value else "none" @@ -109,42 +118,128 @@ def _hard_floor(bench_result: dict, gpu_model: str, backend: str) -> float: return 0.0 -def _build_summary_table(rows: list[tuple]) -> str: +def _runtime_label(bench_result: dict) -> str: + """Return the compact runtime label shown in the sticky summary.""" + gpu_diag = bench_result.get("gpu_diag") or {} + provenance = bench_result.get("provenance") or {} + software = provenance.get("software") or {} + return ", ".join( + part + for part in ( + f"cuda={gpu_diag.get('cuda_version')}" if gpu_diag.get("cuda_version") else "", + f"driver={gpu_diag.get('nvidia_driver_version')}" if gpu_diag.get("nvidia_driver_version") else "", + f"warp={software.get('warp')}" if software.get("warp") else "", + ) + if part + ) + + +def _collapse_values(values: list[str]) -> str: + """Render a list of possibly repeated values as one summary value.""" + unique = sorted({value for value in values if value}) + if not unique: + return "N/A" + if len(unique) == 1: + return unique[0] + return "varies: " + "; ".join(unique) + + +def _uniform_value(values: list[str]) -> str | None: + """Return the shared value when every row agrees, otherwise ``None``.""" + unique = {value for value in values if value} + return next(iter(unique)) if len(unique) == 1 else None + + +def _threshold_sources(rows: list[tuple]) -> list[str]: + return [result.threshold_source for result, _ in rows] + + +def _build_run_context(rows: list[tuple]) -> str: + """Return run-wide fields that should not be repeated in every table row.""" + gpu_names = [] + runtimes = [] + for _result, bench_result in rows: + gpu_diag = bench_result.get("gpu_diag") or {} + launch_config = bench_result.get("launch_config") or {} + gpu_names.append(gpu_diag.get("gpu_name") or launch_config.get("gpu_model_raw") or launch_config.get("gpu_model", "")) + runtimes.append(_runtime_label(bench_result)) + lines = [ - "| Task | Backend | Verdict | FPS | Baseline | Samples | Regression% | Floor | Threshold | Phase | " - "Retry | GPU | Runtime | Note |", - "|---|---|---|---:|---:|---:|---:|---:|---|---|---|---|---|---|", + "### Run context", + "", + f"- **GPU:** {_collapse_values(gpu_names)}", + f"- **Runtime:** {_collapse_values(runtimes)}", ] + shared_threshold = _uniform_value(_threshold_sources(rows)) + if shared_threshold is not None: + lines.append(f"- **Threshold:** {shared_threshold} (same for every task in this run)") + return "\n".join(lines) + + +def _build_summary_table(rows: list[tuple]) -> str: + show_threshold = _uniform_value(_threshold_sources(rows)) is None + + header = ["Task", "Backend", "Verdict", "FPS", "Baseline", "Delta (+ faster / - slower)", "Noise", "Samples"] + if show_threshold: + header.append("Threshold") + header += ["Phase", "Notes", "Retried"] + + aligns = ["---", "---", "---", "---:", "---:", "---:", "---:", "---:"] + if show_threshold: + aligns.append("---") + aligns += ["---", "---", "---"] + + lines = ["| " + " | ".join(header) + " |", "|" + "|".join(aligns) + "|"] for result, bench_result in rows: - gpu_diag = bench_result.get("gpu_diag") or {} - launch_config = bench_result.get("launch_config") or {} - gpu_name = gpu_diag.get("gpu_name") or launch_config.get("gpu_model_raw") or launch_config.get("gpu_model", "") - provenance = bench_result.get("provenance") or {} - software = provenance.get("software") or {} - runtime = ", ".join( - part - for part in ( - f"cuda={gpu_diag.get('cuda_version')}" if gpu_diag.get("cuda_version") else "", - f"driver={gpu_diag.get('nvidia_driver_version')}" if gpu_diag.get("nvidia_driver_version") else "", - f"warp={software.get('warp')}" if software.get("warp") else "", - ) - if part - ) note_parts = [part for part in (result.note, bench_result.get("config_mismatch")) if part] if bench_result.get("p99_over_median") is not None: note_parts.append(f"p99/med={bench_result['p99_over_median']}") if bench_result.get("outlier_count") is not None: note_parts.append(f"outliers={bench_result['outlier_count']}") - lines.append( - f"| {result.task_id} | {result.backend} | {result.verdict.value}" - f" | {_fmt(result.measured_fps)} | {_fmt(result.baseline_fps)} | {result.baseline_sample_count}" - f" | {_fmt(result.regression_pct, 2)} | {_fmt(result.hard_floor_fps)} | {result.threshold_source}" - f" | {result.failure_phase or ''} | {result.was_retried} | {gpu_name}" - f" | {runtime} | {'; '.join(note_parts)} |" - ) + cells = [ + result.task_id, + result.backend, + result.verdict.value, + _fmt(result.measured_fps), + _fmt(result.baseline_fps), + _fmt_signed_pct(result.regression_pct), + _fmt_pct(result.baseline_noise_pct), + str(result.baseline_sample_count), + ] + if show_threshold: + cells.append(result.threshold_source) + cells += [ + result.failure_phase or "", + "; ".join(note_parts), + "yes" if result.was_retried else "no", + ] + lines.append("| " + " | ".join(cells) + " |") return "\n".join(lines) +def _build_summary_notes() -> str: + return "\n".join( + [ + "### How to read this table", + "", + "- **PASS**: no meaningful slowdown was detected.", + "- **WARN**: suspicious or uncertain result, such as insufficient baselines, retry-only success, or a" + " slowdown in the WARN band.", + "- **BLOCK**: a blocking-level regression signal was detected. In this POC, BLOCK is advisory (it flags a" + " regression) and only fails the check when the gate is explicitly configured as blocking.", + "- **HARD_FAILURE**: the benchmark did not produce usable FPS data, for example an import/init/runtime" + " failure or config mismatch.", + "- **Delta**: change in FPS versus the baseline. ``+`` means faster than baseline (speedup);" + " ``-`` means slower than baseline (slowdown).", + "- **Noise**: baseline MAD as a percent of baseline FPS. Higher noise means the task/backend naturally" + " varies more from run to run.", + "- **Phase**: the stage a HARD_FAILURE happened in (e.g. ``import``, ``init``, ``runtime``); blank when" + " the benchmark ran to completion.", + "- **Retried**: ``yes`` if the cell only passed after an automatic re-run.", + ] + ) + + def _write_github_output(**values) -> None: github_output = os.environ.get("GITHUB_OUTPUT", "") if not github_output: @@ -195,6 +290,7 @@ def main() -> int: baselines_updated = False baseline_update_failed = False pending_git_updates: list[BaselineUpdateRecord] = [] + verdict_records = [] for artifact_dir, bench_result in items: task_id = bench_result["task_id"] @@ -234,6 +330,19 @@ def main() -> int: min_block_regression_pct=min_block_regression_pct, ) rows.append((oracle_result, bench_result)) + verdict_records.append( + { + "task_id": task_id, + "backend": backend, + "verdict": oracle_result.verdict.value, + "artifact_dir": str(artifact_dir), + "measured_fps": oracle_result.measured_fps, + "baseline_fps": oracle_result.baseline_fps, + "regression_pct": oracle_result.regression_pct, + "physics_backend": bench_result.get("physics_backend"), + "render_backend": bench_result.get("render_backend"), + } + ) print( f"[aggregate] {task_id}/{backend}: {oracle_result.verdict.value}" @@ -314,13 +423,22 @@ def main() -> int: print(f"::error::Baseline push failed: {exc}") table = _build_summary_table(rows) + if args.verdicts_file: + with open(args.verdicts_file, "w") as fh: + json.dump(verdict_records, fh, indent=2) + fh.write("\n") + print("\n## Performance Smoke Results\n") + print(_build_run_context(rows)) + print() + print(_build_summary_notes()) + print() print(table) print() if args.summary_file: with open(args.summary_file, "a") as fh: - fh.write("\n## Performance Smoke Results\n\n") + fh.write("## Performance Smoke Results\n\n") if not use_flat: fh.write(f"Baseline read SHA: `{_short_sha(baseline_read_sha)}`\n\n") if baseline_push_result and baseline_push_result.pushed_sha: @@ -328,6 +446,10 @@ def main() -> int: f"Baseline pushed SHA: `{_short_sha(baseline_push_result.pushed_sha)}` " f"after {baseline_push_result.attempts} attempt(s)\n\n" ) + fh.write(_build_run_context(rows)) + fh.write("\n\n") + fh.write(_build_summary_notes()) + fh.write("\n\n") fh.write(table) fh.write("\n") diff --git a/tools/perf_smoke_test/confirm_block_reruns.py b/tools/perf_smoke_test/confirm_block_reruns.py new file mode 100644 index 000000000000..e1dc1048f6c7 --- /dev/null +++ b/tools/perf_smoke_test/confirm_block_reruns.py @@ -0,0 +1,230 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Rerun BLOCK cells and annotate artifacts with confirmation FPS attempts.""" + +import argparse +import json +import statistics +import subprocess +import sys +import time +from pathlib import Path + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--verdicts_file", required=True, type=Path) + parser.add_argument("--workspace", required=True, type=Path) + parser.add_argument("--ci_image_tag", required=True) + parser.add_argument("--reruns", type=int, default=2) + return parser.parse_args() + + +def _safe_name(value: str) -> str: + return "".join(ch if ch.isalnum() else "-" for ch in value) + + +def _extract_fps_series(perf_info: list[dict]) -> list[float]: + for phase in perf_info: + if phase.get("phase_name") == "runtime": + for measurement in phase.get("measurements", []): + value = measurement.get("value", {}) + if measurement.get("name", "").endswith("Step Frametimes") and isinstance(value, dict): + return list(value.get("Environment step effective FPS", [])) + return [] + + +def _excluded_frames(launch_config: dict) -> frozenset[int]: + indices: set[int] = set() + for entry in launch_config.get("excluded_frames_raw") or []: + if isinstance(entry, list): + indices.update(range(int(entry[0]), int(entry[1]) + 1)) + else: + indices.add(int(entry)) + return frozenset(indices) + + +def _gate_mean_fps(perf_info_path: Path, launch_config: dict) -> float: + with perf_info_path.open() as fh: + series = _extract_fps_series(json.load(fh)) + excluded = _excluded_frames(launch_config) + filtered = [fps for idx, fps in enumerate(series) if idx not in excluded] + if not filtered: + raise RuntimeError(f"no FPS samples found in {perf_info_path}") + return statistics.mean(filtered) + + +def _run_confirm_attempt( + *, + workspace: Path, + artifact_dir: Path, + launch_config: dict, + ci_image_tag: str, + task_id: str, + backend_key: str, + attempt: int, +) -> float | None: + attempt_dir = artifact_dir / f"confirm_attempt_{attempt}" + attempt_dir.mkdir(parents=True, exist_ok=True) + log_file = attempt_dir / "benchmark.log" + timeout_s = int(launch_config.get("timeout_minutes", 12)) * 60 + container_name = f"perf-confirm-{_safe_name(task_id)}-{_safe_name(backend_key)}-{int(time.time())}-{attempt}" + + hydra_args = " ".join(str(arg) for arg in launch_config.get("hydra_args") or []) + seed = launch_config.get("seed") + seed_arg = f"--seed {seed}" if seed is not None else "" + + subprocess.run(["docker", "rm", "-f", container_name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + docker_cmd = [ + "docker", + "run", + "-d", + "--name", + container_name, + "--init", + "--stop-timeout", + "10", + "--entrypoint", + "bash", + "--gpus", + "all", + "--network=host", + "--security-opt=no-new-privileges:true", + "--ulimit", + "nofile=65536:65536", + "--ulimit", + "nproc=4096:4096", + "-e", + "OMNI_KIT_ACCEPT_EULA=yes", + "-e", + "ACCEPT_EULA=Y", + "-e", + "OMNI_KIT_DISABLE_CUP=1", + "-e", + "ISAAC_SIM_HEADLESS=1", + "-e", + "PYTHONUNBUFFERED=1", + "-e", + "PYTHONDONTWRITEBYTECODE=1", + "-e", + "WARP_CACHE_PATH=/tmp/jit-cache/warp", + "-e", + "CUDA_CACHE_PATH=/tmp/jit-cache/nv", + "-v", + f"{attempt_dir}:/tmp/bench_out", + "-v", + f"{workspace / 'jit-cache'}:/tmp/jit-cache", + "-v", + f"{workspace / 'kit-cache'}:/isaac-sim/kit/cache", + "-v", + f"{workspace}:/workspace/isaaclab", + ci_image_tag, + "-c", + "\n".join( + [ + "set -e", + "cd /workspace/isaaclab", + "rm -f _isaac_sim", + "ln -s /isaac-sim _isaac_sim", + "./isaaclab.sh -p scripts/benchmarks/benchmark_non_rl.py " + f"--task '{task_id}' " + f"--num_envs {launch_config['num_envs']} " + f"--num_frames {launch_config['num_frames']} " + "--benchmark_backend json " + "--output_path /tmp/bench_out " + f"{seed_arg} " + f"{hydra_args}", + ] + ), + ] + + subprocess.run(docker_cmd, check=True) + wait_returncode = 1 + try: + wait = subprocess.run( + ["timeout", str(timeout_s), "docker", "wait", container_name], + capture_output=True, + text=True, + check=False, + ) + wait_returncode = wait.returncode + exit_code = int((wait.stdout or "1").strip() or "1") if wait_returncode == 0 else 1 + finally: + if wait_returncode != 0: + subprocess.run(["docker", "kill", container_name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + logs = subprocess.run(["docker", "logs", container_name], capture_output=True, check=False) + log_file.write_bytes((logs.stdout or b"") + (logs.stderr or b"")) + subprocess.run(["docker", "kill", container_name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.run(["docker", "rm", container_name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + if exit_code != 0: + print(f"[confirm] attempt {attempt} failed for {task_id}/{backend_key} (exit={exit_code})") + return None + + outputs = sorted(attempt_dir.glob("benchmark_non_rl_*.json")) + if not outputs: + print(f"[confirm] attempt {attempt} produced no benchmark JSON for {task_id}/{backend_key}") + return None + fps = _gate_mean_fps(outputs[-1], launch_config) + print(f"[confirm] {task_id}/{backend_key} attempt {attempt}: {fps:.1f} FPS") + return fps + + +def main() -> int: + args = _parse_args() + with args.verdicts_file.open() as fh: + records = json.load(fh) + + args.workspace.mkdir(parents=True, exist_ok=True) + (args.workspace / "jit-cache" / "warp").mkdir(parents=True, exist_ok=True) + (args.workspace / "jit-cache" / "nv").mkdir(parents=True, exist_ok=True) + (args.workspace / "kit-cache").mkdir(parents=True, exist_ok=True) + + block_records = [record for record in records if record.get("verdict") == "BLOCK"] + if not block_records: + print("[confirm] no BLOCK cells to confirm") + return 0 + + for record in block_records: + artifact_dir = Path(record["artifact_dir"]) + result_path = artifact_dir / "perf_smoke_test_result.json" + info_path = artifact_dir / "perf_smoke_test_info.json" + with result_path.open() as fh: + bench_result = json.load(fh) + launch_config = bench_result.get("launch_config") or {} + task_id = bench_result["task_id"] + backend_key = bench_result.get("backend_key") or bench_result.get("backend") + attempts = [_gate_mean_fps(info_path, launch_config)] + + print(f"[confirm] confirming {task_id}/{backend_key}; initial={attempts[0]:.1f} FPS") + for offset in range(args.reruns): + fps = _run_confirm_attempt( + workspace=args.workspace, + artifact_dir=artifact_dir, + launch_config=launch_config, + ci_image_tag=args.ci_image_tag, + task_id=task_id, + backend_key=backend_key, + attempt=offset + 2, + ) + if fps is not None: + attempts.append(fps) + + bench_result["confirmation_fps_attempts"] = attempts + bench_result["confirmation_policy"] = { + "trigger": "initial_block", + "requested_reruns": args.reruns, + "completed_attempts": len(attempts), + } + with result_path.open("w") as fh: + json.dump(bench_result, fh, indent=2) + fh.write("\n") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/perf_smoke_test/oracle.py b/tools/perf_smoke_test/oracle.py index 964f38b74e2b..b3376b10c2ff 100644 --- a/tools/perf_smoke_test/oracle.py +++ b/tools/perf_smoke_test/oracle.py @@ -57,6 +57,7 @@ class OracleResult: block_threshold_fps: float | None = None hard_floor_fps: float | None = None min_block_regression_pct: float = MIN_BLOCK_REGRESSION_PCT + baseline_noise_pct: float | None = None note: str | None = None @@ -180,9 +181,20 @@ def compare( fps_p5 = _percentile(sorted_filtered, 5.0) fps_p95 = _percentile(sorted_filtered, 95.0) + confirmation_attempts = [ + float(value) + for value in bench_result.get("confirmation_fps_attempts", []) + if isinstance(value, (int, float)) + ] + if len(confirmation_attempts) >= 3: + mean_fps = statistics.median(confirmation_attempts) + baseline_fps = baseline.median_fps if baseline is not None else None baseline_sample_count = baseline.sample_count if baseline is not None else 0 baseline_source = baseline.source if baseline is not None else "none" + baseline_noise_pct = None + if baseline is not None and baseline.median_fps > 0.0: + baseline_noise_pct = baseline.mad_fps / baseline.median_fps * 100.0 regression_pct = None if baseline_fps: regression_pct = ((mean_fps - baseline_fps) / baseline_fps) * 100.0 @@ -223,6 +235,13 @@ def compare( verdict = OracleVerdict.WARN note = note or "was_retried" + if confirmation_attempts: + if verdict == OracleVerdict.BLOCK: + note = note or f"block_confirmed(n={len(confirmation_attempts)})" + else: + verdict = OracleVerdict.WARN + note = f"block_not_reproduced(n={len(confirmation_attempts)})" + return OracleResult( verdict=verdict, bisect_verdict=_bisect_verdict(verdict, was_retried, failure_phase), @@ -246,5 +265,6 @@ def compare( block_threshold_fps=block_threshold, hard_floor_fps=hard_floor_fps, min_block_regression_pct=float(min_block_regression_pct), + baseline_noise_pct=baseline_noise_pct, note=note, ) From 13ac548322f7a008f3e5abfd5b1b95fc9bc79761 Mon Sep 17 00:00:00 2001 From: Neil4561 Date: Fri, 26 Jun 2026 08:59:26 +0000 Subject: [PATCH 5/6] Fix confirm rerun artifact mount path Resolve confirmation attempt artifact directories before mounting them into Docker so reruns can write benchmark outputs successfully. --- tools/perf_smoke_test/confirm_block_reruns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/perf_smoke_test/confirm_block_reruns.py b/tools/perf_smoke_test/confirm_block_reruns.py index e1dc1048f6c7..ef5da0b66bfd 100644 --- a/tools/perf_smoke_test/confirm_block_reruns.py +++ b/tools/perf_smoke_test/confirm_block_reruns.py @@ -69,6 +69,7 @@ def _run_confirm_attempt( ) -> float | None: attempt_dir = artifact_dir / f"confirm_attempt_{attempt}" attempt_dir.mkdir(parents=True, exist_ok=True) + attempt_dir = attempt_dir.resolve() log_file = attempt_dir / "benchmark.log" timeout_s = int(launch_config.get("timeout_minutes", 12)) * 60 container_name = f"perf-confirm-{_safe_name(task_id)}-{_safe_name(backend_key)}-{int(time.time())}-{attempt}" From 74bab6f7eda788506ed3304078fe583281e65df8 Mon Sep 17 00:00:00 2001 From: Neil4561 Date: Fri, 26 Jun 2026 09:35:13 +0000 Subject: [PATCH 6/6] Reset perf smoke demo baseline epoch Bump the demo task matrix to baseline epoch 2 so the clean PR can compare against freshly seeded samples instead of the noisy earlier baseline window. --- tools/perf_smoke_test/tasks.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/perf_smoke_test/tasks.json b/tools/perf_smoke_test/tasks.json index 337fe5a1f340..a3d6143fc02c 100644 --- a/tools/perf_smoke_test/tasks.json +++ b/tools/perf_smoke_test/tasks.json @@ -9,7 +9,8 @@ "excluded_frames": [[0, 100]], "camera_resolution": null, "timeout_minutes": 10, - "tags": ["always"] + "tags": ["always"], + "baseline_epoch": 2 }, "tasks": [ {