plumb more memory stats into artifacts, rename gpu_diag -> runtime_resources

Author: angehu-nv

tools/perf_smoke_test/aggregate.py

@@ -126,16 +126,22 @@ def _build_summary_table(rows: list[tuple]) -> str:
         "|---|---|---|---:|---:|---:|---:|---:|---|---|---|---|---|---|",
     ]
     for result, bench_result in rows:
-        gpu_diag = bench_result.gpu_diag or {}
+        runtime_resources = bench_result.runtime_resources or {}
         launch_config = bench_result.launch_config or {}
-        gpu_name = gpu_diag.get("gpu_name") or launch_config.get("gpu_model_raw") or launch_config.get("gpu_model", "")
+        gpu_name = (
+            runtime_resources.get("gpu_name")
+            or launch_config.get("gpu_model_raw")
+            or launch_config.get("gpu_model", "")
+        )
         provenance = bench_result.provenance or {}
         software = provenance.get("software") or {}
         runtime = ", ".join(
             part
             for part in (
-                f"cuda={gpu_diag.get('cuda_version')}" if gpu_diag.get("cuda_version") else "",
-                f"driver={gpu_diag.get('nvidia_driver_version')}" if gpu_diag.get("nvidia_driver_version") else "",
+                f"cuda={runtime_resources.get('cuda_version')}" if runtime_resources.get("cuda_version") else "",
+                f"driver={runtime_resources.get('nvidia_driver_version')}"
+                if runtime_resources.get("nvidia_driver_version")
+                else "",
                 f"warp={software.get('warp')}" if software.get("warp") else "",
             )
             if part

tools/perf_smoke_test/baseline_manager.py

@@ -311,7 +311,7 @@ def make_sample_metadata(
     provenance = bench_result.get("provenance") or {}
     git_info = provenance.get("git") or {}
     software = provenance.get("software") or {}
-    gpu_diag = bench_result.get("gpu_diag") or {}
+    runtime_resources = bench_result.get("runtime_resources") or {}
     metadata = {
         "schema_version": 1,
         "fps": float(fps),
@@ -345,8 +345,8 @@ def make_sample_metadata(
         "runtime": {
             "isaacsim": software.get("isaacsim"),
             "warp": software.get("warp"),
-            "cuda": (gpu_diag or {}).get("cuda_version"),
-            "driver": (gpu_diag or {}).get("nvidia_driver_version"),
+            "cuda": (runtime_resources or {}).get("cuda_version"),
+            "driver": (runtime_resources or {}).get("nvidia_driver_version"),
         },
     }
     metadata["sample_id"] = _stable_sample_id(metadata)

tools/perf_smoke_test/benchmark_result_adapter.py

@@ -10,7 +10,7 @@
 refactor Part 1, PR #6197) serialized by
 :func:`~isaaclab.test.benchmark.serialize.write_bundle_file`.  This module is the
 single point that reads that JSON and projects it into the flat
-``provenance`` / ``gpu_diag`` / fps / ``benchmark_info`` shapes the gate's
+``provenance`` / ``runtime_resources`` / fps / ``benchmark_info`` shapes the gate's
 :mod:`oracle` and :mod:`build_bench_result` consume, replacing the legacy
 phase-array parsing.
 
@@ -198,11 +198,42 @@ def provenance(bundle: dict) -> dict:
     return {"hardware": hardware, "software": software, "git": git}
 
 
-def gpu_diag(bundle: dict) -> dict:
-    """Return the human/debug GPU diagnostics block (non-gating publish info)."""
+def _gb_to_mb(value: Any) -> float | None:
+    """Return ``value`` GB converted to MB (2 dp), or ``None`` if not numeric."""
+    return round(float(value) * 1024, 2) if isinstance(value, (int, float)) else None
+
+
+def _pct(value: Any) -> float | None:
+    """Return ``value`` as a utilisation percent (2 dp), or ``None`` if not numeric."""
+    return round(float(value), 2) if isinstance(value, (int, float)) else None
+
+
+def runtime_resources(bundle: dict) -> dict:
+    """Return the GPU-diagnostics + resource-utilisation block (non-gating publish info).
+
+    Combines GPU identity (name / total memory / CUDA / driver) with the run's
+    measured resource utilisation from the bundle's ``resources`` section: VRAM
+    (mean + peak), system RAM (mean + peak), and GPU/CPU utilisation (mean). Every
+    field here is informational — it is published for humans but never feeds the
+    gate verdict or the ``runtime_contract_hash``.
+
+    Memory is reported in MB (the schema stores GB); utilisation in percent. Two
+    semantic caveats worth remembering when reading these values:
+
+    * ``gpu_mem_*`` is **device-wide** VRAM (``nvidia-smi memory.used`` includes
+      any other process on the GPU) — accurate on a 1-benchmark-per-GPU runner.
+    * ``system_ram_*`` is the benchmark **process** resident set size (psutil
+      ``memory_info().rss``), not whole-host RAM and excluding child processes.
+
+    Absent/malformed sub-sections drop their fields (``None`` filtered out) so a
+    partial bundle degrades gracefully instead of crashing the projection.
+    """
     gpu = _current_gpu(bundle)
-    gpu_mem = _as_dict(_as_dict(bundle.get("resources")).get("gpu_mem_gb"))
-    mem_used_gb = gpu_mem.get("mean")
+    resources = _as_dict(bundle.get("resources"))
+    gpu_mem = _as_dict(resources.get("gpu_mem_gb"))
+    ram = _as_dict(resources.get("ram_gb"))
+    gpu_util = _as_dict(resources.get("gpu_util_pct"))
+    cpu_util = _as_dict(resources.get("cpu_util_pct"))
     # schema-v1 Hardware has no CUDA-runtime field; use the CUDA bindings version
     # (Versions.cuda_bindings) as the closest available proxy for display.
     cuda_version = _as_dict(bundle.get("versions")).get("cuda_bindings")
@@ -211,7 +242,12 @@ def gpu_diag(bundle: dict) -> dict:
         "gpu_total_memory_gb": gpu.get("mem_gb"),
         "cuda_version": cuda_version,
         "nvidia_driver_version": gpu_driver_version(),
-        "gpu_mem_used_mb": round(float(mem_used_gb) * 1024, 2) if isinstance(mem_used_gb, (int, float)) else None,
+        "gpu_mem_used_mb": _gb_to_mb(gpu_mem.get("mean")),
+        "gpu_mem_peak_mb": _gb_to_mb(gpu_mem.get("peak")),
+        "gpu_util_pct": _pct(gpu_util.get("mean")),
+        "system_ram_used_mb": _gb_to_mb(ram.get("mean")),
+        "system_ram_peak_mb": _gb_to_mb(ram.get("peak")),
+        "cpu_util_pct": _pct(cpu_util.get("mean")),
     }
     return {k: v for k, v in diag.items() if v is not None}
 
@@ -261,5 +297,5 @@ def project_runtime(bundle: dict) -> RuntimeSample | None:
         render_backend=info.get("render_backend"),
         presets=info.get("presets") or [],
         provenance=provenance(bundle),
-        gpu_diag=gpu_diag(bundle),
+        runtime_resources=runtime_resources(bundle),
     )

tools/perf_smoke_test/build_bench_result.py

@@ -236,13 +236,13 @@ def main() -> int:
             observed_backend = backend_identity_from_benchmark_info(benchmark_info)
             runtime_contract, runtime_contract_hash = build_runtime_contract(
                 provenance=sample.provenance,
-                gpu_diag=sample.gpu_diag,
+                runtime_resources=sample.runtime_resources,
                 backend=expected_backend,
                 policy=runtime_policy,
             )
             runtime_info = build_runtime_publish_info(
                 provenance=sample.provenance,
-                gpu_diag=sample.gpu_diag,
+                runtime_resources=sample.runtime_resources,
                 policy=runtime_policy,
             )
             config_mismatch = _config_drift(benchmark_info, launch_config)
@@ -278,7 +278,7 @@ def main() -> int:
         runtime_contract=runtime_contract,
         runtime_contract_hash=runtime_contract_hash,
         runtime_info=runtime_info,
-        gpu_diag=(sample.gpu_diag or None) if sample else None,
+        runtime_resources=(sample.runtime_resources or None) if sample else None,
         provenance=sample.provenance if sample else None,
         launch_config=launch_config,
         launch_config_hash=launch_config.get("launch_config_hash"),

tools/perf_smoke_test/contracts.py

@@ -30,7 +30,7 @@ class RuntimeSample:
     """Adapter projection of a schema-v1 ``RuntimeBundle`` into the gate's fields.
 
     Aggregates (fps/startup) and run identity are typed; ``provenance`` and
-    ``gpu_diag`` remain dicts (open provenance payloads the gate carries through).
+    ``runtime_resources`` remain dicts (open provenance payloads the gate carries through).
     """
 
     fps_mean: float | None = None
@@ -48,7 +48,7 @@ class RuntimeSample:
     render_backend: str | None = None
     presets: list[str] = field(default_factory=list)
     provenance: dict[str, Any] = field(default_factory=dict)
-    gpu_diag: dict[str, Any] = field(default_factory=dict)
+    runtime_resources: dict[str, Any] = field(default_factory=dict)
 
     def benchmark_info(self) -> dict[str, Any]:
         """Return the run's self-reported identity as a dict.
@@ -108,7 +108,7 @@ class BenchResult:
     runtime_contract: dict[str, Any] | None = None
     runtime_contract_hash: str | None = None
     runtime_info: dict[str, Any] | None = None
-    gpu_diag: dict[str, Any] | None = None
+    runtime_resources: dict[str, Any] | None = None
     provenance: dict[str, Any] | None = None
     launch_config: dict[str, Any] = field(default_factory=dict)
     launch_config_hash: str | None = None

tools/perf_smoke_test/docs/example_smoke_test_result.json

@@ -84,16 +84,21 @@
     "publish_only": {
       "hardware.gpu_compute_capability": "12.0",
       "hardware.gpu_total_memory_gb": 94.97,
-      "gpu_diag.cuda_version": "12.8",
-      "gpu_diag.nvidia_driver_version": "595.71.05"
+      "runtime_resources.cuda_version": "12.8",
+      "runtime_resources.nvidia_driver_version": "595.71.05"
     }
   },
-  "gpu_diag": {
+  "runtime_resources": {
     "gpu_name": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
     "gpu_total_memory_gb": 94.97,
     "cuda_version": "12.8",
     "nvidia_driver_version": "595.71.05",
-    "gpu_mem_used_mb": 798.72
+    "gpu_mem_used_mb": 798.72,
+    "gpu_mem_peak_mb": 851.97,
+    "gpu_util_pct": 96.4,
+    "system_ram_used_mb": 11894.27,
+    "system_ram_peak_mb": 12408.83,
+    "cpu_util_pct": 33.7
   },
   "provenance": {					-> Audit context: records the machine/software we actually ran on. Useful for debugging hardware or environment differences.
     "hardware": {

tools/perf_smoke_test/docs/module-interfaces.md

@@ -34,8 +34,8 @@ Examples: `NVIDIA L40S -> l40s`, `RTX6000 -> rtx_6000`, `NVIDIA GeForce RTX 5090
 Builds the runtime compatibility contract used for baseline matching. The matching code only sees `runtime_contract_hash`; package/version field selection stays in `gate_config.py`.
 
 ```python
-def build_runtime_contract(*, provenance: dict | None, gpu_diag: dict | None, backend: BackendIdentity, policy: Mapping[str, Any]) -> tuple[dict, str]
-def build_runtime_publish_info(*, provenance: dict | None, gpu_diag: dict | None, policy: Mapping[str, Any]) -> dict
+def build_runtime_contract(*, provenance: dict | None, runtime_resources: dict | None, backend: BackendIdentity, policy: Mapping[str, Any]) -> tuple[dict, str]
+def build_runtime_publish_info(*, provenance: dict | None, runtime_resources: dict | None, policy: Mapping[str, Any]) -> dict
 ```
 
 Default compatibility fields smoke test on top-level active-path packages: IsaacSim, IsaacLab, Torch, Warp, the active physics package (`isaaclab_physx` or `isaaclab_newton`/`newton`), and `isaaclab_ov` for renderer backends. CUDA version, NVIDIA driver, GPU memory, and compute capability are published for humans but do not affect the compatibility hash by default.
@@ -124,7 +124,7 @@ class OracleResult:
     measured_fps: float | None      # raw_fps_mean (steady-state mean); None on HARD_FAILURE
     baseline_fps: float | None      # baseline.median_fps; None if no baseline
     regression_pct: float | None    # ((measured - baseline) / baseline) × 100; None if no baseline
-    gpu_mem_used_mb: float | None   # From bench_result.gpu_diag [informational]
+    gpu_mem_used_mb: float | None   # From bench_result.runtime_resources [informational]
     startup_time_s: float | None    # From bench_result [informational]
     wall_time_s: float | None       # From bench_result [informational]
     was_retried: bool               # Whether Phase 1 succeeded only after a retry
@@ -550,12 +550,17 @@ Each entry in `backends` becomes one `TaskConfig`. `backend_key = physics` when
   "raw_fps_std": 9800.0,
   "raw_fps_min": 1632000.0,
   "raw_fps_max": 1680000.0,
-  "gpu_diag": {
+  "runtime_resources": {
     "gpu_name": "NVIDIA L40S",
     "gpu_total_memory_gb": 45.62,
     "cuda_version": "12.1",
     "nvidia_driver_version": "550.54.15",
-    "gpu_mem_used_mb": 18432.0
+    "gpu_mem_used_mb": 18432.0,
+    "gpu_mem_peak_mb": 19456.0,
+    "gpu_util_pct": 95.0,
+    "system_ram_used_mb": 30720.0,
+    "system_ram_peak_mb": 31744.0,
+    "cpu_util_pct": 40.0
   },
   "provenance": {
     "hardware": {
@@ -604,7 +609,7 @@ Each entry in `backends` becomes one `TaskConfig`. `backend_key = physics` when
   },
   "runtime_info": {
     "software": {"warp": "1.6.0"},
-    "publish_only": {"gpu_diag.cuda_version": "12.1"}
+    "publish_only": {"runtime_resources.cuda_version": "12.1"}
   },
   "task_config_snapshot": {
     "task_id": "Isaac-Velocity-Flat-G1-v0",
@@ -643,16 +648,18 @@ All fields below are projected from the `RuntimeBundle` by
 | `raw_fps_min` | `run.num_envs / runtime.iteration_time_s.peak` | Recovered from the slowest steady-state step |
 | `raw_fps_median`, `raw_fps_p5`, `raw_fps_p95`, `p99_over_median`, `outlier_count` | — (removed) | Not emitted: the schema keeps only aggregates (these needed the raw series), and they were never gating. `raw_fps_std`/`raw_fps_min` cover the tail |
 | `startup_time_s` | sum of `runtime.startup_time_s.*` phases | app_launch + env_creation + first_step + python_imports |
-| `gpu_diag.gpu_mem_used_mb` | `resources.gpu_mem_gb.mean` | Converted from GB |
-| `gpu_diag.gpu_name`, `.gpu_total_memory_gb` | `hardware.gpu_devices[0].{name,mem_gb}` | |
-| `gpu_diag.cuda_version` | `versions.cuda_bindings` | CUDA bindings version used as a display proxy (schema-v1 has no CUDA-runtime field) |
-| `gpu_diag.nvidia_driver_version` | `nvidia-smi` subprocess at post-processing time | `null` if nvidia-smi unavailable |
+| `runtime_resources.gpu_mem_used_mb`, `.gpu_mem_peak_mb` | `resources.gpu_mem_gb.{mean,peak}` | Device-wide VRAM (nvidia-smi); converted GB→MB |
+| `runtime_resources.system_ram_used_mb`, `.system_ram_peak_mb` | `resources.ram_gb.{mean,peak}` | Benchmark **process** RSS (psutil), not whole-host RAM; converted GB→MB |
+| `runtime_resources.gpu_util_pct`, `.cpu_util_pct` | `resources.{gpu_util_pct,cpu_util_pct}.mean` | Mean utilisation [%] |
+| `runtime_resources.gpu_name`, `.gpu_total_memory_gb` | `hardware.gpu_devices[0].{name,mem_gb}` | |
+| `runtime_resources.cuda_version` | `versions.cuda_bindings` | CUDA bindings version used as a display proxy (schema-v1 has no CUDA-runtime field) |
+| `runtime_resources.nvidia_driver_version` | `nvidia-smi` subprocess at post-processing time | `null` if nvidia-smi unavailable |
 | `provenance.hardware` | `hardware` snapshot | CPU, GPU, RAM identity |
 | `provenance.software` | `versions` map (verbatim, minus `git_*` keys) | Package versions |
 | `provenance.git` | `versions.git_commit`/`git_branch`/`git_dirty` | commit, branch, dirty flag |
 
 Key fields the oracle reads: `perf_smoke_test_info_present`, `raw_fps_mean`,
-`failure_phase`, `config_mismatch`, `was_retried`, `gpu_diag.gpu_mem_used_mb`,
+`failure_phase`, `config_mismatch`, `was_retried`, `runtime_resources.gpu_mem_used_mb`,
 `startup_time_s`, `wall_time_s`.
 
 `raw_fps_mean` is the steady-state mean the oracle gates on; `raw_fps_std`/`_min`/`_max`
@@ -720,7 +727,7 @@ objects or prefixed measurement names. Abbreviated (many `versions.*` fields omi
 `benchmark_result_adapter` reads this bundle: the gate metric `raw_fps_mean` comes from
 `runtime.total_fps.mean` (steady-state — warmup was excluded at the source by
 `perf_runtime.py --warmup_frames`), `raw_fps_min` from `run.num_envs /
-runtime.iteration_time_s.peak`, and provenance/`gpu_diag` from `versions`, `hardware`,
+runtime.iteration_time_s.peak`, and provenance/`runtime_resources` from `versions`, `hardware`,
 and `resources`. The bundle stores only aggregates, so the raw per-frame FPS series (and
 hence percentiles) is not available.
 

tools/perf_smoke_test/docs/system-design.md

@@ -126,7 +126,7 @@ IsaacLab/
         │                                 Isaac Lab benchmark core; writes a schema-v1
         │                                 RuntimeBundle (benchmark_runtime_*.json)
         ├── benchmark_result_adapter.py   RuntimeBundle → typed RuntimeSample projection
-        │                                 (FPS aggregates, provenance, gpu_diag)
+        │                                 (FPS aggregates, provenance, runtime_resources)
         ├── contracts.py                  typed gate artifacts (schema v1): RuntimeSample
         │                                 + BenchResult (perf_smoke_test_result.json)
         ├── build_bench_result.py         Phase 2: copies the runtime bundle, projects it
@@ -403,12 +403,17 @@ for local testing.
   "raw_fps_std": 9800.0,
   "raw_fps_min": 1632000.0,
   "raw_fps_max": 1680000.0,
-  "gpu_diag": {
+  "runtime_resources": {
     "gpu_name": "NVIDIA L40S",
     "gpu_total_memory_gb": 45.62,
     "cuda_version": "12.1",
     "nvidia_driver_version": "550.54.15",
-    "gpu_mem_used_mb": 18432.0
+    "gpu_mem_used_mb": 18432.0,
+    "gpu_mem_peak_mb": 19456.0,
+    "gpu_util_pct": 95.0,
+    "system_ram_used_mb": 30720.0,
+    "system_ram_peak_mb": 31744.0,
+    "cpu_util_pct": 40.0
   },
   "provenance": {
     "hardware": { "cpu_name": "...", "gpu_name": "NVIDIA L40S", "cuda_version": "12.1", "..." : "..." },
@@ -489,8 +494,14 @@ the true run-to-run variance distribution.
 
 ### 13.3 Memory tracking scope
 
-**Current state:** `gpu_diag.gpu_mem_used_mb` is captured and surfaced in `OracleResult`
-as an informational field. No memory regression threshold yet.
+**Current state:** the `runtime_resources` block carries an informational (non-gating)
+snapshot of the run's resource utilisation, projected from the bundle's `resources`
+section: VRAM (`gpu_mem_used_mb` mean + `gpu_mem_peak_mb`), process-RSS system RAM
+(`system_ram_used_mb` mean + `system_ram_peak_mb`), and GPU/CPU utilisation
+(`gpu_util_pct`, `cpu_util_pct`). `gpu_mem_used_mb` is additionally surfaced on
+`OracleResult`. None of these gate the verdict or feed `runtime_contract_hash` — no
+memory regression threshold yet. Note `gpu_mem_*` is device-wide (nvidia-smi) while
+`system_ram_*` is the benchmark process's RSS (psutil), not whole-host RAM.
 
 ---
 

tools/perf_smoke_test/gate_config.py

@@ -52,8 +52,8 @@
     "publish_only": [
         "hardware.gpu_compute_capability",
         "hardware.gpu_total_memory_gb",
-        "gpu_diag.cuda_version",
-        "gpu_diag.nvidia_driver_version",
+        "runtime_resources.cuda_version",
+        "runtime_resources.nvidia_driver_version",
     ],
 }
 

tools/perf_smoke_test/oracle.py

@@ -143,7 +143,7 @@ def compare(
     was_retried: bool = bool(bench_result.was_retried)
     startup_time_s: float | None = bench_result.startup_time_s
     wall_time_s: float | None = bench_result.wall_time_s
-    gpu_mem_used_mb: float | None = (bench_result.gpu_diag or {}).get("gpu_mem_used_mb")
+    gpu_mem_used_mb: float | None = (bench_result.runtime_resources or {}).get("gpu_mem_used_mb")
 
     config_mismatch = bench_result.config_mismatch
     if config_mismatch or failure_phase == FailurePhase.CONFIG_MISMATCH.value: