Skip to content

Commit 49681e7

Browse files
author
Neil4561
committed
Adopt Angelina's perf-gate rewrite and make it runner-compatible
Replace the tools/perf_regression_gate tree wholesale with the latest revision from PR #3 (angehu/perf-gate-poc, "hardening fixes"), then re-layer the runner/ops wiring needed to actually run on the NVIDIA RTX-6000 fleet: - Public Isaac Sim image override (fork cannot pull nvcr.io/nvidian/*), NGC_API_KEY env, bench_matrix output, full checkout for the matrix step. - World-writable jit-cache + artifact bind mounts (chmod 0777) and WARP/CUDA cache env/mounts to fix the wp.init() PermissionError that the lazy task loader masked as a spurious mdp AttributeError. - Rolling JIT cache key so the warm cache refills each run instead of freezing empty, plus an opt-in type=gha image layer cache (gha-cache-scope) for fleets where ECR cannot be resolved. - Trim triggers to the runnable push path (pull-request/<N> + protected branches + manual dispatch); pull_request/merge_group never get fleet runners. - Rename baseline branch angehu/perf-baselines -> perf-baselines.
1 parent 437f60a commit 49681e7

35 files changed

Lines changed: 2733 additions & 2437 deletions

.github/actions/ecr-build-push-pull/action.yml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,16 @@ inputs:
3737
description: Tag used for the ECR layer cache image (e.g. "cache-base", "cache-curobo").
3838
required: false
3939
default: 'cache'
40+
gha-cache-scope:
41+
description: >
42+
When non-empty, enables a GitHub Actions (type=gha) buildx layer cache with
43+
this scope on the full-build path. Acts as a registry-free fallback when ECR
44+
is unavailable (e.g. on non-AWS fleets), and as an extra cache source when
45+
ECR is present. Leave empty (default) to preserve the original ECR-only
46+
behavior for callers that do not opt in. The scope isolates this cache from
47+
other workflows sharing the repository's Actions cache.
48+
required: false
49+
default: ''
4050
runs:
4151
using: composite
4252
steps:
@@ -273,6 +283,17 @@ runs:
273283
)
274284
fi
275285
286+
# Opt-in GitHub Actions layer cache. Registry-free fallback for fleets
287+
# where ECR cannot be resolved (the block above is skipped), keeping the
288+
# build in line with how publish-images.yaml / docker-build cache this
289+
# same Dockerfile. The scope isolates it from other workflows' gha caches.
290+
if [ -n "${{ inputs.gha-cache-scope }}" ]; then
291+
BUILD_ARGS+=(
292+
--cache-from "type=gha,scope=${{ inputs.gha-cache-scope }}"
293+
--cache-to "type=gha,scope=${{ inputs.gha-cache-scope }},mode=max"
294+
)
295+
fi
296+
276297
BUILDER_NAME="ci-builder-${{ github.run_id }}-${{ github.job }}"
277298
docker buildx create --use --driver docker-container --name "${BUILDER_NAME}" \
278299
|| docker buildx use "${BUILDER_NAME}"

.github/workflows/perf-regression-gate.yaml

Lines changed: 430 additions & 316 deletions
Large diffs are not rendered by default.

tools/perf_regression_gate/.gitignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,3 @@ local_baselines/
44
__pycache__/
55
*.pyc
66
.pytest_cache/
7-
test-reports-*.xml

tools/perf_regression_gate/aggregate.py

Lines changed: 218 additions & 172 deletions
Large diffs are not rendered by default.
Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
2+
# All rights reserved.
3+
#
4+
# SPDX-License-Identifier: BSD-3-Clause
5+
6+
"""Canonical backend identity helpers for the performance regression gate"""
7+
8+
from __future__ import annotations
9+
10+
from dataclasses import dataclass
11+
from typing import Any
12+
13+
_DEFAULT_PHYSICS_BACKEND = "physx"
14+
_EMPTY_VALUES = {"", "none", "null"}
15+
_DEFAULT_VALUES = _EMPTY_VALUES | {"default"}
16+
_PHYSICS_PRESET_TO_BACKEND = {
17+
"physx": "physx",
18+
"newton": "newton",
19+
"newton_mjwarp": "newton",
20+
}
21+
_RENDER_PRESET_TOKENS = frozenset(
22+
{
23+
"newton_renderer",
24+
"ovrtx_renderer",
25+
"warp_renderer",
26+
"rtx_renderer",
27+
}
28+
)
29+
_KNOWN_PHYSICS_BACKENDS = ("physx", "newton")
30+
31+
32+
@dataclass(frozen=True)
33+
class BackendIdentity:
34+
physics_backend: str
35+
render_backend: str | None = None
36+
37+
@property
38+
def backend_key(self) -> str:
39+
return make_backend_key(self.physics_backend, self.render_backend)
40+
41+
def to_dict(self) -> dict[str, str | None]:
42+
return {
43+
"physics_backend": self.physics_backend,
44+
"render_backend": self.render_backend,
45+
"backend_key": self.backend_key,
46+
}
47+
48+
49+
def _clean(value: Any) -> str | None:
50+
if value is None:
51+
return None
52+
cleaned = str(value).strip()
53+
return cleaned or None
54+
55+
56+
def normalize_physics_backend(value: Any, *, default: str | None = None) -> str | None:
57+
cleaned = _clean(value)
58+
if cleaned is None:
59+
return default
60+
lowered = cleaned.lower()
61+
if lowered in _DEFAULT_VALUES:
62+
return default
63+
return _PHYSICS_PRESET_TO_BACKEND.get(lowered, lowered)
64+
65+
66+
def normalize_render_backend(value: Any) -> str | None:
67+
cleaned = _clean(value)
68+
if cleaned is None:
69+
return None
70+
lowered = cleaned.lower()
71+
if lowered in _DEFAULT_VALUES:
72+
return None
73+
return lowered
74+
75+
76+
def make_backend_key(physics_backend: str, render_backend: str | None = None) -> str:
77+
physics = normalize_physics_backend(physics_backend)
78+
if not physics:
79+
raise ValueError("physics_backend is required to build backend_key")
80+
render = normalize_render_backend(render_backend)
81+
return f"{physics}_{render}" if render else physics
82+
83+
84+
def identity_from_parts(physics_backend: Any, render_backend: Any = None) -> BackendIdentity | None:
85+
physics = normalize_physics_backend(physics_backend)
86+
if not physics:
87+
return None
88+
return BackendIdentity(physics, normalize_render_backend(render_backend))
89+
90+
91+
def split_backend_key(backend_key: Any) -> BackendIdentity | None:
92+
key = _clean(backend_key)
93+
if not key:
94+
return None
95+
for physics in _KNOWN_PHYSICS_BACKENDS:
96+
if key == physics:
97+
return BackendIdentity(physics, None)
98+
prefix = f"{physics}_"
99+
if key.startswith(prefix):
100+
return BackendIdentity(physics, normalize_render_backend(key[len(prefix):]))
101+
if "_" in key:
102+
physics, render = key.split("_", 1)
103+
return identity_from_parts(physics, render)
104+
return BackendIdentity(key, None)
105+
106+
107+
def preset_tokens(value: Any) -> frozenset[str]:
108+
cleaned = _clean(value)
109+
if not cleaned:
110+
return frozenset()
111+
tokens: set[str] = set()
112+
for chunk in cleaned.replace(";", ",").split(","):
113+
token = chunk.strip().lower()
114+
if token:
115+
tokens.add(token)
116+
return frozenset(tokens)
117+
118+
119+
def identity_from_presets(presets: Any) -> BackendIdentity | None:
120+
tokens = preset_tokens(presets)
121+
if not tokens:
122+
return None
123+
physics = "newton" if "newton_mjwarp" in tokens else _DEFAULT_PHYSICS_BACKEND
124+
render = next((token for token in sorted(tokens) if token in _RENDER_PRESET_TOKENS), None)
125+
return BackendIdentity(physics, render)
126+
127+
128+
def backend_identity_from_launch_config(config: dict[str, Any]) -> BackendIdentity | None:
129+
identity = identity_from_parts(config.get("physics_backend"), config.get("render_backend"))
130+
if identity is not None:
131+
return identity
132+
return split_backend_key(config.get("backend_key") or config.get("backend"))
133+
134+
135+
def backend_identity_from_benchmark_info(info: dict[str, Any]) -> BackendIdentity | None:
136+
direct_key = info.get("backend_key") or info.get("backend")
137+
if direct_key:
138+
return split_backend_key(direct_key)
139+
140+
identity = identity_from_parts(
141+
info.get("physics_backend") or info.get("physics"),
142+
info.get("render_backend") or info.get("render"),
143+
)
144+
if identity is not None:
145+
return identity
146+
147+
return identity_from_presets(info.get("presets") or info.get("preset"))

0 commit comments

Comments
 (0)