From 36bfdaef0be956f6a4e6dbc7771f8fab249a6a8d Mon Sep 17 00:00:00 2001 From: YuTeh Shen Date: Fri, 28 Aug 2026 06:06:36 +0000 Subject: [PATCH 1/4] Report resolved camera resolutions in benchmark KPIs --- .../benchmark-camera-resolutions.rst | 4 + source/isaaclab/isaaclab/benchmark/capture.py | 84 +++++++++++++++++++ .../rl_games/benchmark_play_rl_games.py | 1 + .../rl_games/benchmark_train_rl_games.py | 1 + .../backends/rsl_rl/benchmark_play_rsl_rl.py | 1 + .../backends/rsl_rl/benchmark_train_rsl_rl.py | 1 + .../backends/sb3/benchmark_play_sb3.py | 1 + .../backends/sb3/benchmark_train_sb3.py | 1 + .../backends/skrl/benchmark_play_skrl.py | 1 + .../backends/skrl/benchmark_train_skrl.py | 1 + .../isaaclab/benchmark/entrypoints/runtime.py | 1 + .../isaaclab/benchmark/entrypoints/startup.py | 1 + .../test/benchmark/test_benchmark_core.py | 9 ++ .../isaaclab/test/benchmark/test_capture.py | 40 +++++++++ 14 files changed, 147 insertions(+) create mode 100644 source/isaaclab/changelog.d/benchmark-camera-resolutions.rst diff --git a/source/isaaclab/changelog.d/benchmark-camera-resolutions.rst b/source/isaaclab/changelog.d/benchmark-camera-resolutions.rst new file mode 100644 index 000000000000..c2c38cd097f1 --- /dev/null +++ b/source/isaaclab/changelog.d/benchmark-camera-resolutions.rst @@ -0,0 +1,4 @@ +Added +^^^^^ + +* Added resolved camera config paths, image widths, and image heights to benchmark KPI metadata. diff --git a/source/isaaclab/isaaclab/benchmark/capture.py b/source/isaaclab/isaaclab/benchmark/capture.py index d93c47821039..72a6f80b65fb 100644 --- a/source/isaaclab/isaaclab/benchmark/capture.py +++ b/source/isaaclab/isaaclab/benchmark/capture.py @@ -278,6 +278,90 @@ def _backends_from_env_cfg(env_cfg: object) -> tuple[str | None, str | None]: return physics, rendering +def _is_camera_cfg(node: object) -> bool: + """Return whether *node* derives from a supported camera configuration class.""" + camera_cfg_names = {"CameraCfg", "RayCasterCameraCfg"} + return any(base.__name__ in camera_cfg_names for base in type(node).__mro__) + + +def _camera_resolution(node: object) -> tuple[int, int] | None: + """Return the resolved ``(width, height)`` for one camera configuration.""" + width = getattr(node, "width", None) + height = getattr(node, "height", None) + if not isinstance(width, int) or isinstance(width, bool) or not isinstance(height, int) or isinstance(height, bool): + pattern_cfg = getattr(node, "pattern_cfg", None) + width = getattr(pattern_cfg, "width", None) + height = getattr(pattern_cfg, "height", None) + if ( + not isinstance(width, int) + or isinstance(width, bool) + or width <= 0 + or not isinstance(height, int) + or isinstance(height, bool) + or height <= 0 + ): + return None + return width, height + + +def camera_resolutions_from_env_cfg(env_cfg: object) -> dict[str, dict[str, int]]: + """Collect resolved image dimensions from camera configurations in an environment config. + + The returned keys are config paths rooted at ``env`` so benchmark results identify the + exact camera field that was resolved by Hydra. RTX camera configurations expose dimensions + directly, while ray-cast cameras expose them through their pattern configuration. + + Args: + env_cfg: Concrete task environment configuration after command-line overrides are applied. + + Returns: + Camera config paths mapped to their resolved image width and height in pixels. + """ + resolutions: dict[str, dict[str, int]] = {} + stack: list[tuple[str, object]] = [("env", env_cfg)] + visited: set[int] = set() + + while stack: + path, node = stack.pop() + if node is None or isinstance(node, (str, bytes, int, float, bool, type)) or id(node) in visited: + continue + visited.add(id(node)) + + if _is_camera_cfg(node): + resolution = _camera_resolution(node) + if resolution is not None: + width, height = resolution + resolutions[path] = {"width": width, "height": height} + continue + + if isinstance(node, dict): + children = [(f"{path}.{key}", value) for key, value in node.items()] + elif isinstance(node, (list, tuple)): + children = [(f"{path}[{index}]", value) for index, value in enumerate(node)] + else: + try: + children = [(f"{path}.{name}", value) for name, value in vars(node).items() if not name.startswith("_")] + except TypeError: + continue + stack.extend(reversed(children)) + + return dict(sorted(resolutions.items())) + + +def camera_resolution_metadata_from_env_cfg(env_cfg: object) -> list[dict[str, object]]: + """Build workflow metadata containing resolved camera resolutions, when present. + + Args: + env_cfg: Concrete task environment configuration after command-line overrides are applied. + + Returns: + A workflow metadata entry for ``camera_resolutions``, or an empty list when the task has no + configured cameras with concrete image dimensions. + """ + resolutions = camera_resolutions_from_env_cfg(env_cfg) + return [{"name": "camera_resolutions", "data": resolutions}] if resolutions else [] + + def run_config_from_env_cfg(env_cfg: object) -> RunConfig: """Build a :class:`~isaaclab.benchmark.RunConfig` from a concrete task config. diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_play_rl_games.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_play_rl_games.py index e8c295b61bcf..cfb398fe7f83 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_play_rl_games.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_play_rl_games.py @@ -181,6 +181,7 @@ def run(argv: list[str]) -> BenchmarkResult: "data": ("serialized_synchronized" if args_cli.measure_sync_step else "host_return"), }, {"name": "environment_step_warmup_steps", "data": args_cli.warmup_steps}, + *capture.camera_resolution_metadata_from_env_cfg(env_cfg), ] }, ) diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_train_rl_games.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_train_rl_games.py index b6dd72dd4ad0..38062a66ae3b 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_train_rl_games.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_train_rl_games.py @@ -232,6 +232,7 @@ def run(argv: list[str]) -> BenchmarkResult | None: }, {"name": "environment_step_warmup_steps", "data": args_cli.warmup_steps}, {"name": "world_size", "data": distributed.world_size}, + *capture.camera_resolution_metadata_from_env_cfg(env_cfg), ] }, ) diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_play_rsl_rl.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_play_rsl_rl.py index 52e64262abb4..1f98e8f6de23 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_play_rsl_rl.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_play_rsl_rl.py @@ -178,6 +178,7 @@ def run(argv: list[str]) -> BenchmarkResult: "data": ("serialized_synchronized" if args.measure_sync_step else "host_return"), }, {"name": "environment_step_warmup_steps", "data": args.warmup_steps}, + *capture.camera_resolution_metadata_from_env_cfg(env_cfg), ] }, ) diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_train_rsl_rl.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_train_rsl_rl.py index cec0ce323d98..a777ba8f40cf 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_train_rsl_rl.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_train_rsl_rl.py @@ -220,6 +220,7 @@ def run(argv: list[str]) -> BenchmarkResult | None: }, {"name": "environment_step_warmup_steps", "data": args_cli.warmup_steps}, {"name": "world_size", "data": distributed.world_size}, + *capture.camera_resolution_metadata_from_env_cfg(env_cfg), ] }, ) diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_play_sb3.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_play_sb3.py index 7f3aa1e6f171..ba5584a7a404 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_play_sb3.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_play_sb3.py @@ -181,6 +181,7 @@ def run(argv: list[str]) -> BenchmarkResult: "data": ("serialized_synchronized" if args_cli.measure_sync_step else "host_return"), }, {"name": "environment_step_warmup_steps", "data": args_cli.warmup_steps}, + *capture.camera_resolution_metadata_from_env_cfg(env_cfg), ] }, ) diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_train_sb3.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_train_sb3.py index a1a1a80d14cb..0b7856d7a1b4 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_train_sb3.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/sb3/benchmark_train_sb3.py @@ -268,6 +268,7 @@ def run(argv: list[str]) -> BenchmarkResult: "data": ("serialized_synchronized" if args_cli.measure_sync_step else "host_return"), }, {"name": "environment_step_warmup_steps", "data": args_cli.warmup_steps}, + *capture.camera_resolution_metadata_from_env_cfg(env_cfg), ] }, ) diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/skrl/benchmark_play_skrl.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/skrl/benchmark_play_skrl.py index 697f5e4a5743..3df2a7e7f29d 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/skrl/benchmark_play_skrl.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/skrl/benchmark_play_skrl.py @@ -210,6 +210,7 @@ def run(argv: list[str]) -> BenchmarkResult: "data": ("serialized_synchronized" if args_cli.measure_sync_step else "host_return"), }, {"name": "environment_step_warmup_steps", "data": args_cli.warmup_steps}, + *capture.camera_resolution_metadata_from_env_cfg(env_cfg), ] }, ) diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/skrl/benchmark_train_skrl.py b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/skrl/benchmark_train_skrl.py index a6c09f1dbe39..ca246990fe77 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/backends/skrl/benchmark_train_skrl.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/backends/skrl/benchmark_train_skrl.py @@ -352,6 +352,7 @@ def run(argv: list[str]) -> BenchmarkResult | None: }, {"name": "environment_step_warmup_steps", "data": args_cli.warmup_steps}, {"name": "world_size", "data": distributed.world_size}, + *capture.camera_resolution_metadata_from_env_cfg(env_cfg), ] }, ) diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/runtime.py b/source/isaaclab/isaaclab/benchmark/entrypoints/runtime.py index 840b3b95317b..dec2af487d90 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/runtime.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/runtime.py @@ -169,6 +169,7 @@ def run(argv: list[str]) -> BenchmarkResult | None: "data": ("serialized_synchronized" if args.measure_sync_step else "host_return"), }, {"name": "world_size", "data": distributed.world_size}, + *capture.camera_resolution_metadata_from_env_cfg(env_cfg), ] }, ) diff --git a/source/isaaclab/isaaclab/benchmark/entrypoints/startup.py b/source/isaaclab/isaaclab/benchmark/entrypoints/startup.py index 49a611afe511..62923e4da170 100644 --- a/source/isaaclab/isaaclab/benchmark/entrypoints/startup.py +++ b/source/isaaclab/isaaclab/benchmark/entrypoints/startup.py @@ -357,6 +357,7 @@ def run(argv: list[str]) -> BenchmarkResult | None: {"name": "num_envs", "data": args.num_envs}, {"name": "top_n", "data": args.top_n}, {"name": "world_size", "data": distributed.world_size}, + *capture.camera_resolution_metadata_from_env_cfg(env_cfg), ] }, ) diff --git a/source/isaaclab/test/benchmark/test_benchmark_core.py b/source/isaaclab/test/benchmark/test_benchmark_core.py index 4a29d6105a28..909552602b99 100644 --- a/source/isaaclab/test/benchmark/test_benchmark_core.py +++ b/source/isaaclab/test/benchmark/test_benchmark_core.py @@ -170,6 +170,14 @@ def test_benchmark_collects_metadata_measurements_and_writes_json(tmp_path): output_path=str(output_path), use_recorders=False, output_prefix="test", + workflow_metadata={ + "metadata": [ + { + "name": "camera_resolutions", + "data": {"env.scene.tiled_camera": {"width": 64, "height": 48}}, + } + ] + }, ) benchmark.add_measurement( @@ -191,6 +199,7 @@ def test_benchmark_collects_metadata_measurements_and_writes_json(tmp_path): assert not hasattr(benchmark, "_manual_recorders") or benchmark._manual_recorders is None assert data["benchmark_info"]["workflow_name"] == "my_workflow" assert "timestamp" in data["benchmark_info"] + assert data["benchmark_info"]["camera_resolutions"] == {"env.scene.tiled_camera": {"width": 64, "height": 48}} assert data["runtime"]["metric1"] == 10.0 assert data["runtime"]["metric2"] == 20.0 assert data["runtime"]["custom"] == "value" diff --git a/source/isaaclab/test/benchmark/test_capture.py b/source/isaaclab/test/benchmark/test_capture.py index 34d903149649..6accb90f7ae5 100644 --- a/source/isaaclab/test/benchmark/test_capture.py +++ b/source/isaaclab/test/benchmark/test_capture.py @@ -10,6 +10,8 @@ import pytest from isaaclab.benchmark.capture import ( + camera_resolution_metadata_from_env_cfg, + camera_resolutions_from_env_cfg, capture_hardware, capture_resources, capture_versions, @@ -228,6 +230,44 @@ def test_run_config_uses_concrete_backend_configuration(): run_config_from_env_cfg(SimpleNamespace(sim=SimpleNamespace(physics=object()))) +def test_camera_resolutions_use_resolved_config_paths(): + class CameraCfg: + def __init__(self, width: int, height: int): + self.width = width + self.height = height + + class CartpoleCameraCfg(CameraCfg): + pass + + class RayCasterCameraCfg: + def __init__(self, width: int, height: int): + self.pattern_cfg = SimpleNamespace(width=width, height=height) + + shared_camera = CartpoleCameraCfg(width=64, height=48) + env_cfg = SimpleNamespace( + scene=SimpleNamespace(tiled_camera=shared_camera, duplicate=shared_camera), + ray_camera=RayCasterCameraCfg(width=32, height=24), + viewport=SimpleNamespace(width=1920, height=1080), + ) + + expected = { + "env.ray_camera": {"width": 32, "height": 24}, + "env.scene.tiled_camera": {"width": 64, "height": 48}, + } + assert camera_resolutions_from_env_cfg(env_cfg) == expected + assert camera_resolution_metadata_from_env_cfg(env_cfg) == [{"name": "camera_resolutions", "data": expected}] + + +def test_camera_resolutions_omit_unconfigured_dimensions(): + class CameraCfg: + width = object() + height = object() + + env_cfg = SimpleNamespace(camera=CameraCfg()) + assert camera_resolutions_from_env_cfg(env_cfg) == {} + assert camera_resolution_metadata_from_env_cfg(env_cfg) == [] + + def test_capture_resources_peak_clamped_to_mean_when_peak_row_absent(): # Build a recorder that has mean/std rows but no peak rows. # capture_resources must clamp peak to mean rather than leaving it at 0.0 From 2546f27f35cf7c6019b38d211ee41f8d5b46de94 Mon Sep 17 00:00:00 2001 From: YuTeh Shen Date: Mon, 31 Aug 2026 00:59:23 +0000 Subject: [PATCH 2/4] Fix camera configuration discovery --- source/isaaclab/isaaclab/benchmark/capture.py | 19 +++--- .../isaaclab/test/benchmark/test_capture.py | 65 +++++++++++++------ 2 files changed, 54 insertions(+), 30 deletions(-) diff --git a/source/isaaclab/isaaclab/benchmark/capture.py b/source/isaaclab/isaaclab/benchmark/capture.py index 72a6f80b65fb..4985c1abe1e8 100644 --- a/source/isaaclab/isaaclab/benchmark/capture.py +++ b/source/isaaclab/isaaclab/benchmark/capture.py @@ -278,12 +278,6 @@ def _backends_from_env_cfg(env_cfg: object) -> tuple[str | None, str | None]: return physics, rendering -def _is_camera_cfg(node: object) -> bool: - """Return whether *node* derives from a supported camera configuration class.""" - camera_cfg_names = {"CameraCfg", "RayCasterCameraCfg"} - return any(base.__name__ in camera_cfg_names for base in type(node).__mro__) - - def _camera_resolution(node: object) -> tuple[int, int] | None: """Return the resolved ``(width, height)`` for one camera configuration.""" width = getattr(node, "width", None) @@ -317,23 +311,30 @@ def camera_resolutions_from_env_cfg(env_cfg: object) -> dict[str, dict[str, int] Returns: Camera config paths mapped to their resolved image width and height in pixels. """ + # Keep these imports local so importing benchmark capture does not eagerly load sensor dependencies. + from isaaclab.sensors import CameraCfg, RayCasterCameraCfg + + camera_cfg_types = (CameraCfg, RayCasterCameraCfg) resolutions: dict[str, dict[str, int]] = {} stack: list[tuple[str, object]] = [("env", env_cfg)] visited: set[int] = set() while stack: path, node = stack.pop() - if node is None or isinstance(node, (str, bytes, int, float, bool, type)) or id(node) in visited: + if node is None or isinstance(node, (str, bytes, int, float, bool, type)): continue - visited.add(id(node)) - if _is_camera_cfg(node): + if isinstance(node, camera_cfg_types): resolution = _camera_resolution(node) if resolution is not None: width, height = resolution resolutions[path] = {"width": width, "height": height} continue + if id(node) in visited: + continue + visited.add(id(node)) + if isinstance(node, dict): children = [(f"{path}.{key}", value) for key, value in node.items()] elif isinstance(node, (list, tuple)): diff --git a/source/isaaclab/test/benchmark/test_capture.py b/source/isaaclab/test/benchmark/test_capture.py index 6accb90f7ae5..2610ffd35213 100644 --- a/source/isaaclab/test/benchmark/test_capture.py +++ b/source/isaaclab/test/benchmark/test_capture.py @@ -27,6 +27,7 @@ StringMetadata, ) from isaaclab.benchmark.schema import Hardware, Resources, Versions +from isaaclab.sensors import CameraCfg, RayCasterCameraCfg, patterns class _Rec: @@ -42,6 +43,18 @@ def __init__(self, recorders): self._manual_recorders = recorders +def _camera_cfg(width: int = 64, height: int = 48) -> CameraCfg: + return CameraCfg(prim_path="/World/Camera", spawn=None, width=width, height=height) + + +def _ray_caster_camera_cfg(width: int = 32, height: int = 24) -> RayCasterCameraCfg: + return RayCasterCameraCfg( + prim_path="/World/RayCamera", + mesh_prim_paths=["/World/Ground"], + pattern_cfg=patterns.PinholeCameraPatternCfg(width=width, height=height), + ) + + def test_capture_versions_renames_and_defaults(): md = [ StringMetadata(name="isaaclab_version", data="4.6.8"), @@ -230,40 +243,50 @@ def test_run_config_uses_concrete_backend_configuration(): run_config_from_env_cfg(SimpleNamespace(sim=SimpleNamespace(physics=object()))) -def test_camera_resolutions_use_resolved_config_paths(): - class CameraCfg: - def __init__(self, width: int, height: int): - self.width = width - self.height = height - - class CartpoleCameraCfg(CameraCfg): - pass +def test_camera_resolutions_omit_no_camera_metadata(): + env_cfg = SimpleNamespace(viewport=SimpleNamespace(width=1920, height=1080)) + assert camera_resolutions_from_env_cfg(env_cfg) == {} + assert camera_resolution_metadata_from_env_cfg(env_cfg) == [] - class RayCasterCameraCfg: - def __init__(self, width: int, height: int): - self.pattern_cfg = SimpleNamespace(width=width, height=height) - shared_camera = CartpoleCameraCfg(width=64, height=48) +def test_camera_resolutions_report_distinct_cameras_sorted_by_path(): env_cfg = SimpleNamespace( - scene=SimpleNamespace(tiled_camera=shared_camera, duplicate=shared_camera), - ray_camera=RayCasterCameraCfg(width=32, height=24), - viewport=SimpleNamespace(width=1920, height=1080), + z_camera=_camera_cfg(), + a_ray_camera=_ray_caster_camera_cfg(), ) expected = { - "env.ray_camera": {"width": 32, "height": 24}, - "env.scene.tiled_camera": {"width": 64, "height": 48}, + "env.a_ray_camera": {"width": 32, "height": 24}, + "env.z_camera": {"width": 64, "height": 48}, } assert camera_resolutions_from_env_cfg(env_cfg) == expected assert camera_resolution_metadata_from_env_cfg(env_cfg) == [{"name": "camera_resolutions", "data": expected}] -def test_camera_resolutions_omit_unconfigured_dimensions(): +def test_camera_resolutions_preserve_aliased_camera_paths(): + shared_camera = _camera_cfg() + env_cfg = SimpleNamespace(scene=SimpleNamespace(tiled_camera=shared_camera, duplicate=shared_camera)) + + assert camera_resolutions_from_env_cfg(env_cfg) == { + "env.scene.duplicate": {"width": 64, "height": 48}, + "env.scene.tiled_camera": {"width": 64, "height": 48}, + } + + +def test_camera_resolutions_ignore_unrelated_same_named_class(): class CameraCfg: - width = object() - height = object() + width = 64 + height = 48 + + assert camera_resolutions_from_env_cfg(SimpleNamespace(camera=CameraCfg())) == {} + + +def test_camera_resolutions_omit_unconfigured_dimensions(): + camera_cfg = _camera_cfg() + camera_cfg.width = object() + camera_cfg.height = object() - env_cfg = SimpleNamespace(camera=CameraCfg()) + env_cfg = SimpleNamespace(camera=camera_cfg) assert camera_resolutions_from_env_cfg(env_cfg) == {} assert camera_resolution_metadata_from_env_cfg(env_cfg) == [] From 9acf6a6c2950808b89f6aae9f27c9fb70b8a84f4 Mon Sep 17 00:00:00 2001 From: YuTeh Shen Date: Tue, 1 Sep 2026 00:08:16 +0000 Subject: [PATCH 3/4] Preserve camera paths through shared containers --- source/isaaclab/isaaclab/benchmark/capture.py | 11 +++++------ source/isaaclab/test/benchmark/test_capture.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/source/isaaclab/isaaclab/benchmark/capture.py b/source/isaaclab/isaaclab/benchmark/capture.py index 4985c1abe1e8..0e83429094dd 100644 --- a/source/isaaclab/isaaclab/benchmark/capture.py +++ b/source/isaaclab/isaaclab/benchmark/capture.py @@ -316,11 +316,10 @@ def camera_resolutions_from_env_cfg(env_cfg: object) -> dict[str, dict[str, int] camera_cfg_types = (CameraCfg, RayCasterCameraCfg) resolutions: dict[str, dict[str, int]] = {} - stack: list[tuple[str, object]] = [("env", env_cfg)] - visited: set[int] = set() + stack: list[tuple[str, object, frozenset[int]]] = [("env", env_cfg, frozenset())] while stack: - path, node = stack.pop() + path, node, ancestors = stack.pop() if node is None or isinstance(node, (str, bytes, int, float, bool, type)): continue @@ -331,9 +330,9 @@ def camera_resolutions_from_env_cfg(env_cfg: object) -> dict[str, dict[str, int] resolutions[path] = {"width": width, "height": height} continue - if id(node) in visited: + if id(node) in ancestors: continue - visited.add(id(node)) + child_ancestors = ancestors | {id(node)} if isinstance(node, dict): children = [(f"{path}.{key}", value) for key, value in node.items()] @@ -344,7 +343,7 @@ def camera_resolutions_from_env_cfg(env_cfg: object) -> dict[str, dict[str, int] children = [(f"{path}.{name}", value) for name, value in vars(node).items() if not name.startswith("_")] except TypeError: continue - stack.extend(reversed(children)) + stack.extend((child_path, child, child_ancestors) for child_path, child in reversed(children)) return dict(sorted(resolutions.items())) diff --git a/source/isaaclab/test/benchmark/test_capture.py b/source/isaaclab/test/benchmark/test_capture.py index 2610ffd35213..a54158b630ed 100644 --- a/source/isaaclab/test/benchmark/test_capture.py +++ b/source/isaaclab/test/benchmark/test_capture.py @@ -273,6 +273,16 @@ def test_camera_resolutions_preserve_aliased_camera_paths(): } +def test_camera_resolutions_preserve_paths_through_aliased_containers(): + shared_container = SimpleNamespace(camera=_camera_cfg()) + env_cfg = SimpleNamespace(left=shared_container, right=shared_container) + + assert camera_resolutions_from_env_cfg(env_cfg) == { + "env.left.camera": {"width": 64, "height": 48}, + "env.right.camera": {"width": 64, "height": 48}, + } + + def test_camera_resolutions_ignore_unrelated_same_named_class(): class CameraCfg: width = 64 From 9ba49caa3cfd97d967276d0d79e8347304962a5a Mon Sep 17 00:00:00 2001 From: YuTeh Shen Date: Thu, 3 Sep 2026 08:25:45 +0000 Subject: [PATCH 4/4] Add camera resolutions to benchmark schema --- ...nchmark-camera-resolution-schema.minor.rst | 5 ++++ .../benchmark-camera-resolutions.rst | 4 ---- .../isaaclab/isaaclab/benchmark/builders.py | 5 ++++ source/isaaclab/isaaclab/benchmark/capture.py | 5 ++++ source/isaaclab/isaaclab/benchmark/schema.py | 22 +++++++++++++++--- .../isaaclab/test/benchmark/test_builders.py | 7 ++++++ .../isaaclab/test/benchmark/test_capture.py | 8 ++++++- source/isaaclab/test/benchmark/test_schema.py | 23 +++++++++++++++++++ 8 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 source/isaaclab/changelog.d/benchmark-camera-resolution-schema.minor.rst delete mode 100644 source/isaaclab/changelog.d/benchmark-camera-resolutions.rst diff --git a/source/isaaclab/changelog.d/benchmark-camera-resolution-schema.minor.rst b/source/isaaclab/changelog.d/benchmark-camera-resolution-schema.minor.rst new file mode 100644 index 000000000000..86173c679ed5 --- /dev/null +++ b/source/isaaclab/changelog.d/benchmark-camera-resolution-schema.minor.rst @@ -0,0 +1,5 @@ +Added +^^^^^ + +* Added resolved camera config paths, image widths, and image heights to benchmark KPI metadata and + :attr:`~isaaclab.benchmark.schema.RunConfig.camera_resolutions` in version 1.5 of the benchmark schema. diff --git a/source/isaaclab/changelog.d/benchmark-camera-resolutions.rst b/source/isaaclab/changelog.d/benchmark-camera-resolutions.rst deleted file mode 100644 index c2c38cd097f1..000000000000 --- a/source/isaaclab/changelog.d/benchmark-camera-resolutions.rst +++ /dev/null @@ -1,4 +0,0 @@ -Added -^^^^^ - -* Added resolved camera config paths, image widths, and image heights to benchmark KPI metadata. diff --git a/source/isaaclab/isaaclab/benchmark/builders.py b/source/isaaclab/isaaclab/benchmark/builders.py index 566806d5f657..41bb379bd361 100644 --- a/source/isaaclab/isaaclab/benchmark/builders.py +++ b/source/isaaclab/isaaclab/benchmark/builders.py @@ -18,6 +18,7 @@ from isaaclab.benchmark.metrics import ema, mean_std_peak from isaaclab.benchmark.schema import ( + CameraResolution, EnvironmentStepTiming, Hardware, Learning, @@ -42,6 +43,7 @@ def build_run_config( physics_backend: str, rendering_backend: str = "none", presets: Sequence[str] | None = None, + camera_resolutions: dict[str, CameraResolution] | None = None, ) -> RunConfig: """Assemble a :class:`~isaaclab.benchmark.schema.RunConfig`. @@ -52,6 +54,8 @@ def build_run_config( with no camera sensors. presets: Active Hydra preset tokens (e.g. ``["rgb"]``). ``None`` is treated as an empty list. + camera_resolutions: Resolved camera image dimensions keyed by config + path. ``None`` is treated as an empty mapping. Returns: Populated :class:`~isaaclab.benchmark.schema.RunConfig`. @@ -60,6 +64,7 @@ def build_run_config( physics_backend=physics_backend, rendering_backend=rendering_backend, presets=list(presets) if presets else [], + camera_resolutions=dict(camera_resolutions) if camera_resolutions else {}, ) diff --git a/source/isaaclab/isaaclab/benchmark/capture.py b/source/isaaclab/isaaclab/benchmark/capture.py index 0e83429094dd..954b92ef0b24 100644 --- a/source/isaaclab/isaaclab/benchmark/capture.py +++ b/source/isaaclab/isaaclab/benchmark/capture.py @@ -23,6 +23,7 @@ from typing import Any from isaaclab.benchmark.schema import ( + CameraResolution, GpuDeviceInfo, GpuResources, Hardware, @@ -382,6 +383,10 @@ def run_config_from_env_cfg(env_cfg: object) -> RunConfig: return RunConfig( physics_backend=physics, rendering_backend=rendering or "none", + camera_resolutions={ + path: CameraResolution(width=dimensions["width"], height=dimensions["height"]) + for path, dimensions in camera_resolutions_from_env_cfg(env_cfg).items() + }, ) diff --git a/source/isaaclab/isaaclab/benchmark/schema.py b/source/isaaclab/isaaclab/benchmark/schema.py index 23652ab14898..19ff8f652bfb 100644 --- a/source/isaaclab/isaaclab/benchmark/schema.py +++ b/source/isaaclab/isaaclab/benchmark/schema.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Public schema for Isaac Lab benchmark bundles (v1.4). +"""Public schema for Isaac Lab benchmark bundles (v1.5). Defines the on-disk JSON schema produced by the benchmark workflows in :mod:`isaaclab.benchmark.entrypoints`. @@ -17,7 +17,7 @@ :class:`Versions` and :class:`Hardware` metadata so a reader need not cross-reference other files in the bundle directory. -Current version: 1.4 +Current version: 1.5 """ from __future__ import annotations @@ -26,7 +26,7 @@ from dataclasses import dataclass, field from typing import Literal -SCHEMA_VERSION = "1.4" +SCHEMA_VERSION = "1.5" Framework = Literal["rsl_rl", "rl_games", "skrl", "sb3"] PhysicsBackend = Literal["physx", "newton_mjwarp", "newton_kamino", "ovphysx"] @@ -133,6 +133,19 @@ class Versions: isaaclab_release: str | None = None +@dataclass(frozen=True) +class CameraResolution: + """Resolved image dimensions for one camera configuration. + + Args: + width: Image width in pixels. + height: Image height in pixels. + """ + + width: int + height: int + + @dataclass(frozen=True) class RunConfig: """Physics/rendering backend and active presets for a run. @@ -146,11 +159,14 @@ class RunConfig: resolutions, and any other domain presets are captured without a closed enum; ``physics_backend`` / ``rendering_backend`` surface the two primary grouping dimensions as typed fields. + camera_resolutions: Resolved camera image dimensions keyed by config + path. Empty when the task has no configured camera sensors. """ physics_backend: PhysicsBackend rendering_backend: RenderingBackend = "none" presets: list[str] = field(default_factory=list) + camera_resolutions: dict[str, CameraResolution] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/source/isaaclab/test/benchmark/test_builders.py b/source/isaaclab/test/benchmark/test_builders.py index 71b2b9294926..4b2be5786441 100644 --- a/source/isaaclab/test/benchmark/test_builders.py +++ b/source/isaaclab/test/benchmark/test_builders.py @@ -13,6 +13,7 @@ from isaaclab.benchmark import builders from isaaclab.benchmark.schema import ( + CameraResolution, GpuDeviceInfo, Hardware, MeanStd, @@ -68,6 +69,12 @@ def test_run_config_presets_default_empty(): assert builders.build_run_config("newton_mjwarp", presets=["rgb"]).presets == ["rgb"] +def test_run_config_camera_resolutions_default_empty(): + assert builders.build_run_config("physx").camera_resolutions == {} + resolutions = {"env.scene.camera": CameraResolution(width=640, height=480)} + assert builders.build_run_config("physx", camera_resolutions=resolutions).camera_resolutions == resolutions + + def test_run_identity_computes_duration(): run = builders.build_run_identity( run_id="x", diff --git a/source/isaaclab/test/benchmark/test_capture.py b/source/isaaclab/test/benchmark/test_capture.py index a54158b630ed..91279cccc51c 100644 --- a/source/isaaclab/test/benchmark/test_capture.py +++ b/source/isaaclab/test/benchmark/test_capture.py @@ -26,7 +26,7 @@ SingleMeasurement, StringMetadata, ) -from isaaclab.benchmark.schema import Hardware, Resources, Versions +from isaaclab.benchmark.schema import CameraResolution, Hardware, Resources, Versions from isaaclab.sensors import CameraCfg, RayCasterCameraCfg, patterns @@ -262,6 +262,12 @@ def test_camera_resolutions_report_distinct_cameras_sorted_by_path(): assert camera_resolutions_from_env_cfg(env_cfg) == expected assert camera_resolution_metadata_from_env_cfg(env_cfg) == [{"name": "camera_resolutions", "data": expected}] + env_cfg.sim = SimpleNamespace(physics=None) + assert run_config_from_env_cfg(env_cfg).camera_resolutions == { + "env.a_ray_camera": CameraResolution(width=32, height=24), + "env.z_camera": CameraResolution(width=64, height=48), + } + def test_camera_resolutions_preserve_aliased_camera_paths(): shared_camera = _camera_cfg() diff --git a/source/isaaclab/test/benchmark/test_schema.py b/source/isaaclab/test/benchmark/test_schema.py index 95688ce24bf2..92ad7595b36b 100644 --- a/source/isaaclab/test/benchmark/test_schema.py +++ b/source/isaaclab/test/benchmark/test_schema.py @@ -13,6 +13,7 @@ from isaaclab.benchmark.schema import ( SCHEMA_VERSION, + CameraResolution, CProfileFunction, EnvironmentStepTiming, GpuDeviceInfo, @@ -147,6 +148,7 @@ def test_training_bundle_round_trip(tmp_path): assert data["run"]["config"]["physics_backend"] == "newton_mjwarp" assert data["run"]["config"]["rendering_backend"] == "none" assert data["run"]["config"]["presets"] == [] + assert data["run"]["config"]["camera_resolutions"] == {} assert data["runtime"]["collection_fps"]["mean"] == pytest.approx(1_142_000.0) assert data["runtime"]["total_fps"]["mean"] == pytest.approx(1_071_780.0) timing = data["runtime"]["environment_step_timing"] @@ -166,6 +168,27 @@ def test_training_bundle_round_trip(tmp_path): assert data["versions"]["sb3"] is None +def test_camera_resolutions_round_trip(tmp_path): + bundle = dataclasses.replace( + _minimal_training_bundle(), + run=dataclasses.replace( + _run_identity(), + config=RunConfig( + physics_backend="newton_mjwarp", + rendering_backend="newton", + camera_resolutions={"env.scene.camera": CameraResolution(width=640, height=480)}, + ), + ), + ) + path = os.path.join(tmp_path, "training.json") + write_bundle_file(bundle, path) + + with open(path) as f: + data = json.load(f) + + assert data["run"]["config"]["camera_resolutions"] == {"env.scene.camera": {"width": 640, "height": 480}} + + def test_environment_step_timing_rejects_incomplete_measurement_modes(): timing = _runtime().environment_step_timing assert timing is not None