diff --git a/.github/test-subsets/postmerge-rendering.toml b/.github/test-subsets/postmerge-rendering.toml index d258535a2860..c0b6f406d138 100644 --- a/.github/test-subsets/postmerge-rendering.toml +++ b/.github/test-subsets/postmerge-rendering.toml @@ -31,6 +31,8 @@ rendering-correctness-kitless-legacy = [ "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[legacy-ovphysx-ovrtx-albedo]", "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[legacy-ovphysx-ovrtx-rgb]", "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[legacy-ovphysx-ovrtx-semantic_segmentation]", + "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless_async[legacy-newton-ovrtx-rgb]", + "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless_async[legacy-ovphysx-ovrtx-rgb]", ] rendering-correctness-kitless-ovstage = [ @@ -39,4 +41,6 @@ rendering-correctness-kitless-ovstage = [ "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovstage-ovphysx-ovrtx-albedo]", "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovstage-ovphysx-ovrtx-rgb]", "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovstage-ovphysx-ovrtx-semantic_segmentation]", + "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless_async[ovstage-newton-ovrtx-rgb]", + "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless_async[ovstage-ovphysx-ovrtx-rgb]", ] diff --git a/docs/source/testing/benchmarks.rst b/docs/source/testing/benchmarks.rst index 0b95057b611d..bb63c3f79f7c 100644 --- a/docs/source/testing/benchmarks.rst +++ b/docs/source/testing/benchmarks.rst @@ -78,6 +78,25 @@ JSON. ``schema`` writes the stable, typed JSON bundle used for programmatic comparison. With multiple formatters, their filenames include ``_summary`` and ``_schema`` respectively. +OVRTX camera tasks can compare synchronous against asynchronous (one-frame-latency) rendering +through a Hydra override on the camera's renderer configuration (use ``False`` to force +synchronous): + +.. code-block:: bash + + uv run isaaclab benchmark runtime \ + --task Isaac-Cartpole-Camera-Direct \ + --num_envs 256 \ + --warmup_frames 30 \ + --num_frames 200 \ + --benchmark_formatter schema,omniperf \ + --output_path ./results/ovrtx_async \ + physics=newton_mjwarp renderer=ovrtx presets=rgb \ + env.tiled_camera.renderer_cfg.async_rendering=True + +``OVRTX_ASYNC_RENDERING=1`` in the environment toggles the same path for any task, which avoids +naming a camera that a given task may not define. + Warm-Up ~~~~~~~ diff --git a/source/isaaclab_ov/changelog.d/pv-ovrtx-async-slot-buffers.minor.rst b/source/isaaclab_ov/changelog.d/pv-ovrtx-async-slot-buffers.minor.rst new file mode 100644 index 000000000000..384a499fd805 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/pv-ovrtx-async-slot-buffers.minor.rst @@ -0,0 +1,17 @@ +Added +^^^^^ + +* Added an opt-in asynchronous OVRTX render path controlled by + :attr:`~isaaclab_ov.renderers.ovrtx_renderer_cfg.OVRTXRendererCfg.async_rendering` + (default ``False``). When enabled, rendering overlaps simulation and Python work and camera + outputs arrive one frame later, improving throughput. The first frame after (re)initialization + is consumed immediately so the first camera read returns a valid frame; later frames are + pipelined. Available on both the legacy and ovstage scene-ownership paths; under ovstage each + scene write first drains any render still in flight, since OVRTX reads the stage's storage in + place. +* Added the ``OVRTX_ASYNC_RENDERING`` environment variable to override + :attr:`~isaaclab_ov.renderers.ovrtx_renderer_cfg.OVRTXRendererCfg.async_rendering` for tests. +* Added the ``OVRTX_NUM_BUFFERS`` environment variable to configure the render queue depth: the + number of asynchronous renders kept in flight (default ``2``, values below ``2`` are clamped). + Larger values overlap more simulation with rendering at the cost of extra frames of camera + latency. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index a1da0a05c4a7..70d7b4e70e86 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -10,7 +10,11 @@ - **ovrtx_renderer.py** (this file): Orchestrates the pipeline. Owns the OVRTX Renderer, USD loading/cloning, camera and object bindings, and output buffers. Each frame it: updates camera/object transforms (using kernels), steps the renderer, then extracts - tiles from the tiled framebuffer (kernels). + tiles from the tiled framebuffer (kernels). It delegates *how* a frame is executed + (synchronously or pipelined) to a render strategy. + +- **ovrtx_renderer_strategies.py**: Sync vs async render-execution strategies — how transform + writes are staged and how the OVRTX step is dispatched and consumed (Strategy pattern). - **ovrtx_renderer_kernels.py**: Warp GPU kernels for OVRTX rendering pipeline. @@ -62,6 +66,7 @@ PrimMode, Renderer, RendererConfig, + RenderProductSetOutputs, Semantic, ) except ModuleNotFoundError as exc: @@ -93,6 +98,12 @@ generate_random_colors_from_ids_kernel, sync_newton_transforms_kernel, ) +from .ovrtx_renderer_strategies import ( + _RENDER_DELTA_TIME, + _AsyncRenderStrategy, + _RenderStrategy, + _SyncRenderStrategy, +) from .ovrtx_usd import ( build_render_product_as_string, create_scene_partition_attributes, @@ -114,6 +125,7 @@ RenderBufferKind.SIMPLE_SHADING_FULL_MDL.value: 3, } + _PPISP_IMPORT_ERROR_MESSAGE = ( "isaaclab_ppisp is required when CameraCfg.isp_cfg is set. " "It ships with the Isaac Lab wheel (`pip install isaaclab`); otherwise install the " @@ -186,6 +198,18 @@ def ovrtx_use_ovstage_enabled() -> bool: return value == "1" +def _resolve_render_strategy(cfg: OVRTXRendererCfg) -> _RenderStrategy: + """Return the asynchronous strategy when ``cfg`` enables it, else the synchronous one. + + Both scene-ownership paths support asynchronous rendering. Under ovstage the renderer reads the + stage's storage in place, so a scene write could otherwise mutate data a queued render is still + reading; every ovstage write drains the queue first via + :meth:`_RenderStrategy.settle_before_scene_write`, leaving each render to observe exactly the + publication its ordinal names. + """ + return _AsyncRenderStrategy.try_create(cfg) or _SyncRenderStrategy() + + def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: # Only translate missing isaaclab_ppisp imports into the optional-dependency hint; # unrelated missing modules should surface unchanged for easier debugging. @@ -359,12 +383,6 @@ def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: RenderBufferKind.MOTION_VECTORS: RenderBufferSpec(2, wp.float32), } - @property - def _device_id(self) -> int: - """CUDA device index extracted from ``self._device`` for OVRTX ``binding.map()`` calls.""" - parts = self._device.split(":") - return int(parts[1]) if len(parts) > 1 else 0 - def __init__(self, cfg: OVRTXRendererCfg): self.cfg = cfg self._device = "cuda:0" # default; overridden by create_render_data(spec) @@ -386,6 +404,7 @@ def __init__(self, cfg: OVRTXRendererCfg): # Selected once at construction so every dispatch method below sees a stable path for the # lifetime of the renderer, even if the environment variable changes mid-process. self._use_ovstage = ovrtx_use_ovstage_enabled() + self._strategy: _RenderStrategy = _resolve_render_strategy(cfg) self._init_fields() logger.info("Creating OVRTX renderer...") @@ -403,6 +422,11 @@ def __init__(self, cfg: OVRTXRendererCfg): ) logger.info("OVRTX renderer created successfully") + @property + def is_async_rendering_enabled(self) -> bool: + """Whether asynchronous rendering is enabled.""" + return isinstance(self._strategy, _AsyncRenderStrategy) + def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: """Resolve the camera's PPISP cfg and apply OVRTX-specific USD overrides. @@ -536,6 +560,7 @@ def _initialize_from_spec_legacy(self, spec: CameraRenderSpec): ) self._initialized_scene = True + self._strategy.initialize(num_envs) self._camera_xform_binding = self._renderer.bind_attribute( prim_paths=camera_paths, @@ -847,6 +872,7 @@ def create_render_data(self, spec: CameraRenderSpec) -> OVRTXRenderData: matching the interface of Isaac RTX and Newton Warp which need no separate initialize(). """ self._device = spec.device + self._strategy.set_device(self._device) if not self._initialized_scene: self._initialize_from_spec(spec) return OVRTXRenderData(spec, self._device) @@ -899,12 +925,14 @@ def _update_transforms_legacy(self) -> None: if body_q is None: return - with self._object_xform_binding.map(device=Device.CUDA, device_id=self._device_id) as attr_mapping: - ovrtx_transforms = wp.from_dlpack(attr_mapping.tensor, dtype=wp.mat44d) + # The state stages the transforms (mapped OVRTX buffer when sync, a double-buffered slot when + # async) and publishes them to the binding when the context exits. + num_objects = len(self._object_newton_indices) + with self._strategy.stage_object_transforms(self._object_xform_binding, num_objects) as object_transforms: wp.launch( kernel=sync_newton_transforms_kernel, - dim=len(self._object_newton_indices), - inputs=[ovrtx_transforms, self._object_newton_indices, body_q], + dim=num_objects, + inputs=[object_transforms, self._object_newton_indices, body_q], device=self._device, ) @@ -1007,26 +1035,28 @@ def _update_camera_legacy( intrinsics: ProxyArray, ) -> None: """Update camera transforms in OVRTX binding.""" + if self._camera_xform_binding is None: + return num_envs = positions.shape[0] - converted_wp = wp.empty(num_envs, dtype=wp.quatf, device=self._device) - convert_camera_frame_orientation_convention_wp( - src=orientations.warp, - dst=converted_wp, - origin="world", - target="opengl", - device=self._device, - ) - camera_transforms = wp.zeros(num_envs, dtype=wp.mat44d, device=self._device) - wp.launch( - kernel=create_camera_transforms_kernel, - dim=num_envs, - inputs=[positions, converted_wp, camera_transforms], - device=self._device, - ) - if self._camera_xform_binding is not None: - with self._camera_xform_binding.map(device=Device.CUDA, device_id=self._device_id) as attr_mapping: - wp_transforms_view = wp.from_dlpack(attr_mapping.tensor, dtype=wp.mat44d) - wp.copy(wp_transforms_view, camera_transforms) + # The state stages the quaternion scratch + transform buffers and publishes the transforms to + # the binding when the context exits (mapped copy when sync, queued async write when async). + with self._strategy.stage_camera_transforms(self._camera_xform_binding, num_envs) as ( + converted_wp, + camera_transforms, + ): + convert_camera_frame_orientation_convention_wp( + src=orientations.warp, + dst=converted_wp, + origin="world", + target="opengl", + device=self._device, + ) + wp.launch( + kernel=create_camera_transforms_kernel, + dim=num_envs, + inputs=[positions, converted_wp, camera_transforms], + device=self._device, + ) def read_output( self, @@ -1296,7 +1326,7 @@ def _prepare_ppisp_hdr_source( # FIXME: OVRTX render var mapping can select a different CUDA device # than the camera/output buffers on MGPU systems. Keep this PPISP-only # bridge until render var mapping can be constrained like transform - # bindings, which use ``device_id=self._device_id``. + # binding maps, which pin the CUDA device via ``device_id``. return wp.clone(tiled_data, device=output_device) def _process_render_frame(self, render_data: OVRTXRenderData, frame, output_buffers: dict) -> None: @@ -1391,10 +1421,16 @@ def _render_legacy(self, render_data: OVRTXRenderData) -> None: raise RuntimeError("Scene not initialized. Call initialize() first.") if self._renderer is None or len(self._render_product_paths) == 0: return - products = self._renderer.step( - render_products=set(self._render_product_paths), - delta_time=1.0 / 60.0, + + self._strategy.render( + self._renderer, + set(self._render_product_paths), + _RENDER_DELTA_TIME, + render_data, + self._consume_products, ) + + def _consume_products(self, render_data: OVRTXRenderData, products: RenderProductSetOutputs) -> None: product_path = self._render_product_paths[0] if product_path in products and len(products[product_path].frames) > 0: self._process_render_frame( @@ -1524,7 +1560,8 @@ def cleanup(self, render_data: OVRTXRenderData | None) -> None: The stage queries, tensor bindings and render products this renderer holds are shared by every camera that resolves to it, so releasing them here would tear the scene down while - the other cameras are still rendering. :meth:`close` releases them instead. + the other cameras are still rendering. :meth:`close` releases them instead. The render + strategy is shared the same way, so its in-flight work is drained in :meth:`close` too. """ if render_data is None: return @@ -1534,6 +1571,10 @@ def cleanup(self, render_data: OVRTXRenderData | None) -> None: def close(self) -> None: """Release the shared stage state. See :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.close`.""" + # Drain in-flight work before either backend releases the renderer it belongs to. ``render_data`` + # is None because :meth:`cleanup` already released the per-camera buffers, so queued renders are + # only waited on, not consumed. + self._strategy.cleanup(None, self._consume_products) if self._use_ovstage: self._close_ovstage() else: @@ -1552,6 +1593,16 @@ def close(self) -> None: # ovstage 0.1.1 will address this. # --------------------------------------------------------------------------- + def _write_attribute_ovstage(self, *args, **kwargs) -> None: + """Write one ovstage attribute after draining any render that must not observe the change. + + All ovstage scene mutations funnel through here. OVRTX reads the stage's storage in place, so + this write would land in memory a queued render is still reading; the strategy drains those + renders first. Under synchronous rendering nothing is ever in flight, so the drain is a no-op. + """ + self._strategy.settle_before_scene_write() + self._stage.write_attribute(*args, **kwargs).wait() + def _init_fields_ovstage(self) -> None: self._stage = None self._stage_paths = None @@ -1638,14 +1689,14 @@ def _initialize_from_spec_ovstage(self, spec: CameraRenderSpec) -> None: camera_target_ids = np.array( [self._stage_paths.intern_path(path) for path in camera_paths], dtype=np.uint64 ) - self._stage.write_attribute( + self._write_attribute_ovstage( render_product_query, camera_attribute, ordinal=self._current_ordinal, tensors=camera_target_ids, is_array=True, semantic=ovstage.AttributeSemantic.RELATIONSHIP_PATH_ID, - ).wait() + ) self._stage_paths.destroy_path_list(render_product_paths) self._camera_paths_list = self._stage_paths.create_path_list_from_strings(camera_paths) @@ -1657,13 +1708,13 @@ def _initialize_from_spec_ovstage(self, spec: CameraRenderSpec) -> None: # Resetting the xform stack makes omni:xform the absolute world transform, preventing # ancestor transforms (env root, asset root) from compounding on top of the camera pose. - self._stage.write_attribute( + self._write_attribute_ovstage( self._camera_xform_query, "omni:resetXformStack", ordinal=self._current_ordinal, tensors=np.full(num_envs, True, dtype=np.bool_), is_array=False, - ).wait() + ) self._setup_xform_bindings_ovstage() self._setup_deformable_bindings_ovstage(num_envs) @@ -1751,14 +1802,14 @@ def _clone_sources_ovstage(self): logger.info("Cloned %d sources successfully in OVRTX", num_cloned_sources) # Restore the pre-clone xforms. - self._stage.write_attribute( + self._write_attribute_ovstage( env_query, "omni:xform", ordinal=self._current_ordinal, tensors=_xform_tensor_from_numpy(env_root_xforms), is_array=False, semantic=ovstage.AttributeSemantic.MATRIX, - ).wait() + ) self._env_root_xforms = None logger.info("Restored per-env root transforms after cloning") @@ -1776,28 +1827,28 @@ def _update_scene_partitions_after_clone_ovstage(self, num_envs: int): env_paths_list = self._stage_paths.create_path_list_from_strings(env_prim_paths) env_query = self._stage.query_from_path_list(env_paths_list) - self._stage.write_attribute( + self._write_attribute_ovstage( env_query, "primvars:omni:scenePartition", ordinal=self._current_ordinal, tensors=token_ids, is_array=False, semantic=ovstage.AttributeSemantic.TOKEN_ID, - ).wait() + ) self._stage.release_query(env_query).wait() self._stage_paths.destroy_path_list(env_paths_list) logger.info("Written primvars:omni:scenePartition to %d environments", num_envs) cam_paths_list = self._stage_paths.create_path_list_from_strings(camera_prim_paths) cam_query = self._stage.query_from_path_list(cam_paths_list) - self._stage.write_attribute( + self._write_attribute_ovstage( cam_query, "omni:scenePartition", ordinal=self._current_ordinal, tensors=token_ids, is_array=False, semantic=ovstage.AttributeSemantic.TOKEN_ID, - ).wait() + ) self._stage.release_query(cam_query).wait() self._stage_paths.destroy_path_list(cam_paths_list) logger.info("Written omni:scenePartition to %d cameras", num_envs) @@ -1838,13 +1889,13 @@ def _setup_xform_bindings_ovstage(self) -> None: self._object_paths_list = self._stage_paths.create_path_list_from_strings(object_paths) self._object_xform_query = self._stage.query_from_path_list(self._object_paths_list) - self._stage.write_attribute( + self._write_attribute_ovstage( self._object_xform_query, "omni:resetXformStack", ordinal=self._current_ordinal, tensors=np.full(len(object_paths), True, dtype=np.bool_), is_array=False, - ).wait() + ) if self._object_xform_query is None: raise RuntimeError("Failed to create OVRTX object bindings") @@ -1915,23 +1966,23 @@ def _setup_deformable_bindings_ovstage(self, num_envs: int) -> None: # particle_q is already in world space, so resetting the xform stack and pinning an identity # omni:xform prevents the env-root and asset-root ancestor transforms from being applied on top. - self._stage.write_attribute( + self._write_attribute_ovstage( self._deformable_points_query, "omni:resetXformStack", ordinal=self._current_ordinal, tensors=np.full(prim_count, True, dtype=np.bool_), is_array=False, - ).wait() + ) identity_xforms = np.tile(np.eye(4, dtype=np.float64), (prim_count, 1, 1)) - self._stage.write_attribute( + self._write_attribute_ovstage( self._deformable_points_query, "omni:xform", ordinal=self._current_ordinal, tensors=_xform_tensor_from_numpy(identity_xforms), is_array=False, semantic=ovstage.AttributeSemantic.MATRIX, - ).wait() + ) if self._deformable_points_query is None: raise RuntimeError("Failed to create OVRTX deformable body bindings") @@ -1971,23 +2022,23 @@ def _setup_particle_bindings_ovstage(self) -> None: # # particle_q is already in world space, so resetting the xform stack and pinning an identity # omni:xform prevents the env-root and asset-root ancestor transforms from being applied on top. - self._stage.write_attribute( + self._write_attribute_ovstage( self._particle_points_query, "omni:resetXformStack", ordinal=self._current_ordinal, tensors=np.full(prim_count, True, dtype=np.bool_), is_array=False, - ).wait() + ) identity_xforms = np.tile(np.eye(4, dtype=np.float64), (prim_count, 1, 1)) - self._stage.write_attribute( + self._write_attribute_ovstage( self._particle_points_query, "omni:xform", ordinal=self._current_ordinal, tensors=_xform_tensor_from_numpy(identity_xforms), is_array=False, semantic=ovstage.AttributeSemantic.MATRIX, - ).wait() + ) if self._particle_points_query is None: raise RuntimeError("Failed to create OVRTX particle point bindings") @@ -2026,14 +2077,14 @@ def _update_transforms_ovstage(self) -> None: # dtype override on numpy arrays, not DLPack producers. wp.mat44d exports as (N,4,4) lanes=1 # via DLPack, which conflicts with the lanes=16 omni:xform column created at population time. wp.synchronize_device(self._device) - self._stage.write_attribute( + self._write_attribute_ovstage( self._object_xform_query, "omni:xform", ordinal=self._current_ordinal, tensors=_xform_tensor_from_numpy(object_transforms.numpy().reshape(-1, 4, 4)), is_array=False, semantic=ovstage.AttributeSemantic.MATRIX, - ).wait() + ) def _update_geometries_ovstage(self) -> None: if self._deformable_points_query is None and self._particle_points_query is None: @@ -2099,14 +2150,14 @@ def _write_particle_q_slices_ovstage( for particle_offset, particle_count in zip(particle_offsets, particle_counts, strict=True) ] - self._stage.write_attribute( + self._write_attribute_ovstage( query, "points", ordinal=self._current_ordinal, tensors=particle_slices, is_array=True, semantic=ovstage.AttributeSemantic.POINT, - ).wait() + ) def _update_camera_ovstage( self, @@ -2134,14 +2185,14 @@ def _update_camera_ovstage( if self._camera_xform_query is not None: # Synchronize then copy to CPU numpy: same lanes=16 constraint as object transforms above. wp.synchronize_device(self._device) - self._stage.write_attribute( + self._write_attribute_ovstage( self._camera_xform_query, "omni:xform", ordinal=self._current_ordinal, tensors=_xform_tensor_from_numpy(camera_transforms.numpy().reshape(-1, 4, 4)), is_array=False, semantic=ovstage.AttributeSemantic.MATRIX, - ).wait() + ) def _render_ovstage(self, render_data: OVRTXRenderData) -> None: if not self._initialized_scene: @@ -2151,27 +2202,15 @@ def _render_ovstage(self, render_data: OVRTXRenderData) -> None: # Commit all per-frame writes (transforms, geometries, camera) then step. # advance_write_floor must precede step — the renderer rejects ordinal > write_floor. self._stage.advance_write_floor(ordinal=self._current_ordinal).wait() - products = self._renderer.step( - render_products=set(self._render_product_paths), - delta_time=1.0 / 60.0, + self._strategy.render( + self._renderer, + set(self._render_product_paths), + _RENDER_DELTA_TIME, + render_data, + self._consume_products, ordinal=self._current_ordinal, ) self._current_ordinal += 1 - product_path = self._render_product_paths[0] - if product_path in products and len(products[product_path].frames) > 0: - self._process_render_frame( - render_data, - products[product_path].frames[0], - render_data.warp_buffers, - ) - - # Post-render PPISP: HDR scene-linear → LDR RGBA. Source/destination - # buffers are the same warp buffer map used by extraction. - if render_data.ppisp_pipeline is not None: - render_data.ppisp_pipeline.apply( - render_data.warp_buffers[str(RenderBufferKind.RGB_HDR)], - render_data.warp_buffers[str(RenderBufferKind.RGBA)], - ) def _close_ovstage(self) -> None: """Release the renderer's stage queries, path lists and ovstage stage. See :meth:`close`.""" diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py index 5137e61523fb..48e78631da48 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_cfg.py @@ -67,3 +67,14 @@ class OVRTXRendererCfg(RendererCfg): If True, instance IDs are mapped to RGBA colors and returned as a ``uint8`` 4-channel array. If False, raw instance IDs are returned as a ``uint32`` 1-channel array. """ + + async_rendering: bool = False + """Whether to render asynchronously. Defaults to False. + + When True, the renderer submits a render and returns immediately, overlapping rendering with + simulation and Python work to improve throughput, at the cost of camera outputs arriving one + frame later. + + For tests this can be overridden with the ``OVRTX_ASYNC_RENDERING`` environment variable + (``0``/``false``/``no``/``off`` disable, any other non-empty value enables). + """ diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_strategies.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_strategies.py new file mode 100644 index 000000000000..3166c5900b34 --- /dev/null +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_strategies.py @@ -0,0 +1,456 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Rendering execution strategies for the OVRTX renderer. + +The renderer delegates *how* a frame is executed -- how transform writes are staged and how the OVRTX +step is dispatched and consumed -- to a :class:`_RenderStrategy`, selected once from configuration so +the renderer's call sites carry no ``sync``/``async`` branching. + +- :class:`_SyncRenderStrategy` writes transforms straight into OVRTX and consumes each step inline. +- :class:`_AsyncRenderStrategy` pipelines steps and double-buffers transform staging, so rendering + overlaps simulation at the cost of camera outputs arriving one frame later. + +The interface is mechanism-neutral (:meth:`~_RenderStrategy.initialize`, +:meth:`~_RenderStrategy.stage_object_transforms`, :meth:`~_RenderStrategy.stage_camera_transforms`, +:meth:`~_RenderStrategy.render`, :meth:`~_RenderStrategy.cleanup`); slot and queue vocabulary stays +private to :class:`_AsyncRenderStrategy`. +""" + +from __future__ import annotations + +import logging +import os +from abc import ABC, abstractmethod +from collections import deque +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, TypeAlias + +import warp as wp +from ovrtx import DataAccess, Device + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from ovrtx import Operation, PendingFetch, Renderer, RenderProductSetOutputs + + from .ovrtx_renderer import OVRTXRenderData + from .ovrtx_renderer_cfg import OVRTXRendererCfg + + _AsyncRenderOp: TypeAlias = Operation[PendingFetch[RenderProductSetOutputs]] + _RenderProductConsumer: TypeAlias = Callable[[OVRTXRenderData, RenderProductSetOutputs], None] + + +# OVRTX steps assume a fixed frame delta for temporal accumulation and motion vectors. +_RENDER_DELTA_TIME = 1.0 / 60.0 + + +class _AsyncRenderEntry: + """An in-flight render step together with the destination its products belong to. + + The entry carries its own ``render_data`` and consumer so a drain triggered from an unrelated call + site -- a scene write, or teardown -- still delivers the frame to the buffers the step was + submitted for. + """ + + def __init__( + self, + op: _AsyncRenderOp, + render_data: OVRTXRenderData | None, + consume_products: _RenderProductConsumer, + ) -> None: + self.op = op + self.render_data = render_data + self.consume_products = consume_products + self.products: RenderProductSetOutputs | None = None + + def get_products(self): + if self.products is None: + self.products = self.op.wait().fetch() + return self.products + + def deliver(self) -> bool: + """Wait for the step, then hand its products to the destination it was submitted for.""" + products = self.get_products() + if products is None: + return False + if self.render_data is not None: + self.consume_products(self.render_data, products) + return True + + +class _RenderStrategy(ABC): + """Strategy for how the OVRTX renderer stages transforms and dispatches render steps. + + Two concrete strategies exist: :class:`_SyncRenderStrategy` steps OVRTX and consumes its products + inline, while :class:`_AsyncRenderStrategy` pipelines steps and double-buffers transform staging. + The renderer drives whichever it holds through this neutral interface, so its call sites carry no + ``sync``/``async`` branching. Slot/queue vocabulary stays private to :class:`_AsyncRenderStrategy`. + """ + + def __init__(self) -> None: + self._device: str | None = None + + def set_device(self, device: str) -> None: + """Record the CUDA device used for staging buffers and binding maps.""" + self._device = device + + @property + def _device_id(self) -> int: + """CUDA device index parsed from :attr:`_device` for OVRTX ``binding.map()`` calls.""" + assert self._device is not None + parts = self._device.split(":") + return int(parts[1]) if len(parts) > 1 else 0 + + def initialize(self, num_envs: int) -> None: + """Prepare per-scene staging resources for ``num_envs`` environments. + + Called once the scene is (re)initialized. The default does nothing; strategies that own + staging buffers override it. + """ + + def cleanup(self, render_data: OVRTXRenderData | None, consume_products: _RenderProductConsumer) -> None: + """Flush any in-flight work and release staging resources. + + The default does nothing; strategies that own queued work override it. + """ + + def settle_before_scene_write(self) -> None: + """Drain in-flight renders that could observe a scene mutation issued after them. + + Backends whose writes land in storage the renderer reads in place call this before mutating the + scene. The default does nothing; strategies holding in-flight renders override it. + """ + + @abstractmethod + def stage_object_transforms(self, binding: Any, num_rows: int) -> AbstractContextManager[wp.array]: + """Yield a ``mat44d`` buffer of ``num_rows`` object transforms for the caller's kernel to fill. + + The yielded buffer is published to ``binding`` when the context exits without error. + """ + + @abstractmethod + def stage_camera_transforms(self, binding: Any, num_rows: int) -> AbstractContextManager[tuple[wp.array, wp.array]]: + """Yield ``(quats, transforms)`` staging buffers for ``num_rows`` cameras. + + ``quats`` is a ``quatf`` scratch buffer and ``transforms`` is the ``mat44d`` destination; the + latter is published to ``binding`` when the context exits without error. + """ + + @abstractmethod + def render( + self, + renderer: Renderer, + render_products: set[str], + delta_time: float, + render_data: OVRTXRenderData, + consume_products: _RenderProductConsumer, + ordinal: int | None = None, + ) -> None: + """Step ``renderer`` for one frame and consume its products, immediately or deferred. + + ``ordinal`` is the minimum committed ovstage publication the step must observe. It is required + while an ovstage is attached to ``renderer`` and rejected otherwise, so the caller passes the + value matching its scene-ownership path. + """ + + +class _SyncRenderStrategy(_RenderStrategy): + """Blocking strategy: transform writes go straight to OVRTX and each step is consumed inline.""" + + @contextmanager + def stage_object_transforms(self, binding: Any, num_rows: int) -> Iterator[wp.array]: + """Map ``binding`` and yield its buffer so the caller's kernel writes OVRTX storage directly. + + See :meth:`_RenderStrategy.stage_object_transforms`. ``num_rows`` is implied by the binding. + """ + with binding.map(device=Device.CUDA, device_id=self._device_id) as attr_mapping: + yield wp.from_dlpack(attr_mapping.tensor, dtype=wp.mat44d) + + @contextmanager + def stage_camera_transforms(self, binding: Any, num_rows: int) -> Iterator[tuple[wp.array, wp.array]]: + """Yield freshly allocated staging buffers and copy the result into ``binding`` on exit. + + See :meth:`_RenderStrategy.stage_camera_transforms`. + """ + camera_quats = wp.empty(num_rows, dtype=wp.quatf, device=self._device) + camera_transforms = wp.zeros(num_rows, dtype=wp.mat44d, device=self._device) + yield camera_quats, camera_transforms + with binding.map(device=Device.CUDA, device_id=self._device_id) as attr_mapping: + wp_transforms_view = wp.from_dlpack(attr_mapping.tensor, dtype=wp.mat44d) + wp.copy(wp_transforms_view, camera_transforms) + + def render( + self, + renderer: Renderer, + render_products: set[str], + delta_time: float, + render_data: OVRTXRenderData, + consume_products: _RenderProductConsumer, + ordinal: int | None = None, + ) -> None: + """Step OVRTX and consume its products inline. See :meth:`_RenderStrategy.render`.""" + products = renderer.step(render_products=render_products, delta_time=delta_time, ordinal=ordinal) + consume_products(render_data, products) + + +@dataclass +class _AsyncRenderSlot: + """A reusable set of transform staging buffers for one in-flight async update.""" + + camera_transforms: wp.array + camera_quats: wp.array + object_transforms: wp.array | None + write_ops: list[Operation] + + def record_write(self, binding: Any, data: wp.array, cuda_stream: int) -> None: + """Issue an async binding write and record its op so it can be drained before reuse. + + ``cuda_stream`` is the Warp stream the staging kernel that filled ``data`` ran on. Handing it to + OVRTX lets OVRTX insert a GPU-side wait before its ``DataAccess.ASYNC`` read, so the subsequent + ``step_async`` never observes a partially written transform buffer. Without it OVRTX performs no + cross-stream sync against the fill kernel and the read races the write. + """ + self.write_ops.append(binding.write_async(data, data_access=DataAccess.ASYNC, cuda_stream=cuda_stream)) + + def wait_for_writes(self) -> None: + """Block until this slot's outstanding async writes drain, so its buffers are safe to reuse.""" + if not self.write_ops: + return + try: + for op in self.write_ops: + op.wait() + except Exception as e: + raise RuntimeError("Failed to complete OVRTX async binding write before slot reuse") from e + finally: + self.write_ops = [] + + +class _AsyncRenderStrategy(_RenderStrategy): + """Pipelined strategy: steps are queued and consumed later, with double-buffered staging. + + Transform writes always use two slots. A ``DataAccess.ASYNC`` write copies the data into + OVRTX's own storage, so OVRTX reads a buffer only until its write op completes. A buffer is safe + to recycle once its write has drained. + """ + + # See :meth:`_ensure_slots` for why two is always enough. + _NUM_SLOTS = 2 + # Renders kept in flight before the oldest is drained. Two means one frame of camera latency. + # Overridable via ``OVRTX_NUM_BUFFERS`` (see :meth:`_resolve_render_queue_depth`). + _DEFAULT_RENDER_QUEUE_DEPTH = 2 + + @classmethod + def try_create(cls, cfg: OVRTXRendererCfg) -> _AsyncRenderStrategy | None: + """Create an :class:`_AsyncRenderStrategy` when async rendering is enabled, else return ``None``. + + Async rendering is toggled by :attr:`OVRTXRendererCfg.async_rendering`. The + ``OVRTX_ASYNC_RENDERING`` environment variable overrides the config (``0``/``false``/``no``/``off`` + disable, any other non-empty value enables) so existing golden-image tests can exercise the + async path without editing task configs. + """ + enabled = bool(getattr(cfg, "async_rendering", False)) + env_override = os.environ.get("OVRTX_ASYNC_RENDERING") + if env_override is not None and env_override != "": + enabled = env_override.strip().lower() not in ("0", "false", "no", "off") + return cls() if enabled else None + + @classmethod + def _resolve_render_queue_depth(cls) -> int: + """Return the render queue depth, honoring the ``OVRTX_NUM_BUFFERS`` environment variable. + + The value is the number of ``step_async`` renders kept in flight before the oldest is drained; + larger values overlap more simulation with rendering at the cost of extra frames of camera + latency. Falls back to :attr:`_DEFAULT_RENDER_QUEUE_DEPTH` when unset; a value below 2 would + disable pipelining and is clamped up to 2. + """ + raw = os.environ.get("OVRTX_NUM_BUFFERS") + if raw is None or raw.strip() == "": + return cls._DEFAULT_RENDER_QUEUE_DEPTH + try: + value = int(raw.strip()) + except ValueError: + logger.warning( + "Ignoring invalid OVRTX_NUM_BUFFERS=%r; using default %d.", raw, cls._DEFAULT_RENDER_QUEUE_DEPTH + ) + return cls._DEFAULT_RENDER_QUEUE_DEPTH + if value < 2: + logger.warning("OVRTX_NUM_BUFFERS=%d is below the minimum of 2; clamping to 2.", value) + return 2 + return value + + def __init__(self) -> None: + super().__init__() + self._num_envs = 0 + self._render_queue_depth = self._resolve_render_queue_depth() + self._ring: deque[_AsyncRenderEntry] = deque() + self._slots: list[_AsyncRenderSlot] = [] + self._slot_index = 0 + self._primed = False + self._current_slot: _AsyncRenderSlot | None = None + + def _has_pending_ops(self) -> bool: + """Return whether any render op is still queued.""" + return bool(self._ring) + + def _enqueue_render_op( + self, op: _AsyncRenderOp, render_data: OVRTXRenderData | None, consume_products: _RenderProductConsumer + ) -> _AsyncRenderEntry: + """Record an async render op and end the current frame's slot, draining one slot when the ring is full.""" + entry = _AsyncRenderEntry(op, render_data, consume_products) + self._ring.append(entry) + self._current_slot = None + if len(self._ring) >= self._render_queue_depth: + self._try_drain_one() + return entry + + def initialize(self, num_envs: int) -> None: + """Reset staging slots and record the camera count for future slot builds. + + See :meth:`_RenderStrategy.initialize`. + """ + self._reset_slots(num_envs) + + def _reset_slots(self, num_envs: int) -> None: + """Drain and drop all staging slots, and record the camera count for future slot builds. + + ``num_envs == 0`` means the renderer is unbinding, so slots are simply cleared. + """ + for slot in self._slots: + slot.wait_for_writes() + self._slots.clear() + self._slot_index = 0 + self._current_slot = None + self._num_envs = num_envs + self._primed = False + # An empty ring marks the first frame in :meth:`render`, which primes it synchronously. + self._ring.clear() + + def _ensure_slots(self) -> None: + if self._slots: + return + + # Two slots suffice at any render depth: with one write per frame, a slot's only outstanding + # work on reuse is its write from two frames ago, which :meth:`_begin_slot` waits on. OVRTX + # copies the buffer into its own storage before that write op completes. + assert self._device is not None + for _ in range(self._NUM_SLOTS): + self._slots.append( + _AsyncRenderSlot( + camera_transforms=wp.zeros(self._num_envs, dtype=wp.mat44d, device=self._device), + camera_quats=wp.empty(self._num_envs, dtype=wp.quatf, device=self._device), + object_transforms=None, + write_ops=[], + ) + ) + + def _begin_slot(self) -> _AsyncRenderSlot: + """Rotate to the next staging slot for a new frame's transform writes.""" + self._ensure_slots() + slot = self._slots[self._slot_index] + self._slot_index = (self._slot_index + 1) % len(self._slots) + slot.wait_for_writes() + self._current_slot = slot + return slot + + def _current_or_begin_slot(self) -> _AsyncRenderSlot: + """Return the slot for the current frame, starting one if none is active yet.""" + return self._current_slot or self._begin_slot() + + def _write_binding_async(self, slot: _AsyncRenderSlot, binding: Any, data: wp.array) -> None: + """Record an async binding write on ``slot``, using the device's Warp stream for OVRTX ordering.""" + slot.record_write(binding, data, wp.get_stream(self._device).cuda_stream) + + @contextmanager + def stage_object_transforms(self, binding: Any, num_rows: int) -> Iterator[wp.array]: + """Stage object transforms into a fresh frame slot; publish them on exit. + + See :meth:`_RenderStrategy.stage_object_transforms`. Uses :meth:`_begin_slot`, so each object + update opens (and double-buffers) a new slot for the frame. + """ + slot = self._begin_slot() + object_transforms = slot.object_transforms + if object_transforms is None or object_transforms.shape[0] != num_rows: + object_transforms = wp.zeros(num_rows, dtype=wp.mat44d, device=self._device) + slot.object_transforms = object_transforms + yield object_transforms + self._write_binding_async(slot, binding, object_transforms) + + @contextmanager + def stage_camera_transforms(self, binding: Any, num_rows: int) -> Iterator[tuple[wp.array, wp.array]]: + """Stage camera transforms into the current frame slot; publish them on exit. + + See :meth:`_RenderStrategy.stage_camera_transforms`. Uses :meth:`_current_or_begin_slot` to join + the slot opened by the object update (or open one when there was no object update this frame). + The slot's camera buffers are pre-sized to ``num_envs`` in :meth:`_ensure_slots`; reallocate + them if ``num_rows`` diverges, mirroring :meth:`stage_object_transforms`. + """ + slot = self._current_or_begin_slot() + if slot.camera_transforms.shape[0] != num_rows: + slot.camera_transforms = wp.zeros(num_rows, dtype=wp.mat44d, device=self._device) + slot.camera_quats = wp.empty(num_rows, dtype=wp.quatf, device=self._device) + yield slot.camera_quats, slot.camera_transforms + self._write_binding_async(slot, binding, slot.camera_transforms) + + def render( + self, + renderer: Renderer, + render_products: set[str], + delta_time: float, + render_data: OVRTXRenderData, + consume_products: _RenderProductConsumer, + ordinal: int | None = None, + ) -> None: + """Step OVRTX asynchronously and enqueue the op for deferred consumption. + + The first frame of a scene is primed synchronously: the op is waited on and consumed + immediately so the first read returns a rendered frame rather than the zero-initialized output + buffer. Priming is tracked explicitly rather than inferred from an empty ring, because a + backend that drains the ring on every scene write leaves it empty at every render. The entry + caches its products, so the next frame's drain reuses them instead of fetching twice. Later + frames are pipelined; a drain only replaces buffer contents, so the output stays valid while + the queue fills. See :meth:`_RenderStrategy.render`. + """ + is_first_frame = not self._primed + op = renderer.step_async(render_products=render_products, delta_time=delta_time, ordinal=ordinal) + self._enqueue_render_op(op, render_data, consume_products) + self._primed = True + if is_first_frame: + self._try_drain_one() + + def settle_before_scene_write(self) -> None: + """Drain every queued render so the caller's scene write cannot alter a frame in flight. + + See :meth:`_RenderStrategy.settle_before_scene_write`. Draining here keeps the pipelining + window open across the caller's own work -- simulation, inference and product reads all + overlap the render, and only the next frame's first write closes it. + """ + while self._has_pending_ops(): + self._try_drain_one() + + def _try_drain_one(self) -> bool: + """Complete the oldest queued render and deliver it. Returns False when nothing was queued.""" + return bool(self._ring) and self._ring.popleft().deliver() + + def cleanup( + self, + render_data: OVRTXRenderData | None, + consume_products: _RenderProductConsumer, + ) -> None: + """Drain all queued renders best-effort and drop staging slots. + + See :meth:`_RenderStrategy.cleanup`. + """ + # Log and continue: a failure on one op must not prevent draining the rest or tearing the + # renderer down. The per-frame drain in :meth:`_enqueue_render_op` propagates instead. + while self._has_pending_ops(): + try: + self._try_drain_one() + except Exception as e: + logger.warning("Error draining OVRTX async render op: %s", e, exc_info=True) + + self._reset_slots(0) diff --git a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py index c2166a2a4af3..02c7480dc25a 100644 --- a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py +++ b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py @@ -38,6 +38,7 @@ _resolve_clone_plan, _write_file, ) + from isaaclab_ov.renderers.ovrtx_renderer_strategies import _SyncRenderStrategy # noqa: E402 from pxr import Usd, UsdGeom # noqa: E402 else: @@ -105,6 +106,7 @@ def _make_ovrtx_renderer_without_backend() -> OVRTXRenderer: write_attribute=lambda *args, **kwargs: None, ) renderer._clone_plan = None + renderer._strategy = _SyncRenderStrategy() renderer._camera_rel_path = "Camera" renderer._render_product_paths = [] renderer._exported_usd_string = None diff --git a/source/isaaclab_ov/test/test_ovrtx_render_ordinal.py b/source/isaaclab_ov/test/test_ovrtx_render_ordinal.py new file mode 100644 index 000000000000..af0878d6a2d8 --- /dev/null +++ b/source/isaaclab_ov/test/test_ovrtx_render_ordinal.py @@ -0,0 +1,106 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for ordinal propagation from the render strategies into ovrtx stepping.""" + +from __future__ import annotations + +import importlib.util +from typing import Any + +import pytest + +_REQUIRED_MODULES = ("isaaclab_ov", "ovrtx") +_MISSING_MODULES = [module for module in _REQUIRED_MODULES if importlib.util.find_spec(module) is None] + +pytestmark = [ + pytest.mark.isaacsim_ci, + pytest.mark.skipif( + bool(_MISSING_MODULES), + reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", + ), +] + +if not _MISSING_MODULES: + from isaaclab_ov.renderers.ovrtx_renderer_strategies import _AsyncRenderStrategy, _SyncRenderStrategy + + +class _RecordingRenderer: + """Captures the ordinal ovrtx would receive, mimicking ovrtx's attached/standalone contract.""" + + def __init__(self, *, attached: bool) -> None: + self._attached = attached + self.ordinals: list[int | None] = [] + + def _check(self, ordinal: int | None) -> None: + if self._attached and ordinal is None: + raise RuntimeError("ordinal is required while an ovstage is attached") + if not self._attached and ordinal is not None: + raise RuntimeError("ordinal is only valid while an ovstage is attached") + self.ordinals.append(ordinal) + + def step(self, render_products: set[str], delta_time: float, *, ordinal: int | None = None) -> dict: + self._check(ordinal) + return {} + + def step_async(self, render_products: set[str], delta_time: float, *, ordinal: int | None = None) -> Any: + self._check(ordinal) + return _CompletedOp() + + +class _CompletedOp: + """Stands in for an ovrtx step operation whose products are already available.""" + + def wait(self) -> _CompletedOp: + return self + + def fetch(self) -> dict: + return {} + + +def _render(strategy, renderer, ordinal: int | None) -> None: + strategy.render(renderer, {"/product"}, 1.0 / 60.0, None, lambda render_data, products: None, ordinal=ordinal) + + +def test_sync_strategy_forwards_ordinal_when_attached(): + renderer = _RecordingRenderer(attached=True) + + _render(_SyncRenderStrategy(), renderer, 7) + + assert renderer.ordinals == [7] + + +def test_sync_strategy_omits_ordinal_when_standalone(): + renderer = _RecordingRenderer(attached=False) + + _render(_SyncRenderStrategy(), renderer, None) + + assert renderer.ordinals == [None] + + +def test_async_strategy_forwards_ordinal_when_attached(): + renderer = _RecordingRenderer(attached=True) + + _render(_AsyncRenderStrategy(), renderer, 11) + + assert renderer.ordinals == [11] + + +def test_async_strategy_omits_ordinal_when_standalone(): + renderer = _RecordingRenderer(attached=False) + + _render(_AsyncRenderStrategy(), renderer, None) + + assert renderer.ordinals == [None] + + +def test_async_strategy_forwards_each_frame_ordinal(): + renderer = _RecordingRenderer(attached=True) + strategy = _AsyncRenderStrategy() + + for ordinal in (3, 4, 5): + _render(strategy, renderer, ordinal) + + assert renderer.ordinals == [3, 4, 5] diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index e1a5651dca44..1d6914044ba2 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -73,6 +73,9 @@ def _make_ovrtx_render_data() -> OVRTXRenderData: def _make_ovrtx_renderer_without_backend() -> OVRTXRenderer: renderer = OVRTXRenderer.__new__(OVRTXRenderer) renderer.cfg = OVRTXRendererCfg() + # ``__init__`` is bypassed, so set the strategy it would build: ``close`` drains the strategy + # before releasing the backend. + renderer._strategy = ovrtx_renderer_module._resolve_render_strategy(renderer.cfg) return renderer diff --git a/source/isaaclab_ov/test/test_ovrtx_scene_write_barrier.py b/source/isaaclab_ov/test/test_ovrtx_scene_write_barrier.py new file mode 100644 index 000000000000..28dcb8794428 --- /dev/null +++ b/source/isaaclab_ov/test/test_ovrtx_scene_write_barrier.py @@ -0,0 +1,202 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the scene-write barrier that bounds pipelined renders under ovstage.""" + +from __future__ import annotations + +import importlib.util +from typing import Any + +import pytest + +_REQUIRED_MODULES = ("isaaclab_ov", "ovrtx") +_MISSING_MODULES = [module for module in _REQUIRED_MODULES if importlib.util.find_spec(module) is None] + +pytestmark = [ + pytest.mark.isaacsim_ci, + pytest.mark.skipif( + bool(_MISSING_MODULES), + reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", + ), +] + +if not _MISSING_MODULES: + from isaaclab_ov.renderers.ovrtx_renderer_strategies import _AsyncRenderStrategy, _SyncRenderStrategy + + +class _Timeline: + """Records the order of renderer and stage events so their interleaving can be asserted.""" + + def __init__(self) -> None: + self.events: list[str] = [] + + def of_kind(self, kind: str) -> list[str]: + return [event for event in self.events if event.startswith(kind)] + + +class _PendingOp: + """An ovrtx step operation that only reports completion once waited on.""" + + def __init__(self, timeline: _Timeline, index: int) -> None: + self._timeline = timeline + self._index = index + self.waited = False + + def wait(self) -> _PendingOp: + self.waited = True + self._timeline.events.append(f"drain:{self._index}") + return self + + def fetch(self) -> dict: + return {} + + +class _FakeRenderer: + """Renderer stub that records each submitted step and hands back a pending operation.""" + + def __init__(self, timeline: _Timeline) -> None: + self._timeline = timeline + self.ops: list[_PendingOp] = [] + + def step_async(self, render_products: set[str], delta_time: float, *, ordinal: int | None = None) -> _PendingOp: + op = _PendingOp(self._timeline, len(self.ops)) + self.ops.append(op) + self._timeline.events.append(f"submit:{len(self.ops) - 1}:ordinal={ordinal}") + return op + + def step(self, render_products: set[str], delta_time: float, *, ordinal: int | None = None) -> dict: + self._timeline.events.append(f"step:ordinal={ordinal}") + return {} + + +@pytest.fixture() +def timeline() -> _Timeline: + return _Timeline() + + +@pytest.fixture() +def strategy(monkeypatch) -> _AsyncRenderStrategy: + monkeypatch.delenv("OVRTX_NUM_BUFFERS", raising=False) + strategy = _AsyncRenderStrategy() + strategy.set_device("cuda:0") + return strategy + + +def _render(strategy: Any, renderer: _FakeRenderer, ordinal: int, consumed: list[int]) -> None: + strategy.render( + renderer, + {"/Render/Product"}, + 1.0 / 60.0, + object(), + lambda render_data, products: consumed.append(ordinal), + ordinal=ordinal, + ) + + +def test_settle_drains_every_in_flight_render(strategy, timeline): + renderer = _FakeRenderer(timeline) + consumed: list[int] = [] + + _render(strategy, renderer, 0, consumed) + _render(strategy, renderer, 1, consumed) + strategy.settle_before_scene_write() + + assert all(op.waited for op in renderer.ops) + assert not strategy._has_pending_ops() + + +def test_settle_is_idempotent_without_pending_renders(strategy, timeline): + renderer = _FakeRenderer(timeline) + + strategy.settle_before_scene_write() + strategy.settle_before_scene_write() + + assert timeline.events == [] + assert renderer.ops == [] + + +def test_render_stays_pipelined_when_settle_precedes_each_frame(strategy, timeline): + """A submit must precede the drain of that same frame, otherwise the path is merely synchronous.""" + renderer = _FakeRenderer(timeline) + consumed: list[int] = [] + + for ordinal in range(4): + strategy.settle_before_scene_write() + _render(strategy, renderer, ordinal, consumed) + + # Frame 0 is primed synchronously; frames 1..3 each drain only at the following frame's write. + assert timeline.events == [ + "submit:0:ordinal=0", + "drain:0", + "submit:1:ordinal=1", + "drain:1", + "submit:2:ordinal=2", + "drain:2", + "submit:3:ordinal=3", + ] + assert consumed == [0, 1, 2] + + +def test_every_frame_is_delivered_exactly_once(strategy, timeline): + renderer = _FakeRenderer(timeline) + consumed: list[int] = [] + + for ordinal in range(5): + strategy.settle_before_scene_write() + _render(strategy, renderer, ordinal, consumed) + strategy.cleanup(object(), lambda render_data, products: consumed.append(-1)) + + assert consumed == [0, 1, 2, 3, 4] + + +def test_first_frame_is_primed_after_reinitialize(strategy, timeline): + """Priming must be tracked explicitly: a drained ring still means 'already primed'.""" + renderer = _FakeRenderer(timeline) + consumed: list[int] = [] + + _render(strategy, renderer, 0, consumed) + assert consumed == [0] + + strategy.initialize(4) + timeline.events.clear() + consumed.clear() + + _render(strategy, renderer, 1, consumed) + assert consumed == [1], "the first frame of a new scene must be primed synchronously" + + +def test_frames_are_delivered_to_the_render_data_they_were_submitted_for(strategy, timeline): + """A drain triggered by a scene write must not deliver into another frame's buffers.""" + renderer = _FakeRenderer(timeline) + first_target = object() + second_target = object() + delivered: list[object] = [] + + def consume(render_data, products): + delivered.append(render_data) + + strategy.render(renderer, {"/P"}, 1.0 / 60.0, first_target, consume, ordinal=0) + delivered.clear() + + strategy.render(renderer, {"/P"}, 1.0 / 60.0, second_target, consume, ordinal=1) + strategy.settle_before_scene_write() + + assert delivered == [second_target] + + +def test_sync_strategy_needs_no_barrier(timeline): + """The barrier is a no-op for synchronous rendering, which holds nothing in flight.""" + strategy = _SyncRenderStrategy() + strategy.set_device("cuda:0") + renderer = _FakeRenderer(timeline) + consumed: list[int] = [] + + strategy.settle_before_scene_write() + _render(strategy, renderer, 7, consumed) + strategy.settle_before_scene_write() + + assert timeline.events == ["step:ordinal=7"] + assert consumed == [7] diff --git a/source/isaaclab_ov/test/test_ovrtx_strategy_selection.py b/source/isaaclab_ov/test/test_ovrtx_strategy_selection.py new file mode 100644 index 000000000000..adeb9430cfd3 --- /dev/null +++ b/source/isaaclab_ov/test/test_ovrtx_strategy_selection.py @@ -0,0 +1,60 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for OVRTX render strategy selection.""" + +from __future__ import annotations + +import importlib.util + +import pytest + +_REQUIRED_MODULES = ("isaaclab_ov", "ovrtx") +_MISSING_MODULES = [module for module in _REQUIRED_MODULES if importlib.util.find_spec(module) is None] + +pytestmark = [ + pytest.mark.isaacsim_ci, + pytest.mark.skipif( + bool(_MISSING_MODULES), + reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", + ), +] + +if not _MISSING_MODULES: + from isaaclab_ov.renderers.ovrtx_renderer import _resolve_render_strategy + from isaaclab_ov.renderers.ovrtx_renderer_cfg import OVRTXRendererCfg + from isaaclab_ov.renderers.ovrtx_renderer_strategies import _AsyncRenderStrategy, _SyncRenderStrategy + + +@pytest.fixture() +def async_cfg(monkeypatch) -> OVRTXRendererCfg: + monkeypatch.delenv("OVRTX_ASYNC_RENDERING", raising=False) + return OVRTXRendererCfg(async_rendering=True) + + +@pytest.fixture() +def sync_cfg(monkeypatch) -> OVRTXRendererCfg: + monkeypatch.delenv("OVRTX_ASYNC_RENDERING", raising=False) + return OVRTXRendererCfg(async_rendering=False) + + +def test_async_selected_when_enabled(async_cfg): + assert isinstance(_resolve_render_strategy(async_cfg), _AsyncRenderStrategy) + + +def test_sync_selected_when_disabled(sync_cfg): + assert isinstance(_resolve_render_strategy(sync_cfg), _SyncRenderStrategy) + + +def test_env_override_enables_async(sync_cfg, monkeypatch): + monkeypatch.setenv("OVRTX_ASYNC_RENDERING", "1") + + assert isinstance(_resolve_render_strategy(sync_cfg), _AsyncRenderStrategy) + + +def test_env_override_disables_async(async_cfg, monkeypatch): + monkeypatch.setenv("OVRTX_ASYNC_RENDERING", "0") + + assert isinstance(_resolve_render_strategy(async_cfg), _SyncRenderStrategy) diff --git a/source/isaaclab_tasks/changelog.d/pv-ovrtx-async-slot-buffers.rst b/source/isaaclab_tasks/changelog.d/pv-ovrtx-async-slot-buffers.rst new file mode 100644 index 000000000000..e9008b7488b8 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/pv-ovrtx-async-slot-buffers.rst @@ -0,0 +1,4 @@ +Added +^^^^^ + +* Added opt-in asynchronous OVRTX rendering for rendering benchmark runs. diff --git a/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py index 7d94751116b4..a00eee5cd158 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py @@ -10,6 +10,7 @@ import pytest from rendering_test_utils import ( KITLESS_PHYSICS_RENDERER_AOV_COMBINATIONS, + _make_sensor_data_type_params, make_attach_comparison_properties_fixture, make_determinism_fixture, make_generate_html_report_fixture, @@ -19,6 +20,17 @@ rendering_test_cartpole, ) +# Async variants of the synchronous combinations: ``OVRTX_ASYNC_RENDERING`` toggles the renderer's +# pipelined path independently of the preset, so the output must match the same golden images. The one +# frame of latency is absorbed by the SSIM / pixel-difference tolerances, and the shared harness renders +# warm-up frames until camera outputs are non-zero, which primes the pipeline before capture. +_ASYNC_COMBINATIONS = make_kitless_rendering_params( + [ + *_make_sensor_data_type_params("ovphysx", "ovrtx", ["rgb"]), + *_make_sensor_data_type_params("newton", "ovrtx", ["rgb"]), + ] +) + pytestmark = [pytest.mark.isaacsim_ci, pytest.mark.arm_ci] _OVRTX_TEXTURE_READINESS_XFAIL_REASON = "OVRTX 0.4 may return before textured materials are ready (NVBUG#6505191)." @@ -52,3 +64,12 @@ def test_rendering_cartpole_kitless(ovstage_variant, physics_backend, renderer, data_type): """Camera output must match golden images (Cartpole camera presets env).""" rendering_test_cartpole(physics_backend, renderer, data_type, _COMPARISON_SCORES) + + +@pytest.mark.parametrize( + "ovstage_variant,physics_backend,renderer,data_type", _ASYNC_COMBINATIONS, indirect=["ovstage_variant"] +) +def test_rendering_cartpole_kitless_async(ovstage_variant, physics_backend, renderer, data_type, monkeypatch): + """OVRTX async-rendered camera output must match the synchronous golden images (within tolerance).""" + monkeypatch.setenv("OVRTX_ASYNC_RENDERING", "1") + rendering_test_cartpole(physics_backend, renderer, data_type, _COMPARISON_SCORES) diff --git a/source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py b/source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py index 9e32e8b57c1b..c96fa4dac565 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py +++ b/source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py @@ -18,10 +18,10 @@ import pytest # noqa: E402 import torch # noqa: E402 from rendering_test_utils import ( # noqa: E402 - MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME, make_attach_comparison_properties_fixture, make_determinism_fixture, make_generate_html_report_fixture, + max_different_pixels_percentage_for, maybe_save_stage, validate_camera_outputs, ) @@ -125,7 +125,7 @@ def test_rendering_registered_tasks(task_id: str, presets: str | None, env_name: "default_physics", "default_renderer", camera_outputs, - max_different_pixels_percentage=MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME[env_name], + max_different_pixels_percentage=max_different_pixels_percentage_for(env_name), comparison_scores=_COMPARISON_SCORES, ) finally: diff --git a/source/isaaclab_tasks/test/core/test_rendering_tolerance_lookup.py b/source/isaaclab_tasks/test/core/test_rendering_tolerance_lookup.py new file mode 100644 index 000000000000..86783b531b5c --- /dev/null +++ b/source/isaaclab_tasks/test/core/test_rendering_tolerance_lookup.py @@ -0,0 +1,69 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the per-environment pixel tolerance lookup.""" + +from __future__ import annotations + +import ast +import re +from pathlib import Path + +import pytest + +_UTILS_PATH = Path(__file__).resolve().parent.parent / "rendering_test_utils.py" +_SOURCE = _UTILS_PATH.read_text() + +# Call sites must resolve tolerances through the helper; a bare subscript yields the whole +# ``[sync, async]`` list instead of a float. +_RAW_SUBSCRIPT_RE = re.compile(r"MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME\[") + +# The helper is the only sanctioned reader of the table. +_HELPER_UNPACK = "synchronous, asynchronous = MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME" + + +def test_no_call_site_subscripts_the_tolerance_table_directly(): + offenders = [ + f"{_UTILS_PATH.name}:{index}: {line.strip()}" + for index, line in enumerate(_SOURCE.splitlines(), start=1) + if _RAW_SUBSCRIPT_RE.search(line) and _HELPER_UNPACK not in line + ] + + assert not offenders, "resolve tolerances via max_different_pixels_percentage_for():\n" + "\n".join(offenders) + + +def test_every_environment_declares_a_sync_and_async_tolerance(): + """Every entry must be a two-element ``[sync, async]`` list.""" + tree = ast.parse(_SOURCE) + table = next( + node.value + for node in ast.walk(tree) + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME" + for target in node.targets + ) + ) + + assert isinstance(table, ast.Dict) + for key, value in zip(table.keys, table.values): + assert isinstance(value, ast.List), f"{ast.literal_eval(key)} must be [sync, async]" + assert len(value.elts) == 2, f"{ast.literal_eval(key)} must declare exactly two tolerances" + + +@pytest.mark.parametrize("env_name", ["cartpole", "shadow_hand", "franka_cloth"]) +def test_helper_returns_a_float_for_both_lanes(env_name, monkeypatch): + import sys + + sys.path.insert(0, str(_UTILS_PATH.parent)) + from rendering_test_utils import max_different_pixels_percentage_for + + monkeypatch.delenv("OVRTX_ASYNC_RENDERING", raising=False) + sync = max_different_pixels_percentage_for(env_name) + monkeypatch.setenv("OVRTX_ASYNC_RENDERING", "1") + async_ = max_different_pixels_percentage_for(env_name) + + assert isinstance(sync, float) and isinstance(async_, float) + assert async_ >= sync, "the async tolerance must not be tighter than the synchronous one" diff --git a/source/isaaclab_tasks/test/rendering_test_utils.py b/source/isaaclab_tasks/test/rendering_test_utils.py index c3a4c6837782..a775dbc732e5 100644 --- a/source/isaaclab_tasks/test/rendering_test_utils.py +++ b/source/isaaclab_tasks/test/rendering_test_utils.py @@ -43,25 +43,47 @@ # The max percentage of pixels allowed to differ. If the percentage exceeds this value, the test will fail. # The value is set case by case based on the screen space taken up by the env in camera output images. It # needs to be large enough to tolerate minor rendering noise while small enough to catch unexpected changes. +# Entries are ``[synchronous, asynchronous]``: pipelined rendering captures a slightly different render +# state, so it carries its own tolerance. Read via :func:`max_different_pixels_percentage_for`. MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME = { # RTX anti-aliasing along the ground-plane edges varies slightly across GPU and driver environments. - "cartpole": 1.5, + "cartpole": [1.5, 2.5], # Aliasing artifacts of shadow on the table. - "franka_cloth": 8.0, - "franka_soft": 8.0, + "franka_cloth": [8.0, 8.0], + "franka_soft": [8.0, 8.0], # Shadow-hand renderings (incl. ``Isaac-Reorient-Cube-Shadow-Camera-Direct``) show up to # ~3.28 % per-pixel diff from anti-aliasing noise along the many finger/cube edges. 5.0 gives # headroom above that without masking real regressions, which the SSIM gate still catches. - "shadow_hand": 5.0, + "shadow_hand": [5.0, 5.0], # Texture aliasing artifacts on the ground (NVBUG#6116767) - "lift_kuka_homo": 8.0, - "lift_kuka_hetero": 8.0, + "lift_kuka_homo": [8.0, 8.0], + "lift_kuka_hetero": [8.0, 8.0], } # Allow OVRTX Cartpole RGB/RGBA variation tracked by NVBUG#6152566; the SSIM gate remains enabled. The # deterministic Warp rasterizer and the Isaac RTX reference path keep the stricter env-wide threshold. _CARTPOLE_OVRTX_RGB_MAX_DIFFERENT_PIXELS_PERCENTAGE = 2.0 + +def _async_rendering_enabled() -> bool: + """Whether asynchronous (pipelined) rendering is active, per the ``OVRTX_ASYNC_RENDERING`` toggle.""" + value = os.environ.get("OVRTX_ASYNC_RENDERING", "").strip().lower() + return value not in ("", "0", "false", "no", "off") + + +def max_different_pixels_percentage_for(env_name: str) -> float: + """Return ``env_name``'s pixel-diff tolerance for the render path this run uses. + + Args: + env_name: Key into :data:`MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME`. + + Returns: + The synchronous or asynchronous tolerance, per :func:`_async_rendering_enabled`. + """ + synchronous, asynchronous = MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME[env_name] + return asynchronous if _async_rendering_enabled() else synchronous + + # Minimum SSIM score below which two images are considered structurally different. SSIM is a perceptual metric # robust to uniform per-pixel noise that penalises structural changes (geometry shifts, swapped colours, missing # materials, etc.), so it complements the per-pixel L2 gate by catching regressions that survive a loosened pixel @@ -1388,7 +1410,7 @@ class _ShadowHandCameraTestEnvCfg(ShadowHandCameraEnvCfg): physics_backend, renderer, env._tiled_camera.data.output, - max_different_pixels_percentage=MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME["shadow_hand"], + max_different_pixels_percentage=max_different_pixels_percentage_for("shadow_hand"), comparison_scores=comparison_scores, ) @@ -1471,7 +1493,7 @@ class _YellowBgEnvCfg(ShadowHandCameraEnvCfg): physics_backend, renderer, env._tiled_camera.data.output, - max_different_pixels_percentage=MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME["shadow_hand"], + max_different_pixels_percentage=max_different_pixels_percentage_for("shadow_hand"), comparison_scores=comparison_scores, ) finally: @@ -1585,9 +1607,11 @@ class _CartpoleCameraTestEnvCfg(CartpoleCameraEnvCfg): data_type, compare_golden=compare_golden and data_type == "rgb", ) - max_different_pixels_percentage = MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME["cartpole"] + max_different_pixels_percentage = max_different_pixels_percentage_for("cartpole") if renderer == "ovrtx_renderer" and data_type in ("rgb", "rgba"): - max_different_pixels_percentage = _CARTPOLE_OVRTX_RGB_MAX_DIFFERENT_PIXELS_PERCENTAGE + max_different_pixels_percentage = max( + max_different_pixels_percentage, _CARTPOLE_OVRTX_RGB_MAX_DIFFERENT_PIXELS_PERCENTAGE + ) validate_camera_outputs( "cartpole", physics_backend, @@ -1744,7 +1768,7 @@ class _KukaAllegroLiftCameraTestEnvCfg(KukaAllegroLiftCameraEnvCfg): physics_backend, renderer, env.scene.sensors["base_camera"].data.output, - max_different_pixels_percentage=MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME[test_name], + max_different_pixels_percentage=max_different_pixels_percentage_for(test_name), comparison_scores=comparison_scores, ) finally: @@ -1839,7 +1863,7 @@ def rendering_test_franka_cloth( physics_backend, renderer, env.scene.sensors["base_camera"].data.output, - max_different_pixels_percentage=MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME[test_name], + max_different_pixels_percentage=max_different_pixels_percentage_for(test_name), comparison_scores=comparison_scores, ) finally: @@ -1911,7 +1935,7 @@ def rendering_test_franka_soft( physics_backend, renderer, env.scene.sensors["base_camera"].data.output, - max_different_pixels_percentage=MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME[test_name], + max_different_pixels_percentage=max_different_pixels_percentage_for(test_name), comparison_scores=comparison_scores, ) finally: