perf: Order OVRTX render-var reads on the consuming stream, and block the host on Linux (#7074)

## Description

Reading an OVRTX render var has to be ordered against render completion.
On Linux, blocking the calling thread on the render-completion event
measures faster end to end than a GPU-side wait, so that is now what
Linux does.

Measured across three camera tasks at 256/1024/4096 envs, this is +14%
to +73% throughput (mean +36%), and restores parity with ovrtx 0.3 on
every case measured.

Other platforms order the read on the consuming Warp stream.
`ISAAC_LAB_OVRTX_DISABLE_LINUX_CUDA_CPU_SYNC=1` puts Linux on that
ordering too — an escape hatch for when the trade-off changes; it is not
currently faster.

All mapping sites now go through one helper,
`OVRTXRenderer._map_render_var_to_dlpack()`. Camera outputs are
unchanged — only the read ordering differs.

Reported in https://nvbugspro.nvidia.com/bug/6566453

## Type of change

- Bug fix (non-breaking change which fixes an issue)
- Performance improvement

## Screenshots

None — no visual change.

## Checklist

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format`
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

Co-authored-by: pv-nvidia <197907000+pv-nvidia@users.noreply.github.com>

Author: pv-nvidia

source/isaaclab_ov/changelog.d/pv-ovrtx-render-var-sync.rst

@@ -0,0 +1,9 @@
+Fixed
+^^^^^
+
+* Improved OVRTX camera-output throughput on Linux. A render var has to be read in an order that
+  respects render completion, and on Linux blocking the calling thread on the render-completion
+  event measures faster than a GPU-side wait. Camera outputs are now read that way on Linux, worth
+  15-70% more end-to-end throughput depending on task and environment count. Other platforms order
+  the read on the consuming Warp stream, which Linux can also be switched to by setting
+  ``ISAAC_LAB_OVRTX_DISABLE_LINUX_CUDA_CPU_SYNC=1``. Camera outputs themselves are unchanged.

source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py

@@ -24,6 +24,8 @@
 import math
 import os
 import re
+import sys
+from collections.abc import Iterator
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, NoReturn, cast
 
@@ -125,6 +127,11 @@
 _USE_OVSTAGE_ENV = "ISAAC_LAB_OVRTX_USE_OVSTAGE"
 
 
+# Opts Linux out of the host wait, onto the same GPU-side ordering every other platform uses.
+# See :meth:`OVRTXRenderer._map_render_var_to_dlpack`.
+_DISABLE_LINUX_CUDA_CPU_SYNC_ENV = "ISAAC_LAB_OVRTX_DISABLE_LINUX_CUDA_CPU_SYNC"
+
+
 if _OVSTAGE_AVAILABLE:
     # DLDataType for a 4×4 double matrix (omni:xform column). ovstage stores omni:xform
     # as one 16-lane float64 element per prim; wp.mat44d maps to the same layout via __dlpack__.
@@ -203,6 +210,25 @@ def _read_gpu_transforms_enabled() -> bool:
     return value == "1"
 
 
+def _gpu_side_render_var_sync_enabled() -> bool:
+    """Return whether a render-var mapping is ordered by a GPU-side wait rather than a host wait.
+
+    See :meth:`OVRTXRenderer._map_render_var_to_dlpack` for why Linux is the exception, and
+    :data:`_DISABLE_LINUX_CUDA_CPU_SYNC_ENV` for opting out of it.
+
+    Raises:
+        ValueError: If the environment variable is set to anything other than ``0`` or ``1``.
+    """
+    if not sys.platform.startswith("linux"):
+        return True
+    value = os.environ.get(_DISABLE_LINUX_CUDA_CPU_SYNC_ENV, "0").strip()
+    if value not in {"0", "1"}:
+        raise ValueError(
+            f"Invalid value for environment variable `{_DISABLE_LINUX_CUDA_CPU_SYNC_ENV}`: {value}. Expected 0 or 1."
+        )
+    return value == "1"
+
+
 def _resolve_rtx_minimal_mode(data_types: list[str]) -> int | None:
     """Resolve the RTX minimal mode from data types.
 
@@ -1016,6 +1042,40 @@ def _generate_random_colors_from_ids(self, input_ids: wp.array, output_colors: w
         )
         return output_colors
 
+    @contextlib.contextmanager
+    def _map_render_var_to_dlpack(self, render_var: Any) -> Iterator[wp.array]:
+        """Map ``render_var`` for CUDA reads and yield it as a Warp array.
+
+        The render is still in flight when the mapping returns, so reading it has to be ordered
+        against render completion. Normally that is a ``cudaStreamWaitEvent`` on the Warp stream the
+        consuming kernels run on, which is the ordering the OVRTX API is designed around.
+
+        On Linux that GPU-side wait measures substantially slower end to end, so the mapping is
+        instead requested with no GPU-side barrier and the calling thread blocks on the
+        render-completion event. Setting :data:`_DISABLE_LINUX_CUDA_CPU_SYNC_ENV` to ``1`` puts
+        Linux back on the GPU-side wait; it is an escape hatch for platforms where that trade-off
+        no longer holds, and is worth re-measuring before being relied on.
+
+        Note that ``sync_stream=0`` is OVRTX's "no sync" sentinel, *not* the NULL CUDA stream: the
+        field encodes ``0=no sync, 1=default stream, >1=specific stream``, so omitting the argument
+        entirely means ``1``, not ``0``.
+
+        The yielded array is a zero-copy view of the mapped memory and is only valid inside the
+        ``with`` block -- the mapping is released on exit.
+
+        Args:
+            render_var: OVRTX ``RenderVarOutput`` to map (``frame.render_vars[name]``).
+
+        Yields:
+            The render var's contents as a Warp array, valid for the duration of the context.
+        """
+        gpu_side_sync = _gpu_side_render_var_sync_enabled()
+        sync_stream = wp.get_stream(self._device).cuda_stream if gpu_side_sync else 0
+        with render_var.map(device=Device.CUDA, sync_stream=sync_stream) as mapping:
+            if not gpu_side_sync:
+                mapping.wait()
+            yield wp.from_dlpack(mapping)
+
     def _process_id_segmentation_render_var(
         self,
         render_data: OVRTXRenderData,
@@ -1042,8 +1102,7 @@ def _process_id_segmentation_render_var(
         if render_var_name not in frame.render_vars or buffer_key not in output_buffers:
             return
 
-        with frame.render_vars[render_var_name].map(device=Device.CUDA) as mapping:
-            tiled_data = wp.from_dlpack(mapping)
+        with self._map_render_var_to_dlpack(frame.render_vars[render_var_name]) as tiled_data:
             if tiled_data.dtype != wp.uint32:
                 return
 
@@ -1255,15 +1314,13 @@ def _process_render_frame(self, render_data: OVRTXRenderData, frame, output_buff
                         break
 
             if buffer_key is not None:
-                with frame.render_vars["LdrColor"].map(device=Device.CUDA) as mapping:
-                    tiled_data = wp.from_dlpack(mapping)
+                with self._map_render_var_to_dlpack(frame.render_vars["LdrColor"]) as tiled_data:
                     self._extract_rgba_tiles(render_data, tiled_data, output_buffers, buffer_key)
 
         for depth_var in ["DistanceToCameraSD", "DistanceToImagePlaneSD", "DepthSD"]:
             if depth_var not in frame.render_vars:
                 continue
-            with frame.render_vars[depth_var].map(device=Device.CUDA) as mapping:
-                tiled_depth_data = wp.from_dlpack(mapping)
+            with self._map_render_var_to_dlpack(frame.render_vars[depth_var]) as tiled_depth_data:
                 if tiled_depth_data.dtype == wp.uint32:
                     tiled_depth_data = wp.from_torch(
                         wp.to_torch(tiled_depth_data).view(torch.float32), dtype=wp.float32
@@ -1272,13 +1329,11 @@ def _process_render_frame(self, render_data: OVRTXRenderData, frame, output_buff
             break
 
         if "DiffuseAlbedoSD" in frame.render_vars and "albedo" in output_buffers:
-            with frame.render_vars["DiffuseAlbedoSD"].map(device=Device.CUDA) as mapping:
-                tiled_albedo_data = wp.from_dlpack(mapping)
+            with self._map_render_var_to_dlpack(frame.render_vars["DiffuseAlbedoSD"]) as tiled_albedo_data:
                 self._extract_rgba_tiles(render_data, tiled_albedo_data, output_buffers, "albedo", suffix="albedo")
 
         if "HdrColor" in frame.render_vars and "rgb_hdr" in output_buffers:
-            with frame.render_vars["HdrColor"].map(device=Device.CUDA) as mapping:
-                tiled_hdr_data = wp.from_dlpack(mapping)
+            with self._map_render_var_to_dlpack(frame.render_vars["HdrColor"]) as tiled_hdr_data:
                 tiled_hdr_data = self._prepare_ppisp_hdr_source(render_data, tiled_hdr_data, output_buffers)
                 self._extract_hdr_color_tiles(render_data, tiled_hdr_data, output_buffers)
 
@@ -1308,16 +1363,14 @@ def _process_render_frame(self, render_data: OVRTXRenderData, frame, output_buff
             self._process_instance_segmentation_maps(render_data, frame)
 
         if "NormalSD" in frame.render_vars and "normals" in output_buffers:
-            with frame.render_vars["NormalSD"].map(device=Device.CUDA) as mapping:
-                tiled_normals_data = wp.from_dlpack(mapping)
+            with self._map_render_var_to_dlpack(frame.render_vars["NormalSD"]) as tiled_normals_data:
                 self._launch_extract_all_tiles(render_data, tiled_normals_data, output_buffers["normals"])
 
         # For motion vectors, extract only the first two (u, v) channels from the tiled buffer.
         # Note: mirrors the Isaac RTX renderer's handling of the "TargetMotionSD" AOV
         # (check: https://github.com/isaac-sim/IsaacLab/issues/2003).
         if "TargetMotionSD" in frame.render_vars and "motion_vectors" in output_buffers:
-            with frame.render_vars["TargetMotionSD"].map(device=Device.CUDA) as mapping:
-                tiled_motion_vectors_data = wp.from_dlpack(mapping)
+            with self._map_render_var_to_dlpack(frame.render_vars["TargetMotionSD"]) as tiled_motion_vectors_data:
                 self._launch_extract_all_tiles(render_data, tiled_motion_vectors_data, output_buffers["motion_vectors"])
 
     def _render_legacy(self, render_data: OVRTXRenderData) -> None:

source/isaaclab_ov/test/test_ovrtx_renderer_contract.py

@@ -5,7 +5,10 @@
 
 """Tests for the OVRTX renderer output contract."""
 
+import contextlib
 import importlib.util
+import sys
+import types
 
 import pytest
 import torch
@@ -30,8 +33,10 @@
     from isaaclab_ov.renderers import OVRTXRendererCfg  # noqa: E402
     from isaaclab_ov.renderers import ovrtx_renderer as ovrtx_renderer_module  # noqa: E402
     from isaaclab_ov.renderers.ovrtx_renderer import (  # noqa: E402
+        _DISABLE_LINUX_CUDA_CPU_SYNC_ENV,
         OVRTXRenderData,
         OVRTXRenderer,
+        _gpu_side_render_var_sync_enabled,
         ovrtx_use_ovstage_enabled,
     )
 else:
@@ -40,6 +45,8 @@
     OVRTXRendererCfg = None
     ovrtx_renderer_module = None
     ovrtx_use_ovstage_enabled = None
+    _DISABLE_LINUX_CUDA_CPU_SYNC_ENV = None
+    _gpu_side_render_var_sync_enabled = None
 
 _SPAWN = PinholeCameraCfg(
     focal_length=24.0,
@@ -366,6 +373,90 @@ def test_ovrtx_use_ovstage_rejects_non_boolean_values(monkeypatch):
         ovrtx_use_ovstage_enabled()
 
 
+@pytest.mark.parametrize("platform", ["win32", "darwin"])
+def test_ovrtx_render_var_sync_is_gpu_side_off_linux(monkeypatch, platform):
+    """Everywhere but Linux the mapping is ordered by a GPU-side wait on the Warp stream."""
+    monkeypatch.setattr(sys, "platform", platform)
+    monkeypatch.delenv(_DISABLE_LINUX_CUDA_CPU_SYNC_ENV, raising=False)
+    assert _gpu_side_render_var_sync_enabled() is True
+
+
+def test_ovrtx_render_var_sync_waits_on_host_on_linux(monkeypatch):
+    """Linux blocks the calling thread instead, which measures faster there."""
+    monkeypatch.setattr(sys, "platform", "linux")
+    monkeypatch.delenv(_DISABLE_LINUX_CUDA_CPU_SYNC_ENV, raising=False)
+    assert _gpu_side_render_var_sync_enabled() is False
+
+
+def test_ovrtx_render_var_sync_is_gpu_side_on_linux_when_disabled(monkeypatch):
+    """Opting out of the host wait puts Linux on the same GPU-side wait as every other platform."""
+    monkeypatch.setattr(sys, "platform", "linux")
+    monkeypatch.setenv(_DISABLE_LINUX_CUDA_CPU_SYNC_ENV, "1")
+    assert _gpu_side_render_var_sync_enabled() is True
+
+
+def test_ovrtx_render_var_sync_keeps_host_wait_when_explicitly_enabled(monkeypatch):
+    """``0`` is the default, so setting it explicitly must not change anything."""
+    monkeypatch.setattr(sys, "platform", "linux")
+    monkeypatch.setenv(_DISABLE_LINUX_CUDA_CPU_SYNC_ENV, "0")
+    assert _gpu_side_render_var_sync_enabled() is False
+
+
+@pytest.mark.parametrize("value", ["", "true", "yes", "2"])
+def test_ovrtx_render_var_sync_rejects_non_boolean_values(monkeypatch, value):
+    """Values other than 0/1 are a configuration error, not a silent fallback to the host wait."""
+    monkeypatch.setattr(sys, "platform", "linux")
+    monkeypatch.setenv(_DISABLE_LINUX_CUDA_CPU_SYNC_ENV, value)
+    with pytest.raises(ValueError, match="Expected 0 or 1"):
+        _gpu_side_render_var_sync_enabled()
+
+
+class _RecordingRenderVar:
+    """Stand-in for an OVRTX ``RenderVarOutput`` that records how the read was ordered.
+
+    Any of OVRTX's ordering mechanisms counts, so the test stays about *whether* the read is
+    ordered rather than which call carries it.
+    """
+
+    def __init__(self):
+        self.ordering: list[str] = []
+
+    def map(self, *, device, sync_stream):
+        if sync_stream:
+            self.ordering.append("gpu")
+        recorder = self
+
+        class _Mapping:
+            def wait(self):
+                recorder.ordering.append("host")
+
+            def wait_on(self, stream):
+                recorder.ordering.append("gpu")
+
+        return contextlib.nullcontext(_Mapping())
+
+
+@pytest.mark.parametrize(("gpu_side", "expected"), [(True, "gpu"), (False, "host")])
+def test_ovrtx_map_render_var_orders_the_read_against_render_completion(monkeypatch, gpu_side, expected):
+    """The read is ordered exactly once -- by a GPU-side barrier or a host block, never by neither.
+
+    Ordering by neither is a silent race on half-written render output rather than a failure, so
+    this asserts which mechanism ran and not which API call carries it.
+    """
+    sentinel = object()
+    render_var = _RecordingRenderVar()
+    monkeypatch.setattr(ovrtx_renderer_module, "_gpu_side_render_var_sync_enabled", lambda: gpu_side)
+    monkeypatch.setattr(ovrtx_renderer_module.wp, "get_stream", lambda device: types.SimpleNamespace(cuda_stream=99))
+    monkeypatch.setattr(ovrtx_renderer_module.wp, "from_dlpack", lambda mapping: sentinel)
+
+    renderer = _make_ovrtx_renderer_without_backend()
+    renderer._device = "cuda:0"
+    with renderer._map_render_var_to_dlpack(render_var) as array:
+        assert array is sentinel
+
+    assert render_var.ordering == [expected]
+
+
 def test_ovrtx_cleanup_releases_only_the_given_render_data():
     """``cleanup`` releases the render data's own buffers and leaves the renderer usable.