From 65f7e4c9d551b60e4afd4e3a464faea8bbc368ff Mon Sep 17 00:00:00 2001 From: 2047767028-lang <2047767028@qq.com> Date: Thu, 27 Aug 2026 09:06:22 -0700 Subject: [PATCH 1/3] Keep the CloudXR runtime on the GPU the XR frames are rendered on On a multi-GPU workstation an XR teleop session connects, starts, and streams frames whose contents are garbage -- the headset shows noise. Nothing reports an error: the client connects, `IsaacTeleop session started` is logged, and the CloudXR encoder reports normal per-frame timings. Two independent device selections drift apart: 1. `_ensure_cloudxr_runtime` constructs `CloudXRLauncher` without saying which GPU to use, so the runtime falls back to automatic selection (`gpuIndexVulkan: -1`) and takes the first Vulkan physical device. Vulkan's enumeration is unrelated to the CUDA ordering Isaac Lab selects the simulation and renderer devices with. On the host this was found on, Vulkan index 0 is `nvidia-smi` GPU 2 and Vulkan index 1 is a llvmpipe software device, so the compositor imported swapchain memory from a card that holds no rendered frames. 2. `_resolve_kit_args` only applies `--/renderer/multiGpu/activeCudaGpus` when `launcher_args["multi_gpu"] is False`, and that key is assigned in exactly one place: the `distributed` branch of `_resolve_device_settings`. There is no `--multi_gpu` or `--distributed` CLI argument, and `_sim_app_config` is built by intersecting with the keys actually present, so a plain run never sets it. `--device cuda:1` therefore put physics on GPU 1 while the renderer kept `multiGpu/enabled = True` across every visible GPU -- which also makes "the GPU the frames are rendered on" ill-defined for (1) to match against. Probing carb settings after startup: --device cuda:1 -> /physics/cudaDevice = 1 activeCudaGpus = None --device cuda:1 --kit_args "...=1," -> /physics/cudaDevice = 1 activeCudaGpus = '1,' The comment above `launcher_args["physics_gpu"]` already states that "the renderer device is selected in `_resolve_kit_args`", which is what the gate prevents outside distributed runs. Pin the renderer to the simulation device whenever XR is enabled, and point the CloudXR runtime at that same device through `NV_CXR_GPU_INDEX_CUDA`. The scope is deliberately limited to XR, where a single stereo swapchain must be imported by the compositor; non-XR single-process runs keep their current behaviour, so the trade-off settled in #7057 is untouched. The runtime rejects setting both index variables at once, so an index already present in the environment or in the `--cloudxr_env` profile is left alone. Verified end to end on a 4x RTX 5090 host with Meta Quest 3 over CloudXR.js, Isaac Lab 3.0.0, `IsaacContrib-Stack-Cube-Franka-IK-Abs`, headless (`--visualizer none --xr --device cuda:1`): * before: headset shows noise; `gpuIndexVulkan: -1` and `compositor_set_cuda_device_for_vk` lands on a different card than the renderer * after, with no manual `--kit_args` and no GPU index in the profile: `Pinned the CloudXR runtime to CUDA device 1`, `gpuIndexCuda: 1`, and the compositor logs `Physical device 1 is being used by Vulkan` -- the same physical device the renderer is on. The scene renders correctly and the robot is teleoperable. Signed-off-by: 2047767028-lang <2047767028@qq.com> --- .../pk-xr-multigpu-device-pinning.rst | 9 +++ source/isaaclab/isaaclab/app/app_launcher.py | 5 +- .../pk-xr-multigpu-device-pinning.rst | 10 ++++ .../isaaclab_teleop/session_lifecycle.py | 59 +++++++++++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 source/isaaclab/changelog.d/pk-xr-multigpu-device-pinning.rst create mode 100644 source/isaaclab_teleop/changelog.d/pk-xr-multigpu-device-pinning.rst diff --git a/source/isaaclab/changelog.d/pk-xr-multigpu-device-pinning.rst b/source/isaaclab/changelog.d/pk-xr-multigpu-device-pinning.rst new file mode 100644 index 000000000000..148eb961354b --- /dev/null +++ b/source/isaaclab/changelog.d/pk-xr-multigpu-device-pinning.rst @@ -0,0 +1,9 @@ +Fixed +^^^^^ + +* Fixed the Kit renderer not being restricted to a single GPU under ``--xr``. The + ``--/renderer/multiGpu/activeCudaGpus`` setting was only applied when ``multi_gpu`` was + ``False``, which is set for distributed runs alone, so an XR session left the renderer + spanning every visible GPU while physics ran on the selected device. XR streams a single + stereo swapchain that the CloudXR compositor imports, so the renderer is now pinned to the + simulation device whenever XR is enabled. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index cca32d90a3de..58132fb2c21d 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -1299,7 +1299,10 @@ def _resolve_kit_args(self, launcher_args: dict): self._kit_args.append(argument) # Select the renderer by CUDA index; the trailing comma keeps the setting string-typed. - if launcher_args.get("multi_gpu") is False: + # XR streams a single stereo swapchain that the CloudXR compositor imports, so the + # renderer has to stay on one known device there too -- otherwise the compositor and + # the renderer can end up on different GPUs and the headset receives noise. + if launcher_args.get("multi_gpu") is False or self._xr: argument = f"--/renderer/multiGpu/activeCudaGpus={self.device_id}," setting = argument.partition("=")[0] if not any(arg.partition("=")[0] == setting for arg in sys.argv + self._kit_args): diff --git a/source/isaaclab_teleop/changelog.d/pk-xr-multigpu-device-pinning.rst b/source/isaaclab_teleop/changelog.d/pk-xr-multigpu-device-pinning.rst new file mode 100644 index 000000000000..d86558a696be --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/pk-xr-multigpu-device-pinning.rst @@ -0,0 +1,10 @@ +Fixed +^^^^^ + +* Fixed the XR headset receiving noise instead of the rendered scene on multi-GPU hosts. + The auto-launched CloudXR runtime selected its own device, and because Vulkan's physical + device enumeration is unrelated to the CUDA ordering Isaac Lab picks the simulation and + renderer devices with, the compositor could end up on a different GPU than the one holding + the rendered swapchain. The runtime is now pinned to the renderer's CUDA device via + ``NV_CXR_GPU_INDEX_CUDA``; an index already set in the environment or in the + ``--cloudxr_env`` profile is left untouched. diff --git a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py index c3e285e31502..2f17c435eac5 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py +++ b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py @@ -29,6 +29,35 @@ from .isaac_teleop_cfg import IsaacTeleopCfg from .teleop_message_processor import TeleopMessageProcessor +# The CloudXR runtime accepts at most one of these; setting both is rejected outright. +_CXR_GPU_INDEX_ENV_VARS = ("NV_CXR_GPU_INDEX_CUDA", "NV_CXR_GPU_INDEX_VULKAN") + + +def _env_file_pins_gpu_index(env_file: str | None) -> bool: + """Whether a CloudXR ``.env`` profile already selects a GPU index.""" + if not env_file: + return False + try: + with open(env_file, encoding="utf-8") as handle: + lines = handle.readlines() + except OSError: + return False + return any(line.strip().split("=", 1)[0].strip() in _CXR_GPU_INDEX_ENV_VARS for line in lines) + + +def _renderer_cuda_index() -> int | None: + """CUDA index the Kit renderer is pinned to, or ``None`` when it is not pinned.""" + try: + import carb + except ImportError: + return None + setting = carb.settings.get_settings().get("/renderer/multiGpu/activeCudaGpus") + if not setting: + return None + first = str(setting).split(",")[0].strip() + return int(first) if first.isdigit() else None + + if TYPE_CHECKING: from .haptic_feedback import HapticFeedbackCfg @@ -1310,6 +1339,8 @@ def _ensure_cloudxr_runtime(self) -> None: from isaacteleop.cloudxr import CloudXRLauncher as _CloudXRLauncher + self._pin_cloudxr_to_render_device() + self._cloudxr_launcher = _CloudXRLauncher( install_dir=str(Path.home() / ".cloudxr"), env_config=self._cloudxr_env_file, @@ -1317,6 +1348,34 @@ def _ensure_cloudxr_runtime(self) -> None: ) logger.info("CloudXR runtime auto-launched") + def _pin_cloudxr_to_render_device(self) -> None: + """Point the CloudXR runtime at the GPU the frames are rendered on. + + Left to itself the runtime takes the first Vulkan physical device. That + enumeration is unrelated to the CUDA ordering Isaac Lab selects the + simulation and renderer devices with, so on a multi-GPU host the + compositor routinely lands on a different card than the one holding the + rendered swapchain. Nothing reports an error -- the client connects, the + session starts and the encoder logs normal frame timings -- but the + headset only shows noise. + + An explicit choice, in the process environment or in the + ``--cloudxr_env`` profile, is left untouched. + """ + if any(name in os.environ for name in _CXR_GPU_INDEX_ENV_VARS): + return + if _env_file_pins_gpu_index(self._cloudxr_env_file): + return + + index = _renderer_cuda_index() + if index is None and self._device.type == "cuda": + index = self._device.index + if index is None: + return + + os.environ["NV_CXR_GPU_INDEX_CUDA"] = str(index) + logger.info("Pinned the CloudXR runtime to CUDA device %d", index) + # ------------------------------------------------------------------ # OpenXR handle acquisition # ------------------------------------------------------------------ From c6f7f90eb31d58491e268e43018283f4805d839c Mon Sep 17 00:00:00 2001 From: 2047767028-lang <2047767028@qq.com> Date: Fri, 28 Aug 2026 09:00:29 -0700 Subject: [PATCH 2/3] Leave `--xr` without an explicit device to Kit's own renderer choice Review feedback: pinning the renderer whenever XR is enabled also changed the default `--xr` path. `_resolve_device_settings` resolves a bare `--xr` to `device = "cpu"`, leaving `device_id` at its `0` initialiser, so the renderer would have been pinned to CUDA 0 rather than left to Kit. On a host whose display is not attached to GPU 0 that is a regression: Kit previously followed the same auto-selection the CloudXR runtime used, and the two agreed. Require a CUDA device before pinning. `--device cuda:` with `--xr` still aligns the renderer and the compositor, which is the case this PR set out to fix; a bare `--xr` keeps today's behaviour on both sides, since the teleop side also leaves the runtime alone when the renderer is unpinned and the simulation device is CPU. Signed-off-by: 2047767028-lang <2047767028@qq.com> --- .../changelog.d/pk-xr-multigpu-device-pinning.rst | 14 ++++++++------ source/isaaclab/isaaclab/app/app_launcher.py | 6 ++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/source/isaaclab/changelog.d/pk-xr-multigpu-device-pinning.rst b/source/isaaclab/changelog.d/pk-xr-multigpu-device-pinning.rst index 148eb961354b..293ff0ce4953 100644 --- a/source/isaaclab/changelog.d/pk-xr-multigpu-device-pinning.rst +++ b/source/isaaclab/changelog.d/pk-xr-multigpu-device-pinning.rst @@ -1,9 +1,11 @@ Fixed ^^^^^ -* Fixed the Kit renderer not being restricted to a single GPU under ``--xr``. The - ``--/renderer/multiGpu/activeCudaGpus`` setting was only applied when ``multi_gpu`` was - ``False``, which is set for distributed runs alone, so an XR session left the renderer - spanning every visible GPU while physics ran on the selected device. XR streams a single - stereo swapchain that the CloudXR compositor imports, so the renderer is now pinned to the - simulation device whenever XR is enabled. +* Fixed the Kit renderer not being restricted to a single GPU under ``--xr`` when a CUDA + device is selected. The ``--/renderer/multiGpu/activeCudaGpus`` setting was only applied + when ``multi_gpu`` was ``False``, which is set for distributed runs alone, so an XR session + started with ``--device cuda:`` left the renderer spanning every visible GPU while + physics ran on the selected device. XR streams a single stereo swapchain that the CloudXR + compositor imports, so the renderer is now pinned to the simulation device in that case. + ``--xr`` without an explicit device still resolves to ``cpu`` and leaves the renderer + selection to Kit, as before. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 58132fb2c21d..711036669b75 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -1301,8 +1301,10 @@ def _resolve_kit_args(self, launcher_args: dict): # Select the renderer by CUDA index; the trailing comma keeps the setting string-typed. # XR streams a single stereo swapchain that the CloudXR compositor imports, so the # renderer has to stay on one known device there too -- otherwise the compositor and - # the renderer can end up on different GPUs and the headset receives noise. - if launcher_args.get("multi_gpu") is False or self._xr: + # the renderer can end up on different GPUs and the headset receives noise. This only + # applies once a CUDA device has actually been selected: ``--xr`` on its own resolves + # to ``cpu``, where there is no simulation GPU to align to, so Kit's own choice stands. + if launcher_args.get("multi_gpu") is False or (self._xr and "cuda" in self.device): argument = f"--/renderer/multiGpu/activeCudaGpus={self.device_id}," setting = argument.partition("=")[0] if not any(arg.partition("=")[0] == setting for arg in sys.argv + self._kit_args): From 21f2d440c35a6646dba2d44c43345f74ced469d9 Mon Sep 17 00:00:00 2001 From: 2047767028-lang <2047767028@qq.com> Date: Fri, 28 Aug 2026 23:03:01 -0700 Subject: [PATCH 3/3] Give the argv tests the launcher state `_resolve_kit_args` now reads CI caught two failures in `test_app_launcher_argv.py`: AttributeError: 'AppLauncher' object has no attribute 'device' AttributeError: 'AppLauncher' object has no attribute '_xr' Both tests build a partial launcher with `AppLauncher.__new__` and set only the attributes `_resolve_kit_args` needed before this PR. Gating the renderer pin on XR added two more reads, so the spectator-view tests now set `device` (and `_xr`, which one of them never set) alongside the state they already stub. Also covers the new behaviour directly. `_resolve_devices_and_kit_args` takes an `xr` flag, and `test_xr_pins_the_renderer_only_for_a_cuda_device` asserts both halves of the gate: `--xr --device cuda:1` emits `--/renderer/multiGpu/activeCudaGpus=1,` with `physics_gpu = 1`, while a bare `--xr` resolves to CPU and emits nothing, which is the case raised in review. Reverting the `app_launcher.py` change fails the first parameter set, so the test pins the behaviour rather than passing vacuously. 17 passed locally, up from 15 with 2 failing. Signed-off-by: 2047767028-lang <2047767028@qq.com> --- .../test/app/test_app_launcher_argv.py | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/source/isaaclab/test/app/test_app_launcher_argv.py b/source/isaaclab/test/app/test_app_launcher_argv.py index 21c0823a6d37..0f82f903e567 100644 --- a/source/isaaclab/test/app/test_app_launcher_argv.py +++ b/source/isaaclab/test/app/test_app_launcher_argv.py @@ -41,7 +41,7 @@ def test_sanitize_sys_argv_removes_pytest_marker_pair(monkeypatch): assert result == ["test_script.py", "--keep"] -def _resolve_devices_and_kit_args(launcher_args: dict, monkeypatch) -> tuple[dict, list[str]]: +def _resolve_devices_and_kit_args(launcher_args: dict, monkeypatch, *, xr: bool = False) -> tuple[dict, list[str]]: """Resolve device settings and Kit arguments without constructing an ``AppLauncher``. ``_resolve_kit_args`` extends ``sys.argv``, so the caller's argv is isolated. @@ -50,7 +50,7 @@ def _resolve_devices_and_kit_args(launcher_args: dict, monkeypatch) -> tuple[dic launcher = AppLauncher.__new__(AppLauncher) launcher.device_id = 0 launcher._deferred_cuda_device_id = None - launcher._xr = False + launcher._xr = xr AppLauncher._resolve_device_settings(launcher, launcher_args) AppLauncher._resolve_kit_args(launcher, launcher_args) return launcher_args, launcher._kit_args @@ -82,6 +82,35 @@ def test_devices_selected_by_cuda_index(launcher_args, expected_renderer_args, m assert "active_gpu" not in args +@pytest.mark.parametrize( + ("launcher_args", "expected_renderer_args", "expected_physics_gpu"), + [ + pytest.param( + {"device": "cuda:1", "device_explicit": True}, + ["--/renderer/multiGpu/activeCudaGpus=1,"], + 1, + id="xr-explicit-cuda-device", + ), + pytest.param({}, [], 0, id="xr-default-cpu-device"), + ], +) +def test_xr_pins_the_renderer_only_for_a_cuda_device( + launcher_args, expected_renderer_args, expected_physics_gpu, monkeypatch +): + """Pin the renderer under XR when a CUDA device is selected, and only then. + + XR streams one stereo swapchain that the CloudXR compositor imports, so the renderer and + the compositor have to agree on a device. A bare ``--xr`` resolves to ``cpu``, where there + is no simulation GPU to align to, so Kit keeps its own choice -- forcing CUDA 0 there would + break hosts whose display is not attached to GPU 0. + """ + args, kit_args = _resolve_devices_and_kit_args(launcher_args, monkeypatch, xr=True) + + renderer_args = [arg for arg in kit_args if arg.startswith("--/renderer/multiGpu/activeCudaGpus=")] + assert renderer_args == expected_renderer_args + assert args["physics_gpu"] == expected_physics_gpu + + @pytest.mark.parametrize( ("launcher_state", "expected_enabled"), [ @@ -118,6 +147,7 @@ def test_spectator_view_follows_visual_output_intent(launcher_state, expected_en launcher._video_enabled = False launcher._livestream = 0 launcher._xr = False + launcher.device = "cpu" for name, value in launcher_state.items(): setattr(launcher, name, value) @@ -133,6 +163,8 @@ def test_explicit_spectator_setting_overrides_visualizer_default(monkeypatch): launcher = AppLauncher.__new__(AppLauncher) launcher._cli_visualizer_explicit = True launcher._cli_visualizer_types = ["kit"] + launcher._xr = False + launcher.device = "cpu" explicit_arg = f"--{ISAAC_RTX_SHOW_ALL_PARTITIONS_BY_DEFAULT_SETTING}=false" launcher._resolve_kit_args({"kit_args": explicit_arg})