diff --git a/source/isaaclab/changelog.d/pk-xr-multigpu-device-pinning.rst b/source/isaaclab/changelog.d/pk-xr-multigpu-device-pinning.rst new file mode 100644 index 000000000000..293ff0ce4953 --- /dev/null +++ b/source/isaaclab/changelog.d/pk-xr-multigpu-device-pinning.rst @@ -0,0 +1,11 @@ +Fixed +^^^^^ + +* Fixed the Kit renderer not being restricted to a single GPU under ``--xr`` when a CUDA + device is selected. The ``--/renderer/multiGpu/activeCudaGpus`` setting was only applied + when ``multi_gpu`` was ``False``, which is set for distributed runs alone, so an XR session + started with ``--device cuda:`` left the renderer spanning every visible GPU while + physics ran on the selected device. XR streams a single stereo swapchain that the CloudXR + compositor imports, so the renderer is now pinned to the simulation device in that case. + ``--xr`` without an explicit device still resolves to ``cpu`` and leaves the renderer + selection to Kit, as before. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index cca32d90a3de..711036669b75 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -1299,7 +1299,12 @@ def _resolve_kit_args(self, launcher_args: dict): self._kit_args.append(argument) # Select the renderer by CUDA index; the trailing comma keeps the setting string-typed. - if launcher_args.get("multi_gpu") is False: + # XR streams a single stereo swapchain that the CloudXR compositor imports, so the + # renderer has to stay on one known device there too -- otherwise the compositor and + # the renderer can end up on different GPUs and the headset receives noise. This only + # applies once a CUDA device has actually been selected: ``--xr`` on its own resolves + # to ``cpu``, where there is no simulation GPU to align to, so Kit's own choice stands. + if launcher_args.get("multi_gpu") is False or (self._xr and "cuda" in self.device): argument = f"--/renderer/multiGpu/activeCudaGpus={self.device_id}," setting = argument.partition("=")[0] if not any(arg.partition("=")[0] == setting for arg in sys.argv + self._kit_args): diff --git a/source/isaaclab/test/app/test_app_launcher_argv.py b/source/isaaclab/test/app/test_app_launcher_argv.py index 21c0823a6d37..0f82f903e567 100644 --- a/source/isaaclab/test/app/test_app_launcher_argv.py +++ b/source/isaaclab/test/app/test_app_launcher_argv.py @@ -41,7 +41,7 @@ def test_sanitize_sys_argv_removes_pytest_marker_pair(monkeypatch): assert result == ["test_script.py", "--keep"] -def _resolve_devices_and_kit_args(launcher_args: dict, monkeypatch) -> tuple[dict, list[str]]: +def _resolve_devices_and_kit_args(launcher_args: dict, monkeypatch, *, xr: bool = False) -> tuple[dict, list[str]]: """Resolve device settings and Kit arguments without constructing an ``AppLauncher``. ``_resolve_kit_args`` extends ``sys.argv``, so the caller's argv is isolated. @@ -50,7 +50,7 @@ def _resolve_devices_and_kit_args(launcher_args: dict, monkeypatch) -> tuple[dic launcher = AppLauncher.__new__(AppLauncher) launcher.device_id = 0 launcher._deferred_cuda_device_id = None - launcher._xr = False + launcher._xr = xr AppLauncher._resolve_device_settings(launcher, launcher_args) AppLauncher._resolve_kit_args(launcher, launcher_args) return launcher_args, launcher._kit_args @@ -82,6 +82,35 @@ def test_devices_selected_by_cuda_index(launcher_args, expected_renderer_args, m assert "active_gpu" not in args +@pytest.mark.parametrize( + ("launcher_args", "expected_renderer_args", "expected_physics_gpu"), + [ + pytest.param( + {"device": "cuda:1", "device_explicit": True}, + ["--/renderer/multiGpu/activeCudaGpus=1,"], + 1, + id="xr-explicit-cuda-device", + ), + pytest.param({}, [], 0, id="xr-default-cpu-device"), + ], +) +def test_xr_pins_the_renderer_only_for_a_cuda_device( + launcher_args, expected_renderer_args, expected_physics_gpu, monkeypatch +): + """Pin the renderer under XR when a CUDA device is selected, and only then. + + XR streams one stereo swapchain that the CloudXR compositor imports, so the renderer and + the compositor have to agree on a device. A bare ``--xr`` resolves to ``cpu``, where there + is no simulation GPU to align to, so Kit keeps its own choice -- forcing CUDA 0 there would + break hosts whose display is not attached to GPU 0. + """ + args, kit_args = _resolve_devices_and_kit_args(launcher_args, monkeypatch, xr=True) + + renderer_args = [arg for arg in kit_args if arg.startswith("--/renderer/multiGpu/activeCudaGpus=")] + assert renderer_args == expected_renderer_args + assert args["physics_gpu"] == expected_physics_gpu + + @pytest.mark.parametrize( ("launcher_state", "expected_enabled"), [ @@ -118,6 +147,7 @@ def test_spectator_view_follows_visual_output_intent(launcher_state, expected_en launcher._video_enabled = False launcher._livestream = 0 launcher._xr = False + launcher.device = "cpu" for name, value in launcher_state.items(): setattr(launcher, name, value) @@ -133,6 +163,8 @@ def test_explicit_spectator_setting_overrides_visualizer_default(monkeypatch): launcher = AppLauncher.__new__(AppLauncher) launcher._cli_visualizer_explicit = True launcher._cli_visualizer_types = ["kit"] + launcher._xr = False + launcher.device = "cpu" explicit_arg = f"--{ISAAC_RTX_SHOW_ALL_PARTITIONS_BY_DEFAULT_SETTING}=false" launcher._resolve_kit_args({"kit_args": explicit_arg}) diff --git a/source/isaaclab_teleop/changelog.d/pk-xr-multigpu-device-pinning.rst b/source/isaaclab_teleop/changelog.d/pk-xr-multigpu-device-pinning.rst new file mode 100644 index 000000000000..d86558a696be --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/pk-xr-multigpu-device-pinning.rst @@ -0,0 +1,10 @@ +Fixed +^^^^^ + +* Fixed the XR headset receiving noise instead of the rendered scene on multi-GPU hosts. + The auto-launched CloudXR runtime selected its own device, and because Vulkan's physical + device enumeration is unrelated to the CUDA ordering Isaac Lab picks the simulation and + renderer devices with, the compositor could end up on a different GPU than the one holding + the rendered swapchain. The runtime is now pinned to the renderer's CUDA device via + ``NV_CXR_GPU_INDEX_CUDA``; an index already set in the environment or in the + ``--cloudxr_env`` profile is left untouched. diff --git a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py index c3e285e31502..2f17c435eac5 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py +++ b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py @@ -29,6 +29,35 @@ from .isaac_teleop_cfg import IsaacTeleopCfg from .teleop_message_processor import TeleopMessageProcessor +# The CloudXR runtime accepts at most one of these; setting both is rejected outright. +_CXR_GPU_INDEX_ENV_VARS = ("NV_CXR_GPU_INDEX_CUDA", "NV_CXR_GPU_INDEX_VULKAN") + + +def _env_file_pins_gpu_index(env_file: str | None) -> bool: + """Whether a CloudXR ``.env`` profile already selects a GPU index.""" + if not env_file: + return False + try: + with open(env_file, encoding="utf-8") as handle: + lines = handle.readlines() + except OSError: + return False + return any(line.strip().split("=", 1)[0].strip() in _CXR_GPU_INDEX_ENV_VARS for line in lines) + + +def _renderer_cuda_index() -> int | None: + """CUDA index the Kit renderer is pinned to, or ``None`` when it is not pinned.""" + try: + import carb + except ImportError: + return None + setting = carb.settings.get_settings().get("/renderer/multiGpu/activeCudaGpus") + if not setting: + return None + first = str(setting).split(",")[0].strip() + return int(first) if first.isdigit() else None + + if TYPE_CHECKING: from .haptic_feedback import HapticFeedbackCfg @@ -1310,6 +1339,8 @@ def _ensure_cloudxr_runtime(self) -> None: from isaacteleop.cloudxr import CloudXRLauncher as _CloudXRLauncher + self._pin_cloudxr_to_render_device() + self._cloudxr_launcher = _CloudXRLauncher( install_dir=str(Path.home() / ".cloudxr"), env_config=self._cloudxr_env_file, @@ -1317,6 +1348,34 @@ def _ensure_cloudxr_runtime(self) -> None: ) logger.info("CloudXR runtime auto-launched") + def _pin_cloudxr_to_render_device(self) -> None: + """Point the CloudXR runtime at the GPU the frames are rendered on. + + Left to itself the runtime takes the first Vulkan physical device. That + enumeration is unrelated to the CUDA ordering Isaac Lab selects the + simulation and renderer devices with, so on a multi-GPU host the + compositor routinely lands on a different card than the one holding the + rendered swapchain. Nothing reports an error -- the client connects, the + session starts and the encoder logs normal frame timings -- but the + headset only shows noise. + + An explicit choice, in the process environment or in the + ``--cloudxr_env`` profile, is left untouched. + """ + if any(name in os.environ for name in _CXR_GPU_INDEX_ENV_VARS): + return + if _env_file_pins_gpu_index(self._cloudxr_env_file): + return + + index = _renderer_cuda_index() + if index is None and self._device.type == "cuda": + index = self._device.index + if index is None: + return + + os.environ["NV_CXR_GPU_INDEX_CUDA"] = str(index) + logger.info("Pinned the CloudXR runtime to CUDA device %d", index) + # ------------------------------------------------------------------ # OpenXR handle acquisition # ------------------------------------------------------------------