From 72ec718a16924dab4548bba56b70b5d3bb9a0eae Mon Sep 17 00:00:00 2001 From: Jessica Martinez Date: Fri, 7 Aug 2026 17:25:13 -0500 Subject: [PATCH] OMPE-103001: Render and animate Newton cables on all three renderers Cable curve points are now computed on device from the Newton segment bodies and written GPU/ASYNC each frame, so cables follow their simulated pose instead of drawing at their spawn pose and never moving. Drops the CPU mirror in sync_cables_to_usd, which copied the whole model's body_q device-to-host on every dirty render frame. Adds the first rendering coverage for cables, gated on motion rather than presence, and fixes three defects found alongside: the view_count / camera-prim mismatch, the Isaac RTX sensor pump refreshing transforms only, and the Fabric stage being resolved exactly once at startup. Requires Kit !47946, !48359 and !48511, and OVRTX >= 0.4.1. The rendering cells skip below those floors rather than fail. --- scripts/demos/cables.py | 139 ++++++++- .../jmart-cable-render-binding.rst | 10 + .../isaaclab/renderers/camera_render_spec.py | 13 + .../isaaclab/sensors/camera/camera.py | 6 +- .../isaaclab/test/utils/cable_rendering.py | 201 +++++++++++++ .../test/renderers/test_cable_rendering.py | 204 +++++++++++++ .../jmart-cable-render-binding.minor.rst | 31 ++ .../isaaclab_newton/physics/newton_manager.py | 154 ++++++++-- .../jmart-cable-render-binding.minor.rst | 10 + .../isaaclab_ov/renderers/ovrtx_renderer.py | 275 ++++++++++++++++++ .../renderers/ovrtx_renderer_kernels.py | 44 +++ .../test/test_ovrtx_deformable_bindings.py | 109 +++++++ .../test/test_ovrtx_renderer_contract.py | 11 + .../jmart-cable-render-binding.rst | 11 + .../renderers/isaac_rtx_renderer.py | 14 + .../renderers/isaac_rtx_renderer_utils.py | 8 +- .../test_cable_rendering_isaac_rtx.py | 183 ++++++++++++ 17 files changed, 1393 insertions(+), 30 deletions(-) create mode 100644 source/isaaclab/changelog.d/jmart-cable-render-binding.rst create mode 100644 source/isaaclab/isaaclab/test/utils/cable_rendering.py create mode 100644 source/isaaclab/test/renderers/test_cable_rendering.py create mode 100644 source/isaaclab_newton/changelog.d/jmart-cable-render-binding.minor.rst create mode 100644 source/isaaclab_ov/changelog.d/jmart-cable-render-binding.minor.rst create mode 100644 source/isaaclab_physx/changelog.d/jmart-cable-render-binding.rst create mode 100644 source/isaaclab_physx/test/renderers/test_cable_rendering_isaac_rtx.py diff --git a/scripts/demos/cables.py b/scripts/demos/cables.py index 6153e3b40e12..6c1c6d1d1907 100644 --- a/scripts/demos/cables.py +++ b/scripts/demos/cables.py @@ -16,12 +16,17 @@ # Usage without a visualizer and with a larger cable pile. uv run python scripts/demos/cables.py --visualizer none --num_cables 40 --num_segments 15 + # Usage with a rendering camera, to observe cable rendering on a given renderer. + uv run python scripts/demos/cables.py --visualizer none --camera ovrtx + uv run python scripts/demos/cables.py --visualizer none --camera newton_warp + """ from __future__ import annotations import argparse import math +import os import random from isaaclab.app import add_launcher_args, launch_simulation @@ -30,6 +35,25 @@ parser.add_argument("--num_cables", type=int, default=25, help="Number of cables to spawn.") parser.add_argument("--num_segments", type=int, default=20, help="Number of segments per cable.") parser.add_argument("--max_steps", type=int, default=-1, help="Stop after this many steps; negative runs forever.") +parser.add_argument( + "--seed", + type=int, + default=None, + help="Seed the cable spawn RNG so two runs are comparable frame-for-frame.", +) +parser.add_argument( + "--camera", + nargs="?", + const="ovrtx", + choices=["ovrtx", "newton_warp", "isaac_rtx"], + default=None, + help="Attach a camera that renders the cable pile with the given renderer.", +) +parser.add_argument( + "--frames_dir", + default=None, + help="Write each rendered frame as a PNG into this directory, for assembly into an animation.", +) parser.add_argument("--physics", default="newton_vbd", choices=["newton_vbd"], help="Physics backend.") add_launcher_args(parser) parser.set_defaults(visualizer=["kit"]) @@ -40,11 +64,50 @@ if args_cli.num_segments < 2: parser.error("--num_segments must be at least 2.") -import isaaclab.sim as sim_utils -from isaaclab.assets import CableObject, CableObjectCfg +if args_cli.camera == "isaac_rtx": + # ``launch_simulation`` decides Kit-vs-kitless and camera support by scanning the cfg it is + # handed, but this demo builds its camera after that call, so the scan sees neither. Both flags + # are needed and both are additive. + args_cli.require_kit = True + args_cli.enable_cameras = True + from isaaclab.physics import PhysicsCfg +def _load_scene_modules() -> None: + """Import the USD-backed modules, deferred until after :func:`launch_simulation` has opened. + + ``--camera isaac_rtx`` brings up Kit, which loads its own USD; importing the environment's copy + first makes the two collide and the process dies inside ``libusd_tf`` during startup. Safe + because ``from __future__ import annotations`` leaves the annotations below unevaluated. + """ + global torch, sim_utils, CableObject, CableObjectCfg, RendererCfg, Camera, CameraCfg # noqa: PLW0603 + + import torch + + import isaaclab.sim as sim_utils + from isaaclab.assets import CableObject, CableObjectCfg + from isaaclab.renderers import RendererCfg + from isaaclab.sensors import Camera, CameraCfg + + +def _renderer_cfg(renderer: str) -> RendererCfg: + """Import the requested renderer's config lazily, so the demo does not require all of them.""" + if renderer == "ovrtx": + from isaaclab_ov.renderers import OVRTXRendererCfg + + return OVRTXRendererCfg() + if renderer == "newton_warp": + from isaaclab_newton.renderers.newton_warp_renderer_cfg import NewtonWarpRendererCfg + + return NewtonWarpRendererCfg() + if renderer == "isaac_rtx": + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + return IsaacRtxRendererCfg() + raise ValueError(f"Unknown renderer: {renderer}") + + def design_scene(num_cables: int, num_segments: int, colorize: bool) -> dict[str, CableObject]: """Spawn a ground plane, light, and randomly oriented cable pile. @@ -71,6 +134,9 @@ def design_scene(num_cables: int, num_segments: int, colorize: bool) -> dict[str z_base = 0.8 positions = [(index * segment_length, 0.0, 0.0) for index in range(num_segments + 1)] + if args_cli.seed is not None: + random.seed(args_cli.seed) + print(f"[INFO]: Spawning {num_cables} cables...") entities: dict[str, CableObject] = {} for index in range(num_cables): @@ -87,7 +153,7 @@ def design_scene(num_cables: int, num_segments: int, colorize: bool) -> dict[str diffuse_color=(random.random(), random.random(), random.random()) ) cfg = CableObjectCfg( - prim_path=f"/World/Env_0/Cable{index:03d}", + prim_path=f"/World/envs/env_0/Cable{index:03d}", spawn=sim_utils.CableCfg( positions=positions, visual_material=visual_material, @@ -113,11 +179,68 @@ def reset_cables(entities: dict[str, CableObject]) -> None: cable.write_segment_velocity_to_sim_index(segment_velocity=cable.data.default_segment_velocity_w) -def run_simulator(sim: sim_utils.SimulationContext, entities: dict[str, CableObject], max_steps: int = -1) -> None: +# Framed to keep the whole fall in view: cables spawn near z=0.8 and settle at z~0. +_CAM_EYE = (3.0, 3.0, 1.6) +_CAM_TARGET = (0.0, 0.0, 0.35) + + +def _look_at_quat(eye: tuple[float, float, float], target: tuple[float, float, float]): + """Camera orientation looking from ``eye`` to ``target``, as a world-convention ``(w, x, y, z)``.""" + from isaaclab.utils.math import create_rotation_matrix_from_view, quat_from_matrix + + matrix = create_rotation_matrix_from_view( + torch.tensor([eye], dtype=torch.float32), + torch.tensor([target], dtype=torch.float32), + "Z", + device="cpu", + ) + return tuple(float(value) for value in quat_from_matrix(matrix)[0].tolist()) + + +def make_camera(renderer: str) -> Camera: + """Spawn a camera aimed at the cable pile, using the requested renderer. + + The prim must sit under the standard env namespace: scene-partition primvars are authored by + walking that subtree, so a prim outside it inherits no partition and renders nothing. The pose + is baked into the cfg because :meth:`~isaaclab.sensors.Camera.set_world_poses_from_view` does + not reach the render — the camera stays at the origin and the pile falls out of frame. + """ + return Camera( + CameraCfg( + prim_path="/World/envs/env_0/CableCam", + width=640, + height=480, + data_types=["rgb"], + spawn=sim_utils.PinholeCameraCfg(), + offset=CameraCfg.OffsetCfg(pos=_CAM_EYE, rot=_look_at_quat(_CAM_EYE, _CAM_TARGET), convention="opengl"), + renderer_cfg=_renderer_cfg(renderer), + ) + ) + + +def save_frame(camera: Camera, frames_dir: str, index: int) -> None: + """Write one RGB frame as a zero-padded PNG, so a run can be assembled into an animation.""" + from PIL import Image + + rgb = camera.data.output["rgb"] + tensor = rgb.torch if hasattr(rgb, "torch") else rgb + image = tensor[0, ..., :3].detach().to("cpu", torch.uint8).numpy() + Image.fromarray(image).save(os.path.join(frames_dir, f"frame_{index:05d}.png")) + + +def run_simulator( + sim: sim_utils.SimulationContext, + entities: dict[str, CableObject], + max_steps: int = -1, + camera: Camera | None = None, + frames_dir: str | None = None, +) -> None: """Run the simulation and periodically restore the cable pile.""" sim_dt = sim.get_physics_dt() reset_steps = max(1, int(2.0 / sim_dt)) count = 0 + if frames_dir is not None: + os.makedirs(frames_dir, exist_ok=True) while (max_steps < 0 or count < max_steps) and sim.is_headless_or_exist_active_visualizer(): if count > 0 and count % reset_steps == 0: @@ -129,12 +252,17 @@ def run_simulator(sim: sim_utils.SimulationContext, entities: dict[str, CableObj cable.update(sim_dt) if sim.is_rendering: sim.render() + if camera is not None: + camera.update(sim_dt) + if frames_dir is not None: + save_frame(camera, frames_dir, count) count += 1 def main() -> None: """Launch and run the cable pile demo.""" with launch_simulation(cfg=PhysicsCfg(), launcher_args=args_cli) as physics_cfg: + _load_scene_modules() physics_cfg.solver_cfg.iterations = 20 physics_cfg.num_substeps = 8 sim_cfg = sim_utils.SimulationCfg(dt=0.01, device=args_cli.device, physics=physics_cfg) @@ -142,9 +270,10 @@ def main() -> None: sim.set_camera_view(eye=(2.0, 2.0, 1.0), target=(0.0, 0.0, 0.25)) colorize = bool(args_cli.visualizer and "kit" in args_cli.visualizer) entities = design_scene(args_cli.num_cables, args_cli.num_segments, colorize) + camera = make_camera(args_cli.camera) if args_cli.camera else None sim.reset() print("[INFO]: Setup complete...") - run_simulator(sim, entities, args_cli.max_steps) + run_simulator(sim, entities, args_cli.max_steps, camera, args_cli.frames_dir) if __name__ == "__main__": diff --git a/source/isaaclab/changelog.d/jmart-cable-render-binding.rst b/source/isaaclab/changelog.d/jmart-cable-render-binding.rst new file mode 100644 index 000000000000..b09609f6bfb9 --- /dev/null +++ b/source/isaaclab/changelog.d/jmart-cable-render-binding.rst @@ -0,0 +1,10 @@ +Fixed +^^^^^ + +* Fixed :class:`~isaaclab.renderers.camera_render_spec.CameraRenderSpec` accepting a ``view_count`` + that disagrees with ``camera_prim_paths``. A single fixed camera framing several environments + produces one camera prim while the sensor knows about N environments; the tiled reshape was then + launched over N tiles against a render product built from one camera, reading past the end of the + annotator buffer. That surfaced asynchronously as an illegal memory access inside an unrelated + device free, a long way from its cause. The spec now rejects the mismatch where it is introduced, + and :class:`~isaaclab.sensors.Camera` sizes ``view_count`` from the camera prims it found. diff --git a/source/isaaclab/isaaclab/renderers/camera_render_spec.py b/source/isaaclab/isaaclab/renderers/camera_render_spec.py index 526b90121713..9928d8e72d15 100644 --- a/source/isaaclab/isaaclab/renderers/camera_render_spec.py +++ b/source/isaaclab/isaaclab/renderers/camera_render_spec.py @@ -35,3 +35,16 @@ class CameraRenderSpec: camera_prim_paths: tuple[str, ...] view_count: int camera_path_relative_to_env_0: str + + def __post_init__(self) -> None: + """Enforce the ``view_count`` / ``camera_prim_paths`` agreement documented above. + + Backends size the render product from :attr:`camera_prim_paths` but launch their tiled + kernels over :attr:`view_count`, so a mismatch reads past the end of the annotator buffer + and surfaces asynchronously, far from its cause. + """ + if self.camera_prim_paths and self.view_count != len(self.camera_prim_paths): + raise ValueError( + f"view_count ({self.view_count}) must match the number of camera prims" + f" ({len(self.camera_prim_paths)}): {list(self.camera_prim_paths)}." + ) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 2484cffe00d2..f1b8a9367e04 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -530,7 +530,11 @@ def _initialize_impl(self): device=device_str, num_instances=self._num_envs, camera_prim_paths=cam_paths, - view_count=self._num_envs, + # The number of camera PRIMS, which is not the environment count when a single fixed + # camera frames several environments. The tiled reshape is launched over ``view_count`` + # tiles against a render product built from ``camera_prim_paths``, so a mismatch reads + # past the end of the annotator buffer. + view_count=len(cam_paths) if cam_paths else self._num_envs, camera_path_relative_to_env_0=rel_under_env0, ) diff --git a/source/isaaclab/isaaclab/test/utils/cable_rendering.py b/source/isaaclab/isaaclab/test/utils/cable_rendering.py new file mode 100644 index 000000000000..cdc1e38fbe13 --- /dev/null +++ b/source/isaaclab/isaaclab/test/utils/cable_rendering.py @@ -0,0 +1,201 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared harness for the cable-rendering coverage. + +Lives in the library rather than beside the tests because the cells are split across two modules +that cannot import one another: the kit-less renderers need a process where Kit was never started, +and Isaac RTX one where Kit started *before* anything imported USD. Both need the same scene, +framing and metrics, or the two halves stop being comparable. + +A cable frozen at its spawn pose produces a perfectly stable, plausible pixel count, so presence +alone cannot gate this feature; and a cull leaving a few stray pixels still yields a centroid that +can drift far enough to satisfy a displacement check. Tracking is therefore asserted on centroid +displacement *and* retention, over a segmentation that refuses to measure a frame it cannot segment. +""" + +from __future__ import annotations + +import torch + +# Fraction of the frame above which a "geometry" mask is not geometry but a mis-measurement. A cable +# occupies well under 1% of the frame; anything approaching totality means the mask has latched onto +# the backdrop, and every figure derived from it is meaningless. +MAX_PLAUSIBLE_LIT_FRACTION = 0.5 +# A frozen cable still jitters by a pixel or two under resampling; real motion moves the centroid far +# further. Measured: frozen ~0.5-3 px over 140 steps, tracking >15 px. +MIN_CENTROID_SHIFT_PX = 8.0 +# Measured mid-fall: 30 steps of free fall moves the cable well past the threshold above while +# keeping it inside the view, so a genuine miss cannot be confused with the cable leaving the frame. +TRACK_STEPS = 30 +SPAWN_Z = 0.8 +# Framed to keep the whole fall in view: the cable spawns near z=0.8 and settles at z~0. +EYE = (2.2, 2.2, 1.3) +TARGET = (0.0, 0.0, 0.35) +# OVRTX rejects camera prims outside the standard env namespace outright. Geometry must live there +# too: scene-partition primvars are authored by walking that subtree, so a prim outside it inherits +# no partition and the camera cannot see it. +ENV_NS = "/World/envs/env_0" + + +def look_at_quat(eye: tuple[float, float, float], target: tuple[float, float, float]): + """Camera orientation looking from ``eye`` to ``target``, as a world-convention ``(w, x, y, z)``. + + Baked into the camera cfg as an offset rather than applied afterwards with + ``Camera.set_world_poses_from_view``, which does not stick: the camera stays at the origin and + the cable falls out of frame, which is indistinguishable from the defect these cells catch. + """ + from isaaclab.utils.math import create_rotation_matrix_from_view, quat_from_matrix + + matrix = create_rotation_matrix_from_view( + torch.tensor([eye], dtype=torch.float32), + torch.tensor([target], dtype=torch.float32), + "Z", + device="cpu", + ) + return tuple(float(value) for value in quat_from_matrix(matrix)[0].tolist()) + + +def lit_mask(rgb, threshold: int = 12) -> torch.Tensor: + """Boolean HxW mask of the pixels that differ from the frame's own background. + + Segments against the **modal** luminance, not an absolute threshold: a renderer with a bright + backdrop saturates an absolute mask to the whole frame, pinning the centroid at the image + centre, which reads exactly like a frozen render. + """ + tensor = rgb.torch if hasattr(rgb, "torch") else rgb + lum = tensor[..., :3].float().mean(dim=-1) + while lum.dim() > 2: + lum = lum[0] + # The backdrop is whatever luminance dominates the frame. 64 bins is coarse enough to absorb + # gradient and sampling noise into one bin, fine enough to leave the cable outside it. + histogram = torch.histc(lum, bins=64, min=0.0, max=255.0) + background = (int(torch.argmax(histogram).item()) + 0.5) * (255.0 / 64.0) + return (lum - background).abs().gt(threshold) + + +def lit_pixel_count(rgb, threshold: int = 12) -> int: + """How much cable is on screen. Companion to :func:`geometry_centroid`, which says *where*.""" + return int(lit_mask(rgb, threshold).sum().item()) + + +def geometry_centroid(rgb, threshold: int = 12) -> tuple[float, float] | None: + """Centroid of the lit pixels, or ``None`` when nothing is lit. + + Motion is asserted on the centroid rather than on a count of changed pixels: a frozen cable + still re-rasterizes with sampling jitter, which registers thousands of "changed" pixels at a + tiny per-pixel delta. The centroid does not move under jitter. + """ + mask = lit_mask(rgb, threshold) + if not bool(mask.any()): + return None + ys, xs = torch.nonzero(mask, as_tuple=True) + return float(ys.float().mean().item()), float(xs.float().mean().item()) + + +def assert_mask_is_measuring_geometry(mask: torch.Tensor, renderer: str) -> None: + """Fail loudly when the mask has latched onto the backdrop instead of the cable.""" + fraction = float(mask.float().mean().item()) + assert fraction < MAX_PLAUSIBLE_LIT_FRACTION, ( + f"{renderer}: {100 * fraction:.0f}% of the frame is marked as geometry, so the background" + " segmentation has failed and no centroid or retention figure from this frame means" + " anything. This is a broken measurement, not a renderer result." + ) + + +def cable_cfg(prim_path: str | None = None): + """The cable under test: eleven control points, spawned high enough to fall a measurable way.""" + from isaaclab.assets import CableObjectCfg + from isaaclab.sim import UsdPhysicsCollisionCfg + from isaaclab.sim.spawners.materials import CableMaterialCfg + from isaaclab.sim.spawners.shapes import CableCfg + + return CableObjectCfg( + prim_path=prim_path or f"{ENV_NS}/Cable", + spawn=CableCfg( + positions=[(0.05 * index, 0.0, 0.0) for index in range(11)], + physics_material=CableMaterialCfg( + thickness=0.01, + density=100.0, + stretch_stiffness=3.18309886e8, + bend_stiffness=2.03718327e9, + ), + collision_props=[UsdPhysicsCollisionCfg(collision_enabled=True)], + ), + init_state=CableObjectCfg.InitialStateCfg(pos=(0.0, 0.0, SPAWN_Z)), + ) + + +def camera_cfg(renderer_cfg, prim_path: str | None = None, eye=EYE, target=TARGET): + """A camera framed on the cable, with the pose baked into the cfg. See :func:`look_at_quat`.""" + import isaaclab.sim as sim_utils + from isaaclab.sensors import CameraCfg + + return CameraCfg( + prim_path=prim_path or f"{ENV_NS}/Camera", + width=320, + height=240, + data_types=["rgb"], + spawn=sim_utils.PinholeCameraCfg(), + offset=CameraCfg.OffsetCfg(pos=eye, rot=look_at_quat(eye, target), convention="opengl"), + renderer_cfg=renderer_cfg, + ) + + +def spawn_light() -> None: + """A dome light, and deliberately no ground plane. + + A ground plane fills ~18k pixels of a 320x240 frame against a few hundred for the cable, so it + dominates the mask and pins the centroid on a static silhouette. The cable need not land. + """ + import isaaclab.sim as sim_utils + + light_cfg = sim_utils.DomeLightCfg(intensity=3000.0, color=(0.75, 0.75, 0.75)) + light_cfg.func("/World/light", light_cfg) + + +def sim_cfg(): + """Physics for the cable cells. The device is deliberately left at the default. + + Rendering happens on the GPU, and forcing ``device="cpu"`` — as the physics-only cable tests do — + crashes the OVRTX transform sync on a device mismatch. + """ + from isaaclab_newton.physics import NewtonCfg + + import isaaclab.sim as sim_utils + + from isaaclab_contrib.deformable import VBDSolverCfg + + return sim_utils.SimulationCfg( + dt=0.01, + physics=NewtonCfg(solver_cfg=VBDSolverCfg(iterations=20), num_substeps=8, use_cuda_graph=False), + ) + + +def assert_tracks(renderer, first_lit, first_centroid, last_lit, last_centroid, first_z, last_z, envs=1): + """Assert the render followed the simulation, using both gates. + + The two are complementary, not redundant. A frozen render keeps its pixels (retention ~1.0, which + the retention gate cannot see) and does not move (centroid ~0, which catches it). A culled render + moves whatever remnant survives (centroid can pass) while losing most of its pixels (retention + catches it). Dropping either lets one of the two real defects this suite exists to catch through. + """ + import math + + where = f" across {envs} envs" if envs > 1 else "" + assert first_z - last_z > 0.1, f"cable did not fall{where}: {first_z:.4f} -> {last_z:.4f}" + assert first_centroid is not None, f"no cable geometry rendered in the first frame{where}" + assert last_centroid is not None, f"{renderer} dropped the cable from the render as it moved{where}" + assert last_lit >= 0.5 * first_lit, ( + f"{renderer} culled the cable while it moved{where}: {first_lit} -> {last_lit} lit px" + f" ({100 * (1 - last_lit / max(first_lit, 1)):.0f}% lost). A surviving remnant still produces" + " a centroid, so the displacement gate alone cannot detect this." + ) + shift = math.dist(first_centroid, last_centroid) + assert shift > MIN_CENTROID_SHIFT_PX, ( + f"{renderer} did not move the rendered cable{where} while it fell {first_z - last_z:.3f}" + f" units: centroid {first_centroid} -> {last_centroid} ({shift:.1f} px, need >" + f" {MIN_CENTROID_SHIFT_PX})" + ) diff --git a/source/isaaclab/test/renderers/test_cable_rendering.py b/source/isaaclab/test/renderers/test_cable_rendering.py new file mode 100644 index 000000000000..7a019bda3457 --- /dev/null +++ b/source/isaaclab/test/renderers/test_cable_rendering.py @@ -0,0 +1,204 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Cable (``UsdGeom.BasisCurves``) rendering coverage for the kit-less renderers. + +A cable's control points are rewritten every frame from Newton segment bodies. Nothing in the suite +rendered one before: the cable-object tests all run with ``render=False``, so a renderer could draw a +cable at its spawn pose and never move it again without any test noticing. + +Two properties are asserted, in order, for every renderer: + + 1. a cable renders at all, and + 2. the rendered image tracks the simulation instead of freezing at the spawn pose. + +The second is the load-bearing one. See :mod:`isaaclab.test.utils.cable_rendering` for why the +metrics are shaped the way they are. + +**Isaac RTX is covered separately**, in +``source/isaaclab_physx/test/renderers/test_cable_rendering_isaac_rtx.py``. It needs Kit started +before anything imports USD, which cannot be arranged in this module without making the kit-less +cells run under Kit — at which point they would no longer be testing the kit-less path. +""" + +import pytest +import torch + +pytest.importorskip("isaaclab_newton") +pytest.importorskip("newton") + +from isaaclab_newton.assets import CableObject as NewtonCableObject + +from isaaclab.scene import InteractiveScene, InteractiveSceneCfg +from isaaclab.sensors import Camera +from isaaclab.sim import build_simulation_context +from isaaclab.test.utils.cable_rendering import ( + ENV_NS, + TRACK_STEPS, + assert_mask_is_measuring_geometry, + assert_tracks, + cable_cfg, + camera_cfg, + geometry_centroid, + lit_mask, + lit_pixel_count, + sim_cfg, + spawn_light, +) +from isaaclab.utils.configclass import configclass + +pytestmark = [ + pytest.mark.integration, + pytest.mark.rendering, + pytest.mark.skipif(not torch.cuda.is_available(), reason="cable rendering requires a GPU"), +] + +# Minimum OVRTX that can draw an animated, partitioned cable at all. Below this the renderer +# publishes an empty bounding box for a partitioned BasisCurves and the whole partition goes dark, so +# a failure says nothing about Isaac Lab. A version floor rather than an xfail: an xfail records the +# cell as expected-to-fail and quietly stays red once the renderer is fixed, whereas a skip names the +# reason and turns into a real result the moment the floor is met. +_MIN_OVRTX = (0, 4, 1) + +_RENDERERS = ["ovrtx", "newton_warp"] + + +def _skip_if_ovrtx_too_old() -> None: + """Skip the OVRTX cells when the installed OVRTX predates the curve bounding-box fixes.""" + ovrtx = pytest.importorskip("ovrtx") + raw = getattr(ovrtx, "__version__", "0.0.0") + parts = [] + for piece in raw.split(".")[:3]: + digits = "".join(character for character in piece if character.isdigit()) + parts.append(int(digits) if digits else 0) + version = tuple(parts) + (0,) * (3 - len(parts)) + if version < _MIN_OVRTX: + pytest.skip( + f"OVRTX {raw} predates {'.'.join(str(value) for value in _MIN_OVRTX)}, the first build" + " that renders a BasisCurves inside a scene partition at all. Cable rendering cannot be" + " assessed below this floor." + ) + + +def _renderer_cfg(renderer: str): + """Build the renderer config, skipping the case when that backend is not installed.""" + if renderer == "ovrtx": + pytest.importorskip("isaaclab_ov") + _skip_if_ovrtx_too_old() + from isaaclab_ov.renderers import OVRTXRendererCfg + + return OVRTXRendererCfg() + if renderer == "newton_warp": + from isaaclab_newton.renderers.newton_warp_renderer_cfg import NewtonWarpRendererCfg + + return NewtonWarpRendererCfg() + raise ValueError(f"Unknown renderer: {renderer}") + + +def _build_scene(renderer: str) -> Camera: + """Spawn the light and camera. Returns the camera.""" + spawn_light() + return Camera(camera_cfg(_renderer_cfg(renderer))) + + +@pytest.mark.parametrize("renderer", _RENDERERS) +def test_cable_renders(renderer): + """A cable must produce geometry in a render.""" + cfg = sim_cfg() + with build_simulation_context(sim_cfg=cfg) as sim: + camera = _build_scene(renderer) + NewtonCableObject(cable_cfg()) + sim.reset() + sim.step(render=False) + camera.update(cfg.dt) + + mask = lit_mask(camera.data.output["rgb"]) + # Both bounds matter. Zero means nothing was drawn; near-totality means the background + # segmentation failed, which would silently disarm the tracking cells that follow. + assert_mask_is_measuring_geometry(mask, renderer) + assert int(mask.sum().item()) > 0, f"{renderer} rendered no cable geometry" + + +@pytest.mark.parametrize("renderer", _RENDERERS) +def test_cable_render_tracks_simulation(renderer): + """The rendered image must follow the cable as it falls, not freeze at the spawn pose.""" + cfg = sim_cfg() + with build_simulation_context(sim_cfg=cfg) as sim: + camera = _build_scene(renderer) + cable = NewtonCableObject(cable_cfg()) + sim.reset() + sim.step(render=False) + cable.update(cfg.dt) + camera.update(cfg.dt) + # Compute eagerly: ``camera.data.output["rgb"]`` is a live buffer overwritten in place on the + # next update, so holding the reference and measuring it later reads the LAST frame twice and + # the comparison silently becomes a no-op. + assert_mask_is_measuring_geometry(lit_mask(camera.data.output["rgb"]), renderer) + first_lit = lit_pixel_count(camera.data.output["rgb"]) + first_centroid = geometry_centroid(camera.data.output["rgb"]) + first_z = cable.data.segment_pose_w.torch[..., 2].mean().item() + + for _ in range(TRACK_STEPS): + sim.step(render=False) + cable.update(cfg.dt) + camera.update(cfg.dt) + assert_mask_is_measuring_geometry(lit_mask(camera.data.output["rgb"]), renderer) + last_lit = lit_pixel_count(camera.data.output["rgb"]) + last_centroid = geometry_centroid(camera.data.output["rgb"]) + last_z = cable.data.segment_pose_w.torch[..., 2].mean().item() + + assert_tracks(renderer, first_lit, first_centroid, last_lit, last_centroid, first_z, last_z) + + +@pytest.mark.parametrize("renderer", _RENDERERS) +def test_cable_renders_across_environments(renderer): + """Cables must still render and track once the scene is replicated across environments. + + Replication is exercised elsewhere, but every test there steps with ``render=False``, so the + *rendering* half of replication had no coverage at all. + + Two framing rules are load-bearing. The camera does **not** move with ``num_envs``: pulling it + back to frame more environments shrinks each cable below the detection threshold, which reads + exactly like a cull. And it is placed through the cfg, never with ``set_world_poses_from_view``. + """ + num_envs = 4 + spacing = 0.6 + eye = (spacing, -1.6 * spacing, 1.15) + target = (spacing, 0.0, 0.55) + + @configclass + class _CableSceneCfg(InteractiveSceneCfg): + cable = cable_cfg().replace(prim_path="{ENV_REGEX_NS}/Cable") + camera = camera_cfg(_renderer_cfg(renderer), f"{ENV_NS}/MultiEnvCam", eye, target) + + cfg = sim_cfg() + with build_simulation_context(sim_cfg=cfg) as sim: + spawn_light() + scene = InteractiveScene(_CableSceneCfg(num_envs=num_envs, env_spacing=spacing)) + sim.reset() + cable = scene["cable"] + camera = scene["camera"] + + assert cable.num_instances == num_envs, ( + f"replication produced {cable.num_instances} cables, expected {num_envs}" + ) + + scene.update(cfg.dt) + camera.update(cfg.dt) + assert_mask_is_measuring_geometry(lit_mask(camera.data.output["rgb"]), renderer) + first_lit = lit_pixel_count(camera.data.output["rgb"]) + first_centroid = geometry_centroid(camera.data.output["rgb"]) + first_z = cable.data.segment_pose_w.torch[..., 2].mean().item() + + for _ in range(TRACK_STEPS): + sim.step(render=False) + scene.update(cfg.dt) + camera.update(cfg.dt) + assert_mask_is_measuring_geometry(lit_mask(camera.data.output["rgb"]), renderer) + last_lit = lit_pixel_count(camera.data.output["rgb"]) + last_centroid = geometry_centroid(camera.data.output["rgb"]) + last_z = cable.data.segment_pose_w.torch[..., 2].mean().item() + + assert_tracks(renderer, first_lit, first_centroid, last_lit, last_centroid, first_z, last_z, envs=num_envs) diff --git a/source/isaaclab_newton/changelog.d/jmart-cable-render-binding.minor.rst b/source/isaaclab_newton/changelog.d/jmart-cable-render-binding.minor.rst new file mode 100644 index 000000000000..0e663204d155 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/jmart-cable-render-binding.minor.rst @@ -0,0 +1,31 @@ +Added +^^^^^ + +* Added :meth:`~isaaclab_newton.physics.NewtonManager.collect_cable_segment_shapes`, which maps each + renderable cable prim path to its ordered Newton segment shape ids. It reads only the Newton model + and the USD stage, so kit-less renderers can drive cable points without a Fabric sync path. + +Changed +^^^^^^^ + +* Changed :meth:`~isaaclab_newton.physics.NewtonManager.sync_cables_to_usd` to select its Fabric + prims on the simulation device and run its kernel there, instead of mirroring the Newton model to + the host and running on the CPU. The host mirror existed because the RTX Hydra render delegate + could not read GPU-backed Fabric arrays for ``BasisCurves.points``; that gap is fixed upstream. + This removes a device-to-host copy of the whole model's ``body_q`` on every dirty render frame, + which scaled with total body count rather than with the number of cable segments actually read. + +Fixed +^^^^^ + +* Fixed the Fabric cable sync silently doing nothing for a whole session when the Fabric stage was + not yet available at ``start_simulation``. The stage handle was resolved exactly once and cached, + so cables simulated correctly and rendered frozen at their spawn pose. The cable sync now + re-acquires the stage on first use, and ``start_simulation`` warns instead of dereferencing a + handle it does not have. +* Fixed cables rendering frozen under Kit-based renderers even when the Fabric write succeeded. The + sync writes ``points`` in place from a Warp kernel, which leaves a render delegate's cached curve + untouched, so each cable prim is now invalidated explicitly after the write. +* Fixed :class:`~isaaclab_newton.physics.NewtonManager` being unimportable inside a Kit session + whose bundled Warp lags the one Newton's solvers require, by deferring a module-level + ``newton.solvers`` import. diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py index fd5e0a735f7b..aff5d08f8e67 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py @@ -82,7 +82,6 @@ def _paused_gc(): from newton.sensors import SensorContact as NewtonContactSensor from newton.sensors import SensorFrameTransform from newton.sensors import SensorIMU as NewtonSensorIMU -from newton.solvers import SolverBase, SolverKamino from newton.usd import SchemaResolverNewton, SchemaResolverPhysx from pxr import Usd, UsdGeom @@ -114,6 +113,8 @@ def _paused_gc(): from .newton_manager_cfg import NewtonCfg, NewtonShapeCfg if TYPE_CHECKING: + from newton.solvers import SolverBase + from isaaclab_newton.actuators import NewtonActuatorAdapter from .newton_collision_cfg import NewtonCollisionPipelineCfg @@ -445,7 +446,12 @@ def provides_implicit_damping(cls) -> bool: _newton_cable_offset_attr = "newton:cableOffset" _newton_cable_count_attr = "newton:cableSegmentCount" _cable_shape_ids: wp.array | None = None - _cable_sync_cpu_buffers: tuple[wp.array, ...] | None = None + # Fabric stage recovered lazily by the cable sync. Deliberately NOT written back into + # ``_usdrt_stage``, which also selects the particle-visualisation path and would switch particle + # rendering onto the Fabric route with none of its bindings initialized. + _cable_usdrt_stage = None + # Curve prims the cable init accepted, invalidated individually after each device-side write. + _cable_prim_paths: list[str] = [] _newton_particle_offset_attr = "newton:particleOffset" _newton_particle_count_attr = "newton:particleCount" _particle_visual_prims: dict[str, _ParticleVisualPrim] = {} @@ -728,31 +734,73 @@ def sync_transforms_to_usd(cls) -> None: except Exception: logger.exception("[NewtonManager] sync_transforms_to_usd FAILED") + @classmethod + def _cable_fabric_stage(cls): + """Fabric stage the cable sync writes to: the shared one, or the lazily recovered one.""" + return cls._usdrt_stage if cls._usdrt_stage is not None else cls._cable_usdrt_stage + + @classmethod + def _recover_cable_fabric_path(cls) -> bool: + """Re-acquire the Fabric stage and rebuild the cable bindings if startup could not. + + :meth:`start_simulation` resolves the stage once. If it is unavailable then, the handle + stays unset for the session and every cable sync is a silent no-op: the cables simulate + correctly and render frozen at their spawn pose. It is normally available shortly after. + """ + if cls._cable_fabric_stage() is not None: + return True + try: + import usdrt # noqa: PLC0415 + + stage = get_current_stage(fabric=True) + if stage is None: + return False + fabric_hierarchy = usdrt.hierarchy.IFabricHierarchy().get_fabric_hierarchy( + stage.GetFabricId(), stage.GetStageIdAsStageId() + ) + # Assigned only once the bindings are built. _initialize_fabric_cable_prims raises on a + # malformed cable, and a handle left behind by a failed attempt would make every later + # call report success without ever rebuilding, freezing the render for the session. + NewtonManager._initialize_fabric_cable_prims(stage, fabric_hierarchy, usdrt) + NewtonManager._cable_usdrt_stage = stage + logger.info("[NewtonManager] recovered the Fabric cable path after a late stage acquisition") + return cls._cable_shape_ids is not None + except Exception: + logger.exception("[NewtonManager] Fabric cable path recovery FAILED") + return False + @classmethod def sync_cables_to_usd(cls) -> None: - """Write Newton cable segment endpoints to Fabric curve points.""" + """Write Newton cable segment endpoints to Fabric curve points, on device. + + Selecting the prims on the simulation device makes Fabric mirror ``points`` into GPU + storage, which is what makes the scene delegate advertise ``omni:rtx:isGPUBuffer`` and the + RTX Hydra delegate take its GPU-interop update path. The kernel reads Newton's own device + state, so a render frame costs one kernel launch and no host transfers. + """ if not cls._cables_dirty: return - if cls._usdrt_stage is None or cls._cable_shape_ids is None: + if cls._cable_fabric_stage() is None: + cls._recover_cable_fabric_path() + stage = cls._cable_fabric_stage() + if stage is None or cls._cable_shape_ids is None: NewtonManager._cables_dirty = False return try: import usdrt # noqa: PLC0415 - selection = cls._usdrt_stage.SelectPrims( + selection = stage.SelectPrims( require_attrs=[ (usdrt.Sdf.ValueTypeNames.Point3fArray, "points", usdrt.Usd.Access.ReadWrite), (usdrt.Sdf.ValueTypeNames.UInt, cls._newton_cable_offset_attr, usdrt.Usd.Access.Read), (usdrt.Sdf.ValueTypeNames.UInt, cls._newton_cable_count_attr, usdrt.Usd.Access.Read), (usdrt.Sdf.ValueTypeNames.Matrix4d, "omni:fabric:worldMatrix", usdrt.Usd.Access.Read), ], - device="cpu", + device=str(PhysicsManager._device), ) if selection.GetCount() == 0: NewtonManager._cables_dirty = False return - _, _, body_q, _, _ = cls._cable_sync_cpu_buffers - wp.copy(body_q, cls._state_0.body_q) wp.launch( _sync_cable_points, dim=selection.GetCount(), @@ -761,10 +809,21 @@ def sync_cables_to_usd(cls) -> None: wp.fabricarray(data=selection, attrib="omni:fabric:worldMatrix"), wp.fabricarray(data=selection, attrib=cls._newton_cable_offset_attr), wp.fabricarray(data=selection, attrib=cls._newton_cable_count_attr), - *cls._cable_sync_cpu_buffers, + cls._cable_shape_ids, + cls._model.shape_body, + cls._state_0.body_q, + cls._model.shape_transform, + cls._model.shape_scale, ], - device="cpu", + device=PhysicsManager._device, ) + # The points were written in place on the device. A render delegate caches its curve + # geometry and cannot notice that, so without an explicit invalidation it keeps drawing + # the points it last read and the cable renders frozen. + for cable_path in cls._cable_prim_paths: + points_attr = stage.GetPrimAtPath(cable_path).GetAttribute("points") + if points_attr: + points_attr.InvalidateGpuData() NewtonManager._cables_dirty = False except Exception: logger.exception("[NewtonManager] sync_cables_to_usd FAILED") @@ -954,6 +1013,8 @@ def step(cls) -> None: # joint_q_prev, and joint_lambdas via wp.clone/wp.zeros during the # first step() inside graph capture. Replay once to pin those # memory-pool addresses before any eager solver.reset() call. + from newton.solvers import SolverKamino # noqa: PLC0415 + if isinstance(cls._solver, SolverKamino): wp.capture_launch(cls._graph) logger.info("Newton CUDA graph captured (deferred relaxed mode, RTX-compatible)") @@ -1087,7 +1148,8 @@ def clear(cls): NewtonManager._particles_dirty = False NewtonManager._cables_dirty = False NewtonManager._cable_shape_ids = None - NewtonManager._cable_sync_cpu_buffers = None + NewtonManager._cable_usdrt_stage = None + NewtonManager._cable_prim_paths = [] NewtonManager._particle_visual_prims = {} NewtonManager._mpm_object_registry = [] NewtonManager._deformable_registry = [] @@ -1555,8 +1617,15 @@ def start_simulation(cls) -> None: logger.info("Dispatching PHYSICS_READY callbacks") cls.dispatch_event(PhysicsEvent.PHYSICS_READY) - # Setup USD/Fabric sync for Kit viewport rendering - if not cls._clone_physics_only: + # Setup USD/Fabric sync for Kit viewport rendering. ``get_current_stage`` is annotated + # ``-> Usd.Stage`` but returns None before the thread-local stage context is populated, so + # warn and skip rather than dereference it; the cable sync re-acquires it on first use. + if not cls._clone_physics_only and get_current_stage(fabric=True) is None: + logger.warning( + "[NewtonManager] Fabric stage unavailable at start_simulation; the cable sync will" + " re-acquire it lazily on first use." + ) + elif not cls._clone_physics_only: import usdrt body_paths = list(cls._model.body_label) @@ -1606,6 +1675,49 @@ def _initialize_fabric_body_prims(stage, fabric_hierarchy, usdrt, body_bindings: fabric_hierarchy.update_world_xforms() + @classmethod + def collect_cable_segment_shapes(cls) -> dict[str, list[int]]: + """Map each renderable cable prim path to its ordered Newton segment shape ids. + + Derived from the Newton model labels and the USD stage only, with no Fabric or Kit + dependency, so kit-less renderers (OVRTX) can drive cable points without the Fabric + sync path that :meth:`_initialize_fabric_cable_prims` sets up for Kit. + """ + if cls._model is None: + return {} + + usd_stage = get_current_stage() + cable_shapes: dict[str, dict[int, int]] = {} + for shape_id, label in enumerate(cls._model.shape_label): + if label is None: + continue + prim_path, separator, suffix = label.rpartition("_edge_capsule_") + if not separator or not suffix.isdigit(): + continue + cable_shapes.setdefault(prim_path, {})[int(suffix)] = shape_id + + ordered: dict[str, list[int]] = {} + for prim_path, segments in cable_shapes.items(): + usd_prim = usd_stage.GetPrimAtPath(prim_path) + if not usd_prim.IsValid() or not usd_prim.IsA(UsdGeom.BasisCurves): + continue + if not has_deformable_curve_api(usd_prim): + continue + curve = UsdGeom.BasisCurves(usd_prim) + counts = curve.GetCurveVertexCountsAttr().Get() + if ( + len(counts) != 1 + or int(counts[0]) < 2 + or curve.GetTypeAttr().Get() != UsdGeom.Tokens.linear + or curve.GetWrapAttr().Get() == UsdGeom.Tokens.periodic + ): + continue + segment_count = int(counts[0]) - 1 + if set(segments) != set(range(segment_count)): + continue + ordered[prim_path] = [segments[segment] for segment in range(segment_count)] + return ordered + @classmethod def _initialize_fabric_cable_prims(cls, stage, fabric_hierarchy, usdrt) -> None: """Initialize Fabric curve tags and packed Newton segment mappings.""" @@ -1624,6 +1736,7 @@ def _initialize_fabric_cable_prims(cls, stage, fabric_hierarchy, usdrt) -> None: segments[segment] = shape_id shape_ids: list[int] = [] + accepted_prim_paths: list[str] = [] for prim_path, segments in cable_shapes.items(): usd_prim = usd_stage.GetPrimAtPath(prim_path) if not usd_prim.IsValid() or not usd_prim.IsA(UsdGeom.BasisCurves): @@ -1654,21 +1767,14 @@ def _initialize_fabric_cable_prims(cls, stage, fabric_hierarchy, usdrt) -> None: segment_count ) shape_ids.extend(segment_shape_ids) + accepted_prim_paths.append(prim_path) if not shape_ids: NewtonManager._cable_shape_ids = None - NewtonManager._cable_sync_cpu_buffers = None + NewtonManager._cable_prim_paths = [] return NewtonManager._cable_shape_ids = wp.array(shape_ids, dtype=wp.int32, device=PhysicsManager._device) - # TODO: CPU mirror only needed because RTX Hydra ignores GPU Fabric BasisCurves points. - # Drop these buffers and sync on device once NVBug 6502662 is fixed. - NewtonManager._cable_sync_cpu_buffers = ( - NewtonManager._cable_shape_ids.to("cpu"), - cls._model.shape_body.to("cpu"), - wp.empty_like(cls._state_0.body_q, device="cpu"), - cls._model.shape_transform.to("cpu"), - cls._model.shape_scale.to("cpu"), - ) + NewtonManager._cable_prim_paths = accepted_prim_paths fabric_hierarchy.update_world_xforms() @staticmethod @@ -2090,6 +2196,8 @@ def _capture_or_defer_graph(cls) -> None: # joint_q_prev, and joint_lambdas via wp.clone/wp.zeros during the # first step() inside graph capture. Replay once to pin those # memory-pool addresses before any eager solver.reset() call. + from newton.solvers import SolverKamino # noqa: PLC0415 + if isinstance(cls._solver, SolverKamino): wp.capture_launch(cls._graph) else: diff --git a/source/isaaclab_ov/changelog.d/jmart-cable-render-binding.minor.rst b/source/isaaclab_ov/changelog.d/jmart-cable-render-binding.minor.rst new file mode 100644 index 000000000000..244c7a51e0da --- /dev/null +++ b/source/isaaclab_ov/changelog.d/jmart-cable-render-binding.minor.rst @@ -0,0 +1,10 @@ +Added +^^^^^ + +* Added Newton cable rendering to :class:`~isaaclab_ov.renderers.OVRTXRenderer`. Cable curve points + are computed on device from the Newton segment bodies and written zero-copy through an OVRTX array + binding each frame, so cables follow their simulated pose instead of drawing at their spawn pose + and never moving. +* Added the same cable binding to the ovstage render path. The endpoint kernel still runs on device; + the handover is host-side because ovstage 0.1.0 accepts the ``points`` column's dtype override + only on numpy arrays, not on DLPack producers. The legacy path remains zero-copy. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index a49536548e5c..80436af9f36c 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -88,6 +88,7 @@ ) from .ovrtx_renderer_cfg import OVRTXRendererCfg from .ovrtx_renderer_kernels import ( + compute_cable_points_world_kernel, create_camera_transforms_kernel, extract_all_tiles_kernel, generate_random_colors_from_ids_kernel, @@ -481,6 +482,17 @@ def _init_fields_legacy(self) -> None: self._deformable_points_binding = None self._particle_points_binding = None self._particle_workaround_applied = False + self._cable_points_binding = None + self._cable_segment_counts: list[int] = [] + self._cable_point_offsets: list[int] = [] + # Populated by _setup_cable_bindings_legacy; declared here so a renderer that never binds a + # cable still has the attributes. + self._cable_shape_ids = None + self._cable_offsets = None + self._cable_counts = None + self._cable_point_offsets_wp = None + self._cable_points = None + self._cable_point_slices: list[wp.array] = [] def _initialize_from_spec_legacy(self, spec: CameraRenderSpec): """Initialize the OVRTX renderer with internal environment cloning. @@ -564,6 +576,7 @@ def _initialize_from_spec_legacy(self, spec: CameraRenderSpec): self._setup_xform_bindings() self._setup_deformable_bindings(num_envs) self._setup_particle_bindings() + self._setup_cable_bindings() def _clone_sources_in_ovrtx(self): """Clone sources in OVRTX using the scene :class:`~isaaclab.cloner.ClonePlan`.""" @@ -794,6 +807,83 @@ def _setup_deformable_bindings_legacy(self, num_envs: int): if self._deformable_points_binding is None: raise RuntimeError("Failed to create OVRTX deformable body bindings") + def _setup_cable_bindings_legacy(self) -> None: + """Setup OVRTX ``points`` bindings for Newton cables (UsdGeom.BasisCurves). + + Cables are rigid segment bodies, not particles, so their curve points are derived from + ``body_q`` each frame rather than sliced out of ``particle_q``. + """ + try: + from isaaclab_newton.physics import NewtonManager + except ImportError: + logger.debug("NewtonManager not available, skipping cable point bindings") + return + + cable_shapes = NewtonManager.collect_cable_segment_shapes() + if not cable_shapes: + logger.debug("No renderable Newton cables registered, skipping cable point bindings") + return + + cable_prim_paths: list[str] = [] + flat_shape_ids: list[int] = [] + offsets: list[int] = [] + counts: list[int] = [] + for prim_path, segment_shape_ids in cable_shapes.items(): + cable_prim_paths.append(prim_path) + offsets.append(len(flat_shape_ids)) + counts.append(len(segment_shape_ids)) + flat_shape_ids.extend(segment_shape_ids) + + prim_count = len(cable_prim_paths) + # Points are written in world space, so neutralise the inherited env/asset transform the + # same way the deformable path does; otherwise the transform is applied twice. + self._renderer.write_attribute( + prim_paths=cable_prim_paths, + attribute_name="omni:resetXformStack", + tensor=np.full(prim_count, True, dtype=np.bool_), + prim_mode=PrimMode.MUST_EXIST, + ) + self._renderer.write_attribute( + prim_paths=cable_prim_paths, + attribute_name="omni:xform", + tensor=np.tile(np.eye(4, dtype=np.float64), (prim_count, 1, 1)), + semantic=Semantic.XFORM_MAT4x4, + prim_mode=PrimMode.MUST_EXIST, + ) + + self._cable_points_binding = self._renderer.bind_array_attribute( + prim_paths=cable_prim_paths, + attribute_name="points", + dtype=np.float32, + shape=(3,), + prim_mode=PrimMode.MUST_EXIST, + flags=BindingFlag.OPTIMIZE, + ) + if self._cable_points_binding is None: + raise RuntimeError("Failed to create OVRTX cable point bindings") + + # Allocated on the simulation device and kept there: OVRTX selects its GPU-interop update + # path off the tensor's DLPack device, not off the access mode, so a host buffer would + # silently downgrade every write to the CPU path rather than fail. + device = self._device + self._cable_shape_ids = wp.array(flat_shape_ids, dtype=wp.int32, device=device) + self._cable_offsets = wp.array(offsets, dtype=wp.int32, device=device) + self._cable_counts = wp.array(counts, dtype=wp.int32, device=device) + self._cable_segment_counts = counts + # One flat buffer of curve points; each curve owns counts[i] + 1 entries. + self._cable_point_offsets = [] + total_points = 0 + for count in counts: + self._cable_point_offsets.append(total_points) + total_points += count + 1 + self._cable_points = wp.zeros(total_points, dtype=wp.vec3f, device=device) + self._cable_point_offsets_wp = wp.array(self._cable_point_offsets, dtype=wp.int32, device=device) + # Stable for the lifetime of the binding, so built once rather than every render frame. + self._cable_point_slices = [ + self._cable_points[point_offset : point_offset + segment_count + 1] + for point_offset, segment_count in zip(self._cable_point_offsets, counts, strict=True) + ] + def _setup_particle_bindings_legacy(self) -> None: """Setup OVRTX bindings for Newton particle clouds.""" try: @@ -915,6 +1005,9 @@ def _update_transforms_legacy(self) -> None: def _update_geometries_legacy(self) -> None: """Sync geometries to OVRTX.""" + if self._cable_points_binding is not None: + self._write_cable_points() + if self._deformable_points_binding is None and self._particle_points_binding is None: return @@ -952,6 +1045,45 @@ def _update_geometries_legacy(self) -> None: self._particle_visual_counts, ) + def _write_cable_points(self) -> None: + """Recompute world-space cable curve points from Newton bodies and write them to OVRTX.""" + from isaaclab_newton.physics import NewtonManager + + model = NewtonManager._model + state = NewtonManager.get_state() + if model is None or state is None: + return + + wp.launch( + compute_cable_points_world_kernel, + dim=len(self._cable_segment_counts), + inputs=[ + self._cable_shape_ids, + self._cable_offsets, + self._cable_counts, + self._cable_point_offsets_wp, + model.shape_body, + state.body_q, + model.shape_transform, + model.shape_scale, + self._cable_points, + ], + device=self._device, + ) + + # The slices alias ``_cable_points``, which the kernel above just wrote, so OVRTX must not + # read them until it has finished. ``cuda_stream`` hands over the Warp stream the kernel was + # enqueued on, so OVRTX inserts a GPU-side wait instead of us forcing a host synchronize. + # ``DataAccess.ASYNC`` over device-resident tensors is what selects the GPU-interop update + # path; the ovrtx API defaults to ``SYNC``, and neither that nor a host array reports an + # error -- the write simply takes the CPU path. + cuda_stream = wp.get_stream(self._device).cuda_stream + self._cable_points_binding.write( + cast(Any, self._cable_point_slices), + data_access=DataAccess.ASYNC, + cuda_stream=cuda_stream, + ) + def _write_particle_q_slices( self, binding: Any, @@ -1437,12 +1569,23 @@ def _safe_unbind(binding, name: str) -> None: self._deformable_points_binding = None _safe_unbind(self._particle_points_binding, "particle points") self._particle_points_binding = None + _safe_unbind(self._cable_points_binding, "cable points") + self._cable_points_binding = None self._deformable_particle_offsets = [] self._deformable_particle_counts = [] self._particle_visual_offsets = [] self._particle_visual_counts = [] self._particle_workaround_applied = False + self._cable_segment_counts = [] + self._cable_point_offsets = [] + # Drop the slice views before the buffer they alias, so nothing outlives it. + self._cable_point_slices = [] + self._cable_points = None + self._cable_shape_ids = None + self._cable_offsets = None + self._cable_counts = None + self._cable_point_offsets_wp = None if self._renderer: try: @@ -1490,6 +1633,12 @@ def _setup_particle_bindings(self) -> None: else: self._setup_particle_bindings_legacy() + def _setup_cable_bindings(self) -> None: + if self._use_ovstage: + self._setup_cable_bindings_ovstage() + else: + self._setup_cable_bindings_legacy() + def update_transforms(self) -> None: """Sync transforms to OVRTX.""" if self._use_ovstage: @@ -1570,6 +1719,8 @@ def _init_fields_ovstage(self) -> None: self._deformable_paths_list = None self._particle_points_query = None self._particle_paths_list = None + self._cable_points_query = None + self._cable_paths_list = None self._env_root_xforms: np.ndarray | None = None def _initialize_from_spec_ovstage(self, spec: CameraRenderSpec) -> None: @@ -1673,6 +1824,7 @@ def _initialize_from_spec_ovstage(self, spec: CameraRenderSpec) -> None: self._setup_xform_bindings_ovstage() self._setup_deformable_bindings_ovstage(num_envs) self._setup_particle_bindings_ovstage() + self._setup_cable_bindings() # Commit all init-time writes then attach. attach_ovstage happens last so the renderer # immediately sees the fully-configured scene on its first step. @@ -1941,6 +2093,114 @@ def _setup_deformable_bindings_ovstage(self, num_envs: int) -> None: if self._deformable_points_query is None: raise RuntimeError("Failed to create OVRTX deformable body bindings") + def _setup_cable_bindings_ovstage(self) -> None: + """Setup ovstage ``points`` bindings for Newton cables (``UsdGeom.BasisCurves``). + + Mirrors :meth:`_setup_cable_bindings_legacy`, except that the per-frame write goes through a + host copy: ovstage 0.1.0's ``make_dltensor`` accepts the lanes=3 dtype override only on + numpy arrays, so a warp ``vec3f`` slice is rejected against the ``points`` column. The + endpoint kernel still runs on device; only the handover is host-side. + """ + try: + from isaaclab_newton.physics import NewtonManager + except ImportError: + logger.debug("NewtonManager not available, skipping cable point bindings") + return + + cable_shapes = NewtonManager.collect_cable_segment_shapes() + if not cable_shapes: + logger.debug("No renderable Newton cables registered, skipping cable point bindings") + return + + cable_prim_paths: list[str] = [] + flat_shape_ids: list[int] = [] + offsets: list[int] = [] + counts: list[int] = [] + for prim_path, segment_shape_ids in cable_shapes.items(): + cable_prim_paths.append(prim_path) + offsets.append(len(flat_shape_ids)) + counts.append(len(segment_shape_ids)) + flat_shape_ids.extend(segment_shape_ids) + + prim_count = len(cable_prim_paths) + self._cable_paths_list = self._stage_paths.create_path_list_from_strings(cable_prim_paths) + self._cable_points_query = self._stage.query_from_path_list(self._cable_paths_list) + + # The kernel emits world space, so reset the xform stack and pin an identity omni:xform to + # stop the env-root and asset-root ancestor transforms being applied on top. + self._stage.write_attribute( + self._cable_points_query, + "omni:resetXformStack", + ordinal=self._current_ordinal, + tensors=np.full(prim_count, True, dtype=np.bool_), + is_array=False, + ).wait() + + identity_xforms = np.tile(np.eye(4, dtype=np.float64), (prim_count, 1, 1)) + self._stage.write_attribute( + self._cable_points_query, + "omni:xform", + ordinal=self._current_ordinal, + tensors=_xform_tensor_from_numpy(identity_xforms), + is_array=False, + semantic=ovstage.AttributeSemantic.MATRIX, + ).wait() + + device = self._device + self._cable_shape_ids = wp.array(flat_shape_ids, dtype=wp.int32, device=device) + self._cable_offsets = wp.array(offsets, dtype=wp.int32, device=device) + self._cable_counts = wp.array(counts, dtype=wp.int32, device=device) + self._cable_segment_counts = counts + self._cable_point_offsets = [] + total_points = 0 + for count in counts: + self._cable_point_offsets.append(total_points) + total_points += count + 1 + self._cable_points = wp.zeros(total_points, dtype=wp.vec3f, device=device) + self._cable_point_offsets_wp = wp.array(self._cable_point_offsets, dtype=wp.int32, device=device) + + def _write_cable_points_ovstage(self) -> None: + """Recompute world-space cable curve points on device and write them through ovstage.""" + from isaaclab_newton.physics import NewtonManager + + model = NewtonManager._model + state = NewtonManager.get_state() + if model is None or state is None: + return + + wp.launch( + compute_cable_points_world_kernel, + dim=len(self._cable_segment_counts), + inputs=[ + self._cable_shape_ids, + self._cable_offsets, + self._cable_counts, + self._cable_point_offsets_wp, + model.shape_body, + state.body_q, + model.shape_transform, + model.shape_scale, + self._cable_points, + ], + device=self._device, + ) + + wp.synchronize_device(self._device) + points_np = self._cable_points.numpy() + cable_slices = [ + _points_tensor_from_numpy(points_np[point_offset : point_offset + segment_count + 1]) + for point_offset, segment_count in zip(self._cable_point_offsets, self._cable_segment_counts, strict=True) + ] + + self._stage.write_attribute( + self._cable_points_query, + "points", + ordinal=self._current_ordinal, + tensors=cable_slices, + is_array=True, + semantic=ovstage.AttributeSemantic.POINT, + ).wait() + def _setup_particle_bindings_ovstage(self) -> None: """Setup OVRTX bindings for Newton particle clouds (ovstage path).""" try: @@ -2041,6 +2301,9 @@ def _update_transforms_ovstage(self) -> None: ).wait() def _update_geometries_ovstage(self) -> None: + if self._cable_points_query is not None: + self._write_cable_points_ovstage() + if self._deformable_points_query is None and self._particle_points_query is None: return @@ -2216,11 +2479,23 @@ def _safe_destroy_path_list(path_list, name: str) -> None: _safe_destroy_path_list(self._particle_paths_list, "particle paths") self._particle_paths_list = None + _safe_release_query(self._cable_points_query, "cable points") + self._cable_points_query = None + _safe_destroy_path_list(self._cable_paths_list, "cable paths") + self._cable_paths_list = None + self._object_newton_indices = None self._deformable_particle_offsets = [] self._deformable_particle_counts = [] self._particle_visual_offsets = [] self._particle_visual_counts = [] + self._cable_segment_counts = [] + self._cable_point_offsets = [] + self._cable_points = None + self._cable_shape_ids = None + self._cable_offsets = None + self._cable_counts = None + self._cable_point_offsets_wp = None self._env_root_xforms = None # Detach before closing ExitStack: the renderer holds a live reference into the stage, diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_kernels.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_kernels.py index c993c0e6b9a1..27c3d893cd6d 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_kernels.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer_kernels.py @@ -186,3 +186,47 @@ def sync_newton_transforms_kernel( body_idx = newton_body_indices[i] transform = newton_body_q[body_idx] ovrtx_transforms[i] = wp.transpose(wp.mat44d(wp.transform_to_matrix(transform))) + + +@wp.kernel(enable_backward=False) +def compute_cable_points_world_kernel( + shape_ids: wp.array(dtype=wp.int32), # type: ignore + offsets: wp.array(dtype=wp.int32), # type: ignore + counts: wp.array(dtype=wp.int32), # type: ignore + point_offsets: wp.array(dtype=wp.int32), # type: ignore + shape_body: wp.array(dtype=wp.int32), # type: ignore + body_q: wp.array(dtype=wp.transformf), # type: ignore + shape_transform: wp.array(dtype=wp.transformf), # type: ignore + shape_scale: wp.array(dtype=wp.vec3f), # type: ignore + points_out: wp.array(dtype=wp.vec3f), # type: ignore +): + """Write world-space cable curve points from Newton segment bodies. + + Mirrors NewtonManager._sync_cable_points, but emits world space because the OVRTX cable + prims pin an identity omni:xform with the transform stack reset. Endpoints come from the + first and last capsule; interior points are the midpoint of the two adjacent capsule ends. + """ + curve = wp.tid() + offset = offsets[curve] + segment_count = counts[curve] + point_base = point_offsets[curve] + + for point in range(segment_count + 1): + endpoint_w = wp.vec3f() + if point == 0: + shape = shape_ids[offset] + shape_q = wp.transform_multiply(body_q[shape_body[shape]], shape_transform[shape]) + endpoint_w = wp.transform_point(shape_q, wp.vec3f(0.0, 0.0, -shape_scale[shape][1])) + elif point == segment_count: + shape = shape_ids[offset + segment_count - 1] + shape_q = wp.transform_multiply(body_q[shape_body[shape]], shape_transform[shape]) + endpoint_w = wp.transform_point(shape_q, wp.vec3f(0.0, 0.0, shape_scale[shape][1])) + else: + left_shape = shape_ids[offset + point - 1] + left_q = wp.transform_multiply(body_q[shape_body[left_shape]], shape_transform[left_shape]) + left_w = wp.transform_point(left_q, wp.vec3f(0.0, 0.0, shape_scale[left_shape][1])) + right_shape = shape_ids[offset + point] + right_q = wp.transform_multiply(body_q[shape_body[right_shape]], shape_transform[right_shape]) + right_w = wp.transform_point(right_q, wp.vec3f(0.0, 0.0, -shape_scale[right_shape][1])) + endpoint_w = 0.5 * (left_w + right_w) + points_out[point_base + point] = endpoint_w diff --git a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py index 871730c398b0..d592f035da01 100644 --- a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py +++ b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py @@ -103,6 +103,12 @@ def _make_renderer_without_backend(device: str = "cpu") -> tuple[OVRTXRenderer, renderer._particle_visual_offsets = [] renderer._particle_visual_counts = [] renderer._particle_workaround_applied = False + # Cable bindings are set in __init__, which this fixture bypasses via __new__. Without them + # _update_geometries_legacy raises AttributeError on its cable check before reaching anything + # this module is testing. + renderer._cable_points_binding = None + renderer._cable_segment_counts = [] + renderer._cable_point_offsets = [] renderer._use_ovstage = False return renderer, renderer._renderer @@ -479,3 +485,106 @@ class _FakeStream: assert mpm_written[0].ptr == particle_q[2:4].ptr assert renderer._particle_points_binding.write_kwargs["data_access"] is DataAccess.ASYNC assert len(backend.writes) == 0 + + +def _cable_registry(shapes: dict[str, list[int]]): + """Return a ``collect_cable_segment_shapes`` replacement yielding ``shapes``.""" + return classmethod(lambda cls: dict(shapes)) + + +def test_setup_cable_bindings_binds_curve_points(monkeypatch: pytest.MonkeyPatch): + """Renderable cables create a ``points`` array binding over their curve prims.""" + renderer, backend = _make_renderer_without_backend() + monkeypatch.setattr( + NewtonManager, + "collect_cable_segment_shapes", + _cable_registry({"/World/envs/env_0/Cable/geometry/mesh": [4, 5, 6]}), + ) + + renderer._setup_cable_bindings() + + assert len(backend.calls) == 1 + assert backend.calls[0]["prim_paths"] == ["/World/envs/env_0/Cable/geometry/mesh"] + assert backend.calls[0]["attribute_name"] == "points" + assert backend.calls[0]["dtype"] is np.float32 + assert backend.calls[0]["shape"] == (3,) + assert backend.calls[0]["flags"] is BindingFlag.OPTIMIZE + assert renderer._cable_points_binding is backend.bindings["points"] + + # World-space points are written directly, so the inherited env transform must be neutralised + # or it is applied twice -- the same contract the deformable path relies on. + assert [write["attribute_name"] for write in backend.writes] == ["omni:resetXformStack", "omni:xform"] + + +def test_setup_cable_bindings_offsets_span_every_curve(monkeypatch: pytest.MonkeyPatch): + """Each cable owns ``segments + 1`` points, laid end to end in one flat buffer. + + This is the arithmetic that silently breaks under replication: a single cable is correct for + almost any indexing scheme, so the guard is several cables of *different* lengths. + """ + renderer, _ = _make_renderer_without_backend() + monkeypatch.setattr( + NewtonManager, + "collect_cable_segment_shapes", + _cable_registry( + { + "/World/envs/env_0/Cable/geometry/mesh": [0, 1, 2], + "/World/envs/env_1/Cable/geometry/mesh": [3, 4, 5, 6, 7], + "/World/envs/env_2/Cable/geometry/mesh": [8, 9], + } + ), + ) + + renderer._setup_cable_bindings() + + assert renderer._cable_segment_counts == [3, 5, 2] + # 4 points, then 6, then 3 -- each cable starts where the previous one ended. + assert renderer._cable_point_offsets == [0, 4, 10] + assert len(renderer._cable_points) == 13 + + +def test_setup_cable_bindings_noop_without_cables(monkeypatch: pytest.MonkeyPatch): + """A scene with no renderable cables binds nothing rather than failing.""" + renderer, backend = _make_renderer_without_backend() + monkeypatch.setattr(NewtonManager, "collect_cable_segment_shapes", _cable_registry({})) + + renderer._setup_cable_bindings() + + assert renderer._cable_points_binding is None + assert backend.calls == [] + + +def test_write_cable_points_writes_one_slice_per_cable(monkeypatch: pytest.MonkeyPatch): + """Every cable is handed exactly its own span of the shared point buffer.""" + renderer, _ = _make_renderer_without_backend() + monkeypatch.setattr( + NewtonManager, + "collect_cable_segment_shapes", + _cable_registry( + { + "/World/envs/env_0/Cable/geometry/mesh": [0, 1], + "/World/envs/env_1/Cable/geometry/mesh": [2, 3, 4], + } + ), + ) + renderer._setup_cable_bindings() + + model = SimpleNamespace(shape_body=None, shape_transform=None, shape_scale=None) + monkeypatch.setattr(NewtonManager, "_model", model) + monkeypatch.setattr(NewtonManager, "get_state", classmethod(lambda cls: SimpleNamespace(body_q=None))) + # The kernel needs a live Newton model; this test covers the slicing around it, not the maths in it. + monkeypatch.setattr(ovrtx_renderer_module.wp, "launch", lambda *args, **kwargs: None) + monkeypatch.setattr(ovrtx_renderer_module.wp, "get_stream", lambda device: SimpleNamespace(cuda_stream=1234)) + + renderer._write_cable_points() + + written = renderer._cable_points_binding.written + assert written is not None + assert [len(slice_) for slice_ in written] == [3, 4] + assert written[0].ptr == renderer._cable_points[0:3].ptr + assert written[1].ptr == renderer._cable_points[3:7].ptr + # Zero-copy: OVRTX is handed the Warp stream so it waits on the kernel instead of forcing a host + # round-trip. Switching to SYNC would silently reintroduce a per-frame device copy, and is the + # only guard against that -- the downgrade does not raise, it just renders from a stale copy. + assert renderer._cable_points_binding.write_kwargs["data_access"] is DataAccess.ASYNC + assert renderer._cable_points_binding.write_kwargs["cuda_stream"] == 1234 diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index e1a5651dca44..3b0fb028dbc8 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -425,11 +425,14 @@ def reset_stage(self) -> None: renderer._object_xform_binding = _RecordingBinding(events, "object") renderer._deformable_points_binding = _RecordingBinding(events, "deformable") renderer._particle_points_binding = _RecordingBinding(events, "particle") + renderer._cable_points_binding = _RecordingBinding(events, "cable") renderer._deformable_particle_offsets = [0] renderer._deformable_particle_counts = [1] renderer._particle_visual_offsets = [0] renderer._particle_visual_counts = [1] renderer._particle_workaround_applied = True + renderer._cable_segment_counts = [1] + renderer._cable_point_offsets = [0] renderer._renderer = Backend() renderer._render_product_paths = ["/Render/RenderProduct_camera"] renderer._output_id_color_buffers = {"semantic_segmentation": object()} @@ -473,6 +476,8 @@ def close(self) -> None: renderer._deformable_paths_list = "deformable" renderer._particle_points_query = "particle" renderer._particle_paths_list = "particle" + renderer._cable_points_query = "cable" + renderer._cable_paths_list = "cable" renderer._object_newton_indices = object() renderer._deformable_particle_offsets = [0] renderer._deformable_particle_counts = [1] @@ -500,12 +505,14 @@ def test_ovrtx_close_releases_legacy_renderer_state(): "unbind:object", "unbind:deformable", "unbind:particle", + "unbind:cable", "reset_stage", ] assert renderer._camera_xform_binding is None assert renderer._object_xform_binding is None assert renderer._deformable_points_binding is None assert renderer._particle_points_binding is None + assert renderer._cable_points_binding is None assert renderer._particle_workaround_applied is False assert renderer._renderer is None assert renderer._render_product_paths == [] @@ -535,11 +542,15 @@ def test_ovrtx_close_releases_ovstage_renderer_state(): "destroy_path_list:deformable", "release_query:particle", "destroy_path_list:particle", + "release_query:cable", + "destroy_path_list:cable", "detach_ovstage", "exit_stack_close", ] assert renderer._camera_xform_query is None assert renderer._particle_paths_list is None + assert renderer._cable_points_query is None + assert renderer._cable_paths_list is None assert renderer._object_newton_indices is None assert renderer._env_root_xforms is None assert renderer._renderer is None diff --git a/source/isaaclab_physx/changelog.d/jmart-cable-render-binding.rst b/source/isaaclab_physx/changelog.d/jmart-cable-render-binding.rst new file mode 100644 index 000000000000..9f8855126e6d --- /dev/null +++ b/source/isaaclab_physx/changelog.d/jmart-cable-render-binding.rst @@ -0,0 +1,11 @@ +Fixed +^^^^^ + +* Fixed the Isaac RTX sensor pump refreshing rigid-body transforms only. Newton writes cable curve + points and particle points into Fabric from the ``sync_*_to_usd`` calls that only ``pre_render()`` + makes, so a camera read that skipped them drew bodies at their current pose and cables frozen at + their spawn pose — a stable, entirely plausible image of the wrong thing. +* Fixed :class:`~isaaclab_physx.renderers.IsaacRtxRenderer` launching its tiled-reshape kernel over + an empty annotator buffer. The launch is dimensioned by the destination, so every thread read past + the end of a zero-length source and took the process down with an illegal memory access. A frame + with no data now reports a blank frame and a warning. diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 04da134d6035..463ece9385e0 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -531,6 +531,20 @@ def tiling_grid_shape(): else: tiled_data_buffer = output + # An annotator can hand back an EMPTY buffer, e.g. when the render product produced no + # data for this frame. The reshape launch below is dimensioned by the DESTINATION, so + # every one of its view_count * height * width threads would read past the end of a + # zero-length source: an illegal memory access that surfaces asynchronously, often + # inside an unrelated device free, and takes the whole process down. A renderer with no + # pixels should report no pixels. + if getattr(tiled_data_buffer, "size", None) == 0: + logger.warning( + "[IsaacRtxRenderer] annotator '%s' returned an empty buffer; skipping this" + " frame's reshape. The output for this frame will be blank.", + data_type, + ) + continue + # convert data buffer to warp array if isinstance(tiled_data_buffer, np.ndarray): # Let warp infer the dtype from numpy array instead of hardcoding uint8 diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_utils.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_utils.py index f0b67e4b2afa..a26eed3feb57 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_utils.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_utils.py @@ -242,7 +242,13 @@ def ensure_isaac_rtx_render_update(force: bool = False) -> None: # Sync physics results → Fabric so RTX sees updated positions. # physics_manager.step() only runs simulate()/fetch_results() and does NOT # call _update_fabric(), so without this the render would lag one frame behind. - sim.physics_manager.forward() + # + # ``pre_render`` rather than ``forward``: forward() refreshes rigid-body transforms only, while + # Newton-authored geometry — cable curve points, particle points — is written into Fabric by the + # sync_*_to_usd calls that only pre_render() makes. A sensor read that skips them draws bodies at + # the current pose and cables frozen at their spawn pose, which still yields a stable, plausible + # image. + sim.physics_manager.pre_render() import omni.kit.app diff --git a/source/isaaclab_physx/test/renderers/test_cable_rendering_isaac_rtx.py b/source/isaaclab_physx/test/renderers/test_cable_rendering_isaac_rtx.py new file mode 100644 index 000000000000..e371efeea2b8 --- /dev/null +++ b/source/isaaclab_physx/test/renderers/test_cable_rendering_isaac_rtx.py @@ -0,0 +1,183 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Cable (``UsdGeom.BasisCurves``) rendering coverage for the Isaac RTX renderer. + +The kit-less half of this coverage lives in +``source/isaaclab/test/renderers/test_cable_rendering.py``; the shared scene, framing and metrics +live in :mod:`isaaclab.test.utils.cable_rendering` so the two halves stay comparable. + +**Why this is a separate module.** Isaac RTX needs Kit started *before* anything imports USD, hence +the ``AppLauncher`` call at module scope below. Booting Kit lazily arrives too late — the +module-scope imports have already pulled in USD, and USD-first dies in ``libusd_tf``. Booting it for +all renderers would stop the OVRTX and Newton Warp cells exercising the kit-less path they exist to +cover. Splitting the module gives each half the process it needs. +""" + +from isaaclab.app import AppLauncher + +# Must precede every USD-touching import below. ``enable_cameras`` selects the rendering Kit +# experience; without it no offscreen render product is created and the annotator returns nothing, +# which reads as a renderer that draws no cable. +simulation_app = AppLauncher(headless=True, enable_cameras=True).app + +"""Rest everything follows.""" + +import pytest +import torch +import warp as wp +from isaaclab_newton.assets import CableObject as NewtonCableObject + +from isaaclab.scene import InteractiveScene, InteractiveSceneCfg +from isaaclab.sensors import Camera +from isaaclab.sim import build_simulation_context +from isaaclab.test.utils.cable_rendering import ( + ENV_NS, + TRACK_STEPS, + assert_mask_is_measuring_geometry, + assert_tracks, + cable_cfg, + camera_cfg, + geometry_centroid, + lit_mask, + lit_pixel_count, + sim_cfg, + spawn_light, +) +from isaaclab.utils.configclass import configclass + +# Newton's VBD solver, which cables require, needs symbols that landed in Warp 1.15 +# (``DeterministicMode``) and 1.16 (``quat_twist_angle_signed``). Below that the cells die deep in +# Newton's lazy solver shim as ``ImportError: cannot import name 'SolverVBD'``, which names the wrong +# thing entirely. +_MIN_WARP = (1, 16) + + +def _warp_version() -> tuple[int, int]: + """Major/minor of the Warp this process actually imported.""" + parts = [] + for piece in wp.__version__.split(".")[:2]: + digits = "".join(character for character in piece if character.isdigit()) + parts.append(int(digits) if digits else 0) + return tuple(parts) + (0,) * (2 - len(parts)) + + +# Keyed on the IMPORTED module, not a listing of Isaac Sim's ``extscache``. Isaac Sim bundles its +# own Warp, but whether that copy reaches ``sys.path`` ahead of the environment's depends on the +# venv layout, so a gate that globs ``extscache`` both skips runs that would pass and passes runs +# that would skip. Evaluated after ``AppLauncher``, so it reflects what Kit actually loaded. +pytestmark = [ + pytest.mark.integration, + pytest.mark.rendering, + pytest.mark.isaacsim_ci, + pytest.mark.skipif(not torch.cuda.is_available(), reason="cable rendering requires a GPU"), + pytest.mark.skipif( + _warp_version() < _MIN_WARP, + reason=( + f"the imported Warp is {wp.__version__}, but Newton's VBD solver (required by cables)" + f" needs >= {_MIN_WARP[0]}.{_MIN_WARP[1]}. Isaac Sim bundles its own Warp under" + " extscache; if that copy is winning on sys.path, replacing its payload with a newer" + " Warp clears this — nothing pins the version." + ), + ), +] + +_RENDERER = "isaac_rtx" + + +def _renderer_cfg(): + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + return IsaacRtxRendererCfg() + + +def _build_scene() -> Camera: + spawn_light() + return Camera(camera_cfg(_renderer_cfg())) + + +def test_cable_renders(): + """A cable must produce geometry in a render.""" + cfg = sim_cfg() + with build_simulation_context(sim_cfg=cfg) as sim: + camera = _build_scene() + NewtonCableObject(cable_cfg()) + sim.reset() + sim.step(render=False) + camera.update(cfg.dt) + + mask = lit_mask(camera.data.output["rgb"]) + assert_mask_is_measuring_geometry(mask, _RENDERER) + assert int(mask.sum().item()) > 0, f"{_RENDERER} rendered no cable geometry" + + +def test_cable_render_tracks_simulation(): + """The rendered image must follow the cable as it falls, not freeze at the spawn pose.""" + cfg = sim_cfg() + with build_simulation_context(sim_cfg=cfg) as sim: + camera = _build_scene() + cable = NewtonCableObject(cable_cfg()) + sim.reset() + sim.step(render=False) + cable.update(cfg.dt) + camera.update(cfg.dt) + assert_mask_is_measuring_geometry(lit_mask(camera.data.output["rgb"]), _RENDERER) + first_lit = lit_pixel_count(camera.data.output["rgb"]) + first_centroid = geometry_centroid(camera.data.output["rgb"]) + first_z = cable.data.segment_pose_w.torch[..., 2].mean().item() + + for _ in range(TRACK_STEPS): + sim.step(render=False) + cable.update(cfg.dt) + camera.update(cfg.dt) + assert_mask_is_measuring_geometry(lit_mask(camera.data.output["rgb"]), _RENDERER) + last_lit = lit_pixel_count(camera.data.output["rgb"]) + last_centroid = geometry_centroid(camera.data.output["rgb"]) + last_z = cable.data.segment_pose_w.torch[..., 2].mean().item() + + assert_tracks(_RENDERER, first_lit, first_centroid, last_lit, last_centroid, first_z, last_z) + + +def test_cable_renders_across_environments(): + """Cables must still render and track once the scene is replicated across environments.""" + num_envs = 4 + spacing = 0.6 + eye = (spacing, -1.6 * spacing, 1.15) + target = (spacing, 0.0, 0.55) + + @configclass + class _CableSceneCfg(InteractiveSceneCfg): + cable = cable_cfg().replace(prim_path="{ENV_REGEX_NS}/Cable") + camera = camera_cfg(_renderer_cfg(), f"{ENV_NS}/MultiEnvCam", eye, target) + + cfg = sim_cfg() + with build_simulation_context(sim_cfg=cfg) as sim: + spawn_light() + scene = InteractiveScene(_CableSceneCfg(num_envs=num_envs, env_spacing=spacing)) + sim.reset() + cable = scene["cable"] + camera = scene["camera"] + + assert cable.num_instances == num_envs, ( + f"replication produced {cable.num_instances} cables, expected {num_envs}" + ) + + scene.update(cfg.dt) + camera.update(cfg.dt) + assert_mask_is_measuring_geometry(lit_mask(camera.data.output["rgb"]), _RENDERER) + first_lit = lit_pixel_count(camera.data.output["rgb"]) + first_centroid = geometry_centroid(camera.data.output["rgb"]) + first_z = cable.data.segment_pose_w.torch[..., 2].mean().item() + + for _ in range(TRACK_STEPS): + sim.step(render=False) + scene.update(cfg.dt) + camera.update(cfg.dt) + assert_mask_is_measuring_geometry(lit_mask(camera.data.output["rgb"]), _RENDERER) + last_lit = lit_pixel_count(camera.data.output["rgb"]) + last_centroid = geometry_centroid(camera.data.output["rgb"]) + last_z = cable.data.segment_pose_w.torch[..., 2].mean().item() + + assert_tracks(_RENDERER, first_lit, first_centroid, last_lit, last_centroid, first_z, last_z, envs=num_envs)