diff --git a/docs/source/overview/core-concepts/visualization.rst b/docs/source/overview/core-concepts/visualization.rst index 6b835ec1fa1e..00a91ce6aa03 100644 --- a/docs/source/overview/core-concepts/visualization.rst +++ b/docs/source/overview/core-concepts/visualization.rst @@ -463,6 +463,7 @@ Newton Visualizer - Lightweight OpenGL rendering with low overhead - Simulation and rendering pause controls +- Right-click rigid-body dragging with Newton rigid-body solvers - Adjustable update frequency for performance tuning - Some customizable rendering options (shadows, sky, wireframe) - Visualization markers (joints, contacts, springs, COM, debug markers) @@ -483,6 +484,8 @@ Newton Visualizer - Down / Up * - **Left Click + Drag** - Look around + * - **Right Click + Drag** + - Apply an interactive force to a dynamic Newton rigid body * - **Mouse Scroll** - Zoom in/out * - **H** @@ -523,6 +526,7 @@ Newton Visualizer show_contacts=False, # Show contact points and normals show_springs=False, # Show spring constraints show_com=False, # Show center of mass markers + enable_picking=True, # Enable Newton rigid-body dragging # Rendering options enable_shadows=True, # Enable shadow rendering @@ -535,6 +539,14 @@ Newton Visualizer light_color=(1.0, 1.0, 1.0), # Directional light color (RGB [0,1]) ) +.. note:: + + Object dragging requires an interactive Newton visualizer with a Newton + rigid-body solver (MJWarp, XPBD, VBD, Featherstone, or Kamino), either standalone + or in a supported coupled solver with a rigid-body entry. Static and + kinematic bodies and MPM particles are not moved. Picking is disabled + automatically for headless viewers, standalone MPM, and non-Newton physics. + Rerun Visualizer ~~~~~~~~~~~~~~~~ diff --git a/scripts/demos/assets/nvidia_logo_domino_poses.pth b/scripts/demos/assets/nvidia_logo_domino_poses.pth new file mode 100644 index 000000000000..7973b6fe1145 Binary files /dev/null and b/scripts/demos/assets/nvidia_logo_domino_poses.pth differ diff --git a/scripts/demos/mpm/newton_mpm_twoway_coupling.py b/scripts/demos/mpm/newton_mpm_twoway_coupling.py new file mode 100644 index 000000000000..b71bb3e6a676 --- /dev/null +++ b/scripts/demos/mpm/newton_mpm_twoway_coupling.py @@ -0,0 +1,306 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Drag three rigid spheres in a bath of Newton implicit-MPM sand. + +This Isaac Lab port of Newton's ``mpm_twoway_coupling`` example uses a proxy +coupler to expose dynamic rigid spheres as MPM colliders and feed the resulting +impulses back into the rigid-body solver. + +.. code-block:: bash + + uv run python scripts/demos/mpm/newton_mpm_twoway_coupling.py + +The spheres roll through three V-shaped chutes into the bath. Right-click and +drag any sphere to apply an interactive force. +""" + +from __future__ import annotations + +import argparse +from collections.abc import Callable +from functools import partial + +from isaaclab_visualizers.newton import NewtonGLVisualizerCfg + +from pxr import Gf, Usd, UsdGeom + +import isaaclab.sim as sim_utils +from isaaclab.app import add_launcher_args, launch_simulation + +parser = argparse.ArgumentParser(description="Newton rigid-sphere and MPM-sand two-way coupling demo.") +parser.add_argument("--max_steps", type=int, default=-1, help="Stop after this many frames; negative runs forever.") +parser.add_argument("--voxel_size", type=float, default=0.08, help="MPM grid voxel size [m].") +parser.add_argument("--rigid_substeps", type=int, default=4, help="Rigid-solver substeps per coupled step.") +add_launcher_args(parser) +parser.set_defaults(visualizer=["newton_gl"]) +args_cli = parser.parse_args() + + +FPS = 100.0 +GRAVITY = (0.0, 0.0, -9.81) +PARTICLES_PER_CELL = 2.0 +PARTICLE_COLOR = (0.7, 0.6, 0.4) +SPHERE_BODY_PATTERN = r"/World/envs/env_.*/Sphere_[0-9]+" +SPHERE_RADIUS = 0.30 +SPHERE_MASS = 450.0 +SPHERE_COLORS = ((0.20, 0.45, 0.85), (0.85, 0.25, 0.20), (0.25, 0.70, 0.30)) +SPHERE_POSITIONS = ((-3.392, 0.0, 2.167), (0.0, 2.872, 2.167), (3.392, 0.0, 2.167)) + +BATH_INTERIOR_SIZE = (3.6, 2.6) +BATH_WALL_HEIGHT = 1.2 +BATH_WALL_THICKNESS = 0.15 +SAND_LOWER = (-1.65, -1.15, 0.05) +SAND_UPPER = (1.65, 1.15, 0.72) + +CHUTE_PANEL_SIZE = (2.4, 0.62, 0.08) +CHUTE_COLOR = (0.25, 0.28, 0.32) +# Paired poses form the left, back, and right V-shaped chutes. +CHUTE_PANEL_POSES = ( + ((-2.933, -0.274, 1.771), (-0.23957, 0.13504, 0.03367, 0.96085)), + ((-2.933, 0.274, 1.771), (0.23957, 0.13504, -0.03367, 0.96085)), + ((-0.274, 2.413, 1.771), (-0.07391, 0.26489, -0.65562, 0.70323)), + ((0.274, 2.413, 1.771), (0.26489, -0.07391, -0.70323, 0.65562)), + ((2.933, 0.274, 1.771), (0.13504, 0.23957, -0.96085, 0.03367)), + ((2.933, -0.274, 1.771), (-0.13504, 0.23957, 0.96085, 0.03367)), +) + + +@sim_utils.clone +def _spawn_colored_shape( + prim_path: str, + cfg: sim_utils.SpawnerCfg, + translation: tuple[float, float, float] | None = None, + orientation: tuple[float, float, float, float] | None = None, + *, + spawn_func: Callable[..., Usd.Prim], + color: tuple[float, float, float], +) -> Usd.Prim: + """Spawn a shape with a display color understood by the Newton viewer.""" + prim = spawn_func(prim_path, cfg, translation, orientation) + mesh = UsdGeom.Gprim(prim.GetStage().GetPrimAtPath(f"{prim_path}/geometry/mesh")) + mesh.CreateDisplayColorAttr().Set([Gf.Vec3f(*color)]) + return prim + + +def create_visualizer_cfgs(): + """Create the demo-specific Newton visualizer configuration.""" + if not {"newton", "newton_gl"}.intersection(args_cli.visualizer or []): + return [] + + return [ + NewtonGLVisualizerCfg( + streaming_view=False, + show_particles=True, + particle_color=PARTICLE_COLOR, + update_frequency=1, + ) + ] + + +def create_sim_cfg(): + """Create the proxy-coupled MJWarp and MPM simulation configuration.""" + from isaaclab_newton.physics import MJWarpSolverCfg, MPMSolverCfg, NewtonCfg + + from isaaclab_contrib.coupling import CouplerEntryCfg, CouplerProxyCfg, CouplerProxyMappingCfg + + solver_cfg = CouplerProxyCfg( + entries=[ + CouplerEntryCfg( + name="rigid", + solver_cfg=MJWarpSolverCfg(use_mujoco_contacts=False, njmax=128), + bodies=[SPHERE_BODY_PATTERN], + include_static_shapes=True, + substeps=args_cli.rigid_substeps, + ), + CouplerEntryCfg( + name="mpm", + solver_cfg=MPMSolverCfg( + voxel_size=args_cli.voxel_size, + grid_type="fixed", + grid_padding=50, + max_active_cell_count=1 << 16, + strain_basis="P0", + max_iterations=50, + critical_fraction=0.0, + ), + all_particles=True, + in_place=True, + ), + ], + proxies=[ + CouplerProxyMappingCfg( + source="rigid", + destination="mpm", + bodies=[SPHERE_BODY_PATTERN], + mode="lagged", + collision_pipeline=None, + ) + ], + iterations=1, + ) + return sim_utils.SimulationCfg( + dt=1.0 / FPS, + device=args_cli.device, + gravity=GRAVITY, + visualizer_cfgs=create_visualizer_cfgs(), + physics=NewtonCfg(solver_cfg=solver_cfg), + ) + + +def create_scene_cfg(): + """Create the declarative rigid-sphere and granular-bath scene.""" + from isaaclab_newton.assets.mpm_object import MPMObjectCfg + from isaaclab_newton.sim.spawners.mpm import MPMGridCfg, MPMParticleMaterialCfg + + from isaaclab.assets import AssetBaseCfg, RigidObjectCfg, RigidObjectCollectionCfg + from isaaclab.scene import InteractiveSceneCfg + from isaaclab.utils.configclass import configclass + + def bath_collider( + prim_path: str, + size: tuple[float, float, float], + position: tuple[float, float, float], + orientation: tuple[float, float, float, float] = (0.0, 0.0, 0.0, 1.0), + color: tuple[float, float, float] | None = None, + ) -> AssetBaseCfg: + return AssetBaseCfg( + prim_path=prim_path, + spawn=sim_utils.CuboidCfg( + func=( + partial(_spawn_colored_shape, spawn_func=sim_utils.spawn_cuboid, color=color) + if color is not None + else sim_utils.spawn_cuboid + ), + size=size, + collision_props=sim_utils.NewtonCollisionPropertiesCfg(contact_margin=0.04), + physics_material=sim_utils.NewtonMaterialPropertiesCfg( + static_friction=0.6, + dynamic_friction=0.6, + ), + ), + init_state=AssetBaseCfg.InitialStateCfg(pos=position, rot=orientation), + ) + + rigid_objects = {} + for index, position in enumerate(SPHERE_POSITIONS): + rigid_objects[f"sphere_{index}"] = RigidObjectCfg( + prim_path=f"{{ENV_REGEX_NS}}/Sphere_{index}", + spawn=sim_utils.SphereCfg( + func=partial( + _spawn_colored_shape, + spawn_func=sim_utils.spawn_sphere, + color=SPHERE_COLORS[index], + ), + radius=SPHERE_RADIUS, + rigid_props=sim_utils.RigidBodyPropertiesCfg(), + mass_props=sim_utils.MassPropertiesCfg(mass=SPHERE_MASS), + collision_props=sim_utils.NewtonCollisionPropertiesCfg(), + physics_material=sim_utils.NewtonMaterialPropertiesCfg( + static_friction=0.5, + dynamic_friction=0.5, + ), + ), + init_state=RigidObjectCfg.InitialStateCfg(pos=position), + ) + + bath_x, bath_y = BATH_INTERIOR_SIZE + wall_t = BATH_WALL_THICKNESS + wall_z = 0.5 * BATH_WALL_HEIGHT + chute_panels = tuple( + bath_collider( + f"/World/Chutes/Panel_{index}", + CHUTE_PANEL_SIZE, + position, + orientation, + CHUTE_COLOR, + ) + for index, (position, orientation) in enumerate(CHUTE_PANEL_POSES) + ) + + @configclass + class CoupledSceneCfg(InteractiveSceneCfg): + """Scene containing a static bath, three rigid spheres, and MPM sand.""" + + bath_floor = bath_collider( + "/World/Bath/Floor", + (bath_x + 2.0 * wall_t, bath_y + 2.0 * wall_t, wall_t), + (0.0, 0.0, -0.5 * wall_t), + ) + bath_left = bath_collider( + "/World/Bath/LeftWall", + (wall_t, bath_y, BATH_WALL_HEIGHT), + (-0.5 * (bath_x + wall_t), 0.0, wall_z), + ) + bath_right = bath_collider( + "/World/Bath/RightWall", + (wall_t, bath_y, BATH_WALL_HEIGHT), + (0.5 * (bath_x + wall_t), 0.0, wall_z), + ) + bath_front = bath_collider( + "/World/Bath/FrontWall", + (bath_x + 2.0 * wall_t, wall_t, BATH_WALL_HEIGHT), + (0.0, -0.5 * (bath_y + wall_t), wall_z), + ) + bath_back = bath_collider( + "/World/Bath/BackWall", + (bath_x + 2.0 * wall_t, wall_t, BATH_WALL_HEIGHT), + (0.0, 0.5 * (bath_y + wall_t), wall_z), + ) + + chute_left_a, chute_left_b, chute_back_a, chute_back_b, chute_right_a, chute_right_b = chute_panels + + spheres = RigidObjectCollectionCfg(rigid_objects=rigid_objects) + + sand = MPMObjectCfg( + prim_path="{ENV_REGEX_NS}/Sand", + spawn=MPMGridCfg( + lower=SAND_LOWER, + upper=SAND_UPPER, + voxel_size=args_cli.voxel_size, + particles_per_cell=PARTICLES_PER_CELL, + jitter=args_cli.voxel_size / PARTICLES_PER_CELL, + material=MPMParticleMaterialCfg(density=2500.0, friction=0.5, yield_pressure=1.0e5), + visual_color=PARTICLE_COLOR, + ), + ) + + return CoupledSceneCfg(num_envs=1, env_spacing=0.0) + + +def run_simulator(sim, scene) -> None: + """Run until the viewer closes or the optional step limit is reached.""" + sim_dt = sim.get_physics_dt() + step_count = 0 + while sim.is_headless_or_exist_active_visualizer() and (args_cli.max_steps < 0 or step_count < args_cli.max_steps): + sim.step(render=False) + scene.update(sim_dt) + if sim.is_rendering: + sim.render() + step_count += 1 + + +def main() -> None: + """Launch the two-way rigid-MPM coupling demo.""" + sim_cfg = create_sim_cfg() + with launch_simulation(sim_cfg, args_cli): + from isaaclab.scene import InteractiveScene + + sim = sim_utils.SimulationContext(sim_cfg) + sim.set_camera_view(eye=(6.0, -7.0, 5.0), target=(0.0, 0.4, 1.3)) + scene = InteractiveScene(create_scene_cfg()) + sim.reset() + sand = scene["sand"] + particle_count = sand.num_instances * sand.particles_per_object + print( + f"[INFO]: Isaac Lab Newton two-way MPM demo ready. Spawned {particle_count} particles.", + flush=True, + ) + print("[INFO]: Right-click and drag any sphere in the Newton viewer.", flush=True) + run_simulator(sim, scene) + + +if __name__ == "__main__": + main() diff --git a/scripts/demos/newton_viewer_block_and_tackle.py b/scripts/demos/newton_viewer_block_and_tackle.py new file mode 100644 index 000000000000..5ce4ec1acd3e --- /dev/null +++ b/scripts/demos/newton_viewer_block_and_tackle.py @@ -0,0 +1,332 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Drag a cable handle to lift a load through a single 4:1 pulley system. + +.. code-block:: bash + + uv run python scripts/demos/newton_viewer_block_and_tackle.py +""" + +import argparse +import math + +from isaaclab.app import add_launcher_args, launch_simulation + +parser = argparse.ArgumentParser(description="Newton block-and-tackle viewer dragging demo.") +parser.add_argument("--max_steps", type=int, default=-1, help="Stop after this many steps; negative runs forever.") +add_launcher_args(parser) +parser.set_defaults(visualizer=["newton_gl"]) +args_cli = parser.parse_args() + +import newton +import newton.utils +import warp as wp +from isaaclab_newton.physics import NewtonCfg, NewtonManager, NewtonShapeCfg + +import isaaclab.sim as sim_utils +from isaaclab.utils.configclass import configclass + +from isaaclab_contrib.deformable import VBDSolverCfg + +MECHANICAL_ADVANTAGE = 4 +LOAD_MASS = 5.0 +HANDLE_MASS = 0.5 # Compensated in the load mass; raises Newton's picking-force limit. + +CABLE_RADIUS = 0.004 +CABLE_SEGMENT_LENGTH = 0.02 +CABLE_GAP = 0.5 * CABLE_RADIUS +PULLEY_RADIUS = 0.045 +WRAP_RADIUS = PULLEY_RADIUS + 1.25 * CABLE_RADIUS +SHEAVE_SPACING = 5.5 * CABLE_RADIUS + +BLOCK_X = 0.50 +MOVING_Z = 0.72 +FIXED_Z = 1.10 +LEAD_X = BLOCK_X + 3.0 * WRAP_RADIUS +PULL_X = LEAD_X + WRAP_RADIUS +LOAD_CENTER = wp.vec3(BLOCK_X, 0.0, 0.49) +HANDLE_HALF_EXTENTS = (0.025, 0.025, 0.03) + + +@configclass +class _BlockAndTackleVBDSolverCfg(VBDSolverCfg): + """VBD contact settings for this cable and pulley scene.""" + + rigid_contact_hard: bool = False + rigid_body_contact_buffer_size: int = 512 + + +def _append_arc(points: list[wp.vec3], center: wp.vec3, start: float, end: float) -> None: + """Append a clockwise pulley arc in the vertical XZ plane.""" + delta = (end - start + math.pi) % (2.0 * math.pi) - math.pi + if delta > 0.0: + delta -= 2.0 * math.pi + count = max(3, math.ceil(abs(delta) * WRAP_RADIUS / CABLE_SEGMENT_LENGTH)) + for index in range(count + 1): + angle = start + delta * index / count + point = wp.vec3( + float(center[0]) + WRAP_RADIUS * math.cos(angle), + float(center[1]), + float(center[2]) + WRAP_RADIUS * math.sin(angle), + ) + if float(wp.length(point - points[-1])) > 1.0e-8: + points.append(point) + + +def _resample_route(points: list[wp.vec3]) -> tuple[list[wp.vec3], float]: + """Resample a route into equal-length cable segments.""" + lengths = [0.0] + for start, end in zip(points, points[1:]): + lengths.append(lengths[-1] + float(wp.length(end - start))) + segment_count = math.ceil(lengths[-1] / CABLE_SEGMENT_LENGTH) + spacing = lengths[-1] / segment_count + result = [points[0]] + point_index = 1 + for segment_index in range(1, segment_count): + distance = spacing * segment_index + while lengths[point_index] < distance: + point_index += 1 + alpha = (distance - lengths[point_index - 1]) / (lengths[point_index] - lengths[point_index - 1]) + result.append(points[point_index - 1] * (1.0 - alpha) + points[point_index] * alpha) + result.append(points[-1]) + return result, spacing + + +def _cable_route( + moving_centers: list[wp.vec3], fixed_centers: list[wp.vec3], lead_center: wp.vec3 +) -> tuple[list[wp.vec3], float]: + """Route one cable through the moving and fixed pulley blocks.""" + points = [ + wp.vec3( + float(moving_centers[0][0]) + WRAP_RADIUS, + float(moving_centers[0][1]), + float(fixed_centers[0][2]) - 2.0 * WRAP_RADIUS, + ) + ] + for index, (moving_center, fixed_center) in enumerate(zip(moving_centers, fixed_centers, strict=True)): + _append_arc(points, moving_center, 0.0, -math.pi) + _append_arc(points, fixed_center, math.pi, 0.0 if index == 0 else 0.5 * math.pi) + _append_arc(points, lead_center, 0.5 * math.pi, 0.0) + points.append(wp.vec3(PULL_X, float(lead_center[1]), 0.50)) + return _resample_route(points) + + +def _add_box( + builder: newton.ModelBuilder, + body: int, + center: wp.vec3, + half_extents: tuple[float, float, float], + color: tuple[float, float, float], + *, + density: float = 0.0, + collision: bool = False, +) -> None: + """Add a compact visual or colliding box.""" + builder.add_shape_box( + body=body, + xform=wp.transform(center, wp.quat_identity()), + hx=half_extents[0], + hy=half_extents[1], + hz=half_extents[2], + cfg=newton.ModelBuilder.ShapeConfig( + density=density, + ke=1.0e5, + kd=20.0, + mu=0.8, + has_shape_collision=collision, + has_particle_collision=collision, + ), + color=color, + ) + + +def _add_pulley( + builder: newton.ModelBuilder, + center: wp.vec3, + parent: int, + parent_origin: wp.vec3, + color: tuple[float, float, float], +) -> int: + """Add one passive grooved sheave and return its revolute joint.""" + body = builder.add_link(xform=wp.transform(center, wp.quat_identity())) + joint = builder.add_joint_revolute( + parent=parent, + child=body, + axis=wp.vec3(0.0, 1.0, 0.0), + parent_xform=wp.transform(center - parent_origin, wp.quat_identity()), + armature=1.0e-4, + friction=0.0, + ) + align_to_y = wp.quat_from_axis_angle(wp.vec3(1.0, 0.0, 0.0), -0.5 * math.pi) + groove_half_width = 1.55 * CABLE_RADIUS + flange_half_width = 0.6 * CABLE_RADIUS + for y, radius, half_width, shade, friction in ( + (0.0, PULLEY_RADIUS, groove_half_width, color, 0.1), + ( + -(groove_half_width + flange_half_width), + PULLEY_RADIUS + 3.2 * CABLE_RADIUS, + flange_half_width, + tuple(0.68 * value for value in color), + 0.05, + ), + ( + groove_half_width + flange_half_width, + PULLEY_RADIUS + 3.2 * CABLE_RADIUS, + flange_half_width, + tuple(0.68 * value for value in color), + 0.05, + ), + ): + builder.add_shape_cylinder( + body=body, + xform=wp.transform(wp.vec3(0.0, y, 0.0), align_to_y), + radius=radius, + half_height=half_width, + cfg=newton.ModelBuilder.ShapeConfig(density=0.1, ke=1.0e5, kd=0.0, mu=friction), + color=shade, + ) + return joint + + +def _build_system(builder: newton.ModelBuilder) -> tuple[int, list[wp.transform]]: + """Build one complete 4:1 block-and-tackle system.""" + sheave_y = (-0.5 * SHEAVE_SPACING, 0.5 * SHEAVE_SPACING) + moving_centers = [wp.vec3(BLOCK_X, y, MOVING_Z) for y in sheave_y] + fixed_centers = [wp.vec3(BLOCK_X, y, FIXED_Z) for y in sheave_y] + lead_center = wp.vec3(LEAD_X, sheave_y[-1], FIXED_Z) + + _add_box(builder, -1, wp.vec3(0.5 * (BLOCK_X + PULL_X), 0.0, 1.19), (0.18, 0.07, 0.025), (0.16, 0.22, 0.30)) + _add_box(builder, -1, wp.vec3(BLOCK_X, 0.0, 1.145), (0.025, 0.065, 0.045), (0.16, 0.22, 0.30)) + _add_box(builder, -1, wp.vec3(LEAD_X, sheave_y[-1], 1.145), (0.025, 0.025, 0.045), (0.16, 0.22, 0.30)) + _add_box(builder, -1, wp.vec3(BLOCK_X, 0.0, 0.39), (0.14, 0.12, 0.01), (0.24, 0.27, 0.30), collision=True) + + load_body = builder.add_link(xform=wp.transform(LOAD_CENTER, wp.quat_identity()), label="load") + load_half_extents = (0.085, 0.08, 0.09) + load_volume = 8.0 * math.prod(load_half_extents) + _add_box( + builder, + load_body, + wp.vec3(0.0), + load_half_extents, + (0.72, 0.16, 0.12), + density=(LOAD_MASS + HANDLE_MASS * MECHANICAL_ADVANTAGE) / load_volume, + collision=True, + ) + _add_box(builder, load_body, wp.vec3(0.0, 0.0, 0.16), (0.025, 0.065, 0.07), (0.50, 0.18, 0.12)) + load_joint = builder.add_joint_prismatic( + parent=-1, + child=load_body, + axis=wp.vec3(0.0, 0.0, 1.0), + parent_xform=wp.transform(LOAD_CENTER, wp.quat_identity()), + target_kd=20.0, + ) + + moving_joints = [ + _add_pulley(builder, center, load_body, LOAD_CENTER, (0.88, 0.54, 0.12)) for center in moving_centers + ] + fixed_joints = [_add_pulley(builder, center, -1, wp.vec3(0.0), (0.12, 0.38, 0.78)) for center in fixed_centers] + lead_joint = _add_pulley(builder, lead_center, -1, wp.vec3(0.0), (0.12, 0.38, 0.78)) + builder.add_articulation([load_joint, *moving_joints], label="moving_block") + for joint in [*fixed_joints, lead_joint]: + builder.add_articulation([joint]) + + route, segment_length = _cable_route(moving_centers, fixed_centers, lead_center) + route_quaternions = newton.utils.create_parallel_transport_cable_quaternions(route) + cable_segment_count = len(route) - 1 + straight_points, straight_quaternions = newton.utils.create_straight_cable_points_and_quaternions( + start=route[0], + direction=wp.vec3(1.0, 0.0, 0.0), + length=cable_segment_count * segment_length, + num_segments=cable_segment_count, + ) + cable_bodies, cable_joints = builder.add_rod( + positions=straight_points, + quaternions=straight_quaternions, + radius=CABLE_RADIUS, + body_frame_origin="com", + cfg=newton.ModelBuilder.ShapeConfig(density=10.0, ke=1.0e5, kd=0.0, mu=0.1, gap=CABLE_GAP), + wrap_in_articulation=False, + color=(0.82, 0.72, 0.46), + label="block_and_tackle_cable", + ) + endpoint = 0.5 * segment_length + anchor_joint = builder.add_joint_ball( + parent=-1, + child=cable_bodies[0], + parent_xform=wp.transform(route[0], wp.quat_identity()), + child_xform=wp.transform(wp.vec3(0.0, 0.0, -endpoint), wp.quat_identity()), + label="cable_anchor", + ) + handle_volume = 8.0 * math.prod(HANDLE_HALF_EXTENTS) + _add_box( + builder, + cable_bodies[-1], + wp.vec3(0.0, 0.0, endpoint + HANDLE_HALF_EXTENTS[2]), + HANDLE_HALF_EXTENTS, + (0.96, 0.78, 0.26), + density=HANDLE_MASS / handle_volume, + collision=True, + ) + builder.add_articulation([*cable_joints, anchor_joint], label="cable") + + for index, body_a in enumerate(cable_bodies): + for body_b in cable_bodies[index + 1 :]: + for shape_a in builder.body_shapes[body_a]: + for shape_b in builder.body_shapes[body_b]: + builder.add_shape_collision_filter_pair(shape_a, shape_b) + + wrapped_xforms = [ + wp.transform(route[index] + 0.5 * (route[index + 1] - route[index]), route_quaternions[index]) + for index in range(cable_segment_count) + ] + return cable_bodies[0], wrapped_xforms + + +def _initialize_wrapped_cable(cable_body_start: int, wrapped_xforms: list[wp.transform]) -> None: + """Restore model defaults and wrap the structurally straight cable.""" + model = NewtonManager.get_model() + wrapped = wp.array(wrapped_xforms, dtype=wp.transform, device=model.device) + for state in (NewtonManager.get_state_0(), NewtonManager.get_state_1()): + wp.copy(state.body_q, model.body_q) + wp.copy(state.body_qd, model.body_qd) + wp.copy(state.body_q, wrapped, dest_offset=cable_body_start, count=len(wrapped_xforms)) + state.body_f.zero_() + NewtonManager._solver.reset(NewtonManager.get_state_0(), flags=0) + + +def run_simulator(sim: sim_utils.SimulationContext) -> None: + """Run until the viewer closes or the optional step limit is reached.""" + step_count = 0 + while sim.is_headless_or_exist_active_visualizer() and (args_cli.max_steps < 0 or step_count < args_cli.max_steps): + sim.step() + step_count += 1 + + +def main() -> None: + """Launch the block-and-tackle dragging demo.""" + physics_cfg = NewtonCfg( + num_substeps=16, + collision_decimation=1, + default_shape_cfg=NewtonShapeCfg(gap=CABLE_GAP, ke=1.0e5, kd=20.0, mu=0.5), + solver_cfg=_BlockAndTackleVBDSolverCfg(), + ) + with launch_simulation(cfg=physics_cfg, launcher_args=args_cli) as resolved_physics_cfg: + sim_cfg = sim_utils.SimulationCfg(dt=1.0 / 60.0, device=args_cli.device, physics=resolved_physics_cfg) + sim = sim_utils.SimulationContext(sim_cfg) + sim.set_camera_view(eye=(1.35, -2.1, 1.25), target=(0.50, 0.0, 0.72)) + builder = NewtonManager.create_builder() + builder.rigid_gap = CABLE_GAP + cable_body_start, wrapped_xforms = _build_system(builder) + builder.color(balance_colors=False) + NewtonManager.set_builder(builder) + sim.reset() + _initialize_wrapped_cable(cable_body_start, wrapped_xforms) + print("[INFO]: Setup complete. Right-drag the yellow cable handle downward to lift the red load.", flush=True) + run_simulator(sim) + + +if __name__ == "__main__": + main() diff --git a/scripts/demos/newton_viewer_dominoes.py b/scripts/demos/newton_viewer_dominoes.py new file mode 100644 index 000000000000..e0941ceb1fdb --- /dev/null +++ b/scripts/demos/newton_viewer_dominoes.py @@ -0,0 +1,163 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Topple an NVIDIA-logo domino layout with Newton viewer dragging. + +Right-click and drag the black trigger slab on the right to start the cascade +across all rows. + +.. code-block:: bash + + uv run python scripts/demos/newton_viewer_dominoes.py +""" + +import argparse +from pathlib import Path + +from isaaclab.app import add_launcher_args, launch_simulation + +parser = argparse.ArgumentParser(description="NVIDIA-logo domino dragging demo (XPBD).") +parser.add_argument("--max_steps", type=int, default=-1, help="Stop after this many steps; negative runs forever.") +add_launcher_args(parser) +parser.set_defaults(visualizer=["newton_gl"]) +args_cli = parser.parse_args() + +import torch +from isaaclab_newton.physics import NewtonCfg, NewtonManager, NewtonShapeCfg, XPBDSolverCfg + +from pxr import Gf, UsdGeom + +import isaaclab.sim as sim_utils +from isaaclab.assets import AssetBaseCfg, RigidObjectCfg, RigidObjectCollectionCfg +from isaaclab.scene import InteractiveScene, InteractiveSceneCfg +from isaaclab.utils.configclass import configclass + +DOMINO_SIZE = (0.12, 0.032, 0.36) +DOMINO_SPACING = 0.12 +LOGO_FOOTPRINT = (29.4, 8.4) +NVIDIA_GREEN = (0.24, 0.50, 0.0) + +_POSES_PATH = Path(__file__).with_name("assets") / "nvidia_logo_domino_poses.pth" +_POSES = torch.load(_POSES_PATH, map_location="cpu", weights_only=True).tolist() +LOGO_DOMINO_POSES = [(tuple(pose[:3]), tuple(pose[3:])) for pose in _POSES] + + +def _set_display_color(prim_path: str, color: tuple[float, float, float]) -> None: + """Set a mesh display color for the Newton model builder.""" + mesh = sim_utils.get_current_stage().GetPrimAtPath(f"{prim_path}/geometry/mesh") + UsdGeom.Gprim(mesh).CreateDisplayColorAttr().Set([Gf.Vec3f(*color)]) + + +def _domino_cfg(position: tuple[float, float, float], orientation: tuple[float, float, float, float]) -> RigidObjectCfg: + """Return one domino at a saved pose.""" + return RigidObjectCfg( + prim_path="", + spawn=sim_utils.CuboidCfg( + size=DOMINO_SIZE, + rigid_props=sim_utils.RigidBodyPropertiesCfg(), + mass_props=sim_utils.MassPropertiesCfg(density=580.0), + collision_props=sim_utils.CollisionPropertiesCfg(), + physics_material=sim_utils.RigidBodyMaterialCfg( + static_friction=1.0, + dynamic_friction=1.0, + restitution=0.15, + ), + ), + init_state=RigidObjectCfg.InitialStateCfg(pos=position, rot=orientation), + ) + + +def _trigger_cfg() -> RigidObjectCfg: + """Place a low-mass black trigger slab across every row on the right.""" + right_edge = max(position[0] for position, _ in LOGO_DOMINO_POSES) + return RigidObjectCfg( + prim_path="/World/Dominoes/Trigger", + spawn=sim_utils.CuboidCfg( + size=(DOMINO_SIZE[1], LOGO_FOOTPRINT[1], DOMINO_SIZE[2]), + rigid_props=sim_utils.RigidBodyPropertiesCfg(), + mass_props=sim_utils.MassPropertiesCfg(density=20.0), + collision_props=sim_utils.CollisionPropertiesCfg(), + physics_material=sim_utils.RigidBodyMaterialCfg( + static_friction=1.0, + dynamic_friction=1.0, + restitution=0.15, + ), + ), + init_state=RigidObjectCfg.InitialStateCfg(pos=(right_edge + DOMINO_SPACING, 0.0, DOMINO_SIZE[2] / 2.0)), + ) + + +@configclass +class DominoSceneCfg(InteractiveSceneCfg): + """White floor, saved domino poses, and the trigger slab.""" + + floor: AssetBaseCfg = AssetBaseCfg( + prim_path="/World/Floor", + spawn=sim_utils.CuboidCfg( + size=(LOGO_FOOTPRINT[0] + 4.0, LOGO_FOOTPRINT[1] + 4.0, 0.10), + collision_props=sim_utils.CollisionPropertiesCfg(), + physics_material=sim_utils.RigidBodyMaterialCfg( + static_friction=1.0, + dynamic_friction=1.0, + restitution=0.15, + ), + ), + init_state=AssetBaseCfg.InitialStateCfg(pos=(0.0, 0.0, -0.05)), + ) + dominoes: RigidObjectCollectionCfg = RigidObjectCollectionCfg( + rigid_objects={ + f"domino_{index:04d}": _domino_cfg(position, orientation).replace( + prim_path=f"/World/Dominoes/Domino{index:04d}" + ) + for index, (position, orientation) in enumerate(LOGO_DOMINO_POSES) + } + ) + trigger: RigidObjectCfg = _trigger_cfg() + + +def _apply_display_colors() -> None: + """Apply viewer colors after InteractiveScene has authored the prims.""" + _set_display_color("/World/Floor", (1.0, 1.0, 1.0)) + for index in range(len(LOGO_DOMINO_POSES)): + _set_display_color(f"/World/Dominoes/Domino{index:04d}", NVIDIA_GREEN) + _set_display_color("/World/Dominoes/Trigger", (0.02, 0.02, 0.02)) + + +def run_simulator(sim: sim_utils.SimulationContext) -> None: + """Run until the viewer closes or the optional step limit is reached.""" + step_count = 0 + while sim.is_headless_or_exist_active_visualizer() and (args_cli.max_steps < 0 or step_count < args_cli.max_steps): + sim.step() + step_count += 1 + + +def main() -> None: + """Launch the Newton XPBD domino dragging demo.""" + physics_cfg = NewtonCfg( + num_substeps=10, + collision_decimation=1, + default_shape_cfg=NewtonShapeCfg(gap=0.001, ke=1.0e4, kd=0.0, mu=1.0), + solver_cfg=XPBDSolverCfg(iterations=20, enable_restitution=True), + ) + with launch_simulation(cfg=physics_cfg, launcher_args=args_cli) as resolved_physics_cfg: + sim_cfg = sim_utils.SimulationCfg(dt=1.0 / 120.0, device=args_cli.device, physics=resolved_physics_cfg) + sim = sim_utils.SimulationContext(sim_cfg) + sim.set_camera_view(eye=(0.0, -18.0, 15.0), target=(0.0, 0.0, 0.0)) + _scene = InteractiveScene(DominoSceneCfg(num_envs=1, env_spacing=1.0)) + _apply_display_colors() + if NewtonManager._builder is None: + NewtonManager.instantiate_builder_from_stage() + NewtonManager._builder.rigid_gap = 0.001 + sim.reset() + print( + f"[INFO]: Setup complete with {len(LOGO_DOMINO_POSES)} green dominoes. " + "Right-click and drag the black trigger slab on the right to topple the logo.", + flush=True, + ) + run_simulator(sim) + + +if __name__ == "__main__": + main() diff --git a/source/isaaclab/changelog.d/max-newton-viewer-dragging.minor.rst b/source/isaaclab/changelog.d/max-newton-viewer-dragging.minor.rst new file mode 100644 index 000000000000..90bb1e473b2c --- /dev/null +++ b/source/isaaclab/changelog.d/max-newton-viewer-dragging.minor.rst @@ -0,0 +1,4 @@ +Added +^^^^^ + +* Added XPBD domino, VBD block-and-tackle, and coupled rigid-box/MPM demos for Newton viewer dragging. diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index 380ef2fc1a3e..de1d3fc019d4 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -190,6 +190,7 @@ def __init__(self, cfg: SimulationCfg | None = None): # Initialize visualizer state (visualizers are created lazily during initialize_visualizers()). self._scene_data_provider = SceneDataProvider(self.physics_manager.get_scene_data_backend()) self._visualizers: list[BaseVisualizer] = [] + self._pending_visualizer_cfgs: list[Any] | None = None self._reset_requested: bool = False self._scene_data_requirements = SceneDataRequirement() # Clone plan published by InteractiveScene after cloning. Providers (e.g. the @@ -601,32 +602,58 @@ def _resolve_visualizer_cfgs(self) -> list[Any]: return resolved def initialize_visualizers(self) -> None: - """Initialize visualizers from SimulationCfg.visualizer_cfgs.""" - if self._visualizers: + """Initialize visualizers from ``SimulationCfg.visualizer_cfgs``.""" + if self._pending_visualizer_cfgs == [] or (self._pending_visualizer_cfgs is None and self._visualizers): return + visualizer_cfgs = self._get_visualizer_cfgs() + if not visualizer_cfgs: + return + + self._initialize_visualizers() + + if not self._visualizers and self._scene_data_provider is not None: + close_provider = getattr(self._scene_data_provider, "close", None) + if callable(close_provider): + close_provider() + self._scene_data_provider = None + + def _get_visualizer_cfgs(self) -> list[Any]: + """Resolve visualizer configs for the current initialization cycle.""" + if self._pending_visualizer_cfgs is None: + self._pending_visualizer_cfgs = self._resolve_visualizer_cfgs() + return self._pending_visualizer_cfgs + + def _initialize_visualizers(self, config_filter: Callable[[Any], bool] | None = None) -> None: + """Initialize pending visualizers, optionally restricted by config.""" physics_dt = getattr(self.cfg.physics, "dt", None) self._viz_dt = (physics_dt if physics_dt is not None else self.cfg.dt) * self.cfg.render_interval - visualizer_cfgs = self._resolve_visualizer_cfgs() + visualizer_cfgs = self._get_visualizer_cfgs() if not visualizer_cfgs: return cli_explicit = self._is_cli_visualizer_explicit() # Resolve visualizer-driven requirements once and keep optional artifact payload untouched. + all_visualizer_cfgs = [viz.cfg for viz in self._visualizers] + visualizer_cfgs visualizer_types = [ - cfg.visualizer_type for cfg in visualizer_cfgs if getattr(cfg, "visualizer_type", None) is not None + cfg.visualizer_type for cfg in all_visualizer_cfgs if getattr(cfg, "visualizer_type", None) is not None ] requirements = resolve_scene_data_requirements(visualizer_types=visualizer_types) self._scene_data_requirements = requirements - self._visualizers = [] + pending_cfgs = [] + new_visualizers = [] for cfg in visualizer_cfgs: + if config_filter is not None and not config_filter(cfg): + pending_cfgs.append(cfg) + continue try: visualizer = cfg.create_visualizer() visualizer.initialize(self._scene_data_provider) self._visualizers.append(visualizer) + new_visualizers.append(visualizer) except Exception as exc: if cli_explicit: raise RuntimeError( @@ -639,20 +666,16 @@ def initialize_visualizers(self) -> None: type(cfg).__name__, exc, ) + self._pending_visualizer_cfgs = pending_cfgs # Replay any camera pose requested before visualizers were initialized. pending = getattr(self, "_pending_camera_view", None) if pending is not None: eye, target = pending - for viz in self._visualizers: + for viz in new_visualizers: viz.set_camera_view(eye, target) - self._pending_camera_view = None - - if not self._visualizers and self._scene_data_provider is not None: - close_provider = getattr(self._scene_data_provider, "close", None) - if callable(close_provider): - close_provider() - self._scene_data_provider = None + if not pending_cfgs: + self._pending_camera_view = None def get_scene_data_provider(self) -> SceneDataProvider: return self._scene_data_provider @@ -725,6 +748,23 @@ def forward(self) -> None: """Update kinematics without stepping physics.""" self.physics_manager.forward() + def _prepare_newton_visualizer_for_capture(self, _payload=None) -> None: + """Initialize or rebind the Newton viewer before solver graph capture.""" + # Picking applies forces inside solver substeps, so its kernels and buffers + # must exist during graph capture. Render-only viewers can initialize later. + self._initialize_visualizers(self._requires_pre_capture_newton_init) + for viz in (viz for viz in self._visualizers if self._requires_pre_capture_newton_init(viz.cfg)): + viz.reset(soft=False) + + @staticmethod + def _requires_pre_capture_newton_init(cfg: Any) -> bool: + """Return whether a config contributes Newton picking inputs to capture.""" + return ( + getattr(cfg, "visualizer_type", None) in {"newton_gl", "newton_rtx"} + and bool(getattr(cfg, "enable_picking", False)) + and not bool(getattr(cfg, "headless", False)) + ) + def reset(self, soft: bool = False) -> None: """Reset the simulation. @@ -734,9 +774,8 @@ def reset(self, soft: bool = False) -> None: self.physics_manager.reset(soft) for viz in self._visualizers: viz.reset(soft) - if not self._visualizers: - # Initialize visualizers after PhysX sim views are ready, but before play() pumps timeline events. - self.initialize_visualizers() + # Initialize visualizers not prepared by a backend-specific pre-capture hook. + self.initialize_visualizers() # Start the timeline so the play button is pressed self.physics_manager.play() self._is_playing = True @@ -847,6 +886,8 @@ def update_visualizers(self, dt: float, skip_app_pumping: bool = False) -> None: logger.info("Removed visualizer: %s", type(viz).__name__) except Exception as exc: logger.error("Error closing visualizer: %s", exc) + if visualizers_to_remove and not self._visualizers: + self._pending_visualizer_cfgs = None def _should_forward_before_visualizer_update(self) -> bool: """Return True if any visualizer requires pre-step forward kinematics.""" diff --git a/source/isaaclab/test/app/standalone_script_cases.py b/source/isaaclab/test/app/standalone_script_cases.py index 1288994467af..ce53c529c5b7 100644 --- a/source/isaaclab/test/app/standalone_script_cases.py +++ b/source/isaaclab/test/app/standalone_script_cases.py @@ -159,12 +159,30 @@ class SmokeResult: readiness_pattern=r"Newton granular MPM demo ready", fixed_physics_backend="newton_mpm", ), + "scripts/demos/mpm/newton_mpm_twoway_coupling.py": ScriptOverride( + args=("--max_steps", "2", "--voxel_size", "0.2"), + readiness_pattern=r"Newton two-way MPM demo ready", + fixed_physics_backend="newton_coupler", + visualizers=("newton_gl",), + required_modules=("isaaclab_contrib",), + ), "scripts/demos/mpm/particle_pour.py": ScriptOverride( args=("--max-steps", "200"), readiness_pattern=r"particle-pour MPM demo ready", fixed_physics_backend="newton_mpm", ), "scripts/demos/multi_asset.py": ScriptOverride(args=("--num_envs", "4")), + "scripts/demos/newton_viewer_block_and_tackle.py": ScriptOverride( + args=("--max_steps", "20"), + fixed_physics_backend="newton_vbd", + visualizers=("newton_gl",), + required_modules=("isaaclab_contrib",), + ), + "scripts/demos/newton_viewer_dominoes.py": ScriptOverride( + args=("--max_steps", "20"), + fixed_physics_backend="newton_xpbd", + visualizers=("newton_gl",), + ), "scripts/demos/sensors/cameras.py": ScriptOverride(args=("--num_envs", "1"), startup_timeout=600.0), "scripts/demos/sensors/multi_mesh_raycaster.py": ScriptOverride( args=("--flat_ground",), diff --git a/source/isaaclab/test/markers/test_visualization_markers.py b/source/isaaclab/test/markers/test_visualization_markers.py index ee89733c162a..863a7ed453a8 100644 --- a/source/isaaclab/test/markers/test_visualization_markers.py +++ b/source/isaaclab/test/markers/test_visualization_markers.py @@ -281,6 +281,9 @@ def __init__(self): def is_paused(self): return False + def is_running(self): + return True + def begin_frame(self, sim_time): self.calls.append(("begin_frame", sim_time)) diff --git a/source/isaaclab/test/sim/test_simulation_context_visualizers.py b/source/isaaclab/test/sim/test_simulation_context_visualizers.py index c01875b681d7..1c72233204f7 100644 --- a/source/isaaclab/test/sim/test_simulation_context_visualizers.py +++ b/source/isaaclab/test/sim/test_simulation_context_visualizers.py @@ -200,6 +200,42 @@ def test_update_visualizers_handles_training_pause_loop(): assert viz.step_calls == [0.0, 0.2] +def test_newton_visualizer_is_initialized_and_rebound_before_capture(): + created = [] + reset_calls = [] + + class _Cfg: + def __init__(self, visualizer_type, enable_picking=False): + self.visualizer_type = visualizer_type + self.enable_picking = enable_picking + self.headless = False + + def create_visualizer(self): + viz = _FakeVisualizer() + viz.cfg = self + viz.initialize = lambda _provider: created.append(self.visualizer_type) + viz.reset = lambda soft: reset_calls.append((self.visualizer_type, soft)) + return viz + + ctx = _make_context_with_settings( + {}, visualizer_cfgs=[_Cfg("newton_gl", True), _Cfg("newton_rtx", True), _Cfg("rerun")] + ) + ctx._prepare_newton_visualizer_for_capture() + assert created == ["newton_gl", "newton_rtx"] + + ctx.initialize_visualizers() + ctx._prepare_newton_visualizer_for_capture() + + assert created == ["newton_gl", "newton_rtx", "rerun"] + assert len(ctx._visualizers) == 3 + assert reset_calls == [ + ("newton_gl", False), + ("newton_rtx", False), + ("newton_gl", False), + ("newton_rtx", False), + ] + + def test_reset_initializes_visualizers_before_playing_timeline(): """Initial visualizers must see the PhysX views created by reset before play() pumps timeline events.""" events: list[str] = [] @@ -694,6 +730,7 @@ def _make_context_with_settings( ctx._pending_camera_view = None ctx._render_generation = 0 ctx._visualizers = [] + ctx._pending_visualizer_cfgs = None ctx._scene_data_provider = _FakeProvider() ctx._scene_data_requirements = None ctx._clone_plan = None diff --git a/source/isaaclab_contrib/changelog.d/max-newton-viewer-dragging.minor.rst b/source/isaaclab_contrib/changelog.d/max-newton-viewer-dragging.minor.rst new file mode 100644 index 000000000000..cdea16167d22 --- /dev/null +++ b/source/isaaclab_contrib/changelog.d/max-newton-viewer-dragging.minor.rst @@ -0,0 +1,5 @@ +Added +^^^^^ + +* Added Newton visualizer rigid-body dragging support to VBD and + :class:`~isaaclab_contrib.coupling.NewtonCouplerManager`. diff --git a/source/isaaclab_contrib/isaaclab_contrib/coupling/coupler.py b/source/isaaclab_contrib/isaaclab_contrib/coupling/coupler.py index ff685d4cf11e..f88fae66d9da 100644 --- a/source/isaaclab_contrib/isaaclab_contrib/coupling/coupler.py +++ b/source/isaaclab_contrib/isaaclab_contrib/coupling/coupler.py @@ -113,6 +113,7 @@ def _build_solver(cls, model: Model, solver_cfg: CouplerCfg) -> None: NewtonManager._use_single_state = False NewtonManager._supports_contact_sensors = False NewtonManager._needs_collision_pipeline = needs_collision_pipeline + NewtonManager._supports_rigid_body_force_input = True @classmethod def _validate_config(cls, solver_cfg: CouplerCfg) -> None: diff --git a/source/isaaclab_contrib/isaaclab_contrib/custom_coupling/coupled_mjwarp_vbd_manager.py b/source/isaaclab_contrib/isaaclab_contrib/custom_coupling/coupled_mjwarp_vbd_manager.py index 86b428d1c269..1dcdafdb8479 100644 --- a/source/isaaclab_contrib/isaaclab_contrib/custom_coupling/coupled_mjwarp_vbd_manager.py +++ b/source/isaaclab_contrib/isaaclab_contrib/custom_coupling/coupled_mjwarp_vbd_manager.py @@ -72,6 +72,7 @@ def _build_solver(cls, model: Model, solver_cfg: CoupledMJWarpVBDSolverCfg) -> N NewtonManager._use_single_state = False NewtonManager._supports_contact_sensors = False NewtonManager._needs_collision_pipeline = True + NewtonManager._supports_rigid_body_force_input = True @classmethod def _step_solver( diff --git a/source/isaaclab_contrib/isaaclab_contrib/deformable/vbd_manager.py b/source/isaaclab_contrib/isaaclab_contrib/deformable/vbd_manager.py index 1f4f7cd07ff1..ca49ec379816 100644 --- a/source/isaaclab_contrib/isaaclab_contrib/deformable/vbd_manager.py +++ b/source/isaaclab_contrib/isaaclab_contrib/deformable/vbd_manager.py @@ -249,6 +249,7 @@ def _build_solver(cls, model: Model, solver_cfg: VBDSolverCfg) -> None: NewtonManager._solver = cls._create_solver(model, solver_cfg) NewtonManager._use_single_state = False NewtonManager._needs_collision_pipeline = True + NewtonManager._supports_rigid_body_force_input = not solver_cfg.integrate_with_external_rigid_solver @classmethod def _simulate_physics_only(cls) -> None: diff --git a/source/isaaclab_contrib/test/coupling/test_coupler.py b/source/isaaclab_contrib/test/coupling/test_coupler.py index 7bad53be0aa3..4a9b06ee76e1 100644 --- a/source/isaaclab_contrib/test/coupling/test_coupler.py +++ b/source/isaaclab_contrib/test/coupling/test_coupler.py @@ -666,6 +666,7 @@ def test_proxy_selects_expected_outer_collision_pipeline(monkeypatch, case, expe "_use_single_state", "_needs_collision_pipeline", "_supports_contact_sensors", + "_supports_rigid_body_force_input", "_report_contacts", ): monkeypatch.setattr(coupler.NewtonManager, attribute, getattr(coupler.NewtonManager, attribute)) @@ -709,6 +710,7 @@ def test_proxy_selects_expected_outer_collision_pipeline(monkeypatch, case, expe assert coupler.NewtonManager._needs_collision_pipeline is expected_outer assert coupler.NewtonManager._supports_contact_sensors is False + assert coupler.NewtonManager._supports_rigid_body_force_input is True assert recorded_entries == ["rigid", "soft"] @@ -721,6 +723,7 @@ def test_admm_always_requests_outer_collision_pipeline(monkeypatch): "_use_single_state", "_needs_collision_pipeline", "_supports_contact_sensors", + "_supports_rigid_body_force_input", "_report_contacts", ): monkeypatch.setattr(coupler.NewtonManager, attribute, getattr(coupler.NewtonManager, attribute)) @@ -747,6 +750,7 @@ def test_admm_always_requests_outer_collision_pipeline(monkeypatch): NewtonCouplerManager._build_solver(model, cfg) assert coupler.NewtonManager._needs_collision_pipeline is True + assert coupler.NewtonManager._supports_rigid_body_force_input is True def test_contact_sensor_guard_does_not_mutate_manager_state(monkeypatch): @@ -755,6 +759,7 @@ def test_contact_sensor_guard_does_not_mutate_manager_state(monkeypatch): monkeypatch.setattr(coupler.NewtonManager, "_use_single_state", True) monkeypatch.setattr(coupler.NewtonManager, "_needs_collision_pipeline", True) monkeypatch.setattr(coupler.NewtonManager, "_supports_contact_sensors", True) + monkeypatch.setattr(coupler.NewtonManager, "_supports_rigid_body_force_input", False) monkeypatch.setattr(coupler.NewtonManager, "_report_contacts", True) with pytest.raises(NotImplementedError, match="contact sensors"): @@ -764,6 +769,7 @@ def test_contact_sensor_guard_does_not_mutate_manager_state(monkeypatch): assert coupler.NewtonManager._use_single_state is True assert coupler.NewtonManager._needs_collision_pipeline is True assert coupler.NewtonManager._supports_contact_sensors is True + assert coupler.NewtonManager._supports_rigid_body_force_input is False class _RecordingAdmm: diff --git a/source/isaaclab_contrib/test/coupling/test_coupler_runtime.py b/source/isaaclab_contrib/test/coupling/test_coupler_runtime.py index 816cc037c00c..8bb0627b15f1 100644 --- a/source/isaaclab_contrib/test/coupling/test_coupler_runtime.py +++ b/source/isaaclab_contrib/test/coupling/test_coupler_runtime.py @@ -38,6 +38,7 @@ def isolated_newton_manager(monkeypatch: pytest.MonkeyPatch): "_collision_cfg": None, "_needs_collision_pipeline": False, "_supports_contact_sensors": True, + "_supports_rigid_body_force_input": False, "_report_contacts": False, } for name, value in clean_values.items(): diff --git a/source/isaaclab_contrib/test/custom_coupling/test_manager.py b/source/isaaclab_contrib/test/custom_coupling/test_manager.py index 19f2500b6b0d..3ae1d62e809f 100644 --- a/source/isaaclab_contrib/test/custom_coupling/test_manager.py +++ b/source/isaaclab_contrib/test/custom_coupling/test_manager.py @@ -82,13 +82,14 @@ def test_build_solver_rejects_contact_sensors(monkeypatch: pytest.MonkeyPatch) - NewtonCoupledMJWarpVBDManager._build_solver(MagicMock(), CoupledMJWarpVBDSolverCfg()) -def test_build_solver_disables_contact_sensors(monkeypatch: pytest.MonkeyPatch) -> None: +def test_build_solver_sets_capabilities(monkeypatch: pytest.MonkeyPatch) -> None: solver_cfg = CoupledMJWarpVBDSolverCfg() monkeypatch.setattr(manager_module.NewtonManager, "_report_contacts", False) monkeypatch.setattr(manager_module.NewtonManager, "_supports_contact_sensors", True) monkeypatch.setattr(manager_module.NewtonManager, "_solver", None) monkeypatch.setattr(manager_module.NewtonManager, "_use_single_state", True) monkeypatch.setattr(manager_module.NewtonManager, "_needs_collision_pipeline", False) + monkeypatch.setattr(manager_module.NewtonManager, "_supports_rigid_body_force_input", False) monkeypatch.setattr(NewtonCoupledMJWarpVBDManager, "_rigid_solver", None) monkeypatch.setattr(NewtonCoupledMJWarpVBDManager, "_soft_solver", None) monkeypatch.setattr(NewtonCoupledMJWarpVBDManager, "_coupling_mode", None) @@ -101,6 +102,7 @@ def test_build_solver_disables_contact_sensors(monkeypatch: pytest.MonkeyPatch) NewtonCoupledMJWarpVBDManager._build_solver(MagicMock(), solver_cfg) assert manager_module.NewtonManager._supports_contact_sensors is False + assert manager_module.NewtonManager._supports_rigid_body_force_input is True def test_legacy_mjwarp_solver_config_warns() -> None: diff --git a/source/isaaclab_contrib/test/deformable/test_deformable_builder_hooks.py b/source/isaaclab_contrib/test/deformable/test_deformable_builder_hooks.py index fa886612ca1e..db6dddc3b3a4 100644 --- a/source/isaaclab_contrib/test/deformable/test_deformable_builder_hooks.py +++ b/source/isaaclab_contrib/test/deformable/test_deformable_builder_hooks.py @@ -19,6 +19,7 @@ add_deformable_entry_to_builder, setup_registered_deformable_fabric_sync, ) +from isaaclab_contrib.deformable.vbd_manager import NewtonVBDManager class _FakeBuilder: @@ -84,6 +85,22 @@ def test_deformable_package_exports_public_symbols(): assert VBDSolverCfg.__name__ == "VBDSolverCfg" +@pytest.mark.parametrize("external_rigid_solver", [False, True]) +def test_vbd_solver_force_input_capability(monkeypatch, external_rigid_solver: bool): + """VBD consumes rigid forces only when it owns AVBD rigid integration.""" + solver = object() + monkeypatch.setattr(NewtonVBDManager, "_create_solver", lambda model, cfg: solver) + monkeypatch.setattr(NewtonManager, "_solver", None) + monkeypatch.setattr(NewtonManager, "_use_single_state", True) + monkeypatch.setattr(NewtonManager, "_needs_collision_pipeline", False) + monkeypatch.setattr(NewtonManager, "_supports_rigid_body_force_input", False) + + NewtonVBDManager._build_solver(object(), VBDSolverCfg(integrate_with_external_rigid_solver=external_rigid_solver)) + + assert NewtonManager._solver is solver + assert NewtonManager._supports_rigid_body_force_input is not external_rigid_solver + + def test_newton_material_defaults_match_registry_defaults(): """Test that Newton material cfg defaults match the deformable registry defaults.""" material_cfg = NewtonDeformableMaterialCfg() diff --git a/source/isaaclab_newton/changelog.d/max-newton-viewer-dragging.minor.rst b/source/isaaclab_newton/changelog.d/max-newton-viewer-dragging.minor.rst new file mode 100644 index 000000000000..5f7048b42adc --- /dev/null +++ b/source/isaaclab_newton/changelog.d/max-newton-viewer-dragging.minor.rst @@ -0,0 +1,5 @@ +Added +^^^^^ + +* Added support for applying Newton visualizer dragging forces during + rigid-body solver substeps. diff --git a/source/isaaclab_newton/isaaclab_newton/physics/featherstone_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/featherstone_manager.py index 99f86dc8b3be..1f8f04eff319 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/featherstone_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/featherstone_manager.py @@ -35,3 +35,4 @@ def _build_solver(cls, model: Model, solver_cfg: FeatherstoneSolverCfg) -> None: NewtonManager._solver = cls._create_solver(model, solver_cfg) NewtonManager._use_single_state = False NewtonManager._needs_collision_pipeline = True + NewtonManager._supports_rigid_body_force_input = True diff --git a/source/isaaclab_newton/isaaclab_newton/physics/kamino_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/kamino_manager.py index 257de444f048..51eb1bfb2d9d 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/kamino_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/kamino_manager.py @@ -156,3 +156,4 @@ def _build_solver(cls, model: Model, solver_cfg: KaminoSolverCfg) -> None: NewtonManager._solver = cls._create_solver(model, solver_cfg) NewtonManager._use_single_state = False NewtonManager._needs_collision_pipeline = not solver_cfg.use_collision_detector + NewtonManager._supports_rigid_body_force_input = True diff --git a/source/isaaclab_newton/isaaclab_newton/physics/mjwarp_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/mjwarp_manager.py index 887f81698339..75e2b2e46991 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/mjwarp_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/mjwarp_manager.py @@ -52,6 +52,7 @@ def _build_solver(cls, model: Model, solver_cfg: MJWarpSolverCfg) -> None: NewtonManager._solver = cls._create_solver(model, solver_cfg) NewtonManager._use_single_state = True NewtonManager._needs_collision_pipeline = not solver_cfg.use_mujoco_contacts + NewtonManager._supports_rigid_body_force_input = True cfg = PhysicsManager._cfg # Cross-config validation that needs both halves. diff --git a/source/isaaclab_newton/isaaclab_newton/physics/mpm_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/mpm_manager.py index aba2aa85107c..4cc6f6798270 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/mpm_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/mpm_manager.py @@ -117,6 +117,7 @@ def _build_solver(cls, model: Model, solver_cfg: MPMSolverCfg) -> None: NewtonManager._solver = cls._create_solver(model, solver_cfg) NewtonManager._use_single_state = True NewtonManager._needs_collision_pipeline = False + NewtonManager._supports_rigid_body_force_input = False cls._project_outside_colliders = solver_cfg.project_outside_colliders @classmethod diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py index fd5e0a735f7b..908f90fa7be9 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py @@ -370,6 +370,8 @@ def provides_implicit_damping(cls) -> bool: _decimation: int = 1 _collision_decimation: int = 0 _num_envs: int | None = None + _supports_rigid_body_force_input: bool = False + """Whether the solver consumes applied rigid-body forces from :class:`State`.""" # Newton model and state _builder: ModelBuilder = None @@ -413,6 +415,8 @@ def provides_implicit_damping(cls) -> bool: # substeps, in registration order. Multiple articulations register their # implicit-DOF telemetry / FF-routing kernels here. _post_actuator_callbacks: list[Callable[[], None]] = [] + # In-graph hooks invoked immediately before every solver substep. + _state_force_callbacks: list[Callable[[State], None]] = [] # In-graph hooks invoked after the last solver substep and before sensors, # in registration order. Articulations with non-identity ordering register # their backend-to-user state republish kernels here so the reorders are @@ -1046,6 +1050,7 @@ def clear(cls): NewtonManager._model = None NewtonManager._solver = None NewtonManager._use_single_state = None + NewtonManager._supports_rigid_body_force_input = False NewtonManager._state_0 = None NewtonManager._state_1 = None NewtonManager._control = None @@ -1062,6 +1067,7 @@ def clear(cls): NewtonManager._supports_contact_sensors = True NewtonManager._adapter = None NewtonManager._post_actuator_callbacks = [] + NewtonManager._state_force_callbacks = [] NewtonManager._post_step_callbacks = [] # Set by an articulation that took the ``use_newton_actuators=True`` # branch in ``_process_actuators_cfg``. Together with the adapter @@ -1915,6 +1921,9 @@ def _build_solver(cls, model: Model, solver_cfg) -> None: manager owns Newton's :class:`CollisionPipeline` for contact generation; ``False`` if the solver runs internal collision detection (MuJoCo internal contacts, Kamino with its own detector). + * :attr:`NewtonManager._supports_rigid_body_force_input` — ``True`` if + the solver consumes external rigid-body forces from + :attr:`State.body_f`; ``False`` otherwise. Writing through ``NewtonManager._foo`` (rather than ``cls._foo``) keeps the canonical state visible to external readers regardless of @@ -2018,10 +2027,17 @@ def initialize_solver(cls) -> None: raise RuntimeError( f"{cls.__name__}._build_solver did not assign NewtonManager._solver. " "Subclasses of NewtonManager must populate NewtonManager._solver, " - "NewtonManager._use_single_state, and NewtonManager._needs_collision_pipeline." + "NewtonManager._use_single_state, NewtonManager._needs_collision_pipeline, and " + "NewtonManager._supports_rigid_body_force_input." ) cls._initialize_contacts() + # Picking callbacks must be registered after the concrete solver has + # published its force-input capability, but before CUDA graph capture. + sim = PhysicsManager._sim + if NewtonManager._supports_rigid_body_force_input and sim is not None: + sim._prepare_newton_visualizer_for_capture() + # Bind the solver-specialized FK delegate to the active subclass's _eval_fk_impl so # that forward()/step() dispatch correctly even when forward() is invoked through the # base class (the data layer imports NewtonManager directly). ``cls`` is the concrete @@ -2251,6 +2267,8 @@ def _run_solver_substeps(cls, contacts) -> None: if cls._use_single_state: for i in range(cls._num_substeps): + for callback in cls._state_force_callbacks: + callback(cls._state_0) cls._step_solver(cls._state_0, cls._state_0, cls._control, contacts, cls._solver_dt) cls._state_0.clear_forces() if collide_mid_loop and (i + 1) % collide_every == 0 and i + 1 < cls._num_substeps: @@ -2259,6 +2277,8 @@ def _run_solver_substeps(cls, contacts) -> None: cfg = PhysicsManager._cfg need_copy_on_last = cfg is not None and cls._num_substeps % 2 == 1 for i in range(cls._num_substeps): + for callback in cls._state_force_callbacks: + callback(cls._state_0) cls._step_solver(cls._state_0, cls._state_1, cls._control, contacts, cls._solver_dt) if need_copy_on_last and i == cls._num_substeps - 1: cls._state_0.assign(cls._state_1) @@ -3060,6 +3080,20 @@ def register_post_actuator_callback(cls, callback: Callable[[], None]) -> None: """ cls._post_actuator_callbacks.append(callback) + @classmethod + def register_state_force_callback(cls, callback: Callable[[State], None]) -> None: + """Register a graph-safe callback that applies forces before every solver substep. + + Callbacks must be registered before solver initialization so they are + included in CUDA graph capture. + + Args: + callback: Function that adds forces [N, N·m] to the provided state. + """ + if callback in NewtonManager._state_force_callbacks: + return + NewtonManager._state_force_callbacks.append(callback) + @classmethod def register_post_step_callback(cls, callback: Callable[[], None]) -> None: """Append a hook to the list invoked after the last solver substep on every step. diff --git a/source/isaaclab_newton/isaaclab_newton/physics/xpbd_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/xpbd_manager.py index da90a977d566..d38d098a853a 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/xpbd_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/xpbd_manager.py @@ -35,3 +35,4 @@ def _build_solver(cls, model: Model, solver_cfg: XPBDSolverCfg) -> None: NewtonManager._solver = cls._create_solver(model, solver_cfg) NewtonManager._use_single_state = False NewtonManager._needs_collision_pipeline = True + NewtonManager._supports_rigid_body_force_input = True diff --git a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py index d3661a848e3c..f9b0db4f4aef 100644 --- a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py +++ b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py @@ -117,6 +117,14 @@ ), ] +RIGID_BODY_FORCE_INPUT_SUPPORT = { + NewtonMJWarpManager: True, + NewtonXPBDManager: True, + NewtonFeatherstoneManager: True, + NewtonKaminoManager: True, + NewtonMPMManager: False, +} + # --------------------------------------------------------------------------- # class_type wiring (no SimulationContext required) @@ -659,6 +667,49 @@ def test_subclass_of_newton_manager(manager): assert manager._create_solver is not NewtonManager._create_solver +def test_clear_resets_rigid_body_force_capability(monkeypatch): + """Teardown clears the canonical solver capability without subclass shadowing.""" + monkeypatch.setattr(NewtonManager, "_supports_rigid_body_force_input", True) + + NewtonManager.clear() + + assert NewtonManager._supports_rigid_body_force_input is False + for manager in ( + NewtonMJWarpManager, + NewtonXPBDManager, + NewtonFeatherstoneManager, + NewtonKaminoManager, + NewtonMPMManager, + ): + assert manager._supports_rigid_body_force_input is False + + +def test_initialize_solver_prepares_picking_before_graph_capture(monkeypatch): + """Viewer force callbacks are registered after capability publication and before capture.""" + events: list[str] = [] + sim_cfg = SimulationCfg( + dt=1.0 / 120.0, + device="cuda:0", + physics=NewtonCfg(solver_cfg=MJWarpSolverCfg(), use_cuda_graph=False), + ) + + with build_simulation_context(sim_cfg=sim_cfg) as sim: + builder = sim.physics_manager.create_builder() + body = builder.add_body(mass=1.0) + builder.add_joint_revolute(parent=-1, child=body, axis=(0, 0, 1)) + NewtonManager.set_builder(builder) + monkeypatch.setattr(sim, "_prepare_newton_visualizer_for_capture", lambda: events.append("prepare")) + monkeypatch.setattr( + NewtonMJWarpManager, + "_capture_or_defer_graph", + classmethod(lambda cls: events.append("capture")), + ) + + sim.reset() + + assert events == ["prepare", "capture"] + + def test_abstract_build_solver_raises(): """Calling :meth:`_build_solver` on the abstract base raises.""" with pytest.raises(NotImplementedError): @@ -768,6 +819,8 @@ def test_initialize_solver_populates_canonical_state( NewtonManager.set_builder(builder) # Force resolution and bring up the solver. + expected_supports_force_input = RIGID_BODY_FORCE_INPUT_SUPPORT[expected_manager] + NewtonManager._supports_rigid_body_force_input = not expected_supports_force_input sim.reset() # Canonical state lives on the base class. @@ -775,6 +828,7 @@ def test_initialize_solver_populates_canonical_state( assert isinstance(NewtonManager._solver, expected_solver_cls) assert NewtonManager._use_single_state is expected_use_single_state assert NewtonManager._needs_collision_pipeline is expected_needs_collision_pipeline + assert NewtonManager._supports_rigid_body_force_input is expected_supports_force_input assert NewtonManager._reset_solver_internals_delegate.__self__ is expected_manager assert ( NewtonManager._reset_solver_internals_delegate.__func__ is expected_manager._reset_solver_internals.__func__ @@ -886,6 +940,58 @@ def counting_collide(state, contacts): assert calls["n"] == 1 + expected_mid_loop_collides +@pytest.mark.parametrize("use_single_state", [True, False], ids=["single_state", "double_state"]) +def test_state_force_callback_runs_before_every_solver_substep(monkeypatch, use_single_state): + """Viewer forces are applied to each current input state before solver stepping.""" + events = [] + + class _State: + def __init__(self, name): + self.name = name + + def clear_forces(self): + pass + + state_0 = _State("state_0") + state_1 = _State("state_1") + + monkeypatch.setattr(NewtonManager, "_state_0", state_0) + monkeypatch.setattr(NewtonManager, "_state_1", state_1) + monkeypatch.setattr(NewtonManager, "_control", object()) + monkeypatch.setattr(NewtonManager, "_solver_dt", 0.001) + monkeypatch.setattr(NewtonManager, "_num_substeps", 2) + monkeypatch.setattr(NewtonManager, "_collision_decimation", 0) + monkeypatch.setattr(NewtonManager, "_needs_collision_pipeline", False) + monkeypatch.setattr(NewtonManager, "_use_single_state", use_single_state) + monkeypatch.setattr( + NewtonManager, + "_state_force_callbacks", + [lambda state: events.append(("force", state.name))], + ) + monkeypatch.setattr( + NewtonManager, + "_step_solver", + staticmethod(lambda state_in, state_out, *_args: events.append(("step", state_in.name, state_out.name))), + ) + + NewtonManager._run_solver_substeps(contacts=None) + + if use_single_state: + assert events == [ + ("force", "state_0"), + ("step", "state_0", "state_0"), + ("force", "state_0"), + ("step", "state_0", "state_0"), + ] + else: + assert events == [ + ("force", "state_0"), + ("step", "state_0", "state_1"), + ("force", "state_1"), + ("step", "state_1", "state_0"), + ] + + # --------------------------------------------------------------------------- # Regression: an env reset written through the data layer must land in the # manager's canonical _state_0 after an odd number of steps when CUDA graphs diff --git a/source/isaaclab_visualizers/changelog.d/max-newton-viewer-dragging.minor.rst b/source/isaaclab_visualizers/changelog.d/max-newton-viewer-dragging.minor.rst new file mode 100644 index 000000000000..e13d87627813 --- /dev/null +++ b/source/isaaclab_visualizers/changelog.d/max-newton-viewer-dragging.minor.rst @@ -0,0 +1,5 @@ +Added +^^^^^ + +* Added right-click rigid-body dragging to the Newton visualizer with Newton + rigid-body solvers. diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py index 0b431041b7bd..c2164c4baea2 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer.py @@ -85,6 +85,8 @@ def _newton_scalar_base_name(name: str) -> str: """Length of synthesized contact arrows in meters.""" if TYPE_CHECKING: + from newton import State + from isaaclab.scene_data import SceneDataProvider @@ -128,6 +130,10 @@ class _NewtonViewerUIMixin: duplicating code. """ + def _register_isaaclab_ui_callbacks(self) -> None: + """Register model-dependent Isaac Lab viewer controls.""" + self.register_ui_callback(self._render_training_controls, position="side") + def _patch_scalar_plot_width(self) -> None: """Set up ImPlot and suppress Newton's built-in floating Plots window. @@ -370,7 +376,7 @@ def _render_left_panel(_g=gui): imgui.text("WASD - Move camera") imgui.text("QE - Pan up/down") imgui.text("Left Click - Look around") - imgui.text("Right Click - Pick objects") + imgui.text("Right Click - Pick and drag objects") imgui.text("Middle Click - Orbit") imgui.text("Shift + Middle Click - Pan") imgui.text("Ctrl + Middle Click - Dolly") @@ -807,6 +813,45 @@ class NewtonVisualizer(BaseVisualizer): :class:`NewtonRTXVisualizer`. """ + class _ViewerPickingBinding: + """Stable Newton-manager callback for viewer picking. + + CUDA graphs record picking arrays by address, so closing the window + neutralizes and retains them until the captured graph is gone. + """ + + def __init__(self) -> None: + self._viewer: NewtonViewerGL | NewtonViewerRTX | None = None + self._retained_picking = None + + def bind(self, viewer: NewtonViewerGL | NewtonViewerRTX) -> None: + """Bind picking to the current viewer model.""" + self._viewer = viewer + self._retained_picking = None + + def apply(self, state: State) -> None: + """Apply picking while the viewer is active.""" + if self._viewer is None: + # Host callbacks do not run during graph replay, so reaching + # this branch means captured inputs are no longer needed. + self._retained_picking = None + return + self._viewer.apply_forces(state) + + def deactivate(self) -> None: + """Make captured picking inert while preserving its inputs.""" + viewer = self._viewer + if viewer is None: + return + + picking = getattr(viewer, "picking", None) + if picking is not None: + viewer.picking_enabled = False + picking.release() + + self._retained_picking = picking + self._viewer = None + def __init__(self, cfg: NewtonVisualizerCfg): """Initialize shared Newton visualizer state. @@ -830,6 +875,8 @@ def __init__(self, cfg: NewtonVisualizerCfg): self._camera_env_indices: list[int] = [] self._camera_is_owned = False self._generated_camera_prim_paths: list[str] = [] + self._viewer_picking_binding = self._ViewerPickingBinding() + self._picking_enabled = False self._streaming_camera_key: tuple | None = None self._live_plots_manager_visible: dict[str, bool] = {} self._last_streaming_composite: np.ndarray | None = None @@ -850,17 +897,26 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: """ from isaaclab_newton.physics import NewtonManager + from isaaclab.sim import SimulationContext + if self._is_initialized: logger.debug("[%s] initialize() called while already initialized.", type(self).__name__) return scene_data_provider = self._set_scene_data_provider(scene_data_provider) + newton_backend_active = self.physics_backend == "newton" + physics_manager = SimulationContext.instance().physics_manager + picking_supported = newton_backend_active and bool( + getattr(physics_manager, "_supports_rigid_body_force_input", False) + ) num_envs = scene_data_provider.num_envs metadata = {"num_envs": num_envs} self._env_ids = self._compute_visualized_env_ids() self._resolved_visible_env_ids = resolve_visible_env_indices(self._env_ids, self.cfg.max_visible_envs, num_envs) self._model = NewtonManager.get_model() - self._state = NewtonManager.get_state(self._scene_data_provider) + self._state = ( + NewtonManager.get_state_0() if newton_backend_active else NewtonManager.get_state(self._scene_data_provider) + ) runtime_headless = self.cfg.headless or ( sys.platform not in ("win32", "darwin") and not os.environ.get("DISPLAY") @@ -887,10 +943,14 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: pyglet.options["headless"] = True + self._picking_enabled = self.cfg.enable_picking and picking_supported and not runtime_headless self._viewer = self._create_viewer(runtime_headless, metadata) if self._viewer is not None: self._viewer.set_model(self._model) + if self._picking_enabled: + # Keep Newton's public force path scoped to picking for this integration. + self._viewer.wind = None self._viewer.set_visible_worlds(self._resolved_visible_env_ids) self._viewer.set_world_offsets(self.cfg.world_spacing) self._apply_camera_focal_length() @@ -898,13 +958,8 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: self._apply_camera_pose(initial_pose) self._viewer._paused = False - self._viewer.show_joints = self.cfg.show_joints - self._viewer.show_contacts = self.cfg.show_contacts - self._viewer.show_collision = self.cfg.show_collision - self._viewer.show_springs = self.cfg.show_springs - self._viewer.show_inertia_boxes = self.cfg.show_inertia_boxes - self._viewer.show_com = self.cfg.show_com - self._viewer.show_particles = self.cfg.show_particles + self._apply_model_visualization_options() + self._viewer.picking_enabled = self._picking_enabled self._apply_viewer_post_init() @@ -929,10 +984,31 @@ def initialize(self, scene_data_provider: SceneDataProvider) -> None: ("num_visualized_envs", num_visualized_envs), ("headless", self.cfg.headless), ("show_particles", self.cfg.show_particles), + ("enable_picking", self._picking_enabled), ], ) + if self._viewer is not None and self._picking_enabled: + self._viewer_picking_binding.bind(self._viewer) + NewtonManager.register_state_force_callback(self._viewer_picking_binding.apply) + if self._viewer is not None and self.cfg.enable_picking and not picking_supported: + logger.info( + "[NewtonVisualizer] Object dragging is disabled because the active physics solver does not support" + " rigid-body force input." + ) self._is_initialized = True + def _apply_model_visualization_options(self) -> None: + """Apply configured options reset by Newton model changes.""" + if self._viewer is None: + return + self._viewer.show_joints = self.cfg.show_joints + self._viewer.show_contacts = self.cfg.show_contacts + self._viewer.show_collision = self.cfg.show_collision + self._viewer.show_springs = self.cfg.show_springs + self._viewer.show_inertia_boxes = self.cfg.show_inertia_boxes + self._viewer.show_com = self.cfg.show_com + self._viewer.show_particles = self.cfg.show_particles + def step(self, dt: float) -> None: """Advance visualization by one simulation step. @@ -989,8 +1065,12 @@ def step(self, dt: float) -> None: self._render_live_plots() finally: self._viewer.end_frame() + if not self._viewer.is_running(): + self._viewer_picking_binding.deactivate() else: self._pump_paused() + if not self._viewer.is_running(): + self._viewer_picking_binding.deactivate() except Exception: logger.exception("[%s] Viewer update failed.", type(self).__name__) # Subclasses that cannot recover from a viewer failure (e.g. RTX when OVRTX is @@ -1015,10 +1095,38 @@ def consume_reset_request(self) -> bool: return self._viewer.consume_reset_request() return False + def reset(self, soft: bool = False) -> None: + """Rebind viewer resources after a hard Newton model reset.""" + if soft or not self._picking_enabled or not self._is_initialized or self._is_closed: + return + + from isaaclab_newton.physics import NewtonManager + + model = NewtonManager.get_model() + if model is self._model: + return + self._model = model + self._state = NewtonManager.get_state_0() + if self._viewer is not None: + self._viewer.set_model(self._model) + if self._picking_enabled: + self._viewer.wind = None + self._viewer._register_isaaclab_ui_callbacks() + self._viewer.set_visible_worlds(self._resolved_visible_env_ids) + self._viewer.set_world_offsets(self.cfg.world_spacing) + self._apply_model_visualization_options() + self._viewer.picking_enabled = self._picking_enabled + if self._picking_enabled: + self._viewer_picking_binding.bind(self._viewer) + def close(self) -> None: """Release viewer resources.""" if self._is_closed: return + if self._picking_enabled: + # Keep the stable callback registered: captured graphs replay its + # now-neutral device inputs without retaining the viewer. + self._viewer_picking_binding.deactivate() if self._viewer is not None: self._viewer = None if self._camera_sensor is not None and self._camera_is_owned: diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer_cfg.py b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer_cfg.py index ed90834b9c05..23d2d774df95 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer_cfg.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/newton/newton_visualizer_cfg.py @@ -76,6 +76,14 @@ def __post_init__(self): particle_color: tuple[float, float, float] | None = None """Optional particle color RGB [0, 1]. Uses Newton viewer defaults when ``None``.""" + enable_picking: bool = True + """Enable right-click dragging with Newton rigid-body solvers. + + Supported coupled solvers may expose dragging through a rigid-body entry. + Disabled automatically for headless viewers, standalone MPM, and non-Newton + physics. MPM particles are not pickable. + """ + enable_shadows: bool = True """Enable shadow rendering.""" diff --git a/source/isaaclab_visualizers/test/test_newton_adapter.py b/source/isaaclab_visualizers/test/test_newton_adapter.py index c0f99d29be3b..675c7ff4469c 100644 --- a/source/isaaclab_visualizers/test/test_newton_adapter.py +++ b/source/isaaclab_visualizers/test/test_newton_adapter.py @@ -8,6 +8,7 @@ from __future__ import annotations from types import SimpleNamespace +from unittest.mock import Mock import numpy as np import pytest @@ -116,9 +117,10 @@ def set_visible_worlds(self, worlds): assert calls[-1] is None -def test_newton_visualizer_cfg_exposes_particle_options(): - cfg = NewtonGLVisualizerCfg(show_particles=True, particle_color=(0.1, 0.2, 0.3)) +def test_newton_visualizer_cfg_exposes_viewer_options(): + cfg = NewtonGLVisualizerCfg(enable_picking=False, show_particles=True, particle_color=(0.1, 0.2, 0.3)) + assert cfg.enable_picking is False assert cfg.show_particles is True assert cfg.particle_color == (0.1, 0.2, 0.3) @@ -419,6 +421,9 @@ def __init__(self): def is_paused(self): return False + def is_running(self): + return True + def begin_frame(self, _time): pass @@ -466,22 +471,82 @@ def get_contact_sensors(self): def _make_newton_visualizer(viewer, scene_data_provider=None): - visualizer = NewtonGLVisualizer.__new__(NewtonGLVisualizer) - visualizer.cfg = NewtonGLVisualizerCfg(enable_markers=False) + visualizer = NewtonGLVisualizer(NewtonGLVisualizerCfg(enable_markers=False)) visualizer._is_initialized = True visualizer._is_closed = False visualizer._sim_time = 0.0 visualizer._step_counter = 0 visualizer._runtime_headless = False visualizer._viewer = viewer - visualizer._state = None visualizer._scene_data_provider = scene_data_provider visualizer._resolved_visible_env_ids = None visualizer._live_plot_sources = [] + if viewer is not None: + visualizer._viewer_picking_binding.bind(viewer) visualizer._log_camera_sensor_image = lambda: None return visualizer +def test_newton_visualizer_forwards_and_neutralizes_picking(): + viewer = _Viewer() + viewer.picking_enabled = True + viewer.picking = SimpleNamespace(release=Mock()) + viewer.apply_forces = Mock() + visualizer = _make_newton_visualizer(viewer) + visualizer._picking_enabled = True + callback = visualizer._viewer_picking_binding.apply + + state = object() + callback(state) + viewer.apply_forces.assert_called_once_with(state) + + visualizer.close() + + assert viewer.picking_enabled is False + viewer.picking.release.assert_called_once_with() + assert visualizer._viewer is None + assert visualizer._viewer_picking_binding._viewer is None + assert visualizer._viewer_picking_binding._retained_picking is viewer.picking + + callback(object()) + assert visualizer._viewer_picking_binding._retained_picking is None + + +def test_newton_visualizer_hard_reset_rebinds_viewer_model(monkeypatch): + from isaaclab_newton.physics import NewtonManager + + new_model = object() + new_state = object() + monkeypatch.setattr(NewtonManager, "get_model", lambda: new_model) + monkeypatch.setattr(NewtonManager, "get_state_0", lambda: new_state) + + viewer = _Viewer() + viewer.picking_enabled = False + viewer.set_model = Mock() + viewer._register_isaaclab_ui_callbacks = Mock() + viewer.set_visible_worlds = Mock() + viewer.set_world_offsets = Mock() + visualizer = _make_newton_visualizer(viewer) + visualizer._resolved_visible_env_ids = [1, 3] + visualizer._picking_enabled = True + visualizer.cfg.world_spacing = (2.0, 0.0, 0.0) + visualizer.cfg.show_contacts = True + + visualizer.reset(soft=False) + visualizer.reset(soft=False) + + assert visualizer._model is new_model + assert visualizer._state is new_state + viewer.set_model.assert_called_once_with(new_model) + viewer._register_isaaclab_ui_callbacks.assert_called_once_with() + viewer.set_visible_worlds.assert_called_once_with([1, 3]) + viewer.set_world_offsets.assert_called_once_with((2.0, 0.0, 0.0)) + assert viewer.show_contacts is True + assert viewer.picking_enabled is True + assert viewer.wind is None + assert visualizer._viewer_picking_binding._viewer is viewer + + def test_newton_visualizer_logs_native_contacts_when_available(monkeypatch): from isaaclab_newton.physics import NewtonManager