From 070d48cda2f438f5f6b31dd5b6a0481d55cf4527 Mon Sep 17 00:00:00 2001 From: mtrepte Date: Wed, 2 Sep 2026 22:43:20 +0000 Subject: [PATCH 1/3] Clear self-collision filter pairs before finalizing shadow Newton model The PhysX-backend shadow Newton visualization model never runs collision detection, but USD-authored self-collision filter pairs were still being imported into it and replicated across every cloned env. At real training env counts this could reach billions of entries and OOM ModelBuilder.finalize(). --- ...hadow-model-collision-filter-oom-test.skip | 0 ...test_newton_manager_visualization_state.py | 23 +++++++++++++++++++ .../fix-shadow-model-collision-filter-oom.rst | 10 ++++++++ .../isaaclab_newton/physics/newton_manager.py | 8 +++++++ 4 files changed, 41 insertions(+) create mode 100644 source/isaaclab/changelog.d/mtrepte-shadow-model-collision-filter-oom-test.skip create mode 100644 source/isaaclab_newton/changelog.d/fix-shadow-model-collision-filter-oom.rst diff --git a/source/isaaclab/changelog.d/mtrepte-shadow-model-collision-filter-oom-test.skip b/source/isaaclab/changelog.d/mtrepte-shadow-model-collision-filter-oom-test.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab/test/sim/test_newton_manager_visualization_state.py b/source/isaaclab/test/sim/test_newton_manager_visualization_state.py index 823afc3e222e..d68b326300db 100644 --- a/source/isaaclab/test/sim/test_newton_manager_visualization_state.py +++ b/source/isaaclab/test/sim/test_newton_manager_visualization_state.py @@ -330,6 +330,29 @@ def _finalize(device): assert NewtonManager._state_0 is not None +def test_ensure_visualization_model_clears_shape_collision_filter_pairs_before_finalize(monkeypatch): + """The shadow model never runs collision detection, so USD-authored self-collision + filters (which scale with the number of cloned envs) must not be packed into it. + """ + from isaaclab_newton.physics import NewtonManager + from isaaclab_newton.physics import newton_manager as nm + + _reset_newton_manager_state() + monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls, scene_data_provider=None: False)) + monkeypatch.setattr(nm, "get_current_stage", lambda *args, **kwargs: _make_env_stage()) + monkeypatch.setattr(nm.PhysicsManager, "_sim", None, raising=False) + _set_sim_context(monkeypatch, nm) + monkeypatch.setattr(nm.PhysicsManager, "_device", "cpu", raising=False) + + builder = _make_finalize_builder(body_count=3) + builder.shape_collision_filter_pairs = [(0, 1), (0, 2)] + monkeypatch.setattr(nm, "build_visualization_builder_from_stage_envs", lambda *args, **kwargs: (builder, ([], []))) + + NewtonManager._ensure_visualization_model() + + assert builder.shape_collision_filter_pairs == [] + + def test_physx_shadow_model_is_rebuilt_after_physics_stop(monkeypatch): """Sequential PhysX scenes must not reuse the prior scene's Newton visualization model.""" from isaaclab_newton.physics import NewtonManager diff --git a/source/isaaclab_newton/changelog.d/fix-shadow-model-collision-filter-oom.rst b/source/isaaclab_newton/changelog.d/fix-shadow-model-collision-filter-oom.rst new file mode 100644 index 000000000000..c249893d9935 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/fix-shadow-model-collision-filter-oom.rst @@ -0,0 +1,10 @@ +Fixed +^^^^^ + +* Fixed :class:`~isaaclab_newton.physics.NewtonManager` building the PhysX-backend + shadow Newton visualization model (used by Newton-native visualizers/renderers such + as viser, rerun, and Newton GL/RTX) with USD-authored self-collision filter pairs. + These pairs scale with the number of cloned environments and could reach billions + of entries, causing ``ModelBuilder.finalize()`` to run out of memory. The shadow + model never runs collision detection, so its collision filter pairs are now cleared + before finalization. diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py index 0765df0e3700..534b01e7ade1 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py @@ -2815,6 +2815,14 @@ def _ensure_visualization_model(cls) -> None: device = PhysicsManager._device or "cpu" try: + # The shadow model never runs collision detection -- body_q is overwritten every + # frame from the PhysX SceneDataProvider state, not computed by a Newton solver. + # USD-authored self-collision filters (e.g. physxArticulation:enabledSelfCollisions) + # still get imported by add_usd though, and replicated across every cloned env. + # At real env counts that PhysX tasks train with, that filter-pair set can reach + # billions of entries and blow up ModelBuilder.finalize() trying to pack it. + # Drop it before finalizing since the shadow model has no use for it. + builder.shape_collision_filter_pairs = [] NewtonManager._model = builder.finalize(device=device) NewtonManager._state_0 = cls._model.state() cls._model.num_envs = cls._num_envs From 80018d060e505f7e1b69d71d1c72ffa6bfed2176 Mon Sep 17 00:00:00 2001 From: mtrepte Date: Wed, 2 Sep 2026 23:59:23 +0000 Subject: [PATCH 2/3] Assert collision filter pairs are cleared before finalize, not just after Addresses Greptile review feedback: the regression test previously only checked the builder's post-call state, which would stay green even if a future change cleared the pairs after finalize() instead of before. --- .../sim/test_newton_manager_visualization_state.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/isaaclab/test/sim/test_newton_manager_visualization_state.py b/source/isaaclab/test/sim/test_newton_manager_visualization_state.py index d68b326300db..5648ee5c3e8d 100644 --- a/source/isaaclab/test/sim/test_newton_manager_visualization_state.py +++ b/source/isaaclab/test/sim/test_newton_manager_visualization_state.py @@ -344,12 +344,21 @@ def test_ensure_visualization_model_clears_shape_collision_filter_pairs_before_f _set_sim_context(monkeypatch, nm) monkeypatch.setattr(nm.PhysicsManager, "_device", "cpu", raising=False) - builder = _make_finalize_builder(body_count=3) + builder = _make_finalize_builder(body_count=3, finalize=False) builder.shape_collision_filter_pairs = [(0, 1), (0, 2)] + + filter_pairs_at_finalize: list[list[tuple[int, int]]] = [] + + def _finalize(device): + filter_pairs_at_finalize.append(builder.shape_collision_filter_pairs) + return SimpleNamespace(state=lambda: SimpleNamespace(body_q=None)) + + builder.finalize = _finalize monkeypatch.setattr(nm, "build_visualization_builder_from_stage_envs", lambda *args, **kwargs: (builder, ([], []))) NewtonManager._ensure_visualization_model() + assert filter_pairs_at_finalize == [[]] assert builder.shape_collision_filter_pairs == [] From 4716126cc21aa763c3e4d0333452b07e8485d8f4 Mon Sep 17 00:00:00 2001 From: Octi Zhang Date: Thu, 3 Sep 2026 15:15:19 -0700 Subject: [PATCH 3/3] Disable Newton collision pairs before visualization replication --- ...test_newton_manager_visualization_state.py | 36 ++----------- .../fix-shadow-model-collision-filter-oom.rst | 9 +--- .../isaaclab_newton/physics/newton_manager.py | 8 --- .../physics/visualization_builder.py | 8 +++ .../test/cloner/test_rename_builder_labels.py | 50 ++++++++++++++++++- 5 files changed, 63 insertions(+), 48 deletions(-) diff --git a/source/isaaclab/test/sim/test_newton_manager_visualization_state.py b/source/isaaclab/test/sim/test_newton_manager_visualization_state.py index 5648ee5c3e8d..fdd337cd54da 100644 --- a/source/isaaclab/test/sim/test_newton_manager_visualization_state.py +++ b/source/isaaclab/test/sim/test_newton_manager_visualization_state.py @@ -330,38 +330,6 @@ def _finalize(device): assert NewtonManager._state_0 is not None -def test_ensure_visualization_model_clears_shape_collision_filter_pairs_before_finalize(monkeypatch): - """The shadow model never runs collision detection, so USD-authored self-collision - filters (which scale with the number of cloned envs) must not be packed into it. - """ - from isaaclab_newton.physics import NewtonManager - from isaaclab_newton.physics import newton_manager as nm - - _reset_newton_manager_state() - monkeypatch.setattr(NewtonManager, "_backend_is_newton", classmethod(lambda cls, scene_data_provider=None: False)) - monkeypatch.setattr(nm, "get_current_stage", lambda *args, **kwargs: _make_env_stage()) - monkeypatch.setattr(nm.PhysicsManager, "_sim", None, raising=False) - _set_sim_context(monkeypatch, nm) - monkeypatch.setattr(nm.PhysicsManager, "_device", "cpu", raising=False) - - builder = _make_finalize_builder(body_count=3, finalize=False) - builder.shape_collision_filter_pairs = [(0, 1), (0, 2)] - - filter_pairs_at_finalize: list[list[tuple[int, int]]] = [] - - def _finalize(device): - filter_pairs_at_finalize.append(builder.shape_collision_filter_pairs) - return SimpleNamespace(state=lambda: SimpleNamespace(body_q=None)) - - builder.finalize = _finalize - monkeypatch.setattr(nm, "build_visualization_builder_from_stage_envs", lambda *args, **kwargs: (builder, ([], []))) - - NewtonManager._ensure_visualization_model() - - assert filter_pairs_at_finalize == [[]] - assert builder.shape_collision_filter_pairs == [] - - def test_physx_shadow_model_is_rebuilt_after_physics_stop(monkeypatch): """Sequential PhysX scenes must not reuse the prior scene's Newton visualization model.""" from isaaclab_newton.physics import NewtonManager @@ -858,6 +826,10 @@ def test_clone_visualization_builder_ignores_non_env_deformables_on_world_import UsdGeom.Xform.Define(stage, "/World/envs/env_1") fake_builder = _FakeShadowBuilder(body_count=1, cloth_delta=3, track_usd=True) + fake_builder.shape_collision_filter_pairs = [] + fake_builder.shape_collision_group = [] + fake_builder.shape_count = 0 + fake_builder.add_builder = lambda _builder: None clone_plan = SimpleNamespace( sources=("/World/envs/env_0",), destinations=("/World/envs/env_{}",), diff --git a/source/isaaclab_newton/changelog.d/fix-shadow-model-collision-filter-oom.rst b/source/isaaclab_newton/changelog.d/fix-shadow-model-collision-filter-oom.rst index c249893d9935..269487433e0b 100644 --- a/source/isaaclab_newton/changelog.d/fix-shadow-model-collision-filter-oom.rst +++ b/source/isaaclab_newton/changelog.d/fix-shadow-model-collision-filter-oom.rst @@ -1,10 +1,5 @@ Fixed ^^^^^ -* Fixed :class:`~isaaclab_newton.physics.NewtonManager` building the PhysX-backend - shadow Newton visualization model (used by Newton-native visualizers/renderers such - as viser, rerun, and Newton GL/RTX) with USD-authored self-collision filter pairs. - These pairs scale with the number of cloned environments and could reach billions - of entries, causing ``ModelBuilder.finalize()`` to run out of memory. The shadow - model never runs collision detection, so its collision filter pairs are now cleared - before finalization. +* Fixed PhysX-backend Newton visualization models replicating unused collision + filters and contact pairs, which could exhaust memory during ``ModelBuilder.finalize()``. diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py index 820e8ced6f6d..24363b5c1147 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py @@ -2922,14 +2922,6 @@ def _ensure_visualization_model(cls) -> None: device = PhysicsManager._device or "cpu" try: - # The shadow model never runs collision detection -- body_q is overwritten every - # frame from the PhysX SceneDataProvider state, not computed by a Newton solver. - # USD-authored self-collision filters (e.g. physxArticulation:enabledSelfCollisions) - # still get imported by add_usd though, and replicated across every cloned env. - # At real env counts that PhysX tasks train with, that filter-pair set can reach - # billions of entries and blow up ModelBuilder.finalize() trying to pack it. - # Drop it before finalizing since the shadow model has no use for it. - builder.shape_collision_filter_pairs = [] NewtonManager._model = builder.finalize(device=device) NewtonManager._state_0 = cls._model.state() cls._model.num_envs = cls._num_envs diff --git a/source/isaaclab_newton/isaaclab_newton/physics/visualization_builder.py b/source/isaaclab_newton/isaaclab_newton/physics/visualization_builder.py index cef64e4309d7..dda95cf88b69 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/visualization_builder.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/visualization_builder.py @@ -114,6 +114,8 @@ def build_visualization_builder_from_stage_envs( shadow_entities, registry_groups = add_shadow_deformables_to_builder( builder, stage, env_paths, device=device, entries=deformable_entries, clone_plan=clone_plan ) + builder.shape_collision_filter_pairs = [] + builder.shape_collision_group[:] = [0] * builder.shape_count return builder, (shadow_entities, registry_groups) if not env_paths: @@ -148,6 +150,12 @@ def build_visualization_builder_from_stage_envs( schema_resolvers, ignore_paths=source_deformable_ignore_paths or None, ) + global_builder = builder + builder = ModelBuilder(up_axis=up_axis) # Preserve Newton's compact empty filter store. + for visual_builder in (global_builder, *source_builders.values()): + visual_builder.shape_collision_filter_pairs = [] + visual_builder.shape_collision_group[:] = [0] * visual_builder.shape_count + builder.add_builder(global_builder) replicate_builder_mapping(builder, sources, mapping, positions, quaternions, source_builders) rename_builder_labels(builder, sources, destinations, env_ids, mapping) shadow_entities, registry_groups = add_shadow_deformables_to_builder( diff --git a/source/isaaclab_newton/test/cloner/test_rename_builder_labels.py b/source/isaaclab_newton/test/cloner/test_rename_builder_labels.py index ee81d39e83b7..2fbca9c31549 100644 --- a/source/isaaclab_newton/test/cloner/test_rename_builder_labels.py +++ b/source/isaaclab_newton/test/cloner/test_rename_builder_labels.py @@ -21,7 +21,7 @@ from isaaclab_newton.physics import visualization_deformables as visualization_deformables_module from newton.solvers import SolverMuJoCo -from pxr import Usd, UsdGeom +from pxr import Sdf, Usd, UsdGeom, UsdPhysics from isaaclab.cloner import ClonePlan from isaaclab.scene_data.deformable_discovery import DeformableStageEntry @@ -46,6 +46,8 @@ class _FakeVisualizationModelBuilder: def __init__(self, up_axis=None): self.up_axis = up_axis + self.shape_collision_filter_pairs = [] + self.shape_collision_group = [] for attr in _VIS_BUILTIN_LABEL_ATTRS: setattr(self, attr, []) setattr(self, attr.replace("_label", "_world"), []) @@ -86,6 +88,7 @@ def add_usd(self, stage, root_path=None, ignore_paths=None, schema_resolvers=Non for attr in _VIS_BUILTIN_LABEL_ATTRS: getattr(self, attr).append(f"{root_path}/{_VIS_LABEL_SUFFIXES[attr]}") getattr(self, attr.replace("_label", "_world")).append(self._current_world or 0) + self.shape_collision_group.append(1) self.custom_attributes["mujoco:equality_constraint_label"].values.append( f"{root_path}/{_VIS_LABEL_SUFFIXES['equality_constraint_label']}" ) @@ -102,6 +105,7 @@ def add_builder(self, builder, xform=None): labels = getattr(builder, attr) getattr(self, attr).extend(labels) getattr(self, attr.replace("_label", "_world")).extend([self._current_world] * len(labels)) + self.shape_collision_group.extend(builder.shape_collision_group) eq_labels = builder.custom_attributes["mujoco:equality_constraint_label"].values self.custom_attributes["mujoco:equality_constraint_label"].values.extend(eq_labels) self.custom_attributes["mujoco:equality_constraint_world"].values.extend([self._current_world] * len(eq_labels)) @@ -473,6 +477,9 @@ def test_visualization_builder_imports_standalone_stage_as_one_world(self): self._define_xform(stage, "/World") self._define_xform(stage, "/World/Robot") builder = mock.Mock() + builder.shape_collision_filter_pairs = [] + builder.shape_collision_group = [] + builder.shape_count = 0 builder.add_usd.return_value = {"path_shape_map": {}} with ( @@ -490,6 +497,47 @@ def test_visualization_builder_imports_standalone_stage_as_one_world(self): self.assertEqual(registry_groups, []) builder.add_usd.assert_called_once_with(stage, schema_resolvers=["newton", "physx"], ignore_paths=None) + def test_visualization_builder_disables_collision_pairs(self): + stage = Usd.Stage.CreateInMemory() + robot_path = "/World/envs/env_0/Robot" + self._define_xform(stage, "/World") + self._define_xform(stage, "/World/envs") + self._define_xform(stage, "/World/envs/env_0") + self._define_xform(stage, "/World/envs/env_1", (2.0, 0.0, 0.0)) + robot = UsdGeom.Xform.Define(stage, robot_path).GetPrim() + UsdPhysics.ArticulationRootAPI.Apply(robot) + robot.CreateAttribute("physxArticulation:enabledSelfCollisions", Sdf.ValueTypeNames.Bool).Set(False) + for name, translation in (("A", 0.0), ("B", 1.0)): + body_path = f"{robot_path}/{name}" + body = UsdGeom.Xform.Define(stage, body_path) + body.AddTranslateOp().Set((translation, 0.0, 0.0)) + UsdPhysics.RigidBodyAPI.Apply(body.GetPrim()) + collision = UsdGeom.Cube.Define(stage, f"{body_path}/Collision") + collision.CreateSizeAttr(0.2) + UsdPhysics.CollisionAPI.Apply(collision.GetPrim()) + joint = UsdPhysics.RevoluteJoint.Define(stage, f"{robot_path}/Joint") + joint.CreateBody0Rel().SetTargets([Sdf.Path(f"{robot_path}/A")]) + joint.CreateBody1Rel().SetTargets([Sdf.Path(f"{robot_path}/B")]) + + clone_plan = ClonePlan( + sources=(robot_path,), + destinations=("/World/envs/env_{}/Robot",), + clone_mask=torch.ones((1, 2), dtype=torch.bool), + env_ids=torch.arange(2), + ) + for env_paths, plan, expected_shape_count in ( + ([], None, 2), + ([(0, "/World/envs/env_0"), (1, "/World/envs/env_1")], clone_plan, 4), + ): + builder, _shadow_metadata = visualization_builder_module.build_visualization_builder_from_stage_envs( + stage, env_paths, plan + ) + model = builder.finalize(device="cpu") + + self.assertEqual(model.shape_count, expected_shape_count) + self.assertEqual(len(model.shape_collision_filter_pairs), 0) + self.assertEqual(model.shape_contact_pair_count, 0) + def test_visualization_builder_rejects_clone_plan_without_environment_paths(self): """A cloned scene must not be cached as an incomplete single-world model.""" stage = Usd.Stage.CreateInMemory()