Enable deterministic Newton physics

docs/source/features/reproducibility.rst

@@ -55,6 +55,17 @@ Pass ``--deterministic`` to enable reproducible rendering from the app launcher.
         ./isaaclab.sh train --rl_library rl_games \
           --task Isaac-Cartpole-Camera --deterministic
 
+Newton physics determinism
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Set :attr:`isaaclab_newton.physics.NewtonCfg.deterministic_mode` to
+``"gpu_to_gpu"`` to request reproducibility across GPU architectures, or to
+``"run_to_run"`` to request reproducibility on one GPU. Newton applies the
+selected mode to supported solver kernels and enables deterministic contact
+ordering in its collision pipeline. Deterministic execution can increase
+memory use and reduce simulation performance. MJWarp on the GPU, XPBD, and
+Featherstone are supported; selecting an unsupported solver raises an error.
+
 For results on our determinacy testing for RL training, please check the GitHub Pull Request `#940`_.
 
 .. tip::

source/isaaclab_newton/changelog.d/kellyguo11-newton-gpu-determinism.minor.rst

@@ -0,0 +1,5 @@
+Added
+^^^^^
+
+* Added :attr:`~isaaclab_newton.physics.NewtonCfg.deterministic_mode` to apply
+  one determinism setting to supported Newton solvers and collision handling.

source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py

@@ -111,7 +111,10 @@ def _paused_gc():
 from isaaclab_newton.physics.visualization_builder import build_visualization_builder_from_stage_envs
 from isaaclab_newton.physics.visualization_deformables import populate_shadow_deformable_registry
 
-from .newton_manager_cfg import NewtonCfg, NewtonShapeCfg
+from .featherstone_manager_cfg import FeatherstoneSolverCfg
+from .mjwarp_manager_cfg import MJWarpSolverCfg
+from .newton_manager_cfg import NewtonCfg, NewtonShapeCfg, NewtonSolverCfg
+from .xpbd_manager_cfg import XPBDSolverCfg
 
 if TYPE_CHECKING:
     from isaaclab_newton.actuators import NewtonActuatorAdapter
@@ -369,6 +372,7 @@ def provides_implicit_damping(cls) -> bool:
     _num_substeps: int = 1
     _decimation: int = 1
     _collision_decimation: int = 0
+    _deterministic_mode: wp.DeterministicMode = wp.DeterministicMode.NOT_GUARANTEED
     _num_envs: int | None = None
 
     # Newton model and state
@@ -1051,6 +1055,7 @@ def clear(cls):
         NewtonManager._control = None
         NewtonManager._contacts = None
         NewtonManager._needs_collision_pipeline = False
+        NewtonManager._deterministic_mode = wp.DeterministicMode.NOT_GUARANTEED
         NewtonManager._eval_fk = _eval_fk_unbound
         NewtonManager._reset_solver_internals_delegate = _reset_solver_internals_unbound
         NewtonManager._collision_pipeline = None
@@ -1861,13 +1866,12 @@ def _initialize_contacts(cls) -> None:
         """
         if not cls._needs_collision_pipeline:
             return
+        pipeline_args = {"broad_phase": "explicit"}
+        if cls._collision_cfg is not None:
+            pipeline_args = cls._collision_cfg.to_pipeline_args()
+        pipeline_args["deterministic"] = cls._deterministic_mode != wp.DeterministicMode.NOT_GUARANTEED
         if cls._collision_pipeline is None:
-            if cls._collision_cfg is not None:
-                NewtonManager._collision_pipeline = CollisionPipeline(
-                    cls._model, **cls._collision_cfg.to_pipeline_args()
-                )
-            else:
-                NewtonManager._collision_pipeline = CollisionPipeline(cls._model, broad_phase="explicit")
+            NewtonManager._collision_pipeline = CollisionPipeline(cls._model, **pipeline_args)
         if cls._contacts is None:
             NewtonManager._contacts = cls._collision_pipeline.contacts()
             # Grow the collision-pipeline contact buffer to the solver's max when the
@@ -1880,12 +1884,17 @@ def _initialize_contacts(cls) -> None:
             if _solver is not None and hasattr(_solver, "get_max_contact_count"):
                 _need = _solver.get_max_contact_count()
                 if _need > NewtonManager._contacts.rigid_contact_max:
-                    NewtonManager._contacts = Contacts(
-                        rigid_contact_max=_need,
-                        soft_contact_max=0,
-                        device=PhysicsManager._device,
-                        requested_attributes=cls._model.get_requested_contact_attributes(),
-                    )
+                    if cls._deterministic_mode != wp.DeterministicMode.NOT_GUARANTEED:
+                        pipeline_args["rigid_contact_max"] = _need
+                        NewtonManager._collision_pipeline = CollisionPipeline(cls._model, **pipeline_args)
+                        NewtonManager._contacts = cls._collision_pipeline.contacts()
+                    else:
+                        NewtonManager._contacts = Contacts(
+                            rigid_contact_max=_need,
+                            soft_contact_max=0,
+                            device=PhysicsManager._device,
+                            requested_attributes=cls._model.get_requested_contact_attributes(),
+                        )
 
     # ----- Solver construction (subclass contract) ------------------------
 
@@ -1937,7 +1946,38 @@ def _filter_solver_kwargs(solver_cls: type, solver_cfg) -> dict:
         are always excluded — ``model`` is passed positionally at construction.
         """
         valid = set(inspect.signature(solver_cls.__init__).parameters) - {"self", "model"}
-        return {k: v for k, v in solver_cfg.to_dict().items() if k in valid}
+        kwargs = {k: v for k, v in solver_cfg.to_dict().items() if k in valid}
+        if "deterministic" in valid:
+            kwargs["deterministic"] = NewtonManager._deterministic_mode
+        return kwargs
+
+    @staticmethod
+    def _validate_deterministic_solver_cfg(
+        solver_cfg: NewtonSolverCfg, deterministic_mode: wp.DeterministicMode
+    ) -> None:
+        """Validate that a solver can provide the requested determinism guarantee."""
+        if deterministic_mode == wp.DeterministicMode.NOT_GUARANTEED:
+            return
+        solver_cfg_type = type(solver_cfg).__name__
+        if not isinstance(solver_cfg, (FeatherstoneSolverCfg, MJWarpSolverCfg, XPBDSolverCfg)):
+            raise ValueError(
+                f"Newton deterministic mode {deterministic_mode.name} is not supported by {solver_cfg_type}. "
+                "Use MJWarp on the GPU, XPBD, or Featherstone, or disable deterministic mode."
+            )
+        if getattr(solver_cfg, "use_mujoco_cpu", False):
+            raise ValueError(
+                f"Newton deterministic mode {deterministic_mode.name} is not supported by the MuJoCo CPU backend. "
+                "Set MJWarpSolverCfg.use_mujoco_cpu=False or disable deterministic mode."
+            )
+
+    @staticmethod
+    def _resolve_deterministic_mode(deterministic_mode: str) -> wp.DeterministicMode:
+        """Convert a Newton config value to Warp's deterministic-mode enum."""
+        return {
+            "not_guaranteed": wp.DeterministicMode.NOT_GUARANTEED,
+            "run_to_run": wp.DeterministicMode.RUN_TO_RUN,
+            "gpu_to_gpu": wp.DeterministicMode.GPU_TO_GPU,
+        }[deterministic_mode]
 
     @classmethod
     def _step_solver(
@@ -2010,6 +2050,9 @@ def initialize_solver(cls) -> None:
         with Timer(name="newton_initialize_solver", msg="Initialize solver took:", activity="Initializing solver"):
             NewtonManager._num_substeps = cfg.num_substeps  # type: ignore[union-attr]
             NewtonManager._collision_decimation = cfg.collision_decimation  # type: ignore[union-attr]
+            deterministic_mode = cls._resolve_deterministic_mode(cfg.deterministic_mode)  # type: ignore[union-attr]
+            cls._validate_deterministic_solver_cfg(cfg.solver_cfg, deterministic_mode)  # type: ignore[union-attr]
+            NewtonManager._deterministic_mode = deterministic_mode
             NewtonManager._solver_dt = cls.get_physics_dt() / cls._num_substeps
             NewtonManager._collision_cfg = cfg.collision_cfg  # type: ignore[union-attr]
 

source/isaaclab_newton/isaaclab_newton/physics/newton_manager_cfg.py

@@ -134,6 +134,18 @@ class NewtonCfg(PhysicsCfg):
     If set to False, the simulation performance will be severely degraded.
     """
 
+    deterministic_mode: Literal["not_guaranteed", "run_to_run", "gpu_to_gpu"] = "not_guaranteed"
+    """Determinism guarantee applied to the Newton solver and collision pipeline.
+
+    The values ``"not_guaranteed"``, ``"run_to_run"``, and ``"gpu_to_gpu"``
+    map to the corresponding ``warp.DeterministicMode`` values. Deterministic
+    execution increases memory use and can reduce simulation performance.
+
+    MJWarp on the GPU, XPBD, and Featherstone support this setting. Newton
+    raises an error during solver initialization for unsupported solvers rather
+    than silently running them without the requested guarantee.
+    """
+
     solver_cfg: NewtonSolverCfg | None = None
     """Solver configuration. If None (default), MJWarpSolverCfg is used by default."""
 
@@ -216,6 +228,11 @@ def __post_init__(self):
         # previously silently overwritten.
         if self.class_type is not None:
             raise TypeError("Cannot manually set NewtonCfg.class_type; it is auto-derived from solver_cfg.class_type.")
+        if self.deterministic_mode not in ("not_guaranteed", "run_to_run", "gpu_to_gpu"):
+            raise ValueError(
+                "NewtonCfg.deterministic_mode must be 'not_guaranteed', 'run_to_run', or 'gpu_to_gpu', "
+                f"got {self.deterministic_mode!r}."
+            )
         if self.solver_cfg is None:
             from .mjwarp_manager_cfg import MJWarpSolverCfg
 

source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py

@@ -27,6 +27,7 @@
 
 from types import SimpleNamespace
 
+import isaaclab_newton.physics.newton_manager as newton_manager_module
 import numpy as np
 import pytest
 import warp as wp
@@ -173,6 +174,72 @@ def test_newton_cfg_collision_decimation_warning(num_substeps, collision_decimat
     assert cfg.collision_decimation == collision_decimation
 
 
+def test_solver_kwargs_include_newton_deterministic_mode(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Solver construction should receive the mode configured on the outer Newton config."""
+    monkeypatch.setattr(NewtonManager, "_deterministic_mode", wp.DeterministicMode.GPU_TO_GPU)
+
+    kwargs = NewtonManager._filter_solver_kwargs(SolverXPBD, XPBDSolverCfg())
+
+    assert kwargs["deterministic"] == wp.DeterministicMode.GPU_TO_GPU
+
+
+@pytest.mark.parametrize(
+    "solver_cfg",
+    [
+        pytest.param(KaminoSolverCfg(), id="kamino"),
+        pytest.param(MPMSolverCfg(), id="implicit_mpm"),
+        pytest.param(MJWarpSolverCfg(use_mujoco_cpu=True), id="mujoco_cpu"),
+    ],
+)
+def test_deterministic_mode_rejects_unsupported_solver_cfg(solver_cfg) -> None:
+    """Unsupported solvers should not silently ignore a determinism guarantee."""
+    with pytest.raises(ValueError, match="not supported"):
+        NewtonManager._validate_deterministic_solver_cfg(solver_cfg, wp.DeterministicMode.GPU_TO_GPU)
+
+
+@pytest.mark.parametrize("solver_cfg_cls", [FeatherstoneSolverCfg, MJWarpSolverCfg, XPBDSolverCfg])
+def test_deterministic_mode_accepts_supported_solver_cfg_subclasses(solver_cfg_cls) -> None:
+    """Custom subclasses of supported solver configs should retain deterministic support."""
+
+    class CustomSolverCfg(solver_cfg_cls):
+        pass
+
+    NewtonManager._validate_deterministic_solver_cfg(CustomSolverCfg(), wp.DeterministicMode.GPU_TO_GPU)
+
+
+def test_deterministic_collision_pipeline_matches_expanded_contact_capacity(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Deterministic sorting buffers should grow with the solver contact buffer."""
+    pipeline_calls: list[dict] = []
+
+    class FakeCollisionPipeline:
+        def __init__(self, _model, **kwargs):
+            pipeline_calls.append(kwargs)
+            self._rigid_contact_max = kwargs.get("rigid_contact_max", 1)
+
+        def contacts(self):
+            return SimpleNamespace(rigid_contact_max=self._rigid_contact_max)
+
+    solver = SimpleNamespace(get_max_contact_count=lambda: 2)
+    monkeypatch.setattr(newton_manager_module, "CollisionPipeline", FakeCollisionPipeline)
+    monkeypatch.setattr(NewtonManager, "_needs_collision_pipeline", True)
+    monkeypatch.setattr(NewtonManager, "_collision_pipeline", None)
+    monkeypatch.setattr(NewtonManager, "_collision_cfg", None)
+    monkeypatch.setattr(NewtonManager, "_contacts", None)
+    monkeypatch.setattr(NewtonManager, "_solver", solver)
+    monkeypatch.setattr(NewtonManager, "_model", SimpleNamespace())
+    monkeypatch.setattr(NewtonManager, "_deterministic_mode", wp.DeterministicMode.GPU_TO_GPU)
+
+    NewtonManager._initialize_contacts()
+
+    assert pipeline_calls == [
+        {"broad_phase": "explicit", "deterministic": True},
+        {"broad_phase": "explicit", "deterministic": True, "rigid_contact_max": 2},
+    ]
+    assert NewtonManager._contacts.rigid_contact_max == 2
+
+
 def test_refit_sensor_bvh_rejects_missing_sensor_state(monkeypatch):
     """BVH refitting raises when a particle BVH exists without an initialized sensor state."""
     model = SimpleNamespace(shape_count=0, particle_count=1, bvh_particles=object())