[Backport] Set Warp's global determinism mode under --deterministic (#7561) (#7586)

# Description

Backports #7561 to `release/3.0.0`.

This branch cherry-picks the canonical merged commit
`838fb6d66288cc5f452c3aa1b1ad57f420e0f0e4` with `-x` provenance. The
change applied without conflicts or release-specific edits. The
repository backport validator confirms an exact patch replay across all
eight source paths, and the stable source/backport patch IDs match.

| Field | Commit |
|---|---|
| Original merged change | `838fb6d66288cc5f452c3aa1b1ad57f420e0f0e4` |
| Release base used | `1c754876008f0806fdcfda8e3a6b2f593b34d6fc` |
| Backport commit | `6044466990af2cb4b54f07f3705e37f16e6a4d58` |
| Stable patch ID | `643b6fbbd03866389488b17070db3504315b4a41` |

## Type of change

- Bug fix (non-breaking change which fixes an issue)
- Documentation update

## Release backport

- This PR targets the active `release/3.0.0` branch directly.

## Validation

- Repository exact-patch backport validation passed; all eight source
paths are preserved.
- Stable source/backport patch IDs match:
`643b6fbbd03866389488b17070db3504315b4a41`.
- `test_entrypoints_common.py` — 36 passed in an isolated uv
environment.
- All changed-file pre-commit hooks passed, including changelog and Git
LFS checks.
- `sphinx-lint` reported no problems for the modified RST files.
- `git diff --check upstream/release/3.0.0...HEAD` passed.
- The canonical project formatter and warning-as-error docs build cannot
resolve the release lock on this macOS/arm64 host; Linux CI remains
authoritative for those checks.

## Checklist

- [x] I have read and understood the contribution guidelines
- [x] I have run the available pre-commit checks
- [x] The original documentation update is preserved exactly
- [x] My changes generate no new warnings
- [x] The original regression coverage is preserved and passes
- [x] The `isaaclab_rl` changelog fragment is preserved
- [x] The original contributor is already listed in `CONTRIBUTORS.md`

Co-authored-by: hujc <jichuanh@nvidia.com>

Author: kellyguo11

docs/source/features/reproducibility.rst

@@ -35,7 +35,20 @@ for **RL-Games**, **skrl**, **RSL-RL**, and **Stable-Baselines3**: each calls
 so library initialization is not disturbed, then training proceeds with the requested global RNG and
 optional PyTorch deterministic algorithms. Whether the **rendering** half of the flag matters depends
 on the workload: **physics-only** simulation does not render at all; **RTX** rendering (non-minimal
-mode) needs it for reproducible imagery; **Newton** rendering is already deterministic.
+mode) needs it for reproducible imagery; **Newton** rendering needs Warp's global determinism mode,
+which the flag sets (see :ref:`below <reproducibility-warp-determinism>`).
+
+.. _reproducibility-warp-determinism:
+
+**Warp determinism.** Newton's solvers accept a ``deterministic`` argument that Isaac Lab supplies
+from :attr:`~isaaclab.physics.PhysicsCfg.deterministic`, and they apply it as a per-module option.
+Its sensor and geometry kernels take no such argument and fall back to ``warp.config.deterministic``,
+so the RL training and play entrypoints set that global to ``RUN_TO_RUN`` when
+``--deterministic`` is passed. A script that builds its own environment without those entrypoints
+must set ``warp.config.deterministic`` itself, before the environment is created. Without it the scene BVH is built
+over an atomically compacted shape list whose order varies between processes, and a tiled camera can
+render a few pixels differently from identical simulation state. An explicitly chosen mode, such as
+``GPU_TO_GPU``, is left untouched.
 
 .. note::
 

source/isaaclab_rl/changelog.d/warp-determinism-global.rst

@@ -0,0 +1,9 @@
+Fixed
+^^^^^
+
+* Fixed ``--deterministic`` not making camera observations reproducible. The flag configured the
+  physics solver but left :attr:`warp.config.deterministic` at ``NOT_GUARANTEED``, and Newton's
+  sensor and geometry kernels -- unlike its solvers -- take no per-module determinism option and
+  fall back to that global. The scene BVH is built over an atomically compacted shape list, so its
+  primitive order varied between processes and a tiled camera rendered a few pixels differently
+  from identical simulation state, which was enough to make image-observation training diverge.

source/isaaclab_rl/isaaclab_rl/entrypoints/backends/play_rl_games.py

@@ -30,6 +30,7 @@
     apply_video_recording,
     create_isaaclab_env,
     pre_launch_video_config,
+    request_determinism,
     resolve_checkpoint_selector,
     resolve_play_task_name,
     show_run_summary,
@@ -107,6 +108,8 @@ def main():
             train_task_name = task_name.replace("-Play", "")
 
             env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs
+            # Warp reads its determinism mode at module build time, so request it before the env exists.
+            request_determinism(args_cli, env_cfg)
             env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device
 
             if args_cli.seed == -1:

source/isaaclab_rl/isaaclab_rl/entrypoints/backends/play_rsl_rl.py

@@ -29,6 +29,7 @@
     apply_video_recording,
     create_isaaclab_env,
     pre_launch_video_config,
+    request_determinism,
     resolve_checkpoint_selector,
     resolve_play_task_name,
     show_run_summary,
@@ -126,6 +127,8 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, agen
 
             agent_cfg = cli_args.update_rsl_rl_cfg(agent_cfg, args_cli)
             env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs
+            # Warp reads its determinism mode at module build time, so request it before the env exists.
+            request_determinism(args_cli, env_cfg)
 
             agent_cfg = handle_deprecated_rsl_rl_cfg(agent_cfg, installed_version)
 

source/isaaclab_rl/isaaclab_rl/entrypoints/backends/play_sb3.py

@@ -27,6 +27,7 @@
     apply_video_recording,
     create_isaaclab_env,
     pre_launch_video_config,
+    request_determinism,
     resolve_checkpoint_selector,
     resolve_play_task_name,
     show_run_summary,
@@ -112,6 +113,8 @@ def main():
                 args_cli.seed = random.randint(0, 10000)
 
             env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs
+            # Warp reads its determinism mode at module build time, so request it before the env exists.
+            request_determinism(args_cli, env_cfg)
             agent_cfg["seed"] = args_cli.seed if args_cli.seed is not None else agent_cfg["seed"]
             env_cfg.seed = agent_cfg["seed"]
             env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device

source/isaaclab_rl/isaaclab_rl/entrypoints/backends/play_skrl.py

@@ -32,6 +32,7 @@
     create_isaaclab_env,
     pre_launch_video_config,
     preserve_attribute,
+    request_determinism,
     resolve_checkpoint_selector,
     resolve_play_task_name,
     show_run_summary,
@@ -165,6 +166,8 @@ def _main():
             train_task_name = task_name.replace("-Play", "")
 
             env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs
+            # Warp reads its determinism mode at module build time, so request it before the env exists.
+            request_determinism(args_cli, env_cfg)
             env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device
 
             # configure the ML framework into the global skrl variable

source/isaaclab_rl/isaaclab_rl/entrypoints/common.py

@@ -493,10 +493,61 @@ def apply_env_overrides(args_cli: argparse.Namespace, env_cfg: Any, *, apply_dev
     # --deterministic is an AppLauncher flag, so it only reaches carb settings on its own.
     # Record the request on the resolved physics config; each backend translates and validates
     # it when the simulation starts.
-    if getattr(args_cli, "deterministic", False):
-        physics_cfg = getattr(getattr(env_cfg, "sim", None), "physics", None)
-        if physics_cfg is not None:
-            physics_cfg.deterministic = True
+    request_determinism(args_cli, env_cfg)
+
+
+def request_determinism(args_cli: argparse.Namespace, env_cfg: Any) -> None:
+    """Record a ``--deterministic`` request on the config tree and on Warp's global.
+
+    Call this before the environment is created: Warp reads its setting at module build time
+    and the scene BVH is built while the environment is constructed.
+
+    Args:
+        args_cli: Parsed command-line arguments.
+        env_cfg: Isaac Lab environment config.
+    """
+    if not getattr(args_cli, "deterministic", False):
+        return
+    physics_cfg = getattr(getattr(env_cfg, "sim", None), "physics", None)
+    if physics_cfg is not None:
+        physics_cfg.deterministic = True
+    request_warp_determinism(physics_cfg)
+
+
+def request_warp_determinism(physics_cfg: Any) -> None:
+    """Ask Warp for deterministic atomics process-wide, matching the configured guarantee.
+
+    Newton's solvers take a ``deterministic`` argument and apply it as a per-module option, so a
+    solver-level request already covers the physics kernels. Its sensor and geometry modules take
+    no such argument and fall back to ``warp.config.deterministic``, which defaults to
+    ``NOT_GUARANTEED``. One of them, the BVH shape compaction in ``newton._src.geometry.bvh``,
+    claims output slots with ``wp.atomic_add``, so the enabled-shape order -- and with it the
+    order of the primitives the scene BVH is built over -- varies between processes. Ray queries
+    then break ties differently and a tiled camera renders a handful of pixels differently from
+    identical simulation state, which is enough to make an image-observation policy diverge.
+
+    A backend that names a stronger guarantee gets it here too: the strings accepted by
+    :attr:`~isaaclab_newton.physics.NewtonCfg.deterministic_mode` are the
+    ``warp.DeterministicMode`` members lowercased, so ``"gpu_to_gpu"`` selects
+    ``GPU_TO_GPU`` rather than being weakened to ``RUN_TO_RUN``. Warp reads the setting at module
+    build time, so it must land before the first kernel launch; :func:`apply_env_overrides` runs
+    before the environment is created.
+
+    The modes are ordered by strength, and this only ever raises the setting. Reading the current
+    value cannot tell a deliberate choice from the shipped default, so rather than guess at intent
+    it never weakens a guarantee already in place -- whoever set it, they wanted at least that much.
+
+    Args:
+        physics_cfg: Resolved physics config, or ``None`` when the config tree carries none.
+    """
+    import warp as wp
+
+    requested = getattr(physics_cfg, "deterministic_mode", None)
+    mode = getattr(wp.DeterministicMode, requested.upper(), None) if isinstance(requested, str) else None
+    if mode is None or mode == wp.DeterministicMode.NOT_GUARANTEED:
+        mode = wp.DeterministicMode.RUN_TO_RUN
+    if wp.config.deterministic < mode:
+        wp.config.deterministic = mode
 
 
 def validate_distributed_device(args_cli: argparse.Namespace) -> None:

source/isaaclab_rl/test/test_entrypoints_common.py

@@ -420,6 +420,51 @@ def test_apply_env_overrides_leaves_physics_alone_without_the_flag(monkeypatch:
     assert env_cfg.sim.physics.deterministic is False
 
 
+@pytest.mark.parametrize(
+    ("already_set", "configured_mode", "expected"),
+    [
+        # The request lands when nothing has asked for a guarantee yet.
+        ("NOT_GUARANTEED", None, "RUN_TO_RUN"),
+        # "not_guaranteed" is the shipped default, so it reads as unset rather than opt-out.
+        ("NOT_GUARANTEED", "not_guaranteed", "RUN_TO_RUN"),
+        ("NOT_GUARANTEED", "run_to_run", "RUN_TO_RUN"),
+        # A backend naming a stronger guarantee gets it, not a weakened one.
+        ("NOT_GUARANTEED", "gpu_to_gpu", "GPU_TO_GPU"),
+        ("RUN_TO_RUN", "gpu_to_gpu", "GPU_TO_GPU"),
+        # A guarantee already in place is never lowered.
+        ("GPU_TO_GPU", None, "GPU_TO_GPU"),
+        ("GPU_TO_GPU", "run_to_run", "GPU_TO_GPU"),
+    ],
+)
+def test_apply_env_overrides_raises_warp_determinism_to_the_configured_mode(
+    already_set: str, configured_mode: str | None, expected: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Warp's global is what covers Newton's sensor kernels, and it only ever moves upward."""
+    import warp as wp
+
+    monkeypatch.setattr(wp.config, "deterministic", getattr(wp.DeterministicMode, already_set))
+    attrs = {} if configured_mode is None else {"deterministic_mode": configured_mode}
+    env_cfg = SimpleNamespace(sim=SimpleNamespace(physics=_fake_physics_cfg("NewtonCfg", **attrs)))
+
+    args_cli = argparse.Namespace(num_envs=None, device=None, deterministic=True)
+    _rl_common.apply_env_overrides(args_cli, env_cfg, apply_device=False)
+
+    assert wp.config.deterministic == getattr(wp.DeterministicMode, expected)
+
+
+def test_apply_env_overrides_leaves_warp_alone_without_the_flag(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Without ``--deterministic`` Warp keeps its default, so no run pays for determinism."""
+    import warp as wp
+
+    monkeypatch.setattr(wp.config, "deterministic", wp.DeterministicMode.NOT_GUARANTEED)
+    env_cfg = SimpleNamespace(sim=SimpleNamespace(physics=_fake_physics_cfg("NewtonCfg")))
+
+    args_cli = argparse.Namespace(num_envs=None, device=None, deterministic=False)
+    _rl_common.apply_env_overrides(args_cli, env_cfg, apply_device=False)
+
+    assert wp.config.deterministic == wp.DeterministicMode.NOT_GUARANTEED
+
+
 @pytest.mark.parametrize("class_name", ["PhysxCfg", "OvPhysxCfg", "NewtonCfg", "SomeFutureBackendCfg"])
 def test_apply_env_overrides_records_the_request_for_every_backend(class_name: str) -> None:
     """The request is backend-agnostic, so the entrypoint needs no per-backend knowledge."""