[Startup] Avoid replicated Shadow Hand camera USD work

scripts/demos/hands.py

@@ -51,8 +51,7 @@
 
 from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg  # isort:skip
 from isaaclab_assets.robots.allegro import ALLEGRO_HAND_CFG  # isort:skip
-from isaaclab_assets.robots.shadow_hand import SHADOW_HAND_CFG  # isort:skip
-from isaaclab_tasks.core.reorient.config.shadow_hand.shadow_hand_common import ShadowHandRobotCfg  # isort:skip
+from isaaclab_assets.robots.shadow_hand import SHADOW_HAND_CFG, SHADOW_HAND_NEWTON_CFG  # isort:skip
 
 if TYPE_CHECKING:
     from isaaclab.assets import Articulation
@@ -95,7 +94,7 @@ def design_scene() -> tuple[dict, list[list[float]]]:
     # Origin 2 with Shadow Hand
     sim_utils.create_prim("/World/Origin2", "Xform", translation=origins[1])
     # -- Robot
-    shadow_hand_cfg = ShadowHandRobotCfg().newton_mjwarp if args_cli.physics == "newton_mjwarp" else SHADOW_HAND_CFG
+    shadow_hand_cfg = SHADOW_HAND_NEWTON_CFG if args_cli.physics == "newton_mjwarp" else SHADOW_HAND_CFG
     shadow_hand_cfg = shadow_hand_cfg.replace(prim_path="/World/Origin2/Robot")
     shadow_hand = shadow_hand_cfg.class_type(shadow_hand_cfg)
 

source/isaaclab/changelog.d/fix-shadow-hand-demo-config.rst

@@ -0,0 +1,5 @@
+Fixed
+^^^^^
+
+* Fixed the Newton Shadow Hand demo failing at startup after inheriting the reorientation task's
+  prototype spawn path.

source/isaaclab/test/app/test_standalone_scripts.py

@@ -242,6 +242,21 @@ def test_commands_respect_script_launcher_capabilities():
     assert "--enable_cameras" not in usd_camera_case.command()
 
 
+def test_hands_demo_uses_asset_owned_shadow_hand_configs():
+    """The generic hands demo must not inherit task-specific spawn policy."""
+    path = script_cases.ROOT / "scripts/demos/hands.py"
+    tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+    imports = {
+        (node.module, alias.name) for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) for alias in node.names
+    }
+
+    assert not {module for module, _ in imports if module and module.startswith("isaaclab_tasks")}
+    assert {
+        ("isaaclab_assets.robots.shadow_hand", "SHADOW_HAND_CFG"),
+        ("isaaclab_assets.robots.shadow_hand", "SHADOW_HAND_NEWTON_CFG"),
+    } <= imports
+
+
 @pytest.mark.parametrize(
     "relative_path",
     [

source/isaaclab_newton/changelog.d/fix-newton-segmentation-prototype-labels.rst

@@ -0,0 +1,9 @@
+Fixed
+^^^^^
+
+* Fixed the Newton Warp renderer reporting UNLABELLED semantic and instance segmentation for every
+  environment but the prototype when the scene spawns only the prototype
+  (:attr:`~isaaclab.sim.spawners.SpawnerCfg.spawn_path`) and relies on backend replication. Shapes
+  cloned by the physics backend now resolve their labels through the prototype environment recorded
+  in the clone plan, with the matched ancestor rebased into the cloned environment so instance ids
+  stay distinct per environment.

source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py

@@ -502,7 +502,8 @@ def create_render_data(self, spec: CameraRenderSpec) -> RenderData:
             or RenderBufferKind.INSTANCE_SEGMENTATION in spec.cfg.data_types
         ):
             if self._seg_mapper is None:
-                self._seg_mapper = NewtonSegmentationMapper(self._newton_model, self._stage, self.cfg)
+                clone_plan = SimulationContext.instance().get_clone_plan()
+                self._seg_mapper = NewtonSegmentationMapper(self._newton_model, self._stage, self.cfg, clone_plan)
         if RenderBufferKind.SEMANTIC_SEGMENTATION in spec.cfg.data_types:
             self._seg_mapper.build_mapping(
                 RenderBufferKind.SEMANTIC_SEGMENTATION, bool(self.cfg.colorize_semantic_segmentation)

source/isaaclab_newton/isaaclab_newton/renderers/segmentation.py

@@ -28,6 +28,8 @@
 
 # Colorization (host ``random_color_from_id`` / ``pack_rgba``) and the reserved BACKGROUND / UNLABELLED
 # ids are shared with the RTX and OVRTX renderers to keep colorized segmentation visually consistent.
+from isaaclab.cloner import path as cloner_path
+from isaaclab.cloner import query as cloner_query
 from isaaclab.renderers.segmentation_colors import BACKGROUND_ID, UNLABELLED_ID, pack_rgba, random_color_from_id
 from isaaclab.utils.timer import Timer
 
@@ -36,6 +38,8 @@
 
     from pxr import Usd
 
+    from isaaclab.cloner import ClonePlan
+
 _UNLABELLED_COLOR: int = 0xFF000000
 """Packed RGBA color for UNLABELLED pixels: ``(0, 0, 0, 255)`` opaque black."""
 
@@ -254,8 +258,8 @@ def convert_shape_index_to_output(self, shape_index: wp.array, out_view: wp.arra
 class NewtonSegmentationMapper:
     """Builds per-shape segmentation lookup tables from a Newton model and its USD stage."""
 
-    def __init__(self, model: newton.Model, stage: Usd.Stage | None, cfg) -> None:
-        """Initialize the mapper from the Newton model, USD stage, and renderer config.
+    def __init__(self, model: newton.Model, stage: Usd.Stage | None, cfg, clone_plan: ClonePlan) -> None:
+        """Initialize the mapper from the Newton model, USD stage, renderer config, and clone plan.
 
         Construction is cheap — it only captures references and snapshots ``model.shape_label``.
         Call :meth:`build_mapping` to do the actual per-shape USD resolution and id assignment.
@@ -265,6 +269,9 @@ def __init__(self, model: newton.Model, stage: Usd.Stage | None, cfg) -> None:
             stage: The live USD stage used to read :class:`UsdSemantics.LabelsAPI` labels. May be
                 ``None`` in stageless setups, in which case every shape is treated as unlabelled.
             cfg: Renderer config exposing ``semantic_filter`` and ``semantic_segmentation_mapping``.
+            clone_plan: The scene's published :class:`~isaaclab.cloner.ClonePlan`, used to fall back
+                to the prototype env when a replicated shape has no prim on the stage (backend-only
+                replication). See :meth:`_resolve_via_prototype`.
         """
         self._model = model
         self._stage = stage
@@ -276,6 +283,7 @@ def __init__(self, model: newton.Model, stage: Usd.Stage | None, cfg) -> None:
         # Cache of prim path -> (matched_labels or None); labels resolved with ancestor inheritance.
         self._matched_cache: dict[str, tuple[dict[SemanticType, SemanticLabels], SemanticPrimPath] | None] = {}
         self._mappings: dict[tuple[str, bool], NewtonSegmentationMapping] = {}
+        self._clone_plan = clone_plan
 
     def build_mapping(self, kind: _SegKind, colorize: bool) -> None:
         """Build and cache the :class:`NewtonSegmentationMapping` for ``kind`` at the requested colorization."""
@@ -311,10 +319,28 @@ def _resolve_semantic_match(
         short-circuits the traversal immediately. All newly-visited paths are back-filled with
         ``result`` at the end, so sibling shapes that share an ancestry prefix resolve in O(1)
         on subsequent calls without re-walking the hierarchy or re-querying USD.
+
+        When the walk finds nothing, :meth:`_resolve_via_prototype` retries against the prototype
+        environment, covering scenes replicated only in the physics backend.
         """
         if prim_path in self._matched_cache:
             return self._matched_cache[prim_path]
 
+        result = self._walk_for_labels(prim_path)
+        if result is None:
+            result = self._resolve_via_prototype(prim_path)
+            # Overwrite the ``None`` the walk back-filled for this path; ancestors keep theirs so
+            # a sibling shape re-enters the prototype fallback rather than reusing a stale miss.
+            self._matched_cache[prim_path] = result
+        return result
+
+    def _walk_for_labels(self, prim_path: str) -> tuple[dict[SemanticType, SemanticLabels], SemanticPrimPath] | None:
+        """Walk ``prim_path`` up to the stage root for the nearest ancestor passing the filter.
+
+        The stage-only half of :meth:`_resolve_semantic_match`: it performs the traversal and the
+        cache back-fill described there, and returns ``None`` when no ancestor carries a matching
+        label — including when ``prim_path`` names no prim at all.
+        """
         result: tuple[dict[SemanticType, SemanticLabels], SemanticPrimPath] | None = None
         # Paths visited this traversal that were not already in the cache; back-filled at the end.
         traversed: list[str] = []
@@ -347,6 +373,52 @@ def _resolve_semantic_match(
             self._matched_cache[prim_path] = result
         return result
 
+    def _resolve_via_prototype(
+        self, prim_path: str
+    ) -> tuple[dict[SemanticType, SemanticLabels], SemanticPrimPath] | None:
+        """Resolve a replicated shape's labels through the prototype env it was cloned from.
+
+        Newton replicates the *model*, rewriting each cloned shape's ``shape_label`` to its per-env
+        path (see ``isaaclab_newton.cloner.rename_builder_labels``), but a scene that spawns only
+        the prototype (:attr:`~isaaclab.sim.spawners.SpawnerCfg.spawn_path`) authors USD prims —
+        and therefore :class:`UsdSemantics.LabelsAPI` labels — for that one env only. Those clones
+        would otherwise resolve to UNLABELLED, so their labels are read off the prototype instead.
+
+        The matched ancestor is rebased back onto the clone before being returned, because
+        ``instance_segmentation`` groups by that path: leaving it on the prototype side would
+        collapse every environment into a single instance id.
+
+        Returns:
+            The prototype's ``(filtered_labels, matched_ancestor_path)`` with the ancestor rebased
+            into ``prim_path``'s environment, or ``None`` when ``prim_path`` is not a clone, or the
+            prototype itself is unlabelled.
+
+        Raises:
+            ValueError: When ``prim_path`` is owned by multiple distinct, equally near destination
+                templates — a malformed clone plan, not a state to resolve around.
+        """
+        resolved = cloner_query.path_to_source(self._clone_plan, prim_path)
+        if resolved is None:
+            # ``prim_path`` is not owned by the clone plan at all (e.g. an un-cloned ground plane or
+            # other static prim) — nothing to fall back to, so it stays unlabelled.
+            return None
+        source_root, _, asset_suffix = resolved
+        prototype_path = source_root + asset_suffix
+        # The prototype path is where the labels actually live, so a plain stage walk — not another
+        # round of clone-plan resolution — is all that's needed here. When ``prim_path`` names the
+        # prototype's own env, this re-walks the same path the caller already walked; the cache it
+        # left behind makes that a cheap no-op rather than a correctness concern.
+        match = self._walk_for_labels(prototype_path)
+        if match is None:
+            return None
+        matched, ancestor_path = match
+        # ``asset_suffix`` is the part of ``prim_path`` below the destination template, so trimming
+        # it yields this clone's counterpart of ``source_root``. An ancestor above ``source_root``
+        # (a label authored outside the cloned subtree) is genuinely shared and ``rebase`` leaves
+        # it alone, keeping such shapes in one instance group across envs.
+        clone_root = prim_path[: len(prim_path) - len(asset_suffix)] if asset_suffix else prim_path
+        return matched, cloner_path.rebase(ancestor_path, source_root, clone_root)
+
     def _apply_filter(self, labels: dict[SemanticType, SemanticLabels]) -> dict[SemanticType, SemanticLabels]:
         """Restrict ``labels`` (``{type: [labels]}``) to the types/labels passing the semantic filter.