Repoint extras-qualified prebundle mirrors alongside the flat package

Isaac Sim ships some prebundled packages twice: a flat directory and an
extras-qualified mirror, <pkg>[extras]/<pkg>-<version>-*/<pkg>, whose contents
are per-file symlinks into the flat copy. Only the flat directory was
repointed, so the mirror kept pointing at the shipped version's file list.

MEASURED on the Isaac Sim image: newton[sim]/newton-1.5.0-py3-none-any/newton
holds 842 symlinks, and 330 of their targets do not exist in the newton the
pin installs, 232 of them under _src/solvers/kamino.

Collapse each mirror to the same symlink the flat directory already gets, so
one directory link replaces a per-file tree that goes stale on every bump.
Mirrors are matched by name rather than glob, since newton[sim] is a valid
character class that would otherwise match newtons.

Author: hujc7

source/isaaclab/changelog.d/jichuanh-repoint-nested-prebundle-mirrors.rst

@@ -0,0 +1,7 @@
+Fixed
+^^^^^
+
+* Fixed :func:`~isaaclab.cli.commands.install.command_install` leaving extras-qualified
+  prebundle mirrors (``<pkg>[extras]/<pkg>-<version>-*/<pkg>``) pointing at the previously
+  shipped file list, which broke Isaac Sim extensions whenever a package was repointed to a
+  version that renamed or dropped files.

source/isaaclab/isaaclab/cli/commands/install.py

@@ -962,6 +962,41 @@ def _discover_prebundle_dirs() -> set[Path]:
     return prebundle_dirs
 
 
+def _nested_prebundle_mirrors(prebundle_dir: Path, pkg_name: str) -> list[Path]:
+    """Return ``<pkg>[extras]/<pkg>-<version>-*/<pkg>`` mirrors of a prebundled package.
+
+    Isaac Sim's ``isaacsim.pip.newton`` prebundle, for example, holds both a flat ``newton``
+    directory and ``newton[sim]/newton-1.5.0-py3-none-any/newton``, the latter built from
+    per-file symlinks into the former. Only the flat copy is named in
+    :data:`_PREBUNDLE_REPOINT_PACKAGES`, so the mirrors have to be discovered by shape.
+
+    Args:
+        prebundle_dir: A ``pip_prebundle`` directory to search.
+        pkg_name: Package directory name being repointed.
+
+    Returns:
+        Existing mirror directories, which may be empty.
+    """
+    mirrors: list[Path] = []
+    # ``pkg_name`` may contain glob metacharacters once the extras suffix is appended
+    # (``newton[sim]`` is a character class), so match by name rather than globbing.
+    prefix = f"{pkg_name}["
+    try:
+        extras_dirs = [p for p in prebundle_dir.iterdir() if p.is_dir() and p.name.startswith(prefix)]
+    except OSError:
+        return mirrors
+    for extras_dir in extras_dirs:
+        try:
+            wheel_dirs = [p for p in extras_dir.iterdir() if p.is_dir()]
+        except OSError:
+            continue
+        for wheel_dir in wheel_dirs:
+            candidate = wheel_dir / pkg_name
+            if candidate.is_symlink() or candidate.is_dir():
+                mirrors.append(candidate)
+    return mirrors
+
+
 def _find_dangling_prebundle_symlinks() -> set[Path]:
     """Find symlinks under Isaac Sim prebundles whose targets do not resolve.
 
@@ -1084,24 +1119,31 @@ def _repoint_prebundle_packages() -> None:
                 print_debug(f"Skipping repoint of {prebundled}: {venv_pkg} lacks CUDA subpackages (cudnn missing).")
                 continue
 
-            try:
-                # Already repointed to the right place — nothing to do.
-                if prebundled.is_symlink() and prebundled.resolve() == venv_pkg.resolve():
-                    continue
-                # Replace the prebundled copy (a stale symlink or a real directory)
-                # with a symlink to the active environment. We remove rather than
-                # rename-to-``.bak``: the env copy is the symlink target, so the
-                # prebundle content is redundant, and renaming a directory on an
-                # overlayfs lower layer (Docker image build) fails with ``EXDEV``.
-                _force_remove(prebundled)
-                if use_symlinks:
-                    prebundled.symlink_to(venv_pkg)
-                else:
-                    shutil.copytree(venv_pkg, prebundled)
-                repointed += 1
-                print_debug(f"Repointed {prebundled} -> {venv_pkg}")
-            except OSError as exc:
-                print_warning(f"Could not repoint {prebundled}: {exc} — skipping.")
+            # Some Isaac Sim builds ship a second copy of the wheel beside the flat directory,
+            # under ``<pkg>[extras]/<pkg>-<version>-*/<pkg>``, whose contents are per-file
+            # symlinks back into the flat copy. That mirror encodes the shipped version's exact
+            # file list, so repointing only the flat directory leaves every symlink whose file
+            # the new version renamed or dropped dangling. Collapse each mirror to the same
+            # target: one symlink cannot go stale the way a per-file tree does.
+            for target in [prebundled, *_nested_prebundle_mirrors(prebundle_dir, pkg_name)]:
+                try:
+                    # Already repointed to the right place — nothing to do.
+                    if target.is_symlink() and target.resolve() == venv_pkg.resolve():
+                        continue
+                    # Replace the prebundled copy (a stale symlink or a real directory)
+                    # with a symlink to the active environment. We remove rather than
+                    # rename-to-``.bak``: the env copy is the symlink target, so the
+                    # prebundle content is redundant, and renaming a directory on an
+                    # overlayfs lower layer (Docker image build) fails with ``EXDEV``.
+                    _force_remove(target)
+                    if use_symlinks:
+                        target.symlink_to(venv_pkg)
+                    else:
+                        shutil.copytree(venv_pkg, target)
+                    repointed += 1
+                    print_debug(f"Repointed {target} -> {venv_pkg}")
+                except OSError as exc:
+                    print_warning(f"Could not repoint {target}: {exc} — skipping.")
 
     if repointed:
         print_info(

source/isaaclab/test/cli/test_install_prebundle.py

@@ -18,6 +18,7 @@
 from isaaclab.cli.commands.install import (
     _assert_no_new_dangling_prebundle_symlinks,
     _find_dangling_prebundle_symlinks,
+    _nested_prebundle_mirrors,
     _torch_first_on_sys_path_is_prebundle,
     split_install_items,
 )
@@ -160,3 +161,56 @@ def test_non_package_dangles_warn_but_pass(self, tmp_path):
         (services / "test_module.py").symlink_to(core / "gone-test.py")
         with mock.patch("isaaclab.cli.commands.install._discover_prebundle_dirs", return_value={core, services}):
             _assert_no_new_dangling_prebundle_symlinks(set())
+
+
+# ---------------------------------------------------------------------------
+# _nested_prebundle_mirrors
+# ---------------------------------------------------------------------------
+
+
+class TestNestedPrebundleMirrors:
+    """Tests for :func:`_nested_prebundle_mirrors`.
+
+    Isaac Sim's ``isaacsim.pip.newton`` prebundle carries both a flat ``newton`` directory and
+    an extras-qualified ``newton[sim]/newton-<version>-py3-none-any/newton`` mirror built from
+    per-file symlinks. Repointing only the flat copy leaves the mirror pointing at the previous
+    version's file list.
+    """
+
+    def test_extras_qualified_mirror_is_found(self, tmp_path):
+        mirror = tmp_path / "newton[sim]" / "newton-1.5.0-py3-none-any" / "newton"
+        mirror.mkdir(parents=True)
+
+        assert _nested_prebundle_mirrors(tmp_path, "newton") == [mirror]
+
+    def test_flat_package_is_not_reported_as_its_own_mirror(self, tmp_path):
+        """The caller already repoints the flat directory; returning it again would double-count."""
+        (tmp_path / "newton").mkdir()
+
+        assert _nested_prebundle_mirrors(tmp_path, "newton") == []
+
+    def test_package_without_an_extras_directory_has_no_mirror(self, tmp_path):
+        (tmp_path / "torch").mkdir()
+        (tmp_path / "newton[sim]" / "newton-1.5.0-py3-none-any" / "newton").mkdir(parents=True)
+
+        assert _nested_prebundle_mirrors(tmp_path, "torch") == []
+
+    def test_bracket_in_the_name_is_matched_literally_not_as_a_glob(self, tmp_path):
+        """``newton[sim]`` is a valid glob character class, so name matching must not glob."""
+        # ``newton[sim]`` as a class matches the single characters s, i and m.
+        (tmp_path / "newtons" / "newton-1.5.0-py3-none-any" / "newton").mkdir(parents=True)
+        real = tmp_path / "newton[sim]" / "newton-1.5.0-py3-none-any" / "newton"
+        real.mkdir(parents=True)
+
+        assert _nested_prebundle_mirrors(tmp_path, "newton") == [real]
+
+    def test_a_mirror_that_is_a_symlink_is_still_reported(self, tmp_path):
+        """Repointing is idempotent, so an already-collapsed mirror must stay discoverable."""
+        target = tmp_path / "env_newton"
+        target.mkdir()
+        wheel_dir = tmp_path / "newton[sim]" / "newton-1.5.0-py3-none-any"
+        wheel_dir.mkdir(parents=True)
+        mirror = wheel_dir / "newton"
+        mirror.symlink_to(target)
+
+        assert _nested_prebundle_mirrors(tmp_path, "newton") == [mirror]