[Workflow] Expand Newton Warp cache coverage (#7135)

## Summary

- Warm the Warp cache with all supported rigid Newton core and
multi-agent environments.
- Skip empty or unchanged cache snapshots and report byte-accurate
growth.

## Validation

- uv run --frozen isaaclab -f
- Collected 62 selected warm-cache cases.

Author: StafaH

.github/actions/run-package-tests/action.yml

@@ -211,6 +211,7 @@ runs:
         HOST_DIR: ${{ steps.warp-cache-key.outputs.host-dir }}
         MATCHED_KEY: ${{ steps.warp-cache-restore.outputs.cache-matched-key }}
         COLLECTION: ${{ steps.warp-cache-key.outputs.collection }}
+        CACHE_MODE: ${{ inputs.warp-cache }}
       run: |
         set -euo pipefail
         mkdir -p "$HOST_DIR"
@@ -228,9 +229,13 @@ runs:
           echo "Warp cache hit: ${MATCHED_KEY}"
           echo "WARP_CACHE_HIT=${MATCHED_KEY}" >> "$GITHUB_ENV"
         fi
-        echo "WARP_CACHE_MB_BEFORE=$(du -sm "$HOST_DIR" | cut -f1)" >> "$GITHUB_ENV"
+        echo "WARP_CACHE_BYTES_BEFORE=$(du -sb "$HOST_DIR" | cut -f1)" >> "$GITHUB_ENV"
 
         python3 .github/actions/run-package-tests/warp_cache_inventory.py "$HOST_DIR" "Warp cache restored"
+        if [ "$CACHE_MODE" = "save" ]; then
+          fingerprint="$(python3 .github/actions/run-package-tests/warp_cache_inventory.py "$HOST_DIR" --fingerprint)"
+          echo "WARP_CACHE_FINGERPRINT_BEFORE=$fingerprint" >> "$GITHUB_ENV"
+        fi
 
     - name: Setup wheelhouse registry authentication
       if: inputs.wheelhouse-image != ''
@@ -333,15 +338,16 @@ runs:
           exit 1
         fi
 
-    # Growth is the coverage signal: it is exactly the kernels this job needed
-    # that the collection did not already hold. Reporting only - no thresholds,
-    # since we have no measured basis for one yet.
+    # Net growth is a useful coverage signal for missing kernels. The writer
+    # also fingerprints the tree because recompilation can replace an existing
+    # artifact without changing the total size.
     - name: Report Warp cache growth
       if: always() && inputs.warp-cache != ''
       id: warp-cache-growth
       shell: bash
       env:
         HOST_DIR: ${{ steps.warp-cache-key.outputs.host-dir }}
+        CACHE_MODE: ${{ inputs.warp-cache }}
       run: |
         set -euo pipefail
 
@@ -351,31 +357,51 @@ runs:
           exit 0
         fi
 
-        before="${WARP_CACHE_MB_BEFORE:-0}"
-        after="$(du -sm "$HOST_DIR" | cut -f1)"
+        before="${WARP_CACHE_BYTES_BEFORE:-0}"
+        after="$(du -sb "$HOST_DIR" | cut -f1)"
         grew=$(( after - before ))
+        before_mb=$(( before / 1000000 ))
+        after_mb=$(( after / 1000000 ))
+        grew_mb=$(( grew / 1000000 ))
 
         if [ "${WARP_CACHE_HIT:-miss}" = "miss" ]; then
           verdict="cold run, nothing restored"
         elif [ "$grew" -le 0 ]; then
-          verdict="fully covered"
+          verdict="no net growth"
         else
           verdict="compiled $(( grew * 100 / (before > 0 ? before : 1) ))% beyond the restored cache"
         fi
 
-        echo "Warp cache: restored ${before} MB, ended at ${after} MB (+${grew} MB) - ${verdict}"
-        echo "🔵 Warp cache: ${before} -> ${after} MB (+${grew}) - ${verdict}" >> "$GITHUB_STEP_SUMMARY"
+        echo "Warp cache: restored ${before_mb} MB, ended at ${after_mb} MB (+${grew_mb} MB) - ${verdict}"
+        echo "🔵 Warp cache: ${before_mb} -> ${after_mb} MB (+${grew_mb}) - ${verdict}" >> "$GITHUB_STEP_SUMMARY"
         python3 .github/actions/run-package-tests/warp_cache_inventory.py "$HOST_DIR" "Warp cache final"
 
         # Publishing an empty tree would replace a good snapshot with nothing.
-        [ "$after" -lt 1 ] && echo "empty=true" >> "$GITHUB_OUTPUT" || echo "empty=false" >> "$GITHUB_OUTPUT"
+        if [ -z "$(find "$HOST_DIR" -type f -print -quit)" ]; then
+          echo "::warning::Warp cache contains no files; skipping publish"
+          echo "empty=true" >> "$GITHUB_OUTPUT"
+          echo "changed=false" >> "$GITHUB_OUTPUT"
+        else
+          echo "empty=false" >> "$GITHUB_OUTPUT"
+          if [ "$CACHE_MODE" = "save" ] && [ "${WARP_CACHE_HIT:-miss}" != "miss" ]; then
+            fingerprint="$(python3 .github/actions/run-package-tests/warp_cache_inventory.py "$HOST_DIR" --fingerprint)"
+            if [ "$fingerprint" = "${WARP_CACHE_FINGERPRINT_BEFORE:-}" ]; then
+              echo "Warp cache contents are unchanged; skipping duplicate snapshot"
+              echo "changed=false" >> "$GITHUB_OUTPUT"
+            else
+              echo "changed=true" >> "$GITHUB_OUTPUT"
+            fi
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+        fi
 
     # Not success(): the product is compiled kernels, and those stay valid when a
     # test assertion fails. Gating on success() let one failing env silently
     # discard the whole cache. Cancellation still skips, so a tree Warp was
     # mid-write on is never published.
     - name: Save Warp kernel cache
-      if: '!cancelled() && inputs.warp-cache == ''save'' && steps.warp-cache-growth.outputs.empty != ''true'''
+      if: '!cancelled() && inputs.warp-cache == ''save'' && steps.warp-cache-growth.outputs.empty != ''true'' && steps.warp-cache-growth.outputs.changed == ''true'''
       uses: actions/cache/save@v4
       with:
         path: ${{ steps.warp-cache-key.outputs.host-dir }}

.github/actions/run-package-tests/warp_cache_inventory.py

@@ -10,10 +10,13 @@
 cache size and retention with measurements - module counts, artifact mix, and
 the spread of GPU targets a shared collection has accumulated.
 
-Usage: warp_cache_inventory.py <cache-dir> [label]
+Usage:
+    warp_cache_inventory.py <cache-dir> [label]
+    warp_cache_inventory.py <cache-dir> --fingerprint
 """
 
 import collections
+import hashlib
 import os
 import pathlib
 import re
@@ -22,8 +25,33 @@
 SM_PATTERN = re.compile(r"\.(sm\d+[a-z]?)\.")
 
 
+def fingerprint(root: pathlib.Path) -> str:
+    """Return a stable digest of every cache file's relative path and contents.
+
+    This lets the writer avoid publishing another immutable GitHub cache entry
+    when a warm run only read the previously restored files.
+    """
+    digest = hashlib.sha256()
+    for path in sorted(path for path in root.rglob("*") if path.is_file()):
+        digest.update(path.relative_to(root).as_posix().encode())
+        digest.update(b"\0")
+        file_digest = hashlib.sha256()
+        try:
+            with path.open("rb") as stream:
+                for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+                    file_digest.update(chunk)
+        except OSError:
+            continue
+        digest.update(file_digest.digest())
+    return digest.hexdigest()
+
+
 def main() -> int:
     root = pathlib.Path(sys.argv[1])
+    if len(sys.argv) > 2 and sys.argv[2] == "--fingerprint":
+        print(fingerprint(root))
+        return 0
+
     label = sys.argv[2] if len(sys.argv) > 2 else "Warp cache"
 
     if not root.is_dir():

.github/workflows/build.yaml

@@ -896,9 +896,9 @@ jobs:
 
   # Publishes the shared Warp kernel cache the test jobs restore. Needed as its
   # own job because a cache written from a PR is readable only by that PR, and
-  # no solver-heavy test job runs on push. Does not restore before saving:
-  # appending to the previous run grows the cache without bound, which is what
-  # exhausted the repository quota last time.
+  # no solver-heavy test job runs on push. Restores the newest compatible
+  # snapshot so coverage accumulates, but only publishes when its contents
+  # change to avoid filling the repository quota with duplicate snapshots.
   warm-warp-cache:
     name: "warp-cache-warm"
     runs-on: [self-hosted, gpu]
@@ -933,20 +933,16 @@ jobs:
         isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
         isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
         filter-pattern: "isaaclab_tasks"
-        # MJWarp specializes kernels per model config (@cache_kernel in
-        # mujoco_warp/_src/warp_util.py keys on array sizes; ~250 wp.static uses
-        # feed Warp's module hash), so these span specialization axes rather than
-        # maximize task count: Cartpole (primitives, contact-free, + camera),
-        # Drawer (mesh collision), AnymalD (contact-rich locomotion), Handover
-        # (dexterous, many contacts). Deformable/MPM kernels are not covered;
-        # widen if a test job reports large cache growth.
+        # Warm every supported rigid core environment on Newton. MJWarp
+        # specializes kernels per model config, so a small representative task
+        # set leaves most of the cache to be compiled again by each PR test
+        # shard. The multi-agent suite supplies Handover.
         include-files: >-
-          test_environments_isaacsim_physx.py,
           test_environments_newton.py,
-          test_environments_ovphysx.py
-        # No Soft/Cloth: the deformable envs depend on optional extras the CI
-        # image does not install (Soft needs pytetwild), so they only ever fail.
-        test-k-expr: "Cartpole or Drawer or AnymalD or Handover"
+          test_multi_agent_environments.py
+        # Deformable tasks need optional dependencies that are not installed in
+        # this image.
+        test-k-expr: "test_environments and not (Soft or Cloth or Cable)"
         warp-cache: ${{ env.PUBLISHES_WARP_CACHE == 'true' && 'save' || 'restore' }}
         container-name: isaac-lab-warp-cache-warm
         omni-github-test-type: warp-cache-warm