Merge branch 'develop' into shauryad/bug_fixes

Author: frlai

.github/actions/ecr-build-push-pull/README.md

@@ -16,6 +16,19 @@ ECR is also used as the BuildKit layer cache.
     ecr-url: (optional, complete url for ECR storage)
 ```
 
+## Verifying a freshly built image
+
+Pass `verify-test-path` to assert against the image before it is published:
+
+```yaml
+    verify-test-path: docker/test/test_image_invariants.py
+```
+
+The tests run only on a full build, with `IMAGE_TAG` set, so the caller's job needs `uv`
+(`astral-sh/setup-uv`). A failure fails the action with nothing pushed, so the next run
+rebuilds rather than serving the bad image from the deps cache. Exact-tag and deps-cache hits skip
+them: that image passed when it was built.
+
 ## ECR URL resolution order
 
 1. `ecr-url` input

.github/actions/ecr-build-push-pull/action.yml

@@ -37,6 +37,16 @@ inputs:
     description: Tag used for the ECR layer cache image (e.g. "cache-base", "cache-curobo").
     required: false
     default: 'cache'
+  verify-test-path:
+    description: >
+      Path to a test file or directory asserted against a freshly built image, before it is
+      tagged or pushed. Tests run with IMAGE_TAG set; a failure fails the action with nothing
+      published, so the next run rebuilds instead of inheriting the bad image from the cache.
+
+      Not run on an exact-tag or deps-cache hit: those serve an image that already passed when it
+      was built.
+    required: false
+    default: ''
   pull-on-deps-hit:
     description: >
       Pull the image locally after a deps-cache hit. Needed by jobs that run
@@ -242,6 +252,23 @@ runs:
         cache-to: ${{ steps.resolve-ecr.outputs.available == 'true' && format('type=registry,ref={0},mode=max', env.CACHE_IMAGE) || '' }}
         deps-hash: ${{ steps.deps-hash.outputs.hash }}
 
+    # Assert against the image while it is only local: the push steps below publish under both
+    # the commit tag and the deps tag, and a deps-cache hit later serves that image without
+    # rebuilding it, so anything published unverified stays unverified.
+    - name: Verify freshly built image
+      if: >
+        inputs.verify-test-path != '' &&
+        steps.pull-exact.outputs.hit != 'true' &&
+        steps.deps-cache.outputs.deps-cache-hit != 'true'
+      shell: bash
+      env:
+        IMAGE_TAG: ${{ inputs.image-tag }}
+        TEST_PATH: ${{ inputs.verify-test-path }}
+      run: |
+        set -euo pipefail
+        uv run --no-project --with pytest \
+          python -m pytest -q "${TEST_PATH}"
+
     - name: Tag built image with ECR-prefixed name
       if: >
         steps.resolve-ecr.outputs.available == 'true' &&

.github/actions/ovrtx-shader-cache/action.yml

@@ -0,0 +1,150 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+name: 'OVRTX Shader Cache'
+description: >
+  Restores, reports on and publishes the NVIDIA driver PSO blobs (nv_shadercache)
+  that Isaac Lab rendering tests would otherwise recompile from scratch on every
+  run. Requires a prior checkout.
+
+  A composite action cannot span the caller's test step, so the phases are
+  separate invocations selected by 'mode': 'restore' before the tests, then
+  either 'report' or 'save' after them. Every mode recomputes the keys from the
+  same key.sh, so the collection a job reads and the one it writes cannot drift.
+
+  kit/ is compiled by the RTX renderer inside the Isaac Sim image and kitless/ by
+  the pip-installed ovrtx wheel, so each tree is a separate entry keyed by its own
+  producer. The mount layout that fills them lives in
+  .github/actions/run-tests/run_tests.sh.
+
+inputs:
+  mode:
+    description: >-
+      'restore' reads the newest compatible snapshot of each tree and never
+      writes. 'report' summarises how far the run compiled beyond what was
+      restored. 'save' reports the same way and writes each tree the run
+      actually added to back as a new snapshot; only the cache warmer uses it.
+    required: true
+  isaacsim-version:
+    description: 'Isaac Sim image tag; identifies the exact Kit RTX build that compiles the kit/ blobs'
+    required: true
+  publishes:
+    description: >-
+      'true' when the same job later invokes this action in 'save' mode. The
+      restore pass then fingerprints what it restored, so the save pass can tell
+      a run that compiled new blobs from one that only read back the snapshot it
+      started with. Other modes ignore it; leaving it false only costs a
+      duplicate snapshot, never a lost one.
+    required: false
+    default: 'false'
+  trees:
+    description: >-
+      Which cache tree(s) this job needs: 'kit', 'kitless', or 'both'. A job that
+      only exercises one render path should request just that tree, so it does
+      not pay the restore/save cost of the tree it never populates.
+    default: 'both'
+    required: false
+
+outputs:
+  host-dir:
+    description: 'Parent directory holding both cache trees; bind-mounted into the test container'
+    value: ${{ steps.compute.outputs.host-dir }}
+
+runs:
+  using: composite
+  steps:
+    # always() here and on every post-test step below: the caller already gates
+    # this invocation on job status, and without it the composite inherits a
+    # failed job context and skips work that stays valid after a test failure.
+    # This step in particular has to run in every mode, since every later step
+    # reads its outputs.
+    - name: Compute OVRTX shader cache keys
+      id: compute
+      if: always()
+      shell: bash
+      env:
+        ISAACSIM_VERSION: ${{ inputs.isaacsim-version }}
+      run: bash "$GITHUB_ACTION_PATH/key.sh"
+
+    - name: Restore OVRTX kit shader cache
+      if: inputs.mode == 'restore' && inputs.trees != 'kitless'
+      id: restore-kit
+      uses: actions/cache/restore@v4
+      with:
+        path: ${{ steps.compute.outputs.kit-dir }}
+        key: ${{ steps.compute.outputs.kit-key }}
+        restore-keys: ${{ steps.compute.outputs.kit-restore-keys }}
+
+    - name: Restore OVRTX kitless shader cache
+      if: inputs.mode == 'restore' && inputs.trees != 'kit'
+      id: restore-kitless
+      uses: actions/cache/restore@v4
+      with:
+        path: ${{ steps.compute.outputs.kitless-dir }}
+        key: ${{ steps.compute.outputs.kitless-key }}
+        restore-keys: ${{ steps.compute.outputs.kitless-restore-keys }}
+
+    - name: Report restored OVRTX shader cache
+      if: inputs.mode == 'restore'
+      shell: bash
+      env:
+        HOST_DIR: ${{ steps.compute.outputs.host-dir }}
+        TREES: ${{ inputs.trees }}
+        KIT_MATCHED_KEY: ${{ steps.restore-kit.outputs.cache-matched-key }}
+        KITLESS_MATCHED_KEY: ${{ steps.restore-kitless.outputs.cache-matched-key }}
+        KIT_COLLECTION: ${{ steps.compute.outputs.kit-collection }}
+        KITLESS_COLLECTION: ${{ steps.compute.outputs.kitless-collection }}
+        PUBLISHES: ${{ inputs.publishes }}
+      run: bash "$GITHUB_ACTION_PATH/report.sh" restore
+
+    - name: Report OVRTX shader cache growth
+      if: always() && inputs.mode != 'restore'
+      id: growth
+      shell: bash
+      env:
+        HOST_DIR: ${{ steps.compute.outputs.host-dir }}
+        TREES: ${{ inputs.trees }}
+      run: bash "$GITHUB_ACTION_PATH/report.sh" growth
+
+    # Each tree is gated on its own file count so a populated kit/ never carries
+    # an empty kitless/ into a published snapshot. The count is empty rather than
+    # '0' when the growth step aborted before counting; both mean "not measured"
+    # and both must block the save, since the snapshot would publish under a key
+    # every consumer prefers over the last good one.
+    #
+    # The changed gate is a quota guard: every save writes a new immutable entry,
+    # so a warm run that only read back what it restored would spend ~270 MB of
+    # the repository's cache allowance on a duplicate of the snapshot it started
+    # from. It is 'true' whenever the growth step could not rule that out.
+    - name: Save OVRTX kit shader cache
+      if: >-
+        always() && inputs.mode == 'save' && inputs.trees != 'kitless'
+        && steps.growth.outputs.kit-files != ''
+        && steps.growth.outputs.kit-files != '0'
+        && steps.growth.outputs.kit-changed == 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: ${{ steps.compute.outputs.kit-dir }}
+        key: ${{ steps.compute.outputs.kit-key }}
+
+    - name: Save OVRTX kitless shader cache
+      if: >-
+        always() && inputs.mode == 'save' && inputs.trees != 'kit'
+        && steps.growth.outputs.kitless-files != ''
+        && steps.growth.outputs.kitless-files != '0'
+        && steps.growth.outputs.kitless-changed == 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: ${{ steps.compute.outputs.kitless-dir }}
+        key: ${{ steps.compute.outputs.kitless-key }}
+
+    - name: Verify requested OVRTX shader cache tree(s) were populated
+      if: always() && inputs.mode == 'save'
+      shell: bash
+      env:
+        TREES: ${{ inputs.trees }}
+        KIT_FILES: ${{ steps.growth.outputs.kit-files }}
+        KITLESS_FILES: ${{ steps.growth.outputs.kitless-files }}
+      run: bash "$GITHUB_ACTION_PATH/verify.sh"

.github/actions/ovrtx-shader-cache/key.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# Computes the collection prefix, per-run write key and host directory for each
+# of the two OVRTX shader cache trees, and appends them to $GITHUB_OUTPUT.
+#
+# Every mode of the action runs this, so the collection a job restores from and
+# the one it saves to cannot drift. Only values that stay constant for the
+# lifetime of a job may be keyed on.
+#
+# Reads: ISAACSIM_VERSION, GITHUB_WORKSPACE (for pyproject.toml), RUNNER_OS,
+# RUNNER_ARCH, RUNNER_TEMP, GITHUB_SHA, GITHUB_RUN_ID, GITHUB_RUN_ATTEMPT.
+
+set -euo pipefail
+
+: "${ISAACSIM_VERSION:?isaacsim-version input is required}"
+
+# nv_shadercache holds NVIDIA Vulkan driver PSO blobs, which are only valid for
+# the GPU architecture and driver version that compiled them, so both gate every
+# entry. Missing values fail rather than default: head and tr succeed on empty
+# input, so the captured values have to be tested directly.
+driver_ver="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1 || true)"
+gpu_arch="$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.' || true)"
+if [ -z "${driver_ver}" ] || [ -z "${gpu_arch}" ]; then
+  echo "::error::nvidia-smi reported no driver version or compute capability; cannot key the OVRTX shader cache"
+  exit 1
+fi
+
+# [tool.isaaclab.versions].ovrtx in pyproject.toml, the single source of truth
+# resolve-ov-pins reads to build the pip specifier CI actually installs from
+# (a special internal release, not the public wheel uv.lock would resolve), so
+# the key stays a function of the commit and an upstream release cannot re-key
+# every open PR onto a cold collection.
+ovrtx_ver="$(awk -F'"' '
+  /^\[tool\.isaaclab\.versions\]/ { in_section = 1; next }
+  /^\[/ { in_section = 0 }
+  in_section && /^ovrtx[[:space:]]*=/ { print $2; exit }
+' "${GITHUB_WORKSPACE}/pyproject.toml" || true)"
+if [ -z "${ovrtx_ver}" ]; then
+  echo "::error::pyproject.toml [tool.isaaclab.versions] has no entry for ovrtx"
+  exit 1
+fi
+
+# Cache keys may not contain commas, and the isaacsim tag is only conventionally
+# bare.
+sanitize() { printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '_'; }
+
+base="v1-${RUNNER_OS}-${RUNNER_ARCH}-sm${gpu_arch}-drv$(sanitize "$driver_ver")"
+kit_collection="ovrtx-kit-${base}-isaacsim$(sanitize "$ISAACSIM_VERSION")"
+kitless_collection="ovrtx-kitless-${base}-ovrtx$(sanitize "$ovrtx_ver")"
+host_dir="${RUNNER_TEMP}/isaaclab-ovrtx-shader-cache"
+
+# Write key is unique per run so each cumulative snapshot is its own immutable
+# entry; restore matches the collection prefix, newest first.
+emit() {
+  echo "$1-collection=$2" >> "$GITHUB_OUTPUT"
+  echo "$1-key=$2-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
+  echo "$1-restore-keys=$2-" >> "$GITHUB_OUTPUT"
+  echo "$1-dir=${host_dir}/$1" >> "$GITHUB_OUTPUT"
+}
+emit kit "$kit_collection"
+emit kitless "$kitless_collection"
+echo "host-dir=${host_dir}" >> "$GITHUB_OUTPUT"
+
+echo "OVRTX shader cache collections (driver ${driver_ver}, sm${gpu_arch}):"
+echo "  kit/      ${kit_collection}"
+echo "  kitless/  ${kitless_collection}"