[Backport] PR #6905 to release/3.0.0

.github/actions/ovrtx-shader-cache/action.yml

@@ -0,0 +1,150 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+name: 'OVRTX Shader Cache'
+description: >
+  Restores, reports on and publishes the NVIDIA driver PSO blobs (nv_shadercache)
+  that Isaac Lab rendering tests would otherwise recompile from scratch on every
+  run. Requires a prior checkout.
+
+  A composite action cannot span the caller's test step, so the phases are
+  separate invocations selected by 'mode': 'restore' before the tests, then
+  either 'report' or 'save' after them. Every mode recomputes the keys from the
+  same key.sh, so the collection a job reads and the one it writes cannot drift.
+
+  kit/ is compiled by the RTX renderer inside the Isaac Sim image and kitless/ by
+  the pip-installed ovrtx wheel, so each tree is a separate entry keyed by its own
+  producer. The mount layout that fills them lives in
+  .github/actions/run-tests/run_tests.sh.
+
+inputs:
+  mode:
+    description: >-
+      'restore' reads the newest compatible snapshot of each tree and never
+      writes. 'report' summarises how far the run compiled beyond what was
+      restored. 'save' reports the same way and writes each tree the run
+      actually added to back as a new snapshot; only the cache warmer uses it.
+    required: true
+  isaacsim-version:
+    description: 'Isaac Sim image tag; identifies the exact Kit RTX build that compiles the kit/ blobs'
+    required: true
+  publishes:
+    description: >-
+      'true' when the same job later invokes this action in 'save' mode. The
+      restore pass then fingerprints what it restored, so the save pass can tell
+      a run that compiled new blobs from one that only read back the snapshot it
+      started with. Other modes ignore it; leaving it false only costs a
+      duplicate snapshot, never a lost one.
+    required: false
+    default: 'false'
+  trees:
+    description: >-
+      Which cache tree(s) this job needs: 'kit', 'kitless', or 'both'. A job that
+      only exercises one render path should request just that tree, so it does
+      not pay the restore/save cost of the tree it never populates.
+    default: 'both'
+    required: false
+
+outputs:
+  host-dir:
+    description: 'Parent directory holding both cache trees; bind-mounted into the test container'
+    value: ${{ steps.compute.outputs.host-dir }}
+
+runs:
+  using: composite
+  steps:
+    # always() here and on every post-test step below: the caller already gates
+    # this invocation on job status, and without it the composite inherits a
+    # failed job context and skips work that stays valid after a test failure.
+    # This step in particular has to run in every mode, since every later step
+    # reads its outputs.
+    - name: Compute OVRTX shader cache keys
+      id: compute
+      if: always()
+      shell: bash
+      env:
+        ISAACSIM_VERSION: ${{ inputs.isaacsim-version }}
+      run: bash "$GITHUB_ACTION_PATH/key.sh"
+
+    - name: Restore OVRTX kit shader cache
+      if: inputs.mode == 'restore' && inputs.trees != 'kitless'
+      id: restore-kit
+      uses: actions/cache/restore@v4
+      with:
+        path: ${{ steps.compute.outputs.kit-dir }}
+        key: ${{ steps.compute.outputs.kit-key }}
+        restore-keys: ${{ steps.compute.outputs.kit-restore-keys }}
+
+    - name: Restore OVRTX kitless shader cache
+      if: inputs.mode == 'restore' && inputs.trees != 'kit'
+      id: restore-kitless
+      uses: actions/cache/restore@v4
+      with:
+        path: ${{ steps.compute.outputs.kitless-dir }}
+        key: ${{ steps.compute.outputs.kitless-key }}
+        restore-keys: ${{ steps.compute.outputs.kitless-restore-keys }}
+
+    - name: Report restored OVRTX shader cache
+      if: inputs.mode == 'restore'
+      shell: bash
+      env:
+        HOST_DIR: ${{ steps.compute.outputs.host-dir }}
+        TREES: ${{ inputs.trees }}
+        KIT_MATCHED_KEY: ${{ steps.restore-kit.outputs.cache-matched-key }}
+        KITLESS_MATCHED_KEY: ${{ steps.restore-kitless.outputs.cache-matched-key }}
+        KIT_COLLECTION: ${{ steps.compute.outputs.kit-collection }}
+        KITLESS_COLLECTION: ${{ steps.compute.outputs.kitless-collection }}
+        PUBLISHES: ${{ inputs.publishes }}
+      run: bash "$GITHUB_ACTION_PATH/report.sh" restore
+
+    - name: Report OVRTX shader cache growth
+      if: always() && inputs.mode != 'restore'
+      id: growth
+      shell: bash
+      env:
+        HOST_DIR: ${{ steps.compute.outputs.host-dir }}
+        TREES: ${{ inputs.trees }}
+      run: bash "$GITHUB_ACTION_PATH/report.sh" growth
+
+    # Each tree is gated on its own file count so a populated kit/ never carries
+    # an empty kitless/ into a published snapshot. The count is empty rather than
+    # '0' when the growth step aborted before counting; both mean "not measured"
+    # and both must block the save, since the snapshot would publish under a key
+    # every consumer prefers over the last good one.
+    #
+    # The changed gate is a quota guard: every save writes a new immutable entry,
+    # so a warm run that only read back what it restored would spend ~270 MB of
+    # the repository's cache allowance on a duplicate of the snapshot it started
+    # from. It is 'true' whenever the growth step could not rule that out.
+    - name: Save OVRTX kit shader cache
+      if: >-
+        always() && inputs.mode == 'save' && inputs.trees != 'kitless'
+        && steps.growth.outputs.kit-files != ''
+        && steps.growth.outputs.kit-files != '0'
+        && steps.growth.outputs.kit-changed == 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: ${{ steps.compute.outputs.kit-dir }}
+        key: ${{ steps.compute.outputs.kit-key }}
+
+    - name: Save OVRTX kitless shader cache
+      if: >-
+        always() && inputs.mode == 'save' && inputs.trees != 'kit'
+        && steps.growth.outputs.kitless-files != ''
+        && steps.growth.outputs.kitless-files != '0'
+        && steps.growth.outputs.kitless-changed == 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: ${{ steps.compute.outputs.kitless-dir }}
+        key: ${{ steps.compute.outputs.kitless-key }}
+
+    - name: Verify requested OVRTX shader cache tree(s) were populated
+      if: always() && inputs.mode == 'save'
+      shell: bash
+      env:
+        TREES: ${{ inputs.trees }}
+        KIT_FILES: ${{ steps.growth.outputs.kit-files }}
+        KITLESS_FILES: ${{ steps.growth.outputs.kitless-files }}
+      run: bash "$GITHUB_ACTION_PATH/verify.sh"

.github/actions/ovrtx-shader-cache/key.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# Computes the collection prefix, per-run write key and host directory for each
+# of the two OVRTX shader cache trees, and appends them to $GITHUB_OUTPUT.
+#
+# Every mode of the action runs this, so the collection a job restores from and
+# the one it saves to cannot drift. Only values that stay constant for the
+# lifetime of a job may be keyed on.
+#
+# Reads: ISAACSIM_VERSION, GITHUB_WORKSPACE (for pyproject.toml), RUNNER_OS,
+# RUNNER_ARCH, RUNNER_TEMP, GITHUB_SHA, GITHUB_RUN_ID, GITHUB_RUN_ATTEMPT.
+
+set -euo pipefail
+
+: "${ISAACSIM_VERSION:?isaacsim-version input is required}"
+
+# nv_shadercache holds NVIDIA Vulkan driver PSO blobs, which are only valid for
+# the GPU architecture and driver version that compiled them, so both gate every
+# entry. Missing values fail rather than default: head and tr succeed on empty
+# input, so the captured values have to be tested directly.
+driver_ver="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1 || true)"
+gpu_arch="$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.' || true)"
+if [ -z "${driver_ver}" ] || [ -z "${gpu_arch}" ]; then
+  echo "::error::nvidia-smi reported no driver version or compute capability; cannot key the OVRTX shader cache"
+  exit 1
+fi
+
+# [tool.isaaclab.versions].ovrtx in pyproject.toml, the single source of truth
+# resolve-ov-pins reads to build the pip specifier CI actually installs from
+# (a special internal release, not the public wheel uv.lock would resolve), so
+# the key stays a function of the commit and an upstream release cannot re-key
+# every open PR onto a cold collection.
+ovrtx_ver="$(awk -F'"' '
+  /^\[tool\.isaaclab\.versions\]/ { in_section = 1; next }
+  /^\[/ { in_section = 0 }
+  in_section && /^ovrtx[[:space:]]*=/ { print $2; exit }
+' "${GITHUB_WORKSPACE}/pyproject.toml" || true)"
+if [ -z "${ovrtx_ver}" ]; then
+  echo "::error::pyproject.toml [tool.isaaclab.versions] has no entry for ovrtx"
+  exit 1
+fi
+
+# Cache keys may not contain commas, and the isaacsim tag is only conventionally
+# bare.
+sanitize() { printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '_'; }
+
+base="v1-${RUNNER_OS}-${RUNNER_ARCH}-sm${gpu_arch}-drv$(sanitize "$driver_ver")"
+kit_collection="ovrtx-kit-${base}-isaacsim$(sanitize "$ISAACSIM_VERSION")"
+kitless_collection="ovrtx-kitless-${base}-ovrtx$(sanitize "$ovrtx_ver")"
+host_dir="${RUNNER_TEMP}/isaaclab-ovrtx-shader-cache"
+
+# Write key is unique per run so each cumulative snapshot is its own immutable
+# entry; restore matches the collection prefix, newest first.
+emit() {
+  echo "$1-collection=$2" >> "$GITHUB_OUTPUT"
+  echo "$1-key=$2-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
+  echo "$1-restore-keys=$2-" >> "$GITHUB_OUTPUT"
+  echo "$1-dir=${host_dir}/$1" >> "$GITHUB_OUTPUT"
+}
+emit kit "$kit_collection"
+emit kitless "$kitless_collection"
+echo "host-dir=${host_dir}" >> "$GITHUB_OUTPUT"
+
+echo "OVRTX shader cache collections (driver ${driver_ver}, sm${gpu_arch}):"
+echo "  kit/      ${kit_collection}"
+echo "  kitless/  ${kitless_collection}"

.github/actions/ovrtx-shader-cache/report.sh

@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# Reports the state of the OVRTX shader cache, either side of the test run.
+#
+# Usage: report.sh restore|growth
+#
+#   restore  prepares the requested tree(s), checks the mount is writable, prints
+#            the per-tree hit/miss and records the baseline for the growth pass
+#   growth   prints how far the run compiled beyond what was restored, and
+#            emits the per-tree file counts and changed flags the save gates read
+#
+# TREES selects which tree(s) to process: 'kit', 'kitless' or 'both' (default).
+# A tree the caller did not request is skipped entirely.
+#
+# The passes are separate invocations of this action, so the restore pass hands
+# its baseline to the growth pass through OVRTX_CACHE_HIT, OVRTX_CACHE_MB_BEFORE
+# and OVRTX_<TREE>_FINGERPRINT_BEFORE in $GITHUB_ENV. All are internal to this
+# action.
+
+set -euo pipefail
+
+mode="${1:?usage: report.sh restore|growth}"
+: "${HOST_DIR:?HOST_DIR is required}"
+
+# 'kit', 'kitless' or 'both' (default); the tree(s) this job requested.
+trees="${TREES:-both}"
+requested() {
+  [ "$trees" = "both" ] || [ "$trees" = "$1" ]
+}
+
+# Files, not bytes: an empty tree still measures ~1 MB. tr strips wc's padding,
+# which would otherwise make the '0' comparisons in the save gates miss.
+count_files() {
+  find "$1" -type f 2>/dev/null | wc -l | tr -d '[:space:]' || true
+}
+
+fingerprint_var() {
+  printf 'OVRTX_%s_FINGERPRINT_BEFORE' "$(printf '%s' "$1" | tr '[:lower:]' '[:upper:]')"
+}
+
+# Digest of every file's path and contents in one tree, so the save pass can
+# recognise a run that added nothing and skip publishing a duplicate snapshot.
+# Contents rather than size or file count, because the driver rewrites blobs in
+# place as well as appending them. Prints nothing when the tree is missing;
+# callers read an empty digest as "not measured" and publish anyway.
+fingerprint_tree() {
+  [ -d "$1" ] || return 0
+  (cd "$1" && find . -type f -print0 | LC_ALL=C sort -z | xargs -0 -r sha256sum) |
+    sha256sum | cut -d' ' -f1
+}
+
+case "$mode" in
+  restore)
+    hit=""
+    if requested kit; then
+      mkdir -p "$HOST_DIR/kit"
+      # cache-matched-key is authoritative; directory size is not, since an empty
+      # directory still measures as ~1 MB.
+      if [ -z "${KIT_MATCHED_KEY:-}" ]; then
+        echo "::warning::OVRTX kit shader cache miss - no entry in collection ${KIT_COLLECTION:-}"
+      else
+        echo "OVRTX kit shader cache hit: ${KIT_MATCHED_KEY}"
+        hit="${hit}${KIT_MATCHED_KEY}/"
+      fi
+    fi
+    if requested kitless; then
+      mkdir -p "$HOST_DIR/kitless"
+      if [ -z "${KITLESS_MATCHED_KEY:-}" ]; then
+        echo "::warning::OVRTX kitless shader cache miss - no entry in collection ${KITLESS_COLLECTION:-}"
+      else
+        echo "OVRTX kitless shader cache hit: ${KITLESS_MATCHED_KEY}"
+        hit="${hit}${KITLESS_MATCHED_KEY}/"
+      fi
+    fi
+    if [ ! -w "$HOST_DIR" ]; then
+      echo "::error::OVRTX shader cache directory is not writable: $HOST_DIR"
+      exit 1
+    fi
+
+    echo "OVRTX_CACHE_HIT=${hit:-miss}" >> "$GITHUB_ENV"
+    echo "OVRTX_CACHE_MB_BEFORE=$(du -sm "$HOST_DIR" | cut -f1)" >> "$GITHUB_ENV"
+
+    # Only the warmer compares fingerprints, and hashing both trees is the one
+    # part of this pass that scales with the restored snapshot.
+    if [ "${PUBLISHES:-false}" = "true" ]; then
+      for tree in kit kitless; do
+        requested "$tree" || continue
+        echo "$(fingerprint_var "$tree")=$(fingerprint_tree "$HOST_DIR/$tree")" >> "$GITHUB_ENV"
+      done
+    fi
+    ;;
+
+  growth)
+    if [ ! -d "$HOST_DIR" ]; then
+      echo "OVRTX shader cache directory missing; nothing to report"
+      for tree in kit kitless; do
+        requested "$tree" || continue
+        echo "${tree}-files=0" >> "$GITHUB_OUTPUT"
+        echo "${tree}-changed=false" >> "$GITHUB_OUTPUT"
+      done
+      exit 0
+    fi
+
+    before="${OVRTX_CACHE_MB_BEFORE:-0}"
+    after="$(du -sm "$HOST_DIR" | cut -f1)"
+    grew=$(( after - before ))
+
+    if [ "${OVRTX_CACHE_HIT:-miss}" = "miss" ]; then
+      verdict="cold run, nothing restored"
+    elif [ "$grew" -le 0 ]; then
+      verdict="fully covered"
+    else
+      verdict="compiled $(( grew * 100 / (before > 0 ? before : 1) ))% beyond the restored cache"
+    fi
+
+    echo "OVRTX shader cache: restored ${before} MB, ended at ${after} MB (+${grew} MB) - ${verdict}"
+    echo "🔵 OVRTX shader cache: ${before} -> ${after} MB (+${grew}) - ${verdict}" >> "$GITHUB_STEP_SUMMARY"
+
+    for tree in kit kitless; do
+      requested "$tree" || continue
+      files="$(count_files "$HOST_DIR/$tree")"
+      files="${files:-0}"
+
+      # A baseline this job never took - a consumer, or a save whose restore
+      # pass was skipped - leaves the digest empty and publishes, so an
+      # unmeasured tree costs a duplicate snapshot rather than a lost one.
+      before_var="$(fingerprint_var "$tree")"
+      before_fp="${!before_var:-}"
+      if [ -n "$before_fp" ] && [ "$(fingerprint_tree "$HOST_DIR/$tree")" = "$before_fp" ]; then
+        changed=false
+        echo "  ${tree}/ ${files} file(s), unchanged since restore"
+      else
+        changed=true
+        echo "  ${tree}/ ${files} file(s)"
+      fi
+      echo "${tree}-files=${files}" >> "$GITHUB_OUTPUT"
+      echo "${tree}-changed=${changed}" >> "$GITHUB_OUTPUT"
+    done
+    ;;
+
+  *)
+    echo "::error::unknown report mode: ${mode}"
+    exit 1
+    ;;
+esac