Skip to content

Commit 261de9c

Browse files
StafaHStafaH
committed
Cache renderer runtime artifacts in CI
1 parent 10c0070 commit 261de9c

7 files changed

Lines changed: 319 additions & 4 deletions

File tree

‎.github/actions/renderer-cache-key/action.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
2+
# All rights reserved.
3+
#
4+
# SPDX-License-Identifier: BSD-3-Clause
5+
6+
name: 'Renderer Cache Key'
7+
description: 'Computes a hardware- and Isaac-Sim-specific renderer cache key and host directory'
8+
9+
inputs:
10+
isaacsim-base-image:
11+
description: 'Isaac Sim base image repository'
12+
required: true
13+
isaacsim-version:
14+
description: 'Isaac Sim base image version'
15+
required: true
16+
17+
outputs:
18+
collection:
19+
description: 'Collection prefix shared by mutually compatible renderer caches'
20+
value: ${{ steps.compute.outputs.collection }}
21+
key:
22+
description: 'Immutable cache key for this workflow run'
23+
value: ${{ steps.compute.outputs.key }}
24+
restore-keys:
25+
description: 'Collection prefix used to restore the newest compatible snapshot'
26+
value: ${{ steps.compute.outputs.restore-keys }}
27+
host-dir:
28+
description: 'Host directory bind-mounted over renderer cache paths'
29+
value: ${{ steps.compute.outputs.host-dir }}
30+
31+
runs:
32+
using: composite
33+
steps:
34+
- name: Compute renderer cache key
35+
id: compute
36+
shell: bash
37+
env:
38+
ISAACSIM_BASE_IMAGE: ${{ inputs.isaacsim-base-image }}
39+
ISAACSIM_VERSION: ${{ inputs.isaacsim-version }}
40+
run: |
41+
set -euo pipefail
42+
43+
gpu_inventory="$(nvidia-smi --query-gpu=name,driver_version --format=csv,noheader | sort -u)"
44+
compatibility="$ISAACSIM_BASE_IMAGE:$ISAACSIM_VERSION|$RUNNER_OS|$RUNNER_ARCH|$gpu_inventory"
45+
compatibility_hash="$(printf '%s' "$compatibility" | sha256sum | cut -c1-20)"
46+
collection="renderer-v1-${RUNNER_OS}-${RUNNER_ARCH}-${compatibility_hash}"
47+
48+
echo "collection=${collection}" >> "$GITHUB_OUTPUT"
49+
echo "key=${collection}-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
50+
echo "restore-keys=${collection}-" >> "$GITHUB_OUTPUT"
51+
echo "host-dir=${RUNNER_TEMP}/isaaclab-renderer-cache" >> "$GITHUB_OUTPUT"
52+
53+
echo "Renderer cache collection: ${collection}"
54+
echo "Renderer cache compatibility: ${compatibility}"

‎.github/actions/run-package-tests/action.yml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,13 @@ inputs:
107107
post-merge a cumulative writeback; only warm-warp-cache uses it.
108108
default: ''
109109
required: false
110+
renderer-cache:
111+
description: >-
112+
Kit, RTX, and NVIDIA compute cache mode. Empty disables the cache,
113+
'restore' reads the newest compatible snapshot, and 'save' additionally
114+
publishes a cumulative snapshot. PR jobs must use restore only.
115+
default: ''
116+
required: false
110117
standalone-script-scope:
111118
description: 'Enable standalone script smoke tests for this scripts/ subdirectory'
112119
default: ''
@@ -192,6 +199,14 @@ runs:
192199
id: warp-cache-key
193200
uses: ./.github/actions/warp-cache-key
194201

202+
- name: Resolve renderer cache key
203+
if: inputs.renderer-cache != ''
204+
id: renderer-cache-key
205+
uses: ./.github/actions/renderer-cache-key
206+
with:
207+
isaacsim-base-image: ${{ inputs.isaacsim-base-image }}
208+
isaacsim-version: ${{ inputs.isaacsim-version }}
209+
195210
# Both modes restore. 'save' additionally writes the result back, so a
196211
# post-merge run gets the same speedup as a pull request and the collection
197212
# accumulates whatever was missing instead of restarting from empty.
@@ -237,6 +252,45 @@ runs:
237252
echo "WARP_CACHE_FINGERPRINT_BEFORE=$fingerprint" >> "$GITHUB_ENV"
238253
fi
239254
255+
- name: Restore renderer cache
256+
if: inputs.renderer-cache != ''
257+
id: renderer-cache-restore
258+
uses: actions/cache/restore@v4
259+
with:
260+
path: ${{ steps.renderer-cache-key.outputs.host-dir }}
261+
key: ${{ steps.renderer-cache-key.outputs.key }}
262+
restore-keys: ${{ steps.renderer-cache-key.outputs.restore-keys }}
263+
264+
- name: Report restored renderer cache
265+
if: inputs.renderer-cache != ''
266+
shell: bash
267+
env:
268+
HOST_DIR: ${{ steps.renderer-cache-key.outputs.host-dir }}
269+
MATCHED_KEY: ${{ steps.renderer-cache-restore.outputs.cache-matched-key }}
270+
COLLECTION: ${{ steps.renderer-cache-key.outputs.collection }}
271+
CACHE_MODE: ${{ inputs.renderer-cache }}
272+
run: |
273+
set -euo pipefail
274+
mkdir -p "$HOST_DIR"
275+
if [ ! -w "$HOST_DIR" ]; then
276+
echo "::error::Renderer cache directory is not writable: $HOST_DIR"
277+
exit 1
278+
fi
279+
280+
if [ -z "$MATCHED_KEY" ]; then
281+
echo "::warning::Renderer cache miss - no entry in collection ${COLLECTION}"
282+
echo "RENDERER_CACHE_HIT=miss" >> "$GITHUB_ENV"
283+
else
284+
echo "Renderer cache hit: ${MATCHED_KEY}"
285+
echo "RENDERER_CACHE_HIT=${MATCHED_KEY}" >> "$GITHUB_ENV"
286+
fi
287+
echo "RENDERER_CACHE_BYTES_BEFORE=$(du -sb "$HOST_DIR" | cut -f1)" >> "$GITHUB_ENV"
288+
python3 .github/actions/run-package-tests/renderer_cache_inventory.py "$HOST_DIR" "Renderer cache restored"
289+
if [ "$CACHE_MODE" = "save" ]; then
290+
fingerprint="$(python3 .github/actions/run-package-tests/renderer_cache_inventory.py "$HOST_DIR" --fingerprint)"
291+
echo "RENDERER_CACHE_FINGERPRINT_BEFORE=$fingerprint" >> "$GITHUB_ENV"
292+
fi
293+
240294
- name: Setup wheelhouse registry authentication
241295
if: inputs.wheelhouse-image != ''
242296
uses: ./.github/actions/_lib/setup-docker-config
@@ -318,6 +372,7 @@ runs:
318372
wheelhouse-packages: ${{ inputs.wheelhouse-packages }}
319373
omni-github-test-type: ${{ inputs.omni-github-test-type }}
320374
warp-cache-host-dir: ${{ steps.warp-cache-key.outputs.host-dir }}
375+
renderer-cache-host-dir: ${{ steps.renderer-cache-key.outputs.host-dir }}
321376
standalone-script-scope: ${{ inputs.standalone-script-scope }}
322377
standalone-script-visualizer: ${{ inputs.standalone-script-visualizer }}
323378
standalone-script-runtime-group: ${{ inputs.standalone-script-runtime-group }}
@@ -407,6 +462,60 @@ runs:
407462
path: ${{ steps.warp-cache-key.outputs.host-dir }}
408463
key: ${{ steps.warp-cache-key.outputs.key }}
409464

465+
- name: Report renderer cache growth
466+
if: always() && inputs.renderer-cache != ''
467+
id: renderer-cache-growth
468+
shell: bash
469+
env:
470+
HOST_DIR: ${{ steps.renderer-cache-key.outputs.host-dir }}
471+
CACHE_MODE: ${{ inputs.renderer-cache }}
472+
run: |
473+
set -euo pipefail
474+
475+
if [ ! -d "$HOST_DIR" ]; then
476+
echo "Renderer cache directory is missing; nothing to report"
477+
echo "empty=true" >> "$GITHUB_OUTPUT"
478+
exit 0
479+
fi
480+
481+
before="${RENDERER_CACHE_BYTES_BEFORE:-0}"
482+
after="$(du -sb "$HOST_DIR" | cut -f1)"
483+
grew=$(( after - before ))
484+
echo "Renderer cache: restored $(( before / 1000000 )) MB, ended at $(( after / 1000000 )) MB (+$(( grew / 1000000 )) MB)"
485+
python3 .github/actions/run-package-tests/renderer_cache_inventory.py "$HOST_DIR" "Renderer cache final"
486+
487+
if [ -z "$(find "$HOST_DIR" -type f -print -quit)" ]; then
488+
echo "::warning::Renderer cache contains no files; skipping publish"
489+
echo "empty=true" >> "$GITHUB_OUTPUT"
490+
echo "changed=false" >> "$GITHUB_OUTPUT"
491+
elif [ "$after" -gt 8000000000 ]; then
492+
echo "::warning::Renderer cache exceeds 8 GB; skipping publish"
493+
echo "empty=false" >> "$GITHUB_OUTPUT"
494+
echo "too-large=true" >> "$GITHUB_OUTPUT"
495+
echo "changed=false" >> "$GITHUB_OUTPUT"
496+
else
497+
echo "empty=false" >> "$GITHUB_OUTPUT"
498+
echo "too-large=false" >> "$GITHUB_OUTPUT"
499+
if [ "$CACHE_MODE" = "save" ] && [ "${RENDERER_CACHE_HIT:-miss}" != "miss" ]; then
500+
fingerprint="$(python3 .github/actions/run-package-tests/renderer_cache_inventory.py "$HOST_DIR" --fingerprint)"
501+
if [ "$fingerprint" = "${RENDERER_CACHE_FINGERPRINT_BEFORE:-}" ]; then
502+
echo "Renderer cache contents are unchanged; skipping duplicate snapshot"
503+
echo "changed=false" >> "$GITHUB_OUTPUT"
504+
else
505+
echo "changed=true" >> "$GITHUB_OUTPUT"
506+
fi
507+
else
508+
echo "changed=true" >> "$GITHUB_OUTPUT"
509+
fi
510+
fi
511+
512+
- name: Save renderer cache
513+
if: '!cancelled() && inputs.renderer-cache == ''save'' && steps.renderer-cache-growth.outputs.empty != ''true'' && steps.renderer-cache-growth.outputs.too-large != ''true'' && steps.renderer-cache-growth.outputs.changed == ''true'''
514+
uses: actions/cache/save@v4
515+
with:
516+
path: ${{ steps.renderer-cache-key.outputs.host-dir }}
517+
key: ${{ steps.renderer-cache-key.outputs.key }}
518+
410519
- name: Cleanup wheelhouse artifacts
411520
if: always() && steps.extract-wheelhouse.outputs.wheelhouse_host_dir != ''
412521
shell: bash

‎.github/actions/run-package-tests/renderer_cache_inventory.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
2+
# All rights reserved.
3+
#
4+
# SPDX-License-Identifier: BSD-3-Clause
5+
6+
"""Report the size and stable metadata fingerprint of a renderer cache tree."""
7+
8+
import hashlib
9+
import os
10+
import pathlib
11+
import sys
12+
13+
14+
def fingerprint(root: pathlib.Path) -> str:
15+
"""Return a digest of cache file paths, sizes, and modification times."""
16+
digest = hashlib.sha256()
17+
for path in sorted(path for path in root.rglob("*") if path.is_file()):
18+
try:
19+
stat = path.stat()
20+
except OSError:
21+
continue
22+
digest.update(path.relative_to(root).as_posix().encode())
23+
digest.update(f"\0{stat.st_size}\0{stat.st_mtime_ns}\0".encode())
24+
return digest.hexdigest()
25+
26+
27+
def inventory(root: pathlib.Path) -> tuple[int, int, dict[str, int]]:
28+
"""Return total bytes, file count, and bytes grouped by top-level directory."""
29+
total_bytes = 0
30+
file_count = 0
31+
groups: dict[str, int] = {}
32+
for dirpath, _, filenames in os.walk(root):
33+
for name in filenames:
34+
path = pathlib.Path(dirpath, name)
35+
try:
36+
size = path.stat().st_size
37+
except OSError:
38+
continue
39+
relative = path.relative_to(root)
40+
group = relative.parts[0] if relative.parts else "."
41+
groups[group] = groups.get(group, 0) + size
42+
total_bytes += size
43+
file_count += 1
44+
return total_bytes, file_count, groups
45+
46+
47+
def main() -> int:
48+
"""Print a fingerprint or human-readable renderer cache inventory."""
49+
root = pathlib.Path(sys.argv[1])
50+
if len(sys.argv) > 2 and sys.argv[2] == "--fingerprint":
51+
print(fingerprint(root))
52+
return 0
53+
54+
label = sys.argv[2] if len(sys.argv) > 2 else "Renderer cache"
55+
if not root.is_dir():
56+
print(f"{label}: directory is missing")
57+
return 0
58+
59+
total_bytes, file_count, groups = inventory(root)
60+
breakdown = ", ".join(f"{name}={size / 1e6:.0f} MB" for name, size in sorted(groups.items())) or "empty"
61+
print(f"{label}: {total_bytes / 1e6:.0f} MB across {file_count} files ({breakdown})")
62+
63+
summary = os.environ.get("GITHUB_STEP_SUMMARY")
64+
if summary:
65+
with open(summary, "a", encoding="utf-8") as handle:
66+
handle.write(f"🔵 {label}: {total_bytes / 1e6:.0f} MB across {file_count} files ({breakdown})\n")
67+
return 0
68+
69+
70+
if __name__ == "__main__":
71+
raise SystemExit(main())

‎.github/actions/run-package-tests/test_renderer_cache_inventory.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
2+
# All rights reserved.
3+
#
4+
# SPDX-License-Identifier: BSD-3-Clause
5+
6+
"""Tests for renderer cache inventory and change detection."""
7+
8+
import importlib.util
9+
from pathlib import Path
10+
11+
12+
def _load_inventory_module():
13+
module_path = Path(__file__).with_name("renderer_cache_inventory.py")
14+
spec = importlib.util.spec_from_file_location("renderer_cache_inventory", module_path)
15+
assert spec is not None
16+
assert spec.loader is not None
17+
module = importlib.util.module_from_spec(spec)
18+
spec.loader.exec_module(module)
19+
return module
20+
21+
22+
def test_inventory_groups_cache_files_by_top_level_directory(tmp_path: Path) -> None:
23+
"""Inventory should report every cache file without reading its contents."""
24+
inventory_module = _load_inventory_module()
25+
(tmp_path / "home").mkdir()
26+
(tmp_path / "isaac-sim").mkdir()
27+
(tmp_path / "home" / "shader.bin").write_bytes(b"123")
28+
(tmp_path / "isaac-sim" / "kit.bin").write_bytes(b"12345")
29+
30+
total_bytes, file_count, groups = inventory_module.inventory(tmp_path)
31+
32+
assert total_bytes == 8
33+
assert file_count == 2
34+
assert groups == {"home": 3, "isaac-sim": 5}
35+
36+
37+
def test_fingerprint_changes_when_cache_file_changes(tmp_path: Path) -> None:
38+
"""A writer must publish a new snapshot when a cache artifact changes."""
39+
inventory_module = _load_inventory_module()
40+
artifact = tmp_path / "shader.bin"
41+
artifact.write_bytes(b"before")
42+
before = inventory_module.fingerprint(tmp_path)
43+
44+
artifact.write_bytes(b"after-content")
45+
46+
assert inventory_module.fingerprint(tmp_path) != before

‎.github/actions/run-tests/action.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@ inputs:
100100
description: 'Host Warp kernel cache directory bind-mounted into the container as WARP_CACHE_PATH'
101101
default: ''
102102
required: false
103+
renderer-cache-host-dir:
104+
description: 'Host directory containing persistent Kit, RTX, and NVIDIA compute caches'
105+
default: ''
106+
required: false
103107
ci-marker:
104108
description: 'CI_MARKER value forwarded to the container (read by tools/conftest.py to select test files by pytest marker)'
105109
default: ''
@@ -134,7 +138,7 @@ runs:
134138
TEST_K_EXPR_INPUT: ${{ inputs.test-k-expr }}
135139
CI_MARKER_INPUT: ${{ inputs.ci-marker }}
136140
run: |
137-
bash .github/actions/run-tests/run_tests.sh "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "$TEST_K_EXPR_INPUT" "$CI_MARKER_INPUT" "${{ inputs.standalone-script-scope }}" "${{ inputs.standalone-script-visualizer }}" "${{ inputs.standalone-script-runtime-group }}" "${{ inputs.warp-cache-host-dir }}" "${{ inputs.extra-uv-packages }}"
141+
bash .github/actions/run-tests/run_tests.sh "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "$TEST_K_EXPR_INPUT" "$CI_MARKER_INPUT" "${{ inputs.standalone-script-scope }}" "${{ inputs.standalone-script-visualizer }}" "${{ inputs.standalone-script-runtime-group }}" "${{ inputs.warp-cache-host-dir }}" "${{ inputs.extra-uv-packages }}" "${{ inputs.renderer-cache-host-dir }}"
138142
- name: Kill container on cancellation
139143
if: cancelled()
140144
shell: bash

‎.github/actions/run-tests/run_tests.sh‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,15 @@ run_tests() {
3737
local standalone_script_runtime_group="${24}"
3838
local warp_cache_host_dir="${25}"
3939
local extra_uv_packages="${26}"
40+
local renderer_cache_host_dir="${27}"
4041
local logs_pid=""
4142
local wait_pid=""
4243
local docker_wait_file="/tmp/.docker_exit_${container_name}"
4344
local docker_runtime_dir=""
45+
local home_cache_args=""
46+
local isaacsim_cache_dir=""
47+
local isaacsim_compute_cache_dir=""
48+
local kit_cache_dir=""
4449

4550
# Kill the container immediately if the runner is cancelled.
4651
# The GitHub Actions runner can deliver HUP, INT, or TERM on cancellation
@@ -240,14 +245,36 @@ run_tests() {
240245
"${docker_runtime_dir}/isaac-sim/data" \
241246
"${docker_runtime_dir}/isaac-sim/logs" \
242247
"${docker_runtime_dir}/isaac-sim/pkg"
248+
249+
kit_cache_dir="${docker_runtime_dir}/isaac-sim/kit/cache"
250+
isaacsim_cache_dir="${docker_runtime_dir}/isaac-sim/cache"
251+
isaacsim_compute_cache_dir="${docker_runtime_dir}/isaac-sim/computecache"
252+
home_cache_args=""
253+
if [ -n "$renderer_cache_host_dir" ]; then
254+
mkdir -p \
255+
"${renderer_cache_host_dir}/home/cache" \
256+
"${renderer_cache_host_dir}/home/computecache" \
257+
"${renderer_cache_host_dir}/isaac-sim/kit-cache" \
258+
"${renderer_cache_host_dir}/isaac-sim/cache" \
259+
"${renderer_cache_host_dir}/isaac-sim/computecache"
260+
kit_cache_dir="${renderer_cache_host_dir}/isaac-sim/kit-cache"
261+
isaacsim_cache_dir="${renderer_cache_host_dir}/isaac-sim/cache"
262+
isaacsim_compute_cache_dir="${renderer_cache_host_dir}/isaac-sim/computecache"
263+
home_cache_args="\
264+
-v ${renderer_cache_host_dir}/home/cache:/tmp/isaaclab-ci-home/.cache:rw \
265+
-v ${renderer_cache_host_dir}/home/computecache:/tmp/isaaclab-ci-home/.nv/ComputeCache:rw"
266+
echo "🔵 Mounting persistent renderer caches from ${renderer_cache_host_dir}"
267+
fi
268+
243269
docker_volume_args="\
244270
-v ${volume_mount_source}:/workspace/isaaclab:rw \
245271
-v ${docker_runtime_dir}/home:/tmp/isaaclab-ci-home:rw \
246-
-v ${docker_runtime_dir}/isaac-sim/kit/cache:/isaac-sim/kit/cache:rw \
272+
${home_cache_args} \
273+
-v ${kit_cache_dir}:/isaac-sim/kit/cache:rw \
247274
-v ${docker_runtime_dir}/isaac-sim/kit/data:/isaac-sim/kit/data:rw \
248275
-v ${docker_runtime_dir}/isaac-sim/kit/logs:/isaac-sim/kit/logs:rw \
249-
-v ${docker_runtime_dir}/isaac-sim/cache:/isaac-sim/.cache:rw \
250-
-v ${docker_runtime_dir}/isaac-sim/computecache:/isaac-sim/.nv/ComputeCache:rw \
276+
-v ${isaacsim_cache_dir}:/isaac-sim/.cache:rw \
277+
-v ${isaacsim_compute_cache_dir}:/isaac-sim/.nv/ComputeCache:rw \
251278
-v ${docker_runtime_dir}/isaac-sim/config:/isaac-sim/.nvidia-omniverse/config:rw \
252279
-v ${docker_runtime_dir}/isaac-sim/data:/isaac-sim/.local/share/ov/data:rw \
253280
-v ${docker_runtime_dir}/isaac-sim/logs:/isaac-sim/.nvidia-omniverse/logs:rw \

0 commit comments

Comments
 (0)