Skip to content

Commit b0903a5

Browse files
committed
Merge remote-tracking branch 'upstream/develop' into jichuanh/uv-lock-docker-install
# Conflicts: # docker/Dockerfile.kitless # uv.lock
2 parents 355e1f9 + 5615143 commit b0903a5

272 files changed

Lines changed: 7514 additions & 3924 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/check-links.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ jobs:
9797
--exclude 'graphics\.pixar\.com'
9898
--exclude 'openpbs\.org'
9999
--exclude 'docutils\.sourceforge\.io'
100+
--exclude '^https://opensource\.org/'
100101
--exclude 'huggingface\.co/datasets/nvidia/PhysicalAI-Robotics-NuRec'
101102
--exclude 'huggingface\.co/nvidia/COMPASS'
102103
--exclude 'huggingface\.co/nvidia/X-Mobility'

.github/workflows/config.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,6 @@
1010
# which the CI credential can reach.
1111
isaacsim_image_name: nvcr.io/0947644777160149/internal/isaac-sim
1212
# Isaac Sim 6.1.0-alpha.50 (b86cf6ce) includes Kit 110.3.0-360924's fix for NVBug 6566677.
13-
isaacsim_image_tag: latest-develop@sha256:e9272374eddc69155fa12c7a9e0bb6d36778071c6cb649db08a67a25d9c563b8
13+
isaacsim_image_tag: latest-develop@sha256:50d2491d9e04da52bb39f0c8a993fba991d2d13939e88f2008577e00a17c4b03
1414
isaaclab_image_name: nvcr.io/0947644777160149/internal/isaac-lab
1515
ovphysx_wheelhouse_image: ""

.github/workflows/docs.yaml

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,6 @@ jobs:
5757
steps:
5858
- name: Checkout code
5959
uses: actions/checkout@v6
60-
with:
61-
lfs: true
6260

6361
- name: Set up uv
6462
uses: astral-sh/setup-uv@v6
@@ -91,8 +89,6 @@ jobs:
9189
steps:
9290
- name: Checkout code
9391
uses: actions/checkout@v6
94-
with:
95-
lfs: true
9692

9793
- name: Set up uv
9894
uses: astral-sh/setup-uv@v6

.github/workflows/test-multi-gpu-pytest.yaml

Lines changed: 168 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,10 @@
1818
name: Multi-GPU pytest
1919

2020
on:
21+
# No ``paths:`` filter on purpose. Gating lives in the ``changes`` job below so
22+
# these jobs are always created and report green when skipped -- a required check
23+
# that is never created stays pending forever.
2124
pull_request:
22-
paths:
23-
- "source/isaaclab/isaaclab/test/utils/**"
24-
- "source/isaaclab/isaaclab/app/app_launcher.py"
25-
- "source/**/test/**/test_*.py"
26-
- ".github/workflows/test-multi-gpu-pytest.yaml"
27-
- ".github/actions/run-package-tests/**"
28-
- ".github/actions/run-tests/**"
29-
- ".github/actions/ecr-build-push-pull/**"
3025
workflow_dispatch:
3126

3227
concurrency:
@@ -35,8 +30,61 @@ concurrency:
3530

3631
env:
3732
CI_IMAGE_TAG: isaac-lab-ci:${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}-${{ github.sha }}
33+
# Same tag family as kitless-docker.yml so the layer cache is shared, but built here: the ECR
34+
# cache repo is resolved per runner pool, so an image built on the `gpu` pool is invisible to
35+
# this one.
36+
CI_KITLESS_IMAGE_TAG: isaac-lab-ci:${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}-${{ github.sha }}-kitless
3837

3938
jobs:
39+
changes:
40+
name: Detect Changes
41+
runs-on: ubuntu-latest
42+
outputs:
43+
unit: ${{ steps.unit.outputs.should_run }}
44+
smoke: ${{ steps.smoke.outputs.should_run }}
45+
steps:
46+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
47+
with:
48+
fetch-depth: 1
49+
50+
# Narrow: the sharded device-parametrized tests. Their blast radius really is
51+
# the launcher plus the device-scope helpers they are discovered by.
52+
- name: Detect changes (unit shards)
53+
id: unit
54+
uses: ./.github/actions/detect-changes
55+
with:
56+
triggered-jobs-pr: multi-GPU sharded unit tests
57+
patterns: |
58+
^source/isaaclab/isaaclab/test/utils/ :: Device-scope test helpers
59+
^source/isaaclab/isaaclab/app/app_launcher\.py$ :: Device selection at launch
60+
^source/.+/test/.+/test_.*\.py$ :: Device-parametrized test files
61+
^\.github/workflows/test-multi-gpu-pytest\.yaml$ :: This workflow
62+
^\.github/actions/(run-tests|run-package-tests|ecr-build-push-pull|multi-gpu|detect-changes)/ :: Shared actions
63+
64+
# The training smoke drives real multi-rank RL runs, so the runtime it loads is matched
65+
# broadly. The task side is not: these tests exercise exactly two gym ids,
66+
# ``Isaac-Cartpole-Direct`` and ``Isaac-Cartpole-Camera-Direct``, so only their own
67+
# sources are matched rather than all of isaaclab_tasks/isaaclab_assets. An unrelated
68+
# manager task, contrib task, or robot asset must not cost an hour of multi-GPU runner.
69+
- name: Detect changes (training smoke)
70+
id: smoke
71+
uses: ./.github/actions/detect-changes
72+
with:
73+
triggered-jobs-pr: multi-GPU training smoke
74+
patterns: |
75+
^source/isaaclab/ :: Core library (app, sim, scene, renderers, sensors)
76+
^source/isaaclab_(newton|physx|ov)/ :: Physics and renderer backends
77+
^source/isaaclab_ppisp/ :: Render pipeline the Newton renderer imports
78+
^source/isaaclab_rl/ :: RL library integration (rsl_rl entry point)
79+
^source/isaaclab_visualizers/ :: Visualizer and renderer coupling
80+
^source/isaaclab_tasks/isaaclab_tasks/(__init__\.py$|utils/|core/cartpole/(agents/|__init__\.py$|cartpole_direct_)) :: Direct Cartpole envs, their registration and agent configs, plus the preset resolution the presets= selector goes through
81+
^source/isaaclab_assets/isaaclab_assets/robots/cartpole\.py$ :: CARTPOLE_CFG, the only asset these envs load
82+
^scripts/reinforcement_learning/ :: Multi-GPU training entry point
83+
^docker/Dockerfile\.(base|kitless)$ :: CI image definitions
84+
^(pyproject\.toml|uv\.lock)$ :: Dependency graph (image identity)
85+
^\.github/workflows/test-multi-gpu-pytest\.yaml$ :: This workflow
86+
^\.github/actions/(run-tests|run-package-tests|ecr-build-push-pull|multi-gpu|detect-changes)/ :: Shared actions
87+
4088
config:
4189
name: Load Config
4290
runs-on: ubuntu-latest
@@ -58,7 +106,8 @@ jobs:
58106
59107
build:
60108
name: Build / cache image
61-
needs: [config]
109+
needs: [config, changes]
110+
if: needs.changes.outputs.unit == 'true' || needs.changes.outputs.smoke == 'true'
62111
# Must run on the SAME pool as the test job. The ECR cache repo is resolved
63112
# per runner pool (single-GPU `gpu` runners -> gitci-docker-cache; multi-GPU
64113
# runners -> multigpu-docker-cache). If this built on `[self-hosted, gpu]`
@@ -88,16 +137,48 @@ jobs:
88137
dockerfile-path: docker/Dockerfile.base
89138
cache-tag: cache-base
90139

140+
build-kitless:
141+
name: Build / cache kit-less image
142+
needs: [config, changes]
143+
# Only the training smoke uses this image; the sharded unit tests are Kit-only.
144+
if: needs.changes.outputs.smoke == 'true'
145+
# Same pool as the test job for the same reason as ``build`` above -- the ECR cache repo is
146+
# per pool, so building elsewhere would force a rebuild on the scarce multi-GPU runner.
147+
runs-on: [self-hosted, linux, x64, multi-gpu]
148+
steps:
149+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
150+
with:
151+
fetch-depth: 1
152+
lfs: true
153+
154+
# Builds from ubuntu rather than the Isaac Sim image, and already installs ``ov[all]``:
155+
# with no Isaac Sim there is no prebundled ``packaging`` for ovphysx's ``<24`` cap to
156+
# overwrite, which is exactly why that install cannot go in Dockerfile.base.
157+
- uses: ./.github/actions/ecr-build-push-pull
158+
env:
159+
NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
160+
with:
161+
image-tag: ${{ env.CI_KITLESS_IMAGE_TAG }}
162+
isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
163+
isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
164+
dockerfile-path: docker/Dockerfile.kitless
165+
cache-tag: cache-kitless
166+
91167
test-multi-gpu-pytest:
92168
name: Multi-GPU unit tests
93-
needs: [config, build]
94-
if: needs.build.result == 'success'
169+
needs: [config, build, changes]
170+
# ``always()`` is load-bearing: ``build`` is skipped when neither gate fires, and GitHub skips
171+
# a job whose dependency skipped unless the condition carries a status function. Without it a
172+
# gate-miss would silently skip the shards too.
173+
if: always() && needs.build.result == 'success'
95174
# The ``multi-gpu`` label is the multi-GPU pool. Do NOT add ``gpu`` here:
96175
# in this fleet ``gpu`` tags single-GPU runners, so requiring it routes the
97176
# job onto a 1-GPU box and the shard runner aborts with "Need at least 2
98177
# visible devices; found 1".
99178
runs-on: [self-hosted, linux, x64, multi-gpu]
100-
timeout-minutes: 60
179+
# Shards plus the full training-smoke matrix. The smoke alone is ~30 min
180+
# (Kit boot dominates the Kit-renderer cases), so 60 no longer fits.
181+
timeout-minutes: 120
101182
steps:
102183
- name: Checkout repository
103184
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
@@ -177,12 +258,87 @@ jobs:
177258
IMAGE_TAG: ${{ env.CI_IMAGE_TAG }}
178259
INCLUDE_FILES: ${{ steps.discover.outputs.include }}
179260
PATHS: ${{ steps.discover.outputs.paths }}
261+
if: needs.changes.outputs.unit == 'true'
180262
run: bash .github/actions/multi-gpu/multi_gpu_host_launcher.sh
181263

264+
- name: Multi-GPU training smoke (Kit renderer)
265+
# Real multi-rank training runs, outside the sharded device-parametrized
266+
# path above: a multi-GPU job owns several GPUs at once, so a per-device
267+
# shard cannot express it. Unsharded, all GPUs exposed; the tests choose
268+
# their own device order.
269+
#
270+
# Uses the shared run-tests action rather than a bespoke ``docker run``:
271+
# passing a file path (not ``tools``) skips the conftest sharding
272+
# orchestrator, so this is a plain single pytest run, while still
273+
# inheriting the standard container contract -- writable HOME, the Isaac
274+
# Sim cache mounts, and junit results collected into ``reports/``.
275+
# Runs on failure too, so a red shard does not hide smoke coverage.
276+
if: always() && needs.changes.outputs.smoke == 'true'
277+
uses: ./.github/actions/run-tests
278+
with:
279+
test-path: source/isaaclab/test/multi_gpu/test_multi_gpu_training_smoke.py
280+
result-file: multi-gpu-smoke-report.xml
281+
container-name: isaac-lab-mgpu-smoke-${{ github.run_id }}-${{ github.run_attempt }}
282+
image-tag: ${{ env.CI_IMAGE_TAG }}
283+
pytest-options: -m "not kitless"
284+
# Run the checkout, not the image's baked-in copy. ecr-build-push-pull serves a
285+
# commit-tagged or deps-cache image, so on a cache hit the container carries source
286+
# from whenever that image was built -- CI then reports a pass or fail for code this
287+
# commit never contained. The shard launcher already bind-mounts for this reason.
288+
volume-mount-source: ${{ github.workspace }}
289+
182290
- name: Aggregated test summary
183291
# Per-shard + per-file pass/total/walltime, plus a combined table, also
184292
# written to $GITHUB_STEP_SUMMARY. Runs on success or failure.
185293
if: always()
186294
env:
187295
RUNTIME_DIR: ${{ env.MGPU_RUNTIME_DIR }}
188296
run: python3 .github/actions/multi-gpu/aggregate_test_summary.py
297+
298+
test-multi-gpu-smoke-kitless:
299+
name: Multi-GPU training smoke (kit-less)
300+
needs: [config, build-kitless, changes]
301+
# Split out of ``Multi-GPU unit tests`` so a kit-less image build that fails -- or a runner
302+
# that dies under it -- cannot take down the Kit-renderer lane, which needs a different image
303+
# entirely. When two multi-GPU runners are free the lanes also overlap; when only one is, the
304+
# scheduler serialises them onto it, which is the previous behaviour.
305+
if: always() && needs.build-kitless.result == 'success' && needs.changes.outputs.smoke == 'true'
306+
runs-on: [self-hosted, linux, x64, multi-gpu]
307+
# One lane only; the 120 min ceiling on the sharded job covers shards + Kit smoke together.
308+
timeout-minutes: 60
309+
steps:
310+
- name: Checkout repository
311+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
312+
with:
313+
fetch-depth: 1
314+
lfs: true
315+
316+
- name: Pull kit-less image from ECR
317+
# ``build-kitless`` pushes to ECR from whichever multi-GPU runner it landed on, so the
318+
# image is not necessarily present locally here. Without this the kitless step's
319+
# ``docker run`` fails with ``pull access denied`` whenever the two jobs are scheduled
320+
# on different runners.
321+
uses: ./.github/actions/ecr-build-push-pull
322+
env:
323+
NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
324+
with:
325+
image-tag: ${{ env.CI_KITLESS_IMAGE_TAG }}
326+
isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
327+
isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
328+
dockerfile-path: docker/Dockerfile.kitless
329+
cache-tag: cache-kitless
330+
331+
- name: Multi-GPU training smoke (kitless renderer)
332+
# Same tests as the Kit lane with the marker inverted, but run in the kit-less image,
333+
# which already carries ``ov[all]``. Each stack runs in the image built for it, and the
334+
# job split keeps a kitless-renderer fault from taking down the Kit-renderer coverage
335+
# that guards renderer device selection.
336+
uses: ./.github/actions/run-tests
337+
with:
338+
test-path: source/isaaclab/test/multi_gpu/test_multi_gpu_training_smoke.py
339+
result-file: multi-gpu-smoke-kitless-report.xml
340+
container-name: isaac-lab-mgpu-smoke-kitless-${{ github.run_id }}-${{ github.run_attempt }}
341+
image-tag: ${{ env.CI_KITLESS_IMAGE_TAG }}
342+
pytest-options: -m kitless
343+
# Same reason as the Kit step above.
344+
volume-mount-source: ${{ github.workspace }}

CONTRIBUTORS.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ Guidelines for modifications:
8383
* Daniela Hasenbring
8484
* Dhananjay Shendre
8585
* Dhyan Thakkar
86+
* Diego Ferigo
8687
* Dongxuan Fan
8788
* Dorsa Rohani
8889
* Ege Sekkin
@@ -220,6 +221,7 @@ Guidelines for modifications:
220221
* Yun Liu
221222
* YuTeh Shen
222223
* Zehao Wang
224+
* Zeng Qingcheng
223225
* Zijian Li
224226
* Ziqi Fan
225227
* Zoe McCarthy

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ Our [documentation page](https://isaac-sim.github.io/IsaacLab) provides everythi
5757
detailed tutorials and step-by-step guides. Follow these links to learn more about:
5858

5959
- [Installation steps](https://isaac-sim.github.io/IsaacLab/develop/source/setup/installation/index.html#local-installation)
60-
- [Reinforcement learning](https://isaac-sim.github.io/IsaacLab/develop/source/overview/reinforcement-learning/rl_existing_scripts.html)
60+
- [Reinforcement learning](docs/source/concepts/reinforcement_learning.rst)
6161
- [Tutorials](https://isaac-sim.github.io/IsaacLab/develop/source/tutorials/index.html)
6262
- [Available environments](https://isaac-sim.github.io/IsaacLab/develop/source/setup/environments.html)
6363

0 commit comments

Comments
 (0)