1818name : Multi-GPU pytest
1919
2020on :
21+ # No ``paths:`` filter on purpose. Gating lives in the ``changes`` job below so
22+ # these jobs are always created and report green when skipped -- a required check
23+ # that is never created stays pending forever.
2124 pull_request :
22- paths :
23- - " source/isaaclab/isaaclab/test/utils/**"
24- - " source/isaaclab/isaaclab/app/app_launcher.py"
25- - " source/**/test/**/test_*.py"
26- - " .github/workflows/test-multi-gpu-pytest.yaml"
27- - " .github/actions/run-package-tests/**"
28- - " .github/actions/run-tests/**"
29- - " .github/actions/ecr-build-push-pull/**"
3025 workflow_dispatch :
3126
3227concurrency :
@@ -35,8 +30,61 @@ concurrency:
3530
3631env :
3732 CI_IMAGE_TAG : isaac-lab-ci:${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}-${{ github.sha }}
33+ # Same tag family as kitless-docker.yml so the layer cache is shared, but built here: the ECR
34+ # cache repo is resolved per runner pool, so an image built on the `gpu` pool is invisible to
35+ # this one.
36+ CI_KITLESS_IMAGE_TAG : isaac-lab-ci:${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}-${{ github.sha }}-kitless
3837
3938jobs :
39+ changes :
40+ name : Detect Changes
41+ runs-on : ubuntu-latest
42+ outputs :
43+ unit : ${{ steps.unit.outputs.should_run }}
44+ smoke : ${{ steps.smoke.outputs.should_run }}
45+ steps :
46+ - uses : actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
47+ with :
48+ fetch-depth : 1
49+
50+ # Narrow: the sharded device-parametrized tests. Their blast radius really is
51+ # the launcher plus the device-scope helpers they are discovered by.
52+ - name : Detect changes (unit shards)
53+ id : unit
54+ uses : ./.github/actions/detect-changes
55+ with :
56+ triggered-jobs-pr : multi-GPU sharded unit tests
57+ patterns : |
58+ ^source/isaaclab/isaaclab/test/utils/ :: Device-scope test helpers
59+ ^source/isaaclab/isaaclab/app/app_launcher\.py$ :: Device selection at launch
60+ ^source/.+/test/.+/test_.*\.py$ :: Device-parametrized test files
61+ ^\.github/workflows/test-multi-gpu-pytest\.yaml$ :: This workflow
62+ ^\.github/actions/(run-tests|run-package-tests|ecr-build-push-pull|multi-gpu|detect-changes)/ :: Shared actions
63+
64+ # The training smoke drives real multi-rank RL runs, so the runtime it loads is matched
65+ # broadly. The task side is not: these tests exercise exactly two gym ids,
66+ # ``Isaac-Cartpole-Direct`` and ``Isaac-Cartpole-Camera-Direct``, so only their own
67+ # sources are matched rather than all of isaaclab_tasks/isaaclab_assets. An unrelated
68+ # manager task, contrib task, or robot asset must not cost an hour of multi-GPU runner.
69+ - name : Detect changes (training smoke)
70+ id : smoke
71+ uses : ./.github/actions/detect-changes
72+ with :
73+ triggered-jobs-pr : multi-GPU training smoke
74+ patterns : |
75+ ^source/isaaclab/ :: Core library (app, sim, scene, renderers, sensors)
76+ ^source/isaaclab_(newton|physx|ov)/ :: Physics and renderer backends
77+ ^source/isaaclab_ppisp/ :: Render pipeline the Newton renderer imports
78+ ^source/isaaclab_rl/ :: RL library integration (rsl_rl entry point)
79+ ^source/isaaclab_visualizers/ :: Visualizer and renderer coupling
80+ ^source/isaaclab_tasks/isaaclab_tasks/(__init__\.py$|utils/|core/cartpole/(agents/|__init__\.py$|cartpole_direct_)) :: Direct Cartpole envs, their registration and agent configs, plus the preset resolution the presets= selector goes through
81+ ^source/isaaclab_assets/isaaclab_assets/robots/cartpole\.py$ :: CARTPOLE_CFG, the only asset these envs load
82+ ^scripts/reinforcement_learning/ :: Multi-GPU training entry point
83+ ^docker/Dockerfile\.(base|kitless)$ :: CI image definitions
84+ ^(pyproject\.toml|uv\.lock)$ :: Dependency graph (image identity)
85+ ^\.github/workflows/test-multi-gpu-pytest\.yaml$ :: This workflow
86+ ^\.github/actions/(run-tests|run-package-tests|ecr-build-push-pull|multi-gpu|detect-changes)/ :: Shared actions
87+
4088 config :
4189 name : Load Config
4290 runs-on : ubuntu-latest
58106
59107 build :
60108 name : Build / cache image
61- needs : [config]
109+ needs : [config, changes]
110+ if : needs.changes.outputs.unit == 'true' || needs.changes.outputs.smoke == 'true'
62111 # Must run on the SAME pool as the test job. The ECR cache repo is resolved
63112 # per runner pool (single-GPU `gpu` runners -> gitci-docker-cache; multi-GPU
64113 # runners -> multigpu-docker-cache). If this built on `[self-hosted, gpu]`
@@ -88,16 +137,48 @@ jobs:
88137 dockerfile-path : docker/Dockerfile.base
89138 cache-tag : cache-base
90139
140+ build-kitless :
141+ name : Build / cache kit-less image
142+ needs : [config, changes]
143+ # Only the training smoke uses this image; the sharded unit tests are Kit-only.
144+ if : needs.changes.outputs.smoke == 'true'
145+ # Same pool as the test job for the same reason as ``build`` above -- the ECR cache repo is
146+ # per pool, so building elsewhere would force a rebuild on the scarce multi-GPU runner.
147+ runs-on : [self-hosted, linux, x64, multi-gpu]
148+ steps :
149+ - uses : actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
150+ with :
151+ fetch-depth : 1
152+ lfs : true
153+
154+ # Builds from ubuntu rather than the Isaac Sim image, and already installs ``ov[all]``:
155+ # with no Isaac Sim there is no prebundled ``packaging`` for ovphysx's ``<24`` cap to
156+ # overwrite, which is exactly why that install cannot go in Dockerfile.base.
157+ - uses : ./.github/actions/ecr-build-push-pull
158+ env :
159+ NGC_API_KEY : ${{ secrets.NGC_API_KEY }}
160+ with :
161+ image-tag : ${{ env.CI_KITLESS_IMAGE_TAG }}
162+ isaacsim-base-image : ${{ needs.config.outputs.isaacsim_image_name }}
163+ isaacsim-version : ${{ needs.config.outputs.isaacsim_image_tag }}
164+ dockerfile-path : docker/Dockerfile.kitless
165+ cache-tag : cache-kitless
166+
91167 test-multi-gpu-pytest :
92168 name : Multi-GPU unit tests
93- needs : [config, build]
94- if : needs.build.result == 'success'
169+ needs : [config, build, changes]
170+ # ``always()`` is load-bearing: ``build`` is skipped when neither gate fires, and GitHub skips
171+ # a job whose dependency skipped unless the condition carries a status function. Without it a
172+ # gate-miss would silently skip the shards too.
173+ if : always() && needs.build.result == 'success'
95174 # The ``multi-gpu`` label is the multi-GPU pool. Do NOT add ``gpu`` here:
96175 # in this fleet ``gpu`` tags single-GPU runners, so requiring it routes the
97176 # job onto a 1-GPU box and the shard runner aborts with "Need at least 2
98177 # visible devices; found 1".
99178 runs-on : [self-hosted, linux, x64, multi-gpu]
100- timeout-minutes : 60
179+ # Shards plus the full training-smoke matrix. The smoke alone is ~30 min
180+ # (Kit boot dominates the Kit-renderer cases), so 60 no longer fits.
181+ timeout-minutes : 120
101182 steps :
102183 - name : Checkout repository
103184 uses : actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
@@ -177,12 +258,87 @@ jobs:
177258 IMAGE_TAG : ${{ env.CI_IMAGE_TAG }}
178259 INCLUDE_FILES : ${{ steps.discover.outputs.include }}
179260 PATHS : ${{ steps.discover.outputs.paths }}
261+ if : needs.changes.outputs.unit == 'true'
180262 run : bash .github/actions/multi-gpu/multi_gpu_host_launcher.sh
181263
264+ - name : Multi-GPU training smoke (Kit renderer)
265+ # Real multi-rank training runs, outside the sharded device-parametrized
266+ # path above: a multi-GPU job owns several GPUs at once, so a per-device
267+ # shard cannot express it. Unsharded, all GPUs exposed; the tests choose
268+ # their own device order.
269+ #
270+ # Uses the shared run-tests action rather than a bespoke ``docker run``:
271+ # passing a file path (not ``tools``) skips the conftest sharding
272+ # orchestrator, so this is a plain single pytest run, while still
273+ # inheriting the standard container contract -- writable HOME, the Isaac
274+ # Sim cache mounts, and junit results collected into ``reports/``.
275+ # Runs on failure too, so a red shard does not hide smoke coverage.
276+ if : always() && needs.changes.outputs.smoke == 'true'
277+ uses : ./.github/actions/run-tests
278+ with :
279+ test-path : source/isaaclab/test/multi_gpu/test_multi_gpu_training_smoke.py
280+ result-file : multi-gpu-smoke-report.xml
281+ container-name : isaac-lab-mgpu-smoke-${{ github.run_id }}-${{ github.run_attempt }}
282+ image-tag : ${{ env.CI_IMAGE_TAG }}
283+ pytest-options : -m "not kitless"
284+ # Run the checkout, not the image's baked-in copy. ecr-build-push-pull serves a
285+ # commit-tagged or deps-cache image, so on a cache hit the container carries source
286+ # from whenever that image was built -- CI then reports a pass or fail for code this
287+ # commit never contained. The shard launcher already bind-mounts for this reason.
288+ volume-mount-source : ${{ github.workspace }}
289+
182290 - name : Aggregated test summary
183291 # Per-shard + per-file pass/total/walltime, plus a combined table, also
184292 # written to $GITHUB_STEP_SUMMARY. Runs on success or failure.
185293 if : always()
186294 env :
187295 RUNTIME_DIR : ${{ env.MGPU_RUNTIME_DIR }}
188296 run : python3 .github/actions/multi-gpu/aggregate_test_summary.py
297+
298+ test-multi-gpu-smoke-kitless :
299+ name : Multi-GPU training smoke (kit-less)
300+ needs : [config, build-kitless, changes]
301+ # Split out of ``Multi-GPU unit tests`` so a kit-less image build that fails -- or a runner
302+ # that dies under it -- cannot take down the Kit-renderer lane, which needs a different image
303+ # entirely. When two multi-GPU runners are free the lanes also overlap; when only one is, the
304+ # scheduler serialises them onto it, which is the previous behaviour.
305+ if : always() && needs.build-kitless.result == 'success' && needs.changes.outputs.smoke == 'true'
306+ runs-on : [self-hosted, linux, x64, multi-gpu]
307+ # One lane only; the 120 min ceiling on the sharded job covers shards + Kit smoke together.
308+ timeout-minutes : 60
309+ steps :
310+ - name : Checkout repository
311+ uses : actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
312+ with :
313+ fetch-depth : 1
314+ lfs : true
315+
316+ - name : Pull kit-less image from ECR
317+ # ``build-kitless`` pushes to ECR from whichever multi-GPU runner it landed on, so the
318+ # image is not necessarily present locally here. Without this the kitless step's
319+ # ``docker run`` fails with ``pull access denied`` whenever the two jobs are scheduled
320+ # on different runners.
321+ uses : ./.github/actions/ecr-build-push-pull
322+ env :
323+ NGC_API_KEY : ${{ secrets.NGC_API_KEY }}
324+ with :
325+ image-tag : ${{ env.CI_KITLESS_IMAGE_TAG }}
326+ isaacsim-base-image : ${{ needs.config.outputs.isaacsim_image_name }}
327+ isaacsim-version : ${{ needs.config.outputs.isaacsim_image_tag }}
328+ dockerfile-path : docker/Dockerfile.kitless
329+ cache-tag : cache-kitless
330+
331+ - name : Multi-GPU training smoke (kitless renderer)
332+ # Same tests as the Kit lane with the marker inverted, but run in the kit-less image,
333+ # which already carries ``ov[all]``. Each stack runs in the image built for it, and the
334+ # job split keeps a kitless-renderer fault from taking down the Kit-renderer coverage
335+ # that guards renderer device selection.
336+ uses : ./.github/actions/run-tests
337+ with :
338+ test-path : source/isaaclab/test/multi_gpu/test_multi_gpu_training_smoke.py
339+ result-file : multi-gpu-smoke-kitless-report.xml
340+ container-name : isaac-lab-mgpu-smoke-kitless-${{ github.run_id }}-${{ github.run_attempt }}
341+ image-tag : ${{ env.CI_KITLESS_IMAGE_TAG }}
342+ pytest-options : -m kitless
343+ # Same reason as the Kit step above.
344+ volume-mount-source : ${{ github.workspace }}
0 commit comments