Merge remote-tracking branch 'upstream/develop' into angehu/perf-smoke-integration

Author: angehu-nv

.github/actions/validate-kitless-image/action.yml

@@ -66,7 +66,7 @@ runs:
       uv pip install --python "$VIRTUAL_ENV/bin/python" pytest
       python -m pytest \
         source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py::test_train_cartpole_state_completes \
-        source/isaaclab_newton/test/test_mock_interfaces \
+        source/isaaclab/test/benchmark/test_asset_suite_runtime_semantics.py \
         -q -p no:cacheprovider
       '
 

.github/workflows/config.yaml

@@ -9,7 +9,7 @@
 # (frozen at 6.1.0-alpha.2). Both images use Isaac Sim's NGC org, which is current and
 # which the CI credential can reach.
 isaacsim_image_name: nvcr.io/0947644777160149/internal/isaac-sim
-# Pinned to Isaac Sim commit 307e5486 for reproducible CI.
-isaacsim_image_tag: latest-develop@sha256:a13867a6403488a9e640288f96492e6ecd44c3a525b5b85793a2da68fdac48ad
+# Pinned to Isaac Sim 6.1.0-alpha.47 (commit 0d707bba) for reproducible CI.
+isaacsim_image_tag: latest-develop@sha256:dc49acc482221d5302cc7095771b6a807de97a17c7835064f812dd013ff2b806
 isaaclab_image_name: nvcr.io/0947644777160149/internal/isaac-lab
 ovphysx_wheelhouse_image: ""

.github/workflows/kitless-docker.yml

@@ -56,8 +56,9 @@ jobs:
           ^uv\.lock$ :: Locked dependency graph
           ^source/[^/]+/pyproject\.toml$ :: Package manifests
           ^source/isaaclab/isaaclab/cli/ :: Install CLI implementation
-          ^source/isaaclab_newton/isaaclab_newton/test/mock_interfaces/ :: Mock-interface implementation
-          ^source/isaaclab_newton/test/test_mock_interfaces/ :: Kit-less pytest subset
+          ^source/isaaclab/isaaclab/test/fixtures/ :: Isolated asset fixture implementation
+          ^source/isaaclab_(physx|newton|ovphysx)/isaaclab_.*/test/fixtures/ :: Backend fixture implementation
+          ^source/isaaclab/test/benchmark/test_asset_suite_runtime_semantics\.py$ :: Kit-less pytest subset
           ^\.github/workflows/kitless-docker\.yml$ :: This workflow file
           ^\.github/actions/detect-changes/ :: Change-detection action
           ^\.github/actions/_lib/compute-deps-hash/ :: Dependency-cache identity

docs/source/features/multi_gpu.rst

@@ -481,3 +481,7 @@ For skrl JAX training, pass an integer GPU count and the ``--coordinator_address
 For multi-node torch jobs, pass torchrun settings such as ``--nnodes``, ``--node_rank``,
 ``--rdzv_backend``, ``--rdzv_endpoint``, and ``--rdzv_id`` before the training arguments. For
 skrl JAX multi-node jobs, pass ``--nnodes``, ``--node_rank``, and ``--coordinator_address``.
+
+To measure aggregate throughput and timing rather than to train a policy, use the multi-GPU
+benchmark workflows instead; see :ref:`testing_benchmarks_multigpu`. They take the same launcher
+options and support RSL-RL, RL-Games, and skrl with Torch.

docs/source/policy_deployment/05_leapp/exporting_policies_with_leapp.rst

@@ -117,7 +117,7 @@ Set the EULA variables in non-interactive shells so Isaac Sim can start without
                    --task <TASK_NAME> ^
                    --checkpoint <PATH_TO_CHECKPOINT>
 
-For example, to export a UR10 reach policy trained with RSL-RL:
+For example, to export a Humanoid policy trained with RSL-RL:
 
 .. tab-set::
    :sync-group: os
@@ -133,17 +133,17 @@ For example, to export a UR10 reach policy trained with RSL-RL:
 
                OMNI_KIT_ACCEPT_EULA=Y ACCEPT_EULA=Y uv run --extra leapp python \
                    scripts/reinforcement_learning/leapp/rsl_rl/export.py \
-                   --task Isaac-Reach-UR10 \
-                   --checkpoint logs/rsl_rl/ur10_reach/<date timestamp>/model_4999.pt
+                   --task Isaac-Humanoid \
+                   --checkpoint logs/rsl_rl/humanoid/<date timestamp>/model_999.pt
 
          .. tab-item:: isaaclab.sh / isaaclab.bat
 
             .. code-block:: bash
 
                OMNI_KIT_ACCEPT_EULA=Y ACCEPT_EULA=Y ./isaaclab.sh -p \
                    scripts/reinforcement_learning/leapp/rsl_rl/export.py \
-                   --task Isaac-Reach-UR10 \
-                   --checkpoint logs/rsl_rl/ur10_reach/<date timestamp>/model_4999.pt
+                   --task Isaac-Humanoid \
+                   --checkpoint logs/rsl_rl/humanoid/<date timestamp>/model_999.pt
 
    .. tab-item:: :icon:`fa-brands fa-windows` Windows
       :sync: windows
@@ -157,8 +157,8 @@ For example, to export a UR10 reach policy trained with RSL-RL:
                set OMNI_KIT_ACCEPT_EULA=Y
                set ACCEPT_EULA=Y
                uv run --extra leapp python scripts\reinforcement_learning\leapp\rsl_rl\export.py ^
-                   --task Isaac-Reach-UR10 ^
-                   --checkpoint logs\rsl_rl\ur10_reach\<date timestamp>\model_4999.pt
+                   --task Isaac-Humanoid ^
+                   --checkpoint logs\rsl_rl\humanoid\<date timestamp>\model_999.pt
 
          .. tab-item:: isaaclab.sh / isaaclab.bat
 
@@ -167,8 +167,8 @@ For example, to export a UR10 reach policy trained with RSL-RL:
                set OMNI_KIT_ACCEPT_EULA=Y
                set ACCEPT_EULA=Y
                isaaclab.bat -p scripts\reinforcement_learning\leapp\rsl_rl\export.py ^
-                   --task Isaac-Reach-UR10 ^
-                   --checkpoint logs\rsl_rl\ur10_reach\<date timestamp>\model_4999.pt
+                   --task Isaac-Humanoid ^
+                   --checkpoint logs\rsl_rl\humanoid\<date timestamp>\model_999.pt
 
 By default, the export artifacts are saved in the same directory as the checkpoint. The
 exported graph is named after the task.

docs/source/testing/benchmarks.rst

@@ -4,9 +4,9 @@ Benchmarking Isaac Lab
 ======================
 
 Isaac Lab provides supported benchmark workflows for environment stepping,
-trained-policy playback, reinforcement-learning training, and startup profiling.
-This guide explains which workflow to use, how to run it, and how to interpret
-its results.
+trained-policy playback, reinforcement-learning training, and startup profiling,
+each of which can also be run across several GPUs. This guide explains which
+workflow to use, how to run it, and how to interpret its results.
 
 .. seealso::
 
@@ -31,6 +31,8 @@ Choose A Workflow
      - ``training``
    * - Launch, import, configuration, scene creation, or first-step latency
      - ``startup``
+   * - Any of the above across several GPUs
+     - ``<workflow>-multigpu``
    * - One asset or sensor operation
      - :ref:`testing_micro_benchmarks`
 
@@ -119,7 +121,7 @@ same random-action stepping workload.
    .. code-block:: json
 
       {
-        "schema_version": "1.3",
+        "schema_version": "1.4",
         "run": {
           "config": {"physics_backend": "physx", "rendering_backend": "none", "presets": ["physx"]},
           "task": "Isaac-Cartpole-Direct", "seed": 42, "status": "completed", "num_envs": 4096
@@ -154,7 +156,7 @@ same random-action stepping workload.
    .. code-block:: json
 
       {
-        "schema_version": "1.3",
+        "schema_version": "1.4",
         "run": {
           "config": {
             "physics_backend": "physx", "rendering_backend": "isaacsim_rtx",
@@ -336,6 +338,96 @@ Do not treat a faster environment-step rate as proof of faster end-to-end
 training or equivalent learning. Do not compare short training curves as if
 they established final policy quality.
 
+.. _testing_benchmarks_multigpu:
+
+Multi-GPU
+---------
+
+Use It When
+~~~~~~~~~~~
+
+Append ``-multigpu`` to ``startup``, ``runtime``, or ``training`` to run the same
+workflow with one rank per GPU. Use it to measure synchronized multi-GPU training
+throughput, or to measure how much a workflow slows down when every GPU on the
+node is busy.
+
+Command
+~~~~~~~
+
+.. code-block:: bash
+
+   ./isaaclab.sh benchmark training-multigpu \
+       --rl_library rsl_rl \
+       --num_gpus 2 \
+       --task Isaac-Cartpole-Direct \
+       --num_envs 4096 \
+       --max_iterations 100 \
+       --seed 42 \
+       --visualizer none \
+       --output_path ./benchmark_results/multigpu \
+       physics=isaacsim_physx
+
+The launcher accepts ``--num_gpus``, ``--nnodes``, ``--node_rank``, and the
+``torchrun`` rendezvous options, exactly like :ref:`train-multigpu-command`. Every
+other argument is forwarded to the single-GPU workflow unchanged. Add ``--dry_run``
+to print the ``torchrun`` command without running it, and ``--log_all_ranks`` to
+show console output from every rank instead of local rank 0 only.
+
+For a multi-node run, issue the same command on every node with a distinct
+``--node_rank``:
+
+.. code-block:: bash
+
+   ./isaaclab.sh benchmark training-multigpu \
+       --rl_library rsl_rl --nnodes 2 --node_rank 0 --num_gpus 8 \
+       --rdzv_backend c10d --rdzv_endpoint host0:29400 --rdzv_id bench \
+       --task Isaac-Cartpole-Direct
+
+``training-multigpu`` supports ``rsl_rl``, ``rl_games``, and ``skrl`` with Torch. It
+does not support skrl JAX or SB3, and it rejects ``--video``,
+``--capture_env_sensors``, and ``--check_success``, none of which are meaningful
+across ranks. Use :ref:`train-multigpu-command` for general distributed training.
+
+Read The Result
+~~~~~~~~~~~~~~~
+
+``--num_envs`` is the number of environments **per rank**, and each rank creates its
+own Isaac Lab instance on its own GPU. Only global rank 0 writes a bundle. What that
+bundle covers depends on the workflow, and ``extra`` records it:
+
+.. list-table::
+   :header-rows: 1
+   :widths: 30 70
+
+   * - ``extra`` field
+     - Meaning
+   * - ``world_size``, ``local_world_size``, ``num_nodes``
+     - Rank layout of the job.
+   * - ``num_envs_per_rank``
+     - Environments hosted by each rank.
+   * - ``workload_scope``
+     - ``global`` for ``training-multigpu``: ranks train in lockstep, so ``run.num_envs``,
+       ``runtime.steps_per_iteration``, and every FPS field cover all ranks.
+       ``rank0`` for ``startup-multigpu`` and ``runtime-multigpu``: those ranks run
+       independent workloads, so the reported values are rank 0's own, measured while
+       the other ranks contend for the same host.
+   * - ``measurement_scope``
+     - ``rank0_process`` — timings, learning curves, CPU, and RAM come from rank 0 alone.
+   * - ``gpu_measurement_scope``
+     - ``rank0_node`` — ``resources.devices`` reports every GPU visible to rank 0, so a
+       single-node run shows all ranks. ``resources.gpu_util_pct`` and
+       ``resources.gpu_mem_gb`` remain scoped to rank 0's own device.
+
+Do Not Infer
+~~~~~~~~~~~~
+
+Do not compare a multi-GPU result against a single-GPU result at the same
+``--num_envs``: the multi-GPU run has ``world_size`` times as many environments. To
+measure scaling, compare the global throughput of an ``N``-GPU run against ``N``
+times the throughput of a single-GPU run at the same per-rank environment count. Do
+not read ``startup-multigpu`` or ``runtime-multigpu`` throughput as a global rate;
+their ``workload_scope`` is ``rank0``, and the other ranks were not measured.
+
 Startup Profiling
 -----------------
 

docs/source/testing/index.rst

@@ -16,4 +16,3 @@ when automation or a custom benchmark producer requires the Python API.
    benchmarks
    micro_benchmarks
    benchmark_framework
-   mock_interfaces