Angehu/perf smoke integration

.agents/skills/isaaclab-training-multi-gpu

@@ -0,0 +1 @@
+../../skills/user/train-multi-gpu

.github/CODEOWNERS

@@ -19,10 +19,10 @@
 
 
 # App experience files
-/apps/ @kellyguo11 @hujc @matthewtrepte @StafaH
+/apps/ @kellyguo11 @hujc7 @matthewtrepte @StafaH
 
 # Core framework
-/source/isaaclab/ @ooctipus @AntoineRichard @kellyguo11 @StafaH @pbarejko
+/source/isaaclab/ @ooctipus @AntoineRichard @kellyguo11 @StafaH @pbarejko @fatimaanes
 /source/isaaclab/isaaclab/envs/mimic_* @peterd-NV @kellyguo11
 /source/isaaclab/isaaclab/envs/manager_based_rl_mimic_env.py @peterd-NV @kellyguo11
 /source/isaaclab/isaaclab/envs/leapp_deployment_env.py @frlai @StafaH @kellyguo11
@@ -35,10 +35,10 @@
 /source/isaaclab_assets/isaaclab_assets/robots/dr_legs.py @david-cao-mueller @aserifi @rubengrandia @AntoineRichard
 
 /source/isaaclab_contrib/ @ooctipus @StafaH @kellyguo11
-/source/isaaclab_experimental/ @hujc @AntoineRichard @kellyguo11
+/source/isaaclab_experimental/ @hujc7 @AntoineRichard @kellyguo11
 /source/isaaclab_mimic/ @kellyguo11 @peterd-NV
 
-/source/isaaclab_newton/ @AntoineRichard @StafaH @hujc @kellyguo11
+/source/isaaclab_newton/ @AntoineRichard @StafaH @hujc7 @kellyguo11
 /source/isaaclab_newton/isaaclab_newton/physics/kamino_manager.py @david-cao-mueller @aserifi @rubengrandia @AntoineRichard
 
 /source/isaaclab_ov/ @huidongc @rilei-nvidia @pbarejko @kellyguo11
@@ -54,30 +54,30 @@
 /source/isaaclab_rl/ @ooctipus @StafaH @ClemensSchwarke @kellyguo11
 /source/isaaclab_rl/test/export/ @frlai @StafaH @kellyguo11
 
-/source/isaaclab_tasks/ @ooctipus @StafaH @kellyguo11 @hujc
+/source/isaaclab_tasks/ @ooctipus @StafaH @kellyguo11 @hujc7
 /source/isaaclab_tasks/isaaclab_tasks/contrib/dr_legs/ @david-cao-mueller @aserifi @rubengrandia @AntoineRichard
 
 # Standalone scripts
 /scripts/ @StafaH @kellyguo11 @ooctipus
-/scripts/benchmarks/ @AntoineRichard @StafaH @kellyguo11 @r-schmitt @fanes
+/scripts/benchmarks/ @AntoineRichard @StafaH @kellyguo11 @r-schmitt @fatimaanes
 /scripts/imitation_learning/ @peterd-NV @rwiltz @kellyguo11
 /scripts/imitation_learning/locomanipulation_sdg/ @jaybdub @peterd-NV @kellyguo11
 /scripts/reinforcement_learning/leapp/ @frlai @StafaH @kellyguo11
 
 # GitHub Actions and infrastructure
 /uv.lock @StafaH @kellyguo11
-/.github/ @kellyguo11 @myurasov-nv @hujc
-/.vscode/ @kellyguo11 @hujc @myurasov-nv @Toni-SM
-/docker/ @myurasov-nv @kellyguo11 @hujc
+/.github/ @kellyguo11 @myurasov-nv @hujc7
+/.vscode/ @kellyguo11 @hujc7 @myurasov-nv @Toni-SM
+/docker/ @myurasov-nv @kellyguo11 @hujc7
 /docs/ @AntoineRichard @kellyguo11 @StafaH @ooctipus
 /tools/ @AntoineRichard @kellyguo11 @StafaH @ooctipus
 
 # Cross-stack review caps
 /source/*/changelog.d/
 
 # RLiNF
-/source/isaaclab_contrib/isaaclab_contrib/rl/rlinf/ @mingxueg @kellyguo11
-/source/isaaclab_contrib/test/rl/test_rlinf_extension.py @mingxueg @kellyguo11
-/source/isaaclab_rl/isaaclab_rl/entrypoints/backends/*rlinf.py @mingxueg @kellyguo11
-/docs/source/experimental-features/rlinf_vla_posttraining.rst @mingxueg @kellyguo11
-/docs/source/api/lab_contrib/isaaclab_contrib.rl.rst @mingxueg @kellyguo11
+/source/isaaclab_contrib/isaaclab_contrib/rl/rlinf/ @mingxueg-nv @kellyguo11
+/source/isaaclab_contrib/test/rl/test_rlinf_extension.py @mingxueg-nv @kellyguo11
+/source/isaaclab_rl/isaaclab_rl/entrypoints/backends/*rlinf.py @mingxueg-nv @kellyguo11
+/docs/source/experimental-features/rlinf_vla_posttraining.rst @mingxueg-nv @kellyguo11
+/docs/source/api/lab_contrib/isaaclab_contrib.rl.rst @mingxueg-nv @kellyguo11

.github/actions/validate-kitless-image/action.yml

@@ -66,7 +66,7 @@ runs:
       uv pip install --python "$VIRTUAL_ENV/bin/python" pytest
       python -m pytest \
         source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py::test_train_cartpole_state_completes \
-        source/isaaclab_newton/test/test_mock_interfaces \
+        source/isaaclab/test/benchmark/test_asset_suite_runtime_semantics.py \
         -q -p no:cacheprovider
       '
 

.github/workflows/build.yaml

@@ -789,7 +789,7 @@ jobs:
     name: "rendering-correctness-kitless (${{ matrix.variant }})"
     runs-on: [self-hosted, gpu]
     timeout-minutes: 120
-    continue-on-error: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }}
+    continue-on-error: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && matrix.variant == 'ovstage') }}
     needs: [build, config]
     if: needs.build.result == 'success'
     strategy:
@@ -835,6 +835,211 @@ jobs:
         container-name: "isaac-lab-rendering-correctness-kitless-${{ matrix.variant }}-test"
         omni-github-test-type: "rendering-correctness-kitless-${{ matrix.variant }}"
 
+  performance-smoke:
+    name: "performance-smoke (${{ matrix.name }})"
+    runs-on: [self-hosted, gpu]
+    # Budget for two attempts plus image pull and cache restore.
+    # Each attempt is bounded individually by `timeout` to prevent starving retry.
+    timeout-minutes: ${{ matrix.timeout_minutes * 2 + 10 }}
+    needs: [build, config]
+    # Pull requests run in gate mode, pushes to develop additionally record baseline.
+    if: >-
+      (github.event_name != 'push' || github.ref == 'refs/heads/develop') &&
+      needs.build.result == 'success'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - name: cartpole-physx
+          task: Isaac-Cartpole-Direct
+          num_envs: 4096
+          timeout_minutes: 10
+          args: ""
+        - name: cartpole-newton
+          task: Isaac-Cartpole-Direct
+          num_envs: 4096
+          timeout_minutes: 10
+          args: physics=newton_mjwarp
+        - name: gear-mesh-physx
+          task: IsaacContrib-Factory-GearMesh-Direct
+          num_envs: 512
+          timeout_minutes: 15
+          args: ""
+        - name: shadow-camera-physx-rtx
+          task: Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct
+          num_envs: 512
+          timeout_minutes: 20
+          args: renderer=isaacsim_rtx
+        - name: shadow-camera-physx-newton-renderer
+          task: Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct
+          num_envs: 512
+          timeout_minutes: 20
+          args: renderer=newton_renderer
+        - name: shadow-camera-newton-rtx
+          task: Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct
+          num_envs: 512
+          timeout_minutes: 20
+          args: physics=newton_mjwarp renderer=isaacsim_rtx
+        - name: shadow-camera-newton-renderer
+          task: Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct
+          num_envs: 512
+          timeout_minutes: 20
+          args: physics=newton_mjwarp renderer=newton_renderer
+        - name: g1-physx
+          task: Isaac-Velocity-Flat-G1
+          num_envs: 512
+          timeout_minutes: 12
+          args: ""
+        - name: g1-newton
+          task: Isaac-Velocity-Flat-G1
+          num_envs: 512
+          timeout_minutes: 12
+          args: physics=newton_mjwarp
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 1
+    - name: Pull pull-request image
+      uses: ./.github/actions/ecr-build-push-pull
+      with:
+        image-tag: ${{ needs.config.outputs.ci_image_tag }}
+        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
+        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
+        dockerfile-path: docker/Dockerfile.base
+        cache-tag: cache-base
+    - name: Resolve Warp version for the JIT cache key
+      id: warp_version
+      run: |
+        set -euo pipefail
+        version="$(sed -n 's/.*warp-lang[^0-9]*\([0-9][0-9A-Za-z.\-]*\).*/\1/p' pyproject.toml | head -n1)"
+        if [ -z "$version" ]; then
+          echo "::error::Could not resolve the warp-lang pin from pyproject.toml; the JIT cache key would stop tracking Warp"
+          exit 1
+        fi
+        echo "value=$version" >> "$GITHUB_OUTPUT"
+
+    - name: Restore warm JIT cache
+      uses: actions/cache@v4
+      with:
+        path: ${{ github.workspace }}/jit-cache
+        key: perf-jit-warp${{ steps.warp_version.outputs.value }}-${{ matrix.name }}-${{ github.run_id }}
+        restore-keys: |
+          perf-jit-warp${{ steps.warp_version.outputs.value }}-${{ matrix.name }}-
+          perf-jit-warp${{ steps.warp_version.outputs.value }}-
+
+    - name: Run runtime benchmark
+      env:
+        BENCHMARK_ARGS: ${{ matrix.args }}
+        BENCHMARK_TASK: ${{ matrix.task }}
+        CI_IMAGE_TAG: ${{ needs.config.outputs.ci_image_tag }}
+        NUM_ENVS: ${{ matrix.num_envs }}
+        BENCH_TIMEOUT_S: ${{ matrix.timeout_minutes * 60 }}
+      run: |
+        set -uo pipefail
+        install -d -m 0777 benchmark-output
+        # World-writable so the in-container user (uid 1000) can populate the cache
+        install -d -m 0777 "${GITHUB_WORKSPACE}/jit-cache/warp" "${GITHUB_WORKSPACE}/jit-cache/nv"
+
+        run_attempt() {
+          local attempt="$1"
+          local container="performance-smoke-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ matrix.name }}-${attempt}"
+          docker rm -f "$container" >/dev/null 2>&1 || true
+          timeout "${BENCH_TIMEOUT_S}" docker run --name "$container" --gpus all --network=host --ipc=host \
+            -e OMNI_KIT_ACCEPT_EULA=yes \
+            -e ACCEPT_EULA=Y \
+            -e ISAAC_SIM_HEADLESS=1 \
+            -e WARP_CACHE_PATH=/tmp/jit-cache/warp \
+            -e CUDA_CACHE_PATH=/tmp/jit-cache/nv \
+            -v "${GITHUB_WORKSPACE}/jit-cache:/tmp/jit-cache" \
+            --entrypoint bash "$CI_IMAGE_TAG" -lc \
+            "set -euo pipefail
+            mkdir -p /tmp/benchmark-output
+            ./isaaclab.sh benchmark runtime \
+              --task '$BENCHMARK_TASK' \
+              --num_envs '$NUM_ENVS' \
+              --num_steps 200 \
+              --warmup_steps 100 \
+              --seed 42 \
+              --benchmark_formatter schema \
+              --output_path /tmp/benchmark-output \
+              --visualizer none \
+              $BENCHMARK_ARGS"
+          local status=$?
+          docker cp "$container:/tmp/benchmark-output/." benchmark-output/ 2>/dev/null || true
+          docker rm -f "$container" >/dev/null 2>&1 || true
+          return $status
+        }
+
+        if ! run_attempt 1; then
+          echo "::warning::${{ matrix.name }} failed on the first attempt; retrying once"
+          rm -f benchmark-output/benchmark_runtime_*.json
+          run_attempt 2
+        fi
+    # Read-only. Recording step below is the only writer scoped to refs/heads/develop.
+    - name: Compare against baseline history
+      env:
+        PERF_BASELINE_URI: ${{ vars.PERF_BASELINE_URI }}
+      run: |
+        set -euo pipefail
+        mapfile -t results < <(find benchmark-output -maxdepth 1 -type f -name 'benchmark_runtime_*.json')
+        if [ "${#results[@]}" -ne 1 ]; then
+          echo "::error::Expected one runtime benchmark JSON, found ${#results[@]}"
+          exit 1
+        fi
+        # An unset URI yields SKIP, not failure because a missing baseline is not a regression.
+        python3 -m tools.perf_smoke.cli compare \
+          --benchmark_result "${results[0]}" \
+          --baseline_uri "${PERF_BASELINE_URI:-}" \
+          --label "${{ matrix.name }}" \
+          --output_json benchmark-output/comparison.json \
+          | tee -a "$GITHUB_STEP_SUMMARY"
+
+    # Only develop records baselines, so the gate always compares a pull request
+    # against the branch it will merge into rather than against a stale snapshot.
+    - name: Record baseline
+      if: ${{ github.ref == 'refs/heads/develop' && vars.PERF_BASELINE_URI != '' }}
+      env:
+        PERF_BASELINE_URI: ${{ vars.PERF_BASELINE_URI }}
+      run: |
+        set -euo pipefail
+        mapfile -t results < <(find benchmark-output -maxdepth 1 -type f -name 'benchmark_runtime_*.json')
+        python3 -m tools.perf_smoke.cli write \
+          --benchmark_result "${results[0]}" \
+          --baseline_uri "$PERF_BASELINE_URI" \
+          --commit "$GITHUB_SHA" \
+          --timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+          --run_id "$GITHUB_RUN_ID"
+    - name: Upload performance results
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: performance-smoke-${{ matrix.name }}-${{ github.run_id }}
+        path: benchmark-output/
+        if-no-files-found: warn
+        retention-days: 7
+
+  # Centralized summary; individual failed tasks still display their own failed jobs.
+  performance-smoke-summary:
+    name: performance-smoke
+    runs-on: ubuntu-latest
+    needs: [performance-smoke]
+    if: always() && needs.performance-smoke.result != 'skipped'
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 1
+    - uses: actions/download-artifact@v7
+      with:
+        pattern: performance-smoke-*-${{ github.run_id }}
+        path: comparisons
+      continue-on-error: true
+    - name: Summarise performance comparisons
+      run: |
+        set -euo pipefail
+        python3 -m tools.perf_smoke.cli aggregate \
+          --comparison_dir comparisons \
+          | tee -a "$GITHUB_STEP_SUMMARY"
+
   # Publishes the shared Warp kernel cache the test jobs restore. Needed as its
   # own job because a cache written from a PR is readable only by that PR, and
   # no solver-heavy test job runs on push. Does not restore before saving:

.github/workflows/config.yaml

@@ -6,8 +6,10 @@
 # Shared image config for CI workflows. Loaded by the `config` job in each
 # workflow via yq and exposed as job outputs (see e.g. .github/workflows/build.yaml).
 # The `nvidian/isaac-sim` mirror stopped receiving nightly develop builds on 2026-06-24
-# (frozen at 6.1.0-alpha.2). Isaac Sim publishes to its own NGC org, which is current.
+# (frozen at 6.1.0-alpha.2). Both images use Isaac Sim's NGC org, which is current and
+# which the CI credential can reach.
 isaacsim_image_name: nvcr.io/0947644777160149/internal/isaac-sim
-isaacsim_image_tag: latest-develop
-isaaclab_image_name: nvcr.io/nvidian/isaac-lab
+# Pinned to Isaac Sim 6.1.0-alpha.47 (commit 0d707bba) for reproducible CI.
+isaacsim_image_tag: latest-develop@sha256:dc49acc482221d5302cc7095771b6a807de97a17c7835064f812dd013ff2b806
+isaaclab_image_name: nvcr.io/0947644777160149/internal/isaac-lab
 ovphysx_wheelhouse_image: ""

.github/workflows/kitless-docker.yml

@@ -56,8 +56,9 @@ jobs:
           ^uv\.lock$ :: Locked dependency graph
           ^source/[^/]+/pyproject\.toml$ :: Package manifests
           ^source/isaaclab/isaaclab/cli/ :: Install CLI implementation
-          ^source/isaaclab_newton/isaaclab_newton/test/mock_interfaces/ :: Mock-interface implementation
-          ^source/isaaclab_newton/test/test_mock_interfaces/ :: Kit-less pytest subset
+          ^source/isaaclab/isaaclab/test/fixtures/ :: Isolated asset fixture implementation
+          ^source/isaaclab_(physx|newton|ovphysx)/isaaclab_.*/test/fixtures/ :: Backend fixture implementation
+          ^source/isaaclab/test/benchmark/test_asset_suite_runtime_semantics\.py$ :: Kit-less pytest subset
           ^\.github/workflows/kitless-docker\.yml$ :: This workflow file
           ^\.github/actions/detect-changes/ :: Change-detection action
           ^\.github/actions/_lib/compute-deps-hash/ :: Dependency-cache identity

.github/workflows/license-check.yaml

@@ -61,11 +61,11 @@ jobs:
         ACCEPT_EULA: Y
         ISAACSIM_ACCEPT_EULA: YES
       run: |
-        uv sync --extra all --extra test
-        # Isaac Sim conflicts with --extra all under uv, so install it imperatively
-        # after the sync. Read the pinned spec from pyproject (single source of truth).
-        ISAACSIM_SPEC=$(.venv/bin/python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['optional-dependencies']['isaacsim'][0])")
-        uv pip install "$ISAACSIM_SPEC"
+        # ``all`` covers every backend (Isaac Sim included), RL library, and visualizer.
+        # ``rlinf`` and ``mimic`` are outside ``all``, so name them to keep them scanned.
+        # No extras conflict, so this is a single resolution -- Isaac Sim no longer needs
+        # an imperative install after the sync.
+        uv sync --extra all --extra test --extra rlinf --extra mimic
         uv pip install pip-licenses pipdeptree \
           -r tools/template/requirements.txt
         # Put the venv on PATH for later steps.
@@ -175,8 +175,9 @@ jobs:
       run: |
         set -euo pipefail
         config=.github/workflows/config.yaml
-        base_image="$(awk -F ':[[:space:]]*' '$1 == "isaacsim_image_name" { print $2; exit }' "$config")"
-        base_tag="$(awk -F ':[[:space:]]*' '$1 == "isaacsim_image_tag" { print $2; exit }' "$config")"
+        # The tag may carry a digest, so strip the key rather than splitting on every colon.
+        base_image="$(awk '$1 == "isaacsim_image_name:" { sub(/^[^:]+:[[:space:]]*/, ""); print; exit }' "$config")"
+        base_tag="$(awk '$1 == "isaacsim_image_tag:" { sub(/^[^:]+:[[:space:]]*/, ""); print; exit }' "$config")"
         if [[ -z "$base_image" || -z "$base_tag" ]]; then
           echo "::error::Missing Isaac Sim image configuration in $config"
           exit 1

.github/workflows/wheel.yml

@@ -200,4 +200,4 @@ jobs:
             --extra-index-url https://pypi.nvidia.com \
             --index-strategy unsafe-best-match \
             --prerelease=allow \
-            "${wheel}[isaacsim,all]"
+            "${wheel}[all]"

.gitignore

@@ -77,6 +77,8 @@ _build
 
 # No benchmarks output
 /benchmarks/
+benchmark_*.json
+
 
 # Ruff cache
 **/.ruff_cache/

.pre-commit-config.yaml

@@ -21,6 +21,9 @@ repos:
       - id: check-added-large-files
         args: ["--maxkb=2000"]  # restrict files more than 2 MB. Should use git-lfs instead.
       - id: check-yaml
+        # OSMO workflow specs are Jinja templates, so they only become valid
+        # YAML once the submission-time parameters have been substituted.
+        exclude: '^docker/cluster/osmo_multi_gpu_workflow\.yaml$'
       - id: check-merge-conflict
         # These verbatim license headings contain a literal "=======" separator.
         exclude: '^docs/licenses/dependencies/(mujoco|llvmlite|numba)-license\.txt$'

AGENTS.md

@@ -224,7 +224,7 @@ Notes:
 Snake-style `Test_<Filename_Stem_TitleCase>`, mirroring the filename. Each underscore-separated segment is TitleCase. Common acronyms stay uppercase when separate segments. **Do not** use CamelCase here; snake-style is the deliberate convention for these long compound names.
 
 - ✅ `Test_Cli_Install_Core_In_Uvenv_Correctness`
-- ✅ `Test_Uv_Pip_Install_Isaaclab_All_Isaacsim_Trains_Cartpole`
+- ✅ `Test_Uv_Pip_Install_Isaaclab_All_Trains_Cartpole`
 - ❌ `TestCliInstallCoreInUvenvCorrectness`
 
 #### Method naming (strict)