Share one Kit app across test files instead of booting it per file

.github/actions/multi-gpu/mgpu_shard_select.py

@@ -5,7 +5,7 @@
 
 """pytest plugin: select which tests a non-default GPU shard runs.
 
-Loaded only by the multi-GPU lane: ``tools/conftest.py`` injects it via
+Loaded only by the multi-GPU lane: ``tools/run_tests.py`` injects it via
 ``-p mgpu_shard_select`` into each per-file pytest subprocess (and prepends this
 directory to ``PYTHONPATH`` so it is importable). It lives next to the lane
 scripts rather than as a repo-root ``conftest.py`` so it only affects the lane.
@@ -86,7 +86,7 @@ def pytest_collection_modifyitems(config, items):
 def pytest_sessionfinish(session, exitstatus):
     # A file whose tests are all out of scope deselects to zero, so pytest exits
     # NO_TESTS_COLLECTED (5). The lane orchestrator treats any non-zero per-file
-    # exit as a failure (tools/conftest.py), so report "nothing in scope for this
+    # exit as a failure (tools/run_tests.py), so report "nothing in scope for this
     # file" as success rather than a false failure.
     if _shard_mask() is not None and exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED:
         session.exitstatus = pytest.ExitCode.OK

.github/actions/multi-gpu/multi_gpu_shard_runner.sh

@@ -103,10 +103,7 @@ for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do  # C-style loop; start at 1 to sk
     # full $shard_log under a collapsible ``::group::shard cuda:N log``.
     # (tee = full output to the log file; stdbuf -oL = flush per line so the
     # filtered grep/sed stream appears live, not in delayed chunks.)
-    ./isaaclab.sh -p -m pytest \
-      --ignore=tools/conftest.py \
-      --ignore=source/isaaclab/test/install_ci \
-      tools -v 2>&1 \
+    ./isaaclab.sh -p tools/run_tests.py --all 2>&1 \
       | tee "$shard_log" \
       | stdbuf -oL grep -aE \
           '🚀|^source/.*::.* (PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)|^(Total|Passing|Failing|Crashed|Startup Hang|Timeout|Total Wall Time|Total Test Time|Passing Percentage):|^~~~~|^=+ |^E +|^ +File |Traceback|^FAILED|^ERROR ' \

.github/actions/run-package-tests/action.yml

@@ -27,44 +27,21 @@ inputs:
     description: 'ECR cache tag'
     default: 'cache-base'
     required: false
-  filter-pattern:
-    description: >-
-      Pattern to filter test files (e.g., isaaclab_tasks); test files whose path
-      contains this pattern are included. Only one pattern can be used at a time,
-      no comma-separated substrings allowed. Also supports the legacy "not <pattern>"
-      form for exclude-only jobs.
-    default: ''
-    required: false
-  exclude-pattern:
-    description: >-
-      Comma-separated substrings; test files whose path contains any entry are
-      skipped. Combines with filter-pattern (include + exclude).
-    default: ''
-    required: false
   test-k-expr:
     description: >-
       Global pytest -k expression applied inside every per-file pytest run
-      spawned by tools/conftest.py (combined with device-split selectors).
+      spawned by tools/run_tests.py (combined with device-split selectors).
     default: ''
     required: false
-  shard-index:
-    description: 'Zero-based shard index'
-    default: ''
-    required: false
-  shard-count:
-    description: 'Total number of shards'
+  job:
+    description: >-
+      Name of a job in tools/test_plan.toml. The plan decides which files the job covers;
+      tools/run_tests.py runs them. Leave empty only when test-path names a file to run
+      directly through pytest.
     default: ''
     required: false
-  curobo-only:
-    description: 'Run only cuRobo and SkillGen tests'
-    default: 'false'
-    required: false
-  quarantined-only:
-    description: 'Run only quarantined tests'
-    default: 'false'
-    required: false
-  include-files:
-    description: 'Comma-separated list of specific test files to include'
+  shard:
+    description: 'Which shard of a sharded job to run, 0-based. Empty for an unsharded job.'
     default: ''
     required: false
   test-node-ids-file:
@@ -301,14 +278,9 @@ runs:
         container-name: "${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}"
         image-tag: ${{ inputs.image-tag }}
         pytest-options: ${{ inputs.pytest-options }}
-        filter-pattern: ${{ inputs.filter-pattern }}
-        exclude-pattern: ${{ inputs.exclude-pattern }}
+        job: ${{ inputs.job }}
+        shard: ${{ inputs.shard }}
         test-k-expr: ${{ inputs.test-k-expr }}
-        shard-index: ${{ inputs.shard-index }}
-        shard-count: ${{ inputs.shard-count }}
-        curobo-only: ${{ inputs.curobo-only }}
-        quarantined-only: ${{ inputs.quarantined-only }}
-        include-files: ${{ inputs.include-files }}
         test-node-ids-file: ${{ inputs.test-node-ids-file }}
         test-node-ids-key: ${{ inputs.test-node-ids-key }}
         volume-mount-source: ${{ github.workspace }}

.github/actions/run-tests/action.yml

@@ -27,38 +27,23 @@ inputs:
     description: 'Additional pytest options (e.g., -k filter)'
     default: ''
     required: false
-  filter-pattern:
-    description: >-
-      Pattern to filter test files (e.g., isaaclab_tasks); test files whose path
-      contains this pattern are included. Only one pattern can be used at a time,
-      no comma-separated substrings allowed. Also supports the legacy "not <pattern>"
-      form for exclude-only jobs.
-    default: ''
-    required: false
-  exclude-pattern:
-    description: >-
-      Comma-separated substrings; test files whose path contains any entry are
-      excluded. Combines with filter-pattern (include + exclude).
-    default: ''
-    required: false
   test-k-expr:
     description: >-
       Global pytest -k expression applied inside every per-file pytest run
-      spawned by tools/conftest.py (combined with device-split selectors).
+      spawned by tools/run_tests.py (combined with device-split selectors).
       Unlike pytest-options, this reaches the individual test processes, so it
       can deselect parametrized cases (e.g. "not ovphysx").
     default: ''
     required: false
-  curobo-only:
-    description: 'Run only cuRobo and SkillGen tests (requires the cuRobo Docker image)'
-    default: 'false'
-    required: false
-  quarantined-only:
-    description: 'Run only tests listed in QUARANTINED_TESTS (skipped in normal jobs)'
-    default: 'false'
+  job:
+    description: >-
+      Name of a job in tools/test_plan.toml. The plan decides which files the job covers;
+      tools/run_tests.py runs them. Leave empty only when test-path names a file to run
+      directly through pytest.
+    default: ''
     required: false
-  include-files:
-    description: 'Comma-separated list of specific test file paths to include (e.g., source/pkg/test/test_a.py,source/pkg/test/test_b.py)'
+  shard:
+    description: 'Which shard of a sharded job to run, 0-based. Empty for an unsharded job.'
     default: ''
     required: false
   test-node-ids-file:
@@ -69,14 +54,6 @@ inputs:
     description: 'Top-level key in test-node-ids-file containing the node IDs for this job'
     default: ''
     required: false
-  shard-index:
-    description: 'Zero-based index of this shard (used with shard-count to split tests across parallel jobs)'
-    default: ''
-    required: false
-  shard-count:
-    description: 'Total number of shards (used with shard-index to split tests across parallel jobs)'
-    default: ''
-    required: false
   volume-mount-source:
     description: 'Host path to bind-mount at /workspace/isaaclab (for deps-cache-hit mode)'
     default: ''
@@ -100,10 +77,6 @@ inputs:
     description: 'Host Warp kernel cache directory bind-mounted into the container as WARP_CACHE_PATH'
     default: ''
     required: false
-  ci-marker:
-    description: 'CI_MARKER value forwarded to the container (read by tools/conftest.py to select test files by pytest marker)'
-    default: ''
-    required: false
   standalone-script-scope:
     description: 'Enable standalone script smoke tests for this scripts/ subdirectory'
     default: ''
@@ -132,9 +105,8 @@ runs:
         # the run_tests positional arguments if substituted textually.
         PYTEST_OPTIONS: ${{ inputs.pytest-options }}
         TEST_K_EXPR_INPUT: ${{ inputs.test-k-expr }}
-        CI_MARKER_INPUT: ${{ inputs.ci-marker }}
       run: |
-        bash .github/actions/run-tests/run_tests.sh "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "$TEST_K_EXPR_INPUT" "$CI_MARKER_INPUT" "${{ inputs.standalone-script-scope }}" "${{ inputs.standalone-script-visualizer }}" "${{ inputs.standalone-script-runtime-group }}" "${{ inputs.warp-cache-host-dir }}" "${{ inputs.extra-uv-packages }}"
+        bash .github/actions/run-tests/run_tests.sh "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.job }}" "${{ inputs.shard }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "$TEST_K_EXPR_INPUT" "${{ inputs.standalone-script-scope }}" "${{ inputs.standalone-script-visualizer }}" "${{ inputs.standalone-script-runtime-group }}" "${{ inputs.warp-cache-host-dir }}" "${{ inputs.extra-uv-packages }}"
     - name: Kill container on cancellation
       if: cancelled()
       shell: bash

.github/actions/run-tests/run_tests.sh

@@ -17,26 +17,20 @@ run_tests() {
   local image_tag="$4"
   local reports_dir="$5"
   local pytest_options="$6"
-  local filter_pattern="$7"
-  local exclude_pattern="$8"
-  local curobo_only="$9"
-  local include_files="${10}"
-  local quarantined_only="${11}"
-  local shard_index="${12}"
-  local shard_count="${13}"
-  local volume_mount_source="${14}"
-  local extra_pip_packages="${15}"
-  local test_node_ids_file="${16}"
-  local test_node_ids_key="${17}"
-  local wheelhouse_host_dir="${18}"
-  local wheelhouse_packages="${19}"
-  local test_k_expr="${20}"
-  local ci_marker="${21}"
-  local standalone_script_scope="${22}"
-  local standalone_script_visualizer="${23}"
-  local standalone_script_runtime_group="${24}"
-  local warp_cache_host_dir="${25}"
-  local extra_uv_packages="${26}"
+  local job="$7"
+  local shard="$8"
+  local volume_mount_source="$9"
+  local extra_pip_packages="${10}"
+  local test_node_ids_file="${11}"
+  local test_node_ids_key="${12}"
+  local wheelhouse_host_dir="${13}"
+  local wheelhouse_packages="${14}"
+  local test_k_expr="${15}"
+  local standalone_script_scope="${16}"
+  local standalone_script_visualizer="${17}"
+  local standalone_script_runtime_group="${18}"
+  local warp_cache_host_dir="${19}"
+  local extra_uv_packages="${20}"
   local logs_pid=""
   local wait_pid=""
   local docker_wait_file="/tmp/.docker_exit_${container_name}"
@@ -72,26 +66,20 @@ run_tests() {
   if [ -n "$wheelhouse_packages" ]; then
     echo "With wheelhouse packages: $wheelhouse_packages"
   fi
-  if [ -n "$filter_pattern" ]; then
-    echo "With filter pattern: $filter_pattern"
-  fi
-  if [ -n "$exclude_pattern" ]; then
-    echo "With exclude pattern: $exclude_pattern"
-  fi
-  if [ "$curobo_only" = "true" ]; then
-    echo "cuRobo-only mode enabled: running only cuRobo and SkillGen tests"
-  fi
-  if [ -n "$include_files" ]; then
-    echo "Include files: $include_files"
+  if [ -n "$job" ]; then
+    echo "Running test plan job: $job"
   fi
   if [ -n "$test_node_ids_file" ]; then
     echo "Test node IDs file: $test_node_ids_file"
   fi
   if [ -n "$test_node_ids_key" ]; then
     echo "Test node IDs key: $test_node_ids_key"
   fi
-  if [ -n "$shard_index" ] && [ -n "$shard_count" ]; then
-    echo "Shard: $shard_index of $shard_count"
+  # The runner takes the shard as a flag; an unsharded job passes nothing.
+  local shard_arg=""
+  if [ -n "$shard" ]; then
+    shard_arg="--shard $shard"
+    echo "Shard: $shard"
   fi
 
   if [ -n "$test_node_ids_file" ] || [ -n "$test_node_ids_key" ]; then
@@ -121,24 +109,6 @@ run_tests() {
     -e GITHUB_ACTIONS=${GITHUB_ACTIONS:-} \
     -e TEST_RESULT_FILE=$result_file"
 
-  if [ "$curobo_only" = "true" ]; then
-    docker_env_vars="$docker_env_vars -e TEST_CUROBO_ONLY=true"
-    echo "Setting TEST_CUROBO_ONLY=true"
-  fi
-
-  if [ "$quarantined_only" = "true" ]; then
-    docker_env_vars="$docker_env_vars -e TEST_QUARANTINED_ONLY=true"
-    echo "Setting TEST_QUARANTINED_ONLY=true"
-  fi
-
-  if [ -n "$include_files" ]; then
-    # Strip spaces so the value is safe to embed in an unquoted docker_env_vars string.
-    # conftest.py splits on commas and strips whitespace, so compact form works fine.
-    include_files_compact="${include_files// /}"
-    docker_env_vars="$docker_env_vars -e TEST_INCLUDE_FILES=$include_files_compact"
-    echo "Setting TEST_INCLUDE_FILES=$include_files_compact"
-  fi
-
   if [ -n "${TEST_NODE_IDS:-}" ]; then
     docker_env_vars="$docker_env_vars -e TEST_NODE_IDS"
     echo "Setting TEST_NODE_IDS"
@@ -149,35 +119,6 @@ run_tests() {
     echo "Setting TEST_NODE_IDS_FILE=$TEST_NODE_IDS_FILE TEST_NODE_IDS_KEY=$TEST_NODE_IDS_KEY"
   fi
 
-  if [ -n "$shard_index" ] && [ -n "$shard_count" ]; then
-    docker_env_vars="$docker_env_vars -e TEST_SHARD_INDEX=$shard_index -e TEST_SHARD_COUNT=$shard_count"
-    echo "Setting TEST_SHARD_INDEX=$shard_index TEST_SHARD_COUNT=$shard_count"
-  fi
-
-  if [ -n "$filter_pattern" ]; then
-    if [[ "$filter_pattern" == "not "* ]]; then
-      # Handle "not <pattern>" case - note the trailing space to avoid
-      # matching words that happen to start with "not".
-      filter_exclude_pattern="${filter_pattern#not }"
-      if [ -n "$exclude_pattern" ]; then
-        exclude_pattern="${exclude_pattern},${filter_exclude_pattern}"
-      else
-        exclude_pattern="$filter_exclude_pattern"
-      fi
-    else
-      # Handle positive pattern case
-      docker_env_vars="$docker_env_vars -e TEST_FILTER_PATTERN=$filter_pattern"
-      echo "Setting include pattern: $filter_pattern"
-    fi
-  else
-    echo "No filter pattern provided"
-  fi
-
-  if [ -n "$exclude_pattern" ]; then
-    docker_env_vars="$docker_env_vars -e TEST_EXCLUDE_PATTERN=$exclude_pattern"
-    echo "Setting exclude pattern: $exclude_pattern"
-  fi
-
   if [ -n "$extra_pip_packages" ]; then
     export TEST_EXTRA_PIP_PACKAGES="$extra_pip_packages"
     docker_env_vars="$docker_env_vars -e TEST_EXTRA_PIP_PACKAGES"
@@ -193,10 +134,6 @@ run_tests() {
     echo "Setting per-file pytest -k expression: $test_k_expr"
   fi
 
-  if [ -n "$ci_marker" ]; then
-    docker_env_vars="$docker_env_vars -e CI_MARKER=$ci_marker"
-    echo "Setting CI_MARKER=$ci_marker"
-  fi
 
   if [ -n "$standalone_script_scope" ]; then
     docker_env_vars="$docker_env_vars \
@@ -373,8 +310,13 @@ run_tests() {
         bash /with-python-package-retries.sh \"\${uv_executable}\" pip install --python \"\${isaac_python}\" --target \"\${isaac_uv_overlay}\" --no-deps \${TEST_EXTRA_UV_PACKAGES}
         export PYTHONPATH=\"\${isaac_uv_overlay}\${PYTHONPATH:+:\${PYTHONPATH}}\"
       fi
-      echo 'Starting pytest with path: $test_path'
-      ./isaaclab.sh -p -m pytest --ignore=tools/conftest.py $test_path $pytest_options -v --junitxml=tests/$result_file
+      if [ -n '$job' ]; then
+        echo 'Running test plan job: $job'
+        ./isaaclab.sh -p tools/run_tests.py --job '$job' $shard_arg
+      else
+        echo 'Starting pytest with path: $test_path'
+        ./isaaclab.sh -p -m pytest $test_path $pytest_options -v --junitxml=tests/$result_file
+      fi
     "
 
   # Stream container logs in background.

.github/workflows/arm-ci.yml

@@ -184,8 +184,7 @@ jobs:
         container-name: isaac-lab-arm-ci-${{ github.run_id }}-${{ github.run_attempt }}
         image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64
         extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}"
-        test-k-expr: not ovphysx
-        ci-marker: arm_ci
+        job: arm-ci
         volume-mount-source: ${{ github.workspace }}
 
     - name: Run arm_ci OVPhysX marker tests
@@ -196,8 +195,7 @@ jobs:
         container-name: isaac-lab-arm-ci-ovphysx-${{ github.run_id }}-${{ github.run_attempt }}
         image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64
         extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}"
-        test-k-expr: ovphysx
-        ci-marker: arm_ci
+        job: arm-ci-ovphysx
         volume-mount-source: ${{ github.workspace }}