diff --git a/.github/actions/multi-gpu/mgpu_shard_select.py b/.github/actions/multi-gpu/mgpu_shard_select.py index 38d1ac18827c..47dc22d51ea2 100644 --- a/.github/actions/multi-gpu/mgpu_shard_select.py +++ b/.github/actions/multi-gpu/mgpu_shard_select.py @@ -5,7 +5,7 @@ """pytest plugin: select which tests a non-default GPU shard runs. -Loaded only by the multi-GPU lane: ``tools/conftest.py`` injects it via +Loaded only by the multi-GPU lane: ``tools/run_tests.py`` injects it via ``-p mgpu_shard_select`` into each per-file pytest subprocess (and prepends this directory to ``PYTHONPATH`` so it is importable). It lives next to the lane scripts rather than as a repo-root ``conftest.py`` so it only affects the lane. @@ -86,7 +86,7 @@ def pytest_collection_modifyitems(config, items): def pytest_sessionfinish(session, exitstatus): # A file whose tests are all out of scope deselects to zero, so pytest exits # NO_TESTS_COLLECTED (5). The lane orchestrator treats any non-zero per-file - # exit as a failure (tools/conftest.py), so report "nothing in scope for this + # exit as a failure (tools/run_tests.py), so report "nothing in scope for this # file" as success rather than a false failure. if _shard_mask() is not None and exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: session.exitstatus = pytest.ExitCode.OK diff --git a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh index 9d3b5dec5d08..ff7035883e8f 100755 --- a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh +++ b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh @@ -103,10 +103,7 @@ for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do # C-style loop; start at 1 to sk # full $shard_log under a collapsible ``::group::shard cuda:N log``. # (tee = full output to the log file; stdbuf -oL = flush per line so the # filtered grep/sed stream appears live, not in delayed chunks.) - ./isaaclab.sh -p -m pytest \ - --ignore=tools/conftest.py \ - --ignore=source/isaaclab/test/install_ci \ - tools -v 2>&1 \ + ./isaaclab.sh -p tools/run_tests.py --all 2>&1 \ | tee "$shard_log" \ | stdbuf -oL grep -aE \ 'šŸš€|^source/.*::.* (PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)|^(Total|Passing|Failing|Crashed|Startup Hang|Timeout|Total Wall Time|Total Test Time|Passing Percentage):|^~~~~|^=+ |^E +|^ +File |Traceback|^FAILED|^ERROR ' \ diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index d2782786bafb..0730afa24273 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -27,44 +27,21 @@ inputs: description: 'ECR cache tag' default: 'cache-base' required: false - filter-pattern: - description: >- - Pattern to filter test files (e.g., isaaclab_tasks); test files whose path - contains this pattern are included. Only one pattern can be used at a time, - no comma-separated substrings allowed. Also supports the legacy "not " - form for exclude-only jobs. - default: '' - required: false - exclude-pattern: - description: >- - Comma-separated substrings; test files whose path contains any entry are - skipped. Combines with filter-pattern (include + exclude). - default: '' - required: false test-k-expr: description: >- Global pytest -k expression applied inside every per-file pytest run - spawned by tools/conftest.py (combined with device-split selectors). + spawned by tools/run_tests.py (combined with device-split selectors). default: '' required: false - shard-index: - description: 'Zero-based shard index' - default: '' - required: false - shard-count: - description: 'Total number of shards' + job: + description: >- + Name of a job in tools/test_plan.toml. The plan decides which files the job covers; + tools/run_tests.py runs them. Leave empty only when test-path names a file to run + directly through pytest. default: '' required: false - curobo-only: - description: 'Run only cuRobo and SkillGen tests' - default: 'false' - required: false - quarantined-only: - description: 'Run only quarantined tests' - default: 'false' - required: false - include-files: - description: 'Comma-separated list of specific test files to include' + shard: + description: 'Which shard of a sharded job to run, 0-based. Empty for an unsharded job.' default: '' required: false test-node-ids-file: @@ -301,14 +278,9 @@ runs: container-name: "${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}" image-tag: ${{ inputs.image-tag }} pytest-options: ${{ inputs.pytest-options }} - filter-pattern: ${{ inputs.filter-pattern }} - exclude-pattern: ${{ inputs.exclude-pattern }} + job: ${{ inputs.job }} + shard: ${{ inputs.shard }} test-k-expr: ${{ inputs.test-k-expr }} - shard-index: ${{ inputs.shard-index }} - shard-count: ${{ inputs.shard-count }} - curobo-only: ${{ inputs.curobo-only }} - quarantined-only: ${{ inputs.quarantined-only }} - include-files: ${{ inputs.include-files }} test-node-ids-file: ${{ inputs.test-node-ids-file }} test-node-ids-key: ${{ inputs.test-node-ids-key }} volume-mount-source: ${{ github.workspace }} diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index abac275ddf78..aed49abc6832 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -27,38 +27,23 @@ inputs: description: 'Additional pytest options (e.g., -k filter)' default: '' required: false - filter-pattern: - description: >- - Pattern to filter test files (e.g., isaaclab_tasks); test files whose path - contains this pattern are included. Only one pattern can be used at a time, - no comma-separated substrings allowed. Also supports the legacy "not " - form for exclude-only jobs. - default: '' - required: false - exclude-pattern: - description: >- - Comma-separated substrings; test files whose path contains any entry are - excluded. Combines with filter-pattern (include + exclude). - default: '' - required: false test-k-expr: description: >- Global pytest -k expression applied inside every per-file pytest run - spawned by tools/conftest.py (combined with device-split selectors). + spawned by tools/run_tests.py (combined with device-split selectors). Unlike pytest-options, this reaches the individual test processes, so it can deselect parametrized cases (e.g. "not ovphysx"). default: '' required: false - curobo-only: - description: 'Run only cuRobo and SkillGen tests (requires the cuRobo Docker image)' - default: 'false' - required: false - quarantined-only: - description: 'Run only tests listed in QUARANTINED_TESTS (skipped in normal jobs)' - default: 'false' + job: + description: >- + Name of a job in tools/test_plan.toml. The plan decides which files the job covers; + tools/run_tests.py runs them. Leave empty only when test-path names a file to run + directly through pytest. + default: '' required: false - include-files: - description: 'Comma-separated list of specific test file paths to include (e.g., source/pkg/test/test_a.py,source/pkg/test/test_b.py)' + shard: + description: 'Which shard of a sharded job to run, 0-based. Empty for an unsharded job.' default: '' required: false test-node-ids-file: @@ -69,14 +54,6 @@ inputs: description: 'Top-level key in test-node-ids-file containing the node IDs for this job' default: '' required: false - shard-index: - description: 'Zero-based index of this shard (used with shard-count to split tests across parallel jobs)' - default: '' - required: false - shard-count: - description: 'Total number of shards (used with shard-index to split tests across parallel jobs)' - default: '' - required: false volume-mount-source: description: 'Host path to bind-mount at /workspace/isaaclab (for deps-cache-hit mode)' default: '' @@ -100,10 +77,6 @@ inputs: description: 'Host Warp kernel cache directory bind-mounted into the container as WARP_CACHE_PATH' default: '' required: false - ci-marker: - description: 'CI_MARKER value forwarded to the container (read by tools/conftest.py to select test files by pytest marker)' - default: '' - required: false standalone-script-scope: description: 'Enable standalone script smoke tests for this scripts/ subdirectory' default: '' @@ -132,9 +105,8 @@ runs: # the run_tests positional arguments if substituted textually. PYTEST_OPTIONS: ${{ inputs.pytest-options }} TEST_K_EXPR_INPUT: ${{ inputs.test-k-expr }} - CI_MARKER_INPUT: ${{ inputs.ci-marker }} run: | - bash .github/actions/run-tests/run_tests.sh "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "$TEST_K_EXPR_INPUT" "$CI_MARKER_INPUT" "${{ inputs.standalone-script-scope }}" "${{ inputs.standalone-script-visualizer }}" "${{ inputs.standalone-script-runtime-group }}" "${{ inputs.warp-cache-host-dir }}" "${{ inputs.extra-uv-packages }}" + bash .github/actions/run-tests/run_tests.sh "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.job }}" "${{ inputs.shard }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "$TEST_K_EXPR_INPUT" "${{ inputs.standalone-script-scope }}" "${{ inputs.standalone-script-visualizer }}" "${{ inputs.standalone-script-runtime-group }}" "${{ inputs.warp-cache-host-dir }}" "${{ inputs.extra-uv-packages }}" - name: Kill container on cancellation if: cancelled() shell: bash diff --git a/.github/actions/run-tests/run_tests.sh b/.github/actions/run-tests/run_tests.sh index 3779ab10c562..38d62ab53b58 100755 --- a/.github/actions/run-tests/run_tests.sh +++ b/.github/actions/run-tests/run_tests.sh @@ -17,26 +17,20 @@ run_tests() { local image_tag="$4" local reports_dir="$5" local pytest_options="$6" - local filter_pattern="$7" - local exclude_pattern="$8" - local curobo_only="$9" - local include_files="${10}" - local quarantined_only="${11}" - local shard_index="${12}" - local shard_count="${13}" - local volume_mount_source="${14}" - local extra_pip_packages="${15}" - local test_node_ids_file="${16}" - local test_node_ids_key="${17}" - local wheelhouse_host_dir="${18}" - local wheelhouse_packages="${19}" - local test_k_expr="${20}" - local ci_marker="${21}" - local standalone_script_scope="${22}" - local standalone_script_visualizer="${23}" - local standalone_script_runtime_group="${24}" - local warp_cache_host_dir="${25}" - local extra_uv_packages="${26}" + local job="$7" + local shard="$8" + local volume_mount_source="$9" + local extra_pip_packages="${10}" + local test_node_ids_file="${11}" + local test_node_ids_key="${12}" + local wheelhouse_host_dir="${13}" + local wheelhouse_packages="${14}" + local test_k_expr="${15}" + local standalone_script_scope="${16}" + local standalone_script_visualizer="${17}" + local standalone_script_runtime_group="${18}" + local warp_cache_host_dir="${19}" + local extra_uv_packages="${20}" local logs_pid="" local wait_pid="" local docker_wait_file="/tmp/.docker_exit_${container_name}" @@ -72,17 +66,8 @@ run_tests() { if [ -n "$wheelhouse_packages" ]; then echo "With wheelhouse packages: $wheelhouse_packages" fi - if [ -n "$filter_pattern" ]; then - echo "With filter pattern: $filter_pattern" - fi - if [ -n "$exclude_pattern" ]; then - echo "With exclude pattern: $exclude_pattern" - fi - if [ "$curobo_only" = "true" ]; then - echo "cuRobo-only mode enabled: running only cuRobo and SkillGen tests" - fi - if [ -n "$include_files" ]; then - echo "Include files: $include_files" + if [ -n "$job" ]; then + echo "Running test plan job: $job" fi if [ -n "$test_node_ids_file" ]; then echo "Test node IDs file: $test_node_ids_file" @@ -90,8 +75,11 @@ run_tests() { if [ -n "$test_node_ids_key" ]; then echo "Test node IDs key: $test_node_ids_key" fi - if [ -n "$shard_index" ] && [ -n "$shard_count" ]; then - echo "Shard: $shard_index of $shard_count" + # The runner takes the shard as a flag; an unsharded job passes nothing. + local shard_arg="" + if [ -n "$shard" ]; then + shard_arg="--shard $shard" + echo "Shard: $shard" fi if [ -n "$test_node_ids_file" ] || [ -n "$test_node_ids_key" ]; then @@ -121,24 +109,6 @@ run_tests() { -e GITHUB_ACTIONS=${GITHUB_ACTIONS:-} \ -e TEST_RESULT_FILE=$result_file" - if [ "$curobo_only" = "true" ]; then - docker_env_vars="$docker_env_vars -e TEST_CUROBO_ONLY=true" - echo "Setting TEST_CUROBO_ONLY=true" - fi - - if [ "$quarantined_only" = "true" ]; then - docker_env_vars="$docker_env_vars -e TEST_QUARANTINED_ONLY=true" - echo "Setting TEST_QUARANTINED_ONLY=true" - fi - - if [ -n "$include_files" ]; then - # Strip spaces so the value is safe to embed in an unquoted docker_env_vars string. - # conftest.py splits on commas and strips whitespace, so compact form works fine. - include_files_compact="${include_files// /}" - docker_env_vars="$docker_env_vars -e TEST_INCLUDE_FILES=$include_files_compact" - echo "Setting TEST_INCLUDE_FILES=$include_files_compact" - fi - if [ -n "${TEST_NODE_IDS:-}" ]; then docker_env_vars="$docker_env_vars -e TEST_NODE_IDS" echo "Setting TEST_NODE_IDS" @@ -149,35 +119,6 @@ run_tests() { echo "Setting TEST_NODE_IDS_FILE=$TEST_NODE_IDS_FILE TEST_NODE_IDS_KEY=$TEST_NODE_IDS_KEY" fi - if [ -n "$shard_index" ] && [ -n "$shard_count" ]; then - docker_env_vars="$docker_env_vars -e TEST_SHARD_INDEX=$shard_index -e TEST_SHARD_COUNT=$shard_count" - echo "Setting TEST_SHARD_INDEX=$shard_index TEST_SHARD_COUNT=$shard_count" - fi - - if [ -n "$filter_pattern" ]; then - if [[ "$filter_pattern" == "not "* ]]; then - # Handle "not " case - note the trailing space to avoid - # matching words that happen to start with "not". - filter_exclude_pattern="${filter_pattern#not }" - if [ -n "$exclude_pattern" ]; then - exclude_pattern="${exclude_pattern},${filter_exclude_pattern}" - else - exclude_pattern="$filter_exclude_pattern" - fi - else - # Handle positive pattern case - docker_env_vars="$docker_env_vars -e TEST_FILTER_PATTERN=$filter_pattern" - echo "Setting include pattern: $filter_pattern" - fi - else - echo "No filter pattern provided" - fi - - if [ -n "$exclude_pattern" ]; then - docker_env_vars="$docker_env_vars -e TEST_EXCLUDE_PATTERN=$exclude_pattern" - echo "Setting exclude pattern: $exclude_pattern" - fi - if [ -n "$extra_pip_packages" ]; then export TEST_EXTRA_PIP_PACKAGES="$extra_pip_packages" docker_env_vars="$docker_env_vars -e TEST_EXTRA_PIP_PACKAGES" @@ -193,10 +134,6 @@ run_tests() { echo "Setting per-file pytest -k expression: $test_k_expr" fi - if [ -n "$ci_marker" ]; then - docker_env_vars="$docker_env_vars -e CI_MARKER=$ci_marker" - echo "Setting CI_MARKER=$ci_marker" - fi if [ -n "$standalone_script_scope" ]; then docker_env_vars="$docker_env_vars \ @@ -373,8 +310,13 @@ run_tests() { bash /with-python-package-retries.sh \"\${uv_executable}\" pip install --python \"\${isaac_python}\" --target \"\${isaac_uv_overlay}\" --no-deps \${TEST_EXTRA_UV_PACKAGES} export PYTHONPATH=\"\${isaac_uv_overlay}\${PYTHONPATH:+:\${PYTHONPATH}}\" fi - echo 'Starting pytest with path: $test_path' - ./isaaclab.sh -p -m pytest --ignore=tools/conftest.py $test_path $pytest_options -v --junitxml=tests/$result_file + if [ -n '$job' ]; then + echo 'Running test plan job: $job' + ./isaaclab.sh -p tools/run_tests.py --job '$job' $shard_arg + else + echo 'Starting pytest with path: $test_path' + ./isaaclab.sh -p -m pytest $test_path $pytest_options -v --junitxml=tests/$result_file + fi " # Stream container logs in background. diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index 3e992dcef3cb..87482cf53939 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -184,8 +184,7 @@ jobs: container-name: isaac-lab-arm-ci-${{ github.run_id }}-${{ github.run_attempt }} image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}" - test-k-expr: not ovphysx - ci-marker: arm_ci + job: arm-ci volume-mount-source: ${{ github.workspace }} - name: Run arm_ci OVPhysX marker tests @@ -196,8 +195,7 @@ jobs: container-name: isaac-lab-arm-ci-ovphysx-${{ github.run_id }}-${{ github.run_attempt }} image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}" - test-k-expr: ovphysx - ci-marker: arm_ci + job: arm-ci-ovphysx volume-mount-source: ${{ github.workspace }} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2ff42e847a07..210fface7cd5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -27,12 +27,9 @@ # if: false # TEMP: Disabled for debugging # # 2. RUN ONLY SPECIFIC TEST FILES (within a job): -# Add `include-files:` parameter to run-package-tests action -# Example: -# - uses: ./.github/actions/run-package-tests -# with: -# ... -# include-files: "test_rigid_object_collection.py" # Comma-separated +# Edit the job's entry in tools/test_plan.toml -- `files = [...]` limits it to those +# basenames -- then run `python tools/generate_workflows.py` if the job is generated. +# Locally the same selection is `python tools/run_tests.py --job `. # # 3. SKIP CONCURRENCY WAIT (run immediately without waiting for other runs): # Comment out the concurrency block: @@ -217,7 +214,9 @@ jobs: #endregion #region test jobs - test-isaaclab-tasks: + # >>> generated from tools/test_plan.toml -- edit the plan, then run tools/generate_workflows.py + + test-isaaclab-tasks-1: name: isaaclab_tasks [1/3] runs-on: [self-hosted, gpu] timeout-minutes: 180 @@ -236,11 +235,9 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" - exclude-pattern: "test_rendering_,test_video_recording.py" + job: isaaclab-tasks + shard: "0" extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - shard-index: "0" - shard-count: "3" warp-cache: restore container-name: isaac-lab-tasks-1-test @@ -263,11 +260,9 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" - exclude-pattern: "test_rendering_,test_video_recording.py" + job: isaaclab-tasks + shard: "1" extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - shard-index: "1" - shard-count: "3" warp-cache: restore container-name: isaac-lab-tasks-2-test @@ -290,15 +285,13 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" - exclude-pattern: "test_rendering_,test_video_recording.py" + job: isaaclab-tasks + shard: "2" extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - shard-index: "2" - shard-count: "3" warp-cache: restore container-name: isaac-lab-tasks-3-test - test-isaaclab-core: + test-isaaclab-core-1: name: isaaclab (core) [1/3] runs-on: [self-hosted, gpu] timeout-minutes: 180 @@ -316,9 +309,8 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "not isaaclab_" - shard-index: "0" - shard-count: "3" + job: isaaclab-core + shard: "0" warp-cache: restore container-name: isaac-lab-core-1-test @@ -340,9 +332,8 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "not isaaclab_" - shard-index: "1" - shard-count: "3" + job: isaaclab-core + shard: "1" warp-cache: restore container-name: isaac-lab-core-2-test @@ -364,18 +355,13 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "not isaaclab_" - shard-index: "2" - shard-count: "3" + job: isaaclab-core + shard: "2" warp-cache: restore container-name: isaac-lab-core-3-test - # Kit and non-Kit are written out rather than expressed as a matrix: these job - # names are required status checks, and a skipped matrix job publishes its - # check run with the matrix expression unexpanded, so the required context - # never appears and branch protection blocks the pull request. - test-standalone-demos-kit: - name: standalone demos (headless, Kit) + test-isaaclab-rl: + name: isaaclab_rl runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -392,18 +378,12 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - # deformables.py tetrahedralizes its volume meshes at startup - extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - include-files: "test_standalone_scripts.py" - standalone-script-scope: "demos" - standalone-script-visualizer: "none" - standalone-script-runtime-group: kit - result-file: "test-standalone-demos-kit-report.xml" - container-name: isaac-lab-standalone-demos-kit-test - omni-github-test-type: standalone-demo + job: isaaclab-rl + extra-pip-packages: "leapp" + container-name: isaac-lab-rl-test - test-standalone-demos-non-kit: - name: standalone demos (headless, non-Kit) + test-isaaclab-mimic: + name: isaaclab_mimic runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -415,28 +395,16 @@ jobs: with: fetch-depth: 1 lfs: true - - name: Resolve OVPhysX runtime pin from pyproject - uses: ./.github/actions/resolve-ov-pins - id: ov_pins - uses: ./.github/actions/run-package-tests with: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - # deformables.py tetrahedralizes its volume meshes at startup - extra-pip-packages: >- - pytetwild[all]>=0.3.0,<0.4 - ${{ steps.ov_pins.outputs.ovphysx }} - include-files: "test_standalone_scripts.py" - standalone-script-scope: "demos" - standalone-script-visualizer: "none" - standalone-script-runtime-group: non-kit - result-file: "test-standalone-demos-non-kit-report.xml" - container-name: isaac-lab-standalone-demos-non-kit-test - omni-github-test-type: standalone-demo + job: isaaclab-mimic + container-name: isaac-lab-mimic-test - test-isaaclab-rl: - name: isaaclab_rl + test-isaaclab-contrib: + name: isaaclab_contrib runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -453,12 +421,12 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_rl" - extra-pip-packages: "leapp" - container-name: isaac-lab-rl-test + job: isaaclab-contrib + extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" + container-name: isaac-lab-contrib-test - test-isaaclab-mimic: - name: isaaclab_mimic + test-isaaclab-teleop: + name: isaaclab_teleop runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -475,11 +443,11 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_mimic" - container-name: isaac-lab-mimic-test + job: isaaclab-teleop + container-name: isaac-lab-teleop-test - test-isaaclab-contrib: - name: isaaclab_contrib + test-isaaclab-visualizers: + name: isaaclab_visualizers runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -496,12 +464,11 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_contrib" - extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - container-name: isaac-lab-contrib-test + job: isaaclab-visualizers + container-name: isaac-lab-visualizers-test - test-isaaclab-teleop: - name: isaaclab_teleop + test-isaaclab-assets: + name: isaaclab_assets runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -518,11 +485,11 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_teleop" - container-name: isaac-lab-teleop-test + job: isaaclab-assets + container-name: isaac-lab-assets-test - test-isaaclab-visualizers: - name: isaaclab_visualizers + test-isaaclab-experimental: + name: isaaclab_experimental runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -539,11 +506,11 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_visualizers" - container-name: isaac-lab-visualizers-test + job: isaaclab-experimental + container-name: isaac-lab-experimental-test - test-isaaclab-assets: - name: isaaclab_assets + test-isaaclab-newton: + name: isaaclab_newton runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -560,11 +527,12 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_assets" - container-name: isaac-lab-assets-test + job: isaaclab-newton + warp-cache: restore + container-name: isaac-lab-newton-test - test-isaaclab-experimental: - name: isaaclab_experimental + test-isaaclab-physx: + name: isaaclab_physx runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -581,11 +549,18 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_experimental" - container-name: isaac-lab-experimental-test + job: isaaclab-physx + extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" + container-name: isaac-lab-physx-test - test-isaaclab-newton: - name: isaaclab_newton + # <<< end generated jobs + + # Kit and non-Kit are written out rather than expressed as a matrix: these job + # names are required status checks, and a skipped matrix job publishes its + # check run with the matrix expression unexpanded, so the required context + # never appears and branch protection blocks the pull request. + test-standalone-demos-kit: + name: standalone demos (headless, Kit) runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -602,12 +577,18 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_newton" - warp-cache: restore - container-name: isaac-lab-newton-test + job: standalone-demos-kit + # deformables.py tetrahedralizes its volume meshes at startup + extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" + standalone-script-scope: "demos" + standalone-script-visualizer: "none" + standalone-script-runtime-group: kit + result-file: "test-standalone-demos-kit-report.xml" + container-name: isaac-lab-standalone-demos-kit-test + omni-github-test-type: standalone-demo - test-isaaclab-physx: - name: isaaclab_physx + test-standalone-demos-non-kit: + name: standalone demos (headless, non-Kit) runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -619,14 +600,25 @@ jobs: with: fetch-depth: 1 lfs: true + - name: Resolve OVPhysX runtime pin from pyproject + uses: ./.github/actions/resolve-ov-pins + id: ov_pins - uses: ./.github/actions/run-package-tests with: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_physx" - extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - container-name: isaac-lab-physx-test + job: standalone-demos-non-kit + # deformables.py tetrahedralizes its volume meshes at startup + extra-pip-packages: >- + pytetwild[all]>=0.3.0,<0.4 + ${{ steps.ov_pins.outputs.ovphysx }} + standalone-script-scope: "demos" + standalone-script-visualizer: "none" + standalone-script-runtime-group: non-kit + result-file: "test-standalone-demos-non-kit-report.xml" + container-name: isaac-lab-standalone-demos-non-kit-test + omni-github-test-type: standalone-demo test-isaaclab-ov: name: isaaclab_ov @@ -651,7 +643,7 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_ov" + job: isaaclab-ov # Volume deformable tests tetrahedralize their meshes at startup. extra-pip-packages: pytetwild[all]>=0.3.0,<0.4 ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && steps.ov_pins.outputs.ovrtx || format('{0} {1}', steps.ov_pins.outputs.ovrtx, steps.ov_pins.outputs.ovphysx) }} wheelhouse-image: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && needs.config.outputs.ovphysx_wheelhouse_image || '' }} @@ -703,9 +695,9 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }}-curobo isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + job: curobo dockerfile-path: docker/Dockerfile.curobo cache-tag: cache-curobo - include-files: "test_curobo_planner_franka.py,test_curobo_planner_cube_stack.py,test_pink_ik.py" container-name: isaac-lab-curobo-test # Folded from the former standalone verify-curobo-non-root job: reuses @@ -753,9 +745,9 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }}-curobo isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + job: contrib-environments dockerfile-path: docker/Dockerfile.curobo cache-tag: cache-curobo - include-files: "test_generate_dataset_skillgen.py,test_contrib_environments.py" container-name: isaac-lab-contrib-environments-test test-record-video: @@ -777,8 +769,7 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" - include-files: "test_video_recording.py" + job: record-video extra-uv-packages: "moviepy>=1.0.3,<2.0.0.dev0 decorator<5" container-name: isaac-lab-record-video-test omni-github-test-type: video-e2e @@ -800,17 +791,8 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" + job: rendering-correctness extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - include-files: >- - test_rendering_cartpole.py, - test_rendering_lift_kuka_hetero.py, - test_rendering_lift_kuka_homo.py, - test_rendering_franka_cloth.py, - test_rendering_franka_soft.py, - test_rendering_franka_cable.py, - test_rendering_registered_tasks.py, - test_rendering_shadow_hand.py test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }} test-node-ids-key: ${{ github.event_name == 'push' && 'rendering-correctness' || '' }} warp-cache: restore @@ -844,21 +826,12 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" + job: rendering-correctness-kitless-legacy extra-pip-packages: >- pytetwild[all]>=0.3.0,<0.4 ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && steps.ov_pins.outputs.ovrtx || format('{0} {1}', steps.ov_pins.outputs.ovrtx, steps.ov_pins.outputs.ovphysx) }} wheelhouse-image: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && needs.config.outputs.ovphysx_wheelhouse_image || '' }} wheelhouse-packages: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && 'ovphysx' || '' }} - include-files: >- - test_rendering_cartpole_kitless.py, - test_rendering_lift_kuka_hetero_kitless.py, - test_rendering_lift_kuka_homo_kitless.py, - test_rendering_franka_cloth_kitless.py, - test_rendering_franka_soft_kitless.py, - test_rendering_franka_cable_kitless.py, - test_rendering_shadow_hand_kitless.py - test-k-expr: legacy test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }} test-node-ids-key: ${{ github.event_name == 'push' && 'rendering-correctness-kitless-legacy' || '' }} warp-cache: restore @@ -890,21 +863,12 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" + job: rendering-correctness-kitless-ovstage extra-pip-packages: >- pytetwild[all]>=0.3.0,<0.4 ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && steps.ov_pins.outputs.ovrtx || format('{0} {1}', steps.ov_pins.outputs.ovrtx, steps.ov_pins.outputs.ovphysx) }} wheelhouse-image: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && needs.config.outputs.ovphysx_wheelhouse_image || '' }} wheelhouse-packages: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && 'ovphysx' || '' }} - include-files: >- - test_rendering_cartpole_kitless.py, - test_rendering_lift_kuka_hetero_kitless.py, - test_rendering_lift_kuka_homo_kitless.py, - test_rendering_franka_cloth_kitless.py, - test_rendering_franka_soft_kitless.py, - test_rendering_franka_cable_kitless.py, - test_rendering_shadow_hand_kitless.py - test-k-expr: ovstage test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }} test-node-ids-key: ${{ github.event_name == 'push' && 'rendering-correctness-kitless-ovstage' || '' }} container-name: "isaac-lab-rendering-correctness-kitless-ovstage-test" @@ -948,17 +912,13 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" + job: warp-cache-warm # Warm every supported rigid core environment on Newton. MJWarp # specializes kernels per model config, so a small representative task # set leaves most of the cache to be compiled again by each PR test # shard. The multi-agent suite supplies Handover. - include-files: >- - test_environments_newton.py, - test_multi_agent_environments.py # Deformable tasks need optional dependencies that are not installed in # this image. - test-k-expr: "test_environments and not (Soft or Cloth or Cable)" warp-cache: ${{ env.PUBLISHES_WARP_CACHE == 'true' && 'save' || 'restore' }} container-name: isaac-lab-warp-cache-warm omni-github-test-type: warp-cache-warm diff --git a/.github/workflows/daily-compatibility.yml b/.github/workflows/daily-compatibility.yml index eaf05e4d3186..5b24d754f61f 100644 --- a/.github/workflows/daily-compatibility.yml +++ b/.github/workflows/daily-compatibility.yml @@ -123,6 +123,7 @@ jobs: image-tag: ${{ env.DOCKER_IMAGE_TAG }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ matrix.isaacsim_version }} + job: isaaclab-tasks-compat cache-from: type=gha cache-to: type=gha,mode=max @@ -134,7 +135,6 @@ jobs: container-name: "isaac-lab-tasks-compat-test-$$" image-tag: ${{ env.DOCKER_IMAGE_TAG }} pytest-options: "" - filter-pattern: "isaaclab_tasks" extra-pip-packages: ${{ format('{0} {1}', steps.ov_pins.outputs.ovphysx, steps.ov_pins.outputs.ovrtx) }} - name: Copy All Test Results from IsaacLab Tasks Container @@ -183,6 +183,7 @@ jobs: image-tag: ${{ env.DOCKER_IMAGE_TAG }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ matrix.isaacsim_version }} + job: general-compat cache-from: type=gha cache-to: type=gha,mode=max @@ -194,7 +195,6 @@ jobs: container-name: "isaac-lab-general-compat-test-$$" image-tag: ${{ env.DOCKER_IMAGE_TAG }} pytest-options: "" - filter-pattern: "not isaaclab_tasks" - name: Copy All Test Results from General Tests Container run: | diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index 63c03a73779a..a4787a54116c 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -114,7 +114,7 @@ jobs: # # Within a discovered file, tests that are NOT parametrized over the # ``device`` argument are deselected at collection time by the - # ``mgpu_shard_select`` plugin (injected per shard by ``tools/conftest.py``): + # ``mgpu_shard_select`` plugin (injected per shard by ``tools/run_tests.py``): # single-GPU CI already covers them on ``cuda:0`` and re-running on every # non-default shard adds wall-time without surfacing any new failure mode. id: discover diff --git a/.github/workflows/tools-tests.yml b/.github/workflows/tools-tests.yml index 70977f92dc39..f28c076ff277 100644 --- a/.github/workflows/tools-tests.yml +++ b/.github/workflows/tools-tests.yml @@ -60,7 +60,6 @@ jobs: # Files are listed explicitly rather than collected from tools/: test_settings.py is a # configuration module, not a test, and tools/test/ needs the full Isaac Lab install. # - # --noconftest keeps tools/conftest.py out of the session. That file is the CI test - # orchestrator - its pytest_sessionstart scans source/ and scripts/ and runs the whole - # suite, so loading it here would ignore the files named below. - run: python3 -m pytest tools/test_crash_journal.py tools/test_device_split.py -v --noconftest + # The runner is tools/run_tests.py, an ordinary script, so an ordinary pytest run + # over these files no longer has to defend itself against a conftest that hijacks it. + run: python3 -m pytest tools/test_crash_journal.py tools/test_device_split.py -v diff --git a/conftest.py b/conftest.py index d5bf93c8ca11..75c243507a5f 100644 --- a/conftest.py +++ b/conftest.py @@ -11,7 +11,7 @@ Also maintains a crash journal (see :data:`JOURNAL_ENV_VAR`). pytest writes its JUnit XML once, at the end of the session, so a run killed before then - a Kit shutdown crash, an OOM kill, a hard timeout - loses every verdict it had already printed. The journal records collection, -per-test start/finish, and per-test outcomes as they happen, letting ``tools/conftest.py`` +per-test start/finish, and per-test outcomes as they happen, letting ``tools/run_tests.py`` rebuild a real report instead of a single synthetic ``test_execution`` entry. Level markers (``unit`` / ``integration`` / ``benchmark``) are applied per file via a module-level ``pytestmark`` @@ -19,12 +19,14 @@ e.g. ``pytest -m unit source/isaaclab/test`` or ``pytest -m "not unit" source/isaaclab/test``. Also loads ``tools/ovrtx_log.py``, which replays the OVRTX renderer log per test, so every suite that -builds a renderer reports what it logged the same way, and ``tools/hang_dump.py``, which lets the CI -runner ask this process for a stack dump before it kills it for hanging. +builds a renderer reports what it logged the same way; ``tools/hang_dump.py``, which lets the CI +runner ask this process for a stack dump before it kills it for hanging; and ``isaaclab.test.kit``, +which boots one Kit app per process for the test files whose ``pytestmark`` declares they need one. """ from __future__ import annotations +import importlib.util import json import os @@ -38,6 +40,19 @@ pytest_plugins = ["tools.ovrtx_log", "tools.hang_dump"] +# The Kit launch plugin is the only entry here that needs Isaac Lab itself. Lanes that test the +# repository's own tooling install pytest and nothing else, and none of their tests declare a +# launch marker, so a missing package there is not an error. A lane that does need Kit has Isaac +# Lab installed by definition -- its tests import it -- so this cannot quietly skip the launch. +# The exact module is probed rather than the top-level package: an Isaac Lab old enough to +# predate the plugin would otherwise pass the check and fail on import. +try: + _has_kit_plugin = importlib.util.find_spec("isaaclab.test.kit") is not None +except (ImportError, ValueError): + _has_kit_plugin = False +if _has_kit_plugin: + pytest_plugins.append("isaaclab.test.kit") + JOURNAL_ENV_VAR = "ISAACLAB_TEST_JOURNAL" """Environment variable naming the crash-journal file. Unset (the default) disables journaling.""" @@ -96,7 +111,7 @@ def pytest_collection_finish(session): Journaling from :func:`pytest_collection_modifyitems` would record tests that are about to be dropped: pytest's own mark plugin applies ``-k`` / ``-m`` deselection from a ``trylast`` hook, - which runs after this file's. Since ``tools/conftest.py`` splits a run into passes selected by + which runs after this file's. Since ``tools/run_tests.py`` splits a run into passes selected by marker and device, a rebuilt crash report would then emit every other pass's tests as "not run" skips, inflating the counts and duplicating node IDs whose real verdicts came from the sibling pass. ``session.items`` is post-deselection, so it holds exactly the tests this pass runs. diff --git a/docs/source/refs/contributing.rst b/docs/source/refs/contributing.rst index 29ec5961d098..9907daeb656f 100644 --- a/docs/source/refs/contributing.rst +++ b/docs/source/refs/contributing.rst @@ -764,6 +764,49 @@ Please make sure that you add tests for your changes. isaaclab.bat -p -m pytest source/isaaclab/test/deps/test_torch.py::test_array_slicing +Running what CI runs +^^^^^^^^^^^^^^^^^^^^ + +``isaaclab -t`` runs the whole suite the same way CI does. What each CI lane covers is +described in ``tools/test_plan.toml``, and any lane can be run on its own: + +.. code-block:: bash + + isaaclab -t --list-jobs # the CI lanes, and how many files each covers + isaaclab -t --job isaaclab-core --shard 0 # one lane, exactly as CI runs it + isaaclab -t source/isaaclab/test/sim # an ad-hoc directory + isaaclab -t --job isaaclab-rl --list-files # what a lane would run, without running it + +The plan is the single source of truth: ``tools/generate_workflows.py`` renders the uniform +workflow jobs from it, and a test fails if the checked-in YAML has drifted. To change what a +lane covers, edit the plan rather than the workflow file. + + +Tests that need Isaac Sim +^^^^^^^^^^^^^^^^^^^^^^^^^ + +A test file that imports ``omni``, ``carb``, or ``isaacsim`` at module scope needs Isaac Sim +running by the time pytest imports it. Declare that with a marker rather than constructing +:class:`~isaaclab.app.AppLauncher` yourself: + +.. code-block:: python + + import pytest + + import omni.timeline + + pytestmark = [pytest.mark.kit, pytest.mark.integration] + +Use ``pytest.mark.kit_cameras`` instead when the test needs the renderer, which starts the app +with cameras enabled. The two are alternatives: cameras cannot be enabled after startup, so a +file of each kind cannot share a process. + +The app is started once per pytest process and shared by every marked file in it, so a run +covering many such files pays Kit startup once rather than once per file. Add +``pytest.mark.solo`` to keep a file out of that sharing when it depends on having a process +to itself. Tests that need the app object itself request the ``kit_app`` fixture. + + Tools ----- diff --git a/pyproject.toml b/pyproject.toml index e35ada57136c..b8e5507685d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -341,6 +341,7 @@ ignore-words-list = "aheared,collet,followng,haa,publically,rouines,slq,collapsa markers = [ "isaacsim_ci: mark test to run in isaacsim ci", "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5)", + "solo: run this file in a process of its own; it is never grouped with other files", "windows_ci: mark test to run on Windows platforms in CI", "arm_ci: mark test to run on ARM platforms in CI (e.g. NVIDIA DGX Spark)", "unit: test exercises isolated logic and does not launch the simulator", @@ -349,6 +350,8 @@ markers = [ "rendering: test exercises the rendering / camera / visualizer pipeline", "smoke: tests for core installation, task, and RL functionality", "kitless: test must pass inside the Kit-less container, which has no Isaac Sim runtime", + "kit: test file needs a headless Kit app; the runner boots one before importing the module and shares it with the other `kit` files in the process", + "kit_cameras: like `kit`, but the app is booted with cameras enabled; the two configurations never share a process", ] # Add pypi.nvidia.com so that `uv pip install isaaclab[isaacsim]` works without --extra-index-url. diff --git a/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst b/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst new file mode 100644 index 000000000000..13fc0e4b413a --- /dev/null +++ b/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst @@ -0,0 +1,32 @@ +Added +^^^^^ + +* Added the ``kit``, ``kit_cameras``, and ``solo`` pytest markers, and the + :mod:`isaaclab.test.kit` plugin that acts on them. A test file that needs Isaac Sim now + declares it in its module-level ``pytestmark``; the plugin reads that declaration out of the + file's source and boots Kit before pytest imports the module, so files sharing a launch + configuration share one app instead of each starting their own. Test files no longer + construct :class:`~isaaclab.app.AppLauncher` at module scope, and tests that need the app + object request the new ``kit_app`` fixture. +* Added ``test_kit_marker_contract.py`` and ``test_kit_plugin.py``, which keep a file's markers + from drifting from what it does at module scope, and check that the app is started before the + module that needs it is imported. + +Changed +^^^^^^^ + +* Changed ``isaaclab --test`` to drive the new ``tools/run_tests.py``. With no arguments it + runs the whole suite; ``--job `` runs a single CI lane locally, ``--list-jobs`` lists + them, and directories run an ad-hoc selection. It previously ran ``pytest tools``, which went + through the CI orchestrator and silently dropped any pytest arguments passed after it. +* Changed the runner to group same-marker test files into a single pytest invocation rather + than giving every file its own process, so Kit startup is paid once per group. Only files + carrying the new markers are grouped; every other file keeps a process of its own, and a file + a dead group never reached is re-run individually. Set ``ISAACLAB_TEST_BATCH_KIT=0`` to turn + the grouping off. + +Fixed +^^^^^ + +* Fixed ``test_operational_space.py`` assigning ``pytestmark`` twice, which silently dropped + its ``arm_ci`` marker and kept the file out of the ARM CI lane. diff --git a/source/isaaclab/isaaclab/cli/__init__.py b/source/isaaclab/isaaclab/cli/__init__.py index 3707deec9f56..b5daa1f41319 100644 --- a/source/isaaclab/isaaclab/cli/__init__.py +++ b/source/isaaclab/isaaclab/cli/__init__.py @@ -235,7 +235,10 @@ def cli() -> None: "-t", "--test", nargs=argparse.REMAINDER, - help="Run all python pytest tests.", + help=( + "Run the tests. No arguments runs the whole suite; '--job ' runs one CI lane," + " '--list-jobs' lists them, and paths run an ad-hoc selection." + ), ) parser.add_argument( "-o", diff --git a/source/isaaclab/isaaclab/cli/commands/misc.py b/source/isaaclab/isaaclab/cli/commands/misc.py index 44e684413c14..fb233586542e 100644 --- a/source/isaaclab/isaaclab/cli/commands/misc.py +++ b/source/isaaclab/isaaclab/cli/commands/misc.py @@ -56,12 +56,19 @@ def command_new(new_args: list[str]) -> None: def command_test(test_args: list[str]) -> None: - """Run pytest for Isaac Lab tests (-t). + """Run Isaac Lab's tests (-t). + + With no arguments this runs the whole suite, the same way CI does. Pass ``--job `` to + run one CI lane, ``--list-jobs`` to see them, or one or more directories to run an ad-hoc + selection. Every argument goes to ``tools/run_tests.py``; see ``isaaclab -t --help``. Args: - test_args: Additional pytest arguments. + test_args: Arguments for ``tools/run_tests.py``. """ - run_python_command("-m", ["pytest", str(ISAACLAB_ROOT / "tools")] + test_args) + runner = ISAACLAB_ROOT / "tools" / "run_tests.py" + # Bare `isaaclab -t` used to mean "run everything"; keep that, since the runner itself + # refuses an empty selection rather than guessing. + run_python_command(runner, test_args or ["--all"]) def command_vscode_settings() -> None: diff --git a/source/isaaclab/isaaclab/test/kit.py b/source/isaaclab/isaaclab/test/kit.py new file mode 100644 index 000000000000..ae666b319806 --- /dev/null +++ b/source/isaaclab/isaaclab/test/kit.py @@ -0,0 +1,254 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Marker-driven Kit startup for the Isaac Lab test suite. + +A test module that needs Isaac Sim declares it with a marker and nothing else:: + + import pytest + + from pxr import Usd + + import isaaclab.sim as sim_utils + + pytestmark = [pytest.mark.kit, pytest.mark.integration] + +This module is a pytest plugin, loaded from the repo-root ``conftest.py``. It reads that +marker out of the file's source *before* pytest imports the module and boots Kit if it is +there. Reading the source rather than the imported module is what makes the marker usable at +all: a Kit-dependent test module imports ``pxr``, ``omni``, ... at module scope, so Kit has to +be running by the time the module is imported -- which is before any fixture, and before the +module's own ``pytestmark`` exists. + +Kit is booted once per process. The first marked module collected pays startup; every later +one is imported into the app that is already running, so a pytest process covering a directory +of such files boots Kit once instead of once per file. + +:data:`KIT_MARKERS` gives the launch configurations. They are alternatives, not nested: a +camera-enabled app is not a superset of a plain one, because cameras cannot be enabled after +startup and some tests assert that offscreen rendering is off. A process that is handed both +kinds of file raises rather than importing one of them into the wrong app. + +Tests that need the app object itself request the :func:`kit_app` fixture. +""" + +from __future__ import annotations + +import ast +import os +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from isaacsim import SimulationApp + +KIT_MARKERS: dict[str, bool] = {"kit": False, "kit_cameras": True} +"""The launch markers, mapped to the ``enable_cameras`` setting each one asks for.""" + +SOLO_MARKER = "solo" +"""Marker that keeps a file in a process of its own, never grouped with other files. + +A separate axis from :data:`KIT_MARKERS`, which say *which* app a file needs rather than who +else may share the process with it, so the two compose: ``[pytest.mark.kit, pytest.mark.solo]`` +still gets an app booted for it, just not one anybody else is using. +""" + +_app: SimulationApp | None = None +"""The app booted for this process, or None before the first marked module is collected.""" + +_cameras: bool = False +"""Whether :data:`_app` was booted with cameras enabled.""" + + +""" +Marker inspection. +""" + + +def module_markers(source: str) -> frozenset[str]: + """Return the marker names a test module declares, without importing it. + + Only module-scope ``pytestmark`` assignments are read, because only those are known + before the module is imported and can therefore influence how Kit is launched. Assignments + inside module-level ``if`` / ``try`` blocks count; per-test ``@pytest.mark`` decorators do + not. + + Args: + source: The module's text. + + Returns: + Every marker name found, or an empty set if the module declares none or does not parse. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return frozenset() + + names: set[str] = set() + for node in _module_scope_nodes(tree): + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "pytestmark" for target in node.targets + ): + names.update(_marker_names(node.value)) + return frozenset(names) + + +def kit_marker(source: str) -> str | None: + """Return the launch marker a test module declares, or None if it needs no Kit app. + + Args: + source: The module's text. + + Returns: + A key of :data:`KIT_MARKERS`, or None. + + Raises: + ValueError: If the module declares more than one launch marker. They are alternatives, + so there is no configuration that satisfies both. + """ + declared = sorted(module_markers(source) & KIT_MARKERS.keys()) + if len(declared) > 1: + raise ValueError(f"a test module declares more than one launch marker: {', '.join(declared)}") + return declared[0] if declared else None + + +def kit_marker_of_file(path: str | os.PathLike[str]) -> str | None: + """Return the launch marker declared by the test file at ``path``. + + Args: + path: Path to a test file. + + Returns: + A key of :data:`KIT_MARKERS`, or None when the file declares none or cannot be read. + """ + try: + source = Path(path).read_text(encoding="utf-8", errors="replace") + except OSError: + return None + return kit_marker(source) + + +def _module_scope_nodes(tree: ast.Module): + """Yield every node that executes at module import, without entering callables. + + Descends through module-level control flow (``if`` / ``try`` / ``with``) because those + bodies still run at import, but stops at function, class, and lambda boundaries because + those bodies only run when called. + """ + stack = list(tree.body) + while stack: + node = stack.pop() + yield node + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda): + continue + stack.extend(ast.iter_child_nodes(node)) + + +def _marker_names(node: ast.AST) -> list[str]: + """Return the marker names in a ``pytest.mark.`` expression, or a list of them.""" + if isinstance(node, ast.List | ast.Tuple): + return [name for element in node.elts for name in _marker_names(element)] + if isinstance(node, ast.Call): + return _marker_names(node.func) + # pytest.mark., i.e. an attribute whose parent attribute is `mark` + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Attribute) and node.value.attr == "mark": + return [node.attr] + return [] + + +""" +Launching. +""" + + +def _launch(*, cameras: bool) -> SimulationApp: + """Boot this process's Kit app, or return the one already running. + + Args: + cameras: Whether the app must have camera and render extensions enabled. + + Returns: + The running ``SimulationApp``. + + Raises: + RuntimeError: If Kit is already running in a configuration other than the one asked + for, or was started by something other than this plugin. Both mean the files + sharing this process do not share a launch configuration and must be split up. + """ + global _app, _cameras + + if _app is not None: + if cameras != _cameras: + wanted, running = ("with", "without") if cameras else ("without", "with") + raise RuntimeError( + f"a `{_marker_for(cameras)}` file wants a Kit app {wanted} cameras, but this process is" + f" already running one {running} them, and that cannot be changed after startup." + f" `{_marker_for(False)}` and `{_marker_for(True)}` files need separate processes: a" + " camera-enabled app is not a drop-in replacement for a plain one, because" + " test_simulation_context.py::test_headless_mode asserts that offscreen rendering" + " is off." + ) + return _app + + from isaaclab.utils import has_kit + + if has_kit(): + raise RuntimeError( + "Kit is already running but was not started by this plugin, so its launch" + " configuration is unknown. Another test file in this process constructs AppLauncher" + " itself; mark that file `solo` so it keeps a process of its own." + ) + + from isaaclab.app import AppLauncher + + from .utils import resolve_test_sim_device + + _app = AppLauncher(headless=True, enable_cameras=cameras, device=resolve_test_sim_device()).app + _cameras = cameras + return _app + + +def _marker_for(cameras: bool) -> str: + """Return the marker name that asks for the given ``enable_cameras`` setting.""" + return next(name for name, wants in KIT_MARKERS.items() if wants == cameras) + + +""" +Pytest plugin. +""" + + +def pytest_collectstart(collector: pytest.Collector) -> None: + """Boot Kit before pytest imports a module that declares it needs one. + + ``Module.collect()`` is what imports the module, and this hook runs immediately before it. + That is the last point at which the app can still be started early enough for the module's + own ``pxr`` / ``omni`` imports to succeed. + """ + if isinstance(collector, pytest.Module): + marker = kit_marker_of_file(collector.path) + if marker is not None: + _launch(cameras=KIT_MARKERS[marker]) + + +@pytest.fixture(scope="session") +def kit_app() -> SimulationApp: + """The Kit app shared by every launch-marked module in this pytest process. + + Returns: + The running ``SimulationApp``. + + Raises: + RuntimeError: If the requesting module declares no launch marker, so nothing booted + an app for it. + """ + if _app is None: + raise RuntimeError( + "the `kit_app` fixture was requested but no Kit app is running: add one of" + f" {', '.join(sorted(KIT_MARKERS))} to the test module's `pytestmark`." + ) + return _app diff --git a/source/isaaclab/test/cli/test_test_orchestrator_result_handling.py b/source/isaaclab/test/cli/test_test_orchestrator_result_handling.py index 64cd08d9c316..08822b68885c 100644 --- a/source/isaaclab/test/cli/test_test_orchestrator_result_handling.py +++ b/source/isaaclab/test/cli/test_test_orchestrator_result_handling.py @@ -32,8 +32,8 @@ def _load_orchestrator_module() -> ModuleType: - """Load ``tools/conftest.py`` without registering it as a pytest plugin.""" - module_path = TOOLS_DIR / "conftest.py" + """Load ``tools/run_tests.py`` under a private name, leaving any real import untouched.""" + module_path = TOOLS_DIR / "run_tests.py" module_name = "isaaclab_test_orchestrator" tools_dir = str(module_path.parent) if tools_dir not in sys.path: diff --git a/source/isaaclab/test/controllers/test_operational_space.py b/source/isaaclab/test/controllers/test_operational_space.py index 95a965c76aef..e9927a2fdc4e 100644 --- a/source/isaaclab/test/controllers/test_operational_space.py +++ b/source/isaaclab/test/controllers/test_operational_space.py @@ -16,8 +16,6 @@ import torch from flaky import flaky -pytestmark = pytest.mark.arm_ci - import isaaclab.envs.mdp as mdp import isaaclab.sim as sim_utils from isaaclab import cloner @@ -51,7 +49,7 @@ from isaaclab_assets import FRANKA_PANDA_CFG, G1_29DOF_CFG # isort:skip -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.arm_ci, pytest.mark.integration] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_articulation_fragments.py b/source/isaaclab/test/sim/test_articulation_fragments.py index 57cfa0f86dbe..f16581f340d6 100644 --- a/source/isaaclab/test/sim/test_articulation_fragments.py +++ b/source/isaaclab/test/sim/test_articulation_fragments.py @@ -3,15 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import os import pytest @@ -21,6 +12,8 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext +pytestmark = pytest.mark.kit + def _make_xform(stage, path="/World/Art"): UsdGeom.Xform.Define(stage, path) diff --git a/source/isaaclab/test/sim/test_build_simulation_context_headless.py b/source/isaaclab/test/sim/test_build_simulation_context_headless.py index cf266f73f4fe..40f575643308 100644 --- a/source/isaaclab/test/sim/test_build_simulation_context_headless.py +++ b/source/isaaclab/test/sim/test_build_simulation_context_headless.py @@ -13,21 +13,12 @@ ``test_build_simulation_context_nonheadless.py``. """ -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import pytest from isaaclab.sim.simulation_cfg import SimulationCfg from isaaclab.sim.simulation_context import build_simulation_context -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] @pytest.mark.parametrize("gravity_enabled", [True, False]) diff --git a/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py b/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py index 2ce2345062c8..92f79cba8f40 100644 --- a/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py +++ b/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py @@ -12,21 +12,12 @@ ``test_build_simulation_context_headless.py``. """ -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import pytest from isaaclab.sim.simulation_cfg import SimulationCfg from isaaclab.sim.simulation_context import build_simulation_context -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] @pytest.mark.parametrize("gravity_enabled", [True, False]) diff --git a/source/isaaclab/test/sim/test_cloner.py b/source/isaaclab/test/sim/test_cloner.py index 69f97aaf73fc..9a099d13f12c 100644 --- a/source/isaaclab/test/sim/test_cloner.py +++ b/source/isaaclab/test/sim/test_cloner.py @@ -5,15 +5,6 @@ """Tests for USD cloner utilities (no PhysX dependency).""" -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - from types import SimpleNamespace from unittest.mock import MagicMock @@ -38,7 +29,7 @@ from isaaclab.sim import build_simulation_context from isaaclab.sim.utils import queries -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture(params=["cpu", "cuda"]) diff --git a/source/isaaclab/test/sim/test_collision_fragments.py b/source/isaaclab/test/sim/test_collision_fragments.py index f5da64c8b255..c3400f9b20bd 100644 --- a/source/isaaclab/test/sim/test_collision_fragments.py +++ b/source/isaaclab/test/sim/test_collision_fragments.py @@ -3,15 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import pytest from pxr import Sdf, UsdGeom, UsdPhysics @@ -19,7 +10,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def _make_xform(stage, path="/World/Body"): diff --git a/source/isaaclab/test/sim/test_joint_drive_fragments.py b/source/isaaclab/test/sim/test_joint_drive_fragments.py index be4c19de6424..2cc64e494345 100644 --- a/source/isaaclab/test/sim/test_joint_drive_fragments.py +++ b/source/isaaclab/test/sim/test_joint_drive_fragments.py @@ -3,15 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import math import pytest @@ -21,7 +12,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def _make_revolute_joint(stage, path="/World/Articulation/joint_0"): diff --git a/source/isaaclab/test/sim/test_mass_fragments.py b/source/isaaclab/test/sim/test_mass_fragments.py index b57808d501ae..61f1a98ffcac 100644 --- a/source/isaaclab/test/sim/test_mass_fragments.py +++ b/source/isaaclab/test/sim/test_mass_fragments.py @@ -3,15 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import pytest from pxr import UsdGeom, UsdPhysics @@ -19,7 +10,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def _make_xform(stage, path="/World/Body"): diff --git a/source/isaaclab/test/sim/test_material_fragments.py b/source/isaaclab/test/sim/test_material_fragments.py index c09362c5efd0..a5a8843bdc7b 100644 --- a/source/isaaclab/test/sim/test_material_fragments.py +++ b/source/isaaclab/test/sim/test_material_fragments.py @@ -3,15 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import pytest from pxr import UsdPhysics, UsdShade @@ -19,7 +10,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] # ------------------------------------------------------------------------------------- # RigidBodyMaterialFragment marker + metadata diff --git a/source/isaaclab/test/sim/test_mesh_collision_fragments.py b/source/isaaclab/test/sim/test_mesh_collision_fragments.py index ae33dbb938d2..5c1b56016374 100644 --- a/source/isaaclab/test/sim/test_mesh_collision_fragments.py +++ b/source/isaaclab/test/sim/test_mesh_collision_fragments.py @@ -3,15 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import pytest from pxr import UsdGeom, UsdPhysics @@ -19,7 +10,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def _make_xform(stage, path="/World/Mesh"): diff --git a/source/isaaclab/test/sim/test_mesh_converter.py b/source/isaaclab/test/sim/test_mesh_converter.py index f4551b4ba829..d2eb64177e22 100644 --- a/source/isaaclab/test/sim/test_mesh_converter.py +++ b/source/isaaclab/test/sim/test_mesh_converter.py @@ -3,15 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import math import os import random @@ -27,7 +18,7 @@ from isaaclab.sim.schemas import MESH_APPROXIMATION_TOKENS, schemas_cfg from isaaclab.utils.assets import ISAACLAB_NUCLEUS_DIR, retrieve_file_path -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def random_quaternion(): diff --git a/source/isaaclab/test/sim/test_schema_fragments.py b/source/isaaclab/test/sim/test_schema_fragments.py index 5ce4adef2920..8f7fbf03c776 100644 --- a/source/isaaclab/test/sim/test_schema_fragments.py +++ b/source/isaaclab/test/sim/test_schema_fragments.py @@ -3,15 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import pytest from pxr import UsdGeom, UsdPhysics @@ -19,7 +10,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def _make_xform(stage, path="/World/Body"): diff --git a/source/isaaclab/test/sim/test_schema_writer_nested_targets.py b/source/isaaclab/test/sim/test_schema_writer_nested_targets.py index ca084c45bfed..f61baa105900 100644 --- a/source/isaaclab/test/sim/test_schema_writer_nested_targets.py +++ b/source/isaaclab/test/sim/test_schema_writer_nested_targets.py @@ -3,15 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import os import pytest @@ -23,7 +14,7 @@ from isaaclab.sim import SimulationCfg, SimulationContext from isaaclab.sim.schemas import MassCfg, UsdPhysicsCollisionCfg, UsdPhysicsRigidBodyCfg -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] LINK_REL_PATHS = ("link1", "link2", "link2/link3") diff --git a/source/isaaclab/test/sim/test_schemas.py b/source/isaaclab/test/sim/test_schemas.py index de9832f73c4c..2f3f514cf60f 100644 --- a/source/isaaclab/test/sim/test_schemas.py +++ b/source/isaaclab/test/sim/test_schemas.py @@ -3,15 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import math import warnings @@ -45,7 +36,7 @@ from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR from isaaclab.utils.string import to_camel_case -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_simulation_context.py b/source/isaaclab/test/sim/test_simulation_context.py index 325ec6e08176..c338af5e4762 100644 --- a/source/isaaclab/test/sim/test_simulation_context.py +++ b/source/isaaclab/test/sim/test_simulation_context.py @@ -3,16 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher -from isaaclab.test.utils import resolve_test_sim_device, test_devices - -# launch omniverse app -simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app - -"""Rest everything follows.""" - import weakref import numpy as np @@ -25,8 +15,9 @@ import isaaclab.sim as sim_utils from isaaclab.physics import PhysicsEvent from isaaclab.sim import SimulationCfg, SimulationContext +from isaaclab.test.utils import test_devices -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] @pytest.fixture(autouse=True) diff --git a/source/isaaclab/test/sim/test_simulation_stage_in_memory.py b/source/isaaclab/test/sim/test_simulation_stage_in_memory.py index 081496376092..e537cc5a78a9 100644 --- a/source/isaaclab/test/sim/test_simulation_stage_in_memory.py +++ b/source/isaaclab/test/sim/test_simulation_stage_in_memory.py @@ -5,17 +5,6 @@ """Integration tests for simulation context with stage in memory.""" -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -# FIXME (mmittal): Stage in memory requires cameras to be enabled. -simulation_app = AppLauncher(headless=True, enable_cameras=True).app - -"""Rest everything follows.""" - - import pytest import torch @@ -28,7 +17,14 @@ from isaaclab.utils.assets import ISAACLAB_NUCLEUS_DIR from isaaclab.utils.version import get_isaac_sim_version -pytestmark = pytest.mark.integration +# kit_cameras: FIXME (mmittal): stage in memory requires cameras to be enabled. +# +# solo: sharing a Kit app with other test files killed the pytest process here. In the +# kit-reuse-probe-batched CI job this file's first test aborted the interpreter immediately +# after collection, with no Python traceback, while the same test is fine in its own process. +# The cause is not yet understood -- creating the stage in memory is sensitive to what else has +# already touched the stage or the extension set -- so keep the file on its own until it is. +pytestmark = [pytest.mark.kit_cameras, pytest.mark.solo, pytest.mark.integration] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_spawn_from_files.py b/source/isaaclab/test/sim/test_spawn_from_files.py index 64dad8825094..4a9c54fdcdd9 100644 --- a/source/isaaclab/test/sim/test_spawn_from_files.py +++ b/source/isaaclab/test/sim/test_spawn_from_files.py @@ -3,15 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.app import AppLauncher - -"""Launch Isaac Sim Simulator first.""" - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import pytest import omni.kit.app @@ -22,7 +13,7 @@ from isaaclab.sim.spawners.materials.physics_materials_cfg import UsdPhysicsRigidBodyMaterialCfg from isaaclab.utils.assets import ISAACLAB_NUCLEUS_DIR -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_spawn_lights.py b/source/isaaclab/test/sim/test_spawn_lights.py index 59c771880782..7489d86e1a63 100644 --- a/source/isaaclab/test/sim/test_spawn_lights.py +++ b/source/isaaclab/test/sim/test_spawn_lights.py @@ -3,16 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - - import pytest from pxr import Usd, UsdLux @@ -21,7 +11,7 @@ from isaaclab.sim import SimulationCfg, SimulationContext from isaaclab.utils.string import to_camel_case -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture(autouse=True) diff --git a/source/isaaclab/test/sim/test_spawn_materials.py b/source/isaaclab/test/sim/test_spawn_materials.py index d1cb86c87029..7ae14e830fee 100644 --- a/source/isaaclab/test/sim/test_spawn_materials.py +++ b/source/isaaclab/test/sim/test_spawn_materials.py @@ -3,16 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - - import pytest from pxr import UsdPhysics, UsdShade @@ -21,7 +11,7 @@ from isaaclab.sim import SimulationCfg, SimulationContext from isaaclab.utils.assets import NVIDIA_NUCLEUS_DIR -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_spawn_meshes.py b/source/isaaclab/test/sim/test_spawn_meshes.py index 8dd2f5f35f6c..f259e32a8d8d 100644 --- a/source/isaaclab/test/sim/test_spawn_meshes.py +++ b/source/isaaclab/test/sim/test_spawn_meshes.py @@ -3,23 +3,13 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - - import numpy as np import pytest import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_spawn_sensors.py b/source/isaaclab/test/sim/test_spawn_sensors.py index 9e50b54496bc..7b818ef38dd1 100644 --- a/source/isaaclab/test/sim/test_spawn_sensors.py +++ b/source/isaaclab/test/sim/test_spawn_sensors.py @@ -3,16 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - - import pytest from pxr import Usd @@ -22,7 +12,7 @@ from isaaclab.sim.spawners.sensors.sensors import CUSTOM_FISHEYE_CAMERA_ATTRIBUTES, CUSTOM_PINHOLE_CAMERA_ATTRIBUTES from isaaclab.utils.string import to_camel_case -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_spawn_shapes.py b/source/isaaclab/test/sim/test_spawn_shapes.py index fb501e15a771..7c731eec5ac3 100644 --- a/source/isaaclab/test/sim/test_spawn_shapes.py +++ b/source/isaaclab/test/sim/test_spawn_shapes.py @@ -3,21 +3,12 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import pytest import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_spawn_wrappers.py b/source/isaaclab/test/sim/test_spawn_wrappers.py index 9523633f2a11..4877f4e146fc 100644 --- a/source/isaaclab/test/sim/test_spawn_wrappers.py +++ b/source/isaaclab/test/sim/test_spawn_wrappers.py @@ -3,23 +3,13 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - - import pytest import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext from isaaclab.utils.assets import ISAACLAB_NUCLEUS_DIR -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_tendon_fragments.py b/source/isaaclab/test/sim/test_tendon_fragments.py index 60032271c70d..510c67ae3838 100644 --- a/source/isaaclab/test/sim/test_tendon_fragments.py +++ b/source/isaaclab/test/sim/test_tendon_fragments.py @@ -3,15 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import pytest from pxr import PhysxSchema, Sdf, Usd, UsdGeom @@ -19,7 +10,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def _new_sim(): diff --git a/source/isaaclab/test/sim/test_utils_prims.py b/source/isaaclab/test/sim/test_utils_prims.py index 52b570300070..bd5b19deeb7d 100644 --- a/source/isaaclab/test/sim/test_utils_prims.py +++ b/source/isaaclab/test/sim/test_utils_prims.py @@ -3,16 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -# note: need to enable cameras to be able to make replicator core available -simulation_app = AppLauncher(headless=True, enable_cameras=True).app - -"""Rest everything follows.""" - import math import numpy as np @@ -25,7 +15,8 @@ from isaaclab.sim.utils.prims import _to_tuple # type: ignore[reportPrivateUsage] from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR, retrieve_file_path -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +# kit_cameras: replicator core is only available when the app is booted with cameras enabled. +pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture(autouse=True) diff --git a/source/isaaclab/test/sim/test_utils_queries.py b/source/isaaclab/test/sim/test_utils_queries.py index 531be667f73c..03730ca49254 100644 --- a/source/isaaclab/test/sim/test_utils_queries.py +++ b/source/isaaclab/test/sim/test_utils_queries.py @@ -3,16 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -# note: need to enable cameras to be able to make replicator core available -simulation_app = AppLauncher(headless=True, enable_cameras=True).app - -"""Rest everything follows.""" - import ast import inspect import textwrap @@ -25,7 +15,8 @@ from isaaclab.sim.utils import queries from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR -pytestmark = pytest.mark.integration +# kit_cameras: replicator core is only available when the app is booted with cameras enabled. +pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration] @pytest.fixture(autouse=True) diff --git a/source/isaaclab/test/sim/test_utils_semantics.py b/source/isaaclab/test/sim/test_utils_semantics.py index 926a2d0d80a4..6cec2b26b9a5 100644 --- a/source/isaaclab/test/sim/test_utils_semantics.py +++ b/source/isaaclab/test/sim/test_utils_semantics.py @@ -3,21 +3,12 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -# note: need to enable cameras to be able to make replicator core available -simulation_app = AppLauncher(headless=True, enable_cameras=True).app - -"""Rest everything follows.""" - import pytest import isaaclab.sim as sim_utils -pytestmark = pytest.mark.integration +# kit_cameras: replicator core is only available when the app is booted with cameras enabled. +pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration] @pytest.fixture(autouse=True) diff --git a/source/isaaclab/test/sim/test_utils_stage.py b/source/isaaclab/test/sim/test_utils_stage.py index 39a70a076f71..5a9e11439ddf 100644 --- a/source/isaaclab/test/sim/test_utils_stage.py +++ b/source/isaaclab/test/sim/test_utils_stage.py @@ -5,15 +5,6 @@ """Tests for stage utilities.""" -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import tempfile from pathlib import Path @@ -23,7 +14,7 @@ import isaaclab.sim as sim_utils -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] def test_create_new_stage(): diff --git a/source/isaaclab/test/sim/test_utils_transforms.py b/source/isaaclab/test/sim/test_utils_transforms.py index e7cc178b65d5..31c3773b1e88 100644 --- a/source/isaaclab/test/sim/test_utils_transforms.py +++ b/source/isaaclab/test/sim/test_utils_transforms.py @@ -3,15 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" - import math import numpy as np @@ -23,7 +14,7 @@ import isaaclab.sim as sim_utils import isaaclab.utils.math as math_utils -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] @pytest.fixture(autouse=True) diff --git a/source/isaaclab/test/sim/test_views_xform_prim.py b/source/isaaclab/test/sim/test_views_xform_prim.py index 0121e779f82e..88f896ed4cb4 100644 --- a/source/isaaclab/test/sim/test_views_xform_prim.py +++ b/source/isaaclab/test/sim/test_views_xform_prim.py @@ -10,33 +10,39 @@ prim ordering, xformOp standardization, and Isaac Sim comparison. """ -from isaaclab.app import AppLauncher -from isaaclab.test.utils import resolve_test_sim_device, test_devices +import pytest +import torch +import warp as wp -simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app +from pxr import Gf, UsdGeom -import pytest # noqa: E402 -import torch # noqa: E402 -import warp as wp # noqa: E402 - -from pxr import Gf, UsdGeom # noqa: E402 +from isaaclab.test.utils import test_devices try: - from isaaclab.sim.utils import enable_extension # noqa: E402 + from isaaclab.sim.utils import enable_extension + # NOTE: this runs at import, so in a process shared with other test files it changes the + # running app's extension set during collection, before any test executes. Harmless when + # this file has the process to itself; a hazard once files are batched together. enable_extension("isaacsim.core.experimental.prims") from isaacsim.core.experimental.prims import XformPrim as _IsaacSimXformPrimView except (ModuleNotFoundError, ImportError, RuntimeError): _IsaacSimXformPrimView = None -from frame_view_contract_utils import * # noqa: F401, F403, E402 -from frame_view_contract_utils import CHILD_OFFSET, ViewBundle # noqa: E402 +from frame_view_contract_utils import * # noqa: F401, F403 +from frame_view_contract_utils import CHILD_OFFSET, ViewBundle -import isaaclab.sim as sim_utils # noqa: E402 -from isaaclab.sim.views import UsdFrameView as FrameView # noqa: E402 -from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR # noqa: E402 +import isaaclab.sim as sim_utils +from isaaclab.sim.views import UsdFrameView as FrameView +from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +# solo: test_compare_get_world_poses_with_isaacsim goes through Isaac Sim's +# SimulationManager, a process-global singleton that caches the PhysxScene wrapping +# /physicsScene. In a process shared with other test files that prim belongs to a stage an +# earlier file already tore down, so the cached wrapper is dangling and the test dies with +# "Accessed invalid expired 'PhysicsScene' prim". Nothing in this file owns that state, so the +# file needs a process to itself until SimulationManager can be reset between files. +pytestmark = [pytest.mark.kit, pytest.mark.solo, pytest.mark.integration, pytest.mark.isaacsim_ci] PARENT_POS = (0.0, 0.0, 1.0) diff --git a/source/isaaclab/test/test_kit_batching.py b/source/isaaclab/test/test_kit_batching.py new file mode 100644 index 000000000000..923e0ad4b6af --- /dev/null +++ b/source/isaaclab/test/test_kit_batching.py @@ -0,0 +1,229 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the Kit batching grouping and JUnit demultiplexing. + +Both are pure functions over paths and strings, so they run anywhere; the process machinery +they feed is POSIX-only and only exercisable in CI. +""" + +from __future__ import annotations + +import sys +import textwrap +from pathlib import Path + +import pytest +from junitparser import JUnitXml + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tools")) + +from _kit_batching import ( # noqa: E402 + Batch, + batch_size, + batching_enabled, + file_profile, + group_test_files, + split_batch_status, +) + +pytestmark = pytest.mark.unit + + +KIT = "pytestmark = pytest.mark.kit\n" +CAMERAS = "pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration]\n" +SOLO = "pytestmark = [pytest.mark.kit, pytest.mark.solo]\n" +UNMARKED = "pytestmark = pytest.mark.unit\n" +BOTH = "pytestmark = [pytest.mark.kit, pytest.mark.kit_cameras]\n" +LEGACY = "simulation_app = AppLauncher(headless=True).app\n" + + +class TestFileProfile: + """`file_profile` classifies a file from the markers its source declares.""" + + @pytest.mark.parametrize( + "source,expected", + [ + (KIT, "kit"), + (CAMERAS, "kit_cameras"), + (SOLO, None), + (UNMARKED, None), + (LEGACY, None), + (BOTH, None), + ("", None), + ], + ) + def test_profile_matches_markers(self, source: str, expected: str | None): + assert file_profile(source) == expected + + @pytest.mark.parametrize( + "source", + [ + '"""A docstring that mentions pytest.mark.kit."""\n', + "# pytest.mark.kit in a comment\n", + "@pytest.mark.kit\ndef test_one():\n pass\n", + "def helper():\n pytestmark = pytest.mark.kit\n", + ], + ) + def test_markers_outside_a_module_scope_pytestmark_do_not_count(self, source: str): + """Only what the plugin launches from counts, and it launches from ``pytestmark``. + + A per-test decorator resolves after the module is imported, which is already too late + to start Kit, so batching on one would group a file the plugin never launches for. + """ + assert file_profile(source) is None + + def test_unparsable_source_is_not_batched(self): + """A file that does not parse cannot be classified, so it must not be grouped.""" + assert file_profile("pytestmark = [pytest.mark.kit\n") is None + + +class TestGrouping: + """`group_test_files` batches same-profile files and isolates everything else.""" + + def test_same_profile_files_share_one_batch(self): + files = ["a.py", "b.py", "c.py"] + batches = group_test_files(files, dict.fromkeys(files, KIT)) + assert len(batches) == 1 + assert batches[0].profile == "kit" + assert batches[0].files == files + + def test_profiles_never_mix(self): + sources = {"a.py": KIT, "b.py": CAMERAS, "c.py": KIT} + batches = group_test_files(list(sources), sources) + by_profile = {b.profile: b.files for b in batches} + assert by_profile["kit"] == ["a.py", "c.py"] + assert by_profile["kit_cameras"] == ["b.py"] + + @pytest.mark.parametrize("source", [SOLO, UNMARKED, LEGACY]) + def test_unbatchable_files_get_their_own_batch(self, source: str): + sources = {"a.py": KIT, "b.py": source, "c.py": KIT} + batches = group_test_files(list(sources), sources) + solo = [b for b in batches if b.files == ["b.py"]] + assert solo and solo[0].profile is None + assert not solo[0].is_batched + + def test_explicit_unbatchable_overrides_the_marker(self): + sources = {"a.py": KIT, "b.py": KIT} + batches = group_test_files(list(sources), sources, unbatchable={"b.py"}) + assert Batch(profile=None, files=["b.py"]) in batches + + def test_missing_source_is_treated_as_unbatchable(self): + """An unreadable file must not be assumed safe to share a process.""" + batches = group_test_files(["a.py", "b.py"], {"a.py": KIT}) + assert any(b.files == ["b.py"] and b.profile is None for b in batches) + + def test_batches_are_capped(self): + files = [f"f{i}.py" for i in range(7)] + batches = group_test_files(files, dict.fromkeys(files, KIT), max_size=3) + assert [len(b.files) for b in batches] == [3, 3, 1] + + def test_labels_are_unique_across_batches(self): + """A label becomes a JUnit report filename, so two batches must never collide. + + Two same-profile batches of equal size are the case that matters: without the index + they would produce the same label and the second would overwrite the first's report. + """ + files = [f"f{i}.py" for i in range(6)] + batches = group_test_files(files, dict.fromkeys(files, KIT), max_size=3) + labels = [b.label for b in batches] + assert len(batches) == 2 + assert len(labels) == len(set(labels)), f"colliding labels: {labels}" + + def test_every_file_appears_exactly_once(self): + sources = {"a.py": KIT, "b.py": CAMERAS, "c.py": SOLO, "d.py": KIT, "e.py": LEGACY} + batches = group_test_files(list(sources), sources) + covered = [f for b in batches for f in b.files] + assert sorted(covered) == sorted(sources) + assert len(covered) == len(set(covered)) + + +def _report(*cases: tuple[str, str, str, float]) -> JUnitXml: + """Build a JUnit report from ``(classname, name, outcome, time)`` tuples.""" + body = "".join( + f'' + + {"pass": "", "fail": "", "error": "", "skip": ""}[ + outcome + ] + + "" + for cls, name, outcome, t in cases + ) + xml = textwrap.dedent(f"""\ + + {body} + """) + return JUnitXml.fromstring(xml.encode("utf-8")) + + +class TestSplitBatchStatus: + """`split_batch_status` attributes a batch's report back to individual files.""" + + def test_counts_are_attributed_per_file(self): + report = _report( + ("source.sim.test_a", "test_one", "pass", 1.0), + ("source.sim.test_a", "test_two", "fail", 2.0), + ("source.sim.test_b", "test_three", "pass", 3.0), + ) + status = split_batch_status( + report, ["source/sim/test_a.py", "source/sim/test_b.py"], wall_time=60.0, batch_result="CRASHED" + ) + a = status["source/sim/test_a.py"] + b = status["source/sim/test_b.py"] + assert (a["tests"], a["failures"], a["result"]) == (2, 1, "FAILED") + assert (b["tests"], b["failures"], b["result"]) == (1, 0, "passed") + assert a["time_elapsed"] == pytest.approx(3.0) + assert b["time_elapsed"] == pytest.approx(3.0) + + def test_files_that_never_ran_take_the_batch_result(self): + """A file with no testcases means the shared process died before reaching it.""" + report = _report(("source.sim.test_a", "test_one", "pass", 1.0)) + status = split_batch_status( + report, ["source/sim/test_a.py", "source/sim/test_b.py"], wall_time=10.0, batch_result="CRASHED" + ) + assert status["source/sim/test_a.py"]["result"] == "passed" + assert status["source/sim/test_b.py"]["result"] == "CRASHED" + assert status["source/sim/test_b.py"]["errors"] == 1 + + def test_wall_time_is_shared_only_between_files_that_ran(self): + report = _report( + ("source.sim.test_a", "t", "pass", 1.0), + ("source.sim.test_b", "t", "pass", 1.0), + ) + files = ["source/sim/test_a.py", "source/sim/test_b.py", "source/sim/test_c.py"] + status = split_batch_status(report, files, wall_time=90.0, batch_result="CRASHED") + assert status["source/sim/test_a.py"]["wall_time"] == pytest.approx(45.0) + assert status["source/sim/test_b.py"]["wall_time"] == pytest.approx(45.0) + assert status["source/sim/test_c.py"]["wall_time"] == 0.0 + + def test_errors_and_skips_are_counted_separately(self): + report = _report( + ("source.sim.test_a", "t1", "error", 0.5), + ("source.sim.test_a", "t2", "skip", 0.0), + ) + status = split_batch_status(report, ["source/sim/test_a.py"], wall_time=5.0, batch_result="CRASHED") + a = status["source/sim/test_a.py"] + assert (a["errors"], a["skipped"], a["result"]) == (1, 1, "FAILED") + + def test_ambiguous_stems_are_not_misattributed(self): + """Two members sharing a basename cannot be told apart, so neither claims the case.""" + report = _report(("pkg.one.test_dup", "t", "pass", 1.0)) + files = ["pkg/one/test_dup.py", "pkg/two/test_dup.py"] + status = split_batch_status(report, files, wall_time=10.0, batch_result="CRASHED") + assert all(status[f]["result"] == "CRASHED" for f in files) + + +class TestEnvironmentToggles: + """Batching is on unless a lane explicitly turns it off.""" + + @pytest.mark.parametrize("value,expected", [("0", False), ("false", False), ("NO", False), ("1", True), ("", True)]) + def test_disable_flag(self, value: str, expected: bool): + assert batching_enabled({"ISAACLAB_TEST_BATCH_KIT": value}) is expected + + def test_enabled_when_unset(self): + assert batching_enabled({}) is True + + @pytest.mark.parametrize("value,expected", [("5", 5), ("", 12), ("nonsense", 12), ("0", 12), ("-3", 12)]) + def test_batch_size_override(self, value: str, expected: int): + assert batch_size({"ISAACLAB_TEST_BATCH_SIZE": value}) == expected diff --git a/source/isaaclab/test/test_kit_marker_contract.py b/source/isaaclab/test/test_kit_marker_contract.py new file mode 100644 index 000000000000..356166cdd9f7 --- /dev/null +++ b/source/isaaclab/test/test_kit_marker_contract.py @@ -0,0 +1,280 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Test that every test file's Kit markers agree with what the file actually does. + +Kit-dependence is a property of *importing* a test module: a module that imports ``omni`` at +module scope, or constructs :class:`~isaaclab.app.AppLauncher` there, needs Isaac Sim running +by the time pytest imports it -- before any fixture exists. :mod:`isaaclab.test.kit` turns +that property into a declaration, reading a module's ``pytestmark`` out of its source and +booting Kit before the import, so files sharing a launch configuration can share one app. + +A declaration is only useful if it cannot drift from what the file does, which is what this +test enforces: + +* At most one module-scope ``pytestmark`` assignment, since a second assignment rebinds the + name and silently discards the markers from the first. +* At most one launch marker per file. ``kit`` and ``kit_cameras`` are alternatives, so there + is no single app that satisfies both. +* A file declaring a launch marker does not also construct ``AppLauncher`` or + ``SimulationApp``, which would boot a second, unshared app inside the shared one. +* A file marked ``unit`` neither declares a launch marker nor imports a Kit runtime package at + module scope, which turns that marker's registered description ("does not launch the + simulator") into a checked invariant. +* Within :data:`_MIGRATED_ROOTS`, a module-scope Kit runtime import is backed by something + that actually starts Kit -- a launch marker, or the file's own ``AppLauncher``. +* ``solo`` appears only alongside a launch marker. It is the only case where it changes + anything, and writing it alone reads like a launch marker while starting no app at all. + +The checks are AST-based rather than text-based because a source-text search cannot tell an +``AppLauncher`` reference in a docstring from a real call -- several Kit-free files mention +``AppLauncher`` only to document that they do not use it. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +from isaaclab.test.kit import KIT_MARKERS, SOLO_MARKER, module_markers + +pytestmark = pytest.mark.unit + +_REPO_ROOT = Path(__file__).resolve().parents[3] + +_SCAN_ROOTS = ("source", "scripts") + +_EXCLUDED_PARTS = frozenset( + { + # Own pytest.ini / rootdir; deliberately excluded from the main collector too. + "install_ci", + # Vendored copies of the source tree produced by the wheel builder. + "build", + # Virtual environments and the Isaac Sim symlink. + ".venv", + "env_isaaclab", + "_isaac_sim", + } +) + +# Packages that only exist inside a running Kit application. ``pxr`` is deliberately absent: +# OpenUSD is importable Kit-lessly through the ``usd-core`` wheel, so importing it says nothing +# about whether Kit is running. +_KIT_RUNTIME_PREFIXES = ("omni", "carb", "isaacsim") + +# Names whose construction starts an app the plugin does not own. +_LAUNCHER_NAMES = ("AppLauncher", "SimulationApp") + +# Directories where a module-scope Kit runtime import must be backed by a launch marker or by +# the file's own AppLauncher. Grows one package at a time as files are migrated. +_MIGRATED_ROOTS = ("source/isaaclab/test/sim/",) + + +class _FileFacts: + """What a single test file declares and what it actually does at module scope.""" + + def __init__(self, path: Path, source: str, tree: ast.Module): + self.path = path + self.markers = set(module_markers(source)) + self.pytestmark_lines: list[int] = [] + self.module_scope_launchers: list[tuple[str, int]] = [] + self.kit_runtime_imports: list[tuple[str, int]] = [] + + for node in _module_scope_nodes(tree): + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "pytestmark" for target in node.targets + ): + self.pytestmark_lines.append(node.lineno) + + name = _call_name(node) + if name in _LAUNCHER_NAMES: + self.module_scope_launchers.append((name, node.lineno)) + + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name.split(".")[0] in _KIT_RUNTIME_PREFIXES: + self.kit_runtime_imports.append((alias.name, node.lineno)) + elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0: + if node.module.split(".")[0] in _KIT_RUNTIME_PREFIXES: + self.kit_runtime_imports.append((node.module, node.lineno)) + + # Decorator markers (e.g. a per-test ``@pytest.mark.unit``) count toward the file's + # marker set even though they resolve too late to influence the launch. + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): + for decorator in node.decorator_list: + self.markers.update(_decorator_marker_names(decorator)) + + @property + def rel(self) -> str: + """Repo-relative POSIX path, as it appears in an assertion message.""" + return self.path.relative_to(_REPO_ROOT).as_posix() + + @property + def launch_markers(self) -> list[str]: + """The launch markers this file declares, sorted.""" + return sorted(self.markers & KIT_MARKERS.keys()) + + @property + def launchers(self) -> str: + """The file's own app constructions, formatted for an assertion message.""" + return ", ".join(f"{name} at line {line}" for name, line in self.module_scope_launchers) + + +def _module_scope_nodes(tree: ast.Module): + """Yield every node that executes at module import, without entering callables. + + Descends through module-level control flow (``if`` / ``try`` / ``with``) because those + bodies still run at import, but stops at function, class, and lambda boundaries because + those bodies only run when called. + """ + stack = list(tree.body) + while stack: + node = stack.pop() + yield node + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda): + continue + stack.extend(ast.iter_child_nodes(node)) + + +def _call_name(node: ast.AST) -> str | None: + """Return the called function's bare name, for ``f()`` and ``mod.f()`` alike.""" + if not isinstance(node, ast.Call): + return None + if isinstance(node.func, ast.Name): + return node.func.id + if isinstance(node.func, ast.Attribute): + return node.func.attr + return None + + +def _decorator_marker_names(node: ast.AST) -> list[str]: + """Return the marker names in a ``@pytest.mark.`` decorator expression.""" + if isinstance(node, ast.Call): + return _decorator_marker_names(node.func) + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Attribute) and node.value.attr == "mark": + return [node.attr] + return [] + + +def _iter_test_files(): + for root in _SCAN_ROOTS: + for path in sorted((_REPO_ROOT / root).rglob("test_*.py")): + if _EXCLUDED_PARTS.isdisjoint(path.parts): + yield path + + +@pytest.fixture(scope="module") +def facts() -> list[_FileFacts]: + """Parse every test file once and return the extracted facts.""" + collected = [] + for path in _iter_test_files(): + source = path.read_text(encoding="utf-8", errors="replace") + try: + tree = ast.parse(source, filename=str(path)) + except SyntaxError as exc: + pytest.fail(f"{path.relative_to(_REPO_ROOT).as_posix()} failed to parse: {exc}") + collected.append(_FileFacts(path, source, tree)) + assert collected, f"no test files discovered under {_SCAN_ROOTS} -- the scan roots are wrong" + return collected + + +def test_pytestmark_is_assigned_at_most_once(facts: list[_FileFacts]): + """A second module-scope ``pytestmark`` rebinds the name and drops the first one's markers.""" + offenders = [f"{f.rel}: lines {sorted(f.pytestmark_lines)}" for f in facts if len(f.pytestmark_lines) > 1] + assert not offenders, ( + "These files assign `pytestmark` more than once at module scope. The later assignment" + " replaces the earlier one, so the markers declared first are silently lost:\n " + + "\n ".join(offenders) + + "\n\nFix: merge them into a single list, e.g. `pytestmark = [pytest.mark.a, pytest.mark.b]`." + ) + + +def test_launch_markers_are_mutually_exclusive(facts: list[_FileFacts]): + """A file runs in exactly one launch configuration, so it declares at most one.""" + offenders = [f"{f.rel}: {', '.join(f.launch_markers)}" for f in facts if len(f.launch_markers) > 1] + assert not offenders, ( + f"These files declare more than one of {', '.join(sorted(KIT_MARKERS))}, which are" + " alternatives rather than nested configurations, so no single app satisfies both:\n " + "\n ".join(offenders) + ) + + +def test_solo_accompanies_a_launch_marker(facts: list[_FileFacts]): + """`solo` only changes anything for a file that gets an app booted for it. + + A file with no launch marker already runs on its own, so `solo` alone is not merely + redundant -- it reads like a launch marker while starting no app, which is how a file ends + up importing ``omni`` into a process where Kit was never started. + """ + offenders = [f.rel for f in facts if SOLO_MARKER in f.markers and not f.launch_markers] + assert not offenders, ( + f"These files declare `{SOLO_MARKER}` without one of {', '.join(sorted(KIT_MARKERS))}, so" + " nothing boots an app for them and the marker changes nothing:\n " + + "\n ".join(offenders) + + f"\n\nFix: add the launch marker the file needs, or drop `{SOLO_MARKER}` -- an unmarked" + " file is never grouped with another anyway." + ) + + +def test_launch_marked_files_do_not_build_their_own_app(facts: list[_FileFacts]): + """A marked file is handed the process app; building another one defeats the sharing.""" + offenders = [ + f"{f.rel}: declares `{f.launch_markers[0]}` but constructs {f.launchers}" + for f in facts + if f.launch_markers and f.module_scope_launchers + ] + assert not offenders, ( + "These files declare a launch marker and also construct their own app:\n " + + "\n ".join(offenders) + + "\n\nFix: drop the AppLauncher construction and let `isaaclab.test.kit` launch for the" + " marker, or drop the marker and keep the file on a process of its own." + ) + + +def test_unit_files_do_not_touch_kit(facts: list[_FileFacts]): + """A `unit` file must run in a process where Kit was never started.""" + offenders = [] + for f in facts: + if "unit" not in f.markers: + continue + if f.module_scope_launchers: + offenders.append(f"{f.rel}: constructs {f.launchers}") + if f.launch_markers: + offenders.append(f"{f.rel}: declares `{f.launch_markers[0]}`") + if f.kit_runtime_imports: + where = ", ".join(f"`{name}` at line {line}" for name, line in f.kit_runtime_imports) + offenders.append(f"{f.rel}: imports {where} at module scope") + + assert not offenders, ( + "These files are marked `unit` but depend on a running Kit:\n " + + "\n ".join(offenders) + + f"\n\nKit runtime packages: {_KIT_RUNTIME_PREFIXES}." + "\nFix: mark the file `integration` and declare `kit`, or move the Kit import inside the" + " test function so it is not paid at collection." + ) + + +def test_migrated_files_start_kit_before_importing_it(facts: list[_FileFacts]): + """In a migrated package, a Kit import at module scope needs something that booted Kit. + + Either the file declares a launch marker, in which case the plugin boots for it, or it + still constructs its own ``AppLauncher``. A file with neither imports ``omni`` into a + process where Kit was never started, which fails at collection. + """ + offenders = [ + f"{f.rel}: imports `{f.kit_runtime_imports[0][0]}` at line {f.kit_runtime_imports[0][1]}" + for f in facts + if f.rel.startswith(_MIGRATED_ROOTS) + and f.kit_runtime_imports + and not f.launch_markers + and not f.module_scope_launchers + ] + assert not offenders, ( + "These files import a Kit runtime package at module scope but neither declare one of" + f" {', '.join(sorted(KIT_MARKERS))} nor construct an app themselves, so nothing starts" + " Kit before pytest imports them:\n " + "\n ".join(offenders) + ) diff --git a/source/isaaclab/test/test_kit_plugin.py b/source/isaaclab/test/test_kit_plugin.py new file mode 100644 index 000000000000..d50861ae62df --- /dev/null +++ b/source/isaaclab/test/test_kit_plugin.py @@ -0,0 +1,167 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the marker-driven Kit launch plugin. + +The claim the whole arrangement rests on is that :func:`isaaclab.test.kit.pytest_collectstart` +runs *before* pytest imports the test module. If it ever ran after, every ``kit`` file would +fail at its first ``import omni``, so the ordering is asserted here directly rather than left +to CI to discover. + +The launch itself is stubbed out. Booting Kit is what these tests exist to schedule correctly, +not something they need to do. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +import isaaclab.test.kit as kit + +pytestmark = pytest.mark.unit + +_LOG_ENV_VAR = "ISAACLAB_KIT_PLUGIN_PROBE_LOG" + +_CONFTEST = """\ + import os + + import isaaclab.test.kit as kit + + + def _record(entry): + with open(os.environ["{log_env_var}"], "a", encoding="utf-8") as handle: + handle.write(entry + "\\n") + + + def _fake_launch(*, cameras): + _record(f"launch:{{cameras}}") + return object() + + + kit._launch = _fake_launch +""" + +_PYTEST_INI = """\ + [pytest] + markers = + kit: needs a headless Kit app + kit_cameras: needs a Kit app with cameras enabled +""" + +_MODULE = """\ + import os + + import pytest + + with open(os.environ["{log_env_var}"], "a", encoding="utf-8") as handle: + handle.write("import:{name}\\n") + + {pytestmark} + + + def test_placeholder({args}): + pass +""" + + +def _write_module(directory: Path, name: str, *, marker: str | None, args: str = "") -> None: + """Write a scratch test module that logs its own import.""" + (directory / f"test_{name}.py").write_text( + textwrap.dedent(_MODULE).format( + log_env_var=_LOG_ENV_VAR, + name=name, + pytestmark=f"pytestmark = pytest.mark.{marker}" if marker else "", + args=args, + ), + encoding="utf-8", + ) + + +def _run_scratch_pytest(directory: Path) -> tuple[subprocess.CompletedProcess, list[str]]: + """Run pytest over ``directory`` with the plugin loaded, and return its output and log. + + The scratch directory carries its own ``pytest.ini`` so it becomes the rootdir, which keeps + the repo's own conftest -- and therefore the real, unstubbed plugin -- out of the run. + """ + (directory / "conftest.py").write_text( + textwrap.dedent(_CONFTEST).format(log_env_var=_LOG_ENV_VAR), encoding="utf-8" + ) + (directory / "pytest.ini").write_text(textwrap.dedent(_PYTEST_INI), encoding="utf-8") + log = directory / "probe.log" + + result = subprocess.run( + [sys.executable, "-m", "pytest", "-p", "isaaclab.test.kit", "-q", "-p", "no:cacheprovider"], + cwd=directory, + env={**os.environ, _LOG_ENV_VAR: str(log)}, + capture_output=True, + text=True, + timeout=300, + ) + entries = log.read_text(encoding="utf-8").splitlines() if log.exists() else [] + return result, entries + + +def test_kit_is_launched_before_the_module_is_imported(tmp_path: Path): + """The marker only works if the app is up before the module's own imports run.""" + _write_module(tmp_path, "marked", marker="kit") + result, entries = _run_scratch_pytest(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + assert entries == ["launch:False", "import:marked"], ( + f"expected the launch to precede the import, got {entries}\n{result.stdout}" + ) + + +def test_the_marker_selects_the_camera_setting(tmp_path: Path): + """`kit_cameras` must reach AppLauncher as ``enable_cameras=True``, and `kit` as False.""" + _write_module(tmp_path, "plain", marker="kit") + _write_module(tmp_path, "with_cameras", marker="kit_cameras") + result, entries = _run_scratch_pytest(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + launches = {entry for entry in entries if entry.startswith("launch:")} + assert launches == {"launch:False", "launch:True"} + + +def test_an_unmarked_module_does_not_launch_anything(tmp_path: Path): + """Most of the suite is unmarked, and collecting it must stay Kit-free.""" + _write_module(tmp_path, "unmarked", marker=None) + result, entries = _run_scratch_pytest(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + assert entries == ["import:unmarked"] + + +def test_requesting_kit_app_without_a_marker_says_what_is_missing(tmp_path: Path): + """The fixture is only meaningful in a module that declared a launch marker.""" + _write_module(tmp_path, "unmarked", marker=None, args="kit_app") + result, _ = _run_scratch_pytest(tmp_path) + + assert result.returncode != 0 + assert "no Kit app is running" in result.stdout + + +def test_a_second_configuration_in_one_process_is_refused(monkeypatch: pytest.MonkeyPatch): + """`kit` and `kit_cameras` files in one process must fail loudly, not silently share.""" + monkeypatch.setattr(kit, "_app", object()) + monkeypatch.setattr(kit, "_cameras", False) + + with pytest.raises(RuntimeError, match="cannot be changed after startup"): + kit._launch(cameras=True) + + +def test_an_app_started_by_something_else_is_refused(monkeypatch: pytest.MonkeyPatch): + """An app of unknown configuration cannot be handed to a file that asked for a known one.""" + monkeypatch.setattr(kit, "_app", None) + monkeypatch.setattr("isaaclab.utils.has_kit", lambda: True) + + with pytest.raises(RuntimeError, match="not started by this plugin"): + kit._launch(cameras=False) diff --git a/source/isaaclab_tasks/changelog.d/mataylor-kit-test-markers.skip b/source/isaaclab_tasks/changelog.d/mataylor-kit-test-markers.skip new file mode 100644 index 000000000000..fc49ef0e3ac0 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/mataylor-kit-test-markers.skip @@ -0,0 +1 @@ +Test-only: updated a docstring reference to the renamed test runner. diff --git a/source/isaaclab_tasks/test/benchmarking/conftest.py b/source/isaaclab_tasks/test/benchmarking/conftest.py index 51096c6de144..6cce81604d4d 100644 --- a/source/isaaclab_tasks/test/benchmarking/conftest.py +++ b/source/isaaclab_tasks/test/benchmarking/conftest.py @@ -89,7 +89,7 @@ def kpi_store(): # Shard parametrized test items across parallel CI jobs. -# Reads the same TEST_SHARD_INDEX / TEST_SHARD_COUNT env vars used by tools/conftest.py +# Reads the same TEST_SHARD_INDEX / TEST_SHARD_COUNT env vars used by tools/run_tests.py # for file-level sharding, but applies them at the test-item level so a single # parametrized file can be split across multiple runners. # This is a pytest hook ā€” pytest calls it automatically during test collection. diff --git a/tools/_device_split.py b/tools/_device_split.py index e8bad4069133..94ac2663f8e1 100644 --- a/tools/_device_split.py +++ b/tools/_device_split.py @@ -9,7 +9,7 @@ scope must be re-invoked once per device (CPU and GPU) in separate processes to work around process-global device locks such as ``ovphysx<=0.3.7`` gap G5. The :func:`is_device_split_file` predicate lets the per-file CI runner in -``tools/conftest.py`` detect this without importing the test module. +``tools/run_tests.py`` detect this without importing the test module. """ from __future__ import annotations @@ -29,7 +29,7 @@ a future test needs that, expand the parsing rule. """ -# Per-pass pytest ``-k`` selectors used by ``tools/conftest.py`` when a file +# Per-pass pytest ``-k`` selectors used by ``tools/run_tests.py`` when a file # declares the ``device_split`` marker. Each entry is ``(suffix, k_expr)``: # - ``suffix`` is appended to the JUnit report filename to keep both passes' XML. # - ``k_expr`` is the ``-k`` keyword expression. ``"cpu or not cuda"`` keeps diff --git a/tools/_kit_batching.py b/tools/_kit_batching.py new file mode 100644 index 000000000000..7e9784a2395d --- /dev/null +++ b/tools/_kit_batching.py @@ -0,0 +1,279 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Group test files that can share one Kit app into a single pytest invocation. + +A test file that boots Kit at module scope pays Kit startup on its own, and the runner +gives every file its own subprocess, so a directory of 23 such files boots Kit 23 times. +Files that declare a launch marker (see :mod:`isaaclab.test.kit`) share the app when they +land in one process, which turns those 23 boots into one. + +Only files carrying the same launch marker may be grouped. ``kit`` and ``kit_cameras`` +cannot share a process in either direction: cameras cannot be enabled after startup, and a +camera-enabled app is not a substitute for a plain one because some tests assert that +offscreen rendering is off. Anything whose behaviour depends on having a process to itself +stays on the per-file path, and so does every file that declares no marker at all -- which is +still most of the suite. + +Apart from reading :data:`BATCH_ENV_VAR` and :data:`BATCH_SIZE_ENV_VAR`, this module is +deliberately free of process machinery: the grouping and the report demultiplexing are pure +functions over paths and strings, so they can be exercised on any platform, unlike the +POSIX-only subprocess handling in ``tools/run_tests.py``. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field + +from isaaclab.test.kit import SOLO_MARKER, kit_marker, module_markers + +BATCH_ENV_VAR = "ISAACLAB_TEST_BATCH_KIT" +"""Environment variable that turns batching off. Unset (the default) groups what it can.""" + +BATCH_SIZE_ENV_VAR = "ISAACLAB_TEST_BATCH_SIZE" +"""Environment variable overriding :data:`DEFAULT_BATCH_SIZE`.""" + +DEFAULT_BATCH_SIZE = 12 +"""Files per batch. + +Bounded so that one crash cannot cost a whole lane, and so accumulated GPU memory in a long +shared process does not become its own failure mode. +""" + +BATCH_TIMEOUT_CUTOFF = 2000 +"""Files whose own timeout reaches this stay unbatched. + +A batch's timeout is the sum of its members', so one file hanging consumes the whole budget. +The long-running files are also the ones where Kit startup is a rounding error, so excluding +them removes most of the risk and almost none of the benefit. +""" + + +@dataclass +class Batch: + """One pytest invocation covering one or more test files. + + Attributes: + profile: Launch profile shared by every member, or None for an unbatched file. + files: Test files to hand to pytest, in invocation order. + index: Position among the batches of this profile. Part of :attr:`label`, which + becomes a JUnit report filename, so two batches of the same profile and size + cannot write to the same path. + """ + + profile: str | None + files: list[str] = field(default_factory=list) + index: int = 0 + + @property + def is_batched(self) -> bool: + """Whether this covers more than one file.""" + return len(self.files) > 1 + + @property + def label(self) -> str: + """Short identifier used in logs and JUnit report filenames.""" + return f"batch-{self.profile}-{self.index}-{len(self.files)}files" if self.is_batched else self.files[0] + + +def batching_enabled(env: dict | None = None) -> bool: + """Whether this run may group files, i.e. :data:`BATCH_ENV_VAR` is not set to a false value. + + Batching is on by default because only marker-carrying files can be grouped, and an + unreached member of a dead batch is re-run on the per-file path anyway. The escape hatch + exists so a lane that hits a grouping-specific failure can be unblocked without a revert. + """ + env = os.environ if env is None else env + return env.get(BATCH_ENV_VAR, "").strip().lower() not in ("0", "false", "no") + + +def batch_size(env: dict | None = None) -> int: + """Resolve the per-batch file cap, falling back to :data:`DEFAULT_BATCH_SIZE`.""" + env = os.environ if env is None else env + raw = env.get(BATCH_SIZE_ENV_VAR, "").strip() + if not raw: + return DEFAULT_BATCH_SIZE + try: + value = int(raw) + except ValueError: + return DEFAULT_BATCH_SIZE + return value if value > 0 else DEFAULT_BATCH_SIZE + + +def file_profile(source: str) -> str | None: + """Return the launch marker a test file can be grouped under, or None if it cannot be. + + This is :func:`isaaclab.test.kit.kit_marker` -- the same reader the plugin launches from, + so a batch cannot be built around a marker the launch does not honour -- plus the + ``solo`` opt-out, which is a batching concern rather than a launch one. + + Args: + source: The test file's text. Markers are read from the source rather than by + importing the module, because importing a Kit-dependent module boots Kit. + + Returns: + ``"kit_cameras"``, ``"kit"``, or None when the file is unmarked, opts out, or declares + markers the launch plugin would itself reject. + """ + if SOLO_MARKER in module_markers(source): + return None + try: + return kit_marker(source) + except ValueError: + return None # more than one launch marker; the marker-contract test reports it + + +def group_test_files( + test_files: list[str], + sources: dict[str, str], + *, + unbatchable: set[str] | None = None, + max_size: int = DEFAULT_BATCH_SIZE, +) -> list[Batch]: + """Partition ``test_files`` into batches, preserving the given order. + + Files that cannot be grouped -- unmarked, ``solo``, or listed in ``unbatchable`` -- + each become a batch of one, which is exactly the current per-file behaviour. + + Args: + test_files: Test file paths, in the order the runner would execute them. + sources: Map from a path in ``test_files`` to that file's text. A path missing from + the map is treated as unbatchable rather than assumed safe. + unbatchable: Paths to keep on the per-file path regardless of their markers. + max_size: Maximum files per batch. + + Returns: + Batches covering every input file exactly once, in input order. + """ + unbatchable = unbatchable or set() + batches: list[Batch] = [] + pending: dict[str, Batch] = {} + counts: dict[str, int] = {} + + def flush(profile: str) -> None: + if profile in pending: + batches.append(pending.pop(profile)) + + for path in test_files: + source = sources.get(path) + profile = None if source is None or path in unbatchable else file_profile(source) + + if profile is None: + batches.append(Batch(profile=None, files=[path])) + continue + + current = pending.get(profile) + if current is None: + current = Batch(profile=profile, index=counts.get(profile, 0)) + counts[profile] = current.index + 1 + pending[profile] = current + current.files.append(path) + if len(current.files) >= max_size: + flush(profile) + + # Emit any partially filled batches in a stable order. + for profile in sorted(pending): + batches.append(pending[profile]) + return batches + + +def _testcase_files(report, batch_files: list[str]) -> dict[str, list]: + """Map each batch member to the testcases attributed to it in a JUnit report. + + JUnit ``classname`` encodes the dotted module path, so a file is matched by its stem. + Where two members share a stem the match is ambiguous and those testcases are dropped + from the per-file split rather than assigned to the wrong file. + """ + stems: dict[str, list[str]] = {} + for path in batch_files: + stem = os.path.splitext(os.path.basename(path))[0] + stems.setdefault(stem, []).append(path) + + per_file: dict[str, list] = {path: [] for path in batch_files} + for suite in report: + for case in suite: + classname = getattr(case, "classname", "") or "" + name = getattr(case, "name", "") or "" + for part in reversed(classname.split(".")): + owners = stems.get(part) + if owners and len(owners) == 1: + per_file[owners[0]].append(case) + break + else: + # Fall back to the test name for parametrized ids that carry the module. + for stem, owners in stems.items(): + if len(owners) == 1 and stem in name: + per_file[owners[0]].append(case) + break + return per_file + + +def split_batch_status( + report, + batch_files: list[str], + *, + wall_time: float, + batch_result: str, +) -> dict[str, dict]: + """Attribute a batch's JUnit report back to its individual files. + + The summary table, the failed-file list, and the per-file JUnit artifact are all keyed by + file, so a batch has to be taken apart again before its results are reported. + + A file with no testcases in the report never ran -- the shared process died before + reaching it -- and is marked with ``batch_result`` so the caller can re-run it. + + Args: + report: Parsed JUnit XML for the whole batch. + batch_files: The batch's members. + wall_time: Wall seconds for the whole batch, shared out across members that ran. + batch_result: Result to record for members that produced no testcases. + + Returns: + Map from file path to a status dict of the same shape the per-file path produces. + """ + per_file = _testcase_files(report, batch_files) + ran = [path for path, cases in per_file.items() if cases] + share = wall_time / len(ran) if ran else 0.0 + + statuses: dict[str, dict] = {} + for path in batch_files: + cases = per_file[path] + if not cases: + statuses[path] = { + "errors": 1, + "failures": 0, + "skipped": 0, + "tests": 1, + "result": batch_result, + "time_elapsed": 0.0, + "wall_time": 0.0, + } + continue + + errors = failures = skipped = 0 + elapsed = 0.0 + for case in cases: + elapsed += float(getattr(case, "time", 0.0) or 0.0) + result = getattr(case, "result", None) or [] + kinds = {type(entry).__name__ for entry in result} + if "Error" in kinds: + errors += 1 + elif "Failure" in kinds: + failures += 1 + elif "Skipped" in kinds: + skipped += 1 + + statuses[path] = { + "errors": errors, + "failures": failures, + "skipped": skipped, + "tests": len(cases), + "result": "FAILED" if (errors or failures) else "passed", + "time_elapsed": elapsed, + "wall_time": share, + } + return statuses diff --git a/tools/changelog/pyproject.toml b/tools/changelog/pyproject.toml index b43578b10039..04a1ef7865f6 100644 --- a/tools/changelog/pyproject.toml +++ b/tools/changelog/pyproject.toml @@ -3,7 +3,7 @@ # # 1. ``pythonpath = ["."]`` adds ``tools/changelog/`` to ``sys.path``, # making ``import cli`` work from the test files without any shim. -# 2. ``tools/conftest.py`` (a session-takeover hook for the IsaacLab +# 2. ``tools/run_tests.py`` (a session-takeover hook for the IsaacLab # source/ test suite) sits *above* rootdir and is therefore not # loaded ā€” no ``--noconftest`` flag required. # diff --git a/tools/crash_journal.py b/tools/crash_journal.py index 8936a2c24413..ed9696b1e918 100644 --- a/tools/crash_journal.py +++ b/tools/crash_journal.py @@ -7,7 +7,7 @@ pytest writes its JUnit XML once, in ``pytest_sessionfinish``. A run killed before that point ā€” a Kit shutdown crash, an OOM kill, a hard timeout ā€” leaves no report at all, even though every -test verdict was already printed to stdout. ``tools/conftest.py`` used to answer that by +test verdict was already printed to stdout. ``tools/run_tests.py`` used to answer that by synthesizing a single ``test_execution`` error, which discarded which tests passed, which failed, and which one was in flight when the process died. diff --git a/tools/generate_workflows.py b/tools/generate_workflows.py new file mode 100644 index 000000000000..0f7c1acbcd8e --- /dev/null +++ b/tools/generate_workflows.py @@ -0,0 +1,181 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Render the uniform test lanes in the workflow files from ``tools/test_plan.toml``. + +Most package lanes are the same twenty-odd lines of YAML differing only in a name, a container +and a package list, so they were maintained by copy-paste: adding a package meant adding a +block, and a change to the shape had to be applied to every copy by hand. Those lanes are +generated here, between sentinel comments, and :mod:`tools.test.test_test_plan` fails if the +checked-in YAML has drifted from what this produces. + +Only jobs marked ``generate = true`` are rendered. The lanes with bespoke setup -- extra image +builds, wheelhouse expressions, artifact uploads -- stay hand-written; they are still required +to name a job the plan defines, which is what keeps the two from diverging. + +Usage:: + + python tools/generate_workflows.py # rewrite the generated blocks + python tools/generate_workflows.py --check # report drift, change nothing +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import testplan +from testplan import REPO_ROOT, Job + +WORKFLOW_DIR = REPO_ROOT / ".github" / "workflows" + +BEGIN = " # >>> generated from tools/test_plan.toml -- edit the plan, then run tools/generate_workflows.py" +END = " # <<< end generated jobs" + + +def _job_id(job: Job, shard: int | None) -> str: + """Workflow job id; sharded jobs are numbered from 1 to read like their display name.""" + return f"test-{job.name}" if shard is None else f"test-{job.name}-{shard + 1}" + + +def _title(job: Job, shard: int | None) -> str: + """Display name; a sharded job carries its position.""" + return job.title if shard is None else f"{job.title} [{shard + 1}/{job.shards}]" + + +def _container(job: Job, shard: int | None) -> str: + """Container name, kept distinct per shard so two shards cannot collide on one runner.""" + base = job.container_name or f"isaac-lab-{job.name}" + if shard is None: + return base + return base[: -len("-test")] + f"-{shard + 1}-test" if base.endswith("-test") else f"{base}-{shard + 1}" + + +def render_job(job: Job, shard: int | None) -> str: + """Render one workflow job block. + + Args: + job: Job from the plan. + shard: Shard index, or None for an unsharded job. + + Returns: + The YAML block, ending in a blank line. + """ + lines = [ + f" {_job_id(job, shard)}:", + f" name: {_title(job, shard)}", + " runs-on: [self-hosted, gpu]", + f" timeout-minutes: {job.timeout_minutes}", + ] + if job.continue_on_error: + lines.append(" continue-on-error: true") + lines += [ + " needs: [build, config]", + " if: >-", + " github.event_name != 'push' &&", + " needs.build.result == 'success'", + " steps:", + " - uses: actions/checkout@v6", + " with:", + " fetch-depth: 1", + " lfs: true", + " - uses: ./.github/actions/run-package-tests", + " with:", + " image-tag: ${{ needs.config.outputs.ci_image_tag }}", + " isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}", + " isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}", + f" job: {job.name}", + ] + if shard is not None: + lines.append(f' shard: "{shard}"') + if job.extra_pip_packages: + lines.append(f' extra-pip-packages: "{job.extra_pip_packages}"') + if job.warp_cache: + lines.append(f" warp-cache: {job.warp_cache}") + lines.append(f" container-name: {_container(job, shard)}") + return "\n".join(lines) + "\n" + + +def render(workflow: str) -> str: + """Render every generated job for one workflow file, in plan order.""" + blocks = [] + for job in testplan.load_plan(): + if not job.generate or job.workflow != workflow: + continue + shards = range(job.shards) if job.shards > 1 else [None] + blocks.extend(render_job(job, shard) for shard in shards) + return "\n".join(blocks) + + +def _splice(text: str, body: str) -> str: + """Replace the text between the sentinels, keeping everything outside untouched. + + Raises: + ValueError: If the sentinels are missing or out of order, which would otherwise let + the generator silently write nothing. + """ + start, end = text.find(BEGIN), text.find(END) + if start == -1 or end == -1 or end < start: + raise ValueError(f"generated-block sentinels not found in order; expected\n{BEGIN}\n...\n{END}") + return text[: start + len(BEGIN)] + "\n\n" + body + "\n" + text[end:] + + +def _workflow_path(workflow: str) -> Path: + for suffix in (".yaml", ".yml"): + candidate = WORKFLOW_DIR / f"{workflow}{suffix}" + if candidate.exists(): + return candidate + raise FileNotFoundError(f"no workflow file for {workflow!r}") + + +def _generated_workflows() -> list[str]: + return sorted({job.workflow for job in testplan.load_plan() if job.generate}) + + +def write() -> list[str]: + """Rewrite the generated blocks. Returns the workflow files that changed.""" + changed = [] + for workflow in _generated_workflows(): + path = _workflow_path(workflow) + text = path.read_text(encoding="utf-8") + updated = _splice(text, render(workflow)) + if updated != text: + path.write_text(updated, encoding="utf-8") + changed.append(path.name) + return changed + + +def check() -> list[str]: + """Return the workflow files whose generated blocks are out of date.""" + stale = [] + for workflow in _generated_workflows(): + path = _workflow_path(workflow) + text = path.read_text(encoding="utf-8") + if _splice(text, render(workflow)) != text: + stale.append(path.name) + return stale + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--check", action="store_true", help="report drift instead of rewriting") + args = parser.parse_args(argv) + + if args.check: + stale = check() + for name in stale: + print(f"out of date: {name}") + return 1 if stale else 0 + + changed = write() + for name in changed: + print(f"updated: {name}") + if not changed: + print("already up to date") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/hang_dump.py b/tools/hang_dump.py index ad519be57c59..3468575b9455 100644 --- a/tools/hang_dump.py +++ b/tools/hang_dump.py @@ -6,7 +6,7 @@ """On-demand stack dump for a test process the CI runner believes is hung. A test that crashes reports a traceback, because ``PYTHONFAULTHANDLER=1`` (set per test file in -``tools/conftest.py``) installs ``faulthandler`` for ``SIGSEGV`` and friends. A test that *hangs* reported +``tools/run_tests.py``) installs ``faulthandler`` for ``SIGSEGV`` and friends. A test that *hangs* reported nothing: the runner detects the hang and kills the process group with ``SIGKILL``, which cannot be caught, so no handler ever ran. This module closes that gap by giving the runner a signal to ask for a stack first. @@ -41,7 +41,7 @@ DUMP_SIGNAL = getattr(signal, "SIGUSR1", None) """Signal the CI runner sends to ask a hung test process for a stack dump. -``None`` off POSIX. ``tools/conftest.py`` reads this so the sender and the receiver cannot disagree. +``None`` off POSIX. ``tools/run_tests.py`` reads this so the sender and the receiver cannot disagree. """ DUMP_PATH_ENV_VAR = "ISAACLAB_HANG_DUMP" diff --git a/tools/ovrtx_log.py b/tools/ovrtx_log.py index 32fe6e3a0422..aaf36a819fa0 100644 --- a/tools/ovrtx_log.py +++ b/tools/ovrtx_log.py @@ -14,7 +14,7 @@ * loaded as a pytest plugin by the repo-root ``conftest.py``, it claims the log for the test process and replays each test's share of it into pytest's capture, where a failing test's report picks it up; -* imported by ``tools/conftest.py``, it quotes a bounded tail in the crash, hang, or timeout report of a +* imported by ``tools/run_tests.py``, it quotes a bounded tail in the crash, hang, or timeout report of a process that died before it could replay anything. Both readers are needed: the replay attributes output to the test that produced it and stays out of the job @@ -25,7 +25,7 @@ the log alone. So when :data:`LOG_DIR_ENV_VAR` names a directory, everything a test left in the renderer's own directory -- its own share of the log, uncapped, and any dump beside it -- is additionally saved there for CI to upload as an artifact, which is what a diagnosis reads when the quoted tail is not enough. The -test that a crash, hang, or timeout killed never reaches the fixture that saves, so ``tools/conftest.py`` +test that a crash, hang, or timeout killed never reaches the fixture that saves, so ``tools/run_tests.py`` saves what that process left behind on its behalf -- the one test in the run whose output would otherwise be missing from the artifact is the one the artifact exists for. """ @@ -57,7 +57,7 @@ LOG_DIR_ENV_VAR = "ISAACLAB_OVRTX_LOG_DIR" """Environment variable naming the directory each test's renderer output is saved under, uncapped. -Set per pytest invocation by ``tools/conftest.py``, to a directory under ``tests/`` that CI collects as a +Set per pytest invocation by ``tools/run_tests.py``, to a directory under ``tests/`` that CI collects as a job artifact. Unset by default, which leaves a local run writing nothing beyond the replay. """ @@ -172,7 +172,7 @@ def pytest_configure(config): Whatever is at the path belongs to a session that has ended, so it is dropped rather than reasoned about: the byte offsets recorded per test below are only meaningful against this session's own output. The log is left in place at session end instead, since a crashed run is diagnosed by reading it after - the process is gone; ``tools/conftest.py`` clears it before starting the next one. + the process is gone; ``tools/run_tests.py`` clears it before starting the next one. """ with contextlib.suppress(OSError): os.remove(LOG_PATH) @@ -191,7 +191,7 @@ def _echo_ovrtx_log(request): suppressed so that the reverse cannot happen either: this runs in the teardown of every test that rendered, so a full or unwritable artifact directory would otherwise turn each of them into an error and take the replay below down with it. An artifact matters less than the test result and the replay, - as it does to ``_make_crash_pass_result`` in ``tools/conftest.py``. + as it does to ``_make_crash_pass_result`` in ``tools/run_tests.py``. """ label = request.node.name diff --git a/tools/conftest.py b/tools/run_tests.py similarity index 80% rename from tools/conftest.py rename to tools/run_tests.py index 02511473ddf3..b06a3d28b98a 100644 --- a/tools/conftest.py +++ b/tools/run_tests.py @@ -3,6 +3,33 @@ # # SPDX-License-Identifier: BSD-3-Clause +"""Run Isaac Lab's tests, locally or in CI. + +``tools/test_plan.toml`` says what each job covers and :mod:`testplan` resolves it to a file +list; this module executes that list. It owns everything that makes a long test run +survivable -- per-file timeouts, startup-hang detection, stack dumps from a wedged process, +crash reports for a process that died before writing one, retries, the work queue, and the +merged JUnit output -- none of which pytest does on its own. + +Files declaring a launch marker (see :mod:`isaaclab.test.kit`) are handed to pytest together +so they share one Kit app; everything else gets a process to itself. That grouping is the +normal schedule rather than a special case, because sharing a process is the only way a +directory of Kit-dependent files boots Kit once instead of once per file. + +This was ``tools/conftest.py``, which disabled collection and did all of the above from +``pytest_sessionstart``. Being a conftest meant it hijacked any pytest run rooted at the +repository, which is why ``tools-tests.yml`` had to pass ``--noconftest`` to run the tools' +own tests. It is a script now:: + + python tools/run_tests.py --list-jobs + python tools/run_tests.py --job isaaclab-core --shard 0 + python tools/run_tests.py source/isaaclab/test/sim + python tools/run_tests.py --all +""" + +from __future__ import annotations + +import argparse import contextlib import logging import os @@ -18,22 +45,34 @@ from junitparser import Error, JUnitXml, TestCase, TestSuite from prettytable import PrettyTable +from isaaclab.test.kit import kit_marker, module_markers from isaaclab.test.utils import resolve_test_sim_device # Local imports import hang_dump # isort: skip import ovrtx_log # isort: skip import test_settings as test_settings # isort: skip +import testplan # isort: skip from crash_journal import JOURNAL_ENV_VAR, create_crash_report # isort: skip from _device_split import DEVICE_SPLIT_PASSES, is_device_split_file # isort: skip +from _kit_batching import ( # isort: skip + BATCH_TIMEOUT_CUTOFF, + batch_size, + batching_enabled, + group_test_files, + split_batch_status, +) logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) -def pytest_ignore_collect(collection_path, config): - # Skip collection and run each test script individually - return True +class TestRunError(Exception): + """A run could not be set up. Carries the exit code the process should end with.""" + + def __init__(self, message: str, returncode: int = 1): + super().__init__(message) + self.returncode = returncode COLD_CACHE_BUFFER = 700 @@ -45,6 +84,22 @@ def pytest_ignore_collect(collection_path, config): on-disk cache is populated. """ + +def _enables_cameras(test_content: str) -> bool: + """Whether the given test file's source starts Kit with cameras enabled. + + Decided from the file's text rather than by importing it, because importing a + Kit-dependent test module boots Kit. A file either declares ``kit_cameras`` and lets + :mod:`isaaclab.test.kit` launch for it, or still constructs ``AppLauncher`` itself. + """ + try: + if kit_marker(test_content) == "kit_cameras": + return True + except ValueError: + pass # contradictory markers; the file fails at collection and the contract test says why + return "enable_cameras=True" in test_content + + STARTUP_DEADLINE = 120 """Seconds to wait for AppLauncher init or pytest collection before declaring a startup hang. @@ -1338,7 +1393,7 @@ def run_individual_tests(test_files, workspace_root, ci_marker, test_node_ids_by # The first camera-enabled test in a fresh container compiles shaders # (~600 s). Give it extra time so that doesn't look like a test timeout. - is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content + is_cold_cache_test = not cold_cache_applied and _enables_cameras(test_content) if is_cold_cache_test: timeout += COLD_CACHE_BUFFER cold_cache_applied = True @@ -1401,76 +1456,109 @@ def run_individual_tests(test_files, workspace_root, ci_marker, test_node_ids_by return failed_tests, test_status, xml_reports -def _collect_test_files( - source_dirs, - filter_pattern, - exclude_pattern, - include_files, - quarantined_only, - curobo_only, -): - """Collect test files from source directories, applying all active filters.""" - test_files = [] - for source_dir in source_dirs: - if not os.path.exists(source_dir): - logger.error(f"Error: source directory not found at {source_dir}") - pytest.exit("Source directory not found", returncode=1) - - for root, _, files in os.walk(source_dir): - # source/isaaclab/test/install_ci/ has its own pytest config and conftest. - # It is run via .github/actions/install-ci-run, never via this collector, - # so skip the whole subtree to keep install_ci tests out of build.yaml jobs. - if "install_ci" in root.replace("\\", "/").split("/"): +def _batching_exclusions(test_files, test_node_ids_by_file, sources): + """Files that must keep a process to themselves even when batching is on. + + Batching only changes how files are grouped, so anything whose current behaviour depends + on process isolation, on its own timeout, or on being invoked more than once is left on + the per-file path. + """ + excluded = set() + for path in test_files: + name = os.path.basename(path) + source = sources.get(path, "") + if name in PROCESS_FAILURE_RETRIES_BY_FILE: + excluded.add(path) # retried in a fresh process after stale render state + elif os.path.normpath(path) in test_node_ids_by_file: + excluded.add(path) # node-ID selection is expressed per file + elif is_device_split_file(path, source=source): + excluded.add(path) # already invoked once per device with different -k + elif test_settings.PER_TEST_TIMEOUTS.get(name, 0) >= BATCH_TIMEOUT_CUTOFF: + excluded.add(path) # a batch timeout is the sum of its members' + elif name in getattr(test_settings, "NEVER_BATCH", ()): + excluded.add(path) + return excluded + + +def run_batched_tests(batches, workspace_root, ci_marker, cold_cache_applied=False): + """Run each batch as a single pytest invocation and split the results per file. + + Args: + batches: Batches to run; each must contain more than one file. + workspace_root: Repository root, passed to pytest's ``--config-file``. + ci_marker: Optional marker expression applied to every invocation. + cold_cache_applied: Whether the cold-shader-cache buffer was already granted. + + Returns: + A 4-tuple ``(failed_tests, test_status, xml_reports, leftovers)``. ``leftovers`` are + files the batch never reached because the shared process died; the caller re-runs + them on the per-file path, which is the floor this can degrade to. + """ + failed_tests, test_status, xml_reports, leftovers = [], {}, [], [] + global_k_expr = os.environ.get("TEST_K_EXPR", "").strip() or None + + for batch in batches: + logger.info(f"\n\nšŸš€ Running {len(batch.files)} '{batch.profile}' files in one Kit process...\n") + for path in batch.files: + logger.info(f" {path}") + + env = os.environ.copy() + env["PYTHONFAULTHANDLER"] = "1" + + # A batch's budget is the sum of its members', so no file gets less time than it + # would have had alone. + timeout = sum( + test_settings.PER_TEST_TIMEOUTS.get(os.path.basename(p), test_settings.DEFAULT_TIMEOUT) for p in batch.files + ) + is_cold_cache = not cold_cache_applied and batch.profile == "kit_cameras" + if is_cold_cache: + timeout += COLD_CACHE_BUFFER + cold_cache_applied = True + logger.info(f"ā±ļø Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") + startup_deadline = min(timeout, STARTUP_DEADLINE + (COLD_CACHE_BUFFER if is_cold_cache else 0)) + + ctx = _PassContext( + test_file=batch.label, + file_name=batch.label, + workspace_root=workspace_root, + ci_marker=ci_marker, + timeout=timeout, + startup_deadline=startup_deadline, + env=env, + inject_shard_select=False, + pytest_targets=list(batch.files), + ) + + report, status, _ = _run_one_pass(ctx, k_expr=global_k_expr, suffix="") + if report is not None: + xml_reports.append(report) + + if report is None: + # Nothing landed, so nothing can be attributed; hand the whole batch back. + logger.warning(f"āš ļø batch {batch.label} produced no report; re-running its files individually") + leftovers.extend(batch.files) + continue + + per_file = split_batch_status( + report, batch.files, wall_time=status.get("wall_time", 0.0), batch_result=status.get("result", "CRASHED") + ) + unreached = [] + for path, file_status in per_file.items(): + if file_status["result"] in ("CRASHED", "TIMEOUT", "STARTUP_HANG"): + unreached.append(path) continue + test_status[path] = file_status + if file_status["result"] == "FAILED": + failed_tests.append(path) - for file in files: - if not (file.startswith("test_") and file.endswith(".py")): - continue - - # Mode-exclusive filters (each bypasses TESTS_TO_SKIP) - if quarantined_only: - if file not in test_settings.QUARANTINED_TESTS: - continue - elif curobo_only: - if file not in test_settings.CUROBO_TESTS: - continue - else: - # An explicit include_files entry overrides TESTS_TO_SKIP, allowing - # dedicated jobs (e.g. test-environments-training) to run tests that - # are otherwise excluded from general CI runs. - if file in test_settings.TESTS_TO_SKIP and file not in include_files: - logger.debug(f"Skipping {file} as it's in the skip list") - continue - - full_path = os.path.join(root, file) - - if filter_pattern and filter_pattern not in full_path: - logger.debug(f"Skipping {full_path} (does not match include pattern: {filter_pattern})") - continue - if exclude_pattern and any(p.strip() in full_path for p in exclude_pattern.split(",")): - logger.debug(f"Skipping {full_path} (matches exclude pattern: {exclude_pattern})") - continue - if include_files and file not in include_files: - logger.debug(f"Skipping {full_path} (not in include files list)") - continue - - test_files.append(full_path) - - # Sort test files deterministically to ensure consistent test ordering. - test_files.sort() - - # Apply file-level sharding: select every Nth file from the deterministic order. - # Skip when include_files is set ā€” in that case the test's own conftest handles - # sharding at the test-item level (e.g. parametrized test cases). - shard_index = os.environ.get("TEST_SHARD_INDEX", "") - shard_count = os.environ.get("TEST_SHARD_COUNT", "") - if shard_index and shard_count and not include_files: - shard_index = int(shard_index) - shard_count = int(shard_count) - test_files = [f for i, f in enumerate(test_files) if i % shard_count == shard_index] - logger.info(f"Shard {shard_index}/{shard_count}: selected {len(test_files)} test files") - - return test_files + if unreached: + logger.warning( + f"āš ļø batch {batch.label} ended at {unreached[0]} ({status.get('result')});" + f" re-running {len(unreached)} remaining file(s) individually" + ) + leftovers.extend(unreached) + + return failed_tests, test_status, xml_reports, leftovers def _load_test_node_ids_from_toml(workspace_root: str) -> list[str]: @@ -1480,7 +1568,7 @@ def _load_test_node_ids_from_toml(workspace_root: str) -> list[str]: if not (node_ids_file or node_ids_key): return [] if not (node_ids_file and node_ids_key): - pytest.exit("Both TEST_NODE_IDS_FILE and TEST_NODE_IDS_KEY must be set together", returncode=1) + raise TestRunError("Both TEST_NODE_IDS_FILE and TEST_NODE_IDS_KEY must be set together", 1) path = node_ids_file if os.path.isabs(node_ids_file) else os.path.join(workspace_root, node_ids_file) @@ -1488,14 +1576,14 @@ def _load_test_node_ids_from_toml(workspace_root: str) -> list[str]: with open(os.path.normpath(path), "rb") as stream: node_ids = tomllib.load(stream).get(node_ids_key) except OSError as exc: - pytest.exit(f"Could not read TEST_NODE_IDS_FILE {node_ids_file!r}: {exc}", returncode=1) + raise TestRunError(f"Could not read TEST_NODE_IDS_FILE {node_ids_file!r}: {exc}", 1) except tomllib.TOMLDecodeError as exc: - pytest.exit(f"{node_ids_file}: invalid TOML: {exc}", returncode=1) + raise TestRunError(f"{node_ids_file}: invalid TOML: {exc}", 1) if not node_ids: - pytest.exit(f"{node_ids_key!r} not found or empty in {node_ids_file}", returncode=1) + raise TestRunError(f"{node_ids_key!r} not found or empty in {node_ids_file}", 1) if not isinstance(node_ids, list) or not all(isinstance(node_id, str) for node_id in node_ids): - pytest.exit(f"{node_ids_key!r} must be a TOML array of strings in {node_ids_file}", returncode=1) + raise TestRunError(f"{node_ids_key!r} must be a TOML array of strings in {node_ids_file}", 1) return node_ids @@ -1505,13 +1593,13 @@ def _collect_test_node_ids_by_file(workspace_root: str) -> dict[str, list[str]]: node_ids = [line.strip() for line in os.environ.get("TEST_NODE_IDS", "").splitlines() if line.strip()] node_ids.extend(_load_test_node_ids_from_toml(workspace_root)) if len(node_ids) != len(set(node_ids)): - pytest.exit("Configured test node IDs contain duplicates", returncode=1) + raise TestRunError("Configured test node IDs contain duplicates", 1) grouped: dict[str, list[str]] = {} for node_id in node_ids: normalized_node_id = node_id.replace("\\", "/") if "::" not in normalized_node_id: - pytest.exit(f"Configured test node ID must include '::': {node_id}", returncode=1) + raise TestRunError(f"Configured test node ID must include '::': {node_id}", 1) file_part, test_part = normalized_node_id.split("::", 1) if os.path.isabs(file_part): @@ -1520,7 +1608,7 @@ def _collect_test_node_ids_by_file(workspace_root: str) -> dict[str, list[str]]: abs_file = os.path.normpath(os.path.join(workspace_root, file_part)) if not os.path.exists(abs_file): - pytest.exit(f"Configured test node ID file does not exist: {node_id}", returncode=1) + raise TestRunError(f"Configured test node ID file does not exist: {node_id}", 1) grouped.setdefault(abs_file, []).append(f"{normalized_node_id.split('::', 1)[0]}::{test_part}") @@ -1564,134 +1652,124 @@ def _format_test_file_results(test_files: list[str], test_status: dict[str, dict return summary + table.get_string() -def pytest_sessionstart(session): - """Intercept pytest startup to execute tests in the correct order.""" - # Get the workspace root directory (one level up from tools) - workspace_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - source_dirs = [ - os.path.join(workspace_root, "scripts"), - os.path.join(workspace_root, "source"), - ] +def _resolve_selection(args) -> tuple[list[str], str, str | None]: + """Turn the command line into the files to run, the ``-m`` marker and the ``-k`` expression. - # Get filter pattern from environment variable or command line - filter_pattern = os.environ.get("TEST_FILTER_PATTERN", "") - exclude_pattern = os.environ.get("TEST_EXCLUDE_PATTERN", "") - include_files_str = os.environ.get("TEST_INCLUDE_FILES", "") - quarantined_only = os.environ.get("TEST_QUARANTINED_ONLY", "false") == "true" - curobo_only = os.environ.get("TEST_CUROBO_ONLY", "false") == "true" - - isaacsim_ci = os.environ.get("ISAACSIM_CI_SHORT", "false") == "true" - - # CI_MARKER env var is a separate, parallel mechanism for cross-platform - # jobs (arm-ci, windows-ci, ...) to reuse this orchestrator with their own - # markers. Deliberately NOT aliased to ISAACSIM_CI_SHORT: the isaacsim_ci - # filter is owned by Isaac Sim's external CI pipeline; the CI_MARKER path - # leaves that contract untouched. - ci_marker = os.environ.get("CI_MARKER", "") - test_node_ids_by_file = _collect_test_node_ids_by_file(workspace_root) + Args: + args: Parsed command-line arguments. - # Parse include files list (comma-separated paths) - include_files = set() - if include_files_str: - for f in include_files_str.split(","): - f = f.strip() - if f: - include_files.add(os.path.basename(f)) - include_files.update(os.path.basename(path) for path in test_node_ids_by_file) - - # Also try to get from pytest config - if hasattr(session.config, "option") and hasattr(session.config.option, "filter_pattern"): - filter_pattern = filter_pattern or getattr(session.config.option, "filter_pattern", "") - if hasattr(session.config, "option") and hasattr(session.config.option, "exclude_pattern"): - exclude_pattern = exclude_pattern or getattr(session.config.option, "exclude_pattern", "") - - logger.debug("=" * 50) - logger.debug("CONFTEST.PY DEBUG INFO") - logger.debug("=" * 50) - logger.debug(f"Filter pattern: '{filter_pattern}'") - logger.debug(f"Exclude pattern: '{exclude_pattern}'") - logger.debug(f"Include files: {include_files if include_files else 'none'}") - logger.debug(f"Test node IDs: {sum(len(node_ids) for node_ids in test_node_ids_by_file.values())}") - logger.debug(f"Quarantined-only mode: {quarantined_only}") - logger.debug(f"Curobo-only mode: {curobo_only}") - logger.debug(f"TEST_FILTER_PATTERN env var: '{os.environ.get('TEST_FILTER_PATTERN', 'NOT_SET')}'") - logger.debug(f"TEST_EXCLUDE_PATTERN env var: '{os.environ.get('TEST_EXCLUDE_PATTERN', 'NOT_SET')}'") - logger.debug(f"TEST_INCLUDE_FILES env var: '{os.environ.get('TEST_INCLUDE_FILES', 'NOT_SET')}'") - logger.debug(f"TEST_NODE_IDS env var: '{'SET' if os.environ.get('TEST_NODE_IDS') else 'NOT_SET'}'") - logger.debug(f"TEST_NODE_IDS_FILE env var: '{os.environ.get('TEST_NODE_IDS_FILE', 'NOT_SET')}'") - logger.debug(f"TEST_NODE_IDS_KEY env var: '{os.environ.get('TEST_NODE_IDS_KEY', 'NOT_SET')}'") - logger.debug(f"TEST_QUARANTINED_ONLY env var: '{os.environ.get('TEST_QUARANTINED_ONLY', 'NOT_SET')}'") - logger.debug(f"TEST_CUROBO_ONLY env var: '{os.environ.get('TEST_CUROBO_ONLY', 'NOT_SET')}'") - logger.debug("=" * 50) - - # Get all test files in the source directories - test_files = _collect_test_files( - source_dirs, - filter_pattern, - exclude_pattern, - include_files, - quarantined_only, - curobo_only, - ) + Returns: + ``(test_files, marker, k_expr)``; ``test_files`` are absolute paths. - if isaacsim_ci: - new_test_files = [] - for test_file in test_files: - with open(test_file) as f: - if "@pytest.mark.isaacsim_ci" in f.read(): - new_test_files.append(test_file) - test_files = new_test_files - - if ci_marker: - # Match both `@pytest.mark.` (per-function) and - # `pytestmark = pytest.mark.` / `pytestmark = [..., pytest.mark., ...]` - # (module-level) by looking for the common `pytest.mark.` substring. - marker_token = f"pytest.mark.{ci_marker}" - new_test_files = [] - for test_file in test_files: - try: - with open(test_file) as f: - if marker_token in f.read(): - new_test_files.append(test_file) - except OSError as exc: - raise RuntimeError( - f"ci_marker post-scan could not read {test_file}; refusing to" - f" silently drop a potentially marker-tagged file" - ) from exc - test_files = new_test_files + Raises: + TestRunError: If the selection names nothing runnable. + """ + root = testplan.REPO_ROOT + + if args.job: + job = testplan.get_job(args.job) + relative = testplan.resolve(job, shard=args.shard) + marker, k_expr = job.marker or "", job.k_expr + else: + paths = args.paths or (["source", "scripts"] if args.all else None) + if not paths: + raise TestRunError("nothing selected: pass --job, --all, or one or more paths", 2) + job = testplan.Job(name="ad-hoc", title="ad-hoc", workflow="", paths=tuple(paths)) + relative = testplan.resolve(job) + marker, k_expr = "", None + + # ISAACSIM_CI_SHORT is Isaac Sim's external pipeline asking for its own subset. It is a + # separate contract from the plan, so it narrows whatever the job selected rather than + # replacing it, and the job's own marker wins when both are present -- `-m` takes one + # expression. + if os.environ.get("ISAACSIM_CI_SHORT", "false") == "true": + relative = [ + path for path in relative if "isaacsim_ci" in module_markers((root / path).read_text(errors="replace")) + ] + marker = marker or "isaacsim_ci" + + if args.k_expr is not None: + k_expr = args.k_expr + + test_files = [str(root / path) for path in relative] + if not test_files: + _write_empty_report() + raise TestRunError(f"no test files selected for {args.job or 'the given paths'}", 0) + return test_files, marker, k_expr + + +def run(args) -> int: + """Resolve the selection, run it, and report. + + Args: + args: Parsed command-line arguments. + + Returns: + The process exit code; see :data:`EXIT_CODE_LABELS`. + + Raises: + TestRunError: If the run could not be set up. + """ + workspace_root = str(testplan.REPO_ROOT) + test_files, effective_marker, k_expr = _resolve_selection(args) + test_node_ids_by_file = _collect_test_node_ids_by_file(workspace_root) if test_node_ids_by_file: configured_files = set(test_node_ids_by_file) test_files = [test_file for test_file in test_files if os.path.normpath(test_file) in configured_files] missing_files = sorted(configured_files - {os.path.normpath(test_file) for test_file in test_files}) if missing_files: - pytest.exit(f"Configured test node ID files were not collected: {missing_files}", returncode=1) + raise TestRunError(f"Configured test node ID files were not collected: {missing_files}", 1) - if not test_files: - if quarantined_only: - logger.info("No quarantined tests configured ā€” nothing to run.") - _write_empty_report() - pytest.exit("No quarantined tests configured", returncode=0) - if filter_pattern: - logger.info(f"No test files found matching filter pattern '{filter_pattern}' ā€” nothing to run.") - _write_empty_report() - pytest.exit("No test files found for filter", returncode=0) - logger.warning("No test files found in source directory") - pytest.exit("No test files found", returncode=1) - - logger.info(f"Found {len(test_files)} test files after filtering") + if k_expr: + os.environ["TEST_K_EXPR"] = k_expr + + logger.info(f"Found {len(test_files)} test files") for test_file in test_files: node_ids = test_node_ids_by_file.get(os.path.normpath(test_file), []) if test_node_ids_by_file else [] suffix = f" ({', '.join(node_ids)})" if node_ids else "" - logger.info(f" - {test_file}{suffix}") + logger.info(f" - {os.path.relpath(test_file, workspace_root)}{suffix}") + + # Files that declare a launch marker share one Kit app when they land in the same process, + # so group them and pay startup once per group instead of once per file. Unmarked files -- + # still most of the suite -- keep a process each. Disabled by ISAACLAB_TEST_BATCH_KIT=0, and + # under the work queue, which hands out files one at a time across containers and so cannot + # offer coherent groups. + batched_files, batch_results = [], ([], {}, []) + if batching_enabled() and not os.environ.get("ISAACLAB_TEST_QUEUE"): + sources = {} + for path in test_files: + try: + with open(path) as fh: + sources[path] = fh.read() + except OSError: + pass # left out of `sources`, which group_test_files treats as unbatchable + batches = group_test_files( + test_files, + sources, + unbatchable=_batching_exclusions(test_files, test_node_ids_by_file, sources), + max_size=batch_size(), + ) + multi = [b for b in batches if b.is_batched] + if multi: + batched_files = [f for b in multi for f in b.files] + logger.info( + f"āš” Kit batching: {len(batched_files)} of {len(test_files)} files grouped into" + f" {len(multi)} process(es); the rest run individually" + ) + failed, status, reports, leftovers = run_batched_tests(multi, workspace_root, effective_marker) + batch_results = (failed, status, reports) + # Files a batch never reached fall back to the per-file path, so batching can + # never do worse than the behaviour it replaces. + batched_files = [f for f in batched_files if f not in leftovers] - # Run all tests individually. CI_MARKER takes precedence when both env - # vars are set; falls back to "isaacsim_ci" when only ISAACSIM_CI_SHORT - # is set. The pytest -m flag only accepts one expression. - effective_marker = ci_marker or ("isaacsim_ci" if isaacsim_ci else "") + remaining = [f for f in test_files if f not in batched_files] failed_tests, test_status, xml_reports = run_individual_tests( - test_files, workspace_root, effective_marker, test_node_ids_by_file + remaining, workspace_root, effective_marker, test_node_ids_by_file ) + failed_tests = batch_results[0] + failed_tests + test_status = {**batch_results[1], **test_status} + xml_reports = batch_results[2] + xml_reports # In work-queue mode this container ran only the files it claimed; report on those. if os.environ.get("ISAACLAB_TEST_QUEUE"): @@ -1767,5 +1845,64 @@ def pytest_sessionstart(session): # Print summary to console and log file logger.info(summary_str) - # Exit pytest after custom execution to prevent normal pytest from overwriting our report - pytest.exit("Custom test execution completed", returncode=exit_code) + return exit_code + + +def _build_parser() -> argparse.ArgumentParser: + """Build the command-line interface.""" + parser = argparse.ArgumentParser( + prog="run_tests.py", + description=__doc__.splitlines()[0], + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=( + "examples:\n" + " python tools/run_tests.py --list-jobs\n" + " python tools/run_tests.py --job isaaclab-core --shard 0\n" + " python tools/run_tests.py source/isaaclab/test/sim\n" + " python tools/run_tests.py --all\n" + ), + ) + what = parser.add_mutually_exclusive_group() + what.add_argument("--job", help="run a job from tools/test_plan.toml") + what.add_argument("--all", action="store_true", help="run every test file under source/ and scripts/") + parser.add_argument("paths", nargs="*", help="directories to walk for test_*.py") + parser.add_argument("--shard", type=int, help="which shard of a sharded job to run, 0-based") + parser.add_argument("-k", dest="k_expr", help="pytest -k expression; overrides the job's own") + parser.add_argument("--list-jobs", action="store_true", help="list the jobs in the test plan and exit") + parser.add_argument("--list-files", action="store_true", help="print the resolved file list and exit") + return parser + + +def main(argv: list[str] | None = None) -> int: + """Entry point. + + Args: + argv: Command-line arguments; defaults to ``sys.argv[1:]``. + + Returns: + The process exit code. + """ + args = _build_parser().parse_args(argv) + + if args.list_jobs: + rows = PrettyTable(["job", "workflow", "shards", "files"]) + rows.align = "l" + for job in testplan.load_plan(): + rows.add_row([job.name, job.workflow, job.shards or 1, len(testplan.resolve(job))]) + print(rows) + return 0 + + try: + if args.list_files: + test_files, _, _ = _resolve_selection(args) + print("\n".join(os.path.relpath(path, testplan.REPO_ROOT) for path in test_files)) + return 0 + return run(args) + except TestRunError as exc: + level = logger.info if exc.returncode == 0 else logger.error + level(str(exc)) + return exc.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/skills/pyproject.toml b/tools/skills/pyproject.toml index 1197a9bd1189..552a8c3f5169 100644 --- a/tools/skills/pyproject.toml +++ b/tools/skills/pyproject.toml @@ -3,7 +3,7 @@ # # 1. ``pythonpath = ["."]`` adds ``tools/skills/`` to ``sys.path``, # making ``import cli`` work from the test files without any shim. -# 2. ``tools/conftest.py`` sits above rootdir and is therefore not loaded. +# 2. ``tools/run_tests.py`` sits above rootdir and is therefore not loaded. # # Run with: ``uv run python -m pytest tools/skills/`` [tool.pytest.ini_options] diff --git a/tools/test/test_test_plan.py b/tools/test/test_test_plan.py new file mode 100644 index 000000000000..bf0a77ba437f --- /dev/null +++ b/tools/test/test_test_plan.py @@ -0,0 +1,138 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the test plan, the workflow generator, and the CI argument plumbing. + +The plan is the single source of truth for what every CI job runs, so the things that can go +quietly wrong are: a workflow naming a job the plan does not define, checked-in YAML drifting +from what the generator produces, a job resolving to nothing, and the positional arguments +``run-tests/action.yml`` passes to ``run_tests.sh`` sliding out of alignment with the ``local`` +bindings at the top of that script. Each has a test here. +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "tools")) +for _package in sorted((REPO_ROOT / "source").iterdir()): + if (_package / _package.name).is_dir(): + sys.path.insert(0, str(_package)) + +import generate_workflows # noqa: E402 +import testplan # noqa: E402 + +pytestmark = pytest.mark.unit + +_RUN_TESTS_SH = REPO_ROOT / ".github/actions/run-tests/run_tests.sh" +_RUN_TESTS_ACTION = REPO_ROOT / ".github/actions/run-tests/action.yml" + + +@pytest.fixture(scope="module") +def plan() -> list[testplan.Job]: + return testplan.load_plan() + + +def test_every_job_resolves_to_at_least_one_file(plan: list[testplan.Job]): + """A job that selects nothing is a lane that silently tests nothing.""" + empty = [job.name for job in plan if not testplan.resolve(job)] + assert not empty, "these jobs resolve to no test files:\n " + "\n ".join(empty) + + +def test_shards_cover_the_job_exactly_once(plan: list[testplan.Job]): + """Every file in a sharded job runs in exactly one shard.""" + for job in plan: + if job.shards <= 1: + continue + whole = testplan.resolve(job) + pieces = [path for shard in range(job.shards) for path in testplan.resolve(job, shard=shard)] + assert sorted(pieces) == whole, f"{job.name}: shards do not partition the job" + assert len(pieces) == len(set(pieces)), f"{job.name}: a file appears in more than one shard" + + +def test_workflows_only_reference_jobs_the_plan_defines(plan: list[testplan.Job]): + """A workflow naming an unknown job fails at run time, in CI, after an image build.""" + known = {job.name for job in plan} + offenders = [] + for path in sorted((REPO_ROOT / ".github/workflows").glob("*.y*ml")): + data = yaml.safe_load(path.read_text(encoding="utf-8")) + for job_id, definition in (data.get("jobs") or {}).items(): + for step in definition.get("steps") or []: + name = (step.get("with") or {}).get("job") + if name and name not in known: + offenders.append(f"{path.name}::{job_id} -> {name!r}") + assert not offenders, "these workflow steps name a job absent from the test plan:\n " + "\n ".join(offenders) + + +def test_generated_workflow_blocks_are_up_to_date(): + """The checked-in YAML must match what the generator produces. + + Editing a generated block by hand is silently undone by the next regeneration, so the + mismatch has to fail here instead. + """ + stale = generate_workflows.check() + assert not stale, ( + "these workflow files are out of date with tools/test_plan.toml:\n " + + "\n ".join(stale) + + "\n\nFix: run `python tools/generate_workflows.py`." + ) + + +def _shell_bindings() -> list[str]: + """Return run_tests.sh's positional bindings, ordered by position.""" + text = _RUN_TESTS_SH.read_text(encoding="utf-8") + found = {} + for name, position in re.findall(r'^ local ([a-z_]+)="\$\{?(\d+)\}?"', text, re.M): + found[int(position)] = name + return [found[i] for i in sorted(found)] + + +def _action_arguments() -> list[str]: + """Return the arguments the action passes to run_tests.sh, ordered.""" + text = _RUN_TESTS_ACTION.read_text(encoding="utf-8") + line = next(ln for ln in text.splitlines() if "run_tests.sh" in ln and "bash" in ln) + call = line.split("run_tests.sh", 1)[1] + names = [] + for raw in re.findall(r'"([^"]*)"', call): + match = re.search(r"inputs\.([a-z0-9-]+)", raw) + names.append(match.group(1).replace("-", "_") if match else raw.lstrip("$").lower()) + return names + + +def test_run_tests_sh_arguments_line_up_with_the_action(): + """A positional interface this long silently misbinds when one side is edited alone. + + Names are compared rather than counts: an off-by-one that happens to preserve the count + would otherwise pass while feeding, say, the container name in as the job. + """ + bindings = _shell_bindings() + arguments = _action_arguments() + assert len(bindings) == len(arguments), ( + f"run_tests.sh binds {len(bindings)} positional arguments but action.yml passes" + f" {len(arguments)}:\n binds: {bindings}\n passes: {arguments}" + ) + mismatched = [ + f"${i + 1}: script binds {b!r}, action passes {a!r}" + for i, (b, a) in enumerate(zip(bindings, arguments)) + # The action spells a few values as env vars (PYTEST_OPTIONS) or literals rather than + # `inputs.`; those are matched loosely on the shared stem. + if b not in a and a not in b + ] + assert not mismatched, "run_tests.sh and action.yml disagree on argument order:\n " + "\n ".join(mismatched) + + +@pytest.mark.skipif(shutil.which("bash") is None, reason="needs bash to parse the script") +def test_run_tests_sh_is_valid_shell(): + """``bash -n`` catches the quoting mistakes that are easy to make editing this by hand.""" + result = subprocess.run(["bash", "-n", str(_RUN_TESTS_SH)], capture_output=True, text=True, timeout=60) + assert result.returncode == 0, result.stderr diff --git a/tools/test_crash_journal.py b/tools/test_crash_journal.py index 86eb6531c1c8..88b8b7bcd9e8 100644 --- a/tools/test_crash_journal.py +++ b/tools/test_crash_journal.py @@ -406,7 +406,7 @@ def test_ok(): def test_deselected_tests_are_not_journaled_as_collected(tmp_path): """Regression test for a rebuilt report claiming tests that this pass never selected. - ``tools/conftest.py`` splits a run into passes selected by marker and device, so journaling + ``tools/run_tests.py`` splits a run into passes selected by marker and device, so journaling from ``pytest_collection_modifyitems`` ā€” which runs before pytest's own ``trylast`` deselection hook ā€” would record the other passes' tests too. A crash would then rebuild them as "not run" skips, inflating the counts and duplicating node IDs the sibling pass reported. @@ -614,7 +614,7 @@ def test_never_reached(): marks=pytest.mark.skipif(not _HAS_FLAKY, reason="the rerun this case needs is driven by the flaky plugin"), ), pytest.param( - # The startup-hang shape ``tools/conftest.py`` guards against: collection has finished + # The startup-hang shape ``tools/run_tests.py`` guards against: collection has finished # journaling by the time the run loop starts, so this kills the session in the window where # the journal knows every test but none has a verdict. They must come back as "not run" # rather than disappear from the uploaded results, which would silently shrink the suite. diff --git a/tools/test_plan.toml b/tools/test_plan.toml new file mode 100644 index 000000000000..e0e5ea1447a4 --- /dev/null +++ b/tools/test_plan.toml @@ -0,0 +1,280 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# The test plan: one entry per CI job, and the single source of truth for what any run +# covers. `tools/run_tests.py` executes a job from here; `tools/generate_workflows.py` +# renders the workflow YAML from the same entries, so a job cannot exist in CI without +# being described here. See tools/testplan.py for the field semantics. +# +# Selection fields: +# paths directories to walk for test_*.py (required) +# exclude substrings; a file whose repo-relative path contains any of them is dropped +# files basenames; when set, only these files run, and they override TESTS_TO_SKIP +# shards split the resolved list across N jobs, round-robin by sorted position +# pool "quarantined" or "curobo" selects that list from tools/test_settings.py +# instead of walking `paths`, and bypasses TESTS_TO_SKIP +# +# Execution fields (k-expr, marker) narrow within a file rather than choosing files. + +# --------------------------------------------------------------------------- +# build.yaml -- the per-package lanes that run on every PR +# --------------------------------------------------------------------------- + +[[job]] +name = "isaaclab-tasks" +title = "isaaclab_tasks" +workflow = "build" +paths = ["source/isaaclab_tasks"] +exclude = ["test_rendering_", "test_video_recording.py"] +shards = 3 +container-name = "isaac-lab-tasks-test" +warp-cache = "restore" +generate = true +continue-on-error = true +extra-pip-packages = "pytetwild[all]>=0.3.0,<0.4" + +[[job]] +name = "isaaclab-core" +title = "isaaclab (core)" +workflow = "build" +# Everything that is not a source/isaaclab_* package: the core library plus the +# repo scripts. Walking the directories rather than matching "not isaaclab_" as a +# substring is what keeps `source/isaaclab` from also selecting `source/isaaclab_rl`. +paths = ["source/isaaclab", "scripts"] +shards = 3 +container-name = "isaac-lab-core-test" +warp-cache = "restore" +generate = true + +[[job]] +name = "isaaclab-rl" +title = "isaaclab_rl" +workflow = "build" +paths = ["source/isaaclab_rl"] +container-name = "isaac-lab-rl-test" +generate = true +extra-pip-packages = "leapp" + +[[job]] +name = "isaaclab-mimic" +title = "isaaclab_mimic" +workflow = "build" +paths = ["source/isaaclab_mimic"] +container-name = "isaac-lab-mimic-test" +generate = true + +[[job]] +name = "isaaclab-contrib" +title = "isaaclab_contrib" +workflow = "build" +paths = ["source/isaaclab_contrib"] +container-name = "isaac-lab-contrib-test" +generate = true +extra-pip-packages = "pytetwild[all]>=0.3.0,<0.4" + +[[job]] +name = "isaaclab-teleop" +title = "isaaclab_teleop" +workflow = "build" +paths = ["source/isaaclab_teleop"] +container-name = "isaac-lab-teleop-test" +generate = true + +[[job]] +name = "isaaclab-visualizers" +title = "isaaclab_visualizers" +workflow = "build" +paths = ["source/isaaclab_visualizers"] +container-name = "isaac-lab-visualizers-test" +generate = true + +[[job]] +name = "isaaclab-assets" +title = "isaaclab_assets" +workflow = "build" +paths = ["source/isaaclab_assets"] +container-name = "isaac-lab-assets-test" +generate = true + +[[job]] +name = "isaaclab-experimental" +title = "isaaclab_experimental" +workflow = "build" +paths = ["source/isaaclab_experimental"] +container-name = "isaac-lab-experimental-test" +generate = true + +[[job]] +name = "isaaclab-newton" +title = "isaaclab_newton" +workflow = "build" +paths = ["source/isaaclab_newton"] +container-name = "isaac-lab-newton-test" +generate = true +warp-cache = "restore" + +[[job]] +name = "isaaclab-physx" +title = "isaaclab_physx" +workflow = "build" +paths = ["source/isaaclab_physx"] +container-name = "isaac-lab-physx-test" +generate = true +extra-pip-packages = "pytetwild[all]>=0.3.0,<0.4" + +[[job]] +name = "isaaclab-ov" +title = "isaaclab_ov" +workflow = "build" +paths = ["source/isaaclab_ov"] +container-name = "isaac-lab-ov-test" + +# --------------------------------------------------------------------------- +# build.yaml -- targeted lanes selecting individual files +# --------------------------------------------------------------------------- + +[[job]] +name = "standalone-demos-kit" +title = "standalone demos (headless, Kit)" +workflow = "build" +paths = ["source", "scripts"] +files = ["test_standalone_scripts.py"] +container-name = "isaac-lab-standalone-kit-test" + +[[job]] +name = "standalone-demos-non-kit" +title = "standalone demos (headless, non-Kit)" +workflow = "build" +paths = ["source", "scripts"] +files = ["test_standalone_scripts.py"] +container-name = "isaac-lab-standalone-non-kit-test" + +[[job]] +name = "curobo" +title = "test-curobo" +workflow = "build" +paths = ["source", "scripts"] +files = ["test_curobo_planner_franka.py", "test_curobo_planner_cube_stack.py", "test_pink_ik.py"] +container-name = "isaac-lab-curobo-test" + +[[job]] +name = "contrib-environments" +title = "test-contrib-environments" +workflow = "build" +paths = ["source", "scripts"] +files = ["test_generate_dataset_skillgen.py", "test_contrib_environments.py"] +container-name = "isaac-lab-contrib-env-test" + +[[job]] +name = "record-video" +title = "record-video" +workflow = "build" +paths = ["source/isaaclab_tasks"] +files = ["test_video_recording.py"] +container-name = "isaac-lab-record-video-test" + +[[job]] +name = "rendering-correctness" +title = "rendering-correctness" +workflow = "build" +paths = ["source/isaaclab_tasks"] +files = [ + "test_rendering_cartpole.py", + "test_rendering_lift_kuka_hetero.py", + "test_rendering_lift_kuka_homo.py", + "test_rendering_franka_cloth.py", + "test_rendering_franka_soft.py", + "test_rendering_franka_cable.py", + "test_rendering_registered_tasks.py", + "test_rendering_shadow_hand.py", +] +node-ids-key = "rendering-correctness" +container-name = "isaac-lab-rendering-test" + +[[job]] +name = "rendering-correctness-kitless-legacy" +title = "rendering-correctness-kitless (legacy)" +workflow = "build" +paths = ["source/isaaclab_tasks"] +files = [ + "test_rendering_cartpole_kitless.py", + "test_rendering_lift_kuka_hetero_kitless.py", + "test_rendering_lift_kuka_homo_kitless.py", + "test_rendering_franka_cloth_kitless.py", + "test_rendering_franka_soft_kitless.py", + "test_rendering_franka_cable_kitless.py", + "test_rendering_shadow_hand_kitless.py", +] +node-ids-key = "rendering-correctness-kitless-legacy" +k-expr = "legacy" +container-name = "isaac-lab-rendering-kitless-legacy-test" + +[[job]] +name = "rendering-correctness-kitless-ovstage" +title = "rendering-correctness-kitless (ovstage)" +workflow = "build" +paths = ["source/isaaclab_tasks"] +files = [ + "test_rendering_cartpole_kitless.py", + "test_rendering_lift_kuka_hetero_kitless.py", + "test_rendering_lift_kuka_homo_kitless.py", + "test_rendering_franka_cloth_kitless.py", + "test_rendering_franka_soft_kitless.py", + "test_rendering_franka_cable_kitless.py", + "test_rendering_shadow_hand_kitless.py", +] +node-ids-key = "rendering-correctness-kitless-ovstage" +k-expr = "ovstage" +container-name = "isaac-lab-rendering-kitless-ovstage-test" + +[[job]] +name = "warp-cache-warm" +title = "warp-cache-warm" +workflow = "build" +paths = ["source/isaaclab_tasks"] +files = ["test_environments_newton.py", "test_multi_agent_environments.py"] +k-expr = "test_environments and not (Soft or Cloth or Cable)" +container-name = "isaac-lab-warp-warm" + +# --------------------------------------------------------------------------- +# daily-compatibility.yml +# --------------------------------------------------------------------------- + +[[job]] +name = "isaaclab-tasks-compat" +title = "test-isaaclab-tasks-compat" +workflow = "daily-compatibility" +paths = ["source/isaaclab_tasks"] +container-name = "isaac-lab-tasks-compat-test" + +[[job]] +name = "general-compat" +title = "test-general-compat" +workflow = "daily-compatibility" +paths = ["source", "scripts"] +exclude = ["isaaclab_tasks"] +container-name = "isaac-lab-general-compat-test" + +# --------------------------------------------------------------------------- +# arm-ci.yml -- one lane, split by -k so the ovphysx tests run separately +# --------------------------------------------------------------------------- + +[[job]] +name = "arm-ci" +title = "Build & Test" +workflow = "arm-ci" +paths = ["source", "scripts"] +marker = "arm_ci" +k-expr = "not ovphysx" +container-name = "isaac-lab-arm-test" + +[[job]] +name = "arm-ci-ovphysx" +title = "Build & Test" +workflow = "arm-ci" +paths = ["source", "scripts"] +marker = "arm_ci" +k-expr = "ovphysx" +container-name = "isaac-lab-arm-ovphysx-test" diff --git a/tools/test_settings.py b/tools/test_settings.py index 0dc337941d4a..54332c140674 100644 --- a/tools/test_settings.py +++ b/tools/test_settings.py @@ -121,7 +121,7 @@ # quarantined tests - run in dedicated CI job that does not block PR merges *QUARANTINED_TESTS, "test_environments_training.py", # Long-running RL training test; runs in dedicated CI job - # Exercises tools/conftest.py itself, including a hang that has to be waited out in real time. + # Exercises tools/run_tests.py itself, including a hang that has to be waited out in real time. # Needs no Isaac Sim and is not worth the CI spend. To run it when changing the orchestrator: # PYTHONPATH=tools:source/isaaclab pytest --noconftest \ # source/isaaclab/test/cli/test_test_orchestrator_result_handling.py diff --git a/tools/testplan.py b/tools/testplan.py new file mode 100644 index 000000000000..47182b487c07 --- /dev/null +++ b/tools/testplan.py @@ -0,0 +1,236 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Resolve a job in ``tools/test_plan.toml`` to the test files it covers. + +Selection used to be spread over four layers: a workflow job's inputs, two composite actions, +a bash translation into ``TEST_*`` environment variables, and a collector in the test runner +that read them back. A job's coverage could only be worked out by tracing all four. This +module is the whole of it: the plan says what a job runs, and :func:`resolve` turns that into +a file list. + +Markers are read with :func:`isaaclab.test.kit.module_markers`, which parses the file rather +than searching its text, so a marker named in a docstring or a comment no longer pulls a file +into a lane that does not want it. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + +import test_settings +import tomllib + +from isaaclab.test.kit import module_markers + +REPO_ROOT = Path(__file__).resolve().parent.parent + +PLAN_PATH = REPO_ROOT / "tools" / "test_plan.toml" + +# Has its own pytest.ini and conftest, and runs through .github/actions/install-ci-run rather +# than this plan. Skipping the whole subtree keeps its tests out of every job. +_EXCLUDED_DIRS = frozenset({"install_ci"}) + +_POOLS = {"quarantined": "QUARANTINED_TESTS", "curobo": "CUROBO_TESTS"} + + +@dataclass(frozen=True) +class Job: + """One entry in the test plan. + + Attributes: + name: Stable identifier, used on the command line and as the workflow job suffix. + title: Display name; a sharded job renders as ``" [i/n]"``. + workflow: Which workflow file the generated job block belongs to. + paths: Repo-relative directories walked for ``test_*.py``. + exclude: Substrings; a file whose repo-relative path contains any of them is dropped. + files: Basenames; when set, only these run, and they override ``TESTS_TO_SKIP``. + shards: Number of jobs the resolved list is split across. + pool: Named list in ``tools/test_settings.py`` to run instead of walking ``paths``. + marker: Only files declaring this pytest marker are selected. + k_expr: ``-k`` expression passed to each pytest invocation; narrows within a file. + node_ids_key: Key in ``.github/test-subsets/`` selecting exact node IDs on push. + container_name: Docker container name for the CI job. + warp_cache: Warp cache mode for the CI job. + generate: Whether ``tools/generate_workflows.py`` renders this job's workflow block. + False for the lanes whose CI setup is bespoke -- extra build steps, wheelhouse + expressions, artifact uploads -- which stay hand-written and are only checked + against the plan. + continue_on_error: Whether a failure in this lane leaves the run green. + extra_pip_packages: Packages installed in the container before the tests start. + timeout_minutes: Job timeout. + """ + + name: str + title: str + workflow: str + paths: tuple[str, ...] + exclude: tuple[str, ...] = () + files: tuple[str, ...] = () + shards: int = 1 + pool: str | None = None + marker: str | None = None + k_expr: str | None = None + node_ids_key: str | None = None + container_name: str | None = None + warp_cache: str | None = None + generate: bool = False + continue_on_error: bool = False + extra_pip_packages: str | None = None + timeout_minutes: int = 180 + + +def load_plan(path: Path | None = None) -> list[Job]: + """Read the plan file. + + Args: + path: Plan to read; defaults to :data:`PLAN_PATH`. + + Returns: + Every job, in file order. + + Raises: + ValueError: If a job is missing a name or two jobs share one. + """ + raw = tomllib.loads((path or PLAN_PATH).read_text(encoding="utf-8")) + jobs = [] + seen = set() + for entry in raw.get("job", []): + name = entry.get("name") + if not name: + raise ValueError(f"a job in {path or PLAN_PATH} has no name: {entry}") + if name in seen: + raise ValueError(f"duplicate job name in the test plan: {name!r}") + seen.add(name) + jobs.append( + Job( + name=name, + title=entry.get("title", name), + workflow=entry["workflow"], + paths=tuple(entry.get("paths", ())), + exclude=tuple(entry.get("exclude", ())), + files=tuple(entry.get("files", ())), + shards=int(entry.get("shards", 1)), + pool=entry.get("pool"), + marker=entry.get("marker"), + k_expr=entry.get("k-expr"), + node_ids_key=entry.get("node-ids-key"), + container_name=entry.get("container-name"), + warp_cache=entry.get("warp-cache"), + generate=bool(entry.get("generate", False)), + continue_on_error=bool(entry.get("continue-on-error", False)), + extra_pip_packages=entry.get("extra-pip-packages"), + timeout_minutes=int(entry.get("timeout-minutes", 180)), + ) + ) + return jobs + + +def get_job(name: str, path: Path | None = None) -> Job: + """Return the named job. + + Args: + name: Job name from the plan. + path: Plan to read; defaults to :data:`PLAN_PATH`. + + Returns: + The matching job. + + Raises: + KeyError: If no job has that name. + """ + for job in load_plan(path): + if job.name == name: + return job + raise KeyError(f"no job named {name!r} in the test plan; try --list-jobs") + + +def _display_path(path: Path, root: Path) -> str: + """Return ``path`` relative to ``root``, or absolute when it lies outside the repository. + + The local runner accepts any directory, including one outside the checkout, so this cannot + assume every result is repo-relative. + """ + try: + return path.relative_to(root).as_posix() + except ValueError: + return path.as_posix() + + +def walk_test_files(paths: tuple[str, ...] | list[str], root: Path | None = None) -> list[str]: + """Return every ``test_*.py`` under ``paths``, sorted. + + Args: + paths: Directories to walk, repo-relative or absolute. + root: Repository root; defaults to :data:`REPO_ROOT`. + + Returns: + Sorted paths, repo-relative where they lie inside the repository. + + Raises: + FileNotFoundError: If a listed directory does not exist, which would otherwise show up + as a job that silently runs nothing. + """ + root = root or REPO_ROOT + found = set() + for entry in paths: + base = root / entry + if not base.is_dir(): + raise FileNotFoundError(f"test plan path does not exist: {entry}") + for directory, _, names in os.walk(base): + if not _EXCLUDED_DIRS.isdisjoint(Path(directory).parts): + continue + for name in names: + if name.startswith("test_") and name.endswith(".py"): + found.add(_display_path(Path(directory) / name, root)) + return sorted(found) + + +def resolve(job: Job, *, shard: int | None = None, root: Path | None = None) -> list[str]: + """Return the test files ``job`` covers, repo-relative and sorted. + + Args: + job: Job to resolve. + shard: Which shard to take, for a job with ``shards > 1``. None returns every shard. + root: Repository root; defaults to :data:`REPO_ROOT`. + + Returns: + Sorted repo-relative paths. + + Raises: + ValueError: If ``shard`` is out of range for the job. + """ + root = root or REPO_ROOT + candidates = walk_test_files(job.paths, root=root) + wanted = set(job.files) + + selected = [] + for path in candidates: + name = os.path.basename(path) + if job.pool is not None: + if name not in getattr(test_settings, _POOLS[job.pool], ()): + continue + elif wanted: + # An explicit file list is the job's whole point, so it overrides the skip list. + if name not in wanted: + continue + elif name in test_settings.TESTS_TO_SKIP: + continue + if any(token in path for token in job.exclude): + continue + selected.append(path) + + if job.marker: + selected = [ + path for path in selected if job.marker in module_markers((root / path).read_text(errors="replace")) + ] + + if job.shards > 1 and shard is not None: + if not 0 <= shard < job.shards: + raise ValueError(f"shard {shard} out of range for job {job.name!r} with {job.shards} shards") + selected = [path for index, path in enumerate(selected) if index % job.shards == shard] + return selected