From d3a9c152628fd82347cc91e37e7cc01372fa7690 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Sun, 2 Aug 2026 16:10:27 -0400 Subject: [PATCH 01/13] Add Kit test markers and a shared launch_kit() helper Kit-dependence is currently a property of importing a test file: 156 test modules construct AppLauncher at module scope, so Isaac Sim boots during pytest collection. Because nothing declares that dependency, tools/conftest.py has to run every test file in its own subprocess, paying Kit startup once per file. Introduce the two pieces needed to change that: launch_kit() is an idempotent module-scope replacement for AppLauncher. The first test module in a process boots Kit; later modules receive the running app, so a pytest run covering several files pays startup once. It raises rather than silently returning a mismatched app when a file asks for cameras after a camera-less boot. The kit / kit_cameras / kitless markers let a file declare which launch configuration it needs, so files that can share a process can be grouped without importing them. kit_solo opts a file out of any such grouping. test_kit_marker_contract.py keeps the markers from drifting: it checks by AST that a file's declaration matches what it does at module scope. The checks are AST-based rather than text-based because several kit-free files mention AppLauncher only in a docstring saying they do not use it. Files are not yet required to carry a marker; _ENFORCED_ROOTS is empty and grows per package as files are migrated. No test file changes behaviour: nothing is marked kit or kitless yet, and no file calls launch_kit() yet. The guard found one pre-existing bug on its first run. test_operational_space assigned pytestmark twice, and the second assignment discarded arm_ci, so the file had been excluded from the ARM CI lane. Merged into a single list. --- pyproject.toml | 5 + .../changelog.d/mataylor-kit-test-markers.rst | 15 + source/isaaclab/isaaclab/test/launch.py | 84 +++++ .../controllers/test_operational_space.py | 4 +- .../isaaclab/test/test_kit_marker_contract.py | 348 ++++++++++++++++++ 5 files changed, 453 insertions(+), 3 deletions(-) create mode 100644 source/isaaclab/changelog.d/mataylor-kit-test-markers.rst create mode 100644 source/isaaclab/isaaclab/test/launch.py create mode 100644 source/isaaclab/test/test_kit_marker_contract.py diff --git a/pyproject.toml b/pyproject.toml index d743d6a2d54b..b34bf53c547c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -338,6 +338,11 @@ markers = [ "benchmark: test covers the Isaac Lab benchmark framework and infrastructure", "rendering: test exercises the rendering / camera / visualizer pipeline", "smoke: tests for core installation, task, and RL functionality", + "kit: test file needs a booted headless Kit app; it calls isaaclab.test.launch.launch_kit() at module scope rather than constructing AppLauncher", + "kit_cameras: like `kit`, but the app is booted with cameras enabled via launch_kit(cameras=True)", + "kitless: test file runs without Kit; no AppLauncher and no module-scope import of omni/carb/isaacsim", + "kit_solo: keep this file in its own process; it is never grouped with other files", + "newton_ci: mark test to run in the Newton CI lane", ] # Add pypi.nvidia.com so that `uv pip install isaaclab[isaacsim]` works without --extra-index-url. diff --git a/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst b/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst new file mode 100644 index 000000000000..1b2acabc992e --- /dev/null +++ b/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst @@ -0,0 +1,15 @@ +Added +^^^^^ + +* Added :func:`~isaaclab.test.launch.launch_kit` so test modules can share one Kit app per + pytest process instead of each launching their own. It is idempotent: the first module to + call it boots Kit and later modules receive the running app. +* Added the ``kit``, ``kit_cameras``, ``kitless``, and ``kit_solo`` pytest markers so a test + file can declare its Kit launch configuration, plus a test that checks each file's markers + against what it actually does at module scope. + +Fixed +^^^^^ + +* Fixed ``test_operational_space.py`` assigning ``pytestmark`` twice, which silently dropped + its ``arm_ci`` marker and kept the file out of the ARM CI lane. diff --git a/source/isaaclab/isaaclab/test/launch.py b/source/isaaclab/isaaclab/test/launch.py new file mode 100644 index 000000000000..1b9f423f6b4e --- /dev/null +++ b/source/isaaclab/isaaclab/test/launch.py @@ -0,0 +1,84 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared Kit launch helper for Isaac Lab tests. + +Test modules that need Isaac Sim call :func:`launch_kit` at module scope in place of +constructing :class:`~isaaclab.app.AppLauncher` directly:: + + from isaaclab.test.launch import launch_kit + + launch_kit() # or launch_kit(cameras=True) + +The call must stay at module scope: a test module's own imports (``pxr``, ``omni``, +``isaaclab_physx``, ...) run during pytest collection, before any fixture executes, so Kit +must already be running by then. + +:func:`launch_kit` is idempotent within a process. The first test module to call it boots +Kit; every later module gets the running app back. A pytest process covering several test +files therefore pays Kit startup once rather than once per file. + +Declare the matching marker on the module so the test runner can group files that share a +launch configuration into one process:: + + pytestmark = pytest.mark.kit # launch_kit() + pytestmark = pytest.mark.kit_cameras # launch_kit(cameras=True) +""" + +from __future__ import annotations + +from typing import Any + +_app: Any = None +"""The Kit application booted by :func:`launch_kit`, or None before the first call.""" + +_cameras: bool = False +"""Whether :attr:`_app` was booted with camera and render extensions enabled.""" + + +def launch_kit(*, cameras: bool = False) -> Any: + """Boot the shared Kit app for this process, or return the one already running. + + Args: + cameras: Whether the app must be booted with camera and render extensions enabled. + Passed through to :paramref:`~isaaclab.app.AppLauncher.enable_cameras`. + + Returns: + The running ``SimulationApp``. + + Raises: + RuntimeError: If a camera-enabled app is requested but Kit is already running in + this process without cameras, or if Kit was started by something other than + this function. Both mean the test files sharing this process do not share a + launch configuration and must be split across processes. + """ + global _app, _cameras + + if _app is not None: + if cameras and not _cameras: + raise RuntimeError( + "launch_kit(cameras=True) was called, but Kit is already running in this process" + " without cameras. Camera extensions cannot be enabled after startup. Mark this" + " file `pytest.mark.kit_cameras` so it is grouped with other camera tests instead" + " of with plain `pytest.mark.kit` files." + ) + return _app + + from isaaclab.utils import has_kit + + if has_kit(): + raise RuntimeError( + "Kit is already running but was not started by launch_kit(), so its launch" + " configuration is unknown. Another test file in this process still constructs" + " AppLauncher directly; run that file in its own process." + ) + + from isaaclab.app import AppLauncher + + from .utils import resolve_test_sim_device + + _app = AppLauncher(headless=True, enable_cameras=cameras, device=resolve_test_sim_device()).app + _cameras = cameras + return _app diff --git a/source/isaaclab/test/controllers/test_operational_space.py b/source/isaaclab/test/controllers/test_operational_space.py index 1925c6673a0d..8db637450a33 100644 --- a/source/isaaclab/test/controllers/test_operational_space.py +++ b/source/isaaclab/test/controllers/test_operational_space.py @@ -16,8 +16,6 @@ import torch from flaky import flaky -pytestmark = pytest.mark.arm_ci - import isaaclab.envs.mdp as mdp import isaaclab.sim as sim_utils from isaaclab import cloner @@ -51,7 +49,7 @@ from isaaclab_assets import FRANKA_PANDA_CFG, G1_29DOF_CFG # isort:skip -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.arm_ci, pytest.mark.integration] @pytest.fixture diff --git a/source/isaaclab/test/test_kit_marker_contract.py b/source/isaaclab/test/test_kit_marker_contract.py new file mode 100644 index 000000000000..20c6ba41ed5c --- /dev/null +++ b/source/isaaclab/test/test_kit_marker_contract.py @@ -0,0 +1,348 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Test that every test file's Kit markers agree with what the file actually does. + +Kit-dependence is a property of *importing* a test module: a module that constructs +:class:`~isaaclab.app.AppLauncher` at module scope boots Isaac Sim during pytest collection, +before any fixture runs. The ``kit`` / ``kit_cameras`` / ``kitless`` markers make that +property declarative so the runner can group files that share a launch configuration into a +single process instead of paying Kit startup once per file. + +A marker is only useful if it cannot drift from reality, which is what this test enforces: + +* ``kit`` / ``kit_cameras`` -- the file calls :func:`~isaaclab.test.launch.launch_kit` at + module scope with the matching ``cameras`` argument, and never constructs ``AppLauncher`` + or ``SimulationApp`` itself. Direct construction would boot a second, unshared app. +* ``kitless`` -- the file never launches Kit and does not import a Kit runtime package at + module scope, so it can run in a process where Kit was never started. +* ``unit`` -- same requirement as ``kitless``, which turns the marker's registered + description ("does not launch the simulator") into a checked invariant. +* At most one module-scope ``pytestmark`` assignment, since a second assignment silently + rebinds the name and discards the markers from the first. + +The checks are AST-based rather than text-based because a source-text search cannot tell an +``AppLauncher`` reference in a docstring from a real call -- several kit-free files mention +``AppLauncher`` only to document that they do not use it. + +Files outside :data:`_ENFORCED_ROOTS` are not yet *required* to carry a marker; the +consistency rules above still apply to them whenever they do. Extend that tuple as each +package is migrated. +""" + +from __future__ import annotations + +import ast +import json +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.unit, pytest.mark.kitless] + +_REPO_ROOT = Path(__file__).resolve().parents[3] + +_SCAN_ROOTS = ("source", "scripts") + +_EXCLUDED_PARTS = frozenset( + { + # Own pytest.ini / rootdir; deliberately excluded from the main collector too. + "install_ci", + # Vendored copies of the source tree produced by the wheel builder. + "build", + # Virtual environments and the Isaac Sim symlink. + ".venv", + "env_isaaclab", + "_isaac_sim", + } +) + +# Packages that only exist inside a running Kit application. ``pxr`` is deliberately absent: +# OpenUSD is importable kit-less through the ``usd-core`` wheel, so importing it says nothing +# about whether Kit is running. +_KIT_RUNTIME_PREFIXES = ("omni", "carb", "isaacsim") + +# Directories where a test file is required to declare `kit`, `kit_cameras`, or `kitless`. +# Grows one package at a time as files are migrated off module-scope ``AppLauncher``. +_ENFORCED_ROOTS: tuple[str, ...] = () + +_PROFILE_MARKERS = ("kit", "kit_cameras", "kitless") + + +# --------------------------------------------------------------------------- +# AST helpers +# --------------------------------------------------------------------------- + + +def _module_scope_nodes(tree: ast.Module): + """Yield every node that executes at module import, without entering callables. + + Descends through module-level control flow (``if`` / ``try`` / ``with``) because those + bodies still run at import, but stops at function, class, and lambda boundaries because + those bodies only run when called. + """ + stack = list(tree.body) + while stack: + node = stack.pop() + yield node + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda): + continue + for child in ast.iter_child_nodes(node): + stack.append(child) + + +def _call_name(node: ast.AST) -> str | None: + """Return the called function's bare name, for ``f()`` and ``mod.f()`` alike.""" + if not isinstance(node, ast.Call): + return None + func = node.func + if isinstance(func, ast.Name): + return func.id + if isinstance(func, ast.Attribute): + return func.attr + return None + + +def _marker_names(node: ast.AST) -> list[str]: + """Return the marker names in a ``pytest.mark.`` expression or a list of them.""" + if isinstance(node, ast.List | ast.Tuple): + return [name for element in node.elts for name in _marker_names(element)] + if isinstance(node, ast.Call): + return _marker_names(node.func) + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Attribute): + # pytest.mark. + if node.value.attr == "mark": + return [node.attr] + return [] + + +class _FileFacts: + """What a single test file declares and what it actually does at module scope.""" + + def __init__(self, path: Path, tree: ast.Module): + self.path = path + self.pytestmark_assignments: list[int] = [] + self.markers: set[str] = set() + self.launch_kit_cameras: bool | None = None + self.module_scope_launcher: list[tuple[str, int]] = [] + self.launch_kit_anywhere = False + self.kit_runtime_imports: list[tuple[str, int]] = [] + + module_scope = set() + for node in _module_scope_nodes(tree): + module_scope.add(id(node)) + + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "pytestmark" for target in node.targets + ): + self.pytestmark_assignments.append(node.lineno) + self.markers.update(_marker_names(node.value)) + + name = _call_name(node) + if name in ("AppLauncher", "SimulationApp"): + self.module_scope_launcher.append((name, node.lineno)) + elif name == "launch_kit": + self.launch_kit_cameras = any( + keyword.arg == "cameras" and isinstance(keyword.value, ast.Constant) and keyword.value.value + for keyword in node.keywords + ) + + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name.split(".")[0] in _KIT_RUNTIME_PREFIXES: + self.kit_runtime_imports.append((alias.name, node.lineno)) + elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0: + if node.module.split(".")[0] in _KIT_RUNTIME_PREFIXES: + self.kit_runtime_imports.append((node.module, node.lineno)) + + # Decorator markers (e.g. a per-test `@pytest.mark.unit`) count toward the file's + # marker set, and AppLauncher use anywhere -- not just module scope -- disqualifies + # a file from claiming `kitless`. + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): + for decorator in node.decorator_list: + self.markers.update(_marker_names(decorator)) + name = _call_name(node) + if name == "launch_kit": + self.launch_kit_anywhere = True + elif name in ("AppLauncher", "SimulationApp") and id(node) not in module_scope: + self.module_scope_launcher.append((f"{name} (deferred)", node.lineno)) + + @property + def rel(self) -> str: + return self.path.relative_to(_REPO_ROOT).as_posix() + + @property + def profile_markers(self) -> list[str]: + return [marker for marker in _PROFILE_MARKERS if marker in self.markers] + + @property + def launches_kit_directly(self) -> list[tuple[str, int]]: + return self.module_scope_launcher + + +# --------------------------------------------------------------------------- +# Collection +# --------------------------------------------------------------------------- + + +def _iter_test_files(): + for root in _SCAN_ROOTS: + for path in sorted((_REPO_ROOT / root).rglob("test_*.py")): + if _EXCLUDED_PARTS.isdisjoint(path.parts): + yield path + + +@pytest.fixture(scope="module") +def facts() -> list[_FileFacts]: + """Parse every test file once and return the extracted facts.""" + collected = [] + for path in _iter_test_files(): + try: + tree = ast.parse(path.read_text(encoding="utf-8", errors="replace"), filename=str(path)) + except SyntaxError as exc: + pytest.fail(f"{path.relative_to(_REPO_ROOT).as_posix()} failed to parse: {exc}") + collected.append(_FileFacts(path, tree)) + assert collected, f"no test files discovered under {_SCAN_ROOTS} -- the scan roots are wrong" + return collected + + +# --------------------------------------------------------------------------- +# Rules +# --------------------------------------------------------------------------- + + +def test_pytestmark_is_assigned_at_most_once(facts: list[_FileFacts]): + """A second module-scope ``pytestmark`` rebinds the name and drops the first one's markers.""" + offenders = [ + f"{f.rel}: lines {sorted(f.pytestmark_assignments)}" for f in facts if len(f.pytestmark_assignments) > 1 + ] + assert not offenders, ( + "These files assign `pytestmark` more than once at module scope. The later assignment" + " replaces the earlier one, so the markers declared first are silently lost:\n " + + "\n ".join(offenders) + + "\n\nFix: merge them into a single list, e.g. `pytestmark = [pytest.mark.a, pytest.mark.b]`." + ) + + +def test_profile_markers_are_mutually_exclusive(facts: list[_FileFacts]): + """A file runs in exactly one of the launch configurations, so it declares only one.""" + offenders = [f"{f.rel}: {', '.join(f.profile_markers)}" for f in facts if len(f.profile_markers) > 1] + assert not offenders, "These files declare more than one of `kit`, `kit_cameras`, `kitless`:\n " + "\n ".join( + offenders + ) + + +def test_kit_marked_files_use_launch_kit(facts: list[_FileFacts]): + """`kit` / `kit_cameras` files share the process app; they must not build their own.""" + offenders = [] + for f in facts: + markers = f.profile_markers + if not markers or markers[0] == "kitless": + continue + if f.launches_kit_directly: + where = ", ".join(f"{name} at line {line}" for name, line in f.launches_kit_directly) + offenders.append(f"{f.rel}: declares `{markers[0]}` but constructs {where}") + continue + if f.launch_kit_cameras is None: + offenders.append(f"{f.rel}: declares `{markers[0]}` but never calls launch_kit() at module scope") + continue + wants_cameras = markers[0] == "kit_cameras" + if f.launch_kit_cameras != wants_cameras: + expected = "launch_kit(cameras=True)" if wants_cameras else "launch_kit()" + offenders.append(f"{f.rel}: declares `{markers[0]}` but does not call {expected}") + + assert not offenders, ( + "These files' Kit markers disagree with how they launch Kit:\n " + + "\n ".join(offenders) + + "\n\nFix: call `launch_kit()` (or `launch_kit(cameras=True)`) from" + " `isaaclab.test.launch` at module scope instead of constructing AppLauncher, and make" + " the marker match the `cameras` argument." + ) + + +@pytest.mark.parametrize("marker", ["kitless", "unit"]) +def test_kit_free_files_do_not_touch_kit(marker: str, facts: list[_FileFacts]): + """`kitless` and `unit` files must run in a process where Kit was never started.""" + offenders = [] + for f in facts: + if marker not in f.markers: + continue + if f.launches_kit_directly: + where = ", ".join(f"{name} at line {line}" for name, line in f.launches_kit_directly) + offenders.append(f"{f.rel}: constructs {where}") + if f.launch_kit_anywhere: + offenders.append(f"{f.rel}: calls launch_kit()") + if f.kit_runtime_imports: + where = ", ".join(f"`{name}` at line {line}" for name, line in f.kit_runtime_imports) + offenders.append(f"{f.rel}: imports {where} at module scope") + + assert not offenders, ( + f"These files are marked `{marker}` but depend on a running Kit:\n " + + "\n ".join(offenders) + + f"\n\nKit runtime packages: {_KIT_RUNTIME_PREFIXES}." + f"\nFix: drop the `{marker}` marker and declare `kit`, or move the Kit import inside the" + " test function so it is not paid at collection." + ) + + +def test_migrated_packages_declare_a_marker(facts: list[_FileFacts]): + """Within a migrated package, every test file states its launch configuration.""" + if not _ENFORCED_ROOTS: + pytest.skip("no packages are enforced yet; extend _ENFORCED_ROOTS as files are migrated") + + offenders = [f.rel for f in facts if f.rel.startswith(_ENFORCED_ROOTS) and not f.profile_markers] + assert not offenders, ( + "These files are in a migrated package but declare none of `kit`, `kit_cameras`," + " `kitless`:\n " + "\n ".join(offenders) + ) + + +def test_kitless_files_import_without_kit(facts: list[_FileFacts]): + """Importing every `kitless` module must not pull in Kit through a helper module. + + The AST rules only see each file's own imports. A shared test utility that imports Kit + would slip past them, so this imports the real modules in one subprocess and checks that + ``omni.kit.app`` never appears in :data:`sys.modules`. + """ + modules = sorted(f.rel for f in facts if "kitless" in f.markers) + if not modules: + pytest.skip("no files are marked `kitless` yet") + + script = textwrap.dedent(f""" + import importlib.util, json, os, sys + + offenders = [] + for rel in {modules!r}: + # pytest puts a test file's own directory on sys.path (rootdir/conftest handling), + # which is how these modules reach their sibling helpers. Mirror that here. + directory = os.path.dirname(rel) + if directory not in sys.path: + sys.path.insert(0, directory) + + name = "_kitless_probe_" + rel.replace("/", "_")[:-3] + spec = importlib.util.spec_from_file_location(name, rel) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + except Exception as exc: + offenders.append(f"{{rel}}: import failed: {{type(exc).__name__}}: {{exc}}") + continue + if "omni.kit.app" in sys.modules: + offenders.append(f"{{rel}}: importing it started Kit") + break + print("__RESULTS__" + json.dumps(offenders)) + """) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, cwd=_REPO_ROOT, timeout=600) + line = next((ln for ln in result.stdout.splitlines() if ln.startswith("__RESULTS__")), None) + assert line is not None, ( + f"kitless import probe did not report results\n--- stdout ---\n{result.stdout}\n--- stderr ---\n{result.stderr}" + ) + offenders = json.loads(line[len("__RESULTS__") :]) + assert not offenders, "These `kitless` files pull in Kit transitively:\n " + "\n ".join(offenders) From 02010869e90056302b5c9e2c15b548f1ab7c6907 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Sun, 2 Aug 2026 18:18:54 -0400 Subject: [PATCH 02/13] Migrate source/isaaclab/test/sim to launch_kit() Replace the module-scope AppLauncher construction in the Kit-dependent files under source/isaaclab/test/sim with launch_kit(), and declare the matching kit or kit_cameras marker on each file. Because launch_kit() is idempotent, a pytest process covering several of these files now boots Kit once instead of once per file. Nothing forces them into one process yet -- tools/conftest.py still runs a subprocess per file -- so this changes how the files launch Kit, not how CI schedules them. 24 files map to `kit` and 4 to `kit_cameras`. The two groups must not share a process in that order: a camera-enabled app can serve tests that do not need cameras, but cameras cannot be enabled after startup, so launch_kit() raises rather than handing back an app that would silently fail to render. The transform is applied by tools/codemods/kit_launch_migration.py, added here because ~125 files in other packages remain to migrate. It edits line ranges in place rather than round-tripping through ast.unparse, which would discard comments and isort directives, and it preserves each launch call's position so the Kit-dependent imports below it still run after Kit starts. The codemod refuses anything it cannot rewrite without changing behaviour, and reports it. In particular it rejects a conditional launch such as `AppLauncher(...).app if _USE_KIT else None`, which test_mjcf_converter.py and test_urdf_converter.py use so they can run kitlessly when the standalone importer wheel is installed; collapsing that ternary would have made the boot unconditional. It also refuses a file that references AppLauncher for anything other than the launch call, since the import is removed. --- .../test/sim/test_articulation_fragments.py | 11 +- .../test_build_simulation_context_headless.py | 11 +- ...st_build_simulation_context_nonheadless.py | 11 +- source/isaaclab/test/sim/test_cloner.py | 11 +- .../test/sim/test_collision_fragments.py | 11 +- .../test/sim/test_joint_drive_fragments.py | 11 +- .../isaaclab/test/sim/test_mass_fragments.py | 11 +- .../test/sim/test_material_fragments.py | 11 +- .../test/sim/test_mesh_collision_fragments.py | 11 +- .../isaaclab/test/sim/test_mesh_converter.py | 11 +- .../test/sim/test_schema_fragments.py | 11 +- .../sim/test_schema_writer_nested_targets.py | 11 +- source/isaaclab/test/sim/test_schemas.py | 11 +- .../test/sim/test_simulation_context.py | 13 +- .../sim/test_simulation_stage_in_memory.py | 12 +- .../test/sim/test_spawn_from_files.py | 11 +- source/isaaclab/test/sim/test_spawn_lights.py | 12 +- .../isaaclab/test/sim/test_spawn_materials.py | 12 +- source/isaaclab/test/sim/test_spawn_meshes.py | 12 +- .../isaaclab/test/sim/test_spawn_sensors.py | 12 +- source/isaaclab/test/sim/test_spawn_shapes.py | 11 +- .../isaaclab/test/sim/test_spawn_wrappers.py | 12 +- .../test/sim/test_tendon_fragments.py | 11 +- source/isaaclab/test/sim/test_utils_prims.py | 11 +- .../isaaclab/test/sim/test_utils_queries.py | 11 +- .../isaaclab/test/sim/test_utils_semantics.py | 11 +- source/isaaclab/test/sim/test_utils_stage.py | 11 +- .../test/sim/test_utils_transforms.py | 11 +- .../test/sim/test_views_xform_prim.py | 8 +- tools/codemods/kit_launch_migration.py | 325 ++++++++++++++++++ 30 files changed, 415 insertions(+), 234 deletions(-) create mode 100644 tools/codemods/kit_launch_migration.py diff --git a/source/isaaclab/test/sim/test_articulation_fragments.py b/source/isaaclab/test/sim/test_articulation_fragments.py index 2319363122eb..68de1554d211 100644 --- a/source/isaaclab/test/sim/test_articulation_fragments.py +++ b/source/isaaclab/test/sim/test_articulation_fragments.py @@ -3,14 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import os @@ -21,6 +16,8 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext +pytestmark = pytest.mark.kit + def _make_xform(stage, path="/World/Art"): UsdGeom.Xform.Define(stage, path) diff --git a/source/isaaclab/test/sim/test_build_simulation_context_headless.py b/source/isaaclab/test/sim/test_build_simulation_context_headless.py index cf266f73f4fe..cc3e98ebabfb 100644 --- a/source/isaaclab/test/sim/test_build_simulation_context_headless.py +++ b/source/isaaclab/test/sim/test_build_simulation_context_headless.py @@ -13,21 +13,16 @@ ``test_build_simulation_context_nonheadless.py``. """ -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import pytest from isaaclab.sim.simulation_cfg import SimulationCfg from isaaclab.sim.simulation_context import build_simulation_context -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] @pytest.mark.parametrize("gravity_enabled", [True, False]) diff --git a/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py b/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py index 2ce2345062c8..fd5fe7137ab1 100644 --- a/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py +++ b/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py @@ -12,21 +12,16 @@ ``test_build_simulation_context_headless.py``. """ -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import pytest from isaaclab.sim.simulation_cfg import SimulationCfg from isaaclab.sim.simulation_context import build_simulation_context -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] @pytest.mark.parametrize("gravity_enabled", [True, False]) diff --git a/source/isaaclab/test/sim/test_cloner.py b/source/isaaclab/test/sim/test_cloner.py index 7bf436e70234..485f0ee09ba1 100644 --- a/source/isaaclab/test/sim/test_cloner.py +++ b/source/isaaclab/test/sim/test_cloner.py @@ -5,14 +5,9 @@ """Tests for USD cloner utilities (no PhysX dependency).""" -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() from types import SimpleNamespace from unittest.mock import MagicMock @@ -37,7 +32,7 @@ ) from isaaclab.sim import build_simulation_context -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture(params=["cpu", "cuda"]) diff --git a/source/isaaclab/test/sim/test_collision_fragments.py b/source/isaaclab/test/sim/test_collision_fragments.py index 712390bc2f56..c3c4005c4490 100644 --- a/source/isaaclab/test/sim/test_collision_fragments.py +++ b/source/isaaclab/test/sim/test_collision_fragments.py @@ -3,14 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import pytest @@ -19,7 +14,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def _make_xform(stage, path="/World/Body"): diff --git a/source/isaaclab/test/sim/test_joint_drive_fragments.py b/source/isaaclab/test/sim/test_joint_drive_fragments.py index a9c5534ede37..1a6be46df6a6 100644 --- a/source/isaaclab/test/sim/test_joint_drive_fragments.py +++ b/source/isaaclab/test/sim/test_joint_drive_fragments.py @@ -3,14 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import math @@ -21,7 +16,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def _make_revolute_joint(stage, path="/World/Articulation/joint_0"): diff --git a/source/isaaclab/test/sim/test_mass_fragments.py b/source/isaaclab/test/sim/test_mass_fragments.py index f017d9d2d16b..f08578ac18d3 100644 --- a/source/isaaclab/test/sim/test_mass_fragments.py +++ b/source/isaaclab/test/sim/test_mass_fragments.py @@ -3,14 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import pytest @@ -19,7 +14,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def _make_xform(stage, path="/World/Body"): diff --git a/source/isaaclab/test/sim/test_material_fragments.py b/source/isaaclab/test/sim/test_material_fragments.py index c09362c5efd0..c69d51c71e8b 100644 --- a/source/isaaclab/test/sim/test_material_fragments.py +++ b/source/isaaclab/test/sim/test_material_fragments.py @@ -3,14 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import pytest @@ -19,7 +14,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] # ------------------------------------------------------------------------------------- # RigidBodyMaterialFragment marker + metadata diff --git a/source/isaaclab/test/sim/test_mesh_collision_fragments.py b/source/isaaclab/test/sim/test_mesh_collision_fragments.py index ae33dbb938d2..5be29b24ea08 100644 --- a/source/isaaclab/test/sim/test_mesh_collision_fragments.py +++ b/source/isaaclab/test/sim/test_mesh_collision_fragments.py @@ -3,14 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import pytest @@ -19,7 +14,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def _make_xform(stage, path="/World/Mesh"): diff --git a/source/isaaclab/test/sim/test_mesh_converter.py b/source/isaaclab/test/sim/test_mesh_converter.py index f4551b4ba829..2120df259f2f 100644 --- a/source/isaaclab/test/sim/test_mesh_converter.py +++ b/source/isaaclab/test/sim/test_mesh_converter.py @@ -3,14 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import math import os @@ -27,7 +22,7 @@ from isaaclab.sim.schemas import MESH_APPROXIMATION_TOKENS, schemas_cfg from isaaclab.utils.assets import ISAACLAB_NUCLEUS_DIR, retrieve_file_path -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def random_quaternion(): diff --git a/source/isaaclab/test/sim/test_schema_fragments.py b/source/isaaclab/test/sim/test_schema_fragments.py index e6ca68c3ddda..c8a00f31cfff 100644 --- a/source/isaaclab/test/sim/test_schema_fragments.py +++ b/source/isaaclab/test/sim/test_schema_fragments.py @@ -3,14 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import pytest @@ -19,7 +14,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def _make_xform(stage, path="/World/Body"): diff --git a/source/isaaclab/test/sim/test_schema_writer_nested_targets.py b/source/isaaclab/test/sim/test_schema_writer_nested_targets.py index b1cf4331a048..1aa9146a2c32 100644 --- a/source/isaaclab/test/sim/test_schema_writer_nested_targets.py +++ b/source/isaaclab/test/sim/test_schema_writer_nested_targets.py @@ -3,14 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import os @@ -23,7 +18,7 @@ from isaaclab.sim import SimulationCfg, SimulationContext from isaaclab.sim.schemas import MassCfg -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def _author_robot_usd(path: str) -> None: diff --git a/source/isaaclab/test/sim/test_schemas.py b/source/isaaclab/test/sim/test_schemas.py index 337dd2b69304..92f9adcfbf83 100644 --- a/source/isaaclab/test/sim/test_schemas.py +++ b/source/isaaclab/test/sim/test_schemas.py @@ -3,14 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import math import warnings @@ -45,7 +40,7 @@ from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR from isaaclab.utils.string import to_camel_case -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_simulation_context.py b/source/isaaclab/test/sim/test_simulation_context.py index 34de268685d3..02c9445d04fd 100644 --- a/source/isaaclab/test/sim/test_simulation_context.py +++ b/source/isaaclab/test/sim/test_simulation_context.py @@ -3,15 +3,10 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit +from isaaclab.test.utils import test_devices -from isaaclab.app import AppLauncher -from isaaclab.test.utils import resolve_test_sim_device, test_devices - -# launch omniverse app -simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app - -"""Rest everything follows.""" +launch_kit() import weakref @@ -24,7 +19,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] @pytest.fixture(autouse=True) diff --git a/source/isaaclab/test/sim/test_simulation_stage_in_memory.py b/source/isaaclab/test/sim/test_simulation_stage_in_memory.py index f91947fc32b9..bda761131630 100644 --- a/source/isaaclab/test/sim/test_simulation_stage_in_memory.py +++ b/source/isaaclab/test/sim/test_simulation_stage_in_memory.py @@ -5,16 +5,10 @@ """Integration tests for simulation context with stage in memory.""" -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app # FIXME (mmittal): Stage in memory requires cameras to be enabled. -simulation_app = AppLauncher(headless=True, enable_cameras=True).app - -"""Rest everything follows.""" - +launch_kit(cameras=True) import pytest import torch @@ -28,7 +22,7 @@ from isaaclab.utils.assets import ISAACLAB_NUCLEUS_DIR from isaaclab.utils.version import get_isaac_sim_version -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_spawn_from_files.py b/source/isaaclab/test/sim/test_spawn_from_files.py index 0a771c956f2c..4515555fb1bd 100644 --- a/source/isaaclab/test/sim/test_spawn_from_files.py +++ b/source/isaaclab/test/sim/test_spawn_from_files.py @@ -3,14 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.app import AppLauncher +from isaaclab.test.launch import launch_kit -"""Launch Isaac Sim Simulator first.""" - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import pytest @@ -20,7 +15,7 @@ from isaaclab.sim import SimulationCfg, SimulationContext from isaaclab.utils.assets import ISAACLAB_NUCLEUS_DIR -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_spawn_lights.py b/source/isaaclab/test/sim/test_spawn_lights.py index 59c771880782..bea78e909159 100644 --- a/source/isaaclab/test/sim/test_spawn_lights.py +++ b/source/isaaclab/test/sim/test_spawn_lights.py @@ -3,15 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +from isaaclab.test.launch import launch_kit +launch_kit() import pytest @@ -21,7 +15,7 @@ from isaaclab.sim import SimulationCfg, SimulationContext from isaaclab.utils.string import to_camel_case -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture(autouse=True) diff --git a/source/isaaclab/test/sim/test_spawn_materials.py b/source/isaaclab/test/sim/test_spawn_materials.py index d1cb86c87029..93ccd392f7c6 100644 --- a/source/isaaclab/test/sim/test_spawn_materials.py +++ b/source/isaaclab/test/sim/test_spawn_materials.py @@ -3,15 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +from isaaclab.test.launch import launch_kit +launch_kit() import pytest @@ -21,7 +15,7 @@ from isaaclab.sim import SimulationCfg, SimulationContext from isaaclab.utils.assets import NVIDIA_NUCLEUS_DIR -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_spawn_meshes.py b/source/isaaclab/test/sim/test_spawn_meshes.py index a9ad5158c2f3..1a2fc76f2964 100644 --- a/source/isaaclab/test/sim/test_spawn_meshes.py +++ b/source/isaaclab/test/sim/test_spawn_meshes.py @@ -3,22 +3,16 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +from isaaclab.test.launch import launch_kit +launch_kit() import pytest import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_spawn_sensors.py b/source/isaaclab/test/sim/test_spawn_sensors.py index 9e50b54496bc..af0df8b714a5 100644 --- a/source/isaaclab/test/sim/test_spawn_sensors.py +++ b/source/isaaclab/test/sim/test_spawn_sensors.py @@ -3,15 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +from isaaclab.test.launch import launch_kit +launch_kit() import pytest @@ -22,7 +16,7 @@ from isaaclab.sim.spawners.sensors.sensors import CUSTOM_FISHEYE_CAMERA_ATTRIBUTES, CUSTOM_PINHOLE_CAMERA_ATTRIBUTES from isaaclab.utils.string import to_camel_case -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_spawn_shapes.py b/source/isaaclab/test/sim/test_spawn_shapes.py index be59ea011d01..def648d5e7e4 100644 --- a/source/isaaclab/test/sim/test_spawn_shapes.py +++ b/source/isaaclab/test/sim/test_spawn_shapes.py @@ -3,21 +3,16 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import pytest import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_spawn_wrappers.py b/source/isaaclab/test/sim/test_spawn_wrappers.py index a0be9336a56f..c66d9fd7dafa 100644 --- a/source/isaaclab/test/sim/test_spawn_wrappers.py +++ b/source/isaaclab/test/sim/test_spawn_wrappers.py @@ -3,15 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" - -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +from isaaclab.test.launch import launch_kit +launch_kit() import pytest @@ -19,7 +13,7 @@ from isaaclab.sim import SimulationCfg, SimulationContext from isaaclab.utils.assets import ISAACLAB_NUCLEUS_DIR -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_tendon_fragments.py b/source/isaaclab/test/sim/test_tendon_fragments.py index c7569081a164..65e485911db9 100644 --- a/source/isaaclab/test/sim/test_tendon_fragments.py +++ b/source/isaaclab/test/sim/test_tendon_fragments.py @@ -3,14 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import pytest @@ -19,7 +14,7 @@ import isaaclab.sim as sim_utils from isaaclab.sim import SimulationCfg, SimulationContext -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] def _new_sim(): diff --git a/source/isaaclab/test/sim/test_utils_prims.py b/source/isaaclab/test/sim/test_utils_prims.py index 117aaced1608..c1703011b082 100644 --- a/source/isaaclab/test/sim/test_utils_prims.py +++ b/source/isaaclab/test/sim/test_utils_prims.py @@ -3,15 +3,10 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app # note: need to enable cameras to be able to make replicator core available -simulation_app = AppLauncher(headless=True, enable_cameras=True).app - -"""Rest everything follows.""" +launch_kit(cameras=True) import math @@ -25,7 +20,7 @@ from isaaclab.sim.utils.prims import _to_tuple # type: ignore[reportPrivateUsage] from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR, retrieve_file_path -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration, pytest.mark.isaacsim_ci] @pytest.fixture(autouse=True) diff --git a/source/isaaclab/test/sim/test_utils_queries.py b/source/isaaclab/test/sim/test_utils_queries.py index 973e7e718565..92997d04b09f 100644 --- a/source/isaaclab/test/sim/test_utils_queries.py +++ b/source/isaaclab/test/sim/test_utils_queries.py @@ -3,15 +3,10 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app # note: need to enable cameras to be able to make replicator core available -simulation_app = AppLauncher(headless=True, enable_cameras=True).app - -"""Rest everything follows.""" +launch_kit(cameras=True) import pytest @@ -20,7 +15,7 @@ import isaaclab.sim as sim_utils from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration] @pytest.fixture(autouse=True) diff --git a/source/isaaclab/test/sim/test_utils_semantics.py b/source/isaaclab/test/sim/test_utils_semantics.py index 926a2d0d80a4..c88f9e0d8dfe 100644 --- a/source/isaaclab/test/sim/test_utils_semantics.py +++ b/source/isaaclab/test/sim/test_utils_semantics.py @@ -3,21 +3,16 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app # note: need to enable cameras to be able to make replicator core available -simulation_app = AppLauncher(headless=True, enable_cameras=True).app - -"""Rest everything follows.""" +launch_kit(cameras=True) import pytest import isaaclab.sim as sim_utils -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration] @pytest.fixture(autouse=True) diff --git a/source/isaaclab/test/sim/test_utils_stage.py b/source/isaaclab/test/sim/test_utils_stage.py index 39a70a076f71..3bcd26e66361 100644 --- a/source/isaaclab/test/sim/test_utils_stage.py +++ b/source/isaaclab/test/sim/test_utils_stage.py @@ -5,14 +5,9 @@ """Tests for stage utilities.""" -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import tempfile from pathlib import Path @@ -23,7 +18,7 @@ import isaaclab.sim as sim_utils -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] def test_create_new_stage(): diff --git a/source/isaaclab/test/sim/test_utils_transforms.py b/source/isaaclab/test/sim/test_utils_transforms.py index e7cc178b65d5..1af8ce75bea1 100644 --- a/source/isaaclab/test/sim/test_utils_transforms.py +++ b/source/isaaclab/test/sim/test_utils_transforms.py @@ -3,14 +3,9 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Launch Isaac Sim Simulator first.""" +from isaaclab.test.launch import launch_kit -from isaaclab.app import AppLauncher - -# launch omniverse app -simulation_app = AppLauncher(headless=True).app - -"""Rest everything follows.""" +launch_kit() import math @@ -23,7 +18,7 @@ import isaaclab.sim as sim_utils import isaaclab.utils.math as math_utils -pytestmark = pytest.mark.integration +pytestmark = [pytest.mark.kit, pytest.mark.integration] @pytest.fixture(autouse=True) diff --git a/source/isaaclab/test/sim/test_views_xform_prim.py b/source/isaaclab/test/sim/test_views_xform_prim.py index 9217ca537d05..dfa5ad2372ad 100644 --- a/source/isaaclab/test/sim/test_views_xform_prim.py +++ b/source/isaaclab/test/sim/test_views_xform_prim.py @@ -10,10 +10,10 @@ prim ordering, xformOp standardization, and Isaac Sim comparison. """ -from isaaclab.app import AppLauncher -from isaaclab.test.utils import resolve_test_sim_device, test_devices +from isaaclab.test.launch import launch_kit +from isaaclab.test.utils import test_devices -simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app +launch_kit() import pytest # noqa: E402 import torch # noqa: E402 @@ -36,7 +36,7 @@ from isaaclab.sim.views import UsdFrameView as FrameView # noqa: E402 from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR # noqa: E402 -pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] PARENT_POS = (0.0, 0.0, 1.0) diff --git a/tools/codemods/kit_launch_migration.py b/tools/codemods/kit_launch_migration.py new file mode 100644 index 000000000000..f9eec8dad2cd --- /dev/null +++ b/tools/codemods/kit_launch_migration.py @@ -0,0 +1,325 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Rewrite test modules from a module-scope ``AppLauncher`` to the shared ``launch_kit()``. + +A test module that constructs :class:`~isaaclab.app.AppLauncher` at module scope boots its +own Kit app during pytest collection, so a process covering several such files pays Kit +startup once per file. :func:`~isaaclab.test.launch.launch_kit` is idempotent, so migrated +files share one app per process. + +The rewrite is deliberately in-place and line-based rather than an ``ast.unparse`` round +trip, which would discard comments, ``# isort:skip`` directives, and docstring formatting. +Each edit replaces a statement's own line range, so import ordering -- which matters here, +because Kit must boot before the Kit-dependent imports below it -- is preserved exactly. + +Usage:: + + uv run python tools/codemods/kit_launch_migration.py source/isaaclab/test/sim + uv run python tools/codemods/kit_launch_migration.py --check source/isaaclab/test/sim + +Files the transform cannot handle safely are reported and left untouched. +""" + +from __future__ import annotations + +import argparse +import ast +import sys +from pathlib import Path + +_LAUNCH_IMPORT = "from isaaclab.test.launch import launch_kit" +_APP_IMPORT_MODULE = "isaaclab.app" + +# Docstrings used purely as section separators around the old launch block. They document a +# launch step that no longer exists in the file once it is migrated. +_BOILERPLATE_DOCSTRINGS = ("Launch Isaac Sim Simulator first.", "Rest everything follows.") + +_BOILERPLATE_COMMENTS = ("# launch omniverse app", "# launch the simulator") + + +class Unsupported(Exception): + """Raised when a file needs manual attention rather than a mechanical rewrite.""" + + +def _module_scope_nodes(tree: ast.Module): + """Yield nodes that execute at import, without descending into callables.""" + stack = list(tree.body) + while stack: + node = stack.pop() + yield node + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda): + continue + stack.extend(ast.iter_child_nodes(node)) + + +def _call_name(node: ast.AST) -> str | None: + if not isinstance(node, ast.Call): + return None + func = node.func + if isinstance(func, ast.Name): + return func.id + if isinstance(func, ast.Attribute): + return func.attr + return None + + +def _name_usage_count(tree: ast.Module, name: str) -> int: + """Count how many times ``name`` is loaded anywhere in the module.""" + return sum(1 for node in ast.walk(tree) if isinstance(node, ast.Name) and node.id == name) + + +def _find_launcher(tree: ast.Module) -> tuple[ast.stmt, ast.Call]: + """Return the module-scope statement that builds the app, and the ``AppLauncher`` call.""" + found = [] + for statement in tree.body: + for node in ast.walk(statement): + if _call_name(node) == "SimulationApp": + raise Unsupported("constructs SimulationApp directly") + if _call_name(node) == "AppLauncher": + found.append((statement, node)) + + if not found: + raise Unsupported("no module-scope AppLauncher call") + if len(found) > 1: + raise Unsupported(f"{len(found)} module-scope AppLauncher calls") + + statement, call = found[0] + + # The whole statement is replaced by a bare launch_kit() call, so the launch must be + # unconditional. A file that boots Kit only on some branch -- e.g. + # `AppLauncher(...).app if _USE_KIT else None`, used where a standalone wheel lets the + # tests run kitlessly -- would silently become an unconditional boot. Accept only + # ` = AppLauncher(...)`, ` = AppLauncher(...).app`, or a bare call. + value = statement.value if isinstance(statement, ast.Assign | ast.Expr) else None + if isinstance(value, ast.Attribute): + value = value.value + if value is not call: + raise Unsupported(f"AppLauncher launch is conditional or nested: `{ast.unparse(statement).splitlines()[0]}`") + + # `AppLauncher` must not be referenced for anything else, since its import is removed. + if _name_usage_count(tree, "AppLauncher") > 1: + raise Unsupported("`AppLauncher` is referenced beyond the launch call") + + return statement, call + + +def _resolve_cameras(call: ast.Call) -> bool: + """Map the AppLauncher keywords onto the ``cameras`` argument of ``launch_kit``.""" + if call.args: + raise Unsupported("AppLauncher called with positional arguments") + + cameras = False + for keyword in call.keywords: + if keyword.arg is None: + raise Unsupported("AppLauncher called with **kwargs") + value = keyword.value + literal = value.value if isinstance(value, ast.Constant) else None + + if keyword.arg == "headless": + # `headless=True`, or `headless=HEADLESS` where HEADLESS is a True constant. + if literal is not True and not isinstance(value, ast.Name): + raise Unsupported(f"headless={ast.unparse(value)} is not a literal True") + elif keyword.arg == "enable_cameras": + if not isinstance(literal, bool): + raise Unsupported(f"enable_cameras={ast.unparse(value)} is not a literal bool") + cameras = literal + elif keyword.arg == "device": + # launch_kit always applies resolve_test_sim_device(); anything else is a real + # difference in behaviour and must be looked at by hand. + if ast.unparse(value) != "resolve_test_sim_device()": + raise Unsupported(f"device={ast.unparse(value)} is not resolve_test_sim_device()") + else: + raise Unsupported(f"unsupported AppLauncher keyword {keyword.arg}=") + + return cameras + + +def _pytestmark_statement(tree: ast.Module) -> ast.Assign | None: + marks = [ + node + for node in tree.body + if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "pytestmark" for target in node.targets) + ] + if len(marks) > 1: + raise Unsupported("multiple module-scope pytestmark assignments; merge them first") + return marks[0] if marks else None + + +def _render_pytestmark(existing: ast.Assign | None, marker: str) -> str: + """Build the new ``pytestmark`` line with the Kit marker in front.""" + new = f"pytest.mark.{marker}" + if existing is None: + return f"pytestmark = {new}" + value = existing.value + if isinstance(value, ast.List | ast.Tuple): + parts = [new] + [ast.unparse(element) for element in value.elts] + else: + parts = [new, ast.unparse(value)] + return f"pytestmark = [{', '.join(parts)}]" + + +def _is_boilerplate_docstring(node: ast.stmt) -> bool: + return ( + isinstance(node, ast.Expr) + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + and node.value.value.strip() in _BOILERPLATE_DOCSTRINGS + ) + + +def migrate_source(source: str) -> tuple[str, str]: + """Return the rewritten source and the marker it should carry. + + Raises: + Unsupported: If the file needs manual attention. + """ + tree = ast.parse(source) + statement, call = _find_launcher(tree) + cameras = _resolve_cameras(call) + marker = "kit_cameras" if cameras else "kit" + existing_mark = _pytestmark_statement(tree) + + lines = source.splitlines() + # 1-indexed line numbers to drop entirely. + drop: set[int] = set() + # 1-indexed line number -> replacement text. + replace: dict[int, str] = {} + # 1-indexed line number -> text appended after that line. + insert_after: dict[int, list[str]] = {} + + # The launch statement becomes the launch_kit() call, in place, so that the Kit-dependent + # imports below it still run after Kit has started. + replace[statement.lineno] = "launch_kit(cameras=True)" if cameras else "launch_kit()" + drop.update(range(statement.lineno + 1, (statement.end_lineno or statement.lineno) + 1)) + + # `from isaaclab.app import AppLauncher` becomes the launch_kit import, keeping its slot. + app_import_replaced = False + for node in tree.body: + if isinstance(node, ast.ImportFrom) and node.module == _APP_IMPORT_MODULE: + names = [alias.name for alias in node.names] + if names == ["AppLauncher"]: + replace[node.lineno] = _LAUNCH_IMPORT + drop.update(range(node.lineno + 1, (node.end_lineno or node.lineno) + 1)) + app_import_replaced = True + else: + raise Unsupported(f"`from isaaclab.app import {', '.join(names)}` imports more than AppLauncher") + if not app_import_replaced: + raise Unsupported("no `from isaaclab.app import AppLauncher` to replace") + + # Drop `resolve_test_sim_device` imports that only existed to feed AppLauncher, and + # `HEADLESS = True` constants that nothing else reads. launch_kit covers both. + for node in tree.body: + if isinstance(node, ast.ImportFrom) and node.module == "isaaclab.test.utils": + names = [alias.name for alias in node.names] + if "resolve_test_sim_device" not in names or _name_usage_count(tree, "resolve_test_sim_device") != 1: + continue + remaining = [name for name in names if name != "resolve_test_sim_device"] + span = range(node.lineno, (node.end_lineno or node.lineno) + 1) + if remaining: + # Keep the other names; re-emit as a single line, which is how these imports + # are already written and how the formatter would leave them. + replace[node.lineno] = f"from {node.module} import {', '.join(remaining)}" + drop.update(list(span)[1:]) + else: + drop.update(span) + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if ( + isinstance(target, ast.Name) + and target.id in ("HEADLESS", "headless") + and _name_usage_count(tree, target.id) == 1 + ): + drop.update(range(node.lineno, (node.end_lineno or node.lineno) + 1)) + + # Drop the separator docstrings and comments that described the removed launch block. + for node in tree.body: + if _is_boilerplate_docstring(node): + drop.update(range(node.lineno, (node.end_lineno or node.lineno) + 1)) + for index, line in enumerate(lines, start=1): + if line.strip().lower() in _BOILERPLATE_COMMENTS: + drop.add(index) + + # Attach the marker, either by extending the existing pytestmark or by adding one after + # the last module-scope import (where such a declaration conventionally sits). + marked = _render_pytestmark(existing_mark, marker) + if existing_mark is not None: + replace[existing_mark.lineno] = marked + drop.update(range(existing_mark.lineno + 1, (existing_mark.end_lineno or existing_mark.lineno) + 1)) + else: + import_ends = [ + node.end_lineno or node.lineno for node in tree.body if isinstance(node, ast.Import | ast.ImportFrom) + ] + if not import_ends: + raise Unsupported("no imports to anchor a new pytestmark to") + if _name_usage_count(tree, "pytest") == 0 and not any( + isinstance(node, ast.Import) and any(a.name == "pytest" for a in node.names) for node in tree.body + ): + raise Unsupported("pytest is not imported, so a pytestmark cannot be added") + insert_after.setdefault(max(import_ends), []).append(marked) + + # Only the header is rewritten, so blank-line cleanup is confined to it. Collapsing + # runs across the whole file would also eat the blank lines PEP 8 requires between + # top-level definitions and produce a diff far larger than the change being made. + header_end = max([*drop, *replace, *insert_after, 1]) + + out: list[str] = [] + for index, line in enumerate(lines, start=1): + if index in replace: + emitted = replace[index] + elif index not in drop: + emitted = line + else: + emitted = None + + if emitted is not None: + in_header = index <= header_end + if not (in_header and not emitted.strip() and out and not out[-1].strip()): + out.append(emitted) + + for extra in insert_after.get(index, []): + out.extend(["", extra]) + + result = "\n".join(out).rstrip("\n") + "\n" + ast.parse(result) # refuse to emit anything that does not parse + return result, marker + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("paths", nargs="+", type=Path, help="files or directories to migrate") + parser.add_argument("--check", action="store_true", help="report what would change without writing") + args = parser.parse_args(argv) + + targets: list[Path] = [] + for path in args.paths: + targets.extend(sorted(path.rglob("test_*.py")) if path.is_dir() else [path]) + + changed, skipped = [], [] + for path in targets: + source = path.read_text(encoding="utf-8") + try: + new_source, marker = migrate_source(source) + except Unsupported as exc: + skipped.append((path, str(exc))) + continue + except SyntaxError as exc: + skipped.append((path, f"produced invalid syntax: {exc}")) + continue + if new_source != source and not args.check: + path.write_text(new_source, encoding="utf-8", newline="\n") + changed.append((path, marker)) + + for path, marker in changed: + print(f"{'would migrate' if args.check else 'migrated'}: {path.as_posix()} -> {marker}") + for path, reason in skipped: + print(f"skipped: {path.as_posix()}: {reason}", file=sys.stderr) + print(f"\n{len(changed)} migrated, {len(skipped)} skipped, {len(targets)} scanned") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From ded9dfd1ce59e694b862bbf3eedd8945bfd1b61e Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Sun, 2 Aug 2026 18:18:55 -0400 Subject: [PATCH 03/13] Add a CI probe measuring what sharing one Kit app saves Whether migrating the remaining ~125 test files off module-scope AppLauncher is worth doing depends on how much Kit startup actually costs, which is not something the current pipeline reports directly. Add two temporary jobs that run the same 30 files from source/isaaclab/test/sim and differ only in how many Kit apps they boot. kit-reuse-probe-per-file keeps the default test-path of "tools", so tools/conftest.py gives each file its own subprocess and Kit boots 30 times. kit-reuse-probe-batched points pytest at the files directly, so they share one process and launch_kit() boots Kit once. The difference between the two job durations is what reuse is worth per 30 files. Both jobs list their files explicitly instead of selecting with `-m kit`, because pytest's marker filtering deselects tests but still imports every collected module, and importing a kit_cameras module calls launch_kit(cameras=True) regardless of whether its tests will run. The batched job lists the four kit_cameras files first: a camera-enabled app can serve tests that do not need cameras, but cameras cannot be enabled after startup, so the opposite order makes launch_kit() raise. Files in TESTS_TO_SKIP are excluded from both sides so the jobs cover the same tests. To let a job bypass the per-file orchestrator, run-package-tests gains a test-path input. It defaults to "tools", the value that was previously hard-coded, so every existing caller is unaffected. Both jobs are continue-on-error and are meant to be deleted once the measurement is recorded. --- .github/actions/run-package-tests/action.yml | 9 +- .github/workflows/build.yaml | 116 +++++++++++++++++++ 2 files changed, 124 insertions(+), 1 deletion(-) diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index 6afdde7cbc73..ccd3fa55b68a 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -79,6 +79,13 @@ inputs: description: 'Additional pytest options' default: '' required: false + test-path: + description: >- + Path handed to pytest. Defaults to "tools", which loads tools/conftest.py and runs each + test file in its own subprocess. Point it at a test directory instead to run those files + together in a single pytest process, bypassing the per-file orchestrator. + default: 'tools' + required: false extra-pip-packages: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' @@ -291,7 +298,7 @@ runs: - name: Run Tests uses: ./.github/actions/run-tests with: - test-path: "tools" + test-path: ${{ inputs.test-path }} result-file: "${{ inputs.result-file != '' && inputs.result-file || format('{0}-report.xml', github.job) }}" container-name: "${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}" image-tag: ${{ inputs.image-tag }} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 853698d0aece..8b2699f5b9b1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -892,6 +892,122 @@ jobs: omni-github-test-type: warp-cache-warm #endregion + #region kit-reuse timing probe + # TEMPORARY. Measures what sharing one Kit app across test files is worth, so the decision to + # migrate the remaining ~125 files is based on a number rather than an estimate. Both jobs run + # the same 30 files from source/isaaclab/test/sim; the only difference is how many Kit apps get + # booted. Compare the two job durations in the Actions UI, then delete this region. + # + # The file lists are spelled out rather than selected with `-m kit` because pytest's marker + # filtering deselects tests but still imports every collected module, and importing a + # kit_cameras module calls launch_kit(cameras=True). Files in TESTS_TO_SKIP are left out of + # both sides so the two jobs cover exactly the same tests. + test-kit-reuse-probe-per-file: + name: "kit-reuse-probe-per-file" + runs-on: [self-hosted, gpu] + timeout-minutes: 120 + continue-on-error: true + needs: [build, config] + if: needs.build.result == 'success' + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + lfs: true + # Baseline: the default test-path of "tools" runs tools/conftest.py, which gives each file + # its own subprocess, so Kit boots 30 times. + - uses: ./.github/actions/run-package-tests + with: + image-tag: ${{ needs.config.outputs.ci_image_tag }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + filter-pattern: "isaaclab/test/sim" + include-files: >- + test_simulation_stage_in_memory.py, + test_utils_prims.py, + test_utils_queries.py, + test_utils_semantics.py, + test_articulation_fragments.py, + test_build_simulation_context_headless.py, + test_cloner.py, + test_collision_fragments.py, + test_joint_drive_fragments.py, + test_mass_fragments.py, + test_material_fragments.py, + test_mesh_collision_fragments.py, + test_mesh_converter.py, + test_schema_fragments.py, + test_schema_writer_nested_targets.py, + test_schemas.py, + test_simulation_context.py, + test_spawn_from_files.py, + test_spawn_lights.py, + test_spawn_materials.py, + test_spawn_meshes.py, + test_spawn_sensors.py, + test_spawn_shapes.py, + test_spawn_wrappers.py, + test_tendon_fragments.py, + test_utils_stage.py, + test_utils_transforms.py, + test_views_xform_prim.py + container-name: isaac-lab-kit-reuse-probe-per-file + omni-github-test-type: kit-reuse-probe-per-file + + test-kit-reuse-probe-batched: + name: "kit-reuse-probe-batched" + runs-on: [self-hosted, gpu] + timeout-minutes: 120 + continue-on-error: true + needs: [build, config] + if: needs.build.result == 'success' + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + lfs: true + # Batched: pointing test-path at the files themselves bypasses tools/conftest.py, so all 30 + # run in one pytest process and launch_kit() boots Kit once. The four kit_cameras files are + # listed first on purpose: a camera-enabled app can serve tests that do not need cameras, but + # cameras cannot be turned on after startup, so the reverse order makes launch_kit() raise. + - uses: ./.github/actions/run-package-tests + with: + image-tag: ${{ needs.config.outputs.ci_image_tag }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + test-path: >- + source/isaaclab/test/sim/test_simulation_stage_in_memory.py + source/isaaclab/test/sim/test_utils_prims.py + source/isaaclab/test/sim/test_utils_queries.py + source/isaaclab/test/sim/test_utils_semantics.py + source/isaaclab/test/sim/test_articulation_fragments.py + source/isaaclab/test/sim/test_build_simulation_context_headless.py + source/isaaclab/test/sim/test_cloner.py + source/isaaclab/test/sim/test_collision_fragments.py + source/isaaclab/test/sim/test_joint_drive_fragments.py + source/isaaclab/test/sim/test_mass_fragments.py + source/isaaclab/test/sim/test_material_fragments.py + source/isaaclab/test/sim/test_mesh_collision_fragments.py + source/isaaclab/test/sim/test_mesh_converter.py + source/isaaclab/test/sim/test_schema_fragments.py + source/isaaclab/test/sim/test_schema_writer_nested_targets.py + source/isaaclab/test/sim/test_schemas.py + source/isaaclab/test/sim/test_simulation_context.py + source/isaaclab/test/sim/test_spawn_from_files.py + source/isaaclab/test/sim/test_spawn_lights.py + source/isaaclab/test/sim/test_spawn_materials.py + source/isaaclab/test/sim/test_spawn_meshes.py + source/isaaclab/test/sim/test_spawn_sensors.py + source/isaaclab/test/sim/test_spawn_shapes.py + source/isaaclab/test/sim/test_spawn_wrappers.py + source/isaaclab/test/sim/test_tendon_fragments.py + source/isaaclab/test/sim/test_utils_stage.py + source/isaaclab/test/sim/test_utils_transforms.py + source/isaaclab/test/sim/test_views_xform_prim.py + container-name: isaac-lab-kit-reuse-probe-batched + omni-github-test-type: kit-reuse-probe-batched + #endregion + #region disabled quarantined tests # test-quarantined: # name: "Quarantined Tests" From 0c714ba82d9a556443c55e38607cff37f3f0f015 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Sun, 2 Aug 2026 19:48:42 -0400 Subject: [PATCH 04/13] Keep the cold-cache buffer working for migrated camera tests The per-file runner grants the first camera-enabled test file an extra 700 s of timeout, because that file compiles RTX shaders (~600 s) on a cold cache. It identified such files by searching their source for the literal string "enable_cameras=True". Migrating a file to launch_kit(cameras=True) removes that literal, so the buffer stopped being applied and the file was killed at the 120 s startup deadline instead. That is what happened to test_simulation_stage_in_memory.py in the kit-reuse-probe-per-file job: it was reported as a startup hang at 120.94 s having run no tests. Match the marker and the launch_kit call as well as the old literal, so the buffer applies both before and after a file is migrated. Also narrow the probe to the 24 `kit` files and drop the four `kit_cameras` ones from both sides. The cold shader compile is roughly thirty times the Kit startup the probe is trying to measure, so including those files tells us about shader caching rather than about app reuse. --- .github/workflows/build.yaml | 28 +++++++++++----------------- tools/conftest.py | 17 ++++++++++++++++- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8b2699f5b9b1..3a29cdb3dcbe 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -895,13 +895,17 @@ jobs: #region kit-reuse timing probe # TEMPORARY. Measures what sharing one Kit app across test files is worth, so the decision to # migrate the remaining ~125 files is based on a number rather than an estimate. Both jobs run - # the same 30 files from source/isaaclab/test/sim; the only difference is how many Kit apps get - # booted. Compare the two job durations in the Actions UI, then delete this region. + # the same 24 `kit` files from source/isaaclab/test/sim; the only difference is how many Kit + # apps get booted. Compare the two job durations in the Actions UI, then delete this region. + # + # The four `kit_cameras` files in that directory are excluded from both sides. The first + # camera-enabled boot in a fresh container compiles shaders for ~600 s, which is an order of + # magnitude larger than the Kit startup being measured and would swamp the comparison. # # The file lists are spelled out rather than selected with `-m kit` because pytest's marker - # filtering deselects tests but still imports every collected module, and importing a - # kit_cameras module calls launch_kit(cameras=True). Files in TESTS_TO_SKIP are left out of - # both sides so the two jobs cover exactly the same tests. + # filtering deselects tests but still imports every collected module, so `-m` alone cannot keep + # a kit_cameras module from calling launch_kit(cameras=True). Files in TESTS_TO_SKIP are left + # out of both sides so the two jobs cover exactly the same tests. test-kit-reuse-probe-per-file: name: "kit-reuse-probe-per-file" runs-on: [self-hosted, gpu] @@ -923,10 +927,6 @@ jobs: isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} filter-pattern: "isaaclab/test/sim" include-files: >- - test_simulation_stage_in_memory.py, - test_utils_prims.py, - test_utils_queries.py, - test_utils_semantics.py, test_articulation_fragments.py, test_build_simulation_context_headless.py, test_cloner.py, @@ -966,20 +966,14 @@ jobs: with: fetch-depth: 1 lfs: true - # Batched: pointing test-path at the files themselves bypasses tools/conftest.py, so all 30 - # run in one pytest process and launch_kit() boots Kit once. The four kit_cameras files are - # listed first on purpose: a camera-enabled app can serve tests that do not need cameras, but - # cameras cannot be turned on after startup, so the reverse order makes launch_kit() raise. + # Batched: pointing test-path at the files themselves bypasses tools/conftest.py, so all 24 + # run in one pytest process and launch_kit() boots Kit once. - uses: ./.github/actions/run-package-tests with: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} test-path: >- - source/isaaclab/test/sim/test_simulation_stage_in_memory.py - source/isaaclab/test/sim/test_utils_prims.py - source/isaaclab/test/sim/test_utils_queries.py - source/isaaclab/test/sim/test_utils_semantics.py source/isaaclab/test/sim/test_articulation_fragments.py source/isaaclab/test/sim/test_build_simulation_context_headless.py source/isaaclab/test/sim/test_cloner.py diff --git a/tools/conftest.py b/tools/conftest.py index b391c8ba0dee..ad4023345225 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -42,6 +42,21 @@ def pytest_ignore_collect(collection_path, config): on-disk cache is populated. """ +_CAMERA_MARKERS = ("enable_cameras=True", "launch_kit(cameras=True)", "pytest.mark.kit_cameras") +"""Source-text signatures of a test file that starts Kit with cameras enabled. + +Matched against the file's text rather than by importing it, because importing a test +module boots Kit. ``enable_cameras=True`` covers files that still construct +``AppLauncher`` directly; the other two cover files migrated to +:func:`~isaaclab.test.launch.launch_kit`, which no longer contain that literal. +""" + + +def _enables_cameras(test_content: str) -> bool: + """Whether the given test file's source starts Kit with cameras enabled.""" + return any(marker in test_content for marker in _CAMERA_MARKERS) + + STARTUP_DEADLINE = 120 """Seconds to wait for AppLauncher init or pytest collection before declaring a startup hang. @@ -1001,7 +1016,7 @@ def run_individual_tests(test_files, workspace_root, ci_marker, test_node_ids_by # The first camera-enabled test in a fresh container compiles shaders # (~600 s). Give it extra time so that doesn't look like a test timeout. - is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content + is_cold_cache_test = not cold_cache_applied and _enables_cameras(test_content) if is_cold_cache_test: timeout += COLD_CACHE_BUFFER cold_cache_applied = True From 505971275a80dbc4e6308bc752bba064007a349d Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 3 Aug 2026 15:13:27 -0400 Subject: [PATCH 05/13] Record the two files that do not tolerate a shared Kit app The kit-reuse-probe-batched job surfaced two ways a test file can misbehave once it no longer has a Kit process to itself. test_simulation_stage_in_memory.py aborted the interpreter immediately after collection, with no Python traceback, while the same test passes in its own process. Creating the stage in memory is sensitive to what else has already touched the stage or the extension set. The cause is not understood yet, so mark the file kit_solo to keep it out of any future batching rather than leave a landmine for whoever wires that up. test_views_xform_prim.py calls enable_extension() at module scope, so in a shared process it mutates the running app's extension set during collection, before any test runs. That is harmless today and the file is not being changed, but it is the kind of import-time side effect that batching turns into a cross-file interaction, so say so at the call site. Neither change affects how these tests run today; both files still get their own process from tools/conftest.py. --- .../isaaclab/test/sim/test_simulation_stage_in_memory.py | 7 ++++++- source/isaaclab/test/sim/test_views_xform_prim.py | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/source/isaaclab/test/sim/test_simulation_stage_in_memory.py b/source/isaaclab/test/sim/test_simulation_stage_in_memory.py index bda761131630..450d893d4ed3 100644 --- a/source/isaaclab/test/sim/test_simulation_stage_in_memory.py +++ b/source/isaaclab/test/sim/test_simulation_stage_in_memory.py @@ -22,7 +22,12 @@ from isaaclab.utils.assets import ISAACLAB_NUCLEUS_DIR from isaaclab.utils.version import get_isaac_sim_version -pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration] +# kit_solo: sharing a Kit app with other test files killed the pytest process here. In the +# kit-reuse-probe-batched CI job this file's first test aborted the interpreter immediately +# after collection, with no Python traceback, while the same test is fine in its own process. +# The cause is not yet understood -- creating the stage in memory is sensitive to what else has +# already touched the stage or the extension set -- so keep the file on its own until it is. +pytestmark = [pytest.mark.kit_cameras, pytest.mark.kit_solo, pytest.mark.integration] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_views_xform_prim.py b/source/isaaclab/test/sim/test_views_xform_prim.py index dfa5ad2372ad..c3b5a91ae9ad 100644 --- a/source/isaaclab/test/sim/test_views_xform_prim.py +++ b/source/isaaclab/test/sim/test_views_xform_prim.py @@ -24,6 +24,9 @@ try: from isaaclab.sim.utils import enable_extension # noqa: E402 + # NOTE: this runs at import, so in a process shared with other test files it changes the + # running app's extension set during collection, before any test executes. Harmless when + # this file has the process to itself; a hazard once files are batched together. enable_extension("isaacsim.core.experimental.prims") from isaacsim.core.experimental.prims import XformPrim as _IsaacSimXformPrimView except (ModuleNotFoundError, ImportError, RuntimeError): From 6f10aec250193119af1cca92ad063ccdcd96360d Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 3 Aug 2026 15:26:10 -0400 Subject: [PATCH 06/13] Batch every sim test file that can share a Kit app The batched probe was passing over files rather than classifying them: the four kit_cameras files had been dropped from both sides to keep the measurement clean, which meant three files that can share an app were being run one-app-each for no reason. Mark the files that genuinely cannot share, and batch everything else. test_views_xform_prim.py is the one this run identified. Its test_compare_get_world_poses_with_isaacsim reaches Isaac Sim's SimulationManager, a process-global singleton that caches the PhysxScene wrapping /physicsScene. In a shared process that prim belongs to a stage an earlier file has already torn down, so the cached wrapper is dangling and the test fails with "Accessed invalid expired 'PhysicsScene' prim". The other 62 tests in the file are fine; the marker is per file, so the file goes solo until SimulationManager can be reset between files. That leaves 26 of the directory's files sharing one app, up from 24, with two marked kit_solo and one already in TESTS_TO_SKIP. The three kit_cameras files are listed first in the batched job because a camera-enabled app can serve tests that do not need cameras while the reverse makes launch_kit() raise. Both jobs run the identical 26 so the durations stay comparable. --- .github/workflows/build.yaml | 32 ++++++++++++------- .../test/sim/test_views_xform_prim.py | 8 ++++- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3a29cdb3dcbe..a152d31cec7b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -895,17 +895,23 @@ jobs: #region kit-reuse timing probe # TEMPORARY. Measures what sharing one Kit app across test files is worth, so the decision to # migrate the remaining ~125 files is based on a number rather than an estimate. Both jobs run - # the same 24 `kit` files from source/isaaclab/test/sim; the only difference is how many Kit - # apps get booted. Compare the two job durations in the Actions UI, then delete this region. + # the same 26 files from source/isaaclab/test/sim; the only difference is how many Kit apps get + # booted. Compare the two job durations in the Actions UI, then delete this region. # - # The four `kit_cameras` files in that directory are excluded from both sides. The first - # camera-enabled boot in a fresh container compiles shaders for ~600 s, which is an order of - # magnitude larger than the Kit startup being measured and would swamp the comparison. + # The 26 are every file in that directory that can share a Kit app. Excluded are the two marked + # kit_solo, which demonstrably cannot -- see the comments on their pytestmark for what each one + # does to a shared process -- and anything in TESTS_TO_SKIP. Both jobs use the identical set so + # the durations stay comparable. # - # The file lists are spelled out rather than selected with `-m kit` because pytest's marker + # The three `kit_cameras` files come first in the batched list. A camera-enabled app can serve + # tests that do not need cameras, but cameras cannot be turned on after startup, so the reverse + # order would make launch_kit() raise. Both sides pay the one-off ~600 s cold shader compile + # that the first camera-enabled boot in a fresh container incurs, so it does not bias the + # comparison. + # + # The file lists are spelled out rather than selected with `-m` because pytest's marker # filtering deselects tests but still imports every collected module, so `-m` alone cannot keep - # a kit_cameras module from calling launch_kit(cameras=True). Files in TESTS_TO_SKIP are left - # out of both sides so the two jobs cover exactly the same tests. + # a kit_cameras module from calling launch_kit(cameras=True). test-kit-reuse-probe-per-file: name: "kit-reuse-probe-per-file" runs-on: [self-hosted, gpu] @@ -927,6 +933,9 @@ jobs: isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} filter-pattern: "isaaclab/test/sim" include-files: >- + test_utils_prims.py, + test_utils_queries.py, + test_utils_semantics.py, test_articulation_fragments.py, test_build_simulation_context_headless.py, test_cloner.py, @@ -949,8 +958,7 @@ jobs: test_spawn_wrappers.py, test_tendon_fragments.py, test_utils_stage.py, - test_utils_transforms.py, - test_views_xform_prim.py + test_utils_transforms.py container-name: isaac-lab-kit-reuse-probe-per-file omni-github-test-type: kit-reuse-probe-per-file @@ -974,6 +982,9 @@ jobs: isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} test-path: >- + source/isaaclab/test/sim/test_utils_prims.py + source/isaaclab/test/sim/test_utils_queries.py + source/isaaclab/test/sim/test_utils_semantics.py source/isaaclab/test/sim/test_articulation_fragments.py source/isaaclab/test/sim/test_build_simulation_context_headless.py source/isaaclab/test/sim/test_cloner.py @@ -997,7 +1008,6 @@ jobs: source/isaaclab/test/sim/test_tendon_fragments.py source/isaaclab/test/sim/test_utils_stage.py source/isaaclab/test/sim/test_utils_transforms.py - source/isaaclab/test/sim/test_views_xform_prim.py container-name: isaac-lab-kit-reuse-probe-batched omni-github-test-type: kit-reuse-probe-batched #endregion diff --git a/source/isaaclab/test/sim/test_views_xform_prim.py b/source/isaaclab/test/sim/test_views_xform_prim.py index c3b5a91ae9ad..8b56f5f39f61 100644 --- a/source/isaaclab/test/sim/test_views_xform_prim.py +++ b/source/isaaclab/test/sim/test_views_xform_prim.py @@ -39,7 +39,13 @@ from isaaclab.sim.views import UsdFrameView as FrameView # noqa: E402 from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR # noqa: E402 -pytestmark = [pytest.mark.kit, pytest.mark.integration, pytest.mark.isaacsim_ci] +# kit_solo: test_compare_get_world_poses_with_isaacsim goes through Isaac Sim's +# SimulationManager, a process-global singleton that caches the PhysxScene wrapping +# /physicsScene. In a process shared with other test files that prim belongs to a stage an +# earlier file already tore down, so the cached wrapper is dangling and the test dies with +# "Accessed invalid expired 'PhysicsScene' prim". Nothing in this file owns that state, so the +# file needs a process to itself until SimulationManager can be reset between files. +pytestmark = [pytest.mark.kit, pytest.mark.kit_solo, pytest.mark.integration, pytest.mark.isaacsim_ci] PARENT_POS = (0.0, 0.0, 1.0) From 207b26c0a19bba9d5c8ac1ffccc3c95a338d8b89 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 3 Aug 2026 15:31:37 -0400 Subject: [PATCH 07/13] Derive the batched file list from the markers instead of listing it Both probe jobs carried a hand-written list of the files that can share a Kit app, duplicated between them in two different formats. That has to be edited by hand whenever a file is added, renamed, or reclassified, and the two copies have to be kept identical or the timing comparison silently stops comparing like with like. A stale list is wrong quietly rather than loudly. The markers already record which files can share an app, so make them the only source. tools/kit_test_files.py selects the files marked kit or kit_cameras, drops those marked kit_solo and those in TESTS_TO_SKIP, and puts the kit_cameras files first because a camera-enabled app can serve tests that do not need cameras while the reverse makes launch_kit() raise. Each job calls it in a step and passes the result through, so the two jobs cannot disagree with each other or with the markers. A marker expression still cannot replace this: pytest's -m deselects tests but imports every collected module regardless, so it cannot stop a kit_cameras module from calling launch_kit(cameras=True) in a run that booted without cameras. The list has to be settled before pytest starts. Markers are read from the source text rather than by importing the modules, since importing a Kit-dependent test module boots Kit. test_kit_marker_contract.py now also checks the two invariants a caller depends on: the derived list matches the markers, and cameras sort first. Verified the ordering check fails when the script's ordering is reversed. --- .github/workflows/build.yaml | 101 +++++------------ .../isaaclab/test/test_kit_marker_contract.py | 42 +++++++ tools/kit_test_files.py | 103 ++++++++++++++++++ 3 files changed, 175 insertions(+), 71 deletions(-) create mode 100644 tools/kit_test_files.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a152d31cec7b..8abd1cba9669 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -895,23 +895,18 @@ jobs: #region kit-reuse timing probe # TEMPORARY. Measures what sharing one Kit app across test files is worth, so the decision to # migrate the remaining ~125 files is based on a number rather than an estimate. Both jobs run - # the same 26 files from source/isaaclab/test/sim; the only difference is how many Kit apps get + # the same files from source/isaaclab/test/sim; the only difference is how many Kit apps get # booted. Compare the two job durations in the Actions UI, then delete this region. # - # The 26 are every file in that directory that can share a Kit app. Excluded are the two marked - # kit_solo, which demonstrably cannot -- see the comments on their pytestmark for what each one - # does to a shared process -- and anything in TESTS_TO_SKIP. Both jobs use the identical set so - # the durations stay comparable. + # Neither job hardcodes a file list. tools/kit_test_files.py derives it from the kit / + # kit_cameras / kit_solo markers, so the two jobs cannot drift apart from each other or from + # the markers as files are added, renamed, or reclassified. It also fixes the order: the + # kit_cameras files come first, because a camera-enabled app can serve tests that do not need + # cameras while the reverse makes launch_kit() raise. # - # The three `kit_cameras` files come first in the batched list. A camera-enabled app can serve - # tests that do not need cameras, but cameras cannot be turned on after startup, so the reverse - # order would make launch_kit() raise. Both sides pay the one-off ~600 s cold shader compile - # that the first camera-enabled boot in a fresh container incurs, so it does not bias the - # comparison. - # - # The file lists are spelled out rather than selected with `-m` because pytest's marker - # filtering deselects tests but still imports every collected module, so `-m` alone cannot keep - # a kit_cameras module from calling launch_kit(cameras=True). + # A marker expression cannot replace the explicit list here: pytest's -m deselects tests but + # still imports every collected module, so it cannot stop a kit_cameras module from calling + # launch_kit(cameras=True) in a run that booted without cameras. test-kit-reuse-probe-per-file: name: "kit-reuse-probe-per-file" runs-on: [self-hosted, gpu] @@ -924,41 +919,23 @@ jobs: with: fetch-depth: 1 lfs: true + - name: Resolve shareable test files + id: files + shell: bash + run: | + set -euo pipefail + names=$(python3 tools/kit_test_files.py source/isaaclab/test/sim --format names) + echo "names=$names" >> "$GITHUB_OUTPUT" + echo "Resolved $(echo "$names" | tr ',' ' ' | wc -w) shareable test files" # Baseline: the default test-path of "tools" runs tools/conftest.py, which gives each file - # its own subprocess, so Kit boots 30 times. + # its own subprocess, so Kit boots once per file. - uses: ./.github/actions/run-package-tests with: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} filter-pattern: "isaaclab/test/sim" - include-files: >- - test_utils_prims.py, - test_utils_queries.py, - test_utils_semantics.py, - test_articulation_fragments.py, - test_build_simulation_context_headless.py, - test_cloner.py, - test_collision_fragments.py, - test_joint_drive_fragments.py, - test_mass_fragments.py, - test_material_fragments.py, - test_mesh_collision_fragments.py, - test_mesh_converter.py, - test_schema_fragments.py, - test_schema_writer_nested_targets.py, - test_schemas.py, - test_simulation_context.py, - test_spawn_from_files.py, - test_spawn_lights.py, - test_spawn_materials.py, - test_spawn_meshes.py, - test_spawn_sensors.py, - test_spawn_shapes.py, - test_spawn_wrappers.py, - test_tendon_fragments.py, - test_utils_stage.py, - test_utils_transforms.py + include-files: ${{ steps.files.outputs.names }} container-name: isaac-lab-kit-reuse-probe-per-file omni-github-test-type: kit-reuse-probe-per-file @@ -974,40 +951,22 @@ jobs: with: fetch-depth: 1 lfs: true - # Batched: pointing test-path at the files themselves bypasses tools/conftest.py, so all 24 - # run in one pytest process and launch_kit() boots Kit once. + - name: Resolve shareable test files + id: files + shell: bash + run: | + set -euo pipefail + paths=$(python3 tools/kit_test_files.py source/isaaclab/test/sim --format paths) + echo "paths=$paths" >> "$GITHUB_OUTPUT" + echo "Resolved $(echo "$paths" | wc -w) shareable test files" + # Batched: pointing test-path at the files themselves bypasses tools/conftest.py, so they + # all run in one pytest process and launch_kit() boots Kit once. - uses: ./.github/actions/run-package-tests with: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - test-path: >- - source/isaaclab/test/sim/test_utils_prims.py - source/isaaclab/test/sim/test_utils_queries.py - source/isaaclab/test/sim/test_utils_semantics.py - source/isaaclab/test/sim/test_articulation_fragments.py - source/isaaclab/test/sim/test_build_simulation_context_headless.py - source/isaaclab/test/sim/test_cloner.py - source/isaaclab/test/sim/test_collision_fragments.py - source/isaaclab/test/sim/test_joint_drive_fragments.py - source/isaaclab/test/sim/test_mass_fragments.py - source/isaaclab/test/sim/test_material_fragments.py - source/isaaclab/test/sim/test_mesh_collision_fragments.py - source/isaaclab/test/sim/test_mesh_converter.py - source/isaaclab/test/sim/test_schema_fragments.py - source/isaaclab/test/sim/test_schema_writer_nested_targets.py - source/isaaclab/test/sim/test_schemas.py - source/isaaclab/test/sim/test_simulation_context.py - source/isaaclab/test/sim/test_spawn_from_files.py - source/isaaclab/test/sim/test_spawn_lights.py - source/isaaclab/test/sim/test_spawn_materials.py - source/isaaclab/test/sim/test_spawn_meshes.py - source/isaaclab/test/sim/test_spawn_sensors.py - source/isaaclab/test/sim/test_spawn_shapes.py - source/isaaclab/test/sim/test_spawn_wrappers.py - source/isaaclab/test/sim/test_tendon_fragments.py - source/isaaclab/test/sim/test_utils_stage.py - source/isaaclab/test/sim/test_utils_transforms.py + test-path: ${{ steps.files.outputs.paths }} container-name: isaac-lab-kit-reuse-probe-batched omni-github-test-type: kit-reuse-probe-batched #endregion diff --git a/source/isaaclab/test/test_kit_marker_contract.py b/source/isaaclab/test/test_kit_marker_contract.py index 20c6ba41ed5c..0b9ec15da9c0 100644 --- a/source/isaaclab/test/test_kit_marker_contract.py +++ b/source/isaaclab/test/test_kit_marker_contract.py @@ -303,6 +303,48 @@ def test_migrated_packages_declare_a_marker(facts: list[_FileFacts]): ) +def test_shareable_file_list_is_derived_from_the_markers(): + """``tools/kit_test_files.py`` must agree with the markers, and order cameras first. + + CI batches test files by asking that script which ones can share a Kit app, instead of + carrying a hand-written list that goes stale as files are added or reclassified. These are + the invariants a caller relies on. + """ + sys.path.insert(0, str(_REPO_ROOT / "tools")) + from kit_test_files import shareable_test_files # noqa: PLC0415 + from test_settings import TESTS_TO_SKIP # noqa: PLC0415 + + directory = _REPO_ROOT / "source" / "isaaclab" / "test" / "sim" + selected = shareable_test_files(directory) + names = [path.name for path in selected] + assert names, f"no shareable files found in {directory}" + assert len(names) == len(set(names)), f"duplicate entries: {names}" + + sources = {path.name: path.read_text(encoding="utf-8") for path in directory.glob("test_*.py")} + + def marks(name: str, marker: str) -> bool: + return f"pytest.mark.{marker}" in sources[name] + + expected = { + name + for name, source in sources.items() + if name not in TESTS_TO_SKIP and "pytest.mark.kit" in source and "pytest.mark.kit_solo" not in source + } + assert set(names) == expected, ( + "the derived list disagrees with the markers:" + f"\n only in list: {sorted(set(names) - expected)}" + f"\n only in markers: {sorted(expected - set(names))}" + ) + + # A camera-enabled app can serve tests that do not need cameras, but cameras cannot be + # enabled after startup, so every kit_cameras file must precede every plain kit file. + is_camera = [marks(name, "kit_cameras") for name in names] + assert is_camera == sorted(is_camera, reverse=True), ( + "kit_cameras files must come first, otherwise a plain `kit` file boots the app without" + f" cameras and the later launch_kit(cameras=True) raises. Got: {names}" + ) + + def test_kitless_files_import_without_kit(facts: list[_FileFacts]): """Importing every `kitless` module must not pull in Kit through a helper module. diff --git a/tools/kit_test_files.py b/tools/kit_test_files.py new file mode 100644 index 000000000000..370fe02a0205 --- /dev/null +++ b/tools/kit_test_files.py @@ -0,0 +1,103 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""List the test files in a directory that can share one Kit app, in a safe order. + +The ``kit`` / ``kit_cameras`` / ``kit_solo`` markers already record which files can share a +Kit app; this turns that into the file list a runner needs, so the two never drift. Anything +that hardcodes such a list has to be updated by hand whenever a file is added, renamed, or +reclassified, and a stale list is silently wrong rather than loudly broken. + +Selection: every file marked ``kit`` or ``kit_cameras``, minus those marked ``kit_solo`` and +those in :data:`tools.test_settings.TESTS_TO_SKIP`. + +Order: ``kit_cameras`` files first. A camera-enabled app can serve tests that do not need +cameras, but cameras cannot be enabled after startup, so a plain ``kit`` file booting first +would make a later ``launch_kit(cameras=True)`` raise. + +Markers are read from the file's source rather than by importing it, because importing a +Kit-dependent test module boots Kit. + +Usage:: + + python3 tools/kit_test_files.py source/isaaclab/test/sim --format paths + python3 tools/kit_test_files.py source/isaaclab/test/sim --format names +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +# `kit` must not match `kit_cameras` or `kit_solo`, hence the boundary on the plain pattern. +_MARK_KIT = re.compile(r"pytest\.mark\.kit(?![\w])") +_MARK_CAMERAS = re.compile(r"pytest\.mark\.kit_cameras\b") +_MARK_SOLO = re.compile(r"pytest\.mark\.kit_solo\b") + + +def _tests_to_skip() -> frozenset[str]: + """Names from ``tools/test_settings.py``, which the per-file runner also honours.""" + sys.path.insert(0, str(Path(__file__).resolve().parent)) + try: + from test_settings import TESTS_TO_SKIP # noqa: PLC0415 + except ImportError: + return frozenset() + return frozenset(TESTS_TO_SKIP) + + +def shareable_test_files(directory: Path) -> list[Path]: + """Return the files under ``directory`` that can share a Kit app, cameras first. + + Args: + directory: Directory to scan, non-recursively matching ``test_*.py``. + + Returns: + The selected files: ``kit_cameras`` ones first, each group sorted by name. + """ + skip = _tests_to_skip() + cameras, plain = [], [] + for path in sorted(directory.glob("test_*.py")): + if path.name in skip: + continue + source = path.read_text(encoding="utf-8", errors="replace") + if _MARK_SOLO.search(source): + continue + if _MARK_CAMERAS.search(source): + cameras.append(path) + elif _MARK_KIT.search(source): + plain.append(path) + return cameras + plain + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path, help="directory to scan for test files") + parser.add_argument( + "--format", + choices=("paths", "names"), + default="paths", + help="'paths' for space-separated repo paths (pytest arguments); " + "'names' for comma-separated file names (the include-files input)", + ) + args = parser.parse_args(argv) + + if not args.directory.is_dir(): + parser.error(f"not a directory: {args.directory}") + + files = shareable_test_files(args.directory) + if not files: + parser.error(f"no Kit-marked test files found in {args.directory}") + + if args.format == "paths": + print(" ".join(path.as_posix() for path in files)) + else: + print(",".join(path.name for path in files)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 12b8bb69bded0619190ad47747fd28b66d15960d Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Tue, 4 Aug 2026 15:40:46 -0400 Subject: [PATCH 08/13] Keep kit and kit_cameras files in separate processes The probe batched the kit_cameras files together with the plain kit ones, on the assumption that a camera-enabled app is a superset: it can serve tests that do not need cameras, so booting cameras first would satisfy everyone. That is wrong, and CI showed exactly where. test_simulation_context.py's test_headless_mode asserts not sim.has_gui and not sim.has_offscreen_render so it fails in an app that was booted with cameras. The evidence is clean: in the batch that contained no camera files that test passed 43/43, and in the batch that booted cameras first it was the single failure out of 462 tests. So the relationship is not superset but mutual exclusion. Cameras cannot be enabled after startup, which rules out one order, and some tests require them to be off, which rules out the other. Treat the two as separate batches. launch_kit() now raises on any mismatch rather than only when cameras are requested after a plain boot, so a file can never silently receive an app configured differently from what its marker declares. kit_test_files.py takes a --profile and returns one group, which also removes the cameras-first ordering it previously had to arrange. Both probe jobs ask for the kit group: it is much the larger, and a camera batch would mostly measure the one-off ~600 s cold shader compile rather than Kit startup. The contract test now checks each profile against the markers separately and asserts the two groups do not overlap, replacing the ordering check that this change makes meaningless. --- .github/workflows/build.yaml | 21 ++++--- source/isaaclab/isaaclab/test/launch.py | 28 +++++++--- .../isaaclab/test/test_kit_marker_contract.py | 56 ++++++++++--------- tools/kit_test_files.py | 53 ++++++++++++------ 4 files changed, 96 insertions(+), 62 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8abd1cba9669..f18a72398487 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -900,13 +900,18 @@ jobs: # # Neither job hardcodes a file list. tools/kit_test_files.py derives it from the kit / # kit_cameras / kit_solo markers, so the two jobs cannot drift apart from each other or from - # the markers as files are added, renamed, or reclassified. It also fixes the order: the - # kit_cameras files come first, because a camera-enabled app can serve tests that do not need - # cameras while the reverse makes launch_kit() raise. + # the markers as files are added, renamed, or reclassified. # - # A marker expression cannot replace the explicit list here: pytest's -m deselects tests but - # still imports every collected module, so it cannot stop a kit_cameras module from calling - # launch_kit(cameras=True) in a run that booted without cameras. + # Both select --profile kit. The kit and kit_cameras groups are separate batches and never + # share a process: cameras cannot be enabled after startup, and a camera-enabled app is not a + # drop-in for a plain one either, since test_simulation_context.py::test_headless_mode asserts + # that offscreen rendering is off. The kit group is the one worth measuring -- it is much the + # larger, and a camera batch is dominated by the one-off ~600 s cold shader compile rather + # than by Kit startup. + # + # A marker expression cannot replace the resolved list: pytest's -m deselects tests but still + # imports every collected module, so it cannot stop a module from calling launch_kit() with + # the wrong profile in a run that booted the other one. test-kit-reuse-probe-per-file: name: "kit-reuse-probe-per-file" runs-on: [self-hosted, gpu] @@ -924,7 +929,7 @@ jobs: shell: bash run: | set -euo pipefail - names=$(python3 tools/kit_test_files.py source/isaaclab/test/sim --format names) + names=$(python3 tools/kit_test_files.py source/isaaclab/test/sim --profile kit --format names) echo "names=$names" >> "$GITHUB_OUTPUT" echo "Resolved $(echo "$names" | tr ',' ' ' | wc -w) shareable test files" # Baseline: the default test-path of "tools" runs tools/conftest.py, which gives each file @@ -956,7 +961,7 @@ jobs: shell: bash run: | set -euo pipefail - paths=$(python3 tools/kit_test_files.py source/isaaclab/test/sim --format paths) + paths=$(python3 tools/kit_test_files.py source/isaaclab/test/sim --profile kit --format paths) echo "paths=$paths" >> "$GITHUB_OUTPUT" echo "Resolved $(echo "$paths" | wc -w) shareable test files" # Batched: pointing test-path at the files themselves bypasses tools/conftest.py, so they diff --git a/source/isaaclab/isaaclab/test/launch.py b/source/isaaclab/isaaclab/test/launch.py index 1b9f423f6b4e..1234ef57501a 100644 --- a/source/isaaclab/isaaclab/test/launch.py +++ b/source/isaaclab/isaaclab/test/launch.py @@ -25,6 +25,12 @@ pytestmark = pytest.mark.kit # launch_kit() pytestmark = pytest.mark.kit_cameras # launch_kit(cameras=True) + +The two groups cannot be merged. Cameras cannot be enabled after startup, so a plain ``kit`` +file cannot run in a process a ``kit_cameras`` file will later join; and a camera-enabled app +is not a drop-in replacement for a plain one either, because some tests assert that offscreen +rendering is off. :func:`launch_kit` therefore raises on any mismatch rather than handing back +an app whose configuration is not the one the caller asked for. """ from __future__ import annotations @@ -49,20 +55,24 @@ def launch_kit(*, cameras: bool = False) -> Any: The running ``SimulationApp``. Raises: - RuntimeError: If a camera-enabled app is requested but Kit is already running in - this process without cameras, or if Kit was started by something other than - this function. Both mean the test files sharing this process do not share a - launch configuration and must be split across processes. + RuntimeError: If the running app was booted with a different ``cameras`` setting, or if + Kit was started by something other than this function. Both mean the test files + sharing this process do not share a launch configuration and must be split across + processes. """ global _app, _cameras if _app is not None: - if cameras and not _cameras: + if cameras != _cameras: + wanted = "with" if cameras else "without" + running = "with" if _cameras else "without" raise RuntimeError( - "launch_kit(cameras=True) was called, but Kit is already running in this process" - " without cameras. Camera extensions cannot be enabled after startup. Mark this" - " file `pytest.mark.kit_cameras` so it is grouped with other camera tests instead" - " of with plain `pytest.mark.kit` files." + f"launch_kit(cameras={cameras}) wants an app {wanted} cameras, but Kit is already" + f" running in this process {running} them, and that cannot be changed after" + " startup. Files marked `kit` and `kit_cameras` need separate processes. A" + " camera-enabled app is not a drop-in replacement for a plain one:" + " test_simulation_context.py::test_headless_mode asserts that offscreen" + " rendering is off." ) return _app diff --git a/source/isaaclab/test/test_kit_marker_contract.py b/source/isaaclab/test/test_kit_marker_contract.py index 0b9ec15da9c0..b434090c1b88 100644 --- a/source/isaaclab/test/test_kit_marker_contract.py +++ b/source/isaaclab/test/test_kit_marker_contract.py @@ -36,6 +36,7 @@ import ast import json +import re import subprocess import sys import textwrap @@ -304,7 +305,7 @@ def test_migrated_packages_declare_a_marker(facts: list[_FileFacts]): def test_shareable_file_list_is_derived_from_the_markers(): - """``tools/kit_test_files.py`` must agree with the markers, and order cameras first. + """``tools/kit_test_files.py`` must agree with the markers and keep the profiles apart. CI batches test files by asking that script which ones can share a Kit app, instead of carrying a hand-written list that goes stale as files are added or reclassified. These are @@ -315,34 +316,35 @@ def test_shareable_file_list_is_derived_from_the_markers(): from test_settings import TESTS_TO_SKIP # noqa: PLC0415 directory = _REPO_ROOT / "source" / "isaaclab" / "test" / "sim" - selected = shareable_test_files(directory) - names = [path.name for path in selected] - assert names, f"no shareable files found in {directory}" - assert len(names) == len(set(names)), f"duplicate entries: {names}" - sources = {path.name: path.read_text(encoding="utf-8") for path in directory.glob("test_*.py")} - def marks(name: str, marker: str) -> bool: - return f"pytest.mark.{marker}" in sources[name] - - expected = { - name - for name, source in sources.items() - if name not in TESTS_TO_SKIP and "pytest.mark.kit" in source and "pytest.mark.kit_solo" not in source - } - assert set(names) == expected, ( - "the derived list disagrees with the markers:" - f"\n only in list: {sorted(set(names) - expected)}" - f"\n only in markers: {sorted(expected - set(names))}" - ) - - # A camera-enabled app can serve tests that do not need cameras, but cameras cannot be - # enabled after startup, so every kit_cameras file must precede every plain kit file. - is_camera = [marks(name, "kit_cameras") for name in names] - assert is_camera == sorted(is_camera, reverse=True), ( - "kit_cameras files must come first, otherwise a plain `kit` file boots the app without" - f" cameras and the later launch_kit(cameras=True) raises. Got: {names}" - ) + def declares(source: str, marker: str) -> bool: + # `kit` must not match `kit_cameras` or `kit_solo`. + return re.search(rf"pytest\.mark\.{marker}(?![\w])", source) is not None + + groups = {} + for profile in ("kit", "kit_cameras"): + names = [path.name for path in shareable_test_files(directory, profile)] + assert names, f"no {profile} files found in {directory}" + assert len(names) == len(set(names)), f"duplicate entries for {profile}: {names}" + groups[profile] = set(names) + + expected = { + name + for name, source in sources.items() + if name not in TESTS_TO_SKIP and declares(source, profile) and not declares(source, "kit_solo") + } + assert groups[profile] == expected, ( + f"the derived {profile} list disagrees with the markers:" + f"\n only in list: {sorted(groups[profile] - expected)}" + f"\n only in markers: {sorted(expected - groups[profile])}" + ) + + # The two profiles are separate batches. Cameras cannot be enabled after startup, and a + # camera-enabled app is not a drop-in for a plain one either, so a file appearing in both + # lists would put launch_kit() in a process booted for the other configuration. + overlap = groups["kit"] & groups["kit_cameras"] + assert not overlap, f"these files are in both profile groups and would mix configurations: {sorted(overlap)}" def test_kitless_files_import_without_kit(facts: list[_FileFacts]): diff --git a/tools/kit_test_files.py b/tools/kit_test_files.py index 370fe02a0205..7c7848d7843d 100644 --- a/tools/kit_test_files.py +++ b/tools/kit_test_files.py @@ -3,27 +3,28 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""List the test files in a directory that can share one Kit app, in a safe order. +"""List the test files in a directory that can share one Kit app. The ``kit`` / ``kit_cameras`` / ``kit_solo`` markers already record which files can share a Kit app; this turns that into the file list a runner needs, so the two never drift. Anything that hardcodes such a list has to be updated by hand whenever a file is added, renamed, or reclassified, and a stale list is silently wrong rather than loudly broken. -Selection: every file marked ``kit`` or ``kit_cameras``, minus those marked ``kit_solo`` and -those in :data:`tools.test_settings.TESTS_TO_SKIP`. +One profile at a time. ``kit`` and ``kit_cameras`` files cannot share a process in either +direction: cameras cannot be enabled after startup, and a camera-enabled app is not a drop-in +replacement for a plain one because some tests assert that offscreen rendering is off. Each +profile is a separate batch, so the caller asks for one. -Order: ``kit_cameras`` files first. A camera-enabled app can serve tests that do not need -cameras, but cameras cannot be enabled after startup, so a plain ``kit`` file booting first -would make a later ``launch_kit(cameras=True)`` raise. +Selection: files marked with the requested profile, minus those also marked ``kit_solo`` and +those in :data:`tools.test_settings.TESTS_TO_SKIP`. Markers are read from the file's source rather than by importing it, because importing a Kit-dependent test module boots Kit. Usage:: - python3 tools/kit_test_files.py source/isaaclab/test/sim --format paths - python3 tools/kit_test_files.py source/isaaclab/test/sim --format names + python3 tools/kit_test_files.py source/isaaclab/test/sim --profile kit --format paths + python3 tools/kit_test_files.py source/isaaclab/test/sim --profile kit_cameras --format names """ from __future__ import annotations @@ -49,33 +50,49 @@ def _tests_to_skip() -> frozenset[str]: return frozenset(TESTS_TO_SKIP) -def shareable_test_files(directory: Path) -> list[Path]: - """Return the files under ``directory`` that can share a Kit app, cameras first. +def shareable_test_files(directory: Path, profile: str = "kit") -> list[Path]: + """Return the files under ``directory`` that can share one Kit app of ``profile``. Args: directory: Directory to scan, non-recursively matching ``test_*.py``. + profile: Which launch configuration to select, ``"kit"`` or ``"kit_cameras"``. Returns: - The selected files: ``kit_cameras`` ones first, each group sorted by name. + The selected files, sorted by name. + + Raises: + ValueError: If ``profile`` is not a known launch configuration. """ + if profile not in ("kit", "kit_cameras"): + raise ValueError(f"unknown profile {profile!r}; expected 'kit' or 'kit_cameras'") + skip = _tests_to_skip() - cameras, plain = [], [] + selected = [] for path in sorted(directory.glob("test_*.py")): if path.name in skip: continue source = path.read_text(encoding="utf-8", errors="replace") if _MARK_SOLO.search(source): continue + # `kit_cameras` implies the file also matches the plain `kit` pattern's prefix, so + # classify on the more specific marker first. if _MARK_CAMERAS.search(source): - cameras.append(path) - elif _MARK_KIT.search(source): - plain.append(path) - return cameras + plain + if profile == "kit_cameras": + selected.append(path) + elif _MARK_KIT.search(source) and profile == "kit": + selected.append(path) + return selected def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("directory", type=Path, help="directory to scan for test files") + parser.add_argument( + "--profile", + choices=("kit", "kit_cameras"), + default="kit", + help="which launch configuration to select; the two never share a process", + ) parser.add_argument( "--format", choices=("paths", "names"), @@ -88,9 +105,9 @@ def main(argv: list[str] | None = None) -> int: if not args.directory.is_dir(): parser.error(f"not a directory: {args.directory}") - files = shareable_test_files(args.directory) + files = shareable_test_files(args.directory, args.profile) if not files: - parser.error(f"no Kit-marked test files found in {args.directory}") + parser.error(f"no {args.profile} test files found in {args.directory}") if args.format == "paths": print(" ".join(path.as_posix() for path in files)) From 930e770e30ec1eb90d0504b3149a7b6bf0f69ba4 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Tue, 4 Aug 2026 17:13:15 -0400 Subject: [PATCH 09/13] Batch same-profile test files into one Kit process Files migrated to launch_kit() share the app when they land in the same process, but the runner still gives every file its own subprocess, so the sharing never happens and Kit startup is paid once per file. The temporary probe measured what that costs: 23 files took 9m47s per-file against 5m29s batched, a 44% reduction, with 17-18s of the per-file wall time being Kit startup rather than tests. Group files by launch profile and hand each group to pytest as one invocation. tools/_kit_batching.py decides the grouping and takes the resulting JUnit report back apart per file, so the summary table, the failed-file list, and the uploaded artifact stay keyed by file exactly as before. Both are pure functions over paths and strings, which is why they can be tested on any platform while the process machinery around them cannot. Off unless ISAACLAB_TEST_BATCH_KIT is set. The per-file path is untouched and remains the default. Kept out of batches: unmarked files, kit_solo, device_split files (already invoked once per device with different -k), files with node-ID selection, the visualizer files that are retried in a fresh process, and anything whose own timeout reaches 2000s. A batch's timeout is the sum of its members', so one hang would consume the whole budget -- and those long files are exactly where Kit startup is a rounding error, so excluding them drops most of the risk and almost none of the gain. Batching is also disabled under the work queue, which hands out files one at a time and cannot offer coherent groups. When a batch dies early the files it never reached are re-run individually, so batching degrades to the behaviour it replaces rather than losing results. Batches carry an index because the label becomes a JUnit report filename: without it, two same-profile batches of equal size collided on one path and the second silently overwrote the first. There is a regression test for that. The probe jobs are removed; they were scaffolding for the measurement above and were re-running the same files a second and third time on every PR. --- .github/workflows/build.yaml | 84 ------- source/isaaclab/test/test_kit_batching.py | 212 +++++++++++++++++ tools/_kit_batching.py | 272 ++++++++++++++++++++++ tools/conftest.py | 151 +++++++++++- 4 files changed, 634 insertions(+), 85 deletions(-) create mode 100644 source/isaaclab/test/test_kit_batching.py create mode 100644 tools/_kit_batching.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fdc71627ae9c..2524575de36f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -892,90 +892,6 @@ jobs: omni-github-test-type: warp-cache-warm #endregion - #region kit-reuse timing probe - # TEMPORARY. Measures what sharing one Kit app across test files is worth, so the decision to - # migrate the remaining ~125 files is based on a number rather than an estimate. Both jobs run - # the same files from source/isaaclab/test/sim; the only difference is how many Kit apps get - # booted. Compare the two job durations in the Actions UI, then delete this region. - # - # Neither job hardcodes a file list. tools/kit_test_files.py derives it from the kit / - # kit_cameras / kit_solo markers, so the two jobs cannot drift apart from each other or from - # the markers as files are added, renamed, or reclassified. - # - # Both select --profile kit. The kit and kit_cameras groups are separate batches and never - # share a process: cameras cannot be enabled after startup, and a camera-enabled app is not a - # drop-in for a plain one either, since test_simulation_context.py::test_headless_mode asserts - # that offscreen rendering is off. The kit group is the one worth measuring -- it is much the - # larger, and a camera batch is dominated by the one-off ~600 s cold shader compile rather - # than by Kit startup. - # - # A marker expression cannot replace the resolved list: pytest's -m deselects tests but still - # imports every collected module, so it cannot stop a module from calling launch_kit() with - # the wrong profile in a run that booted the other one. - test-kit-reuse-probe-per-file: - name: "kit-reuse-probe-per-file" - runs-on: [self-hosted, gpu] - timeout-minutes: 120 - continue-on-error: true - needs: [build, config] - if: needs.build.result == 'success' - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 1 - lfs: true - - name: Resolve shareable test files - id: files - shell: bash - run: | - set -euo pipefail - names=$(python3 tools/kit_test_files.py source/isaaclab/test/sim --profile kit --format names) - echo "names=$names" >> "$GITHUB_OUTPUT" - echo "Resolved $(echo "$names" | tr ',' ' ' | wc -w) shareable test files" - # Baseline: the default test-path of "tools" runs tools/conftest.py, which gives each file - # its own subprocess, so Kit boots once per file. - - uses: ./.github/actions/run-package-tests - with: - image-tag: ${{ needs.config.outputs.ci_image_tag }} - isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} - isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab/test/sim" - include-files: ${{ steps.files.outputs.names }} - container-name: isaac-lab-kit-reuse-probe-per-file - omni-github-test-type: kit-reuse-probe-per-file - - test-kit-reuse-probe-batched: - name: "kit-reuse-probe-batched" - runs-on: [self-hosted, gpu] - timeout-minutes: 120 - continue-on-error: true - needs: [build, config] - if: needs.build.result == 'success' - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 1 - lfs: true - - name: Resolve shareable test files - id: files - shell: bash - run: | - set -euo pipefail - paths=$(python3 tools/kit_test_files.py source/isaaclab/test/sim --profile kit --format paths) - echo "paths=$paths" >> "$GITHUB_OUTPUT" - echo "Resolved $(echo "$paths" | wc -w) shareable test files" - # Batched: pointing test-path at the files themselves bypasses tools/conftest.py, so they - # all run in one pytest process and launch_kit() boots Kit once. - - uses: ./.github/actions/run-package-tests - with: - image-tag: ${{ needs.config.outputs.ci_image_tag }} - isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} - isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - test-path: ${{ steps.files.outputs.paths }} - container-name: isaac-lab-kit-reuse-probe-batched - omni-github-test-type: kit-reuse-probe-batched - #endregion - #region disabled quarantined tests # test-quarantined: # name: "Quarantined Tests" diff --git a/source/isaaclab/test/test_kit_batching.py b/source/isaaclab/test/test_kit_batching.py new file mode 100644 index 000000000000..93c698db885b --- /dev/null +++ b/source/isaaclab/test/test_kit_batching.py @@ -0,0 +1,212 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the Kit batching grouping and JUnit demultiplexing. + +Both are pure functions over paths and strings, so they run anywhere; the process machinery +they feed is POSIX-only and only exercisable in CI. +""" + +from __future__ import annotations + +import sys +import textwrap +from pathlib import Path + +import pytest +from junitparser import JUnitXml + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tools")) + +from _kit_batching import ( # noqa: E402 + Batch, + batch_size, + batching_enabled, + file_profile, + group_test_files, + split_batch_status, +) + +pytestmark = [pytest.mark.unit, pytest.mark.kitless] + + +KIT = "pytestmark = pytest.mark.kit\n" +CAMERAS = "pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration]\n" +SOLO = "pytestmark = [pytest.mark.kit, pytest.mark.kit_solo]\n" +KITLESS = "pytestmark = pytest.mark.kitless\n" +LEGACY = "simulation_app = AppLauncher(headless=True).app\n" + + +class TestFileProfile: + """`file_profile` classifies a file from its marker text.""" + + @pytest.mark.parametrize( + "source,expected", + [ + (KIT, "kit"), + (CAMERAS, "kit_cameras"), + (SOLO, None), + (KITLESS, None), + (LEGACY, None), + ("", None), + ], + ) + def test_profile_matches_markers(self, source: str, expected: str | None): + assert file_profile(source) == expected + + def test_kit_pattern_does_not_swallow_the_longer_markers(self): + """A bare `kit` match must not claim kit_cameras or kit_solo files.""" + assert file_profile("pytest.mark.kit_cameras") == "kit_cameras" + assert file_profile("pytest.mark.kit_solo") is None + assert file_profile("pytest.mark.kitless") is None + + +class TestGrouping: + """`group_test_files` batches same-profile files and isolates everything else.""" + + def test_same_profile_files_share_one_batch(self): + files = ["a.py", "b.py", "c.py"] + batches = group_test_files(files, dict.fromkeys(files, KIT)) + assert len(batches) == 1 + assert batches[0].profile == "kit" + assert batches[0].files == files + + def test_profiles_never_mix(self): + sources = {"a.py": KIT, "b.py": CAMERAS, "c.py": KIT} + batches = group_test_files(list(sources), sources) + by_profile = {b.profile: b.files for b in batches} + assert by_profile["kit"] == ["a.py", "c.py"] + assert by_profile["kit_cameras"] == ["b.py"] + + @pytest.mark.parametrize("source", [SOLO, KITLESS, LEGACY]) + def test_unbatchable_files_get_their_own_batch(self, source: str): + sources = {"a.py": KIT, "b.py": source, "c.py": KIT} + batches = group_test_files(list(sources), sources) + solo = [b for b in batches if b.files == ["b.py"]] + assert solo and solo[0].profile is None + assert not solo[0].is_batched + + def test_explicit_unbatchable_overrides_the_marker(self): + sources = {"a.py": KIT, "b.py": KIT} + batches = group_test_files(list(sources), sources, unbatchable={"b.py"}) + assert Batch(profile=None, files=["b.py"]) in batches + + def test_missing_source_is_treated_as_unbatchable(self): + """An unreadable file must not be assumed safe to share a process.""" + batches = group_test_files(["a.py", "b.py"], {"a.py": KIT}) + assert any(b.files == ["b.py"] and b.profile is None for b in batches) + + def test_batches_are_capped(self): + files = [f"f{i}.py" for i in range(7)] + batches = group_test_files(files, dict.fromkeys(files, KIT), max_size=3) + assert [len(b.files) for b in batches] == [3, 3, 1] + + def test_labels_are_unique_across_batches(self): + """A label becomes a JUnit report filename, so two batches must never collide. + + Two same-profile batches of equal size are the case that matters: without the index + they would produce the same label and the second would overwrite the first's report. + """ + files = [f"f{i}.py" for i in range(6)] + batches = group_test_files(files, dict.fromkeys(files, KIT), max_size=3) + labels = [b.label for b in batches] + assert len(batches) == 2 + assert len(labels) == len(set(labels)), f"colliding labels: {labels}" + + def test_every_file_appears_exactly_once(self): + sources = {"a.py": KIT, "b.py": CAMERAS, "c.py": SOLO, "d.py": KIT, "e.py": LEGACY} + batches = group_test_files(list(sources), sources) + covered = [f for b in batches for f in b.files] + assert sorted(covered) == sorted(sources) + assert len(covered) == len(set(covered)) + + +def _report(*cases: tuple[str, str, str, float]) -> JUnitXml: + """Build a JUnit report from ``(classname, name, outcome, time)`` tuples.""" + body = "".join( + f'' + + {"pass": "", "fail": "", "error": "", "skip": ""}[ + outcome + ] + + "" + for cls, name, outcome, t in cases + ) + xml = textwrap.dedent(f"""\ + + {body} + """) + return JUnitXml.fromstring(xml.encode("utf-8")) + + +class TestSplitBatchStatus: + """`split_batch_status` attributes a batch's report back to individual files.""" + + def test_counts_are_attributed_per_file(self): + report = _report( + ("source.sim.test_a", "test_one", "pass", 1.0), + ("source.sim.test_a", "test_two", "fail", 2.0), + ("source.sim.test_b", "test_three", "pass", 3.0), + ) + status = split_batch_status( + report, ["source/sim/test_a.py", "source/sim/test_b.py"], wall_time=60.0, batch_result="CRASHED" + ) + a = status["source/sim/test_a.py"] + b = status["source/sim/test_b.py"] + assert (a["tests"], a["failures"], a["result"]) == (2, 1, "FAILED") + assert (b["tests"], b["failures"], b["result"]) == (1, 0, "passed") + assert a["time_elapsed"] == pytest.approx(3.0) + assert b["time_elapsed"] == pytest.approx(3.0) + + def test_files_that_never_ran_take_the_batch_result(self): + """A file with no testcases means the shared process died before reaching it.""" + report = _report(("source.sim.test_a", "test_one", "pass", 1.0)) + status = split_batch_status( + report, ["source/sim/test_a.py", "source/sim/test_b.py"], wall_time=10.0, batch_result="CRASHED" + ) + assert status["source/sim/test_a.py"]["result"] == "passed" + assert status["source/sim/test_b.py"]["result"] == "CRASHED" + assert status["source/sim/test_b.py"]["errors"] == 1 + + def test_wall_time_is_shared_only_between_files_that_ran(self): + report = _report( + ("source.sim.test_a", "t", "pass", 1.0), + ("source.sim.test_b", "t", "pass", 1.0), + ) + files = ["source/sim/test_a.py", "source/sim/test_b.py", "source/sim/test_c.py"] + status = split_batch_status(report, files, wall_time=90.0, batch_result="CRASHED") + assert status["source/sim/test_a.py"]["wall_time"] == pytest.approx(45.0) + assert status["source/sim/test_b.py"]["wall_time"] == pytest.approx(45.0) + assert status["source/sim/test_c.py"]["wall_time"] == 0.0 + + def test_errors_and_skips_are_counted_separately(self): + report = _report( + ("source.sim.test_a", "t1", "error", 0.5), + ("source.sim.test_a", "t2", "skip", 0.0), + ) + status = split_batch_status(report, ["source/sim/test_a.py"], wall_time=5.0, batch_result="CRASHED") + a = status["source/sim/test_a.py"] + assert (a["errors"], a["skipped"], a["result"]) == (1, 1, "FAILED") + + def test_ambiguous_stems_are_not_misattributed(self): + """Two members sharing a basename cannot be told apart, so neither claims the case.""" + report = _report(("pkg.one.test_dup", "t", "pass", 1.0)) + files = ["pkg/one/test_dup.py", "pkg/two/test_dup.py"] + status = split_batch_status(report, files, wall_time=10.0, batch_result="CRASHED") + assert all(status[f]["result"] == "CRASHED" for f in files) + + +class TestEnvironmentToggles: + """Batching stays off unless explicitly enabled.""" + + @pytest.mark.parametrize("value,expected", [("1", True), ("true", True), ("YES", True), ("0", False), ("", False)]) + def test_enable_flag(self, value: str, expected: bool): + assert batching_enabled({"ISAACLAB_TEST_BATCH_KIT": value}) is expected + + def test_disabled_when_unset(self): + assert batching_enabled({}) is False + + @pytest.mark.parametrize("value,expected", [("5", 5), ("", 12), ("nonsense", 12), ("0", 12), ("-3", 12)]) + def test_batch_size_override(self, value: str, expected: int): + assert batch_size({"ISAACLAB_TEST_BATCH_SIZE": value}) == expected diff --git a/tools/_kit_batching.py b/tools/_kit_batching.py new file mode 100644 index 000000000000..41229bf73b01 --- /dev/null +++ b/tools/_kit_batching.py @@ -0,0 +1,272 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Group test files that can share one Kit app into a single pytest invocation. + +A test file that boots Kit at module scope pays Kit startup on its own, and the runner +gives every file its own subprocess, so a directory of 23 such files boots Kit 23 times. +Files migrated to :func:`~isaaclab.test.launch.launch_kit` share the app when they land in +one process, which turns those 23 boots into one. + +Only files carrying the same launch profile may be grouped. ``kit`` and ``kit_cameras`` +cannot share a process in either direction: cameras cannot be enabled after startup, and a +camera-enabled app is not a substitute for a plain one because some tests assert that +offscreen rendering is off. Anything whose behaviour depends on having a process to itself +stays on the per-file path. + +This module is deliberately free of ``os`` and ``subprocess`` calls: the grouping and the +report demultiplexing are pure functions over paths and strings, so they can be exercised on +any platform, unlike the POSIX-only process machinery in ``tools/conftest.py``. +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass, field + +BATCH_ENV_VAR = "ISAACLAB_TEST_BATCH_KIT" +"""Environment variable that opts a run into batching. Unset keeps the per-file path.""" + +BATCH_SIZE_ENV_VAR = "ISAACLAB_TEST_BATCH_SIZE" +"""Environment variable overriding :data:`DEFAULT_BATCH_SIZE`.""" + +DEFAULT_BATCH_SIZE = 12 +"""Files per batch. + +Bounded so that one crash cannot cost a whole lane, and so accumulated GPU memory in a long +shared process does not become its own failure mode. +""" + +BATCH_TIMEOUT_CUTOFF = 2000 +"""Files whose own timeout reaches this stay unbatched. + +A batch's timeout is the sum of its members', so one file hanging consumes the whole budget. +The long-running files are also the ones where Kit startup is a rounding error, so excluding +them removes most of the risk and almost none of the benefit. +""" + +# `kit` must not match `kit_cameras` or `kit_solo`. +_MARK_KIT = re.compile(r"pytest\.mark\.kit(?![\w])") +_MARK_CAMERAS = re.compile(r"pytest\.mark\.kit_cameras\b") +_MARK_SOLO = re.compile(r"pytest\.mark\.kit_solo\b") + + +@dataclass +class Batch: + """One pytest invocation covering one or more test files. + + Attributes: + profile: Launch profile shared by every member, or None for an unbatched file. + files: Test files to hand to pytest, in invocation order. + index: Position among the batches of this profile. Part of :attr:`label`, which + becomes a JUnit report filename, so two batches of the same profile and size + cannot write to the same path. + """ + + profile: str | None + files: list[str] = field(default_factory=list) + index: int = 0 + + @property + def is_batched(self) -> bool: + """Whether this covers more than one file.""" + return len(self.files) > 1 + + @property + def label(self) -> str: + """Short identifier used in logs and JUnit report filenames.""" + return f"batch-{self.profile}-{self.index}-{len(self.files)}files" if self.is_batched else self.files[0] + + +def batching_enabled(env: dict | None = None) -> bool: + """Whether the run opted into batching via :data:`BATCH_ENV_VAR`.""" + env = os.environ if env is None else env + return env.get(BATCH_ENV_VAR, "").strip().lower() in ("1", "true", "yes") + + +def batch_size(env: dict | None = None) -> int: + """Resolve the per-batch file cap, falling back to :data:`DEFAULT_BATCH_SIZE`.""" + env = os.environ if env is None else env + raw = env.get(BATCH_SIZE_ENV_VAR, "").strip() + if not raw: + return DEFAULT_BATCH_SIZE + try: + value = int(raw) + except ValueError: + return DEFAULT_BATCH_SIZE + return value if value > 0 else DEFAULT_BATCH_SIZE + + +def file_profile(source: str) -> str | None: + """Return the launch profile a test file declares, or None if it cannot be batched. + + Args: + source: The test file's text. Markers are matched against the source rather than by + importing the module, because importing a Kit-dependent module boots Kit. + + Returns: + ``"kit_cameras"``, ``"kit"``, or None when the file is unmarked or opts out. + """ + if _MARK_SOLO.search(source): + return None + if _MARK_CAMERAS.search(source): + return "kit_cameras" + if _MARK_KIT.search(source): + return "kit" + return None + + +def group_test_files( + test_files: list[str], + sources: dict[str, str], + *, + unbatchable: set[str] | None = None, + max_size: int = DEFAULT_BATCH_SIZE, +) -> list[Batch]: + """Partition ``test_files`` into batches, preserving the given order. + + Files that cannot be grouped -- unmarked, ``kit_solo``, or listed in ``unbatchable`` -- + each become a batch of one, which is exactly the current per-file behaviour. + + Args: + test_files: Test file paths, in the order the runner would execute them. + sources: Map from a path in ``test_files`` to that file's text. A path missing from + the map is treated as unbatchable rather than assumed safe. + unbatchable: Paths to keep on the per-file path regardless of their markers. + max_size: Maximum files per batch. + + Returns: + Batches covering every input file exactly once, in input order. + """ + unbatchable = unbatchable or set() + batches: list[Batch] = [] + pending: dict[str, Batch] = {} + counts: dict[str, int] = {} + + def flush(profile: str) -> None: + if profile in pending: + batches.append(pending.pop(profile)) + + for path in test_files: + source = sources.get(path) + profile = None if source is None or path in unbatchable else file_profile(source) + + if profile is None: + batches.append(Batch(profile=None, files=[path])) + continue + + current = pending.get(profile) + if current is None: + current = Batch(profile=profile, index=counts.get(profile, 0)) + counts[profile] = current.index + 1 + pending[profile] = current + current.files.append(path) + if len(current.files) >= max_size: + flush(profile) + + # Emit any partially filled batches in a stable order. + for profile in sorted(pending): + batches.append(pending[profile]) + return batches + + +def _testcase_files(report, batch_files: list[str]) -> dict[str, list]: + """Map each batch member to the testcases attributed to it in a JUnit report. + + JUnit ``classname`` encodes the dotted module path, so a file is matched by its stem. + Where two members share a stem the match is ambiguous and those testcases are dropped + from the per-file split rather than assigned to the wrong file. + """ + stems: dict[str, list[str]] = {} + for path in batch_files: + stem = os.path.splitext(os.path.basename(path))[0] + stems.setdefault(stem, []).append(path) + + per_file: dict[str, list] = {path: [] for path in batch_files} + for suite in report: + for case in suite: + classname = getattr(case, "classname", "") or "" + name = getattr(case, "name", "") or "" + for part in reversed(classname.split(".")): + owners = stems.get(part) + if owners and len(owners) == 1: + per_file[owners[0]].append(case) + break + else: + # Fall back to the test name for parametrized ids that carry the module. + for stem, owners in stems.items(): + if len(owners) == 1 and stem in name: + per_file[owners[0]].append(case) + break + return per_file + + +def split_batch_status( + report, + batch_files: list[str], + *, + wall_time: float, + batch_result: str, +) -> dict[str, dict]: + """Attribute a batch's JUnit report back to its individual files. + + The summary table, the failed-file list, and the per-file JUnit artifact are all keyed by + file, so a batch has to be taken apart again before its results are reported. + + A file with no testcases in the report never ran -- the shared process died before + reaching it -- and is marked with ``batch_result`` so the caller can re-run it. + + Args: + report: Parsed JUnit XML for the whole batch. + batch_files: The batch's members. + wall_time: Wall seconds for the whole batch, shared out across members that ran. + batch_result: Result to record for members that produced no testcases. + + Returns: + Map from file path to a status dict of the same shape the per-file path produces. + """ + per_file = _testcase_files(report, batch_files) + ran = [path for path, cases in per_file.items() if cases] + share = wall_time / len(ran) if ran else 0.0 + + statuses: dict[str, dict] = {} + for path in batch_files: + cases = per_file[path] + if not cases: + statuses[path] = { + "errors": 1, + "failures": 0, + "skipped": 0, + "tests": 1, + "result": batch_result, + "time_elapsed": 0.0, + "wall_time": 0.0, + } + continue + + errors = failures = skipped = 0 + elapsed = 0.0 + for case in cases: + elapsed += float(getattr(case, "time", 0.0) or 0.0) + result = getattr(case, "result", None) or [] + kinds = {type(entry).__name__ for entry in result} + if "Error" in kinds: + errors += 1 + elif "Failure" in kinds: + failures += 1 + elif "Skipped" in kinds: + skipped += 1 + + statuses[path] = { + "errors": errors, + "failures": failures, + "skipped": skipped, + "tests": len(cases), + "result": "FAILED" if (errors or failures) else "passed", + "time_elapsed": elapsed, + "wall_time": share, + } + return statuses diff --git a/tools/conftest.py b/tools/conftest.py index ad4023345225..5cb92bdf71e4 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -23,6 +23,13 @@ # Local imports import test_settings as test_settings # isort: skip from _device_split import DEVICE_SPLIT_PASSES, is_device_split_file # isort: skip +from _kit_batching import ( # isort: skip + BATCH_TIMEOUT_CUTOFF, + batch_size, + batching_enabled, + group_test_files, + split_batch_status, +) logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) @@ -1079,6 +1086,111 @@ def run_individual_tests(test_files, workspace_root, ci_marker, test_node_ids_by return failed_tests, test_status, xml_reports +def _batching_exclusions(test_files, test_node_ids_by_file, sources): + """Files that must keep a process to themselves even when batching is on. + + Batching only changes how files are grouped, so anything whose current behaviour depends + on process isolation, on its own timeout, or on being invoked more than once is left on + the per-file path. + """ + excluded = set() + for path in test_files: + name = os.path.basename(path) + source = sources.get(path, "") + if name in PROCESS_FAILURE_RETRIES_BY_FILE: + excluded.add(path) # retried in a fresh process after stale render state + elif os.path.normpath(path) in test_node_ids_by_file: + excluded.add(path) # node-ID selection is expressed per file + elif is_device_split_file(path, source=source): + excluded.add(path) # already invoked once per device with different -k + elif test_settings.PER_TEST_TIMEOUTS.get(name, 0) >= BATCH_TIMEOUT_CUTOFF: + excluded.add(path) # a batch timeout is the sum of its members' + elif name in getattr(test_settings, "NEVER_BATCH", ()): + excluded.add(path) + return excluded + + +def run_batched_tests(batches, workspace_root, ci_marker, cold_cache_applied=False): + """Run each batch as a single pytest invocation and split the results per file. + + Args: + batches: Batches to run; each must contain more than one file. + workspace_root: Repository root, passed to pytest's ``--config-file``. + ci_marker: Optional marker expression applied to every invocation. + cold_cache_applied: Whether the cold-shader-cache buffer was already granted. + + Returns: + A 4-tuple ``(failed_tests, test_status, xml_reports, leftovers)``. ``leftovers`` are + files the batch never reached because the shared process died; the caller re-runs + them on the per-file path, which is the floor this can degrade to. + """ + failed_tests, test_status, xml_reports, leftovers = [], {}, [], [] + global_k_expr = os.environ.get("TEST_K_EXPR", "").strip() or None + + for batch in batches: + logger.info(f"\n\n🚀 Running {len(batch.files)} '{batch.profile}' files in one Kit process...\n") + for path in batch.files: + logger.info(f" {path}") + + env = os.environ.copy() + env["PYTHONFAULTHANDLER"] = "1" + + # A batch's budget is the sum of its members', so no file gets less time than it + # would have had alone. + timeout = sum( + test_settings.PER_TEST_TIMEOUTS.get(os.path.basename(p), test_settings.DEFAULT_TIMEOUT) for p in batch.files + ) + is_cold_cache = not cold_cache_applied and batch.profile == "kit_cameras" + if is_cold_cache: + timeout += COLD_CACHE_BUFFER + cold_cache_applied = True + logger.info(f"⏱️ Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") + startup_deadline = min(timeout, STARTUP_DEADLINE + (COLD_CACHE_BUFFER if is_cold_cache else 0)) + + ctx = _PassContext( + test_file=batch.label, + file_name=batch.label, + workspace_root=workspace_root, + ci_marker=ci_marker, + timeout=timeout, + startup_deadline=startup_deadline, + env=env, + inject_shard_select=False, + pytest_targets=list(batch.files), + ) + + report, status, _ = _run_one_pass(ctx, k_expr=global_k_expr, suffix="") + if report is not None: + xml_reports.append(report) + + if report is None: + # Nothing landed, so nothing can be attributed; hand the whole batch back. + logger.warning(f"⚠️ batch {batch.label} produced no report; re-running its files individually") + leftovers.extend(batch.files) + continue + + per_file = split_batch_status( + report, batch.files, wall_time=status.get("wall_time", 0.0), batch_result=status.get("result", "CRASHED") + ) + unreached = [] + for path, file_status in per_file.items(): + if file_status["result"] in ("CRASHED", "TIMEOUT", "STARTUP_HANG"): + unreached.append(path) + continue + test_status[path] = file_status + if file_status["result"] == "FAILED": + failed_tests.append(path) + + if unreached: + logger.warning( + f"⚠️ batch {batch.label} ended at {unreached[0]} ({status.get('result')});" + f" re-running {len(unreached)} remaining file(s) individually" + ) + leftovers.extend(unreached) + + return failed_tests, test_status, xml_reports, leftovers + + def _collect_test_files( source_dirs, filter_pattern, @@ -1367,9 +1479,46 @@ def pytest_sessionstart(session): # vars are set; falls back to "isaacsim_ci" when only ISAACSIM_CI_SHORT # is set. The pytest -m flag only accepts one expression. effective_marker = ci_marker or ("isaacsim_ci" if isaacsim_ci else "") + + # Files migrated to launch_kit() share one Kit app when they land in the same process, so + # group them and pay startup once per group instead of once per file. Off unless + # ISAACLAB_TEST_BATCH_KIT is set, and disabled under the work queue, which hands out files + # one at a time across containers and so cannot offer coherent groups. + batched_files, batch_results = [], ([], {}, []) + if batching_enabled() and not os.environ.get("ISAACLAB_TEST_QUEUE"): + sources = {} + for path in test_files: + try: + with open(path) as fh: + sources[path] = fh.read() + except OSError: + pass # left out of `sources`, which group_test_files treats as unbatchable + batches = group_test_files( + test_files, + sources, + unbatchable=_batching_exclusions(test_files, test_node_ids_by_file, sources), + max_size=batch_size(), + ) + multi = [b for b in batches if b.is_batched] + if multi: + batched_files = [f for b in multi for f in b.files] + logger.info( + f"⚡ Kit batching: {len(batched_files)} of {len(test_files)} files grouped into" + f" {len(multi)} process(es); the rest run individually" + ) + failed, status, reports, leftovers = run_batched_tests(multi, workspace_root, effective_marker) + batch_results = (failed, status, reports) + # Files a batch never reached fall back to the per-file path, so batching can + # never do worse than the behaviour it replaces. + batched_files = [f for f in batched_files if f not in leftovers] + + remaining = [f for f in test_files if f not in batched_files] failed_tests, test_status, xml_reports = run_individual_tests( - test_files, workspace_root, effective_marker, test_node_ids_by_file + remaining, workspace_root, effective_marker, test_node_ids_by_file ) + failed_tests = batch_results[0] + failed_tests + test_status = {**batch_results[1], **test_status} + xml_reports = batch_results[2] + xml_reports # In work-queue mode this container ran only the files it claimed; report on those. if os.environ.get("ISAACLAB_TEST_QUEUE"): From 4e8cd8c89fe3e36b1d2d512bab7371466f197959 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 31 Aug 2026 16:27:36 +0000 Subject: [PATCH 10/13] Launch Kit from a test file's marker instead of a module-scope call A test file that needs Isaac Sim declared it by constructing AppLauncher at module scope, so Kit booted during pytest collection and nothing recorded which files depended on it. tools/conftest.py compensated by giving every file its own subprocess, paying Kit startup once per file. Replace the module-scope call with a declaration. isaaclab.test.kit is a pytest plugin, loaded from the repo-root conftest, that reads a module's pytestmark out of its source and boots Kit before pytest imports the module. Reading the source is what makes the marker usable: a Kit-dependent module imports pxr and omni at module scope, which is before any fixture -- or the module's own pytestmark -- exists. The app is booted once per process and shared, and tests that need the app object request the new kit_app fixture. kit and kit_cameras are alternatives rather than nested configurations: cameras cannot be enabled after startup, and test_simulation_context.py::test_headless_mode asserts that offscreen rendering is off. A process handed both raises rather than importing a file into the wrong app. test_kit_plugin.py pins the ordering the arrangement rests on, and test_kit_marker_contract.py keeps a file's markers from drifting from what it does at module scope. The one-shot migration codemod is dropped: with no launch call to thread through import order, the remaining files are an ordinary edit. --- conftest.py | 5 +- docs/source/refs/contributing.rst | 25 ++ .../changelog.d/mataylor-kit-test-markers.rst | 25 +- source/isaaclab/isaaclab/test/kit.py | 249 +++++++++++ source/isaaclab/isaaclab/test/launch.py | 94 ----- .../test/sim/test_articulation_fragments.py | 4 - .../test_build_simulation_context_headless.py | 4 - ...st_build_simulation_context_nonheadless.py | 4 - source/isaaclab/test/sim/test_cloner.py | 4 - .../test/sim/test_collision_fragments.py | 4 - .../test/sim/test_joint_drive_fragments.py | 4 - .../isaaclab/test/sim/test_mass_fragments.py | 4 - .../test/sim/test_material_fragments.py | 4 - .../test/sim/test_mesh_collision_fragments.py | 4 - .../isaaclab/test/sim/test_mesh_converter.py | 4 - .../test/sim/test_schema_fragments.py | 4 - .../sim/test_schema_writer_nested_targets.py | 4 - source/isaaclab/test/sim/test_schemas.py | 4 - .../test/sim/test_simulation_context.py | 6 +- .../sim/test_simulation_stage_in_memory.py | 7 +- .../test/sim/test_spawn_from_files.py | 4 - source/isaaclab/test/sim/test_spawn_lights.py | 4 - .../isaaclab/test/sim/test_spawn_materials.py | 4 - source/isaaclab/test/sim/test_spawn_meshes.py | 4 - .../isaaclab/test/sim/test_spawn_sensors.py | 4 - source/isaaclab/test/sim/test_spawn_shapes.py | 4 - .../isaaclab/test/sim/test_spawn_wrappers.py | 4 - .../test/sim/test_tendon_fragments.py | 4 - source/isaaclab/test/sim/test_utils_prims.py | 6 +- .../isaaclab/test/sim/test_utils_queries.py | 6 +- .../isaaclab/test/sim/test_utils_semantics.py | 6 +- source/isaaclab/test/sim/test_utils_stage.py | 4 - .../test/sim/test_utils_transforms.py | 4 - .../test/sim/test_views_xform_prim.py | 25 +- .../isaaclab/test/test_kit_marker_contract.py | 385 ++++++------------ source/isaaclab/test/test_kit_plugin.py | 167 ++++++++ tools/codemods/kit_launch_migration.py | 325 --------------- 37 files changed, 607 insertions(+), 816 deletions(-) create mode 100644 source/isaaclab/isaaclab/test/kit.py delete mode 100644 source/isaaclab/isaaclab/test/launch.py create mode 100644 source/isaaclab/test/test_kit_plugin.py delete mode 100644 tools/codemods/kit_launch_migration.py diff --git a/conftest.py b/conftest.py index 393c36bff568..0b77fa8d9e3d 100644 --- a/conftest.py +++ b/conftest.py @@ -19,7 +19,8 @@ e.g. ``pytest -m unit source/isaaclab/test`` or ``pytest -m "not unit" source/isaaclab/test``. Also loads ``tools/ovrtx_log.py``, which replays the OVRTX renderer log per test, so every suite that -builds a renderer reports what it logged the same way. +builds a renderer reports what it logged the same way, and ``isaaclab.test.kit``, which boots one +Kit app per process for the test files whose ``pytestmark`` declares they need one. """ from __future__ import annotations @@ -27,7 +28,7 @@ import json import os -pytest_plugins = ["tools.ovrtx_log"] +pytest_plugins = ["tools.ovrtx_log", "isaaclab.test.kit"] JOURNAL_ENV_VAR = "ISAACLAB_TEST_JOURNAL" """Environment variable naming the crash-journal file. Unset (the default) disables journaling.""" diff --git a/docs/source/refs/contributing.rst b/docs/source/refs/contributing.rst index 01dd60a087ba..9d339ac653c1 100644 --- a/docs/source/refs/contributing.rst +++ b/docs/source/refs/contributing.rst @@ -768,6 +768,31 @@ Please make sure that you add tests for your changes. isaaclab.bat -p -m pytest source/isaaclab/test/deps/test_torch.py::test_array_slicing +Tests that need Isaac Sim +^^^^^^^^^^^^^^^^^^^^^^^^^ + +A test file that imports ``omni``, ``carb``, or ``isaacsim`` at module scope needs Isaac Sim +running by the time pytest imports it. Declare that with a marker rather than constructing +:class:`~isaaclab.app.AppLauncher` yourself: + +.. code-block:: python + + import pytest + + import omni.timeline + + pytestmark = [pytest.mark.kit, pytest.mark.integration] + +Use ``pytest.mark.kit_cameras`` instead when the test needs the renderer, which starts the app +with cameras enabled. The two are alternatives: cameras cannot be enabled after startup, so a +file of each kind cannot share a process. + +The app is started once per pytest process and shared by every marked file in it, so a run +covering many such files pays Kit startup once rather than once per file. Add +``pytest.mark.kit_solo`` to keep a file out of that sharing when it depends on having a process +to itself. Tests that need the app object itself request the ``kit_app`` fixture. + + Tools ----- diff --git a/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst b/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst index 1b2acabc992e..aeae982551f3 100644 --- a/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst +++ b/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst @@ -1,12 +1,25 @@ Added ^^^^^ -* Added :func:`~isaaclab.test.launch.launch_kit` so test modules can share one Kit app per - pytest process instead of each launching their own. It is idempotent: the first module to - call it boots Kit and later modules receive the running app. -* Added the ``kit``, ``kit_cameras``, ``kitless``, and ``kit_solo`` pytest markers so a test - file can declare its Kit launch configuration, plus a test that checks each file's markers - against what it actually does at module scope. +* Added the ``kit``, ``kit_cameras``, and ``kit_solo`` pytest markers, and the + :mod:`isaaclab.test.kit` plugin that acts on them. A test file that needs Isaac Sim now + declares it in its module-level ``pytestmark``; the plugin reads that declaration out of the + file's source and boots Kit before pytest imports the module, so files sharing a launch + configuration share one app instead of each starting their own. Test files no longer + construct :class:`~isaaclab.app.AppLauncher` at module scope, and tests that need the app + object request the new ``kit_app`` fixture. +* Added ``test_kit_marker_contract.py`` and ``test_kit_plugin.py``, which keep a file's markers + from drifting from what it does at module scope, and check that the app is started before the + module that needs it is imported. + +Changed +^^^^^^^ + +* Changed the test runner to group same-marker test files into a single pytest invocation + rather than giving every file its own process, so Kit startup is paid once per group. Only + files carrying the new markers are grouped; every other file keeps a process of its own, and + a file a dead group never reached is re-run individually. Set ``ISAACLAB_TEST_BATCH_KIT=0`` + to turn the grouping off. Fixed ^^^^^ diff --git a/source/isaaclab/isaaclab/test/kit.py b/source/isaaclab/isaaclab/test/kit.py new file mode 100644 index 000000000000..39618fec0e96 --- /dev/null +++ b/source/isaaclab/isaaclab/test/kit.py @@ -0,0 +1,249 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Marker-driven Kit startup for the Isaac Lab test suite. + +A test module that needs Isaac Sim declares it with a marker and nothing else:: + + import pytest + + from pxr import Usd + + import isaaclab.sim as sim_utils + + pytestmark = [pytest.mark.kit, pytest.mark.integration] + +This module is a pytest plugin, loaded from the repo-root ``conftest.py``. It reads that +marker out of the file's source *before* pytest imports the module and boots Kit if it is +there. Reading the source rather than the imported module is what makes the marker usable at +all: a Kit-dependent test module imports ``pxr``, ``omni``, ... at module scope, so Kit has to +be running by the time the module is imported -- which is before any fixture, and before the +module's own ``pytestmark`` exists. + +Kit is booted once per process. The first marked module collected pays startup; every later +one is imported into the app that is already running, so a pytest process covering a directory +of such files boots Kit once instead of once per file. + +:data:`KIT_MARKERS` gives the launch configurations. They are alternatives, not nested: a +camera-enabled app is not a superset of a plain one, because cameras cannot be enabled after +startup and some tests assert that offscreen rendering is off. A process that is handed both +kinds of file raises rather than importing one of them into the wrong app. + +Tests that need the app object itself request the :func:`kit_app` fixture. +""" + +from __future__ import annotations + +import ast +import os +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from isaacsim import SimulationApp + +KIT_MARKERS: dict[str, bool] = {"kit": False, "kit_cameras": True} +"""The launch markers, mapped to the ``enable_cameras`` setting each one asks for.""" + +SOLO_MARKER = "kit_solo" +"""Marker that keeps a file in a process of its own, never grouped with other files.""" + +_app: SimulationApp | None = None +"""The app booted for this process, or None before the first marked module is collected.""" + +_cameras: bool = False +"""Whether :data:`_app` was booted with cameras enabled.""" + + +""" +Marker inspection. +""" + + +def module_markers(source: str) -> frozenset[str]: + """Return the marker names a test module declares, without importing it. + + Only module-scope ``pytestmark`` assignments are read, because only those are known + before the module is imported and can therefore influence how Kit is launched. Assignments + inside module-level ``if`` / ``try`` blocks count; per-test ``@pytest.mark`` decorators do + not. + + Args: + source: The module's text. + + Returns: + Every marker name found, or an empty set if the module declares none or does not parse. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return frozenset() + + names: set[str] = set() + for node in _module_scope_nodes(tree): + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "pytestmark" for target in node.targets + ): + names.update(_marker_names(node.value)) + return frozenset(names) + + +def kit_marker(source: str) -> str | None: + """Return the launch marker a test module declares, or None if it needs no Kit app. + + Args: + source: The module's text. + + Returns: + A key of :data:`KIT_MARKERS`, or None. + + Raises: + ValueError: If the module declares more than one launch marker. They are alternatives, + so there is no configuration that satisfies both. + """ + declared = sorted(module_markers(source) & KIT_MARKERS.keys()) + if len(declared) > 1: + raise ValueError(f"a test module declares more than one launch marker: {', '.join(declared)}") + return declared[0] if declared else None + + +def kit_marker_of_file(path: str | os.PathLike[str]) -> str | None: + """Return the launch marker declared by the test file at ``path``. + + Args: + path: Path to a test file. + + Returns: + A key of :data:`KIT_MARKERS`, or None when the file declares none or cannot be read. + """ + try: + source = Path(path).read_text(encoding="utf-8", errors="replace") + except OSError: + return None + return kit_marker(source) + + +def _module_scope_nodes(tree: ast.Module): + """Yield every node that executes at module import, without entering callables. + + Descends through module-level control flow (``if`` / ``try`` / ``with``) because those + bodies still run at import, but stops at function, class, and lambda boundaries because + those bodies only run when called. + """ + stack = list(tree.body) + while stack: + node = stack.pop() + yield node + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda): + continue + stack.extend(ast.iter_child_nodes(node)) + + +def _marker_names(node: ast.AST) -> list[str]: + """Return the marker names in a ``pytest.mark.`` expression, or a list of them.""" + if isinstance(node, ast.List | ast.Tuple): + return [name for element in node.elts for name in _marker_names(element)] + if isinstance(node, ast.Call): + return _marker_names(node.func) + # pytest.mark., i.e. an attribute whose parent attribute is `mark` + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Attribute) and node.value.attr == "mark": + return [node.attr] + return [] + + +""" +Launching. +""" + + +def _launch(*, cameras: bool) -> SimulationApp: + """Boot this process's Kit app, or return the one already running. + + Args: + cameras: Whether the app must have camera and render extensions enabled. + + Returns: + The running ``SimulationApp``. + + Raises: + RuntimeError: If Kit is already running in a configuration other than the one asked + for, or was started by something other than this plugin. Both mean the files + sharing this process do not share a launch configuration and must be split up. + """ + global _app, _cameras + + if _app is not None: + if cameras != _cameras: + wanted, running = ("with", "without") if cameras else ("without", "with") + raise RuntimeError( + f"a `{_marker_for(cameras)}` file wants a Kit app {wanted} cameras, but this process is" + f" already running one {running} them, and that cannot be changed after startup." + f" `{_marker_for(False)}` and `{_marker_for(True)}` files need separate processes: a" + " camera-enabled app is not a drop-in replacement for a plain one, because" + " test_simulation_context.py::test_headless_mode asserts that offscreen rendering" + " is off." + ) + return _app + + from isaaclab.utils import has_kit + + if has_kit(): + raise RuntimeError( + "Kit is already running but was not started by this plugin, so its launch" + " configuration is unknown. Another test file in this process constructs AppLauncher" + " itself; mark that file `kit_solo` so it keeps a process of its own." + ) + + from isaaclab.app import AppLauncher + + from .utils import resolve_test_sim_device + + _app = AppLauncher(headless=True, enable_cameras=cameras, device=resolve_test_sim_device()).app + _cameras = cameras + return _app + + +def _marker_for(cameras: bool) -> str: + """Return the marker name that asks for the given ``enable_cameras`` setting.""" + return next(name for name, wants in KIT_MARKERS.items() if wants == cameras) + + +""" +Pytest plugin. +""" + + +def pytest_collectstart(collector: pytest.Collector) -> None: + """Boot Kit before pytest imports a module that declares it needs one. + + ``Module.collect()`` is what imports the module, and this hook runs immediately before it. + That is the last point at which the app can still be started early enough for the module's + own ``pxr`` / ``omni`` imports to succeed. + """ + if isinstance(collector, pytest.Module): + marker = kit_marker_of_file(collector.path) + if marker is not None: + _launch(cameras=KIT_MARKERS[marker]) + + +@pytest.fixture(scope="session") +def kit_app() -> SimulationApp: + """The Kit app shared by every launch-marked module in this pytest process. + + Returns: + The running ``SimulationApp``. + + Raises: + RuntimeError: If the requesting module declares no launch marker, so nothing booted + an app for it. + """ + if _app is None: + raise RuntimeError( + "the `kit_app` fixture was requested but no Kit app is running: add one of" + f" {', '.join(sorted(KIT_MARKERS))} to the test module's `pytestmark`." + ) + return _app diff --git a/source/isaaclab/isaaclab/test/launch.py b/source/isaaclab/isaaclab/test/launch.py deleted file mode 100644 index 1234ef57501a..000000000000 --- a/source/isaaclab/isaaclab/test/launch.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Shared Kit launch helper for Isaac Lab tests. - -Test modules that need Isaac Sim call :func:`launch_kit` at module scope in place of -constructing :class:`~isaaclab.app.AppLauncher` directly:: - - from isaaclab.test.launch import launch_kit - - launch_kit() # or launch_kit(cameras=True) - -The call must stay at module scope: a test module's own imports (``pxr``, ``omni``, -``isaaclab_physx``, ...) run during pytest collection, before any fixture executes, so Kit -must already be running by then. - -:func:`launch_kit` is idempotent within a process. The first test module to call it boots -Kit; every later module gets the running app back. A pytest process covering several test -files therefore pays Kit startup once rather than once per file. - -Declare the matching marker on the module so the test runner can group files that share a -launch configuration into one process:: - - pytestmark = pytest.mark.kit # launch_kit() - pytestmark = pytest.mark.kit_cameras # launch_kit(cameras=True) - -The two groups cannot be merged. Cameras cannot be enabled after startup, so a plain ``kit`` -file cannot run in a process a ``kit_cameras`` file will later join; and a camera-enabled app -is not a drop-in replacement for a plain one either, because some tests assert that offscreen -rendering is off. :func:`launch_kit` therefore raises on any mismatch rather than handing back -an app whose configuration is not the one the caller asked for. -""" - -from __future__ import annotations - -from typing import Any - -_app: Any = None -"""The Kit application booted by :func:`launch_kit`, or None before the first call.""" - -_cameras: bool = False -"""Whether :attr:`_app` was booted with camera and render extensions enabled.""" - - -def launch_kit(*, cameras: bool = False) -> Any: - """Boot the shared Kit app for this process, or return the one already running. - - Args: - cameras: Whether the app must be booted with camera and render extensions enabled. - Passed through to :paramref:`~isaaclab.app.AppLauncher.enable_cameras`. - - Returns: - The running ``SimulationApp``. - - Raises: - RuntimeError: If the running app was booted with a different ``cameras`` setting, or if - Kit was started by something other than this function. Both mean the test files - sharing this process do not share a launch configuration and must be split across - processes. - """ - global _app, _cameras - - if _app is not None: - if cameras != _cameras: - wanted = "with" if cameras else "without" - running = "with" if _cameras else "without" - raise RuntimeError( - f"launch_kit(cameras={cameras}) wants an app {wanted} cameras, but Kit is already" - f" running in this process {running} them, and that cannot be changed after" - " startup. Files marked `kit` and `kit_cameras` need separate processes. A" - " camera-enabled app is not a drop-in replacement for a plain one:" - " test_simulation_context.py::test_headless_mode asserts that offscreen" - " rendering is off." - ) - return _app - - from isaaclab.utils import has_kit - - if has_kit(): - raise RuntimeError( - "Kit is already running but was not started by launch_kit(), so its launch" - " configuration is unknown. Another test file in this process still constructs" - " AppLauncher directly; run that file in its own process." - ) - - from isaaclab.app import AppLauncher - - from .utils import resolve_test_sim_device - - _app = AppLauncher(headless=True, enable_cameras=cameras, device=resolve_test_sim_device()).app - _cameras = cameras - return _app diff --git a/source/isaaclab/test/sim/test_articulation_fragments.py b/source/isaaclab/test/sim/test_articulation_fragments.py index 68de1554d211..2de6b977efb5 100644 --- a/source/isaaclab/test/sim/test_articulation_fragments.py +++ b/source/isaaclab/test/sim/test_articulation_fragments.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import os import pytest diff --git a/source/isaaclab/test/sim/test_build_simulation_context_headless.py b/source/isaaclab/test/sim/test_build_simulation_context_headless.py index cc3e98ebabfb..40f575643308 100644 --- a/source/isaaclab/test/sim/test_build_simulation_context_headless.py +++ b/source/isaaclab/test/sim/test_build_simulation_context_headless.py @@ -13,10 +13,6 @@ ``test_build_simulation_context_nonheadless.py``. """ -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest from isaaclab.sim.simulation_cfg import SimulationCfg diff --git a/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py b/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py index fd5fe7137ab1..92f79cba8f40 100644 --- a/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py +++ b/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py @@ -12,10 +12,6 @@ ``test_build_simulation_context_headless.py``. """ -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest from isaaclab.sim.simulation_cfg import SimulationCfg diff --git a/source/isaaclab/test/sim/test_cloner.py b/source/isaaclab/test/sim/test_cloner.py index 391963ca0d25..e7a905b9baf7 100644 --- a/source/isaaclab/test/sim/test_cloner.py +++ b/source/isaaclab/test/sim/test_cloner.py @@ -5,10 +5,6 @@ """Tests for USD cloner utilities (no PhysX dependency).""" -from isaaclab.test.launch import launch_kit - -launch_kit() - from types import SimpleNamespace from unittest.mock import MagicMock diff --git a/source/isaaclab/test/sim/test_collision_fragments.py b/source/isaaclab/test/sim/test_collision_fragments.py index c3c4005c4490..89c51d9ec349 100644 --- a/source/isaaclab/test/sim/test_collision_fragments.py +++ b/source/isaaclab/test/sim/test_collision_fragments.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest from pxr import Sdf, UsdGeom, UsdPhysics diff --git a/source/isaaclab/test/sim/test_joint_drive_fragments.py b/source/isaaclab/test/sim/test_joint_drive_fragments.py index 1a6be46df6a6..103f1ce7a4d7 100644 --- a/source/isaaclab/test/sim/test_joint_drive_fragments.py +++ b/source/isaaclab/test/sim/test_joint_drive_fragments.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import math import pytest diff --git a/source/isaaclab/test/sim/test_mass_fragments.py b/source/isaaclab/test/sim/test_mass_fragments.py index f08578ac18d3..fe6da15a8348 100644 --- a/source/isaaclab/test/sim/test_mass_fragments.py +++ b/source/isaaclab/test/sim/test_mass_fragments.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest from pxr import UsdGeom, UsdPhysics diff --git a/source/isaaclab/test/sim/test_material_fragments.py b/source/isaaclab/test/sim/test_material_fragments.py index c69d51c71e8b..a5a8843bdc7b 100644 --- a/source/isaaclab/test/sim/test_material_fragments.py +++ b/source/isaaclab/test/sim/test_material_fragments.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest from pxr import UsdPhysics, UsdShade diff --git a/source/isaaclab/test/sim/test_mesh_collision_fragments.py b/source/isaaclab/test/sim/test_mesh_collision_fragments.py index 5be29b24ea08..5c1b56016374 100644 --- a/source/isaaclab/test/sim/test_mesh_collision_fragments.py +++ b/source/isaaclab/test/sim/test_mesh_collision_fragments.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest from pxr import UsdGeom, UsdPhysics diff --git a/source/isaaclab/test/sim/test_mesh_converter.py b/source/isaaclab/test/sim/test_mesh_converter.py index 2120df259f2f..d2eb64177e22 100644 --- a/source/isaaclab/test/sim/test_mesh_converter.py +++ b/source/isaaclab/test/sim/test_mesh_converter.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import math import os import random diff --git a/source/isaaclab/test/sim/test_schema_fragments.py b/source/isaaclab/test/sim/test_schema_fragments.py index c8a00f31cfff..dfeedef15cfa 100644 --- a/source/isaaclab/test/sim/test_schema_fragments.py +++ b/source/isaaclab/test/sim/test_schema_fragments.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest from pxr import UsdGeom, UsdPhysics diff --git a/source/isaaclab/test/sim/test_schema_writer_nested_targets.py b/source/isaaclab/test/sim/test_schema_writer_nested_targets.py index 1aa9146a2c32..2b0d33696b7e 100644 --- a/source/isaaclab/test/sim/test_schema_writer_nested_targets.py +++ b/source/isaaclab/test/sim/test_schema_writer_nested_targets.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import os import pytest diff --git a/source/isaaclab/test/sim/test_schemas.py b/source/isaaclab/test/sim/test_schemas.py index 9be550bf3f42..89fb364552e6 100644 --- a/source/isaaclab/test/sim/test_schemas.py +++ b/source/isaaclab/test/sim/test_schemas.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import math import warnings diff --git a/source/isaaclab/test/sim/test_simulation_context.py b/source/isaaclab/test/sim/test_simulation_context.py index 2cb7cc9878c8..c338af5e4762 100644 --- a/source/isaaclab/test/sim/test_simulation_context.py +++ b/source/isaaclab/test/sim/test_simulation_context.py @@ -3,11 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit -from isaaclab.test.utils import test_devices - -launch_kit() - import weakref import numpy as np @@ -20,6 +15,7 @@ import isaaclab.sim as sim_utils from isaaclab.physics import PhysicsEvent from isaaclab.sim import SimulationCfg, SimulationContext +from isaaclab.test.utils import test_devices pytestmark = [pytest.mark.kit, pytest.mark.integration] diff --git a/source/isaaclab/test/sim/test_simulation_stage_in_memory.py b/source/isaaclab/test/sim/test_simulation_stage_in_memory.py index 2e2743ed7fab..4809eb36b581 100644 --- a/source/isaaclab/test/sim/test_simulation_stage_in_memory.py +++ b/source/isaaclab/test/sim/test_simulation_stage_in_memory.py @@ -5,11 +5,6 @@ """Integration tests for simulation context with stage in memory.""" -from isaaclab.test.launch import launch_kit - -# FIXME (mmittal): Stage in memory requires cameras to be enabled. -launch_kit(cameras=True) - import pytest import torch @@ -22,6 +17,8 @@ from isaaclab.utils.assets import ISAACLAB_NUCLEUS_DIR from isaaclab.utils.version import get_isaac_sim_version +# kit_cameras: FIXME (mmittal): stage in memory requires cameras to be enabled. +# # kit_solo: sharing a Kit app with other test files killed the pytest process here. In the # kit-reuse-probe-batched CI job this file's first test aborted the interpreter immediately # after collection, with no Python traceback, while the same test is fine in its own process. diff --git a/source/isaaclab/test/sim/test_spawn_from_files.py b/source/isaaclab/test/sim/test_spawn_from_files.py index 2e1e4915e85e..4a9c54fdcdd9 100644 --- a/source/isaaclab/test/sim/test_spawn_from_files.py +++ b/source/isaaclab/test/sim/test_spawn_from_files.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest import omni.kit.app diff --git a/source/isaaclab/test/sim/test_spawn_lights.py b/source/isaaclab/test/sim/test_spawn_lights.py index bea78e909159..7489d86e1a63 100644 --- a/source/isaaclab/test/sim/test_spawn_lights.py +++ b/source/isaaclab/test/sim/test_spawn_lights.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest from pxr import Usd, UsdLux diff --git a/source/isaaclab/test/sim/test_spawn_materials.py b/source/isaaclab/test/sim/test_spawn_materials.py index 93ccd392f7c6..7ae14e830fee 100644 --- a/source/isaaclab/test/sim/test_spawn_materials.py +++ b/source/isaaclab/test/sim/test_spawn_materials.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest from pxr import UsdPhysics, UsdShade diff --git a/source/isaaclab/test/sim/test_spawn_meshes.py b/source/isaaclab/test/sim/test_spawn_meshes.py index 6fd116b42fe3..f259e32a8d8d 100644 --- a/source/isaaclab/test/sim/test_spawn_meshes.py +++ b/source/isaaclab/test/sim/test_spawn_meshes.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import numpy as np import pytest diff --git a/source/isaaclab/test/sim/test_spawn_sensors.py b/source/isaaclab/test/sim/test_spawn_sensors.py index af0df8b714a5..7b818ef38dd1 100644 --- a/source/isaaclab/test/sim/test_spawn_sensors.py +++ b/source/isaaclab/test/sim/test_spawn_sensors.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest from pxr import Usd diff --git a/source/isaaclab/test/sim/test_spawn_shapes.py b/source/isaaclab/test/sim/test_spawn_shapes.py index 7791659f8ec6..7c731eec5ac3 100644 --- a/source/isaaclab/test/sim/test_spawn_shapes.py +++ b/source/isaaclab/test/sim/test_spawn_shapes.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest import isaaclab.sim as sim_utils diff --git a/source/isaaclab/test/sim/test_spawn_wrappers.py b/source/isaaclab/test/sim/test_spawn_wrappers.py index bcf629b53f8e..4877f4e146fc 100644 --- a/source/isaaclab/test/sim/test_spawn_wrappers.py +++ b/source/isaaclab/test/sim/test_spawn_wrappers.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest import isaaclab.sim as sim_utils diff --git a/source/isaaclab/test/sim/test_tendon_fragments.py b/source/isaaclab/test/sim/test_tendon_fragments.py index 65e485911db9..1bb2dfc2f0f8 100644 --- a/source/isaaclab/test/sim/test_tendon_fragments.py +++ b/source/isaaclab/test/sim/test_tendon_fragments.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import pytest from pxr import PhysxSchema, Sdf, Usd, UsdGeom diff --git a/source/isaaclab/test/sim/test_utils_prims.py b/source/isaaclab/test/sim/test_utils_prims.py index 55173047e3c1..bd5b19deeb7d 100644 --- a/source/isaaclab/test/sim/test_utils_prims.py +++ b/source/isaaclab/test/sim/test_utils_prims.py @@ -3,11 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -# note: need to enable cameras to be able to make replicator core available -launch_kit(cameras=True) - import math import numpy as np @@ -20,6 +15,7 @@ from isaaclab.sim.utils.prims import _to_tuple # type: ignore[reportPrivateUsage] from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR, retrieve_file_path +# kit_cameras: replicator core is only available when the app is booted with cameras enabled. pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration, pytest.mark.isaacsim_ci] diff --git a/source/isaaclab/test/sim/test_utils_queries.py b/source/isaaclab/test/sim/test_utils_queries.py index 3e8912a769ab..2e086062e044 100644 --- a/source/isaaclab/test/sim/test_utils_queries.py +++ b/source/isaaclab/test/sim/test_utils_queries.py @@ -3,11 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -# note: need to enable cameras to be able to make replicator core available -launch_kit(cameras=True) - import ast import inspect import textwrap @@ -20,6 +15,7 @@ from isaaclab.sim.utils import queries from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR +# kit_cameras: replicator core is only available when the app is booted with cameras enabled. pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration] diff --git a/source/isaaclab/test/sim/test_utils_semantics.py b/source/isaaclab/test/sim/test_utils_semantics.py index c88f9e0d8dfe..6cec2b26b9a5 100644 --- a/source/isaaclab/test/sim/test_utils_semantics.py +++ b/source/isaaclab/test/sim/test_utils_semantics.py @@ -3,15 +3,11 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -# note: need to enable cameras to be able to make replicator core available -launch_kit(cameras=True) - import pytest import isaaclab.sim as sim_utils +# kit_cameras: replicator core is only available when the app is booted with cameras enabled. pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration] diff --git a/source/isaaclab/test/sim/test_utils_stage.py b/source/isaaclab/test/sim/test_utils_stage.py index 3bcd26e66361..5a9e11439ddf 100644 --- a/source/isaaclab/test/sim/test_utils_stage.py +++ b/source/isaaclab/test/sim/test_utils_stage.py @@ -5,10 +5,6 @@ """Tests for stage utilities.""" -from isaaclab.test.launch import launch_kit - -launch_kit() - import tempfile from pathlib import Path diff --git a/source/isaaclab/test/sim/test_utils_transforms.py b/source/isaaclab/test/sim/test_utils_transforms.py index 1af8ce75bea1..31c3773b1e88 100644 --- a/source/isaaclab/test/sim/test_utils_transforms.py +++ b/source/isaaclab/test/sim/test_utils_transforms.py @@ -3,10 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab.test.launch import launch_kit - -launch_kit() - import math import numpy as np diff --git a/source/isaaclab/test/sim/test_views_xform_prim.py b/source/isaaclab/test/sim/test_views_xform_prim.py index dea4625a21b8..4ef944ea48d2 100644 --- a/source/isaaclab/test/sim/test_views_xform_prim.py +++ b/source/isaaclab/test/sim/test_views_xform_prim.py @@ -10,19 +10,16 @@ prim ordering, xformOp standardization, and Isaac Sim comparison. """ -from isaaclab.test.launch import launch_kit -from isaaclab.test.utils import test_devices - -launch_kit() +import pytest +import torch +import warp as wp -import pytest # noqa: E402 -import torch # noqa: E402 -import warp as wp # noqa: E402 +from pxr import Gf, UsdGeom -from pxr import Gf, UsdGeom # noqa: E402 +from isaaclab.test.utils import test_devices try: - from isaaclab.sim.utils import enable_extension # noqa: E402 + from isaaclab.sim.utils import enable_extension # NOTE: this runs at import, so in a process shared with other test files it changes the # running app's extension set during collection, before any test executes. Harmless when @@ -32,12 +29,12 @@ except (ModuleNotFoundError, ImportError, RuntimeError): _IsaacSimXformPrimView = None -from frame_view_contract_utils import * # noqa: F401, F403, E402 -from frame_view_contract_utils import CHILD_OFFSET, ViewBundle # noqa: E402 +from frame_view_contract_utils import * # noqa: F401, F403 +from frame_view_contract_utils import CHILD_OFFSET, ViewBundle -import isaaclab.sim as sim_utils # noqa: E402 -from isaaclab.sim.views import UsdFrameView as FrameView # noqa: E402 -from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR # noqa: E402 +import isaaclab.sim as sim_utils +from isaaclab.sim.views import UsdFrameView as FrameView +from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR # kit_solo: test_compare_get_world_poses_with_isaacsim goes through Isaac Sim's # SimulationManager, a process-global singleton that caches the PhysxScene wrapping diff --git a/source/isaaclab/test/test_kit_marker_contract.py b/source/isaaclab/test/test_kit_marker_contract.py index b434090c1b88..387ef8a487d4 100644 --- a/source/isaaclab/test/test_kit_marker_contract.py +++ b/source/isaaclab/test/test_kit_marker_contract.py @@ -5,46 +5,42 @@ """Test that every test file's Kit markers agree with what the file actually does. -Kit-dependence is a property of *importing* a test module: a module that constructs -:class:`~isaaclab.app.AppLauncher` at module scope boots Isaac Sim during pytest collection, -before any fixture runs. The ``kit`` / ``kit_cameras`` / ``kitless`` markers make that -property declarative so the runner can group files that share a launch configuration into a -single process instead of paying Kit startup once per file. - -A marker is only useful if it cannot drift from reality, which is what this test enforces: - -* ``kit`` / ``kit_cameras`` -- the file calls :func:`~isaaclab.test.launch.launch_kit` at - module scope with the matching ``cameras`` argument, and never constructs ``AppLauncher`` - or ``SimulationApp`` itself. Direct construction would boot a second, unshared app. -* ``kitless`` -- the file never launches Kit and does not import a Kit runtime package at - module scope, so it can run in a process where Kit was never started. -* ``unit`` -- same requirement as ``kitless``, which turns the marker's registered - description ("does not launch the simulator") into a checked invariant. -* At most one module-scope ``pytestmark`` assignment, since a second assignment silently - rebinds the name and discards the markers from the first. +Kit-dependence is a property of *importing* a test module: a module that imports ``omni`` at +module scope, or constructs :class:`~isaaclab.app.AppLauncher` there, needs Isaac Sim running +by the time pytest imports it -- before any fixture exists. :mod:`isaaclab.test.kit` turns +that property into a declaration, reading a module's ``pytestmark`` out of its source and +booting Kit before the import, so files sharing a launch configuration can share one app. + +A declaration is only useful if it cannot drift from what the file does, which is what this +test enforces: + +* At most one module-scope ``pytestmark`` assignment, since a second assignment rebinds the + name and silently discards the markers from the first. +* At most one launch marker per file. ``kit`` and ``kit_cameras`` are alternatives, so there + is no single app that satisfies both. +* A file declaring a launch marker does not also construct ``AppLauncher`` or + ``SimulationApp``, which would boot a second, unshared app inside the shared one. +* A file marked ``unit`` neither declares a launch marker nor imports a Kit runtime package at + module scope, which turns that marker's registered description ("does not launch the + simulator") into a checked invariant. +* Within :data:`_MIGRATED_ROOTS`, a module-scope Kit runtime import is backed by something + that actually starts Kit -- a launch marker, or the file's own ``AppLauncher``. The checks are AST-based rather than text-based because a source-text search cannot tell an -``AppLauncher`` reference in a docstring from a real call -- several kit-free files mention +``AppLauncher`` reference in a docstring from a real call -- several Kit-free files mention ``AppLauncher`` only to document that they do not use it. - -Files outside :data:`_ENFORCED_ROOTS` are not yet *required* to carry a marker; the -consistency rules above still apply to them whenever they do. Extend that tuple as each -package is migrated. """ from __future__ import annotations import ast -import json -import re -import subprocess -import sys -import textwrap from pathlib import Path import pytest -pytestmark = [pytest.mark.unit, pytest.mark.kitless] +from isaaclab.test.kit import KIT_MARKERS, module_markers + +pytestmark = pytest.mark.unit _REPO_ROOT = Path(__file__).resolve().parents[3] @@ -64,94 +60,37 @@ ) # Packages that only exist inside a running Kit application. ``pxr`` is deliberately absent: -# OpenUSD is importable kit-less through the ``usd-core`` wheel, so importing it says nothing +# OpenUSD is importable Kit-lessly through the ``usd-core`` wheel, so importing it says nothing # about whether Kit is running. _KIT_RUNTIME_PREFIXES = ("omni", "carb", "isaacsim") -# Directories where a test file is required to declare `kit`, `kit_cameras`, or `kitless`. -# Grows one package at a time as files are migrated off module-scope ``AppLauncher``. -_ENFORCED_ROOTS: tuple[str, ...] = () - -_PROFILE_MARKERS = ("kit", "kit_cameras", "kitless") - - -# --------------------------------------------------------------------------- -# AST helpers -# --------------------------------------------------------------------------- +# Names whose construction starts an app the plugin does not own. +_LAUNCHER_NAMES = ("AppLauncher", "SimulationApp") - -def _module_scope_nodes(tree: ast.Module): - """Yield every node that executes at module import, without entering callables. - - Descends through module-level control flow (``if`` / ``try`` / ``with``) because those - bodies still run at import, but stops at function, class, and lambda boundaries because - those bodies only run when called. - """ - stack = list(tree.body) - while stack: - node = stack.pop() - yield node - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda): - continue - for child in ast.iter_child_nodes(node): - stack.append(child) - - -def _call_name(node: ast.AST) -> str | None: - """Return the called function's bare name, for ``f()`` and ``mod.f()`` alike.""" - if not isinstance(node, ast.Call): - return None - func = node.func - if isinstance(func, ast.Name): - return func.id - if isinstance(func, ast.Attribute): - return func.attr - return None - - -def _marker_names(node: ast.AST) -> list[str]: - """Return the marker names in a ``pytest.mark.`` expression or a list of them.""" - if isinstance(node, ast.List | ast.Tuple): - return [name for element in node.elts for name in _marker_names(element)] - if isinstance(node, ast.Call): - return _marker_names(node.func) - if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Attribute): - # pytest.mark. - if node.value.attr == "mark": - return [node.attr] - return [] +# Directories where a module-scope Kit runtime import must be backed by a launch marker or by +# the file's own AppLauncher. Grows one package at a time as files are migrated. +_MIGRATED_ROOTS = ("source/isaaclab/test/sim/",) class _FileFacts: """What a single test file declares and what it actually does at module scope.""" - def __init__(self, path: Path, tree: ast.Module): + def __init__(self, path: Path, source: str, tree: ast.Module): self.path = path - self.pytestmark_assignments: list[int] = [] - self.markers: set[str] = set() - self.launch_kit_cameras: bool | None = None - self.module_scope_launcher: list[tuple[str, int]] = [] - self.launch_kit_anywhere = False + self.markers = set(module_markers(source)) + self.pytestmark_lines: list[int] = [] + self.module_scope_launchers: list[tuple[str, int]] = [] self.kit_runtime_imports: list[tuple[str, int]] = [] - module_scope = set() for node in _module_scope_nodes(tree): - module_scope.add(id(node)) - if isinstance(node, ast.Assign) and any( isinstance(target, ast.Name) and target.id == "pytestmark" for target in node.targets ): - self.pytestmark_assignments.append(node.lineno) - self.markers.update(_marker_names(node.value)) + self.pytestmark_lines.append(node.lineno) name = _call_name(node) - if name in ("AppLauncher", "SimulationApp"): - self.module_scope_launcher.append((name, node.lineno)) - elif name == "launch_kit": - self.launch_kit_cameras = any( - keyword.arg == "cameras" and isinstance(keyword.value, ast.Constant) and keyword.value.value - for keyword in node.keywords - ) + if name in _LAUNCHER_NAMES: + self.module_scope_launchers.append((name, node.lineno)) if isinstance(node, ast.Import): for alias in node.names: @@ -161,35 +100,63 @@ def __init__(self, path: Path, tree: ast.Module): if node.module.split(".")[0] in _KIT_RUNTIME_PREFIXES: self.kit_runtime_imports.append((node.module, node.lineno)) - # Decorator markers (e.g. a per-test `@pytest.mark.unit`) count toward the file's - # marker set, and AppLauncher use anywhere -- not just module scope -- disqualifies - # a file from claiming `kitless`. + # Decorator markers (e.g. a per-test ``@pytest.mark.unit``) count toward the file's + # marker set even though they resolve too late to influence the launch. for node in ast.walk(tree): if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): for decorator in node.decorator_list: - self.markers.update(_marker_names(decorator)) - name = _call_name(node) - if name == "launch_kit": - self.launch_kit_anywhere = True - elif name in ("AppLauncher", "SimulationApp") and id(node) not in module_scope: - self.module_scope_launcher.append((f"{name} (deferred)", node.lineno)) + self.markers.update(_decorator_marker_names(decorator)) @property def rel(self) -> str: + """Repo-relative POSIX path, as it appears in an assertion message.""" return self.path.relative_to(_REPO_ROOT).as_posix() @property - def profile_markers(self) -> list[str]: - return [marker for marker in _PROFILE_MARKERS if marker in self.markers] + def launch_markers(self) -> list[str]: + """The launch markers this file declares, sorted.""" + return sorted(self.markers & KIT_MARKERS.keys()) @property - def launches_kit_directly(self) -> list[tuple[str, int]]: - return self.module_scope_launcher + def launchers(self) -> str: + """The file's own app constructions, formatted for an assertion message.""" + return ", ".join(f"{name} at line {line}" for name, line in self.module_scope_launchers) + + +def _module_scope_nodes(tree: ast.Module): + """Yield every node that executes at module import, without entering callables. + + Descends through module-level control flow (``if`` / ``try`` / ``with``) because those + bodies still run at import, but stops at function, class, and lambda boundaries because + those bodies only run when called. + """ + stack = list(tree.body) + while stack: + node = stack.pop() + yield node + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda): + continue + stack.extend(ast.iter_child_nodes(node)) + + +def _call_name(node: ast.AST) -> str | None: + """Return the called function's bare name, for ``f()`` and ``mod.f()`` alike.""" + if not isinstance(node, ast.Call): + return None + if isinstance(node.func, ast.Name): + return node.func.id + if isinstance(node.func, ast.Attribute): + return node.func.attr + return None -# --------------------------------------------------------------------------- -# Collection -# --------------------------------------------------------------------------- +def _decorator_marker_names(node: ast.AST) -> list[str]: + """Return the marker names in a ``@pytest.mark.`` decorator expression.""" + if isinstance(node, ast.Call): + return _decorator_marker_names(node.func) + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Attribute) and node.value.attr == "mark": + return [node.attr] + return [] def _iter_test_files(): @@ -204,25 +171,19 @@ def facts() -> list[_FileFacts]: """Parse every test file once and return the extracted facts.""" collected = [] for path in _iter_test_files(): + source = path.read_text(encoding="utf-8", errors="replace") try: - tree = ast.parse(path.read_text(encoding="utf-8", errors="replace"), filename=str(path)) + tree = ast.parse(source, filename=str(path)) except SyntaxError as exc: pytest.fail(f"{path.relative_to(_REPO_ROOT).as_posix()} failed to parse: {exc}") - collected.append(_FileFacts(path, tree)) + collected.append(_FileFacts(path, source, tree)) assert collected, f"no test files discovered under {_SCAN_ROOTS} -- the scan roots are wrong" return collected -# --------------------------------------------------------------------------- -# Rules -# --------------------------------------------------------------------------- - - def test_pytestmark_is_assigned_at_most_once(facts: list[_FileFacts]): """A second module-scope ``pytestmark`` rebinds the name and drops the first one's markers.""" - offenders = [ - f"{f.rel}: lines {sorted(f.pytestmark_assignments)}" for f in facts if len(f.pytestmark_assignments) > 1 - ] + offenders = [f"{f.rel}: lines {sorted(f.pytestmark_lines)}" for f in facts if len(f.pytestmark_lines) > 1] assert not offenders, ( "These files assign `pytestmark` more than once at module scope. The later assignment" " replaces the earlier one, so the markers declared first are silently lost:\n " @@ -231,162 +192,70 @@ def test_pytestmark_is_assigned_at_most_once(facts: list[_FileFacts]): ) -def test_profile_markers_are_mutually_exclusive(facts: list[_FileFacts]): - """A file runs in exactly one of the launch configurations, so it declares only one.""" - offenders = [f"{f.rel}: {', '.join(f.profile_markers)}" for f in facts if len(f.profile_markers) > 1] - assert not offenders, "These files declare more than one of `kit`, `kit_cameras`, `kitless`:\n " + "\n ".join( - offenders +def test_launch_markers_are_mutually_exclusive(facts: list[_FileFacts]): + """A file runs in exactly one launch configuration, so it declares at most one.""" + offenders = [f"{f.rel}: {', '.join(f.launch_markers)}" for f in facts if len(f.launch_markers) > 1] + assert not offenders, ( + f"These files declare more than one of {', '.join(sorted(KIT_MARKERS))}, which are" + " alternatives rather than nested configurations, so no single app satisfies both:\n " + "\n ".join(offenders) ) -def test_kit_marked_files_use_launch_kit(facts: list[_FileFacts]): - """`kit` / `kit_cameras` files share the process app; they must not build their own.""" - offenders = [] - for f in facts: - markers = f.profile_markers - if not markers or markers[0] == "kitless": - continue - if f.launches_kit_directly: - where = ", ".join(f"{name} at line {line}" for name, line in f.launches_kit_directly) - offenders.append(f"{f.rel}: declares `{markers[0]}` but constructs {where}") - continue - if f.launch_kit_cameras is None: - offenders.append(f"{f.rel}: declares `{markers[0]}` but never calls launch_kit() at module scope") - continue - wants_cameras = markers[0] == "kit_cameras" - if f.launch_kit_cameras != wants_cameras: - expected = "launch_kit(cameras=True)" if wants_cameras else "launch_kit()" - offenders.append(f"{f.rel}: declares `{markers[0]}` but does not call {expected}") - +def test_launch_marked_files_do_not_build_their_own_app(facts: list[_FileFacts]): + """A marked file is handed the process app; building another one defeats the sharing.""" + offenders = [ + f"{f.rel}: declares `{f.launch_markers[0]}` but constructs {f.launchers}" + for f in facts + if f.launch_markers and f.module_scope_launchers + ] assert not offenders, ( - "These files' Kit markers disagree with how they launch Kit:\n " + "These files declare a launch marker and also construct their own app:\n " + "\n ".join(offenders) - + "\n\nFix: call `launch_kit()` (or `launch_kit(cameras=True)`) from" - " `isaaclab.test.launch` at module scope instead of constructing AppLauncher, and make" - " the marker match the `cameras` argument." + + "\n\nFix: drop the AppLauncher construction and let `isaaclab.test.kit` launch for the" + " marker, or drop the marker and keep the file on a process of its own." ) -@pytest.mark.parametrize("marker", ["kitless", "unit"]) -def test_kit_free_files_do_not_touch_kit(marker: str, facts: list[_FileFacts]): - """`kitless` and `unit` files must run in a process where Kit was never started.""" +def test_unit_files_do_not_touch_kit(facts: list[_FileFacts]): + """A `unit` file must run in a process where Kit was never started.""" offenders = [] for f in facts: - if marker not in f.markers: + if "unit" not in f.markers: continue - if f.launches_kit_directly: - where = ", ".join(f"{name} at line {line}" for name, line in f.launches_kit_directly) - offenders.append(f"{f.rel}: constructs {where}") - if f.launch_kit_anywhere: - offenders.append(f"{f.rel}: calls launch_kit()") + if f.module_scope_launchers: + offenders.append(f"{f.rel}: constructs {f.launchers}") + if f.launch_markers: + offenders.append(f"{f.rel}: declares `{f.launch_markers[0]}`") if f.kit_runtime_imports: where = ", ".join(f"`{name}` at line {line}" for name, line in f.kit_runtime_imports) offenders.append(f"{f.rel}: imports {where} at module scope") assert not offenders, ( - f"These files are marked `{marker}` but depend on a running Kit:\n " + "These files are marked `unit` but depend on a running Kit:\n " + "\n ".join(offenders) + f"\n\nKit runtime packages: {_KIT_RUNTIME_PREFIXES}." - f"\nFix: drop the `{marker}` marker and declare `kit`, or move the Kit import inside the" + "\nFix: mark the file `integration` and declare `kit`, or move the Kit import inside the" " test function so it is not paid at collection." ) -def test_migrated_packages_declare_a_marker(facts: list[_FileFacts]): - """Within a migrated package, every test file states its launch configuration.""" - if not _ENFORCED_ROOTS: - pytest.skip("no packages are enforced yet; extend _ENFORCED_ROOTS as files are migrated") +def test_migrated_files_start_kit_before_importing_it(facts: list[_FileFacts]): + """In a migrated package, a Kit import at module scope needs something that booted Kit. - offenders = [f.rel for f in facts if f.rel.startswith(_ENFORCED_ROOTS) and not f.profile_markers] - assert not offenders, ( - "These files are in a migrated package but declare none of `kit`, `kit_cameras`," - " `kitless`:\n " + "\n ".join(offenders) - ) - - -def test_shareable_file_list_is_derived_from_the_markers(): - """``tools/kit_test_files.py`` must agree with the markers and keep the profiles apart. - - CI batches test files by asking that script which ones can share a Kit app, instead of - carrying a hand-written list that goes stale as files are added or reclassified. These are - the invariants a caller relies on. + Either the file declares a launch marker, in which case the plugin boots for it, or it + still constructs its own ``AppLauncher``. A file with neither imports ``omni`` into a + process where Kit was never started, which fails at collection. """ - sys.path.insert(0, str(_REPO_ROOT / "tools")) - from kit_test_files import shareable_test_files # noqa: PLC0415 - from test_settings import TESTS_TO_SKIP # noqa: PLC0415 - - directory = _REPO_ROOT / "source" / "isaaclab" / "test" / "sim" - sources = {path.name: path.read_text(encoding="utf-8") for path in directory.glob("test_*.py")} - - def declares(source: str, marker: str) -> bool: - # `kit` must not match `kit_cameras` or `kit_solo`. - return re.search(rf"pytest\.mark\.{marker}(?![\w])", source) is not None - - groups = {} - for profile in ("kit", "kit_cameras"): - names = [path.name for path in shareable_test_files(directory, profile)] - assert names, f"no {profile} files found in {directory}" - assert len(names) == len(set(names)), f"duplicate entries for {profile}: {names}" - groups[profile] = set(names) - - expected = { - name - for name, source in sources.items() - if name not in TESTS_TO_SKIP and declares(source, profile) and not declares(source, "kit_solo") - } - assert groups[profile] == expected, ( - f"the derived {profile} list disagrees with the markers:" - f"\n only in list: {sorted(groups[profile] - expected)}" - f"\n only in markers: {sorted(expected - groups[profile])}" - ) - - # The two profiles are separate batches. Cameras cannot be enabled after startup, and a - # camera-enabled app is not a drop-in for a plain one either, so a file appearing in both - # lists would put launch_kit() in a process booted for the other configuration. - overlap = groups["kit"] & groups["kit_cameras"] - assert not overlap, f"these files are in both profile groups and would mix configurations: {sorted(overlap)}" - - -def test_kitless_files_import_without_kit(facts: list[_FileFacts]): - """Importing every `kitless` module must not pull in Kit through a helper module. - - The AST rules only see each file's own imports. A shared test utility that imports Kit - would slip past them, so this imports the real modules in one subprocess and checks that - ``omni.kit.app`` never appears in :data:`sys.modules`. - """ - modules = sorted(f.rel for f in facts if "kitless" in f.markers) - if not modules: - pytest.skip("no files are marked `kitless` yet") - - script = textwrap.dedent(f""" - import importlib.util, json, os, sys - - offenders = [] - for rel in {modules!r}: - # pytest puts a test file's own directory on sys.path (rootdir/conftest handling), - # which is how these modules reach their sibling helpers. Mirror that here. - directory = os.path.dirname(rel) - if directory not in sys.path: - sys.path.insert(0, directory) - - name = "_kitless_probe_" + rel.replace("/", "_")[:-3] - spec = importlib.util.spec_from_file_location(name, rel) - module = importlib.util.module_from_spec(spec) - sys.modules[name] = module - try: - spec.loader.exec_module(module) - except Exception as exc: - offenders.append(f"{{rel}}: import failed: {{type(exc).__name__}}: {{exc}}") - continue - if "omni.kit.app" in sys.modules: - offenders.append(f"{{rel}}: importing it started Kit") - break - print("__RESULTS__" + json.dumps(offenders)) - """) - result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, cwd=_REPO_ROOT, timeout=600) - line = next((ln for ln in result.stdout.splitlines() if ln.startswith("__RESULTS__")), None) - assert line is not None, ( - f"kitless import probe did not report results\n--- stdout ---\n{result.stdout}\n--- stderr ---\n{result.stderr}" + offenders = [ + f"{f.rel}: imports `{f.kit_runtime_imports[0][0]}` at line {f.kit_runtime_imports[0][1]}" + for f in facts + if f.rel.startswith(_MIGRATED_ROOTS) + and f.kit_runtime_imports + and not f.launch_markers + and not f.module_scope_launchers + ] + assert not offenders, ( + "These files import a Kit runtime package at module scope but neither declare one of" + f" {', '.join(sorted(KIT_MARKERS))} nor construct an app themselves, so nothing starts" + " Kit before pytest imports them:\n " + "\n ".join(offenders) ) - offenders = json.loads(line[len("__RESULTS__") :]) - assert not offenders, "These `kitless` files pull in Kit transitively:\n " + "\n ".join(offenders) diff --git a/source/isaaclab/test/test_kit_plugin.py b/source/isaaclab/test/test_kit_plugin.py new file mode 100644 index 000000000000..d50861ae62df --- /dev/null +++ b/source/isaaclab/test/test_kit_plugin.py @@ -0,0 +1,167 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the marker-driven Kit launch plugin. + +The claim the whole arrangement rests on is that :func:`isaaclab.test.kit.pytest_collectstart` +runs *before* pytest imports the test module. If it ever ran after, every ``kit`` file would +fail at its first ``import omni``, so the ordering is asserted here directly rather than left +to CI to discover. + +The launch itself is stubbed out. Booting Kit is what these tests exist to schedule correctly, +not something they need to do. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +import isaaclab.test.kit as kit + +pytestmark = pytest.mark.unit + +_LOG_ENV_VAR = "ISAACLAB_KIT_PLUGIN_PROBE_LOG" + +_CONFTEST = """\ + import os + + import isaaclab.test.kit as kit + + + def _record(entry): + with open(os.environ["{log_env_var}"], "a", encoding="utf-8") as handle: + handle.write(entry + "\\n") + + + def _fake_launch(*, cameras): + _record(f"launch:{{cameras}}") + return object() + + + kit._launch = _fake_launch +""" + +_PYTEST_INI = """\ + [pytest] + markers = + kit: needs a headless Kit app + kit_cameras: needs a Kit app with cameras enabled +""" + +_MODULE = """\ + import os + + import pytest + + with open(os.environ["{log_env_var}"], "a", encoding="utf-8") as handle: + handle.write("import:{name}\\n") + + {pytestmark} + + + def test_placeholder({args}): + pass +""" + + +def _write_module(directory: Path, name: str, *, marker: str | None, args: str = "") -> None: + """Write a scratch test module that logs its own import.""" + (directory / f"test_{name}.py").write_text( + textwrap.dedent(_MODULE).format( + log_env_var=_LOG_ENV_VAR, + name=name, + pytestmark=f"pytestmark = pytest.mark.{marker}" if marker else "", + args=args, + ), + encoding="utf-8", + ) + + +def _run_scratch_pytest(directory: Path) -> tuple[subprocess.CompletedProcess, list[str]]: + """Run pytest over ``directory`` with the plugin loaded, and return its output and log. + + The scratch directory carries its own ``pytest.ini`` so it becomes the rootdir, which keeps + the repo's own conftest -- and therefore the real, unstubbed plugin -- out of the run. + """ + (directory / "conftest.py").write_text( + textwrap.dedent(_CONFTEST).format(log_env_var=_LOG_ENV_VAR), encoding="utf-8" + ) + (directory / "pytest.ini").write_text(textwrap.dedent(_PYTEST_INI), encoding="utf-8") + log = directory / "probe.log" + + result = subprocess.run( + [sys.executable, "-m", "pytest", "-p", "isaaclab.test.kit", "-q", "-p", "no:cacheprovider"], + cwd=directory, + env={**os.environ, _LOG_ENV_VAR: str(log)}, + capture_output=True, + text=True, + timeout=300, + ) + entries = log.read_text(encoding="utf-8").splitlines() if log.exists() else [] + return result, entries + + +def test_kit_is_launched_before_the_module_is_imported(tmp_path: Path): + """The marker only works if the app is up before the module's own imports run.""" + _write_module(tmp_path, "marked", marker="kit") + result, entries = _run_scratch_pytest(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + assert entries == ["launch:False", "import:marked"], ( + f"expected the launch to precede the import, got {entries}\n{result.stdout}" + ) + + +def test_the_marker_selects_the_camera_setting(tmp_path: Path): + """`kit_cameras` must reach AppLauncher as ``enable_cameras=True``, and `kit` as False.""" + _write_module(tmp_path, "plain", marker="kit") + _write_module(tmp_path, "with_cameras", marker="kit_cameras") + result, entries = _run_scratch_pytest(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + launches = {entry for entry in entries if entry.startswith("launch:")} + assert launches == {"launch:False", "launch:True"} + + +def test_an_unmarked_module_does_not_launch_anything(tmp_path: Path): + """Most of the suite is unmarked, and collecting it must stay Kit-free.""" + _write_module(tmp_path, "unmarked", marker=None) + result, entries = _run_scratch_pytest(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + assert entries == ["import:unmarked"] + + +def test_requesting_kit_app_without_a_marker_says_what_is_missing(tmp_path: Path): + """The fixture is only meaningful in a module that declared a launch marker.""" + _write_module(tmp_path, "unmarked", marker=None, args="kit_app") + result, _ = _run_scratch_pytest(tmp_path) + + assert result.returncode != 0 + assert "no Kit app is running" in result.stdout + + +def test_a_second_configuration_in_one_process_is_refused(monkeypatch: pytest.MonkeyPatch): + """`kit` and `kit_cameras` files in one process must fail loudly, not silently share.""" + monkeypatch.setattr(kit, "_app", object()) + monkeypatch.setattr(kit, "_cameras", False) + + with pytest.raises(RuntimeError, match="cannot be changed after startup"): + kit._launch(cameras=True) + + +def test_an_app_started_by_something_else_is_refused(monkeypatch: pytest.MonkeyPatch): + """An app of unknown configuration cannot be handed to a file that asked for a known one.""" + monkeypatch.setattr(kit, "_app", None) + monkeypatch.setattr("isaaclab.utils.has_kit", lambda: True) + + with pytest.raises(RuntimeError, match="not started by this plugin"): + kit._launch(cameras=False) diff --git a/tools/codemods/kit_launch_migration.py b/tools/codemods/kit_launch_migration.py deleted file mode 100644 index f9eec8dad2cd..000000000000 --- a/tools/codemods/kit_launch_migration.py +++ /dev/null @@ -1,325 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Rewrite test modules from a module-scope ``AppLauncher`` to the shared ``launch_kit()``. - -A test module that constructs :class:`~isaaclab.app.AppLauncher` at module scope boots its -own Kit app during pytest collection, so a process covering several such files pays Kit -startup once per file. :func:`~isaaclab.test.launch.launch_kit` is idempotent, so migrated -files share one app per process. - -The rewrite is deliberately in-place and line-based rather than an ``ast.unparse`` round -trip, which would discard comments, ``# isort:skip`` directives, and docstring formatting. -Each edit replaces a statement's own line range, so import ordering -- which matters here, -because Kit must boot before the Kit-dependent imports below it -- is preserved exactly. - -Usage:: - - uv run python tools/codemods/kit_launch_migration.py source/isaaclab/test/sim - uv run python tools/codemods/kit_launch_migration.py --check source/isaaclab/test/sim - -Files the transform cannot handle safely are reported and left untouched. -""" - -from __future__ import annotations - -import argparse -import ast -import sys -from pathlib import Path - -_LAUNCH_IMPORT = "from isaaclab.test.launch import launch_kit" -_APP_IMPORT_MODULE = "isaaclab.app" - -# Docstrings used purely as section separators around the old launch block. They document a -# launch step that no longer exists in the file once it is migrated. -_BOILERPLATE_DOCSTRINGS = ("Launch Isaac Sim Simulator first.", "Rest everything follows.") - -_BOILERPLATE_COMMENTS = ("# launch omniverse app", "# launch the simulator") - - -class Unsupported(Exception): - """Raised when a file needs manual attention rather than a mechanical rewrite.""" - - -def _module_scope_nodes(tree: ast.Module): - """Yield nodes that execute at import, without descending into callables.""" - stack = list(tree.body) - while stack: - node = stack.pop() - yield node - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda): - continue - stack.extend(ast.iter_child_nodes(node)) - - -def _call_name(node: ast.AST) -> str | None: - if not isinstance(node, ast.Call): - return None - func = node.func - if isinstance(func, ast.Name): - return func.id - if isinstance(func, ast.Attribute): - return func.attr - return None - - -def _name_usage_count(tree: ast.Module, name: str) -> int: - """Count how many times ``name`` is loaded anywhere in the module.""" - return sum(1 for node in ast.walk(tree) if isinstance(node, ast.Name) and node.id == name) - - -def _find_launcher(tree: ast.Module) -> tuple[ast.stmt, ast.Call]: - """Return the module-scope statement that builds the app, and the ``AppLauncher`` call.""" - found = [] - for statement in tree.body: - for node in ast.walk(statement): - if _call_name(node) == "SimulationApp": - raise Unsupported("constructs SimulationApp directly") - if _call_name(node) == "AppLauncher": - found.append((statement, node)) - - if not found: - raise Unsupported("no module-scope AppLauncher call") - if len(found) > 1: - raise Unsupported(f"{len(found)} module-scope AppLauncher calls") - - statement, call = found[0] - - # The whole statement is replaced by a bare launch_kit() call, so the launch must be - # unconditional. A file that boots Kit only on some branch -- e.g. - # `AppLauncher(...).app if _USE_KIT else None`, used where a standalone wheel lets the - # tests run kitlessly -- would silently become an unconditional boot. Accept only - # ` = AppLauncher(...)`, ` = AppLauncher(...).app`, or a bare call. - value = statement.value if isinstance(statement, ast.Assign | ast.Expr) else None - if isinstance(value, ast.Attribute): - value = value.value - if value is not call: - raise Unsupported(f"AppLauncher launch is conditional or nested: `{ast.unparse(statement).splitlines()[0]}`") - - # `AppLauncher` must not be referenced for anything else, since its import is removed. - if _name_usage_count(tree, "AppLauncher") > 1: - raise Unsupported("`AppLauncher` is referenced beyond the launch call") - - return statement, call - - -def _resolve_cameras(call: ast.Call) -> bool: - """Map the AppLauncher keywords onto the ``cameras`` argument of ``launch_kit``.""" - if call.args: - raise Unsupported("AppLauncher called with positional arguments") - - cameras = False - for keyword in call.keywords: - if keyword.arg is None: - raise Unsupported("AppLauncher called with **kwargs") - value = keyword.value - literal = value.value if isinstance(value, ast.Constant) else None - - if keyword.arg == "headless": - # `headless=True`, or `headless=HEADLESS` where HEADLESS is a True constant. - if literal is not True and not isinstance(value, ast.Name): - raise Unsupported(f"headless={ast.unparse(value)} is not a literal True") - elif keyword.arg == "enable_cameras": - if not isinstance(literal, bool): - raise Unsupported(f"enable_cameras={ast.unparse(value)} is not a literal bool") - cameras = literal - elif keyword.arg == "device": - # launch_kit always applies resolve_test_sim_device(); anything else is a real - # difference in behaviour and must be looked at by hand. - if ast.unparse(value) != "resolve_test_sim_device()": - raise Unsupported(f"device={ast.unparse(value)} is not resolve_test_sim_device()") - else: - raise Unsupported(f"unsupported AppLauncher keyword {keyword.arg}=") - - return cameras - - -def _pytestmark_statement(tree: ast.Module) -> ast.Assign | None: - marks = [ - node - for node in tree.body - if isinstance(node, ast.Assign) - and any(isinstance(target, ast.Name) and target.id == "pytestmark" for target in node.targets) - ] - if len(marks) > 1: - raise Unsupported("multiple module-scope pytestmark assignments; merge them first") - return marks[0] if marks else None - - -def _render_pytestmark(existing: ast.Assign | None, marker: str) -> str: - """Build the new ``pytestmark`` line with the Kit marker in front.""" - new = f"pytest.mark.{marker}" - if existing is None: - return f"pytestmark = {new}" - value = existing.value - if isinstance(value, ast.List | ast.Tuple): - parts = [new] + [ast.unparse(element) for element in value.elts] - else: - parts = [new, ast.unparse(value)] - return f"pytestmark = [{', '.join(parts)}]" - - -def _is_boilerplate_docstring(node: ast.stmt) -> bool: - return ( - isinstance(node, ast.Expr) - and isinstance(node.value, ast.Constant) - and isinstance(node.value.value, str) - and node.value.value.strip() in _BOILERPLATE_DOCSTRINGS - ) - - -def migrate_source(source: str) -> tuple[str, str]: - """Return the rewritten source and the marker it should carry. - - Raises: - Unsupported: If the file needs manual attention. - """ - tree = ast.parse(source) - statement, call = _find_launcher(tree) - cameras = _resolve_cameras(call) - marker = "kit_cameras" if cameras else "kit" - existing_mark = _pytestmark_statement(tree) - - lines = source.splitlines() - # 1-indexed line numbers to drop entirely. - drop: set[int] = set() - # 1-indexed line number -> replacement text. - replace: dict[int, str] = {} - # 1-indexed line number -> text appended after that line. - insert_after: dict[int, list[str]] = {} - - # The launch statement becomes the launch_kit() call, in place, so that the Kit-dependent - # imports below it still run after Kit has started. - replace[statement.lineno] = "launch_kit(cameras=True)" if cameras else "launch_kit()" - drop.update(range(statement.lineno + 1, (statement.end_lineno or statement.lineno) + 1)) - - # `from isaaclab.app import AppLauncher` becomes the launch_kit import, keeping its slot. - app_import_replaced = False - for node in tree.body: - if isinstance(node, ast.ImportFrom) and node.module == _APP_IMPORT_MODULE: - names = [alias.name for alias in node.names] - if names == ["AppLauncher"]: - replace[node.lineno] = _LAUNCH_IMPORT - drop.update(range(node.lineno + 1, (node.end_lineno or node.lineno) + 1)) - app_import_replaced = True - else: - raise Unsupported(f"`from isaaclab.app import {', '.join(names)}` imports more than AppLauncher") - if not app_import_replaced: - raise Unsupported("no `from isaaclab.app import AppLauncher` to replace") - - # Drop `resolve_test_sim_device` imports that only existed to feed AppLauncher, and - # `HEADLESS = True` constants that nothing else reads. launch_kit covers both. - for node in tree.body: - if isinstance(node, ast.ImportFrom) and node.module == "isaaclab.test.utils": - names = [alias.name for alias in node.names] - if "resolve_test_sim_device" not in names or _name_usage_count(tree, "resolve_test_sim_device") != 1: - continue - remaining = [name for name in names if name != "resolve_test_sim_device"] - span = range(node.lineno, (node.end_lineno or node.lineno) + 1) - if remaining: - # Keep the other names; re-emit as a single line, which is how these imports - # are already written and how the formatter would leave them. - replace[node.lineno] = f"from {node.module} import {', '.join(remaining)}" - drop.update(list(span)[1:]) - else: - drop.update(span) - if isinstance(node, ast.Assign) and len(node.targets) == 1: - target = node.targets[0] - if ( - isinstance(target, ast.Name) - and target.id in ("HEADLESS", "headless") - and _name_usage_count(tree, target.id) == 1 - ): - drop.update(range(node.lineno, (node.end_lineno or node.lineno) + 1)) - - # Drop the separator docstrings and comments that described the removed launch block. - for node in tree.body: - if _is_boilerplate_docstring(node): - drop.update(range(node.lineno, (node.end_lineno or node.lineno) + 1)) - for index, line in enumerate(lines, start=1): - if line.strip().lower() in _BOILERPLATE_COMMENTS: - drop.add(index) - - # Attach the marker, either by extending the existing pytestmark or by adding one after - # the last module-scope import (where such a declaration conventionally sits). - marked = _render_pytestmark(existing_mark, marker) - if existing_mark is not None: - replace[existing_mark.lineno] = marked - drop.update(range(existing_mark.lineno + 1, (existing_mark.end_lineno or existing_mark.lineno) + 1)) - else: - import_ends = [ - node.end_lineno or node.lineno for node in tree.body if isinstance(node, ast.Import | ast.ImportFrom) - ] - if not import_ends: - raise Unsupported("no imports to anchor a new pytestmark to") - if _name_usage_count(tree, "pytest") == 0 and not any( - isinstance(node, ast.Import) and any(a.name == "pytest" for a in node.names) for node in tree.body - ): - raise Unsupported("pytest is not imported, so a pytestmark cannot be added") - insert_after.setdefault(max(import_ends), []).append(marked) - - # Only the header is rewritten, so blank-line cleanup is confined to it. Collapsing - # runs across the whole file would also eat the blank lines PEP 8 requires between - # top-level definitions and produce a diff far larger than the change being made. - header_end = max([*drop, *replace, *insert_after, 1]) - - out: list[str] = [] - for index, line in enumerate(lines, start=1): - if index in replace: - emitted = replace[index] - elif index not in drop: - emitted = line - else: - emitted = None - - if emitted is not None: - in_header = index <= header_end - if not (in_header and not emitted.strip() and out and not out[-1].strip()): - out.append(emitted) - - for extra in insert_after.get(index, []): - out.extend(["", extra]) - - result = "\n".join(out).rstrip("\n") + "\n" - ast.parse(result) # refuse to emit anything that does not parse - return result, marker - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("paths", nargs="+", type=Path, help="files or directories to migrate") - parser.add_argument("--check", action="store_true", help="report what would change without writing") - args = parser.parse_args(argv) - - targets: list[Path] = [] - for path in args.paths: - targets.extend(sorted(path.rglob("test_*.py")) if path.is_dir() else [path]) - - changed, skipped = [], [] - for path in targets: - source = path.read_text(encoding="utf-8") - try: - new_source, marker = migrate_source(source) - except Unsupported as exc: - skipped.append((path, str(exc))) - continue - except SyntaxError as exc: - skipped.append((path, f"produced invalid syntax: {exc}")) - continue - if new_source != source and not args.check: - path.write_text(new_source, encoding="utf-8", newline="\n") - changed.append((path, marker)) - - for path, marker in changed: - print(f"{'would migrate' if args.check else 'migrated'}: {path.as_posix()} -> {marker}") - for path, reason in skipped: - print(f"skipped: {path.as_posix()}: {reason}", file=sys.stderr) - print(f"\n{len(changed)} migrated, {len(skipped)} skipped, {len(targets)} scanned") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From 867b12f1fa7603ab29e74ce479608e23568574df Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 31 Aug 2026 16:27:46 +0000 Subject: [PATCH 11/13] Group marker-declared test files into one Kit process by default The batching path shipped behind ISAACLAB_TEST_BATCH_KIT, which nothing set, so the saving it exists for never landed. Make it the default and turn the variable into an escape hatch: ISAACLAB_TEST_BATCH_KIT=0 restores the per-file path. Only files carrying a launch marker can be grouped, which today is one directory; every other file still gets a process of its own, and a file a dead batch never reached is re-run individually. On source/isaaclab/test/sim that is 43 files in 19 pytest invocations rather than 43. Read the markers through isaaclab.test.kit rather than re-deriving them, so a batch cannot be built around a marker the launch does not honour. Drop tools/kit_test_files.py and the run-package-tests test-path input, which were written for the measurement jobs and have no callers. --- .github/actions/run-package-tests/action.yml | 9 +- source/isaaclab/test/test_kit_batching.py | 47 +++++--- tools/_kit_batching.py | 57 +++++---- tools/conftest.py | 33 ++--- tools/kit_test_files.py | 120 ------------------- 5 files changed, 83 insertions(+), 183 deletions(-) delete mode 100644 tools/kit_test_files.py diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index ccd3fa55b68a..6afdde7cbc73 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -79,13 +79,6 @@ inputs: description: 'Additional pytest options' default: '' required: false - test-path: - description: >- - Path handed to pytest. Defaults to "tools", which loads tools/conftest.py and runs each - test file in its own subprocess. Point it at a test directory instead to run those files - together in a single pytest process, bypassing the per-file orchestrator. - default: 'tools' - required: false extra-pip-packages: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' @@ -298,7 +291,7 @@ runs: - name: Run Tests uses: ./.github/actions/run-tests with: - test-path: ${{ inputs.test-path }} + test-path: "tools" result-file: "${{ inputs.result-file != '' && inputs.result-file || format('{0}-report.xml', github.job) }}" container-name: "${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}" image-tag: ${{ inputs.image-tag }} diff --git a/source/isaaclab/test/test_kit_batching.py b/source/isaaclab/test/test_kit_batching.py index 93c698db885b..e4551364e12e 100644 --- a/source/isaaclab/test/test_kit_batching.py +++ b/source/isaaclab/test/test_kit_batching.py @@ -29,18 +29,19 @@ split_batch_status, ) -pytestmark = [pytest.mark.unit, pytest.mark.kitless] +pytestmark = pytest.mark.unit KIT = "pytestmark = pytest.mark.kit\n" CAMERAS = "pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration]\n" SOLO = "pytestmark = [pytest.mark.kit, pytest.mark.kit_solo]\n" -KITLESS = "pytestmark = pytest.mark.kitless\n" +UNMARKED = "pytestmark = pytest.mark.unit\n" +BOTH = "pytestmark = [pytest.mark.kit, pytest.mark.kit_cameras]\n" LEGACY = "simulation_app = AppLauncher(headless=True).app\n" class TestFileProfile: - """`file_profile` classifies a file from its marker text.""" + """`file_profile` classifies a file from the markers its source declares.""" @pytest.mark.parametrize( "source,expected", @@ -48,19 +49,35 @@ class TestFileProfile: (KIT, "kit"), (CAMERAS, "kit_cameras"), (SOLO, None), - (KITLESS, None), + (UNMARKED, None), (LEGACY, None), + (BOTH, None), ("", None), ], ) def test_profile_matches_markers(self, source: str, expected: str | None): assert file_profile(source) == expected - def test_kit_pattern_does_not_swallow_the_longer_markers(self): - """A bare `kit` match must not claim kit_cameras or kit_solo files.""" - assert file_profile("pytest.mark.kit_cameras") == "kit_cameras" - assert file_profile("pytest.mark.kit_solo") is None - assert file_profile("pytest.mark.kitless") is None + @pytest.mark.parametrize( + "source", + [ + '"""A docstring that mentions pytest.mark.kit."""\n', + "# pytest.mark.kit in a comment\n", + "@pytest.mark.kit\ndef test_one():\n pass\n", + "def helper():\n pytestmark = pytest.mark.kit\n", + ], + ) + def test_markers_outside_a_module_scope_pytestmark_do_not_count(self, source: str): + """Only what the plugin launches from counts, and it launches from ``pytestmark``. + + A per-test decorator resolves after the module is imported, which is already too late + to start Kit, so batching on one would group a file the plugin never launches for. + """ + assert file_profile(source) is None + + def test_unparsable_source_is_not_batched(self): + """A file that does not parse cannot be classified, so it must not be grouped.""" + assert file_profile("pytestmark = [pytest.mark.kit\n") is None class TestGrouping: @@ -80,7 +97,7 @@ def test_profiles_never_mix(self): assert by_profile["kit"] == ["a.py", "c.py"] assert by_profile["kit_cameras"] == ["b.py"] - @pytest.mark.parametrize("source", [SOLO, KITLESS, LEGACY]) + @pytest.mark.parametrize("source", [SOLO, UNMARKED, LEGACY]) def test_unbatchable_files_get_their_own_batch(self, source: str): sources = {"a.py": KIT, "b.py": source, "c.py": KIT} batches = group_test_files(list(sources), sources) @@ -198,14 +215,14 @@ def test_ambiguous_stems_are_not_misattributed(self): class TestEnvironmentToggles: - """Batching stays off unless explicitly enabled.""" + """Batching is on unless a lane explicitly turns it off.""" - @pytest.mark.parametrize("value,expected", [("1", True), ("true", True), ("YES", True), ("0", False), ("", False)]) - def test_enable_flag(self, value: str, expected: bool): + @pytest.mark.parametrize("value,expected", [("0", False), ("false", False), ("NO", False), ("1", True), ("", True)]) + def test_disable_flag(self, value: str, expected: bool): assert batching_enabled({"ISAACLAB_TEST_BATCH_KIT": value}) is expected - def test_disabled_when_unset(self): - assert batching_enabled({}) is False + def test_enabled_when_unset(self): + assert batching_enabled({}) is True @pytest.mark.parametrize("value,expected", [("5", 5), ("", 12), ("nonsense", 12), ("0", 12), ("-3", 12)]) def test_batch_size_override(self, value: str, expected: int): diff --git a/tools/_kit_batching.py b/tools/_kit_batching.py index 41229bf73b01..abff353ce87f 100644 --- a/tools/_kit_batching.py +++ b/tools/_kit_batching.py @@ -7,28 +7,31 @@ A test file that boots Kit at module scope pays Kit startup on its own, and the runner gives every file its own subprocess, so a directory of 23 such files boots Kit 23 times. -Files migrated to :func:`~isaaclab.test.launch.launch_kit` share the app when they land in -one process, which turns those 23 boots into one. +Files that declare a launch marker (see :mod:`isaaclab.test.kit`) share the app when they +land in one process, which turns those 23 boots into one. -Only files carrying the same launch profile may be grouped. ``kit`` and ``kit_cameras`` +Only files carrying the same launch marker may be grouped. ``kit`` and ``kit_cameras`` cannot share a process in either direction: cameras cannot be enabled after startup, and a camera-enabled app is not a substitute for a plain one because some tests assert that offscreen rendering is off. Anything whose behaviour depends on having a process to itself -stays on the per-file path. +stays on the per-file path, and so does every file that declares no marker at all -- which is +still most of the suite. -This module is deliberately free of ``os`` and ``subprocess`` calls: the grouping and the -report demultiplexing are pure functions over paths and strings, so they can be exercised on -any platform, unlike the POSIX-only process machinery in ``tools/conftest.py``. +Apart from reading :data:`BATCH_ENV_VAR` and :data:`BATCH_SIZE_ENV_VAR`, this module is +deliberately free of process machinery: the grouping and the report demultiplexing are pure +functions over paths and strings, so they can be exercised on any platform, unlike the +POSIX-only subprocess handling in ``tools/conftest.py``. """ from __future__ import annotations import os -import re from dataclasses import dataclass, field +from isaaclab.test.kit import SOLO_MARKER, kit_marker, module_markers + BATCH_ENV_VAR = "ISAACLAB_TEST_BATCH_KIT" -"""Environment variable that opts a run into batching. Unset keeps the per-file path.""" +"""Environment variable that turns batching off. Unset (the default) groups what it can.""" BATCH_SIZE_ENV_VAR = "ISAACLAB_TEST_BATCH_SIZE" """Environment variable overriding :data:`DEFAULT_BATCH_SIZE`.""" @@ -48,11 +51,6 @@ them removes most of the risk and almost none of the benefit. """ -# `kit` must not match `kit_cameras` or `kit_solo`. -_MARK_KIT = re.compile(r"pytest\.mark\.kit(?![\w])") -_MARK_CAMERAS = re.compile(r"pytest\.mark\.kit_cameras\b") -_MARK_SOLO = re.compile(r"pytest\.mark\.kit_solo\b") - @dataclass class Batch: @@ -82,9 +80,14 @@ def label(self) -> str: def batching_enabled(env: dict | None = None) -> bool: - """Whether the run opted into batching via :data:`BATCH_ENV_VAR`.""" + """Whether this run may group files, i.e. :data:`BATCH_ENV_VAR` is not set to a false value. + + Batching is on by default because only marker-carrying files can be grouped, and an + unreached member of a dead batch is re-run on the per-file path anyway. The escape hatch + exists so a lane that hits a grouping-specific failure can be unblocked without a revert. + """ env = os.environ if env is None else env - return env.get(BATCH_ENV_VAR, "").strip().lower() in ("1", "true", "yes") + return env.get(BATCH_ENV_VAR, "").strip().lower() not in ("0", "false", "no") def batch_size(env: dict | None = None) -> int: @@ -101,22 +104,26 @@ def batch_size(env: dict | None = None) -> int: def file_profile(source: str) -> str | None: - """Return the launch profile a test file declares, or None if it cannot be batched. + """Return the launch marker a test file can be grouped under, or None if it cannot be. + + This is :func:`isaaclab.test.kit.kit_marker` -- the same reader the plugin launches from, + so a batch cannot be built around a marker the launch does not honour -- plus the + ``kit_solo`` opt-out, which is a batching concern rather than a launch one. Args: - source: The test file's text. Markers are matched against the source rather than by + source: The test file's text. Markers are read from the source rather than by importing the module, because importing a Kit-dependent module boots Kit. Returns: - ``"kit_cameras"``, ``"kit"``, or None when the file is unmarked or opts out. + ``"kit_cameras"``, ``"kit"``, or None when the file is unmarked, opts out, or declares + markers the launch plugin would itself reject. """ - if _MARK_SOLO.search(source): + if SOLO_MARKER in module_markers(source): return None - if _MARK_CAMERAS.search(source): - return "kit_cameras" - if _MARK_KIT.search(source): - return "kit" - return None + try: + return kit_marker(source) + except ValueError: + return None # more than one launch marker; the marker-contract test reports it def group_test_files( diff --git a/tools/conftest.py b/tools/conftest.py index 8cd7edad252f..d07adbbe8973 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -18,6 +18,7 @@ from junitparser import Error, JUnitXml, TestCase, TestSuite from prettytable import PrettyTable +from isaaclab.test.kit import kit_marker from isaaclab.test.utils import resolve_test_sim_device # Local imports @@ -51,19 +52,20 @@ def pytest_ignore_collect(collection_path, config): on-disk cache is populated. """ -_CAMERA_MARKERS = ("enable_cameras=True", "launch_kit(cameras=True)", "pytest.mark.kit_cameras") -"""Source-text signatures of a test file that starts Kit with cameras enabled. - -Matched against the file's text rather than by importing it, because importing a test -module boots Kit. ``enable_cameras=True`` covers files that still construct -``AppLauncher`` directly; the other two cover files migrated to -:func:`~isaaclab.test.launch.launch_kit`, which no longer contain that literal. -""" - def _enables_cameras(test_content: str) -> bool: - """Whether the given test file's source starts Kit with cameras enabled.""" - return any(marker in test_content for marker in _CAMERA_MARKERS) + """Whether the given test file's source starts Kit with cameras enabled. + + Decided from the file's text rather than by importing it, because importing a + Kit-dependent test module boots Kit. A file either declares ``kit_cameras`` and lets + :mod:`isaaclab.test.kit` launch for it, or still constructs ``AppLauncher`` itself. + """ + try: + if kit_marker(test_content) == "kit_cameras": + return True + except ValueError: + pass # contradictory markers; the file fails at collection and the contract test says why + return "enable_cameras=True" in test_content STARTUP_DEADLINE = 120 @@ -1599,10 +1601,11 @@ def pytest_sessionstart(session): # is set. The pytest -m flag only accepts one expression. effective_marker = ci_marker or ("isaacsim_ci" if isaacsim_ci else "") - # Files migrated to launch_kit() share one Kit app when they land in the same process, so - # group them and pay startup once per group instead of once per file. Off unless - # ISAACLAB_TEST_BATCH_KIT is set, and disabled under the work queue, which hands out files - # one at a time across containers and so cannot offer coherent groups. + # Files that declare a launch marker share one Kit app when they land in the same process, + # so group them and pay startup once per group instead of once per file. Unmarked files -- + # still most of the suite -- keep a process each. Disabled by ISAACLAB_TEST_BATCH_KIT=0, and + # under the work queue, which hands out files one at a time across containers and so cannot + # offer coherent groups. batched_files, batch_results = [], ([], {}, []) if batching_enabled() and not os.environ.get("ISAACLAB_TEST_QUEUE"): sources = {} diff --git a/tools/kit_test_files.py b/tools/kit_test_files.py deleted file mode 100644 index 7c7848d7843d..000000000000 --- a/tools/kit_test_files.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""List the test files in a directory that can share one Kit app. - -The ``kit`` / ``kit_cameras`` / ``kit_solo`` markers already record which files can share a -Kit app; this turns that into the file list a runner needs, so the two never drift. Anything -that hardcodes such a list has to be updated by hand whenever a file is added, renamed, or -reclassified, and a stale list is silently wrong rather than loudly broken. - -One profile at a time. ``kit`` and ``kit_cameras`` files cannot share a process in either -direction: cameras cannot be enabled after startup, and a camera-enabled app is not a drop-in -replacement for a plain one because some tests assert that offscreen rendering is off. Each -profile is a separate batch, so the caller asks for one. - -Selection: files marked with the requested profile, minus those also marked ``kit_solo`` and -those in :data:`tools.test_settings.TESTS_TO_SKIP`. - -Markers are read from the file's source rather than by importing it, because importing a -Kit-dependent test module boots Kit. - -Usage:: - - python3 tools/kit_test_files.py source/isaaclab/test/sim --profile kit --format paths - python3 tools/kit_test_files.py source/isaaclab/test/sim --profile kit_cameras --format names -""" - -from __future__ import annotations - -import argparse -import re -import sys -from pathlib import Path - -# `kit` must not match `kit_cameras` or `kit_solo`, hence the boundary on the plain pattern. -_MARK_KIT = re.compile(r"pytest\.mark\.kit(?![\w])") -_MARK_CAMERAS = re.compile(r"pytest\.mark\.kit_cameras\b") -_MARK_SOLO = re.compile(r"pytest\.mark\.kit_solo\b") - - -def _tests_to_skip() -> frozenset[str]: - """Names from ``tools/test_settings.py``, which the per-file runner also honours.""" - sys.path.insert(0, str(Path(__file__).resolve().parent)) - try: - from test_settings import TESTS_TO_SKIP # noqa: PLC0415 - except ImportError: - return frozenset() - return frozenset(TESTS_TO_SKIP) - - -def shareable_test_files(directory: Path, profile: str = "kit") -> list[Path]: - """Return the files under ``directory`` that can share one Kit app of ``profile``. - - Args: - directory: Directory to scan, non-recursively matching ``test_*.py``. - profile: Which launch configuration to select, ``"kit"`` or ``"kit_cameras"``. - - Returns: - The selected files, sorted by name. - - Raises: - ValueError: If ``profile`` is not a known launch configuration. - """ - if profile not in ("kit", "kit_cameras"): - raise ValueError(f"unknown profile {profile!r}; expected 'kit' or 'kit_cameras'") - - skip = _tests_to_skip() - selected = [] - for path in sorted(directory.glob("test_*.py")): - if path.name in skip: - continue - source = path.read_text(encoding="utf-8", errors="replace") - if _MARK_SOLO.search(source): - continue - # `kit_cameras` implies the file also matches the plain `kit` pattern's prefix, so - # classify on the more specific marker first. - if _MARK_CAMERAS.search(source): - if profile == "kit_cameras": - selected.append(path) - elif _MARK_KIT.search(source) and profile == "kit": - selected.append(path) - return selected - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("directory", type=Path, help="directory to scan for test files") - parser.add_argument( - "--profile", - choices=("kit", "kit_cameras"), - default="kit", - help="which launch configuration to select; the two never share a process", - ) - parser.add_argument( - "--format", - choices=("paths", "names"), - default="paths", - help="'paths' for space-separated repo paths (pytest arguments); " - "'names' for comma-separated file names (the include-files input)", - ) - args = parser.parse_args(argv) - - if not args.directory.is_dir(): - parser.error(f"not a directory: {args.directory}") - - files = shareable_test_files(args.directory, args.profile) - if not files: - parser.error(f"no {args.profile} test files found in {args.directory}") - - if args.format == "paths": - print(" ".join(path.as_posix() for path in files)) - else: - print(",".join(path.name for path in files)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From 8f6d01a8dedf4464003261df17b2a81f3177c47a Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 31 Aug 2026 17:16:55 +0000 Subject: [PATCH 12/13] Rename kit_solo to solo and require it to accompany a launch marker Sitting in the marker list beside kit and kit_cameras, kit_solo read like a third launch configuration. It is not one: kit and kit_cameras say which app a file needs, while this says who else may share the process, so the two compose rather than compete. Dropping the prefix stops the false parallelism, and the marker moves next to device_split, the other marker about how a file is invoked. Nothing stopped a file from carrying solo alone, which launched no app while looking like it did -- exactly how a file ends up importing omni into a process where Kit was never started. The marker contract now rejects that: solo is only meaningful for a file that gets an app booted for it, since an unmarked file is never grouped with another anyway. --- docs/source/refs/contributing.rst | 2 +- pyproject.toml | 2 +- .../changelog.d/mataylor-kit-test-markers.rst | 2 +- source/isaaclab/isaaclab/test/kit.py | 11 +++++++--- .../sim/test_simulation_stage_in_memory.py | 4 ++-- .../test/sim/test_views_xform_prim.py | 4 ++-- source/isaaclab/test/test_kit_batching.py | 2 +- .../isaaclab/test/test_kit_marker_contract.py | 21 ++++++++++++++++++- tools/_kit_batching.py | 4 ++-- 9 files changed, 38 insertions(+), 14 deletions(-) diff --git a/docs/source/refs/contributing.rst b/docs/source/refs/contributing.rst index f278bdcd02ad..eec05829007a 100644 --- a/docs/source/refs/contributing.rst +++ b/docs/source/refs/contributing.rst @@ -785,7 +785,7 @@ file of each kind cannot share a process. The app is started once per pytest process and shared by every marked file in it, so a run covering many such files pays Kit startup once rather than once per file. Add -``pytest.mark.kit_solo`` to keep a file out of that sharing when it depends on having a process +``pytest.mark.solo`` to keep a file out of that sharing when it depends on having a process to itself. Tests that need the app object itself request the ``kit_app`` fixture. diff --git a/pyproject.toml b/pyproject.toml index efe0a4be8fb7..b8e5507685d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -341,6 +341,7 @@ ignore-words-list = "aheared,collet,followng,haa,publically,rouines,slq,collapsa markers = [ "isaacsim_ci: mark test to run in isaacsim ci", "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5)", + "solo: run this file in a process of its own; it is never grouped with other files", "windows_ci: mark test to run on Windows platforms in CI", "arm_ci: mark test to run on ARM platforms in CI (e.g. NVIDIA DGX Spark)", "unit: test exercises isolated logic and does not launch the simulator", @@ -351,7 +352,6 @@ markers = [ "kitless: test must pass inside the Kit-less container, which has no Isaac Sim runtime", "kit: test file needs a headless Kit app; the runner boots one before importing the module and shares it with the other `kit` files in the process", "kit_cameras: like `kit`, but the app is booted with cameras enabled; the two configurations never share a process", - "kit_solo: keep this file in its own process; it is never grouped with other files", ] # Add pypi.nvidia.com so that `uv pip install isaaclab[isaacsim]` works without --extra-index-url. diff --git a/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst b/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst index aeae982551f3..5e83634f2e70 100644 --- a/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst +++ b/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst @@ -1,7 +1,7 @@ Added ^^^^^ -* Added the ``kit``, ``kit_cameras``, and ``kit_solo`` pytest markers, and the +* Added the ``kit``, ``kit_cameras``, and ``solo`` pytest markers, and the :mod:`isaaclab.test.kit` plugin that acts on them. A test file that needs Isaac Sim now declares it in its module-level ``pytestmark``; the plugin reads that declaration out of the file's source and boots Kit before pytest imports the module, so files sharing a launch diff --git a/source/isaaclab/isaaclab/test/kit.py b/source/isaaclab/isaaclab/test/kit.py index 39618fec0e96..ae666b319806 100644 --- a/source/isaaclab/isaaclab/test/kit.py +++ b/source/isaaclab/isaaclab/test/kit.py @@ -49,8 +49,13 @@ KIT_MARKERS: dict[str, bool] = {"kit": False, "kit_cameras": True} """The launch markers, mapped to the ``enable_cameras`` setting each one asks for.""" -SOLO_MARKER = "kit_solo" -"""Marker that keeps a file in a process of its own, never grouped with other files.""" +SOLO_MARKER = "solo" +"""Marker that keeps a file in a process of its own, never grouped with other files. + +A separate axis from :data:`KIT_MARKERS`, which say *which* app a file needs rather than who +else may share the process with it, so the two compose: ``[pytest.mark.kit, pytest.mark.solo]`` +still gets an app booted for it, just not one anybody else is using. +""" _app: SimulationApp | None = None """The app booted for this process, or None before the first marked module is collected.""" @@ -195,7 +200,7 @@ def _launch(*, cameras: bool) -> SimulationApp: raise RuntimeError( "Kit is already running but was not started by this plugin, so its launch" " configuration is unknown. Another test file in this process constructs AppLauncher" - " itself; mark that file `kit_solo` so it keeps a process of its own." + " itself; mark that file `solo` so it keeps a process of its own." ) from isaaclab.app import AppLauncher diff --git a/source/isaaclab/test/sim/test_simulation_stage_in_memory.py b/source/isaaclab/test/sim/test_simulation_stage_in_memory.py index 4809eb36b581..e537cc5a78a9 100644 --- a/source/isaaclab/test/sim/test_simulation_stage_in_memory.py +++ b/source/isaaclab/test/sim/test_simulation_stage_in_memory.py @@ -19,12 +19,12 @@ # kit_cameras: FIXME (mmittal): stage in memory requires cameras to be enabled. # -# kit_solo: sharing a Kit app with other test files killed the pytest process here. In the +# solo: sharing a Kit app with other test files killed the pytest process here. In the # kit-reuse-probe-batched CI job this file's first test aborted the interpreter immediately # after collection, with no Python traceback, while the same test is fine in its own process. # The cause is not yet understood -- creating the stage in memory is sensitive to what else has # already touched the stage or the extension set -- so keep the file on its own until it is. -pytestmark = [pytest.mark.kit_cameras, pytest.mark.kit_solo, pytest.mark.integration] +pytestmark = [pytest.mark.kit_cameras, pytest.mark.solo, pytest.mark.integration] @pytest.fixture diff --git a/source/isaaclab/test/sim/test_views_xform_prim.py b/source/isaaclab/test/sim/test_views_xform_prim.py index 4ef944ea48d2..88f896ed4cb4 100644 --- a/source/isaaclab/test/sim/test_views_xform_prim.py +++ b/source/isaaclab/test/sim/test_views_xform_prim.py @@ -36,13 +36,13 @@ from isaaclab.sim.views import UsdFrameView as FrameView from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR -# kit_solo: test_compare_get_world_poses_with_isaacsim goes through Isaac Sim's +# solo: test_compare_get_world_poses_with_isaacsim goes through Isaac Sim's # SimulationManager, a process-global singleton that caches the PhysxScene wrapping # /physicsScene. In a process shared with other test files that prim belongs to a stage an # earlier file already tore down, so the cached wrapper is dangling and the test dies with # "Accessed invalid expired 'PhysicsScene' prim". Nothing in this file owns that state, so the # file needs a process to itself until SimulationManager can be reset between files. -pytestmark = [pytest.mark.kit, pytest.mark.kit_solo, pytest.mark.integration, pytest.mark.isaacsim_ci] +pytestmark = [pytest.mark.kit, pytest.mark.solo, pytest.mark.integration, pytest.mark.isaacsim_ci] PARENT_POS = (0.0, 0.0, 1.0) diff --git a/source/isaaclab/test/test_kit_batching.py b/source/isaaclab/test/test_kit_batching.py index e4551364e12e..923e0ad4b6af 100644 --- a/source/isaaclab/test/test_kit_batching.py +++ b/source/isaaclab/test/test_kit_batching.py @@ -34,7 +34,7 @@ KIT = "pytestmark = pytest.mark.kit\n" CAMERAS = "pytestmark = [pytest.mark.kit_cameras, pytest.mark.integration]\n" -SOLO = "pytestmark = [pytest.mark.kit, pytest.mark.kit_solo]\n" +SOLO = "pytestmark = [pytest.mark.kit, pytest.mark.solo]\n" UNMARKED = "pytestmark = pytest.mark.unit\n" BOTH = "pytestmark = [pytest.mark.kit, pytest.mark.kit_cameras]\n" LEGACY = "simulation_app = AppLauncher(headless=True).app\n" diff --git a/source/isaaclab/test/test_kit_marker_contract.py b/source/isaaclab/test/test_kit_marker_contract.py index 387ef8a487d4..356166cdd9f7 100644 --- a/source/isaaclab/test/test_kit_marker_contract.py +++ b/source/isaaclab/test/test_kit_marker_contract.py @@ -25,6 +25,8 @@ simulator") into a checked invariant. * Within :data:`_MIGRATED_ROOTS`, a module-scope Kit runtime import is backed by something that actually starts Kit -- a launch marker, or the file's own ``AppLauncher``. +* ``solo`` appears only alongside a launch marker. It is the only case where it changes + anything, and writing it alone reads like a launch marker while starting no app at all. The checks are AST-based rather than text-based because a source-text search cannot tell an ``AppLauncher`` reference in a docstring from a real call -- several Kit-free files mention @@ -38,7 +40,7 @@ import pytest -from isaaclab.test.kit import KIT_MARKERS, module_markers +from isaaclab.test.kit import KIT_MARKERS, SOLO_MARKER, module_markers pytestmark = pytest.mark.unit @@ -201,6 +203,23 @@ def test_launch_markers_are_mutually_exclusive(facts: list[_FileFacts]): ) +def test_solo_accompanies_a_launch_marker(facts: list[_FileFacts]): + """`solo` only changes anything for a file that gets an app booted for it. + + A file with no launch marker already runs on its own, so `solo` alone is not merely + redundant -- it reads like a launch marker while starting no app, which is how a file ends + up importing ``omni`` into a process where Kit was never started. + """ + offenders = [f.rel for f in facts if SOLO_MARKER in f.markers and not f.launch_markers] + assert not offenders, ( + f"These files declare `{SOLO_MARKER}` without one of {', '.join(sorted(KIT_MARKERS))}, so" + " nothing boots an app for them and the marker changes nothing:\n " + + "\n ".join(offenders) + + f"\n\nFix: add the launch marker the file needs, or drop `{SOLO_MARKER}` -- an unmarked" + " file is never grouped with another anyway." + ) + + def test_launch_marked_files_do_not_build_their_own_app(facts: list[_FileFacts]): """A marked file is handed the process app; building another one defeats the sharing.""" offenders = [ diff --git a/tools/_kit_batching.py b/tools/_kit_batching.py index abff353ce87f..839121088aee 100644 --- a/tools/_kit_batching.py +++ b/tools/_kit_batching.py @@ -108,7 +108,7 @@ def file_profile(source: str) -> str | None: This is :func:`isaaclab.test.kit.kit_marker` -- the same reader the plugin launches from, so a batch cannot be built around a marker the launch does not honour -- plus the - ``kit_solo`` opt-out, which is a batching concern rather than a launch one. + ``solo`` opt-out, which is a batching concern rather than a launch one. Args: source: The test file's text. Markers are read from the source rather than by @@ -135,7 +135,7 @@ def group_test_files( ) -> list[Batch]: """Partition ``test_files`` into batches, preserving the given order. - Files that cannot be grouped -- unmarked, ``kit_solo``, or listed in ``unbatchable`` -- + Files that cannot be grouped -- unmarked, ``solo``, or listed in ``unbatchable`` -- each become a batch of one, which is exactly the current per-file behaviour. Args: From b7b11073d8536a256397a4b8235568b467efce05 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 31 Aug 2026 19:01:46 +0000 Subject: [PATCH 13/13] Describe the test plan in one file and stop running it from a conftest What each CI lane covered could only be worked out by tracing four layers: a workflow job's inputs, two composite actions, a bash translation into sixteen TEST_* docker environment variables, and a collector that read them back. The collector lived in tools/conftest.py, which disabled collection outright and ran the entire suite from pytest_sessionstart -- pytest as a process launcher. Being a conftest, it hijacked any pytest run rooted at the repository, which is why tools-tests.yml had to pass --noconftest to test the repository's own tools. tools/test_plan.toml now says what every job covers, tools/testplan.py resolves a job to a file list, and tools/run_tests.py executes it. The runner keeps everything that makes a long run survivable -- per-file timeouts, startup-hang detection, stack dumps, crash reports, retries, the work queue, the merged JUnit output -- but it is a script with a command line rather than a hook. Grouping marker-declared files into one Kit process stops being a special case and becomes the runner's normal schedule, which is what the caller could have asked for all along. The fourteen uniform package lanes are generated from the plan by tools/generate_workflows.py, between sentinel comments, with a test that fails on drift. Lanes with bespoke setup stay hand-written and are checked against the plan instead. Job display names are unchanged, so required status checks still match. Locally the same plan is one command: `isaaclab -t` runs everything, `--job ` runs a single lane the way CI runs it, `--list-jobs` lists them. Selection is unchanged. A harness resolved all 29 call sites both ways -- through the old collector with that job's environment variables, and through the plan -- and the file lists are identical, sharding and skip-list overrides included. --- .../actions/multi-gpu/mgpu_shard_select.py | 4 +- .../multi-gpu/multi_gpu_shard_runner.sh | 5 +- .github/actions/run-package-tests/action.yml | 48 +-- .github/actions/run-tests/action.yml | 48 +-- .github/actions/run-tests/run_tests.sh | 114 ++---- .github/workflows/arm-ci.yml | 6 +- .github/workflows/build.yaml | 240 +++++------- .github/workflows/daily-compatibility.yml | 4 +- .github/workflows/test-multi-gpu-pytest.yaml | 2 +- .github/workflows/tools-tests.yml | 7 +- conftest.py | 20 +- docs/source/refs/contributing.rst | 18 + .../changelog.d/mataylor-kit-test-markers.rst | 14 +- source/isaaclab/isaaclab/cli/__init__.py | 5 +- source/isaaclab/isaaclab/cli/commands/misc.py | 13 +- .../test_test_orchestrator_result_handling.py | 4 +- .../mataylor-kit-test-markers.skip | 1 + .../test/benchmarking/conftest.py | 2 +- tools/_device_split.py | 4 +- tools/_kit_batching.py | 2 +- tools/changelog/pyproject.toml | 2 +- tools/crash_journal.py | 2 +- tools/generate_workflows.py | 181 +++++++++ tools/hang_dump.py | 4 +- tools/ovrtx_log.py | 10 +- tools/{conftest.py => run_tests.py} | 366 ++++++++---------- tools/skills/pyproject.toml | 2 +- tools/test/test_test_plan.py | 138 +++++++ tools/test_crash_journal.py | 4 +- tools/test_plan.toml | 280 ++++++++++++++ tools/test_settings.py | 2 +- tools/testplan.py | 236 +++++++++++ 32 files changed, 1240 insertions(+), 548 deletions(-) create mode 100644 source/isaaclab_tasks/changelog.d/mataylor-kit-test-markers.skip create mode 100644 tools/generate_workflows.py rename tools/{conftest.py => run_tests.py} (87%) create mode 100644 tools/test/test_test_plan.py create mode 100644 tools/test_plan.toml create mode 100644 tools/testplan.py diff --git a/.github/actions/multi-gpu/mgpu_shard_select.py b/.github/actions/multi-gpu/mgpu_shard_select.py index 38d1ac18827c..47dc22d51ea2 100644 --- a/.github/actions/multi-gpu/mgpu_shard_select.py +++ b/.github/actions/multi-gpu/mgpu_shard_select.py @@ -5,7 +5,7 @@ """pytest plugin: select which tests a non-default GPU shard runs. -Loaded only by the multi-GPU lane: ``tools/conftest.py`` injects it via +Loaded only by the multi-GPU lane: ``tools/run_tests.py`` injects it via ``-p mgpu_shard_select`` into each per-file pytest subprocess (and prepends this directory to ``PYTHONPATH`` so it is importable). It lives next to the lane scripts rather than as a repo-root ``conftest.py`` so it only affects the lane. @@ -86,7 +86,7 @@ def pytest_collection_modifyitems(config, items): def pytest_sessionfinish(session, exitstatus): # A file whose tests are all out of scope deselects to zero, so pytest exits # NO_TESTS_COLLECTED (5). The lane orchestrator treats any non-zero per-file - # exit as a failure (tools/conftest.py), so report "nothing in scope for this + # exit as a failure (tools/run_tests.py), so report "nothing in scope for this # file" as success rather than a false failure. if _shard_mask() is not None and exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: session.exitstatus = pytest.ExitCode.OK diff --git a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh index 9d3b5dec5d08..ff7035883e8f 100755 --- a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh +++ b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh @@ -103,10 +103,7 @@ for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do # C-style loop; start at 1 to sk # full $shard_log under a collapsible ``::group::shard cuda:N log``. # (tee = full output to the log file; stdbuf -oL = flush per line so the # filtered grep/sed stream appears live, not in delayed chunks.) - ./isaaclab.sh -p -m pytest \ - --ignore=tools/conftest.py \ - --ignore=source/isaaclab/test/install_ci \ - tools -v 2>&1 \ + ./isaaclab.sh -p tools/run_tests.py --all 2>&1 \ | tee "$shard_log" \ | stdbuf -oL grep -aE \ '🚀|^source/.*::.* (PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)|^(Total|Passing|Failing|Crashed|Startup Hang|Timeout|Total Wall Time|Total Test Time|Passing Percentage):|^~~~~|^=+ |^E +|^ +File |Traceback|^FAILED|^ERROR ' \ diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index d2782786bafb..0730afa24273 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -27,44 +27,21 @@ inputs: description: 'ECR cache tag' default: 'cache-base' required: false - filter-pattern: - description: >- - Pattern to filter test files (e.g., isaaclab_tasks); test files whose path - contains this pattern are included. Only one pattern can be used at a time, - no comma-separated substrings allowed. Also supports the legacy "not " - form for exclude-only jobs. - default: '' - required: false - exclude-pattern: - description: >- - Comma-separated substrings; test files whose path contains any entry are - skipped. Combines with filter-pattern (include + exclude). - default: '' - required: false test-k-expr: description: >- Global pytest -k expression applied inside every per-file pytest run - spawned by tools/conftest.py (combined with device-split selectors). + spawned by tools/run_tests.py (combined with device-split selectors). default: '' required: false - shard-index: - description: 'Zero-based shard index' - default: '' - required: false - shard-count: - description: 'Total number of shards' + job: + description: >- + Name of a job in tools/test_plan.toml. The plan decides which files the job covers; + tools/run_tests.py runs them. Leave empty only when test-path names a file to run + directly through pytest. default: '' required: false - curobo-only: - description: 'Run only cuRobo and SkillGen tests' - default: 'false' - required: false - quarantined-only: - description: 'Run only quarantined tests' - default: 'false' - required: false - include-files: - description: 'Comma-separated list of specific test files to include' + shard: + description: 'Which shard of a sharded job to run, 0-based. Empty for an unsharded job.' default: '' required: false test-node-ids-file: @@ -301,14 +278,9 @@ runs: container-name: "${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}" image-tag: ${{ inputs.image-tag }} pytest-options: ${{ inputs.pytest-options }} - filter-pattern: ${{ inputs.filter-pattern }} - exclude-pattern: ${{ inputs.exclude-pattern }} + job: ${{ inputs.job }} + shard: ${{ inputs.shard }} test-k-expr: ${{ inputs.test-k-expr }} - shard-index: ${{ inputs.shard-index }} - shard-count: ${{ inputs.shard-count }} - curobo-only: ${{ inputs.curobo-only }} - quarantined-only: ${{ inputs.quarantined-only }} - include-files: ${{ inputs.include-files }} test-node-ids-file: ${{ inputs.test-node-ids-file }} test-node-ids-key: ${{ inputs.test-node-ids-key }} volume-mount-source: ${{ github.workspace }} diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index abac275ddf78..aed49abc6832 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -27,38 +27,23 @@ inputs: description: 'Additional pytest options (e.g., -k filter)' default: '' required: false - filter-pattern: - description: >- - Pattern to filter test files (e.g., isaaclab_tasks); test files whose path - contains this pattern are included. Only one pattern can be used at a time, - no comma-separated substrings allowed. Also supports the legacy "not " - form for exclude-only jobs. - default: '' - required: false - exclude-pattern: - description: >- - Comma-separated substrings; test files whose path contains any entry are - excluded. Combines with filter-pattern (include + exclude). - default: '' - required: false test-k-expr: description: >- Global pytest -k expression applied inside every per-file pytest run - spawned by tools/conftest.py (combined with device-split selectors). + spawned by tools/run_tests.py (combined with device-split selectors). Unlike pytest-options, this reaches the individual test processes, so it can deselect parametrized cases (e.g. "not ovphysx"). default: '' required: false - curobo-only: - description: 'Run only cuRobo and SkillGen tests (requires the cuRobo Docker image)' - default: 'false' - required: false - quarantined-only: - description: 'Run only tests listed in QUARANTINED_TESTS (skipped in normal jobs)' - default: 'false' + job: + description: >- + Name of a job in tools/test_plan.toml. The plan decides which files the job covers; + tools/run_tests.py runs them. Leave empty only when test-path names a file to run + directly through pytest. + default: '' required: false - include-files: - description: 'Comma-separated list of specific test file paths to include (e.g., source/pkg/test/test_a.py,source/pkg/test/test_b.py)' + shard: + description: 'Which shard of a sharded job to run, 0-based. Empty for an unsharded job.' default: '' required: false test-node-ids-file: @@ -69,14 +54,6 @@ inputs: description: 'Top-level key in test-node-ids-file containing the node IDs for this job' default: '' required: false - shard-index: - description: 'Zero-based index of this shard (used with shard-count to split tests across parallel jobs)' - default: '' - required: false - shard-count: - description: 'Total number of shards (used with shard-index to split tests across parallel jobs)' - default: '' - required: false volume-mount-source: description: 'Host path to bind-mount at /workspace/isaaclab (for deps-cache-hit mode)' default: '' @@ -100,10 +77,6 @@ inputs: description: 'Host Warp kernel cache directory bind-mounted into the container as WARP_CACHE_PATH' default: '' required: false - ci-marker: - description: 'CI_MARKER value forwarded to the container (read by tools/conftest.py to select test files by pytest marker)' - default: '' - required: false standalone-script-scope: description: 'Enable standalone script smoke tests for this scripts/ subdirectory' default: '' @@ -132,9 +105,8 @@ runs: # the run_tests positional arguments if substituted textually. PYTEST_OPTIONS: ${{ inputs.pytest-options }} TEST_K_EXPR_INPUT: ${{ inputs.test-k-expr }} - CI_MARKER_INPUT: ${{ inputs.ci-marker }} run: | - bash .github/actions/run-tests/run_tests.sh "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "$TEST_K_EXPR_INPUT" "$CI_MARKER_INPUT" "${{ inputs.standalone-script-scope }}" "${{ inputs.standalone-script-visualizer }}" "${{ inputs.standalone-script-runtime-group }}" "${{ inputs.warp-cache-host-dir }}" "${{ inputs.extra-uv-packages }}" + bash .github/actions/run-tests/run_tests.sh "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.job }}" "${{ inputs.shard }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "$TEST_K_EXPR_INPUT" "${{ inputs.standalone-script-scope }}" "${{ inputs.standalone-script-visualizer }}" "${{ inputs.standalone-script-runtime-group }}" "${{ inputs.warp-cache-host-dir }}" "${{ inputs.extra-uv-packages }}" - name: Kill container on cancellation if: cancelled() shell: bash diff --git a/.github/actions/run-tests/run_tests.sh b/.github/actions/run-tests/run_tests.sh index 3779ab10c562..38d62ab53b58 100755 --- a/.github/actions/run-tests/run_tests.sh +++ b/.github/actions/run-tests/run_tests.sh @@ -17,26 +17,20 @@ run_tests() { local image_tag="$4" local reports_dir="$5" local pytest_options="$6" - local filter_pattern="$7" - local exclude_pattern="$8" - local curobo_only="$9" - local include_files="${10}" - local quarantined_only="${11}" - local shard_index="${12}" - local shard_count="${13}" - local volume_mount_source="${14}" - local extra_pip_packages="${15}" - local test_node_ids_file="${16}" - local test_node_ids_key="${17}" - local wheelhouse_host_dir="${18}" - local wheelhouse_packages="${19}" - local test_k_expr="${20}" - local ci_marker="${21}" - local standalone_script_scope="${22}" - local standalone_script_visualizer="${23}" - local standalone_script_runtime_group="${24}" - local warp_cache_host_dir="${25}" - local extra_uv_packages="${26}" + local job="$7" + local shard="$8" + local volume_mount_source="$9" + local extra_pip_packages="${10}" + local test_node_ids_file="${11}" + local test_node_ids_key="${12}" + local wheelhouse_host_dir="${13}" + local wheelhouse_packages="${14}" + local test_k_expr="${15}" + local standalone_script_scope="${16}" + local standalone_script_visualizer="${17}" + local standalone_script_runtime_group="${18}" + local warp_cache_host_dir="${19}" + local extra_uv_packages="${20}" local logs_pid="" local wait_pid="" local docker_wait_file="/tmp/.docker_exit_${container_name}" @@ -72,17 +66,8 @@ run_tests() { if [ -n "$wheelhouse_packages" ]; then echo "With wheelhouse packages: $wheelhouse_packages" fi - if [ -n "$filter_pattern" ]; then - echo "With filter pattern: $filter_pattern" - fi - if [ -n "$exclude_pattern" ]; then - echo "With exclude pattern: $exclude_pattern" - fi - if [ "$curobo_only" = "true" ]; then - echo "cuRobo-only mode enabled: running only cuRobo and SkillGen tests" - fi - if [ -n "$include_files" ]; then - echo "Include files: $include_files" + if [ -n "$job" ]; then + echo "Running test plan job: $job" fi if [ -n "$test_node_ids_file" ]; then echo "Test node IDs file: $test_node_ids_file" @@ -90,8 +75,11 @@ run_tests() { if [ -n "$test_node_ids_key" ]; then echo "Test node IDs key: $test_node_ids_key" fi - if [ -n "$shard_index" ] && [ -n "$shard_count" ]; then - echo "Shard: $shard_index of $shard_count" + # The runner takes the shard as a flag; an unsharded job passes nothing. + local shard_arg="" + if [ -n "$shard" ]; then + shard_arg="--shard $shard" + echo "Shard: $shard" fi if [ -n "$test_node_ids_file" ] || [ -n "$test_node_ids_key" ]; then @@ -121,24 +109,6 @@ run_tests() { -e GITHUB_ACTIONS=${GITHUB_ACTIONS:-} \ -e TEST_RESULT_FILE=$result_file" - if [ "$curobo_only" = "true" ]; then - docker_env_vars="$docker_env_vars -e TEST_CUROBO_ONLY=true" - echo "Setting TEST_CUROBO_ONLY=true" - fi - - if [ "$quarantined_only" = "true" ]; then - docker_env_vars="$docker_env_vars -e TEST_QUARANTINED_ONLY=true" - echo "Setting TEST_QUARANTINED_ONLY=true" - fi - - if [ -n "$include_files" ]; then - # Strip spaces so the value is safe to embed in an unquoted docker_env_vars string. - # conftest.py splits on commas and strips whitespace, so compact form works fine. - include_files_compact="${include_files// /}" - docker_env_vars="$docker_env_vars -e TEST_INCLUDE_FILES=$include_files_compact" - echo "Setting TEST_INCLUDE_FILES=$include_files_compact" - fi - if [ -n "${TEST_NODE_IDS:-}" ]; then docker_env_vars="$docker_env_vars -e TEST_NODE_IDS" echo "Setting TEST_NODE_IDS" @@ -149,35 +119,6 @@ run_tests() { echo "Setting TEST_NODE_IDS_FILE=$TEST_NODE_IDS_FILE TEST_NODE_IDS_KEY=$TEST_NODE_IDS_KEY" fi - if [ -n "$shard_index" ] && [ -n "$shard_count" ]; then - docker_env_vars="$docker_env_vars -e TEST_SHARD_INDEX=$shard_index -e TEST_SHARD_COUNT=$shard_count" - echo "Setting TEST_SHARD_INDEX=$shard_index TEST_SHARD_COUNT=$shard_count" - fi - - if [ -n "$filter_pattern" ]; then - if [[ "$filter_pattern" == "not "* ]]; then - # Handle "not " case - note the trailing space to avoid - # matching words that happen to start with "not". - filter_exclude_pattern="${filter_pattern#not }" - if [ -n "$exclude_pattern" ]; then - exclude_pattern="${exclude_pattern},${filter_exclude_pattern}" - else - exclude_pattern="$filter_exclude_pattern" - fi - else - # Handle positive pattern case - docker_env_vars="$docker_env_vars -e TEST_FILTER_PATTERN=$filter_pattern" - echo "Setting include pattern: $filter_pattern" - fi - else - echo "No filter pattern provided" - fi - - if [ -n "$exclude_pattern" ]; then - docker_env_vars="$docker_env_vars -e TEST_EXCLUDE_PATTERN=$exclude_pattern" - echo "Setting exclude pattern: $exclude_pattern" - fi - if [ -n "$extra_pip_packages" ]; then export TEST_EXTRA_PIP_PACKAGES="$extra_pip_packages" docker_env_vars="$docker_env_vars -e TEST_EXTRA_PIP_PACKAGES" @@ -193,10 +134,6 @@ run_tests() { echo "Setting per-file pytest -k expression: $test_k_expr" fi - if [ -n "$ci_marker" ]; then - docker_env_vars="$docker_env_vars -e CI_MARKER=$ci_marker" - echo "Setting CI_MARKER=$ci_marker" - fi if [ -n "$standalone_script_scope" ]; then docker_env_vars="$docker_env_vars \ @@ -373,8 +310,13 @@ run_tests() { bash /with-python-package-retries.sh \"\${uv_executable}\" pip install --python \"\${isaac_python}\" --target \"\${isaac_uv_overlay}\" --no-deps \${TEST_EXTRA_UV_PACKAGES} export PYTHONPATH=\"\${isaac_uv_overlay}\${PYTHONPATH:+:\${PYTHONPATH}}\" fi - echo 'Starting pytest with path: $test_path' - ./isaaclab.sh -p -m pytest --ignore=tools/conftest.py $test_path $pytest_options -v --junitxml=tests/$result_file + if [ -n '$job' ]; then + echo 'Running test plan job: $job' + ./isaaclab.sh -p tools/run_tests.py --job '$job' $shard_arg + else + echo 'Starting pytest with path: $test_path' + ./isaaclab.sh -p -m pytest $test_path $pytest_options -v --junitxml=tests/$result_file + fi " # Stream container logs in background. diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml index 3e992dcef3cb..87482cf53939 100644 --- a/.github/workflows/arm-ci.yml +++ b/.github/workflows/arm-ci.yml @@ -184,8 +184,7 @@ jobs: container-name: isaac-lab-arm-ci-${{ github.run_id }}-${{ github.run_attempt }} image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}" - test-k-expr: not ovphysx - ci-marker: arm_ci + job: arm-ci volume-mount-source: ${{ github.workspace }} - name: Run arm_ci OVPhysX marker tests @@ -196,8 +195,7 @@ jobs: container-name: isaac-lab-arm-ci-ovphysx-${{ github.run_id }}-${{ github.run_attempt }} image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ${{ steps.ov_pins.outputs.ovphysx }}" - test-k-expr: ovphysx - ci-marker: arm_ci + job: arm-ci-ovphysx volume-mount-source: ${{ github.workspace }} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2ff42e847a07..210fface7cd5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -27,12 +27,9 @@ # if: false # TEMP: Disabled for debugging # # 2. RUN ONLY SPECIFIC TEST FILES (within a job): -# Add `include-files:` parameter to run-package-tests action -# Example: -# - uses: ./.github/actions/run-package-tests -# with: -# ... -# include-files: "test_rigid_object_collection.py" # Comma-separated +# Edit the job's entry in tools/test_plan.toml -- `files = [...]` limits it to those +# basenames -- then run `python tools/generate_workflows.py` if the job is generated. +# Locally the same selection is `python tools/run_tests.py --job `. # # 3. SKIP CONCURRENCY WAIT (run immediately without waiting for other runs): # Comment out the concurrency block: @@ -217,7 +214,9 @@ jobs: #endregion #region test jobs - test-isaaclab-tasks: + # >>> generated from tools/test_plan.toml -- edit the plan, then run tools/generate_workflows.py + + test-isaaclab-tasks-1: name: isaaclab_tasks [1/3] runs-on: [self-hosted, gpu] timeout-minutes: 180 @@ -236,11 +235,9 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" - exclude-pattern: "test_rendering_,test_video_recording.py" + job: isaaclab-tasks + shard: "0" extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - shard-index: "0" - shard-count: "3" warp-cache: restore container-name: isaac-lab-tasks-1-test @@ -263,11 +260,9 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" - exclude-pattern: "test_rendering_,test_video_recording.py" + job: isaaclab-tasks + shard: "1" extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - shard-index: "1" - shard-count: "3" warp-cache: restore container-name: isaac-lab-tasks-2-test @@ -290,15 +285,13 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" - exclude-pattern: "test_rendering_,test_video_recording.py" + job: isaaclab-tasks + shard: "2" extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - shard-index: "2" - shard-count: "3" warp-cache: restore container-name: isaac-lab-tasks-3-test - test-isaaclab-core: + test-isaaclab-core-1: name: isaaclab (core) [1/3] runs-on: [self-hosted, gpu] timeout-minutes: 180 @@ -316,9 +309,8 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "not isaaclab_" - shard-index: "0" - shard-count: "3" + job: isaaclab-core + shard: "0" warp-cache: restore container-name: isaac-lab-core-1-test @@ -340,9 +332,8 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "not isaaclab_" - shard-index: "1" - shard-count: "3" + job: isaaclab-core + shard: "1" warp-cache: restore container-name: isaac-lab-core-2-test @@ -364,18 +355,13 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "not isaaclab_" - shard-index: "2" - shard-count: "3" + job: isaaclab-core + shard: "2" warp-cache: restore container-name: isaac-lab-core-3-test - # Kit and non-Kit are written out rather than expressed as a matrix: these job - # names are required status checks, and a skipped matrix job publishes its - # check run with the matrix expression unexpanded, so the required context - # never appears and branch protection blocks the pull request. - test-standalone-demos-kit: - name: standalone demos (headless, Kit) + test-isaaclab-rl: + name: isaaclab_rl runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -392,18 +378,12 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - # deformables.py tetrahedralizes its volume meshes at startup - extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - include-files: "test_standalone_scripts.py" - standalone-script-scope: "demos" - standalone-script-visualizer: "none" - standalone-script-runtime-group: kit - result-file: "test-standalone-demos-kit-report.xml" - container-name: isaac-lab-standalone-demos-kit-test - omni-github-test-type: standalone-demo + job: isaaclab-rl + extra-pip-packages: "leapp" + container-name: isaac-lab-rl-test - test-standalone-demos-non-kit: - name: standalone demos (headless, non-Kit) + test-isaaclab-mimic: + name: isaaclab_mimic runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -415,28 +395,16 @@ jobs: with: fetch-depth: 1 lfs: true - - name: Resolve OVPhysX runtime pin from pyproject - uses: ./.github/actions/resolve-ov-pins - id: ov_pins - uses: ./.github/actions/run-package-tests with: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - # deformables.py tetrahedralizes its volume meshes at startup - extra-pip-packages: >- - pytetwild[all]>=0.3.0,<0.4 - ${{ steps.ov_pins.outputs.ovphysx }} - include-files: "test_standalone_scripts.py" - standalone-script-scope: "demos" - standalone-script-visualizer: "none" - standalone-script-runtime-group: non-kit - result-file: "test-standalone-demos-non-kit-report.xml" - container-name: isaac-lab-standalone-demos-non-kit-test - omni-github-test-type: standalone-demo + job: isaaclab-mimic + container-name: isaac-lab-mimic-test - test-isaaclab-rl: - name: isaaclab_rl + test-isaaclab-contrib: + name: isaaclab_contrib runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -453,12 +421,12 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_rl" - extra-pip-packages: "leapp" - container-name: isaac-lab-rl-test + job: isaaclab-contrib + extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" + container-name: isaac-lab-contrib-test - test-isaaclab-mimic: - name: isaaclab_mimic + test-isaaclab-teleop: + name: isaaclab_teleop runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -475,11 +443,11 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_mimic" - container-name: isaac-lab-mimic-test + job: isaaclab-teleop + container-name: isaac-lab-teleop-test - test-isaaclab-contrib: - name: isaaclab_contrib + test-isaaclab-visualizers: + name: isaaclab_visualizers runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -496,12 +464,11 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_contrib" - extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - container-name: isaac-lab-contrib-test + job: isaaclab-visualizers + container-name: isaac-lab-visualizers-test - test-isaaclab-teleop: - name: isaaclab_teleop + test-isaaclab-assets: + name: isaaclab_assets runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -518,11 +485,11 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_teleop" - container-name: isaac-lab-teleop-test + job: isaaclab-assets + container-name: isaac-lab-assets-test - test-isaaclab-visualizers: - name: isaaclab_visualizers + test-isaaclab-experimental: + name: isaaclab_experimental runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -539,11 +506,11 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_visualizers" - container-name: isaac-lab-visualizers-test + job: isaaclab-experimental + container-name: isaac-lab-experimental-test - test-isaaclab-assets: - name: isaaclab_assets + test-isaaclab-newton: + name: isaaclab_newton runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -560,11 +527,12 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_assets" - container-name: isaac-lab-assets-test + job: isaaclab-newton + warp-cache: restore + container-name: isaac-lab-newton-test - test-isaaclab-experimental: - name: isaaclab_experimental + test-isaaclab-physx: + name: isaaclab_physx runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -581,11 +549,18 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_experimental" - container-name: isaac-lab-experimental-test + job: isaaclab-physx + extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" + container-name: isaac-lab-physx-test - test-isaaclab-newton: - name: isaaclab_newton + # <<< end generated jobs + + # Kit and non-Kit are written out rather than expressed as a matrix: these job + # names are required status checks, and a skipped matrix job publishes its + # check run with the matrix expression unexpanded, so the required context + # never appears and branch protection blocks the pull request. + test-standalone-demos-kit: + name: standalone demos (headless, Kit) runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -602,12 +577,18 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_newton" - warp-cache: restore - container-name: isaac-lab-newton-test + job: standalone-demos-kit + # deformables.py tetrahedralizes its volume meshes at startup + extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" + standalone-script-scope: "demos" + standalone-script-visualizer: "none" + standalone-script-runtime-group: kit + result-file: "test-standalone-demos-kit-report.xml" + container-name: isaac-lab-standalone-demos-kit-test + omni-github-test-type: standalone-demo - test-isaaclab-physx: - name: isaaclab_physx + test-standalone-demos-non-kit: + name: standalone demos (headless, non-Kit) runs-on: [self-hosted, gpu] timeout-minutes: 180 needs: [build, config] @@ -619,14 +600,25 @@ jobs: with: fetch-depth: 1 lfs: true + - name: Resolve OVPhysX runtime pin from pyproject + uses: ./.github/actions/resolve-ov-pins + id: ov_pins - uses: ./.github/actions/run-package-tests with: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_physx" - extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - container-name: isaac-lab-physx-test + job: standalone-demos-non-kit + # deformables.py tetrahedralizes its volume meshes at startup + extra-pip-packages: >- + pytetwild[all]>=0.3.0,<0.4 + ${{ steps.ov_pins.outputs.ovphysx }} + standalone-script-scope: "demos" + standalone-script-visualizer: "none" + standalone-script-runtime-group: non-kit + result-file: "test-standalone-demos-non-kit-report.xml" + container-name: isaac-lab-standalone-demos-non-kit-test + omni-github-test-type: standalone-demo test-isaaclab-ov: name: isaaclab_ov @@ -651,7 +643,7 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_ov" + job: isaaclab-ov # Volume deformable tests tetrahedralize their meshes at startup. extra-pip-packages: pytetwild[all]>=0.3.0,<0.4 ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && steps.ov_pins.outputs.ovrtx || format('{0} {1}', steps.ov_pins.outputs.ovrtx, steps.ov_pins.outputs.ovphysx) }} wheelhouse-image: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && needs.config.outputs.ovphysx_wheelhouse_image || '' }} @@ -703,9 +695,9 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }}-curobo isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + job: curobo dockerfile-path: docker/Dockerfile.curobo cache-tag: cache-curobo - include-files: "test_curobo_planner_franka.py,test_curobo_planner_cube_stack.py,test_pink_ik.py" container-name: isaac-lab-curobo-test # Folded from the former standalone verify-curobo-non-root job: reuses @@ -753,9 +745,9 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }}-curobo isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + job: contrib-environments dockerfile-path: docker/Dockerfile.curobo cache-tag: cache-curobo - include-files: "test_generate_dataset_skillgen.py,test_contrib_environments.py" container-name: isaac-lab-contrib-environments-test test-record-video: @@ -777,8 +769,7 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" - include-files: "test_video_recording.py" + job: record-video extra-uv-packages: "moviepy>=1.0.3,<2.0.0.dev0 decorator<5" container-name: isaac-lab-record-video-test omni-github-test-type: video-e2e @@ -800,17 +791,8 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" + job: rendering-correctness extra-pip-packages: "pytetwild[all]>=0.3.0,<0.4" - include-files: >- - test_rendering_cartpole.py, - test_rendering_lift_kuka_hetero.py, - test_rendering_lift_kuka_homo.py, - test_rendering_franka_cloth.py, - test_rendering_franka_soft.py, - test_rendering_franka_cable.py, - test_rendering_registered_tasks.py, - test_rendering_shadow_hand.py test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }} test-node-ids-key: ${{ github.event_name == 'push' && 'rendering-correctness' || '' }} warp-cache: restore @@ -844,21 +826,12 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" + job: rendering-correctness-kitless-legacy extra-pip-packages: >- pytetwild[all]>=0.3.0,<0.4 ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && steps.ov_pins.outputs.ovrtx || format('{0} {1}', steps.ov_pins.outputs.ovrtx, steps.ov_pins.outputs.ovphysx) }} wheelhouse-image: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && needs.config.outputs.ovphysx_wheelhouse_image || '' }} wheelhouse-packages: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && 'ovphysx' || '' }} - include-files: >- - test_rendering_cartpole_kitless.py, - test_rendering_lift_kuka_hetero_kitless.py, - test_rendering_lift_kuka_homo_kitless.py, - test_rendering_franka_cloth_kitless.py, - test_rendering_franka_soft_kitless.py, - test_rendering_franka_cable_kitless.py, - test_rendering_shadow_hand_kitless.py - test-k-expr: legacy test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }} test-node-ids-key: ${{ github.event_name == 'push' && 'rendering-correctness-kitless-legacy' || '' }} warp-cache: restore @@ -890,21 +863,12 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" + job: rendering-correctness-kitless-ovstage extra-pip-packages: >- pytetwild[all]>=0.3.0,<0.4 ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && steps.ov_pins.outputs.ovrtx || format('{0} {1}', steps.ov_pins.outputs.ovrtx, steps.ov_pins.outputs.ovphysx) }} wheelhouse-image: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && needs.config.outputs.ovphysx_wheelhouse_image || '' }} wheelhouse-packages: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && 'ovphysx' || '' }} - include-files: >- - test_rendering_cartpole_kitless.py, - test_rendering_lift_kuka_hetero_kitless.py, - test_rendering_lift_kuka_homo_kitless.py, - test_rendering_franka_cloth_kitless.py, - test_rendering_franka_soft_kitless.py, - test_rendering_franka_cable_kitless.py, - test_rendering_shadow_hand_kitless.py - test-k-expr: ovstage test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }} test-node-ids-key: ${{ github.event_name == 'push' && 'rendering-correctness-kitless-ovstage' || '' }} container-name: "isaac-lab-rendering-correctness-kitless-ovstage-test" @@ -948,17 +912,13 @@ jobs: image-tag: ${{ needs.config.outputs.ci_image_tag }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - filter-pattern: "isaaclab_tasks" + job: warp-cache-warm # Warm every supported rigid core environment on Newton. MJWarp # specializes kernels per model config, so a small representative task # set leaves most of the cache to be compiled again by each PR test # shard. The multi-agent suite supplies Handover. - include-files: >- - test_environments_newton.py, - test_multi_agent_environments.py # Deformable tasks need optional dependencies that are not installed in # this image. - test-k-expr: "test_environments and not (Soft or Cloth or Cable)" warp-cache: ${{ env.PUBLISHES_WARP_CACHE == 'true' && 'save' || 'restore' }} container-name: isaac-lab-warp-cache-warm omni-github-test-type: warp-cache-warm diff --git a/.github/workflows/daily-compatibility.yml b/.github/workflows/daily-compatibility.yml index eaf05e4d3186..5b24d754f61f 100644 --- a/.github/workflows/daily-compatibility.yml +++ b/.github/workflows/daily-compatibility.yml @@ -123,6 +123,7 @@ jobs: image-tag: ${{ env.DOCKER_IMAGE_TAG }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ matrix.isaacsim_version }} + job: isaaclab-tasks-compat cache-from: type=gha cache-to: type=gha,mode=max @@ -134,7 +135,6 @@ jobs: container-name: "isaac-lab-tasks-compat-test-$$" image-tag: ${{ env.DOCKER_IMAGE_TAG }} pytest-options: "" - filter-pattern: "isaaclab_tasks" extra-pip-packages: ${{ format('{0} {1}', steps.ov_pins.outputs.ovphysx, steps.ov_pins.outputs.ovrtx) }} - name: Copy All Test Results from IsaacLab Tasks Container @@ -183,6 +183,7 @@ jobs: image-tag: ${{ env.DOCKER_IMAGE_TAG }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ matrix.isaacsim_version }} + job: general-compat cache-from: type=gha cache-to: type=gha,mode=max @@ -194,7 +195,6 @@ jobs: container-name: "isaac-lab-general-compat-test-$$" image-tag: ${{ env.DOCKER_IMAGE_TAG }} pytest-options: "" - filter-pattern: "not isaaclab_tasks" - name: Copy All Test Results from General Tests Container run: | diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index 63c03a73779a..a4787a54116c 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -114,7 +114,7 @@ jobs: # # Within a discovered file, tests that are NOT parametrized over the # ``device`` argument are deselected at collection time by the - # ``mgpu_shard_select`` plugin (injected per shard by ``tools/conftest.py``): + # ``mgpu_shard_select`` plugin (injected per shard by ``tools/run_tests.py``): # single-GPU CI already covers them on ``cuda:0`` and re-running on every # non-default shard adds wall-time without surfacing any new failure mode. id: discover diff --git a/.github/workflows/tools-tests.yml b/.github/workflows/tools-tests.yml index 70977f92dc39..f28c076ff277 100644 --- a/.github/workflows/tools-tests.yml +++ b/.github/workflows/tools-tests.yml @@ -60,7 +60,6 @@ jobs: # Files are listed explicitly rather than collected from tools/: test_settings.py is a # configuration module, not a test, and tools/test/ needs the full Isaac Lab install. # - # --noconftest keeps tools/conftest.py out of the session. That file is the CI test - # orchestrator - its pytest_sessionstart scans source/ and scripts/ and runs the whole - # suite, so loading it here would ignore the files named below. - run: python3 -m pytest tools/test_crash_journal.py tools/test_device_split.py -v --noconftest + # The runner is tools/run_tests.py, an ordinary script, so an ordinary pytest run + # over these files no longer has to defend itself against a conftest that hijacks it. + run: python3 -m pytest tools/test_crash_journal.py tools/test_device_split.py -v diff --git a/conftest.py b/conftest.py index 823cb9f35b07..75c243507a5f 100644 --- a/conftest.py +++ b/conftest.py @@ -11,7 +11,7 @@ Also maintains a crash journal (see :data:`JOURNAL_ENV_VAR`). pytest writes its JUnit XML once, at the end of the session, so a run killed before then - a Kit shutdown crash, an OOM kill, a hard timeout - loses every verdict it had already printed. The journal records collection, -per-test start/finish, and per-test outcomes as they happen, letting ``tools/conftest.py`` +per-test start/finish, and per-test outcomes as they happen, letting ``tools/run_tests.py`` rebuild a real report instead of a single synthetic ``test_execution`` entry. Level markers (``unit`` / ``integration`` / ``benchmark``) are applied per file via a module-level ``pytestmark`` @@ -26,6 +26,7 @@ from __future__ import annotations +import importlib.util import json import os @@ -37,7 +38,20 @@ else: wp.config.enable_backward = False -pytest_plugins = ["tools.ovrtx_log", "tools.hang_dump", "isaaclab.test.kit"] +pytest_plugins = ["tools.ovrtx_log", "tools.hang_dump"] + +# The Kit launch plugin is the only entry here that needs Isaac Lab itself. Lanes that test the +# repository's own tooling install pytest and nothing else, and none of their tests declare a +# launch marker, so a missing package there is not an error. A lane that does need Kit has Isaac +# Lab installed by definition -- its tests import it -- so this cannot quietly skip the launch. +# The exact module is probed rather than the top-level package: an Isaac Lab old enough to +# predate the plugin would otherwise pass the check and fail on import. +try: + _has_kit_plugin = importlib.util.find_spec("isaaclab.test.kit") is not None +except (ImportError, ValueError): + _has_kit_plugin = False +if _has_kit_plugin: + pytest_plugins.append("isaaclab.test.kit") JOURNAL_ENV_VAR = "ISAACLAB_TEST_JOURNAL" """Environment variable naming the crash-journal file. Unset (the default) disables journaling.""" @@ -97,7 +111,7 @@ def pytest_collection_finish(session): Journaling from :func:`pytest_collection_modifyitems` would record tests that are about to be dropped: pytest's own mark plugin applies ``-k`` / ``-m`` deselection from a ``trylast`` hook, - which runs after this file's. Since ``tools/conftest.py`` splits a run into passes selected by + which runs after this file's. Since ``tools/run_tests.py`` splits a run into passes selected by marker and device, a rebuilt crash report would then emit every other pass's tests as "not run" skips, inflating the counts and duplicating node IDs whose real verdicts came from the sibling pass. ``session.items`` is post-deselection, so it holds exactly the tests this pass runs. diff --git a/docs/source/refs/contributing.rst b/docs/source/refs/contributing.rst index eec05829007a..9907daeb656f 100644 --- a/docs/source/refs/contributing.rst +++ b/docs/source/refs/contributing.rst @@ -764,6 +764,24 @@ Please make sure that you add tests for your changes. isaaclab.bat -p -m pytest source/isaaclab/test/deps/test_torch.py::test_array_slicing +Running what CI runs +^^^^^^^^^^^^^^^^^^^^ + +``isaaclab -t`` runs the whole suite the same way CI does. What each CI lane covers is +described in ``tools/test_plan.toml``, and any lane can be run on its own: + +.. code-block:: bash + + isaaclab -t --list-jobs # the CI lanes, and how many files each covers + isaaclab -t --job isaaclab-core --shard 0 # one lane, exactly as CI runs it + isaaclab -t source/isaaclab/test/sim # an ad-hoc directory + isaaclab -t --job isaaclab-rl --list-files # what a lane would run, without running it + +The plan is the single source of truth: ``tools/generate_workflows.py`` renders the uniform +workflow jobs from it, and a test fails if the checked-in YAML has drifted. To change what a +lane covers, edit the plan rather than the workflow file. + + Tests that need Isaac Sim ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst b/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst index 5e83634f2e70..13fc0e4b413a 100644 --- a/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst +++ b/source/isaaclab/changelog.d/mataylor-kit-test-markers.rst @@ -15,11 +15,15 @@ Added Changed ^^^^^^^ -* Changed the test runner to group same-marker test files into a single pytest invocation - rather than giving every file its own process, so Kit startup is paid once per group. Only - files carrying the new markers are grouped; every other file keeps a process of its own, and - a file a dead group never reached is re-run individually. Set ``ISAACLAB_TEST_BATCH_KIT=0`` - to turn the grouping off. +* Changed ``isaaclab --test`` to drive the new ``tools/run_tests.py``. With no arguments it + runs the whole suite; ``--job `` runs a single CI lane locally, ``--list-jobs`` lists + them, and directories run an ad-hoc selection. It previously ran ``pytest tools``, which went + through the CI orchestrator and silently dropped any pytest arguments passed after it. +* Changed the runner to group same-marker test files into a single pytest invocation rather + than giving every file its own process, so Kit startup is paid once per group. Only files + carrying the new markers are grouped; every other file keeps a process of its own, and a file + a dead group never reached is re-run individually. Set ``ISAACLAB_TEST_BATCH_KIT=0`` to turn + the grouping off. Fixed ^^^^^ diff --git a/source/isaaclab/isaaclab/cli/__init__.py b/source/isaaclab/isaaclab/cli/__init__.py index 3707deec9f56..b5daa1f41319 100644 --- a/source/isaaclab/isaaclab/cli/__init__.py +++ b/source/isaaclab/isaaclab/cli/__init__.py @@ -235,7 +235,10 @@ def cli() -> None: "-t", "--test", nargs=argparse.REMAINDER, - help="Run all python pytest tests.", + help=( + "Run the tests. No arguments runs the whole suite; '--job ' runs one CI lane," + " '--list-jobs' lists them, and paths run an ad-hoc selection." + ), ) parser.add_argument( "-o", diff --git a/source/isaaclab/isaaclab/cli/commands/misc.py b/source/isaaclab/isaaclab/cli/commands/misc.py index 44e684413c14..fb233586542e 100644 --- a/source/isaaclab/isaaclab/cli/commands/misc.py +++ b/source/isaaclab/isaaclab/cli/commands/misc.py @@ -56,12 +56,19 @@ def command_new(new_args: list[str]) -> None: def command_test(test_args: list[str]) -> None: - """Run pytest for Isaac Lab tests (-t). + """Run Isaac Lab's tests (-t). + + With no arguments this runs the whole suite, the same way CI does. Pass ``--job `` to + run one CI lane, ``--list-jobs`` to see them, or one or more directories to run an ad-hoc + selection. Every argument goes to ``tools/run_tests.py``; see ``isaaclab -t --help``. Args: - test_args: Additional pytest arguments. + test_args: Arguments for ``tools/run_tests.py``. """ - run_python_command("-m", ["pytest", str(ISAACLAB_ROOT / "tools")] + test_args) + runner = ISAACLAB_ROOT / "tools" / "run_tests.py" + # Bare `isaaclab -t` used to mean "run everything"; keep that, since the runner itself + # refuses an empty selection rather than guessing. + run_python_command(runner, test_args or ["--all"]) def command_vscode_settings() -> None: diff --git a/source/isaaclab/test/cli/test_test_orchestrator_result_handling.py b/source/isaaclab/test/cli/test_test_orchestrator_result_handling.py index 64cd08d9c316..08822b68885c 100644 --- a/source/isaaclab/test/cli/test_test_orchestrator_result_handling.py +++ b/source/isaaclab/test/cli/test_test_orchestrator_result_handling.py @@ -32,8 +32,8 @@ def _load_orchestrator_module() -> ModuleType: - """Load ``tools/conftest.py`` without registering it as a pytest plugin.""" - module_path = TOOLS_DIR / "conftest.py" + """Load ``tools/run_tests.py`` under a private name, leaving any real import untouched.""" + module_path = TOOLS_DIR / "run_tests.py" module_name = "isaaclab_test_orchestrator" tools_dir = str(module_path.parent) if tools_dir not in sys.path: diff --git a/source/isaaclab_tasks/changelog.d/mataylor-kit-test-markers.skip b/source/isaaclab_tasks/changelog.d/mataylor-kit-test-markers.skip new file mode 100644 index 000000000000..fc49ef0e3ac0 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/mataylor-kit-test-markers.skip @@ -0,0 +1 @@ +Test-only: updated a docstring reference to the renamed test runner. diff --git a/source/isaaclab_tasks/test/benchmarking/conftest.py b/source/isaaclab_tasks/test/benchmarking/conftest.py index 51096c6de144..6cce81604d4d 100644 --- a/source/isaaclab_tasks/test/benchmarking/conftest.py +++ b/source/isaaclab_tasks/test/benchmarking/conftest.py @@ -89,7 +89,7 @@ def kpi_store(): # Shard parametrized test items across parallel CI jobs. -# Reads the same TEST_SHARD_INDEX / TEST_SHARD_COUNT env vars used by tools/conftest.py +# Reads the same TEST_SHARD_INDEX / TEST_SHARD_COUNT env vars used by tools/run_tests.py # for file-level sharding, but applies them at the test-item level so a single # parametrized file can be split across multiple runners. # This is a pytest hook — pytest calls it automatically during test collection. diff --git a/tools/_device_split.py b/tools/_device_split.py index e8bad4069133..94ac2663f8e1 100644 --- a/tools/_device_split.py +++ b/tools/_device_split.py @@ -9,7 +9,7 @@ scope must be re-invoked once per device (CPU and GPU) in separate processes to work around process-global device locks such as ``ovphysx<=0.3.7`` gap G5. The :func:`is_device_split_file` predicate lets the per-file CI runner in -``tools/conftest.py`` detect this without importing the test module. +``tools/run_tests.py`` detect this without importing the test module. """ from __future__ import annotations @@ -29,7 +29,7 @@ a future test needs that, expand the parsing rule. """ -# Per-pass pytest ``-k`` selectors used by ``tools/conftest.py`` when a file +# Per-pass pytest ``-k`` selectors used by ``tools/run_tests.py`` when a file # declares the ``device_split`` marker. Each entry is ``(suffix, k_expr)``: # - ``suffix`` is appended to the JUnit report filename to keep both passes' XML. # - ``k_expr`` is the ``-k`` keyword expression. ``"cpu or not cuda"`` keeps diff --git a/tools/_kit_batching.py b/tools/_kit_batching.py index 839121088aee..7e9784a2395d 100644 --- a/tools/_kit_batching.py +++ b/tools/_kit_batching.py @@ -20,7 +20,7 @@ Apart from reading :data:`BATCH_ENV_VAR` and :data:`BATCH_SIZE_ENV_VAR`, this module is deliberately free of process machinery: the grouping and the report demultiplexing are pure functions over paths and strings, so they can be exercised on any platform, unlike the -POSIX-only subprocess handling in ``tools/conftest.py``. +POSIX-only subprocess handling in ``tools/run_tests.py``. """ from __future__ import annotations diff --git a/tools/changelog/pyproject.toml b/tools/changelog/pyproject.toml index b43578b10039..04a1ef7865f6 100644 --- a/tools/changelog/pyproject.toml +++ b/tools/changelog/pyproject.toml @@ -3,7 +3,7 @@ # # 1. ``pythonpath = ["."]`` adds ``tools/changelog/`` to ``sys.path``, # making ``import cli`` work from the test files without any shim. -# 2. ``tools/conftest.py`` (a session-takeover hook for the IsaacLab +# 2. ``tools/run_tests.py`` (a session-takeover hook for the IsaacLab # source/ test suite) sits *above* rootdir and is therefore not # loaded — no ``--noconftest`` flag required. # diff --git a/tools/crash_journal.py b/tools/crash_journal.py index 8936a2c24413..ed9696b1e918 100644 --- a/tools/crash_journal.py +++ b/tools/crash_journal.py @@ -7,7 +7,7 @@ pytest writes its JUnit XML once, in ``pytest_sessionfinish``. A run killed before that point — a Kit shutdown crash, an OOM kill, a hard timeout — leaves no report at all, even though every -test verdict was already printed to stdout. ``tools/conftest.py`` used to answer that by +test verdict was already printed to stdout. ``tools/run_tests.py`` used to answer that by synthesizing a single ``test_execution`` error, which discarded which tests passed, which failed, and which one was in flight when the process died. diff --git a/tools/generate_workflows.py b/tools/generate_workflows.py new file mode 100644 index 000000000000..0f7c1acbcd8e --- /dev/null +++ b/tools/generate_workflows.py @@ -0,0 +1,181 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Render the uniform test lanes in the workflow files from ``tools/test_plan.toml``. + +Most package lanes are the same twenty-odd lines of YAML differing only in a name, a container +and a package list, so they were maintained by copy-paste: adding a package meant adding a +block, and a change to the shape had to be applied to every copy by hand. Those lanes are +generated here, between sentinel comments, and :mod:`tools.test.test_test_plan` fails if the +checked-in YAML has drifted from what this produces. + +Only jobs marked ``generate = true`` are rendered. The lanes with bespoke setup -- extra image +builds, wheelhouse expressions, artifact uploads -- stay hand-written; they are still required +to name a job the plan defines, which is what keeps the two from diverging. + +Usage:: + + python tools/generate_workflows.py # rewrite the generated blocks + python tools/generate_workflows.py --check # report drift, change nothing +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import testplan +from testplan import REPO_ROOT, Job + +WORKFLOW_DIR = REPO_ROOT / ".github" / "workflows" + +BEGIN = " # >>> generated from tools/test_plan.toml -- edit the plan, then run tools/generate_workflows.py" +END = " # <<< end generated jobs" + + +def _job_id(job: Job, shard: int | None) -> str: + """Workflow job id; sharded jobs are numbered from 1 to read like their display name.""" + return f"test-{job.name}" if shard is None else f"test-{job.name}-{shard + 1}" + + +def _title(job: Job, shard: int | None) -> str: + """Display name; a sharded job carries its position.""" + return job.title if shard is None else f"{job.title} [{shard + 1}/{job.shards}]" + + +def _container(job: Job, shard: int | None) -> str: + """Container name, kept distinct per shard so two shards cannot collide on one runner.""" + base = job.container_name or f"isaac-lab-{job.name}" + if shard is None: + return base + return base[: -len("-test")] + f"-{shard + 1}-test" if base.endswith("-test") else f"{base}-{shard + 1}" + + +def render_job(job: Job, shard: int | None) -> str: + """Render one workflow job block. + + Args: + job: Job from the plan. + shard: Shard index, or None for an unsharded job. + + Returns: + The YAML block, ending in a blank line. + """ + lines = [ + f" {_job_id(job, shard)}:", + f" name: {_title(job, shard)}", + " runs-on: [self-hosted, gpu]", + f" timeout-minutes: {job.timeout_minutes}", + ] + if job.continue_on_error: + lines.append(" continue-on-error: true") + lines += [ + " needs: [build, config]", + " if: >-", + " github.event_name != 'push' &&", + " needs.build.result == 'success'", + " steps:", + " - uses: actions/checkout@v6", + " with:", + " fetch-depth: 1", + " lfs: true", + " - uses: ./.github/actions/run-package-tests", + " with:", + " image-tag: ${{ needs.config.outputs.ci_image_tag }}", + " isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}", + " isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}", + f" job: {job.name}", + ] + if shard is not None: + lines.append(f' shard: "{shard}"') + if job.extra_pip_packages: + lines.append(f' extra-pip-packages: "{job.extra_pip_packages}"') + if job.warp_cache: + lines.append(f" warp-cache: {job.warp_cache}") + lines.append(f" container-name: {_container(job, shard)}") + return "\n".join(lines) + "\n" + + +def render(workflow: str) -> str: + """Render every generated job for one workflow file, in plan order.""" + blocks = [] + for job in testplan.load_plan(): + if not job.generate or job.workflow != workflow: + continue + shards = range(job.shards) if job.shards > 1 else [None] + blocks.extend(render_job(job, shard) for shard in shards) + return "\n".join(blocks) + + +def _splice(text: str, body: str) -> str: + """Replace the text between the sentinels, keeping everything outside untouched. + + Raises: + ValueError: If the sentinels are missing or out of order, which would otherwise let + the generator silently write nothing. + """ + start, end = text.find(BEGIN), text.find(END) + if start == -1 or end == -1 or end < start: + raise ValueError(f"generated-block sentinels not found in order; expected\n{BEGIN}\n...\n{END}") + return text[: start + len(BEGIN)] + "\n\n" + body + "\n" + text[end:] + + +def _workflow_path(workflow: str) -> Path: + for suffix in (".yaml", ".yml"): + candidate = WORKFLOW_DIR / f"{workflow}{suffix}" + if candidate.exists(): + return candidate + raise FileNotFoundError(f"no workflow file for {workflow!r}") + + +def _generated_workflows() -> list[str]: + return sorted({job.workflow for job in testplan.load_plan() if job.generate}) + + +def write() -> list[str]: + """Rewrite the generated blocks. Returns the workflow files that changed.""" + changed = [] + for workflow in _generated_workflows(): + path = _workflow_path(workflow) + text = path.read_text(encoding="utf-8") + updated = _splice(text, render(workflow)) + if updated != text: + path.write_text(updated, encoding="utf-8") + changed.append(path.name) + return changed + + +def check() -> list[str]: + """Return the workflow files whose generated blocks are out of date.""" + stale = [] + for workflow in _generated_workflows(): + path = _workflow_path(workflow) + text = path.read_text(encoding="utf-8") + if _splice(text, render(workflow)) != text: + stale.append(path.name) + return stale + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--check", action="store_true", help="report drift instead of rewriting") + args = parser.parse_args(argv) + + if args.check: + stale = check() + for name in stale: + print(f"out of date: {name}") + return 1 if stale else 0 + + changed = write() + for name in changed: + print(f"updated: {name}") + if not changed: + print("already up to date") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/hang_dump.py b/tools/hang_dump.py index ad519be57c59..3468575b9455 100644 --- a/tools/hang_dump.py +++ b/tools/hang_dump.py @@ -6,7 +6,7 @@ """On-demand stack dump for a test process the CI runner believes is hung. A test that crashes reports a traceback, because ``PYTHONFAULTHANDLER=1`` (set per test file in -``tools/conftest.py``) installs ``faulthandler`` for ``SIGSEGV`` and friends. A test that *hangs* reported +``tools/run_tests.py``) installs ``faulthandler`` for ``SIGSEGV`` and friends. A test that *hangs* reported nothing: the runner detects the hang and kills the process group with ``SIGKILL``, which cannot be caught, so no handler ever ran. This module closes that gap by giving the runner a signal to ask for a stack first. @@ -41,7 +41,7 @@ DUMP_SIGNAL = getattr(signal, "SIGUSR1", None) """Signal the CI runner sends to ask a hung test process for a stack dump. -``None`` off POSIX. ``tools/conftest.py`` reads this so the sender and the receiver cannot disagree. +``None`` off POSIX. ``tools/run_tests.py`` reads this so the sender and the receiver cannot disagree. """ DUMP_PATH_ENV_VAR = "ISAACLAB_HANG_DUMP" diff --git a/tools/ovrtx_log.py b/tools/ovrtx_log.py index 32fe6e3a0422..aaf36a819fa0 100644 --- a/tools/ovrtx_log.py +++ b/tools/ovrtx_log.py @@ -14,7 +14,7 @@ * loaded as a pytest plugin by the repo-root ``conftest.py``, it claims the log for the test process and replays each test's share of it into pytest's capture, where a failing test's report picks it up; -* imported by ``tools/conftest.py``, it quotes a bounded tail in the crash, hang, or timeout report of a +* imported by ``tools/run_tests.py``, it quotes a bounded tail in the crash, hang, or timeout report of a process that died before it could replay anything. Both readers are needed: the replay attributes output to the test that produced it and stays out of the job @@ -25,7 +25,7 @@ the log alone. So when :data:`LOG_DIR_ENV_VAR` names a directory, everything a test left in the renderer's own directory -- its own share of the log, uncapped, and any dump beside it -- is additionally saved there for CI to upload as an artifact, which is what a diagnosis reads when the quoted tail is not enough. The -test that a crash, hang, or timeout killed never reaches the fixture that saves, so ``tools/conftest.py`` +test that a crash, hang, or timeout killed never reaches the fixture that saves, so ``tools/run_tests.py`` saves what that process left behind on its behalf -- the one test in the run whose output would otherwise be missing from the artifact is the one the artifact exists for. """ @@ -57,7 +57,7 @@ LOG_DIR_ENV_VAR = "ISAACLAB_OVRTX_LOG_DIR" """Environment variable naming the directory each test's renderer output is saved under, uncapped. -Set per pytest invocation by ``tools/conftest.py``, to a directory under ``tests/`` that CI collects as a +Set per pytest invocation by ``tools/run_tests.py``, to a directory under ``tests/`` that CI collects as a job artifact. Unset by default, which leaves a local run writing nothing beyond the replay. """ @@ -172,7 +172,7 @@ def pytest_configure(config): Whatever is at the path belongs to a session that has ended, so it is dropped rather than reasoned about: the byte offsets recorded per test below are only meaningful against this session's own output. The log is left in place at session end instead, since a crashed run is diagnosed by reading it after - the process is gone; ``tools/conftest.py`` clears it before starting the next one. + the process is gone; ``tools/run_tests.py`` clears it before starting the next one. """ with contextlib.suppress(OSError): os.remove(LOG_PATH) @@ -191,7 +191,7 @@ def _echo_ovrtx_log(request): suppressed so that the reverse cannot happen either: this runs in the teardown of every test that rendered, so a full or unwritable artifact directory would otherwise turn each of them into an error and take the replay below down with it. An artifact matters less than the test result and the replay, - as it does to ``_make_crash_pass_result`` in ``tools/conftest.py``. + as it does to ``_make_crash_pass_result`` in ``tools/run_tests.py``. """ label = request.node.name diff --git a/tools/conftest.py b/tools/run_tests.py similarity index 87% rename from tools/conftest.py rename to tools/run_tests.py index 62327129abb1..b06a3d28b98a 100644 --- a/tools/conftest.py +++ b/tools/run_tests.py @@ -3,6 +3,33 @@ # # SPDX-License-Identifier: BSD-3-Clause +"""Run Isaac Lab's tests, locally or in CI. + +``tools/test_plan.toml`` says what each job covers and :mod:`testplan` resolves it to a file +list; this module executes that list. It owns everything that makes a long test run +survivable -- per-file timeouts, startup-hang detection, stack dumps from a wedged process, +crash reports for a process that died before writing one, retries, the work queue, and the +merged JUnit output -- none of which pytest does on its own. + +Files declaring a launch marker (see :mod:`isaaclab.test.kit`) are handed to pytest together +so they share one Kit app; everything else gets a process to itself. That grouping is the +normal schedule rather than a special case, because sharing a process is the only way a +directory of Kit-dependent files boots Kit once instead of once per file. + +This was ``tools/conftest.py``, which disabled collection and did all of the above from +``pytest_sessionstart``. Being a conftest meant it hijacked any pytest run rooted at the +repository, which is why ``tools-tests.yml`` had to pass ``--noconftest`` to run the tools' +own tests. It is a script now:: + + python tools/run_tests.py --list-jobs + python tools/run_tests.py --job isaaclab-core --shard 0 + python tools/run_tests.py source/isaaclab/test/sim + python tools/run_tests.py --all +""" + +from __future__ import annotations + +import argparse import contextlib import logging import os @@ -18,13 +45,14 @@ from junitparser import Error, JUnitXml, TestCase, TestSuite from prettytable import PrettyTable -from isaaclab.test.kit import kit_marker +from isaaclab.test.kit import kit_marker, module_markers from isaaclab.test.utils import resolve_test_sim_device # Local imports import hang_dump # isort: skip import ovrtx_log # isort: skip import test_settings as test_settings # isort: skip +import testplan # isort: skip from crash_journal import JOURNAL_ENV_VAR, create_crash_report # isort: skip from _device_split import DEVICE_SPLIT_PASSES, is_device_split_file # isort: skip from _kit_batching import ( # isort: skip @@ -39,9 +67,12 @@ logger = logging.getLogger(__name__) -def pytest_ignore_collect(collection_path, config): - # Skip collection and run each test script individually - return True +class TestRunError(Exception): + """A run could not be set up. Carries the exit code the process should end with.""" + + def __init__(self, message: str, returncode: int = 1): + super().__init__(message) + self.returncode = returncode COLD_CACHE_BUFFER = 700 @@ -1530,78 +1561,6 @@ def run_batched_tests(batches, workspace_root, ci_marker, cold_cache_applied=Fal return failed_tests, test_status, xml_reports, leftovers -def _collect_test_files( - source_dirs, - filter_pattern, - exclude_pattern, - include_files, - quarantined_only, - curobo_only, -): - """Collect test files from source directories, applying all active filters.""" - test_files = [] - for source_dir in source_dirs: - if not os.path.exists(source_dir): - logger.error(f"Error: source directory not found at {source_dir}") - pytest.exit("Source directory not found", returncode=1) - - for root, _, files in os.walk(source_dir): - # source/isaaclab/test/install_ci/ has its own pytest config and conftest. - # It is run via .github/actions/install-ci-run, never via this collector, - # so skip the whole subtree to keep install_ci tests out of build.yaml jobs. - if "install_ci" in root.replace("\\", "/").split("/"): - continue - - for file in files: - if not (file.startswith("test_") and file.endswith(".py")): - continue - - # Mode-exclusive filters (each bypasses TESTS_TO_SKIP) - if quarantined_only: - if file not in test_settings.QUARANTINED_TESTS: - continue - elif curobo_only: - if file not in test_settings.CUROBO_TESTS: - continue - else: - # An explicit include_files entry overrides TESTS_TO_SKIP, allowing - # dedicated jobs (e.g. test-environments-training) to run tests that - # are otherwise excluded from general CI runs. - if file in test_settings.TESTS_TO_SKIP and file not in include_files: - logger.debug(f"Skipping {file} as it's in the skip list") - continue - - full_path = os.path.join(root, file) - - if filter_pattern and filter_pattern not in full_path: - logger.debug(f"Skipping {full_path} (does not match include pattern: {filter_pattern})") - continue - if exclude_pattern and any(p.strip() in full_path for p in exclude_pattern.split(",")): - logger.debug(f"Skipping {full_path} (matches exclude pattern: {exclude_pattern})") - continue - if include_files and file not in include_files: - logger.debug(f"Skipping {full_path} (not in include files list)") - continue - - test_files.append(full_path) - - # Sort test files deterministically to ensure consistent test ordering. - test_files.sort() - - # Apply file-level sharding: select every Nth file from the deterministic order. - # Skip when include_files is set — in that case the test's own conftest handles - # sharding at the test-item level (e.g. parametrized test cases). - shard_index = os.environ.get("TEST_SHARD_INDEX", "") - shard_count = os.environ.get("TEST_SHARD_COUNT", "") - if shard_index and shard_count and not include_files: - shard_index = int(shard_index) - shard_count = int(shard_count) - test_files = [f for i, f in enumerate(test_files) if i % shard_count == shard_index] - logger.info(f"Shard {shard_index}/{shard_count}: selected {len(test_files)} test files") - - return test_files - - def _load_test_node_ids_from_toml(workspace_root: str) -> list[str]: """Load exact pytest node IDs from a TOML file configured in the environment.""" node_ids_file = os.environ.get("TEST_NODE_IDS_FILE") @@ -1609,7 +1568,7 @@ def _load_test_node_ids_from_toml(workspace_root: str) -> list[str]: if not (node_ids_file or node_ids_key): return [] if not (node_ids_file and node_ids_key): - pytest.exit("Both TEST_NODE_IDS_FILE and TEST_NODE_IDS_KEY must be set together", returncode=1) + raise TestRunError("Both TEST_NODE_IDS_FILE and TEST_NODE_IDS_KEY must be set together", 1) path = node_ids_file if os.path.isabs(node_ids_file) else os.path.join(workspace_root, node_ids_file) @@ -1617,14 +1576,14 @@ def _load_test_node_ids_from_toml(workspace_root: str) -> list[str]: with open(os.path.normpath(path), "rb") as stream: node_ids = tomllib.load(stream).get(node_ids_key) except OSError as exc: - pytest.exit(f"Could not read TEST_NODE_IDS_FILE {node_ids_file!r}: {exc}", returncode=1) + raise TestRunError(f"Could not read TEST_NODE_IDS_FILE {node_ids_file!r}: {exc}", 1) except tomllib.TOMLDecodeError as exc: - pytest.exit(f"{node_ids_file}: invalid TOML: {exc}", returncode=1) + raise TestRunError(f"{node_ids_file}: invalid TOML: {exc}", 1) if not node_ids: - pytest.exit(f"{node_ids_key!r} not found or empty in {node_ids_file}", returncode=1) + raise TestRunError(f"{node_ids_key!r} not found or empty in {node_ids_file}", 1) if not isinstance(node_ids, list) or not all(isinstance(node_id, str) for node_id in node_ids): - pytest.exit(f"{node_ids_key!r} must be a TOML array of strings in {node_ids_file}", returncode=1) + raise TestRunError(f"{node_ids_key!r} must be a TOML array of strings in {node_ids_file}", 1) return node_ids @@ -1634,13 +1593,13 @@ def _collect_test_node_ids_by_file(workspace_root: str) -> dict[str, list[str]]: node_ids = [line.strip() for line in os.environ.get("TEST_NODE_IDS", "").splitlines() if line.strip()] node_ids.extend(_load_test_node_ids_from_toml(workspace_root)) if len(node_ids) != len(set(node_ids)): - pytest.exit("Configured test node IDs contain duplicates", returncode=1) + raise TestRunError("Configured test node IDs contain duplicates", 1) grouped: dict[str, list[str]] = {} for node_id in node_ids: normalized_node_id = node_id.replace("\\", "/") if "::" not in normalized_node_id: - pytest.exit(f"Configured test node ID must include '::': {node_id}", returncode=1) + raise TestRunError(f"Configured test node ID must include '::': {node_id}", 1) file_part, test_part = normalized_node_id.split("::", 1) if os.path.isabs(file_part): @@ -1649,7 +1608,7 @@ def _collect_test_node_ids_by_file(workspace_root: str) -> dict[str, list[str]]: abs_file = os.path.normpath(os.path.join(workspace_root, file_part)) if not os.path.exists(abs_file): - pytest.exit(f"Configured test node ID file does not exist: {node_id}", returncode=1) + raise TestRunError(f"Configured test node ID file does not exist: {node_id}", 1) grouped.setdefault(abs_file, []).append(f"{normalized_node_id.split('::', 1)[0]}::{test_part}") @@ -1693,131 +1652,83 @@ def _format_test_file_results(test_files: list[str], test_status: dict[str, dict return summary + table.get_string() -def pytest_sessionstart(session): - """Intercept pytest startup to execute tests in the correct order.""" - # Get the workspace root directory (one level up from tools) - workspace_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - source_dirs = [ - os.path.join(workspace_root, "scripts"), - os.path.join(workspace_root, "source"), - ] +def _resolve_selection(args) -> tuple[list[str], str, str | None]: + """Turn the command line into the files to run, the ``-m`` marker and the ``-k`` expression. - # Get filter pattern from environment variable or command line - filter_pattern = os.environ.get("TEST_FILTER_PATTERN", "") - exclude_pattern = os.environ.get("TEST_EXCLUDE_PATTERN", "") - include_files_str = os.environ.get("TEST_INCLUDE_FILES", "") - quarantined_only = os.environ.get("TEST_QUARANTINED_ONLY", "false") == "true" - curobo_only = os.environ.get("TEST_CUROBO_ONLY", "false") == "true" - - isaacsim_ci = os.environ.get("ISAACSIM_CI_SHORT", "false") == "true" - - # CI_MARKER env var is a separate, parallel mechanism for cross-platform - # jobs (arm-ci, windows-ci, ...) to reuse this orchestrator with their own - # markers. Deliberately NOT aliased to ISAACSIM_CI_SHORT: the isaacsim_ci - # filter is owned by Isaac Sim's external CI pipeline; the CI_MARKER path - # leaves that contract untouched. - ci_marker = os.environ.get("CI_MARKER", "") - test_node_ids_by_file = _collect_test_node_ids_by_file(workspace_root) + Args: + args: Parsed command-line arguments. - # Parse include files list (comma-separated paths) - include_files = set() - if include_files_str: - for f in include_files_str.split(","): - f = f.strip() - if f: - include_files.add(os.path.basename(f)) - include_files.update(os.path.basename(path) for path in test_node_ids_by_file) - - # Also try to get from pytest config - if hasattr(session.config, "option") and hasattr(session.config.option, "filter_pattern"): - filter_pattern = filter_pattern or getattr(session.config.option, "filter_pattern", "") - if hasattr(session.config, "option") and hasattr(session.config.option, "exclude_pattern"): - exclude_pattern = exclude_pattern or getattr(session.config.option, "exclude_pattern", "") - - logger.debug("=" * 50) - logger.debug("CONFTEST.PY DEBUG INFO") - logger.debug("=" * 50) - logger.debug(f"Filter pattern: '{filter_pattern}'") - logger.debug(f"Exclude pattern: '{exclude_pattern}'") - logger.debug(f"Include files: {include_files if include_files else 'none'}") - logger.debug(f"Test node IDs: {sum(len(node_ids) for node_ids in test_node_ids_by_file.values())}") - logger.debug(f"Quarantined-only mode: {quarantined_only}") - logger.debug(f"Curobo-only mode: {curobo_only}") - logger.debug(f"TEST_FILTER_PATTERN env var: '{os.environ.get('TEST_FILTER_PATTERN', 'NOT_SET')}'") - logger.debug(f"TEST_EXCLUDE_PATTERN env var: '{os.environ.get('TEST_EXCLUDE_PATTERN', 'NOT_SET')}'") - logger.debug(f"TEST_INCLUDE_FILES env var: '{os.environ.get('TEST_INCLUDE_FILES', 'NOT_SET')}'") - logger.debug(f"TEST_NODE_IDS env var: '{'SET' if os.environ.get('TEST_NODE_IDS') else 'NOT_SET'}'") - logger.debug(f"TEST_NODE_IDS_FILE env var: '{os.environ.get('TEST_NODE_IDS_FILE', 'NOT_SET')}'") - logger.debug(f"TEST_NODE_IDS_KEY env var: '{os.environ.get('TEST_NODE_IDS_KEY', 'NOT_SET')}'") - logger.debug(f"TEST_QUARANTINED_ONLY env var: '{os.environ.get('TEST_QUARANTINED_ONLY', 'NOT_SET')}'") - logger.debug(f"TEST_CUROBO_ONLY env var: '{os.environ.get('TEST_CUROBO_ONLY', 'NOT_SET')}'") - logger.debug("=" * 50) - - # Get all test files in the source directories - test_files = _collect_test_files( - source_dirs, - filter_pattern, - exclude_pattern, - include_files, - quarantined_only, - curobo_only, - ) + Returns: + ``(test_files, marker, k_expr)``; ``test_files`` are absolute paths. - if isaacsim_ci: - new_test_files = [] - for test_file in test_files: - with open(test_file) as f: - if "@pytest.mark.isaacsim_ci" in f.read(): - new_test_files.append(test_file) - test_files = new_test_files - - if ci_marker: - # Match both `@pytest.mark.` (per-function) and - # `pytestmark = pytest.mark.` / `pytestmark = [..., pytest.mark., ...]` - # (module-level) by looking for the common `pytest.mark.` substring. - marker_token = f"pytest.mark.{ci_marker}" - new_test_files = [] - for test_file in test_files: - try: - with open(test_file) as f: - if marker_token in f.read(): - new_test_files.append(test_file) - except OSError as exc: - raise RuntimeError( - f"ci_marker post-scan could not read {test_file}; refusing to" - f" silently drop a potentially marker-tagged file" - ) from exc - test_files = new_test_files + Raises: + TestRunError: If the selection names nothing runnable. + """ + root = testplan.REPO_ROOT + + if args.job: + job = testplan.get_job(args.job) + relative = testplan.resolve(job, shard=args.shard) + marker, k_expr = job.marker or "", job.k_expr + else: + paths = args.paths or (["source", "scripts"] if args.all else None) + if not paths: + raise TestRunError("nothing selected: pass --job, --all, or one or more paths", 2) + job = testplan.Job(name="ad-hoc", title="ad-hoc", workflow="", paths=tuple(paths)) + relative = testplan.resolve(job) + marker, k_expr = "", None + + # ISAACSIM_CI_SHORT is Isaac Sim's external pipeline asking for its own subset. It is a + # separate contract from the plan, so it narrows whatever the job selected rather than + # replacing it, and the job's own marker wins when both are present -- `-m` takes one + # expression. + if os.environ.get("ISAACSIM_CI_SHORT", "false") == "true": + relative = [ + path for path in relative if "isaacsim_ci" in module_markers((root / path).read_text(errors="replace")) + ] + marker = marker or "isaacsim_ci" + + if args.k_expr is not None: + k_expr = args.k_expr + + test_files = [str(root / path) for path in relative] + if not test_files: + _write_empty_report() + raise TestRunError(f"no test files selected for {args.job or 'the given paths'}", 0) + return test_files, marker, k_expr + + +def run(args) -> int: + """Resolve the selection, run it, and report. + + Args: + args: Parsed command-line arguments. + + Returns: + The process exit code; see :data:`EXIT_CODE_LABELS`. + + Raises: + TestRunError: If the run could not be set up. + """ + workspace_root = str(testplan.REPO_ROOT) + test_files, effective_marker, k_expr = _resolve_selection(args) + test_node_ids_by_file = _collect_test_node_ids_by_file(workspace_root) if test_node_ids_by_file: configured_files = set(test_node_ids_by_file) test_files = [test_file for test_file in test_files if os.path.normpath(test_file) in configured_files] missing_files = sorted(configured_files - {os.path.normpath(test_file) for test_file in test_files}) if missing_files: - pytest.exit(f"Configured test node ID files were not collected: {missing_files}", returncode=1) + raise TestRunError(f"Configured test node ID files were not collected: {missing_files}", 1) - if not test_files: - if quarantined_only: - logger.info("No quarantined tests configured — nothing to run.") - _write_empty_report() - pytest.exit("No quarantined tests configured", returncode=0) - if filter_pattern: - logger.info(f"No test files found matching filter pattern '{filter_pattern}' — nothing to run.") - _write_empty_report() - pytest.exit("No test files found for filter", returncode=0) - logger.warning("No test files found in source directory") - pytest.exit("No test files found", returncode=1) - - logger.info(f"Found {len(test_files)} test files after filtering") + if k_expr: + os.environ["TEST_K_EXPR"] = k_expr + + logger.info(f"Found {len(test_files)} test files") for test_file in test_files: node_ids = test_node_ids_by_file.get(os.path.normpath(test_file), []) if test_node_ids_by_file else [] suffix = f" ({', '.join(node_ids)})" if node_ids else "" - logger.info(f" - {test_file}{suffix}") - - # Run all tests individually. CI_MARKER takes precedence when both env - # vars are set; falls back to "isaacsim_ci" when only ISAACSIM_CI_SHORT - # is set. The pytest -m flag only accepts one expression. - effective_marker = ci_marker or ("isaacsim_ci" if isaacsim_ci else "") + logger.info(f" - {os.path.relpath(test_file, workspace_root)}{suffix}") # Files that declare a launch marker share one Kit app when they land in the same process, # so group them and pay startup once per group instead of once per file. Unmarked files -- @@ -1934,5 +1845,64 @@ def pytest_sessionstart(session): # Print summary to console and log file logger.info(summary_str) - # Exit pytest after custom execution to prevent normal pytest from overwriting our report - pytest.exit("Custom test execution completed", returncode=exit_code) + return exit_code + + +def _build_parser() -> argparse.ArgumentParser: + """Build the command-line interface.""" + parser = argparse.ArgumentParser( + prog="run_tests.py", + description=__doc__.splitlines()[0], + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=( + "examples:\n" + " python tools/run_tests.py --list-jobs\n" + " python tools/run_tests.py --job isaaclab-core --shard 0\n" + " python tools/run_tests.py source/isaaclab/test/sim\n" + " python tools/run_tests.py --all\n" + ), + ) + what = parser.add_mutually_exclusive_group() + what.add_argument("--job", help="run a job from tools/test_plan.toml") + what.add_argument("--all", action="store_true", help="run every test file under source/ and scripts/") + parser.add_argument("paths", nargs="*", help="directories to walk for test_*.py") + parser.add_argument("--shard", type=int, help="which shard of a sharded job to run, 0-based") + parser.add_argument("-k", dest="k_expr", help="pytest -k expression; overrides the job's own") + parser.add_argument("--list-jobs", action="store_true", help="list the jobs in the test plan and exit") + parser.add_argument("--list-files", action="store_true", help="print the resolved file list and exit") + return parser + + +def main(argv: list[str] | None = None) -> int: + """Entry point. + + Args: + argv: Command-line arguments; defaults to ``sys.argv[1:]``. + + Returns: + The process exit code. + """ + args = _build_parser().parse_args(argv) + + if args.list_jobs: + rows = PrettyTable(["job", "workflow", "shards", "files"]) + rows.align = "l" + for job in testplan.load_plan(): + rows.add_row([job.name, job.workflow, job.shards or 1, len(testplan.resolve(job))]) + print(rows) + return 0 + + try: + if args.list_files: + test_files, _, _ = _resolve_selection(args) + print("\n".join(os.path.relpath(path, testplan.REPO_ROOT) for path in test_files)) + return 0 + return run(args) + except TestRunError as exc: + level = logger.info if exc.returncode == 0 else logger.error + level(str(exc)) + return exc.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/skills/pyproject.toml b/tools/skills/pyproject.toml index 1197a9bd1189..552a8c3f5169 100644 --- a/tools/skills/pyproject.toml +++ b/tools/skills/pyproject.toml @@ -3,7 +3,7 @@ # # 1. ``pythonpath = ["."]`` adds ``tools/skills/`` to ``sys.path``, # making ``import cli`` work from the test files without any shim. -# 2. ``tools/conftest.py`` sits above rootdir and is therefore not loaded. +# 2. ``tools/run_tests.py`` sits above rootdir and is therefore not loaded. # # Run with: ``uv run python -m pytest tools/skills/`` [tool.pytest.ini_options] diff --git a/tools/test/test_test_plan.py b/tools/test/test_test_plan.py new file mode 100644 index 000000000000..bf0a77ba437f --- /dev/null +++ b/tools/test/test_test_plan.py @@ -0,0 +1,138 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the test plan, the workflow generator, and the CI argument plumbing. + +The plan is the single source of truth for what every CI job runs, so the things that can go +quietly wrong are: a workflow naming a job the plan does not define, checked-in YAML drifting +from what the generator produces, a job resolving to nothing, and the positional arguments +``run-tests/action.yml`` passes to ``run_tests.sh`` sliding out of alignment with the ``local`` +bindings at the top of that script. Each has a test here. +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "tools")) +for _package in sorted((REPO_ROOT / "source").iterdir()): + if (_package / _package.name).is_dir(): + sys.path.insert(0, str(_package)) + +import generate_workflows # noqa: E402 +import testplan # noqa: E402 + +pytestmark = pytest.mark.unit + +_RUN_TESTS_SH = REPO_ROOT / ".github/actions/run-tests/run_tests.sh" +_RUN_TESTS_ACTION = REPO_ROOT / ".github/actions/run-tests/action.yml" + + +@pytest.fixture(scope="module") +def plan() -> list[testplan.Job]: + return testplan.load_plan() + + +def test_every_job_resolves_to_at_least_one_file(plan: list[testplan.Job]): + """A job that selects nothing is a lane that silently tests nothing.""" + empty = [job.name for job in plan if not testplan.resolve(job)] + assert not empty, "these jobs resolve to no test files:\n " + "\n ".join(empty) + + +def test_shards_cover_the_job_exactly_once(plan: list[testplan.Job]): + """Every file in a sharded job runs in exactly one shard.""" + for job in plan: + if job.shards <= 1: + continue + whole = testplan.resolve(job) + pieces = [path for shard in range(job.shards) for path in testplan.resolve(job, shard=shard)] + assert sorted(pieces) == whole, f"{job.name}: shards do not partition the job" + assert len(pieces) == len(set(pieces)), f"{job.name}: a file appears in more than one shard" + + +def test_workflows_only_reference_jobs_the_plan_defines(plan: list[testplan.Job]): + """A workflow naming an unknown job fails at run time, in CI, after an image build.""" + known = {job.name for job in plan} + offenders = [] + for path in sorted((REPO_ROOT / ".github/workflows").glob("*.y*ml")): + data = yaml.safe_load(path.read_text(encoding="utf-8")) + for job_id, definition in (data.get("jobs") or {}).items(): + for step in definition.get("steps") or []: + name = (step.get("with") or {}).get("job") + if name and name not in known: + offenders.append(f"{path.name}::{job_id} -> {name!r}") + assert not offenders, "these workflow steps name a job absent from the test plan:\n " + "\n ".join(offenders) + + +def test_generated_workflow_blocks_are_up_to_date(): + """The checked-in YAML must match what the generator produces. + + Editing a generated block by hand is silently undone by the next regeneration, so the + mismatch has to fail here instead. + """ + stale = generate_workflows.check() + assert not stale, ( + "these workflow files are out of date with tools/test_plan.toml:\n " + + "\n ".join(stale) + + "\n\nFix: run `python tools/generate_workflows.py`." + ) + + +def _shell_bindings() -> list[str]: + """Return run_tests.sh's positional bindings, ordered by position.""" + text = _RUN_TESTS_SH.read_text(encoding="utf-8") + found = {} + for name, position in re.findall(r'^ local ([a-z_]+)="\$\{?(\d+)\}?"', text, re.M): + found[int(position)] = name + return [found[i] for i in sorted(found)] + + +def _action_arguments() -> list[str]: + """Return the arguments the action passes to run_tests.sh, ordered.""" + text = _RUN_TESTS_ACTION.read_text(encoding="utf-8") + line = next(ln for ln in text.splitlines() if "run_tests.sh" in ln and "bash" in ln) + call = line.split("run_tests.sh", 1)[1] + names = [] + for raw in re.findall(r'"([^"]*)"', call): + match = re.search(r"inputs\.([a-z0-9-]+)", raw) + names.append(match.group(1).replace("-", "_") if match else raw.lstrip("$").lower()) + return names + + +def test_run_tests_sh_arguments_line_up_with_the_action(): + """A positional interface this long silently misbinds when one side is edited alone. + + Names are compared rather than counts: an off-by-one that happens to preserve the count + would otherwise pass while feeding, say, the container name in as the job. + """ + bindings = _shell_bindings() + arguments = _action_arguments() + assert len(bindings) == len(arguments), ( + f"run_tests.sh binds {len(bindings)} positional arguments but action.yml passes" + f" {len(arguments)}:\n binds: {bindings}\n passes: {arguments}" + ) + mismatched = [ + f"${i + 1}: script binds {b!r}, action passes {a!r}" + for i, (b, a) in enumerate(zip(bindings, arguments)) + # The action spells a few values as env vars (PYTEST_OPTIONS) or literals rather than + # `inputs.`; those are matched loosely on the shared stem. + if b not in a and a not in b + ] + assert not mismatched, "run_tests.sh and action.yml disagree on argument order:\n " + "\n ".join(mismatched) + + +@pytest.mark.skipif(shutil.which("bash") is None, reason="needs bash to parse the script") +def test_run_tests_sh_is_valid_shell(): + """``bash -n`` catches the quoting mistakes that are easy to make editing this by hand.""" + result = subprocess.run(["bash", "-n", str(_RUN_TESTS_SH)], capture_output=True, text=True, timeout=60) + assert result.returncode == 0, result.stderr diff --git a/tools/test_crash_journal.py b/tools/test_crash_journal.py index 86eb6531c1c8..88b8b7bcd9e8 100644 --- a/tools/test_crash_journal.py +++ b/tools/test_crash_journal.py @@ -406,7 +406,7 @@ def test_ok(): def test_deselected_tests_are_not_journaled_as_collected(tmp_path): """Regression test for a rebuilt report claiming tests that this pass never selected. - ``tools/conftest.py`` splits a run into passes selected by marker and device, so journaling + ``tools/run_tests.py`` splits a run into passes selected by marker and device, so journaling from ``pytest_collection_modifyitems`` — which runs before pytest's own ``trylast`` deselection hook — would record the other passes' tests too. A crash would then rebuild them as "not run" skips, inflating the counts and duplicating node IDs the sibling pass reported. @@ -614,7 +614,7 @@ def test_never_reached(): marks=pytest.mark.skipif(not _HAS_FLAKY, reason="the rerun this case needs is driven by the flaky plugin"), ), pytest.param( - # The startup-hang shape ``tools/conftest.py`` guards against: collection has finished + # The startup-hang shape ``tools/run_tests.py`` guards against: collection has finished # journaling by the time the run loop starts, so this kills the session in the window where # the journal knows every test but none has a verdict. They must come back as "not run" # rather than disappear from the uploaded results, which would silently shrink the suite. diff --git a/tools/test_plan.toml b/tools/test_plan.toml new file mode 100644 index 000000000000..e0e5ea1447a4 --- /dev/null +++ b/tools/test_plan.toml @@ -0,0 +1,280 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# The test plan: one entry per CI job, and the single source of truth for what any run +# covers. `tools/run_tests.py` executes a job from here; `tools/generate_workflows.py` +# renders the workflow YAML from the same entries, so a job cannot exist in CI without +# being described here. See tools/testplan.py for the field semantics. +# +# Selection fields: +# paths directories to walk for test_*.py (required) +# exclude substrings; a file whose repo-relative path contains any of them is dropped +# files basenames; when set, only these files run, and they override TESTS_TO_SKIP +# shards split the resolved list across N jobs, round-robin by sorted position +# pool "quarantined" or "curobo" selects that list from tools/test_settings.py +# instead of walking `paths`, and bypasses TESTS_TO_SKIP +# +# Execution fields (k-expr, marker) narrow within a file rather than choosing files. + +# --------------------------------------------------------------------------- +# build.yaml -- the per-package lanes that run on every PR +# --------------------------------------------------------------------------- + +[[job]] +name = "isaaclab-tasks" +title = "isaaclab_tasks" +workflow = "build" +paths = ["source/isaaclab_tasks"] +exclude = ["test_rendering_", "test_video_recording.py"] +shards = 3 +container-name = "isaac-lab-tasks-test" +warp-cache = "restore" +generate = true +continue-on-error = true +extra-pip-packages = "pytetwild[all]>=0.3.0,<0.4" + +[[job]] +name = "isaaclab-core" +title = "isaaclab (core)" +workflow = "build" +# Everything that is not a source/isaaclab_* package: the core library plus the +# repo scripts. Walking the directories rather than matching "not isaaclab_" as a +# substring is what keeps `source/isaaclab` from also selecting `source/isaaclab_rl`. +paths = ["source/isaaclab", "scripts"] +shards = 3 +container-name = "isaac-lab-core-test" +warp-cache = "restore" +generate = true + +[[job]] +name = "isaaclab-rl" +title = "isaaclab_rl" +workflow = "build" +paths = ["source/isaaclab_rl"] +container-name = "isaac-lab-rl-test" +generate = true +extra-pip-packages = "leapp" + +[[job]] +name = "isaaclab-mimic" +title = "isaaclab_mimic" +workflow = "build" +paths = ["source/isaaclab_mimic"] +container-name = "isaac-lab-mimic-test" +generate = true + +[[job]] +name = "isaaclab-contrib" +title = "isaaclab_contrib" +workflow = "build" +paths = ["source/isaaclab_contrib"] +container-name = "isaac-lab-contrib-test" +generate = true +extra-pip-packages = "pytetwild[all]>=0.3.0,<0.4" + +[[job]] +name = "isaaclab-teleop" +title = "isaaclab_teleop" +workflow = "build" +paths = ["source/isaaclab_teleop"] +container-name = "isaac-lab-teleop-test" +generate = true + +[[job]] +name = "isaaclab-visualizers" +title = "isaaclab_visualizers" +workflow = "build" +paths = ["source/isaaclab_visualizers"] +container-name = "isaac-lab-visualizers-test" +generate = true + +[[job]] +name = "isaaclab-assets" +title = "isaaclab_assets" +workflow = "build" +paths = ["source/isaaclab_assets"] +container-name = "isaac-lab-assets-test" +generate = true + +[[job]] +name = "isaaclab-experimental" +title = "isaaclab_experimental" +workflow = "build" +paths = ["source/isaaclab_experimental"] +container-name = "isaac-lab-experimental-test" +generate = true + +[[job]] +name = "isaaclab-newton" +title = "isaaclab_newton" +workflow = "build" +paths = ["source/isaaclab_newton"] +container-name = "isaac-lab-newton-test" +generate = true +warp-cache = "restore" + +[[job]] +name = "isaaclab-physx" +title = "isaaclab_physx" +workflow = "build" +paths = ["source/isaaclab_physx"] +container-name = "isaac-lab-physx-test" +generate = true +extra-pip-packages = "pytetwild[all]>=0.3.0,<0.4" + +[[job]] +name = "isaaclab-ov" +title = "isaaclab_ov" +workflow = "build" +paths = ["source/isaaclab_ov"] +container-name = "isaac-lab-ov-test" + +# --------------------------------------------------------------------------- +# build.yaml -- targeted lanes selecting individual files +# --------------------------------------------------------------------------- + +[[job]] +name = "standalone-demos-kit" +title = "standalone demos (headless, Kit)" +workflow = "build" +paths = ["source", "scripts"] +files = ["test_standalone_scripts.py"] +container-name = "isaac-lab-standalone-kit-test" + +[[job]] +name = "standalone-demos-non-kit" +title = "standalone demos (headless, non-Kit)" +workflow = "build" +paths = ["source", "scripts"] +files = ["test_standalone_scripts.py"] +container-name = "isaac-lab-standalone-non-kit-test" + +[[job]] +name = "curobo" +title = "test-curobo" +workflow = "build" +paths = ["source", "scripts"] +files = ["test_curobo_planner_franka.py", "test_curobo_planner_cube_stack.py", "test_pink_ik.py"] +container-name = "isaac-lab-curobo-test" + +[[job]] +name = "contrib-environments" +title = "test-contrib-environments" +workflow = "build" +paths = ["source", "scripts"] +files = ["test_generate_dataset_skillgen.py", "test_contrib_environments.py"] +container-name = "isaac-lab-contrib-env-test" + +[[job]] +name = "record-video" +title = "record-video" +workflow = "build" +paths = ["source/isaaclab_tasks"] +files = ["test_video_recording.py"] +container-name = "isaac-lab-record-video-test" + +[[job]] +name = "rendering-correctness" +title = "rendering-correctness" +workflow = "build" +paths = ["source/isaaclab_tasks"] +files = [ + "test_rendering_cartpole.py", + "test_rendering_lift_kuka_hetero.py", + "test_rendering_lift_kuka_homo.py", + "test_rendering_franka_cloth.py", + "test_rendering_franka_soft.py", + "test_rendering_franka_cable.py", + "test_rendering_registered_tasks.py", + "test_rendering_shadow_hand.py", +] +node-ids-key = "rendering-correctness" +container-name = "isaac-lab-rendering-test" + +[[job]] +name = "rendering-correctness-kitless-legacy" +title = "rendering-correctness-kitless (legacy)" +workflow = "build" +paths = ["source/isaaclab_tasks"] +files = [ + "test_rendering_cartpole_kitless.py", + "test_rendering_lift_kuka_hetero_kitless.py", + "test_rendering_lift_kuka_homo_kitless.py", + "test_rendering_franka_cloth_kitless.py", + "test_rendering_franka_soft_kitless.py", + "test_rendering_franka_cable_kitless.py", + "test_rendering_shadow_hand_kitless.py", +] +node-ids-key = "rendering-correctness-kitless-legacy" +k-expr = "legacy" +container-name = "isaac-lab-rendering-kitless-legacy-test" + +[[job]] +name = "rendering-correctness-kitless-ovstage" +title = "rendering-correctness-kitless (ovstage)" +workflow = "build" +paths = ["source/isaaclab_tasks"] +files = [ + "test_rendering_cartpole_kitless.py", + "test_rendering_lift_kuka_hetero_kitless.py", + "test_rendering_lift_kuka_homo_kitless.py", + "test_rendering_franka_cloth_kitless.py", + "test_rendering_franka_soft_kitless.py", + "test_rendering_franka_cable_kitless.py", + "test_rendering_shadow_hand_kitless.py", +] +node-ids-key = "rendering-correctness-kitless-ovstage" +k-expr = "ovstage" +container-name = "isaac-lab-rendering-kitless-ovstage-test" + +[[job]] +name = "warp-cache-warm" +title = "warp-cache-warm" +workflow = "build" +paths = ["source/isaaclab_tasks"] +files = ["test_environments_newton.py", "test_multi_agent_environments.py"] +k-expr = "test_environments and not (Soft or Cloth or Cable)" +container-name = "isaac-lab-warp-warm" + +# --------------------------------------------------------------------------- +# daily-compatibility.yml +# --------------------------------------------------------------------------- + +[[job]] +name = "isaaclab-tasks-compat" +title = "test-isaaclab-tasks-compat" +workflow = "daily-compatibility" +paths = ["source/isaaclab_tasks"] +container-name = "isaac-lab-tasks-compat-test" + +[[job]] +name = "general-compat" +title = "test-general-compat" +workflow = "daily-compatibility" +paths = ["source", "scripts"] +exclude = ["isaaclab_tasks"] +container-name = "isaac-lab-general-compat-test" + +# --------------------------------------------------------------------------- +# arm-ci.yml -- one lane, split by -k so the ovphysx tests run separately +# --------------------------------------------------------------------------- + +[[job]] +name = "arm-ci" +title = "Build & Test" +workflow = "arm-ci" +paths = ["source", "scripts"] +marker = "arm_ci" +k-expr = "not ovphysx" +container-name = "isaac-lab-arm-test" + +[[job]] +name = "arm-ci-ovphysx" +title = "Build & Test" +workflow = "arm-ci" +paths = ["source", "scripts"] +marker = "arm_ci" +k-expr = "ovphysx" +container-name = "isaac-lab-arm-ovphysx-test" diff --git a/tools/test_settings.py b/tools/test_settings.py index 0dc337941d4a..54332c140674 100644 --- a/tools/test_settings.py +++ b/tools/test_settings.py @@ -121,7 +121,7 @@ # quarantined tests - run in dedicated CI job that does not block PR merges *QUARANTINED_TESTS, "test_environments_training.py", # Long-running RL training test; runs in dedicated CI job - # Exercises tools/conftest.py itself, including a hang that has to be waited out in real time. + # Exercises tools/run_tests.py itself, including a hang that has to be waited out in real time. # Needs no Isaac Sim and is not worth the CI spend. To run it when changing the orchestrator: # PYTHONPATH=tools:source/isaaclab pytest --noconftest \ # source/isaaclab/test/cli/test_test_orchestrator_result_handling.py diff --git a/tools/testplan.py b/tools/testplan.py new file mode 100644 index 000000000000..47182b487c07 --- /dev/null +++ b/tools/testplan.py @@ -0,0 +1,236 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Resolve a job in ``tools/test_plan.toml`` to the test files it covers. + +Selection used to be spread over four layers: a workflow job's inputs, two composite actions, +a bash translation into ``TEST_*`` environment variables, and a collector in the test runner +that read them back. A job's coverage could only be worked out by tracing all four. This +module is the whole of it: the plan says what a job runs, and :func:`resolve` turns that into +a file list. + +Markers are read with :func:`isaaclab.test.kit.module_markers`, which parses the file rather +than searching its text, so a marker named in a docstring or a comment no longer pulls a file +into a lane that does not want it. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + +import test_settings +import tomllib + +from isaaclab.test.kit import module_markers + +REPO_ROOT = Path(__file__).resolve().parent.parent + +PLAN_PATH = REPO_ROOT / "tools" / "test_plan.toml" + +# Has its own pytest.ini and conftest, and runs through .github/actions/install-ci-run rather +# than this plan. Skipping the whole subtree keeps its tests out of every job. +_EXCLUDED_DIRS = frozenset({"install_ci"}) + +_POOLS = {"quarantined": "QUARANTINED_TESTS", "curobo": "CUROBO_TESTS"} + + +@dataclass(frozen=True) +class Job: + """One entry in the test plan. + + Attributes: + name: Stable identifier, used on the command line and as the workflow job suffix. + title: Display name; a sharded job renders as ``" [i/n]"``. + workflow: Which workflow file the generated job block belongs to. + paths: Repo-relative directories walked for ``test_*.py``. + exclude: Substrings; a file whose repo-relative path contains any of them is dropped. + files: Basenames; when set, only these run, and they override ``TESTS_TO_SKIP``. + shards: Number of jobs the resolved list is split across. + pool: Named list in ``tools/test_settings.py`` to run instead of walking ``paths``. + marker: Only files declaring this pytest marker are selected. + k_expr: ``-k`` expression passed to each pytest invocation; narrows within a file. + node_ids_key: Key in ``.github/test-subsets/`` selecting exact node IDs on push. + container_name: Docker container name for the CI job. + warp_cache: Warp cache mode for the CI job. + generate: Whether ``tools/generate_workflows.py`` renders this job's workflow block. + False for the lanes whose CI setup is bespoke -- extra build steps, wheelhouse + expressions, artifact uploads -- which stay hand-written and are only checked + against the plan. + continue_on_error: Whether a failure in this lane leaves the run green. + extra_pip_packages: Packages installed in the container before the tests start. + timeout_minutes: Job timeout. + """ + + name: str + title: str + workflow: str + paths: tuple[str, ...] + exclude: tuple[str, ...] = () + files: tuple[str, ...] = () + shards: int = 1 + pool: str | None = None + marker: str | None = None + k_expr: str | None = None + node_ids_key: str | None = None + container_name: str | None = None + warp_cache: str | None = None + generate: bool = False + continue_on_error: bool = False + extra_pip_packages: str | None = None + timeout_minutes: int = 180 + + +def load_plan(path: Path | None = None) -> list[Job]: + """Read the plan file. + + Args: + path: Plan to read; defaults to :data:`PLAN_PATH`. + + Returns: + Every job, in file order. + + Raises: + ValueError: If a job is missing a name or two jobs share one. + """ + raw = tomllib.loads((path or PLAN_PATH).read_text(encoding="utf-8")) + jobs = [] + seen = set() + for entry in raw.get("job", []): + name = entry.get("name") + if not name: + raise ValueError(f"a job in {path or PLAN_PATH} has no name: {entry}") + if name in seen: + raise ValueError(f"duplicate job name in the test plan: {name!r}") + seen.add(name) + jobs.append( + Job( + name=name, + title=entry.get("title", name), + workflow=entry["workflow"], + paths=tuple(entry.get("paths", ())), + exclude=tuple(entry.get("exclude", ())), + files=tuple(entry.get("files", ())), + shards=int(entry.get("shards", 1)), + pool=entry.get("pool"), + marker=entry.get("marker"), + k_expr=entry.get("k-expr"), + node_ids_key=entry.get("node-ids-key"), + container_name=entry.get("container-name"), + warp_cache=entry.get("warp-cache"), + generate=bool(entry.get("generate", False)), + continue_on_error=bool(entry.get("continue-on-error", False)), + extra_pip_packages=entry.get("extra-pip-packages"), + timeout_minutes=int(entry.get("timeout-minutes", 180)), + ) + ) + return jobs + + +def get_job(name: str, path: Path | None = None) -> Job: + """Return the named job. + + Args: + name: Job name from the plan. + path: Plan to read; defaults to :data:`PLAN_PATH`. + + Returns: + The matching job. + + Raises: + KeyError: If no job has that name. + """ + for job in load_plan(path): + if job.name == name: + return job + raise KeyError(f"no job named {name!r} in the test plan; try --list-jobs") + + +def _display_path(path: Path, root: Path) -> str: + """Return ``path`` relative to ``root``, or absolute when it lies outside the repository. + + The local runner accepts any directory, including one outside the checkout, so this cannot + assume every result is repo-relative. + """ + try: + return path.relative_to(root).as_posix() + except ValueError: + return path.as_posix() + + +def walk_test_files(paths: tuple[str, ...] | list[str], root: Path | None = None) -> list[str]: + """Return every ``test_*.py`` under ``paths``, sorted. + + Args: + paths: Directories to walk, repo-relative or absolute. + root: Repository root; defaults to :data:`REPO_ROOT`. + + Returns: + Sorted paths, repo-relative where they lie inside the repository. + + Raises: + FileNotFoundError: If a listed directory does not exist, which would otherwise show up + as a job that silently runs nothing. + """ + root = root or REPO_ROOT + found = set() + for entry in paths: + base = root / entry + if not base.is_dir(): + raise FileNotFoundError(f"test plan path does not exist: {entry}") + for directory, _, names in os.walk(base): + if not _EXCLUDED_DIRS.isdisjoint(Path(directory).parts): + continue + for name in names: + if name.startswith("test_") and name.endswith(".py"): + found.add(_display_path(Path(directory) / name, root)) + return sorted(found) + + +def resolve(job: Job, *, shard: int | None = None, root: Path | None = None) -> list[str]: + """Return the test files ``job`` covers, repo-relative and sorted. + + Args: + job: Job to resolve. + shard: Which shard to take, for a job with ``shards > 1``. None returns every shard. + root: Repository root; defaults to :data:`REPO_ROOT`. + + Returns: + Sorted repo-relative paths. + + Raises: + ValueError: If ``shard`` is out of range for the job. + """ + root = root or REPO_ROOT + candidates = walk_test_files(job.paths, root=root) + wanted = set(job.files) + + selected = [] + for path in candidates: + name = os.path.basename(path) + if job.pool is not None: + if name not in getattr(test_settings, _POOLS[job.pool], ()): + continue + elif wanted: + # An explicit file list is the job's whole point, so it overrides the skip list. + if name not in wanted: + continue + elif name in test_settings.TESTS_TO_SKIP: + continue + if any(token in path for token in job.exclude): + continue + selected.append(path) + + if job.marker: + selected = [ + path for path in selected if job.marker in module_markers((root / path).read_text(errors="replace")) + ] + + if job.shards > 1 and shard is not None: + if not 0 <= shard < job.shards: + raise ValueError(f"shard {shard} out of range for job {job.name!r} with {job.shards} shards") + selected = [path for index, path in enumerate(selected) if index % job.shards == shard] + return selected