diff --git a/.github/workflows/tools-tests.yml b/.github/workflows/tools-tests.yml index 1c217268b708..05eea14d0bc8 100644 --- a/.github/workflows/tools-tests.yml +++ b/.github/workflows/tools-tests.yml @@ -51,9 +51,13 @@ jobs: # The tests import the modules under test (crash_journal, _device_split) directly. PYTHONPATH: tools # Files are listed explicitly rather than collected from tools/: test_settings.py is a - # configuration module, not a test, and tools/test/ needs the full Isaac Lab install. + # configuration module, not a test, and most of tools/test/ needs the full Isaac Lab + # install. test_task_discovery.py is the exception - task_discovery.py defers every + # isaaclab import into a function body, so its unit tests run on pytest alone. # # --noconftest keeps tools/conftest.py out of the session. That file is the CI test # orchestrator - its pytest_sessionstart scans source/ and scripts/ and runs the whole # suite, so loading it here would ignore the files named below. - run: python3 -m pytest tools/test_crash_journal.py tools/test_device_split.py -v --noconftest + run: >- + python3 -m pytest tools/test_crash_journal.py tools/test_device_split.py + tools/test/test_task_discovery.py -v --noconftest diff --git a/tools/environ_docs.py b/tools/environ_docs.py index 7096b853b51e..0042d6901341 100644 --- a/tools/environ_docs.py +++ b/tools/environ_docs.py @@ -23,7 +23,9 @@ import gymnasium as gym -from isaaclab_tasks.utils.preset_cli import enumerate_task_presets +# ``is_training_task`` is re-exported for callers that import it from here. +from task_discovery import RL_LIBRARY_PRIORITY, discover_tasks, is_training_task # noqa: F401 + from isaaclab_tasks.utils.preset_target import PresetTarget if TYPE_CHECKING: @@ -48,13 +50,9 @@ } ) -# RL libraries listed in a stable order across generated docs. -_RL_LIBRARY_ORDER = ("rl_games", "rsl_rl", "skrl", "sb3", "rlinf") - -# Gym IDs excluded from the training list. The ``-Eval`` suffix marks dedicated -# evaluation variants (e.g. ``IsaacContrib-Assemble-Trocar-G129-Dex3-Eval``, an alias -# registered for RLinf eval configs) that should not appear as their own training row. -_EVAL_TASK_SUFFIXES = ("-Eval",) +# RL libraries listed in a stable order across generated docs. Owned by +# ``task_discovery`` so the tables and the task matrix cannot drift apart. +_RL_LIBRARY_ORDER = RL_LIBRARY_PRIORITY # RL libraries not discoverable from Gym ``kwargs`` (e.g. RLinf YAML-based workflows). RL_LIBRARY_OVERRIDES: dict[str, dict[str, list[str]]] = { @@ -117,17 +115,6 @@ def _supports_warp_frontend(task_name: str, workflow: str, presets: dict[PresetT return False -def is_training_task(task_id: str) -> bool: - """Return ``True`` when *task_id* is a training (non-inference) Isaac task.""" - if "Isaac" not in task_id: - return False - if any(task_id.endswith(suffix) for suffix in _EVAL_TASK_SUFFIXES): - return False - if "-Benchmark-" in task_id: - return False - return True - - def parse_rl_libraries_from_kwargs(kwargs: dict) -> dict[str, list[str]]: """Parse RL-library and algorithm labels from Gym registry kwargs. @@ -527,13 +514,17 @@ def collect_environment_doc_rows( rows: list[EnvironmentDocRow] = [] - for spec in specs: - if not is_training_task(spec.id) or spec.kwargs.get("deprecated"): - continue - - preset_map = enumerate_task_presets(spec.id) - if preset_map is not None: - preset_map = dict(preset_map) + specs_by_id = {spec.id: spec for spec in specs} + for task in discover_tasks(specs, resolve=False): + spec = specs_by_id[task.task_id] + + preset_map = None + if task.declared is not None: + preset_map = { + PresetTarget.PHYSICS: list(task.declared["physics"]), + PresetTarget.RENDERER: list(task.declared["renderer"]), + PresetTarget.DOMAIN: list(task.declared["presets"]), + } preset_map[PresetTarget.PHYSICS] = _physics_names_for_docs(spec.id, preset_map) agents = apply_rl_library_overrides(spec.id, parse_rl_libraries_from_kwargs(spec.kwargs)) diff --git a/tools/task_discovery.py b/tools/task_discovery.py new file mode 100644 index 000000000000..9c38d3871299 --- /dev/null +++ b/tools/task_discovery.py @@ -0,0 +1,366 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Enumerate registered training tasks and the backend combinations they support. + +Declared combinations come from one registry walk. Resolved ones cost a config build +and a validator run each, and are the only reliable answer: the cross product is not +all legal, since OVRTX cannot share a process with Kit physics. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any + +logger = logging.getLogger(__name__) + +__all__ = [ + "RL_LIBRARY_PRIORITY", + "is_training_task", + "DiscoveredTask", + "DiscoveryError", + "discover_tasks", +] + +# Shared with the environment tables, but only the ordering: ``RL_LIBRARY_OVERRIDES`` +# there adds libraries declaring no entry point, so ``rlinf`` can appear in the tables +# and not in :attr:`DiscoveredTask.rl_libraries`. +RL_LIBRARY_PRIORITY: tuple[str, ...] = ("rl_games", "rsl_rl", "skrl", "sb3", "rlinf") + +# ``-Eval`` variants are separate registrations against an eval-specific env cfg, and +# ``-Benchmark-`` tasks are perf harnesses; neither is a training row. +_EVAL_TASK_SUFFIXES = ("-Eval",) + +# The validator could not run, as opposed to rejecting the combination. Swallowing +# these would mark every combination illegal. ``ImportError`` counts as drift because a +# missing *extra* raises ``ModuleNotFoundError`` and is handled before them. +_INFRASTRUCTURE_ERRORS = (ImportError, AttributeError, NameError, SyntaxError, TypeError) + + +class DiscoveryError(RuntimeError): + """Raised when the registry cannot be walked, or validation cannot run.""" + + +@dataclass(frozen=True) +class DiscoveredTask: + """One registered training task. + + Args: + task_id: Gym task id. + scope: ``core`` or ``contrib``. + rl_libraries: Libraries the task declares an agent config for, in + :data:`RL_LIBRARY_PRIORITY` order. Empty for IK, teleop and mimic tasks. + declared: Preset names by axis, or ``None`` if the config would not load. An + all-empty mapping means the task declares none. Keeps duplicate spellings. + modes: Ways to run the task; see :func:`_build_modes`. Resolution assumes a + headless launch, reads ``LIVESTREAM`` from the environment, and stops before + the Isaac Sim availability check. + default: The no-token run. ``None`` in declared mode, or if it does not resolve. + resolved: Whether ``modes`` was resolved or merely declared. + """ + + @dataclass(frozen=True) + class Mode: + """The tokens one run passes, ``None`` on an axis it leaves to the config. + + ``presets`` holds at most one name even though the token takes a comma list, so + a task with several independent preset axes is under-approximated. + """ + + physics: str | None + renderer: str | None + presets: str | None + + @dataclass(frozen=True) + class Default: + """The no-token run: the ``modes`` entry it collapsed into, and the physics + cfg class it resolves to, e.g. ``NewtonCfg(MJWarpSolverCfg)``. + """ + + backend: str | None + mode: DiscoveredTask.Mode + + task_id: str + scope: str + rl_libraries: tuple[str, ...] + # Mutable, so instances are not hashable despite ``frozen``. Treat as read-only. + declared: dict[str, tuple[str, ...]] | None + modes: tuple[DiscoveredTask.Mode, ...] + default: DiscoveredTask.Default | None + resolved: bool + + +def _domain_presets(names: list[str], typed_names: tuple[str, ...]) -> tuple[str, ...]: + """Return domain presets, dropping those the task also declares on a typed axis. + + Filtering by name instead would hide backends only reachable as ``presets=NAME``. + """ + typed = set(typed_names) + return tuple(sorted(name for name in names if name not in typed)) + + +def is_training_task(task_id: str) -> bool: + """Return whether *task_id* is a trainable (non-inference) Isaac task.""" + if "Isaac" not in task_id: + return False + if any(task_id.endswith(suffix) for suffix in _EVAL_TASK_SUFFIXES): + return False + return "-Benchmark-" not in task_id + + +def _rl_libraries_from_kwargs(kwargs: dict[str, Any]) -> tuple[str, ...]: + """Return the RL libraries a registration declares an agent config for. + + Matched on the stem before ``_cfg_entry_point`` so variants such as + ``rsl_rl_recurrent_cfg_entry_point`` count towards their library. + """ + declared = set() + for key in kwargs: + if not key.endswith("_cfg_entry_point") or key == "env_cfg_entry_point": + continue + stem = key[: -len("_cfg_entry_point")] + for candidate in RL_LIBRARY_PRIORITY: + if stem == candidate or stem.startswith(f"{candidate}_"): + declared.add(candidate) + break + return tuple(name for name in RL_LIBRARY_PRIORITY if name in declared) + + +def _mode_resolves( + task_id: str, physics: str | None, renderer: str | None, presets: str | None = None +) -> tuple[str, str | None] | None: + """Resolve one combination. + + Returns: + ``None`` if it cannot run. Otherwise ``(fingerprint, backend)``: the fingerprint + digests the resolved config, so two spellings of one run share it, and the + backend carries the solver that separates ``newton_mjwarp`` from + ``newton_kamino``. + + Raises: + DiscoveryError: If validation could not run at all. + """ + import argparse + import hashlib + import sys + + # Two are private. Losing one is drift, not a rejected combination. + try: + from isaaclab.app.sim_launcher import _get_kit_runtime_sources, _validate_runtime, scan + + from isaaclab_tasks.utils import resolve_task_config, setup_preset_cli + except ImportError as exc: + raise DiscoveryError(f"discovery could not import the Isaac Lab APIs it depends on: {exc}") from exc + + parser = argparse.ArgumentParser() + parser.add_argument("--task") + parser.add_argument("--agent", default=None) + argv = ["--task", task_id] + if physics is not None: + argv.append(f"physics={physics}") + if renderer is not None: + argv.append(f"renderer={renderer}") + if presets is not None: + argv.append(f"presets={presets}") + + original_argv = list(sys.argv) + try: + args, remaining = setup_preset_cli(parser, argv) + sys.argv = [sys.argv[0]] + remaining + env_cfg, _ = resolve_task_config(args.task, args.agent) + # The env constructor validates before it builds anything, and tasks put their + # cross-axis rules there -- Reach rejects ``newton_ik`` without Newton physics. + # Skipping it reports those combinations legal and fails them at launch instead. + env_cfg.validate() + config_scan = scan(env_cfg, args) + _validate_runtime(config_scan, _get_kit_runtime_sources(config_scan, args)) + fingerprint = hashlib.sha256(repr(env_cfg.to_dict()).encode()).hexdigest() + physics_cfg = config_scan.resolved_physics_cfg + solver = getattr(physics_cfg, "solver_cfg", None) + backend = None if physics_cfg is None else type(physics_cfg).__name__ + if solver is not None: + backend = f"{backend}({type(solver).__name__})" + return fingerprint, backend + except ModuleNotFoundError: + # An uninstalled extra. Narrower than ``ImportError``, which would hide drift. + return None + except _INFRASTRUCTURE_ERRORS as exc: + raise DiscoveryError( + f"preset validation for {task_id!r} could not run: {type(exc).__name__}: {exc}. This is an Isaac Lab" + " API failure, not a rejected preset combination." + ) from exc + except Exception: # noqa: BLE001 - any other failure means the combination cannot run + return None + finally: + sys.argv = original_argv + + +def _build_modes( + task_id: str, + physics: tuple[str, ...], + renderers: tuple[str, ...], + domains: tuple[str, ...], + *, + resolve: bool, + strict: bool = False, + collapse: bool = True, +) -> tuple[tuple[DiscoveredTask.Mode, ...], DiscoveredTask.Default | None]: + """Return the runs for one task, and what it does when given no tokens. + + Renderers expand across, domain presets one at a time. ``collapse`` deduplicates on + the resolved config, not the backend, which would merge same-backend runs. + + Returns: + ``(modes, default)``. *default* names an explicit spelling even when ``collapse`` + is off, and is ``None`` in declared mode or when the no-token run cannot resolve. + + Raises: + DiscoveryError: If ``strict`` and the validator could not judge a combination. + """ + physics_options: tuple[str | None, ...] = physics or (None,) + renderer_options: tuple[str | None, ...] = renderers or (None,) + domain_options: tuple[str | None, ...] = (None, *domains) if domains else (None,) + + if not resolve: + return ( + tuple( + DiscoveredTask.Mode(physics=p, renderer=r, presets=d) + for p in physics_options + for r in renderer_options + for d in domain_options + ), + None, + ) + + combinations = [(p, r, d) for p in physics_options for r in renderer_options for d in domain_options] + # The no-token run always has to be probed, and is not always in the cross product: + # a task declaring physics presets has no ``physics=None`` column. + if (None, None, None) not in combinations: + combinations.insert(0, (None, None, None)) + + # Keyed by fingerprint, holding the most explicit spelling seen of each run -- naming + # the presets reproduces it even if the config's own defaults move later. Built even + # when uncollapsed, since it identifies the spelling of the no-token run. + unique: dict[str, tuple[int, DiscoveredTask.Mode]] = {} + validated: list[DiscoveredTask.Mode] = [] + default_key: str | None = None + default_backend: str | None = None + for physics_name, renderer, domain in combinations: + try: + resolution = _mode_resolves(task_id, physics_name, renderer, domain) + except DiscoveryError as exc: + if strict: + raise + # Unknown is not the same answer as legal, so drop it -- but loudly, + # because it is a gap in the matrix. + logger.warning( + "%s: dropping physics=%s renderer=%s presets=%s: %s", task_id, physics_name, renderer, domain, exc + ) + continue + if resolution is None: + continue + key, backend = resolution + mode = DiscoveredTask.Mode(physics=physics_name, renderer=renderer, presets=domain) + validated.append(mode) + if (physics_name, renderer, domain) == (None, None, None): + default_key, default_backend = key, backend + explicitness = sum(token is not None for token in (physics_name, renderer, domain)) + incumbent = unique.get(key) + if incumbent is None or explicitness > incumbent[0]: + unique[key] = (explicitness, mode) + + default = None + if default_key is not None: + default = DiscoveredTask.Default(backend=default_backend, mode=unique[default_key][1]) + modes = tuple(mode for _, mode in unique.values()) if collapse else tuple(validated) + return modes, default + + +def discover_tasks( + specs: list[Any] | None = None, *, resolve: bool = True, strict: bool = False, collapse: bool = True +) -> list[DiscoveredTask]: + """Walk the Gym registry and return every registered training task. + + Imports Isaac Lab, so it needs the project environment. ``isaaclab_tasks`` registers + the core and contrib tasks; ``isaaclab_tasks_experimental`` is imported too when + present, for whatever it registers. + + Args: + specs: Gym specs to walk. When ``None``, the whole registry is scanned. + resolve: Build every combination and keep only what the validator accepts. + When ``False``, report what is declared: fast, unverified. + strict: Raise on a combination the validator cannot judge instead of logging + and dropping it. Off by default, so the task is still returned with that + combination missing from ``modes``. + collapse: Reduce spellings resolving to the same config to one. Off keeps every + validated spelling. Ignored when ``resolve`` is off. + + Returns: + Discovered tasks sorted by ``task_id``. + + Raises: + DiscoveryError: If the task packages cannot be imported, or, when ``strict``, if + any task could not be inspected. + """ + import contextlib + + try: + import gymnasium as gym + + import isaaclab_tasks # noqa: F401 + from isaaclab_tasks.utils.preset_cli import enumerate_task_presets + from isaaclab_tasks.utils.preset_target import PresetTarget + except ImportError as exc: + raise DiscoveryError(f"could not import the Isaac Lab task packages: {exc}") from exc + + with contextlib.suppress(ImportError): + import isaaclab_tasks_experimental # noqa: F401 + + if specs is None: + specs = list(gym.registry.values()) + + tasks: list[DiscoveredTask] = [] + for spec in specs: + if not is_training_task(spec.id) or spec.kwargs.get("deprecated"): + continue + # Tasks with no RL entry point (IK, teleop, mimic) are still registered + # environments; callers needing a trainable task filter on ``rl_libraries``. + preset_map = enumerate_task_presets(spec.id) + declared_physics = tuple(sorted(preset_map.get(PresetTarget.PHYSICS, []))) if preset_map else () + renderers = tuple(sorted(preset_map.get(PresetTarget.RENDERER, []))) if preset_map else () + domains = ( + _domain_presets(preset_map.get(PresetTarget.DOMAIN, []), declared_physics + renderers) if preset_map else () + ) + if preset_map is None: + # Nothing is known about this task, and a cross product of nothing known is + # not a cross product of nothing declared -- do not invent a runnable mode. + modes, default = (), None + else: + modes, default = _build_modes( + spec.id, declared_physics, renderers, domains, resolve=resolve, strict=strict, collapse=collapse + ) + tasks.append( + DiscoveredTask( + task_id=spec.id, + scope="contrib" if spec.id.startswith("IsaacContrib-") else "core", + rl_libraries=_rl_libraries_from_kwargs(spec.kwargs), + declared=( + None + if preset_map is None + else { + "physics": declared_physics, + "renderer": renderers, + "presets": tuple(sorted(preset_map.get(PresetTarget.DOMAIN, []))), + } + ), + modes=modes, + default=default, + resolved=resolve, + ) + ) + tasks.sort(key=lambda task: task.task_id) + return tasks diff --git a/tools/test/test_task_discovery.py b/tools/test/test_task_discovery.py new file mode 100644 index 000000000000..f4aa6f402c52 --- /dev/null +++ b/tools/test/test_task_discovery.py @@ -0,0 +1,236 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for the parts of task discovery that need no Isaac Lab. + +Resolution is stubbed here so the module stays importable on pytest alone; the real +resolver is covered by ``test_task_discovery_resolve.py``. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + + +def _bootstrap_paths() -> None: + """Prepend ``tools/`` so the module imports the same way the tool does.""" + tools_dir = Path(__file__).resolve().parents[1] + if str(tools_dir) not in sys.path: + sys.path.insert(0, str(tools_dir)) + + +_bootstrap_paths() + +import task_discovery # noqa: E402 +from task_discovery import ( # noqa: E402 + DiscoveredTask, + DiscoveryError, + _build_modes, + _domain_presets, + _rl_libraries_from_kwargs, + is_training_task, +) + +Mode = DiscoveredTask.Mode +Default = DiscoveredTask.Default + + +@pytest.fixture +def resolver(monkeypatch): + """Install a ``_mode_resolves`` driven by a ``combination -> (fingerprint, backend)`` table. + + A combination absent from the table resolves to ``None``, i.e. cannot run. + """ + + def install(runs): + monkeypatch.setattr( + task_discovery, + "_mode_resolves", + lambda task, physics, renderer, presets=None: runs.get((physics, renderer, presets)), + ) + + return install + + +@pytest.fixture +def broken_validator(monkeypatch): + """Install a ``_mode_resolves`` that fails structurally on one physics token.""" + + def install(backend): + def fake(task, physics, renderer, presets=None): + if physics == backend: + raise DiscoveryError("AttributeError: no attribute 'solver_cfg'") + return ("fp", "PhysxCfg") + + monkeypatch.setattr(task_discovery, "_mode_resolves", fake) + + return install + + +@pytest.mark.parametrize( + ("kwargs", "expected"), + [ + ({"rsl_rl_cfg_entry_point": "x"}, ("rsl_rl",)), + # Variants belong to their library; exact-name matching would drop every + # recurrent, distillation and per-terrain config. + ({"rsl_rl_recurrent_cfg_entry_point": "x"}, ("rsl_rl",)), + ({"skrl_flat_ppo_cfg_entry_point": "x"}, ("skrl",)), + # Ordering follows RL_LIBRARY_PRIORITY, not registration order. + ({"skrl_cfg_entry_point": "x", "rsl_rl_cfg_entry_point": "x"}, ("rsl_rl", "skrl")), + # The env config is not an agent config. + ({"env_cfg_entry_point": "x"}, ()), + ], +) +def test_rl_libraries_are_read_from_entry_point_stems(kwargs: dict, expected: tuple[str, ...]) -> None: + assert _rl_libraries_from_kwargs(kwargs) == expected + + +@pytest.mark.parametrize( + ("task_id", "expected"), + [ + ("Isaac-Ant", True), + ("IsaacContrib-Walk", True), + ("Isaac-Ant-Eval", False), + ("Isaac-Benchmark-Cartpole", False), + ("Some-Other-Env", False), + ], +) +def test_only_trainable_isaac_tasks_are_walked(task_id: str, expected: bool) -> None: + assert is_training_task(task_id) is expected + + +@pytest.mark.parametrize( + ("names", "typed", "expected"), + [ + # Declared on a typed axis too, so reachable as ``physics=``/``renderer=`` and + # reporting them again as ``presets=`` would double-count the same run. + (["rgb", "ovphysx", "depth", "ovrtx"], ("ovphysx", "ovrtx"), ("depth", "rgb")), + # Isaac-Open-Drawer-Franka: backends bucket under DOMAIN because their cfg + # classes do not subclass PhysicsCfg, so ``presets=`` is the only way to select + # them. Dropping them by name left the task reporting one empty mode. + (["newton_mjwarp", "ovphysx", "physx"], (), ("newton_mjwarp", "ovphysx", "physx")), + ], +) +def test_domain_presets_drop_only_what_a_typed_axis_already_offers(names, typed, expected) -> None: + assert _domain_presets(names, typed) == expected + + +@pytest.mark.parametrize( + ("physics", "renderers", "domains", "expected"), + [ + # Renderers are expanded across: a camera task reported headless-only would omit + # the thing under test. + ( + ("physx", "newton_mjwarp"), + ("ovrtx",), + (), + (Mode("physx", "ovrtx", None), Mode("newton_mjwarp", "ovrtx", None)), + ), + # Declaring nothing still leaves one way to run. + ((), (), (), (Mode(None, None, None),)), + # Domain presets go one at a time, never combined; ``None`` keeps the task's own + # default reachable beside them. + ((), (), ("rgb", "depth"), (Mode(None, None, None), Mode(None, None, "rgb"), Mode(None, None, "depth"))), + ], +) +def test_declared_modes_are_the_raw_cross_product(physics, renderers, domains, expected) -> None: + # Nothing has been resolved, so nothing can be collapsed and there is no default. + assert _build_modes("Isaac-X", physics, renderers, domains, resolve=False) == (expected, None) + + +def test_spellings_that_resolve_to_the_same_run_collapse_to_one_mode(resolver) -> None: + # Isaac-Open-Drawer-Franka in miniature: passing nothing lands on the same config as + # naming the preset the task already defaults to, and ``physx`` lands on the same + # config as the concrete backend it aliases. + resolver( + { + (None, None, None): ("fp-physx", "PhysxCfg"), + (None, None, "isaacsim_physx"): ("fp-physx", "PhysxCfg"), + (None, None, "ovphysx"): ("fp-ov", "OvPhysxCfg"), + (None, None, "physx"): ("fp-ov", "OvPhysxCfg"), + (None, None, "newton_mjwarp"): ("fp-newton", "NewtonCfg(MJWarpSolverCfg)"), + } + ) + + modes, default = _build_modes( + "Isaac-X", (), (), ("isaacsim_physx", "newton_mjwarp", "ovphysx", "physx"), resolve=True + ) + + # Five spellings, three distinct runs. + assert modes == ( + Mode(None, None, "isaacsim_physx"), + Mode(None, None, "newton_mjwarp"), + Mode(None, None, "ovphysx"), + ) + assert default == Default(backend="PhysxCfg", mode=Mode(None, None, "isaacsim_physx")) + + +def test_runs_sharing_a_backend_but_not_a_config_stay_separate(resolver) -> None: + # Isaac-Reach-Franka's controller presets all run on Newton MJWarp and are different + # runs, so collapsing on the backend instead of the config would merge them. + resolver( + { + (None, None, None): ("fp-joint", "NewtonCfg(MJWarpSolverCfg)"), + (None, None, "joint_pos"): ("fp-joint", "NewtonCfg(MJWarpSolverCfg)"), + (None, None, "diffik"): ("fp-diffik", "NewtonCfg(MJWarpSolverCfg)"), + } + ) + + modes, default = _build_modes("Isaac-X", (), (), ("joint_pos", "diffik"), resolve=True) + + assert modes == (Mode(None, None, "joint_pos"), Mode(None, None, "diffik")) + assert default.mode == Mode(None, None, "joint_pos") + + +def test_a_default_matching_no_named_preset_survives_as_its_own_mode(resolver) -> None: + resolver({(None, None, None): ("fp-own", "PhysxCfg"), (None, None, "rgb"): ("fp-rgb", "PhysxCfg")}) + + modes, default = _build_modes("Isaac-X", (), (), ("rgb",), resolve=True) + + assert modes == (Mode(None, None, None), Mode(None, None, "rgb")) + assert default == Default(backend="PhysxCfg", mode=Mode(None, None, None)) + + +def test_uncollapsed_keeps_every_spelling_but_still_drops_rejections(resolver) -> None: + # ``shapes`` names the default of its own axis, so it resolves to the default config + # and the collapse drops it -- but it is still a token a reader can type, which is + # what the documentation view needs. ``raycaster`` is absent from the table, so it + # does not resolve and must be dropped either way. + resolver( + { + (None, None, None): ("fp-default", "PhysxCfg"), + (None, None, "shapes"): ("fp-default", "PhysxCfg"), + (None, None, "cube"): ("fp-cube", "PhysxCfg"), + } + ) + domains = ("shapes", "cube", "raycaster") + + collapsed, _ = _build_modes("Isaac-X", (), (), domains, resolve=True) + every, default = _build_modes("Isaac-X", (), (), domains, resolve=True, collapse=False) + + assert collapsed == (Mode(None, None, "shapes"), Mode(None, None, "cube")) + assert every == (Mode(None, None, None), Mode(None, None, "shapes"), Mode(None, None, "cube")) + assert default == Default(backend="PhysxCfg", mode=Mode(None, None, "shapes")) + + +def test_a_combination_the_validator_cannot_judge_is_dropped_and_the_walk_continues(broken_validator) -> None: + # One task breaking the validator must not cost the caller the rest of the registry. + # Unknown is not legal either, so the combination is dropped rather than reported. + broken_validator("newton_mjwarp") + + modes, _ = _build_modes("Isaac-X", ("physx", "newton_mjwarp"), (), (), resolve=True) + + assert modes == (Mode("physx", None, None),) + + +def test_strict_raises_on_a_structural_failure_instead_of_dropping_it(broken_validator) -> None: + # Callers policing Isaac Lab API drift want the canary, not a survivable walk. + broken_validator("newton_mjwarp") + + with pytest.raises(DiscoveryError): + _build_modes("Isaac-X", ("physx", "newton_mjwarp"), (), (), resolve=True, strict=True) diff --git a/tools/test/test_task_discovery_resolve.py b/tools/test/test_task_discovery_resolve.py new file mode 100644 index 000000000000..eda584bd6d26 --- /dev/null +++ b/tools/test/test_task_discovery_resolve.py @@ -0,0 +1,140 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for task discovery against real configs. + +Kept apart from ``test_task_discovery.py`` so that file stays importable with nothing +but pytest. Everything here needs Isaac Lab and resolves real task configs, which costs +a few seconds of warm-up and roughly 0.1s per combination. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + + +def _bootstrap_paths() -> None: + """Prepend ``tools/`` and the editable ``source/*`` packages.""" + repo_root = Path(__file__).resolve().parents[2] + prepend = [str(repo_root / "tools")] + for package_dir in sorted((repo_root / "source").iterdir()): + if (package_dir / package_dir.name).is_dir(): + prepend.append(str(package_dir)) + for path in reversed(prepend): + if path not in sys.path: + sys.path.insert(0, path) + + +_bootstrap_paths() + +pytest.importorskip("isaaclab_tasks", reason="task discovery resolution needs Isaac Lab") + +import task_discovery # noqa: E402 +from task_discovery import DiscoveryError, _mode_resolves # noqa: E402 + + +def _raise(exc: BaseException): + """Return a ``resolve_task_config`` stand-in that raises *exc*.""" + + def fake(*args, **kwargs): + raise exc + + return fake + + +def test_a_kit_backed_physics_and_a_kitless_renderer_are_rejected() -> None: + """The pairing that justifies resolving at all must actually come back rejected. + + OVRTX is kitless and cannot share a process with Isaac Sim PhysX. If the validator + ever starts accepting this, every other test here still passes. + """ + assert _mode_resolves("Isaac-Cartpole-Camera", "isaacsim_physx", "ovrtx", None) is None + assert _mode_resolves("Isaac-Cartpole-Camera", "ovphysx", "isaacsim_rtx", None) is None + assert _mode_resolves("Isaac-Cartpole-Camera", "isaacsim_physx", "isaacsim_rtx", None) is not None + assert _mode_resolves("Isaac-Cartpole-Camera", "ovphysx", "ovrtx", None) is not None + + +def test_a_cross_axis_rule_the_task_declares_is_enforced() -> None: + """Tasks put cross-axis rules in ``validate_config``, which the env constructor runs. + + Reach rejects ``newton_ik`` without Newton physics. Resolving without validating + reported those pairings legal, and they died on the first reset instead. + """ + assert _mode_resolves("Isaac-Reach-Franka", "isaacsim_physx", None, "newton_ik") is None + assert _mode_resolves("Isaac-Reach-Franka", "ovphysx", None, "newton_ik") is None + assert _mode_resolves("Isaac-Reach-Franka", "newton_mjwarp", None, "newton_ik") is not None + + +def test_the_fingerprint_identifies_a_run() -> None: + """The collapse is only as good as this: stable per run, distinct across runs. + + ``to_dict`` erases class identity, so backends differ only by the values it keeps. + Were that discriminator dropped upstream, two backends would merge into one mode + and a dispatcher would silently stop scheduling one of them. + """ + once = _mode_resolves("Isaac-Cartpole", "newton_mjwarp", None, None) + again = _mode_resolves("Isaac-Cartpole", "newton_mjwarp", None, None) + other = _mode_resolves("Isaac-Cartpole", "newton_kamino", None, None) + + assert once is not None and other is not None + assert once == again, "an unstable fingerprint splits one run across several modes" + assert once[1] != other[1], "different backends" + assert once[0] != other[0], "...so they must not share a fingerprint" + + +def test_an_alias_collapses_onto_the_backend_it_resolves_to() -> None: + """``physics=physx`` is an alias, so it must share a fingerprint with its target.""" + assert _mode_resolves("Isaac-Cartpole", "physx", None, None) == _mode_resolves( + "Isaac-Cartpole", "ovphysx", None, None + ) + + +@pytest.mark.parametrize( + ("raised", "expectation"), + [ + # A config needing an uninstalled extra cannot run here -- same answer as a + # rejection, which is what keeps discovery usable from a partial install. + (ModuleNotFoundError("No module named 'isaaclab_absent_extra'"), "rejected"), + # A module that imports but no longer exports what it should is API drift. + (ImportError("cannot import name 'gone'"), "raises"), + (AttributeError("'NoneType' object has no attribute 'solver_cfg'"), "raises"), + # Anything else means the combination cannot run. + (ValueError("Invalid backend combination"), "rejected"), + ], +) +def test_failures_are_sorted_into_rejection_or_api_drift(monkeypatch, raised, expectation) -> None: + import isaaclab_tasks.utils as utils + + monkeypatch.setattr(utils, "resolve_task_config", _raise(raised)) + # ``discover_tasks`` runs in-process from a CLI tool, so an argv leak on any of + # these paths would corrupt the caller. + before = list(sys.argv) + + if expectation == "raises": + with pytest.raises(DiscoveryError): + _mode_resolves("Isaac-Cartpole", "ovphysx", None, "rgb") + else: + assert _mode_resolves("Isaac-Cartpole", "ovphysx", None, "rgb") is None + + assert sys.argv == before + + +def test_an_unloadable_config_reports_unknown_and_no_modes(monkeypatch) -> None: + """``declared is None`` must not sit beside a mode claiming the task runs.""" + import gymnasium as gym + + import isaaclab_tasks.utils.preset_cli as preset_cli + + monkeypatch.setattr(preset_cli, "enumerate_task_presets", lambda task_name: None) + spec = next(spec for spec in gym.registry.values() if spec.id == "Isaac-Cartpole") + + task = task_discovery.discover_tasks([spec], resolve=False)[0] + + assert task.declared is None + assert task.modes == () + assert task.default is None