Make pyproject.toml the single definition for lint tool versions and settings

.github/workflows/cicd_tests.yml

@@ -52,6 +52,34 @@ env:
 # When support is dropped for a version it is important to update these as appropriate.
 
 jobs:
+  pre-commit:  # Run the hooks pre-commit.ci skips, using the tools pyproject.toml pins
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+    # This job executes the hooks named by a PR's own .pre-commit-config.yaml, so
+    # it must not leave a usable token in .git/config for those hooks to reach.
+    - uses: actions/checkout@v7
+      with:
+        persist-credentials: false
+    # reads the pins below with tomllib, which needs 3.11+; the hooks take their
+    # target from pyproject.toml, so the interpreter version does not affect them
+    - name: Set up Python ${{ env.PYTHON_VER3 }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ env.PYTHON_VER3 }}
+        cache: 'pip'
+    - name: Install lint tools
+      run: |
+        # the lint extra alone, so this job needs neither torch nor the optional dependencies
+        python -m pip install --upgrade pip
+        python -c "import tomllib; print('\n'.join(tomllib.load(open('pyproject.toml','rb'))['project']['optional-dependencies']['lint']))" > lint-requirements.txt
+        cat lint-requirements.txt
+        python -m pip install -r lint-requirements.txt
+        rm lint-requirements.txt
+    - name: Run pre-commit
+      run: python -m pre_commit run --all-files --show-diff-on-failure
+
   static-checks:  # Perform static type and other checks using runtests.sh
     runs-on: ubuntu-latest
     strategy:

.pre-commit-config.yaml

@@ -6,6 +6,9 @@ ci:
   autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
   autoupdate_schedule: quarterly
   # submodules: true
+  # The hooks below run the tools from the environment pyproject.toml defines, which
+  # pre-commit.ci does not build; the pre-commit job in cicd_tests.yml runs them instead.
+  skip: [black, isort, pycln, ruff-check]
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -26,19 +29,33 @@ repos:
         args: ['--autofix', '--no-sort-keys', '--indent=4']
       - id: end-of-file-fixer
       - id: mixed-line-ending
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.20
-    hooks:
-    -   id: ruff-check
-        args: ["--fix"]
-        exclude: |
-          (?x)(
-              ^versioneer.py|
-              ^monai/_version.py
-          )
 
-  - repo: https://github.com/hadialqattan/pycln
-    rev: v2.6.0
+  # Versions come from [project.optional-dependencies].lint and settings from the
+  # [tool.*] tables, so these hooks and runtests.sh cannot resolve a different tool.
+  # The --force-exclude/--filter-files flags apply those settings to the explicit
+  # filenames pre-commit passes.
+  - repo: local
     hooks:
+      - id: ruff-check
+        name: ruff check
+        entry: python -m ruff check --force-exclude --fix
+        language: system
+        types_or: [python, pyi]
+
+      - id: isort
+        name: isort
+        entry: python -m isort --filter-files
+        language: system
+        types_or: [python, pyi]
+
+      - id: black
+        name: black
+        entry: python -m black
+        language: system
+        types_or: [python, pyi]
+
       - id: pycln
-        args: [--config=pyproject.toml]
+        name: pycln
+        entry: python -m pycln --config=pyproject.toml
+        language: system
+        types_or: [python, pyi]

monai/config/print_dependencies.py

@@ -18,17 +18,73 @@
 
 from __future__ import annotations
 
+import re
 import sys
 from collections.abc import Collection
 
 BUILD_SYSTEM_KEY = "build-system"
 PROJ_KEY = "project"
 OPTS_KEY = "optional-dependencies"
 DEP_KEY = "dependencies"
+NAME_KEY = "name"
 REQ_KEY = "requires"
 TOML_FILE = "pyproject.toml"
 
 
+# a requirement of the form "<name>[<extra>,...]" and nothing else; a version specifier or an
+# environment marker after the brackets makes it something other than a bare self-reference
+SELF_REF_RE = re.compile(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\s*\[([^\]]+)\]$")
+
+
+def _normalize_name(name: str) -> str:
+    """
+    Normalize a project or extra name so case and ``-``/``_``/``.`` spellings compare equal.
+
+    PEP 503 (project names) and PEP 685 (extra names) specify the same rule, so one function
+    serves both sides of the comparison.
+    """
+    return re.sub(r"[-_.]+", "-", name.strip()).lower()
+
+
+def _expand_self_extras(dependencies: list[str], name: str, opts: dict) -> list[str]:
+    """
+    Replace self-referential requirements such as ``monai[lint]`` with the contents of that group.
+
+    pip resolves such a requirement against the package index, so leaving one in a generated
+    requirements file installs the published release instead of the checkout being worked on.
+
+    Args:
+        dependencies: requirement strings, some of which may be self-references.
+        name: this project's name, the only one treated as a self-reference.
+        opts: the "optional-dependencies" table the groups are read from.
+
+    Returns:
+        List of requirements with every self-reference replaced by the group it names.
+
+    Raises:
+        KeyError: If a self-reference names a group absent from `opts`.
+    """
+    self_name = _normalize_name(name)
+    groups = {_normalize_name(key): value for key, value in opts.items()}
+    expanded: list[str] = []
+    pending = list(dependencies)
+    seen: set[str] = set()
+
+    while pending:
+        req = pending.pop(0)
+        match = SELF_REF_RE.match(req.strip())
+        if match is None or _normalize_name(match.group(1)) != self_name:
+            expanded.append(req)
+            continue
+        for group in (_normalize_name(g) for g in match.group(2).split(",")):
+            if group in seen:  # a group already expanded, or a cycle
+                continue
+            seen.add(group)
+            pending.extend(groups[group])
+
+    return expanded
+
+
 def parse_dependencies(filename: str | None = None, sections: Collection[str] | None = None) -> list[str]:
     """
     Parse the toml file given by `filename` and return the dependency sections selected by `sections`.
@@ -68,6 +124,8 @@ def parse_dependencies(filename: str | None = None, sections: Collection[str] |
         for s in sections:
             dependencies += opts[s]
 
+    dependencies = _expand_self_extras(dependencies, proj[NAME_KEY], opts)
+
     return sorted(set(dependencies))
 
 

pyproject.toml

@@ -157,20 +157,25 @@ torchvision = ["torchvision"]
 tqdm = ["tqdm>=4.47.0"]
 transformers = ["transformers>=5.5.0"]  # 5.x needs the transchex BertLayer/BertConfig updates; re-verify the NGC image float8 concern
 zarr = ["zarr"]
+# the tools .pre-commit-config.yaml and runtests.sh invoke; installable without torch
+lint = [
+  "black>=26.3.1",
+  "isort>=5.1,<6,!=6.0.0",
+  "pre-commit",
+  "pycln==2.6.0",
+  "ruff==0.16.4"
+]
 # these dependencies are for testing/building only, they aren't needed for regular use so don't appear in "all"
 testing = [
-  "black>=26.3.1",
+  "monai[lint]",
   "coverage>=5.5",
-  "isort>=5.1,<6,!=6.0.0",
   "mccabe",
   "packaging",
   "parameterized",
   "pep8-naming",
-  "pre-commit",
   "pycodestyle",
   "pyflakes",
   "pyrefly>=1.0.0",
-  "ruff>=0.14.11,<0.15",
   "tomli",  # used in print_dependencies.py for Python<3.11
   "typeguard<3",  # https://github.com/microsoft/nni/issues/5457
   "types-PyYAML",
@@ -266,7 +271,9 @@ line-length = 120
 target-version = ['py310']
 skip-magic-trailing-comma = true
 include = '\.pyi?$'
-exclude = '''
+# force-exclude, not exclude: black ignores exclude for filenames passed explicitly,
+# which is how pre-commit invokes it.
+force-exclude = '''
 (
   /(
     # exclude a few common directories in the root of the project
@@ -296,6 +303,8 @@ exclude = "monai/bundle/__main__.py"
 [tool.ruff]
 line-length = 120
 target-version = "py310"
+# Vendored/generated; matches [tool.black] exclude and [tool.pyrefly] project-excludes.
+extend-exclude = ["versioneer.py", "monai/_version.py"]
 
 [tool.ruff.lint]
 select = [

runtests.sh

@@ -595,13 +595,13 @@ then
     then
         install_deps
     fi
-    ruff --version
+    ${cmdPrefix}"${PY_EXE}" -m ruff --version
 
     if [ $doRuffFix = true ]
     then
-        ruff check --fix --unsafe-fixes --exclude versioneer.py --exclude "monai/_version.py" "$homedir"
+        ${cmdPrefix}"${PY_EXE}" -m ruff check --fix --unsafe-fixes "$homedir"
     else
-        ruff check --exclude versioneer.py --exclude "monai/_version.py" "$homedir"
+        ${cmdPrefix}"${PY_EXE}" -m ruff check "$homedir"
     fi
 
     ruff_status=$?

tests/config/test_print_dependencies.py

@@ -32,14 +32,52 @@
 
 [project.optional-dependencies]
 all = ["something", "another"]
-testing = ["coverage", "black"]
+lint = ["ruff", "black"]
+testing = ["test[lint]", "coverage"]
+cyclic = ["test[cyclic]", "spam"]
+spell_check = ["eggs"]
+mixed = ["Test[Spell.Check]", "ham"]
+other_pkg = ["versioneer[toml]", "bacon"]
+"""
+
+# a project whose name needs normalizing before a self-reference spelled differently can match it
+PUNCTUATED_NAME_TOML = """
+[project]
+name = "My_Project"
+dependencies = ["torch"]
+
+[project.optional-dependencies]
+lint = ["ruff"]
+testing = ["my-project[lint]", "coverage"]
 """
 
 PARSE_CASES = [
     ([], ["numpy", "torch"]),
-    (["testing"], ["black", "coverage", "numpy", "torch"]),
+    # "test[lint]" expands rather than reaching pip as a requirement on the published package
+    (["testing"], ["black", "coverage", "numpy", "ruff", "torch"]),
     (["build-system"], ["numpy", "setuptools", "torch", "wheel"]),
-    (["*"], ["another", "black", "coverage", "numpy", "something", "torch"]),
+    (
+        ["*"],
+        [
+            "another",
+            "bacon",
+            "black",
+            "coverage",
+            "eggs",
+            "ham",
+            "numpy",
+            "ruff",
+            "something",
+            "spam",
+            "torch",
+            "versioneer[toml]",
+        ],
+    ),
+    (["cyclic"], ["numpy", "spam", "torch"]),
+    # PEP 685: extra names compare equal across case and "-"/"_"/"." spellings
+    (["mixed"], ["eggs", "ham", "numpy", "torch"]),
+    # another project's extra is not a self-reference and passes through untouched
+    (["other_pkg"], ["bacon", "numpy", "torch", "versioneer[toml]"]),
 ]
 
 
@@ -57,6 +95,15 @@ def test_parse_dependencies(self, sections, outputs):
         deps = parse_dependencies(self.toml.name, sections)
         self.assertEqual(outputs, deps)
 
+    def test_self_reference_spelled_differently(self):
+        """PEP 503: "my-project[lint]" is a self-reference to a project named "My_Project"."""
+        toml = NamedTemporaryFile("w", delete=False)
+        toml.write(PUNCTUATED_NAME_TOML)
+        toml.close()
+        self.addCleanup(os.unlink, toml.name)
+
+        self.assertEqual(["coverage", "ruff", "torch"], parse_dependencies(toml.name, ["testing"]))
+
     def test_missing_section(self):
         with self.assertRaises(KeyError):
             parse_dependencies(self.toml.name, ["nonexistent_section"])