Merge branch 'dev' into weekly_preview_fix

Author: ericspod

docs/requirements.txt

@@ -20,8 +20,8 @@ sphinxcontrib-serializinghtml
 sphinx-autodoc-typehints==1.11.1
 pandas
 einops
-transformers>=4.53.0
-mlflow>=2.12.2,<3.13
+transformers>=5.5.0
+mlflow>=3.15.2
 clearml>=1.10.0rc0
 tensorboardX
 imagecodecs; platform_system == "Linux" or platform_system == "Darwin"

monai/bundle/scripts.py

@@ -997,7 +997,8 @@ def run(
             common parameters shown below will be added and can be passed through the `override` parameter of this method.
 
             - ``"output_dir"``: the path to save mlflow tracking outputs locally, default to "<bundle root>/eval".
-            - ``"tracking_uri"``: uri to save mlflow tracking outputs, default to "/output_dir/mlruns".
+            - ``"tracking_uri"``: uri to save mlflow tracking outputs, default to a local SQLite database
+              at "<output_dir>/mlruns.db" with run artifacts kept under "<output_dir>/mlruns".
             - ``"experiment_name"``: experiment name for this run, default to "monai_experiment".
             - ``"run_name"``: the name of current run.
             - ``"save_execute_config"``: whether to save the executed config files. It can be `False`, `/path/to/artifacts`

monai/bundle/utils.py

@@ -116,8 +116,10 @@
     "configs": {
         # if no "output_dir" in the bundle config, default to "<bundle root>/eval"
         "output_dir": "$@bundle_root + '/eval'",
-        # use URI to support linux, mac and windows os
-        "tracking_uri": "$monai.utils.path_to_uri(@output_dir) + '/mlruns'",
+        # MLflow 3.13+ rejects the filesystem (file store) tracking backend, so default tracking
+        # to a local SQLite database. The handler keeps run artifacts under "<output_dir>/mlruns"
+        # (next to the db). A URI is used so the path is valid on linux, mac and windows os.
+        "tracking_uri": "$monai.utils.path_to_sqlite_uri(@output_dir + '/mlruns.db')",
         "experiment_name": "monai_experiment",
         "run_name": None,
         # may fill it at runtime

monai/handlers/mlflow_handler.py

@@ -22,7 +22,16 @@
 from torch.utils.data import Dataset
 
 from monai.apps.utils import get_logger
-from monai.utils import CommonKeys, IgniteInfo, ensure_tuple, flatten_dict, min_version, optional_import
+from monai.utils import (
+    CommonKeys,
+    IgniteInfo,
+    ensure_tuple,
+    flatten_dict,
+    min_version,
+    optional_import,
+    path_to_sqlite_uri,
+    path_to_uri,
+)
 
 Events, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Events")
 mlflow, _ = optional_import("mlflow", descriptor="Please install mlflow before using MLFlowHandler.")
@@ -68,7 +77,16 @@ class MLFlowHandler:
         tracking_uri: connects to a tracking URI. can also set the `MLFLOW_TRACKING_URI` environment
             variable to have MLflow find a URI from there. in both cases, the URI can either be
             an HTTP/HTTPS URI for a remote server, a database connection string, or a local path
-            to log data to a directory. The URI defaults to path `mlruns`.
+            to log data to a directory. When no ``tracking_uri`` is provided and the
+            ``MLFLOW_TRACKING_URI`` environment variable is unset, the handler now
+            defaults to a local SQLite database backend at ``sqlite:///<cwd>/mlruns.db`` with
+            artifacts stored under ``<cwd>/mlruns``. The default was changed from the filesystem
+            (file store) backend because MLflow 3.13+ raises an exception for the file store unless
+            ``MLFLOW_ALLOW_FILE_STORE=true`` is set; SQLite is the backend MLflow recommends and it
+            does not raise. Any explicitly provided ``tracking_uri`` is passed through unchanged
+            unless ``MLFLOW_TRACKING_URI`` is set (which takes precedence); local file paths and
+            ``file://`` URIs are rejected because MLflow no longer supports the filesystem (file
+            store) tracking backend.
             for more details: https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.set_tracking_uri.
         iteration_log: whether to log data to MLFlow when iteration completed, default to `True`.
             ``iteration_log`` can be also a function and it will be interpreted as an event filter
@@ -113,6 +131,11 @@ class MLFlowHandler:
         optimizer_param_names: parameter names in the optimizer that need to be recorded during running the
             workflow, default to `'lr'`.
         close_on_complete: whether to close the mlflow run in `complete` phase in workflow, default to False.
+        artifact_location: the location to store run artifacts in, passed to MLflow when the experiment is
+            created. When ``None`` and a local SQLite backend is used (from the ``tracking_uri`` argument
+            or the ``MLFLOW_TRACKING_URI`` environment variable), it defaults to an ``mlruns`` directory
+            next to the database file; for other backends ``None`` lets MLflow decide based on the
+            ``tracking_uri``. Has no effect if the experiment already exists.
 
     For more details of MLFlow usage, please refer to: https://mlflow.org/docs/latest/index.html.
 
@@ -141,6 +164,7 @@ def __init__(
         artifacts: str | Sequence[Path] | None = None,
         optimizer_param_names: str | Sequence[str] = "lr",
         close_on_complete: bool = False,
+        artifact_location: str | None = None,
     ) -> None:
         self.iteration_log = iteration_log
         self.epoch_log = epoch_log
@@ -156,7 +180,39 @@ def __init__(
         self.experiment_param = experiment_param
         self.artifacts = ensure_tuple(artifacts)
         self.optimizer_param_names = ensure_tuple(optimizer_param_names)
-        self.client = mlflow.MlflowClient(tracking_uri=tracking_uri if tracking_uri else None)
+        # When no tracking_uri is provided, default to a local SQLite backend instead of the
+        # filesystem (file store) backend. MLflow 3.13+ raises for the file store unless
+        # `MLFLOW_ALLOW_FILE_STORE=true` is set, while SQLite is the recommended backend and does
+        # not raise. Artifacts cannot live inside a database, so by default they are stored under
+        # the `./mlruns` directory (where the previous file store default kept them) via the
+        # experiment `artifact_location`. Any explicitly provided tracking_uri is left unchanged.
+        self.artifact_location = artifact_location
+        # Resolve the effective tracking URI. The `MLFLOW_TRACKING_URI` environment variable takes
+        # priority so it can override a hard-coded `tracking_uri` argument; both configure the
+        # artifact location the same way.
+        env_tracking_uri = os.environ.get("MLFLOW_TRACKING_URI")
+        effective_tracking_uri = env_tracking_uri or tracking_uri
+        # When neither is set, fall back to the local SQLite default described above.
+        if not effective_tracking_uri:
+            tracking_uri = effective_tracking_uri = path_to_sqlite_uri(os.path.join(os.getcwd(), "mlruns.db"))
+        # For a local SQLite backend, keep run artifacts in an `mlruns` directory next to the
+        # database file (mirroring the previous file-store layout) unless the caller set
+        # `artifact_location`. Other backends (e.g. a remote server) are left to MLflow to decide.
+        if self.artifact_location is None and effective_tracking_uri.startswith("sqlite:///"):
+            db_path = Path(effective_tracking_uri[len("sqlite:///") :])
+            self.artifact_location = path_to_uri(db_path.parent / "mlruns")
+        # MLflow 3.13+ refuses the filesystem (file store) tracking backend, and 3.14+ resolves
+        # the store eagerly at client construction, so a local path or ``file://`` URI would raise
+        # an opaque MlflowException. Reject those here with an actionable message instead.
+        if effective_tracking_uri.startswith("file://") or "://" not in effective_tracking_uri:
+            raise ValueError(
+                "MLflow no longer supports the filesystem (file store) tracking backend; got "
+                f"tracking_uri={effective_tracking_uri!r}. Use a SQLite URI "
+                "(sqlite:///<path>/mlruns.db) or a remote tracking URI instead."
+            )
+        # Only the argument is passed to the client; when `MLFLOW_TRACKING_URI` took priority it
+        # is left None so MLflow resolves the environment variable itself.
+        self.client = mlflow.MlflowClient(tracking_uri=None if env_tracking_uri else tracking_uri)
         self.run_finish_status = mlflow.entities.RunStatus.to_string(mlflow.entities.RunStatus.FINISHED)
         self.close_on_complete = close_on_complete
         self.experiment = None
@@ -246,7 +302,12 @@ def _set_experiment(self):
                 try:
                     experiment = self.client.get_experiment_by_name(self.experiment_name)
                     if not experiment:
-                        experiment_id = self.client.create_experiment(self.experiment_name)
+                        # pass an explicit artifact_location (set for the default SQLite backend, or
+                        # by the caller) so artifacts land in the intended directory; when it is
+                        # None MLflow decides based on the tracking_uri.
+                        experiment_id = self.client.create_experiment(
+                            self.experiment_name, artifact_location=self.artifact_location
+                        )
                         experiment = self.client.get_experiment(experiment_id)
                     break
                 except MlflowException as e:
@@ -338,14 +399,43 @@ def complete(self) -> None:
             for artifact in artifact_list:
                 self.client.log_artifact(self.cur_run.info.run_id, artifact)
 
+    def _dispose_sqlite_store(self) -> None:
+        """
+        Release MLflow's SQLAlchemy engine when a local SQLite tracking backend is used.
+
+        MLflow keeps the SQLite connection open for the lifetime of the client, which on
+        Windows prevents the database file from being deleted. MLflow exposes no public
+        client close/dispose API, so this reaches into its internals defensively to release
+        the engine. It is a no-op for non-SQLite backends.
+        """
+        tracking_uri = getattr(self.client, "tracking_uri", "")
+        if not isinstance(tracking_uri, str) or not tracking_uri.startswith("sqlite:"):
+            return
+        store = getattr(getattr(self.client, "_tracking_client", None), "store", None)
+        if store is None:
+            return
+        dispose = getattr(store, "_dispose_engine", None)
+        if callable(dispose):
+            dispose()
+        else:
+            engine = getattr(store, "engine", None)
+            if engine is not None:
+                engine.dispose()
+        read_engine = getattr(store, "read_engine", None)
+        if read_engine is not None:
+            read_engine.dispose()
+
     def close(self) -> None:
         """
-        Stop current running logger of MLFlow.
+        Stop current running logger of MLFlow and release local SQLite resources.
 
         """
-        if self.cur_run:
-            self.client.set_terminated(self.cur_run.info.run_id, self.run_finish_status)
-            self.cur_run = None
+        try:
+            if self.cur_run:
+                self.client.set_terminated(self.cur_run.info.run_id, self.run_finish_status)
+                self.cur_run = None
+        finally:
+            self._dispose_sqlite_store()
 
     def epoch_completed(self, engine: Engine) -> None:
         """

monai/networks/nets/transchex.py

@@ -23,6 +23,7 @@
 transformers = optional_import("transformers")
 load_tf_weights_in_bert = optional_import("transformers", name="load_tf_weights_in_bert")[0]
 cached_file = optional_import("transformers.utils", name="cached_file")[0]
+BertConfig = optional_import("transformers", name="BertConfig")[0]
 BertEmbeddings = optional_import("transformers.models.bert.modeling_bert", name="BertEmbeddings")[0]
 BertLayer = optional_import("transformers.models.bert.modeling_bert", name="BertLayer")[0]
 
@@ -222,7 +223,11 @@ def __init__(
 
         """
         super().__init__()
-        self.config = type("obj", (object,), bert_config)
+        self.config = BertConfig(**bert_config)
+        # explicitly select the eager attention path: transformers>=4.48 dispatches attention
+        # implementations via `config._attn_implementation`, which is otherwise left unset since
+        # `bert_config` above does not come from a `from_pretrained` call.
+        self.config._attn_implementation = "eager"
         self.embeddings = BertEmbeddings(self.config)
         self.language_encoder = nn.ModuleList([BertLayer(self.config) for _ in range(num_language_layers)])
         self.vision_encoder = nn.ModuleList([BertLayer(self.config) for _ in range(num_vision_layers)])

monai/utils/__init__.py

@@ -89,6 +89,7 @@
     is_sqrt,
     issequenceiterable,
     list_to_dict,
+    path_to_sqlite_uri,
     path_to_uri,
     pprint_edges,
     progress_bar,

monai/utils/misc.py

@@ -27,6 +27,7 @@
 from math import log10
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, TypeVar, cast, overload
+from urllib.parse import quote
 
 import numpy as np
 import torch
@@ -69,6 +70,7 @@
     "save_obj",
     "label_union",
     "path_to_uri",
+    "path_to_sqlite_uri",
     "pprint_edges",
     "check_key_duplicates",
     "CheckKeyDuplicatesYamlLoader",
@@ -727,6 +729,23 @@ def path_to_uri(path: PathLike) -> str:
     return Path(path).absolute().as_uri()
 
 
+def path_to_sqlite_uri(path: PathLike) -> str:
+    """
+    Convert a database file path to a SQLite connection URI, e.g. for use as an MLflow
+    ``tracking_uri``. If not an absolute path, it is converted to an absolute path first.
+
+    A forward-slash (POSIX) path is used so the URI is valid on Windows as well as POSIX:
+    on Windows this yields ``sqlite:///C:/path/db.sqlite`` and on POSIX ``sqlite:////path/db.sqlite``.
+    URI-special characters in the path (e.g. ``?``, ``#``) are percent-encoded so they are not
+    misparsed as query/fragment components by SQLAlchemy.
+
+    Args:
+        path: input database file path, can be a string or `Path` object.
+
+    """
+    return f"sqlite:///{quote(Path(path).absolute().as_posix(), safe='/:')}"
+
+
 def pprint_edges(val: Any, n_lines: int = 20) -> str:
     """
     Pretty print the head and tail ``n_lines`` of ``val``, and omit the middle part if the part has more than 3 lines.

pyproject.toml

@@ -1,7 +1,7 @@
 
 [build-system]
 requires = [
-  "setuptools",
+  "setuptools>=78.1.1",
   "wheel",
   "versioneer[toml]",
   "more-itertools>=8.0",
@@ -71,7 +71,7 @@ all = [
   "lpips==0.1.4",
   "matplotlib>=3.6.3",
   "MetricsReloaded @ git+https://github.com/Project-MONAI/MetricsReloaded@monai-support",
-  "mlflow>=2.12.2,<3.13",
+  "mlflow>=3.15.2",
   "nibabel",
   "ninja",
   "nni; platform_system == 'Linux' and 'arm' not in platform_machine and 'aarch' not in platform_machine",
@@ -103,7 +103,7 @@ all = [
   "torchio",
   "torchvision",
   "tqdm>=4.47.0",
-  "transformers>=4.53.0, <5.0",
+  "transformers>=5.5.0",
   "zarr"
 ]
 clearml = ["clearml>=1.10.0rc0"]
@@ -126,7 +126,7 @@ lmdb = ["lmdb"]
 lpips = ["lpips==0.1.4"]
 matplotlib = ["matplotlib>=3.6.3"]
 metrics_reloaded = ["MetricsReloaded @ git+https://github.com/Project-MONAI/MetricsReloaded@monai-support"]
-mlflow = ["mlflow>=2.12.2,<3.13"]
+mlflow = ["mlflow>=3.15.2"]
 nibabel = ["nibabel"]
 nni = [
   "nni; platform_system == 'Linux' and 'arm' not in platform_machine and 'aarch' not in platform_machine",
@@ -155,7 +155,7 @@ tifffile = ["tifffile; platform_system == 'Linux' or platform_system == 'Darwin'
 torchio = ["torchio"]
 torchvision = ["torchvision"]
 tqdm = ["tqdm>=4.47.0"]
-transformers = ["transformers>=4.53.0, <5.0"]  # 5.x references torch.float8_e8m0fnu absent in older PyTorch builds
+transformers = ["transformers>=5.5.0"]  # 5.x needs the transchex BertLayer/BertConfig updates; re-verify the NGC image float8 concern
 zarr = ["zarr"]
 # these dependencies are for testing/building only, they aren't needed for regular use so don't appear in "all"
 testing = [