From 81a5b364fda4a257789c389760220a36d57b00b7 Mon Sep 17 00:00:00 2001 From: Justin Larkin Date: Tue, 10 Nov 2026 20:27:31 -0500 Subject: [PATCH] feat(resolver): resolve mapped versions to Git refs Add `versionmap-git` source settings with a separate clone URL and a map of versions to commit IDs or refs. Preserve selected refs in resolved URLs for checkout and replay, and scope cached candidates to the repository and map. Share clone URL validation between Git profiles and reject root-only URLs. Cover configuration, SHA/tag checkout, constraints, replay, and sdist strategies. Closes: #1180 Co-Authored-By: Codex Signed-off-by: Justin Larkin --- docs/concepts/resolver-architecture.rst | 26 ++- docs/reference/config-reference.rst | 6 + src/fromager/packagesettings/__init__.py | 2 + src/fromager/packagesettings/_resolver.py | 104 ++++++++++- src/fromager/resolver.py | 30 ++-- src/fromager/versionmap.py | 33 ++-- tests/test_packagesettings.py | 35 ++++ tests/test_packagesettings_resolver.py | 206 +++++++++++++++++++++- tests/test_resolver.py | 89 +++++++--- tests/test_sources.py | 20 ++- tests/test_versionmap.py | 78 ++++---- 11 files changed, 525 insertions(+), 104 deletions(-) diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 1d5923320..559bbdfbc 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -79,8 +79,15 @@ their common ``find_matches()`` method. - ``GitHubTagProvider`` and ``GitLabTagProvider`` query repository tags and parse versions from tag names. GitLab also reports timestamps for age filtering. -- ``VersionMapProvider`` uses an existing version-to-URL mapping, such - as results from an earlier resolution or a URL template. +- ``VersionMapProvider`` maps versions to Git commit IDs or ref paths, + such as ``abad1dea`` or ``refs/tags/v1.2``. Its separate ``clone_url`` + identifies the repository. Candidate URLs combine both values as + ``git+https://host.example/repo.git@refs/tags/v1.2``, which the Git + downloader uses to clone and check out the selected ref. + +.. versionchanged:: 0.104.0 + ``VersionMap`` values are Git refs, and ``VersionMapProvider`` requires + a separate ``clone_url`` instead of accepting full URLs in the map. Selecting a Provider -------------------- @@ -96,6 +103,16 @@ index URLs are ignored. Without ``source:``, Fromager uses the legacy resolver settings and the package's ``get_resolver_provider`` override when present. +.. versionadded:: 0.104.0 + The ``versionmap-git`` profile accepts a ``versionmap`` of version + strings to Git refs and a separate HTTPS or SSH ``clone_url``. + +``versionmap-git`` selects the highest mapped version matching the +requirement and constraints, then clones the repository at its mapped ref. +The resolved URL includes the ref for build-order replay. Its +``build_sdist`` setting defaults to ``pep517`` and also accepts ``tarball``. +See :class:`~fromager.packagesettings.VersionMapGitResolver` for an example. + Hook Profiles ~~~~~~~~~~~~~ @@ -317,7 +334,8 @@ Per-Package Configuration ------------------------- Package settings files (YAML) configure the resolver for each package -independently: resolver type (PyPI, GitHub tags, GitLab tags, hook-based, -or blocked), index URL, cooldown, tag pattern, and override plugins. +independently: resolver type (PyPI, GitHub tags, GitLab tags, mapped Git +refs, hook-based, or blocked), index URL, cooldown, tag pattern, and +override plugins. See :doc:`package-settings` for how settings are loaded and merged. diff --git a/docs/reference/config-reference.rst b/docs/reference/config-reference.rst index bd5c07f92..f38e7b767 100644 --- a/docs/reference/config-reference.rst +++ b/docs/reference/config-reference.rst @@ -55,6 +55,12 @@ Source Resolver :inherited-members: AbstractPyPIResolver, CooldownMixin :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel +.. versionchanged:: 0.104.0 + ``pypi-git`` rejects host-only or root-only ``clone_url`` values. + +.. autopydantic_model:: VersionMapGitResolver + :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel + .. autopydantic_model:: GitHubTagDownloadResolver :inherited-members: AbstractGitSourceResolver, CooldownMixin :members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel diff --git a/src/fromager/packagesettings/__init__.py b/src/fromager/packagesettings/__init__.py index 5703c7589..e6a85e247 100644 --- a/src/fromager/packagesettings/__init__.py +++ b/src/fromager/packagesettings/__init__.py @@ -31,6 +31,7 @@ PyPIPrebuiltResolver, PyPISDistResolver, SourceResolver, + VersionMapGitResolver, pep440_tag_matcher, ) from ._settings import Settings, SettingsFile @@ -93,6 +94,7 @@ "Variant", "VariantChangelog", "VariantInfo", + "VersionMapGitResolver", "WheelSettings", "default_update_extra_environ", "get_extra_environ", diff --git a/src/fromager/packagesettings/_resolver.py b/src/fromager/packagesettings/_resolver.py index 6a05a5d81..774adf53d 100644 --- a/src/fromager/packagesettings/_resolver.py +++ b/src/fromager/packagesettings/_resolver.py @@ -11,13 +11,14 @@ import pydantic from packaging.utils import canonicalize_name +from packaging.version import Version from .. import downloads, gitutils, overrides, resolver +from ..versionmap import VersionMap from ._typedefs import MODEL_CONFIG if typing.TYPE_CHECKING: from packaging.requirements import Requirement - from packaging.version import Version from .. import context, requirements_file @@ -27,6 +28,15 @@ _PYPI_INDEX_URL = pydantic.HttpUrl("https://pypi.org/simple/") +def _validate_git_clone_url(value: pydantic.AnyUrl) -> pydantic.AnyUrl: + """Require an HTTPS or SSH repository URL with a non-root path.""" + if value.scheme not in {"https", "ssh"}: + raise ValueError(f"invalid scheme in url {value}") + if not value.path or value.path == "/": + raise ValueError(f"url {value} has an empty path") + return value + + class BuildSDist(enum.StrEnum): pep517 = "pep517" tarball = "tarball" @@ -106,8 +116,8 @@ def download( *download_url* must contain everything the download needs, such as the git ref in ``git+https://host/repo.git@refs/tags/v1.0``. It is passed to download hooks unchanged and is not validated, so it can be - a ref instead of a URL, such as a commit SHA from a - ``VersionMapProvider``. + a ref instead of a URL when a custom resolver and download hook + use that convention. """ raise NotImplementedError @@ -455,11 +465,8 @@ class PyPIGitResolver(AbstractPyPIResolver): @pydantic.field_validator("clone_url", mode="after") @classmethod def validate_clone_url(cls, value: pydantic.AnyUrl) -> pydantic.AnyUrl: - if value.scheme not in {"https", "ssh"}: - raise ValueError(f"invalid scheme in url {value}") - if not value.path: - raise ValueError(f"url {value} has an empty path") - return value + """Validate the repository's clone URL.""" + return _validate_git_clone_url(value) def resolver_provider( self, @@ -491,6 +498,86 @@ def download( ) +class VersionMapGitResolver(AbstractResolver): + """Resolve mapped versions and clone the repository at their Git refs. + + ``versionmap`` associates PEP 440 versions with commit IDs or ref paths. + ``clone_url`` identifies the repository. Build the sdist using + ``build_sdist``, which defaults to PEP 517. + + Example:: + + provider: versionmap-git + clone_url: https://git.example/project/repo.git + build_sdist: pep517 + versionmap: + '1.0': abad1dea + '1.1': refs/tags/1.1 + + .. versionadded:: 0.104.0 + """ + + provider: typing.Literal["versionmap-git"] + + download_kinds: typing.ClassVar[DownloadKindSet] = frozenset( + {DownloadKind.git_checkout} + ) + + clone_url: pydantic.AnyUrl + """Repository clone URL using HTTPS or SSH.""" + + versionmap: dict[str, str] = pydantic.Field(min_length=1) + """Nonempty mapping of version strings to Git refs, without download URLs.""" + + build_sdist: BuildSDist = BuildSDist.pep517 + """Source distribution build method.""" + + @pydantic.field_validator("clone_url", mode="after") + @classmethod + def validate_clone_url(cls, value: pydantic.AnyUrl) -> pydantic.AnyUrl: + """Validate the repository's clone URL.""" + return _validate_git_clone_url(value) + + @pydantic.field_validator("versionmap", mode="after") + @classmethod + def validate_versionmap(cls, value: dict[str, str]) -> dict[str, str]: + """Validate versions and refs before constructing checkout URLs.""" + for version, ref in value.items(): + Version(version) + if not ref or any(c.isspace() or c in "@?#:" for c in ref): + raise ValueError(f"invalid Git ref for version {version!r}: {ref!r}") + return value + + def resolver_provider( + self, + ctx: context.WorkContext, + req: Requirement, + req_type: requirements_file.RequirementType | None, + ) -> resolver.VersionMapProvider: + """Return a provider using the mapped refs and current constraints.""" + return resolver.VersionMapProvider( + version_map=VersionMap( + {Version(version): ref for version, ref in self.versionmap.items()} + ), + package_name=req.name, + clone_url=str(self.clone_url), + constraints=ctx.constraints, + req_type=req_type, + ) + + def download( + self, + ctx: context.WorkContext, + req: Requirement, + version: Version, + download_url: str, + ) -> DownloadedSource: + """Clone the repository at the ref recorded in the resolved URL.""" + return self._download( + ctx, req, version, download_url, DownloadKind.git_checkout + ) + + _PEP440_TAG_PATTERN = re.compile(r"^(v?\d.*)$") @@ -1102,6 +1189,7 @@ def download( | PyPIPrebuiltResolver | PyPIDownloadResolver | PyPIGitResolver + | VersionMapGitResolver | GitHubTagCloneResolver | GitHubTagDownloadResolver | GitLabTagCloneResolver diff --git a/src/fromager/resolver.py b/src/fromager/resolver.py index bcdde2476..5c254855d 100644 --- a/src/fromager/resolver.py +++ b/src/fromager/resolver.py @@ -1391,11 +1391,14 @@ def _find_tags( class VersionMapProvider(BaseProvider): - """Lookup package versions from a VersionMap + """Resolve package versions to Git refs in ``clone_url``. - This provider wraps a VersionMap instance to provide versions and URLs - for package resolution. The VersionMap should contain Version keys mapped - to URL strings. + The ``VersionMap`` maps versions to commit IDs or ref paths. Candidate + URLs combine ``clone_url`` with each ref so the Git downloader can + clone the repository and check out the selected ref. + + .. versionchanged:: 0.104.0 + Require ``clone_url`` and map versions to Git refs instead of URLs. """ provider_description: typing.ClassVar[str] = ( @@ -1408,6 +1411,7 @@ def __init__( package_name: str | None, constraints: Constraints | None = None, *, + clone_url: str, req_type: RequirementType | None = None, use_resolver_cache: bool = True, ) -> None: @@ -1420,26 +1424,28 @@ def __init__( ) self.version_map = version_map self.package_name = package_name + self.clone_url = clone_url @property def cache_key(self) -> str: + """Scope cached candidates to the package, repository, and mapped refs.""" if self.package_name is None: raise ValueError("Cannot cache VersionMapProvider without package name") - return f"versionmap:{self.package_name}" + refs = tuple( + (str(version), self.version_map[version]) + for version in self.version_map.versions() + ) + return f"versionmap:{self.package_name}:{self.clone_url}:{refs!r}" def find_candidates(self, identifier: str) -> Candidates: - """Find candidates from the VersionMap - - Iterates through all versions in the VersionMap and creates Candidate - objects with the associated URLs. - """ + """Return candidates with Git URLs containing their mapped refs.""" candidates: list[Candidate] = [] for version in self.version_map.versions(): - url = self.version_map[version] + ref = self.version_map[version] candidate = Candidate( name=identifier, version=version, - url=url, + url=f"git+{self.clone_url}@{ref}", ) candidates.append(candidate) diff --git a/src/fromager/versionmap.py b/src/fromager/versionmap.py index 280d0f9f0..e2e44d033 100644 --- a/src/fromager/versionmap.py +++ b/src/fromager/versionmap.py @@ -1,4 +1,4 @@ -"""VersionMap interface for managing package settings in plugins.""" +"""Map package versions to Git commit IDs or ref paths.""" import typing @@ -7,21 +7,28 @@ class VersionMap: - def __init__( - self, initial_content: dict[Version | str, typing.Any] | None = None - ) -> None: + """Associate package versions with Git refs in a separate repository. + + Values are commit IDs or ref paths, such as ``refs/tags/v1.2``. + The clone URL belongs to the resolver provider. + + .. versionchanged:: 0.104.0 + Values are Git ref strings instead of arbitrary data. + """ + + def __init__(self, initial_content: dict[Version | str, str] | None = None) -> None: """Initialize the VersionMap - Stores the inputs associating versions and arbitrary data. If the + Store the inputs associating versions and Git refs. If the versions are strings, they are converted to Version instances internally. Any exceptions from the conversion are propagated. """ - self._content: dict[Version, typing.Any] = {} + self._content: dict[Version, str] = {} for k, v in (initial_content or {}).items(): self.add(k, v) - def add(self, key: Version | str, value: typing.Any) -> None: - """Add a single value associated with a version + def add(self, key: Version | str, value: str) -> None: + """Associate a Git ref with a version. String keys are converted to Version instances. Any exceptions from the conversion are propagated. @@ -30,8 +37,8 @@ def add(self, key: Version | str, value: typing.Any) -> None: key = Version(key) self._content[key] = value - def __getitem__(self, key: Version | str) -> typing.Any: - """Get the value associated with a version + def __getitem__(self, key: Version | str) -> str: + """Get the Git ref associated with a version. String keys are converted to Version instances. Raises KeyError if the version is not found. @@ -49,12 +56,12 @@ def lookup( req: Requirement, constraint: Requirement | None = None, allow_prerelease: bool = False, - ) -> tuple[Version, typing.Any]: - """Return the matching version and associated value. + ) -> tuple[Version, str]: + """Return the matching version and associated Git ref. Finds the known version that best matches the requirement and optional constraint and returns a tuple containing that version and the - associated value. + associated Git ref. """ for version in self.versions(): if not req.specifier.contains(version, prereleases=allow_prerelease): diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 6a37a8266..e9f67e772 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -26,6 +26,7 @@ SettingsFile, SourceResolver, Variant, + VersionMapGitResolver, substitute_template, ) from fromager.packagesettings._models import DeleteEnvPattern, KeepEnvPattern @@ -1180,6 +1181,7 @@ def test_source_resolver_importable() -> None: union_members = typing.get_args(args[0]) assert PyPISDistResolver in union_members assert PyPIPrebuiltResolver in union_members + assert VersionMapGitResolver in union_members def test_package_settings_with_source() -> None: @@ -1191,6 +1193,39 @@ def test_package_settings_with_source() -> None: assert ps.source.provider == "pypi-sdist" +@pytest.mark.parametrize("variant_source", [False, True]) +def test_package_settings_with_versionmap_source( + tmp_path: pathlib.Path, variant_source: bool +) -> None: + """Select mapped Git sources from package or variant settings.""" + source = { + "provider": "versionmap-git", + "clone_url": "https://git.test/project/repo.git", + "versionmap": {"1.0": "abad1dea", "1.1": "refs/tags/1.1"}, + } + data = ( + {"variants": {"cpu": {"source": source}}} + if variant_source + else {"source": source} + ) + ps = PackageSettings.model_validate( + {"name": "test-pkg", "has_config": True, **data} + ) + settings = Settings( + settings=SettingsFile(), + package_settings=[ps], + variant="cpu", + patches_dir=tmp_path, + max_jobs=1, + ) + + pbi = settings.package_build_info("test-pkg") + + assert isinstance(pbi.source_resolver, VersionMapGitResolver) + assert pbi.source_resolver.versionmap == source["versionmap"] + assert pbi.pre_built is False + + def test_variant_info_with_source() -> None: """``VariantInfo`` with ``source:`` works.""" yaml_str = "variants:\n cpu:\n source:\n provider: pypi-prebuilt\n" diff --git a/tests/test_packagesettings_resolver.py b/tests/test_packagesettings_resolver.py index 5f05ec123..3cbc5d842 100644 --- a/tests/test_packagesettings_resolver.py +++ b/tests/test_packagesettings_resolver.py @@ -14,13 +14,16 @@ import pydantic import pytest +import resolvelib import yaml from packaging.requirements import Requirement from packaging.version import Version from wheel.wheelfile import WheelFile -from fromager import finders, resolver, sources +from fromager import finders, packagesettings, resolver, sources from fromager.context import WorkContext +from fromager.log import req_ctxvar_context +from fromager.packagesettings import VersionMapGitResolver from fromager.packagesettings._resolver import ( AbstractResolver, BuildSDist, @@ -384,11 +387,16 @@ def test_missing_version_in_tag(self) -> None: tag="latest", ) - def test_clone_url_rejects_http(self) -> None: - with pytest.raises(pydantic.ValidationError): + @pytest.mark.parametrize( + "clone_url", + ["http://git.test/repo.git", "https://git.test", "https://git.test/"], + ) + def test_invalid_clone_url(self, clone_url: str) -> None: + """Reject unsupported schemes and missing repository paths.""" + with pytest.raises(pydantic.ValidationError, match="clone_url"): PyPIGitResolver( provider="pypi-git", - clone_url="http://github.com/org/repo.git", # type: ignore[arg-type] + clone_url=pydantic.AnyUrl(clone_url), tag="v{version}", ) @@ -583,6 +591,196 @@ def test_download_does_not_reuse_mismatched_checkout( assert _git(recloned.path, "rev-parse", "HEAD").strip() == expected_head +class TestVersionMapGitResolver: + """Cover configuration, version selection, and mapped Git checkouts.""" + + YAML = """\ + source: + provider: versionmap-git + clone_url: https://git.test/project/repo.git + build_sdist: pep517 + versionmap: + '1.0': abad1dea + '1.1': refs/tags/1.1 + """ + + def test_parse(self) -> None: + """Load the issue's example with refs separate from the clone URL.""" + r = _parse(self.YAML) + + assert isinstance(r, VersionMapGitResolver) + assert str(r.clone_url) == "https://git.test/project/repo.git" + assert r.versionmap == {"1.0": "abad1dea", "1.1": "refs/tags/1.1"} + assert r.build_sdist is BuildSDist.pep517 + assert r.download_kinds == frozenset({DownloadKind.git_checkout}) + assert r.resolves_prebuilt_wheel is False + assert r.supports_override_hooks is False + + def test_default_build_sdist_strategy(self) -> None: + """Default Git source builds to PEP 517.""" + r = _parse(self.YAML.replace(" build_sdist: pep517\n", "")) + + assert r.build_sdist is BuildSDist.pep517 + + @pytest.mark.parametrize( + ("requirement", "constraint", "version", "ref"), + [ + ("test-pkg", None, "1.1", "refs/tags/1.1"), + ("test-pkg==1.0", None, "1.0", "abad1dea"), + ("test-pkg", "test-pkg<1.1", "1.0", "abad1dea"), + ], + ) + def test_resolver_provider( + self, + tmp_context: WorkContext, + requirement: str, + constraint: str | None, + version: str, + ref: str, + ) -> None: + """Select mapped refs using both requirements and constraints.""" + r = _parse(self.YAML) + if constraint: + tmp_context.constraints.add_constraint(constraint) + req = Requirement(requirement) + + p = r.resolver_provider(tmp_context, req, _REQ_TYPE) + results = resolver.find_all_matching_from_provider(p, req) + + assert isinstance(p, resolver.VersionMapProvider) + assert p.package_name == req.name + assert p.clone_url == "https://git.test/project/repo.git" + assert p.version_map["1.0"] == "abad1dea" + assert p.req_type is _REQ_TYPE + assert results[0] == (f"git+{p.clone_url}@{ref}", Version(version)) + + def test_resolver_no_match(self, tmp_context: WorkContext) -> None: + """Fail when no configured version matches the requirement.""" + req = Requirement("test-pkg>=2") + p = _parse(self.YAML).resolver_provider(tmp_context, req, None) + + with pytest.raises(resolvelib.resolvers.ResolverException): + resolver.find_all_matching_from_provider(p, req) + + @pytest.mark.parametrize( + "clone_url", + ["https://git.test/project/repo.git", "ssh://git@git.test/project/repo.git"], + ) + def test_clone_url(self, clone_url: str) -> None: + """Accept HTTPS and SSH repository URLs.""" + r = _parse(self.YAML.replace("https://git.test/project/repo.git", clone_url)) + + assert isinstance(r, VersionMapGitResolver) + assert str(r.clone_url) == clone_url + + @pytest.mark.parametrize( + "clone_url", + ["http://git.test/repo.git", "https://git.test", "https://git.test/"], + ) + def test_invalid_clone_url(self, clone_url: str) -> None: + """Reject unsupported schemes and missing repository paths.""" + with pytest.raises(pydantic.ValidationError, match="clone_url"): + _parse(self.YAML.replace("https://git.test/project/repo.git", clone_url)) + + @pytest.mark.parametrize( + "versionmap", + [ + {}, + {"latest": "abad1dea"}, + {"1.0": ""}, + {"1.0": "https://pkg.test/source.tar.gz"}, + {"1.0": "ref with spaces"}, + {"1.0": "refs/tags/release\n1.0"}, + {"1.0": "refs/tags/release@1.0"}, + {"1.0": "refs/tags/release#1.0"}, + {"1.0": "refs/tags/release?1.0"}, + ], + ) + def test_invalid_versionmap(self, versionmap: dict[str, str]) -> None: + """Reject invalid versions and refs that cannot form checkout URLs.""" + with pytest.raises(pydantic.ValidationError, match="versionmap"): + VersionMapGitResolver( + provider="versionmap-git", + clone_url=pydantic.AnyUrl("https://git.test/project/repo.git"), + versionmap=versionmap, + ) + + @mock.patch("fromager.downloads.download_git_source") + def test_download_uses_recorded_url( + self, mock_dl: mock.MagicMock, tmp_context: WorkContext + ) -> None: + """Replay the resolved URL even when the configuration's ref differs.""" + expected = tmp_context.work_dir / "test-pkg-1.0" / "test-pkg-1.0" + mock_dl.return_value = expected + url = "git+https://git.test/project/repo.git@refs/tags/recorded" + + result = _parse(self.YAML).download(tmp_context, _REQ, Version("1.0"), url) + + assert result == DownloadedSource(expected, DownloadKind.git_checkout) + mock_dl.assert_called_once_with(destination_dir=expected, vcs_url=url) + + @pytest.mark.parametrize("use_commit", [True, False], ids=["commit", "tag"]) + def test_configured_source_checks_out_mapped_ref( + self, + tmp_context: WorkContext, + tmp_path: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, + use_commit: bool, + ) -> None: + """Resolve YAML settings, check out an older ref, and prepare its source.""" + repo = _make_tagged_repo(tmp_path / "repo") + commit = _git(repo, "rev-parse", "HEAD").strip() + (repo / "new.txt").write_text("newer commit") + _git(repo, "add", ".") + _git(repo, "commit", "-m", "New source") + ref = commit if use_commit else "refs/tags/v1.2.3" + # Resolve the configured HTTPS URL against a local repository offline. + monkeypatch.setenv("GIT_CONFIG_COUNT", "1") + monkeypatch.setenv("GIT_CONFIG_KEY_0", f"url.{repo.as_uri()}.insteadOf") + monkeypatch.setenv("GIT_CONFIG_VALUE_0", "https://git.test/project/repo.git") + ps = packagesettings.PackageSettings.from_string( + _REQ.name, + f"""\ +source: + provider: versionmap-git + clone_url: https://git.test/project/repo.git + versionmap: + '1.2.3': {ref} + '2.0': HEAD +""", + ) + tmp_context.settings = packagesettings.Settings( + settings=packagesettings.SettingsFile(), + package_settings=[ps], + variant=tmp_context.variant, + patches_dir=tmp_path / "patches", + max_jobs=1, + ) + tmp_context.constraints.add_constraint("test-pkg<2") + + with req_ctxvar_context(_REQ): + url, version = sources.resolve_source( + ctx=tmp_context, req=_REQ, sdist_server_url=resolver.PYPI_SERVER_URL + ) + downloaded = sources.download_source( + ctx=tmp_context, req=_REQ, version=version, download_url=url + ) + prepared = sources.prepare_source( + ctx=tmp_context, + req=_REQ, + version=version, + source_filename=downloaded.path, + download_kind=downloaded.kind, + ) + + assert version == _VERSION + assert url == f"git+https://git.test/project/repo.git@{ref}" + assert downloaded.kind is DownloadKind.git_checkout + assert prepared == downloaded.path + assert _git(prepared, "rev-parse", "HEAD").strip() == commit + assert not (prepared / "new.txt").exists() + + # -- Git source resolvers ---------------------------------------------------- diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 74ec9b582..ec35fed1f 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -12,6 +12,7 @@ from fromager import constraints, resolver from fromager.__main__ import main as fromager +from fromager.versionmap import VersionMap _hydra_core_simple_response = """ @@ -868,18 +869,19 @@ def _versions(*args: typing.Any, **kwds: typing.Any) -> list[tuple[str, str]]: def test_resolve_versionmap() -> None: - from fromager.versionmap import VersionMap - + """Resolve the newest mapped version to its repository and git ref.""" version_map = VersionMap( { - "1.2": "https://example.com/pkg-1.2.tar.gz", - "1.3": "https://example.com/pkg-1.3.tar.gz", - "1.4.1": "https://example.com/pkg-1.4.1.tar.gz", + "1.2": "abad1dea", + "1.3": "refs/tags/v1.3", + "1.4.1": "refs/tags/v1.4.1", } ) provider = resolver.VersionMapProvider( - version_map=version_map, package_name="testpkg" + version_map=version_map, + package_name="testpkg", + clone_url="https://git.test/pkg.git", ) reporter: resolvelib.BaseReporter = resolvelib.BaseReporter() rslvr = resolvelib.Resolver(provider, reporter) @@ -889,25 +891,24 @@ def test_resolve_versionmap() -> None: candidate = result.mapping["testpkg"] assert str(candidate.version) == "1.4.1" - assert candidate.url == "https://example.com/pkg-1.4.1.tar.gz" + assert candidate.url == "git+https://git.test/pkg.git@refs/tags/v1.4.1" # VersionMapProvider uses resolver cache by default cache = resolver.BaseProvider.resolver_cache assert "testpkg" in cache cached_candidates = cache["testpkg"][ - (resolver.VersionMapProvider, "versionmap:testpkg") + (resolver.VersionMapProvider, provider.cache_key) ] assert len(cached_candidates) == 3 def test_resolve_versionmap_with_constraint() -> None: - from fromager.versionmap import VersionMap - + """Apply constraints before selecting a mapped git ref.""" version_map = VersionMap( { - "1.2": "https://example.com/pkg-1.2.tar.gz", - "1.3": "https://example.com/pkg-1.3.tar.gz", - "1.4.1": "https://example.com/pkg-1.4.1.tar.gz", + "1.2": "abad1dea", + "1.3": "refs/tags/v1.3", + "1.4.1": "refs/tags/v1.4.1", } ) @@ -915,7 +916,10 @@ def test_resolve_versionmap_with_constraint() -> None: c.add_constraint("testpkg<1.4") provider = resolver.VersionMapProvider( - version_map=version_map, package_name="testpkg", constraints=c + version_map=version_map, + package_name="testpkg", + clone_url="https://git.test/pkg.git", + constraints=c, ) reporter: resolvelib.BaseReporter = resolvelib.BaseReporter() rslvr = resolvelib.Resolver(provider, reporter) @@ -925,21 +929,22 @@ def test_resolve_versionmap_with_constraint() -> None: candidate = result.mapping["testpkg"] assert str(candidate.version) == "1.3" - assert candidate.url == "https://example.com/pkg-1.3.tar.gz" + assert candidate.url == "git+https://git.test/pkg.git@refs/tags/v1.3" def test_resolve_versionmap_no_match() -> None: - from fromager.versionmap import VersionMap - + """Reject requirements with no mapped version.""" version_map = VersionMap( { - "1.2": "https://example.com/pkg-1.2.tar.gz", - "1.3": "https://example.com/pkg-1.3.tar.gz", + "1.2": "abad1dea", + "1.3": "refs/tags/v1.3", } ) provider = resolver.VersionMapProvider( - version_map=version_map, package_name="testpkg" + version_map=version_map, + package_name="testpkg", + clone_url="https://git.test/pkg.git", ) reporter: resolvelib.BaseReporter = resolvelib.BaseReporter() rslvr = resolvelib.Resolver(provider, reporter) @@ -948,6 +953,50 @@ def test_resolve_versionmap_no_match() -> None: rslvr.resolve([Requirement("testpkg>=2.0")]) +def test_resolve_versionmap_cache_separates_repositories() -> None: + """Keep cached URLs scoped to the configured Git repository.""" + version_map = VersionMap({"1.2": "refs/tags/v1.2"}) + req = Requirement("testpkg") + for clone_url in ("https://git.test/pkg.git", "https://mirror.test/pkg.git"): + provider = resolver.VersionMapProvider( + version_map=version_map, package_name=req.name, clone_url=clone_url + ) + + results = resolver.find_all_matching_from_provider(provider, req) + + assert results == [(f"git+{clone_url}@refs/tags/v1.2", Version("1.2"))] + + +def test_resolve_versionmap_without_package_name() -> None: + """Resolve without caching when no package name is configured.""" + provider = resolver.VersionMapProvider( + version_map=VersionMap({"1.2": "abad1dea"}), + package_name=None, + clone_url="https://git.test/pkg.git", + ) + + results = resolver.find_all_matching_from_provider(provider, Requirement("testpkg")) + + assert results == [("git+https://git.test/pkg.git@abad1dea", Version("1.2"))] + assert not resolver.BaseProvider.resolver_cache + + +def test_resolve_versionmap_cache_separates_maps() -> None: + """Use each configured map when the package and clone URL are unchanged.""" + for ref in ("abad1dea", "refs/tags/v1.2"): + provider = resolver.VersionMapProvider( + version_map=VersionMap({"1.2": ref}), + package_name="testpkg", + clone_url="https://git.test/pkg.git", + ) + + results = resolver.find_all_matching_from_provider( + provider, Requirement("testpkg") + ) + + assert results == [(f"git+https://git.test/pkg.git@{ref}", Version("1.2"))] + + _gitlab_submodlib_repo_response = """ [ { diff --git a/tests/test_sources.py b/tests/test_sources.py index 54732d583..ea9d39079 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -1804,13 +1804,20 @@ def test_prepare_source_plugin_returns_bad_tuple( packagesettings.HookSDistResolver( provider="hook-sdist", build_sdist=packagesettings.BuildSDist.pep517 ), + packagesettings.VersionMapGitResolver( + provider="versionmap-git", + clone_url=pydantic.AnyUrl("https://git.test/pkg.git"), + versionmap={"1.0": "refs/tags/v1.0"}, + ), ], - ids=["git-clone", "hook-source"], + ids=["git-clone", "hook-source", "versionmap-git"], ) @pytest.mark.parametrize("has_override", [False, True]) def test_build_sdist_uses_pep517_for_configured_source( source_resolver: ( - packagesettings.GitHubTagCloneResolver | packagesettings.HookSDistResolver + packagesettings.GitHubTagCloneResolver + | packagesettings.HookSDistResolver + | packagesettings.VersionMapGitResolver ), has_override: bool, tmp_context: context.WorkContext, @@ -1876,14 +1883,21 @@ def test_build_sdist_uses_pep517_for_configured_source( project_url=pydantic.HttpUrl("https://github.test/org/pkg"), build_sdist=packagesettings.BuildSDist.tarball, ), + packagesettings.VersionMapGitResolver( + provider="versionmap-git", + clone_url=pydantic.AnyUrl("https://git.test/pkg.git"), + versionmap={"1.0": "refs/tags/v1.0"}, + build_sdist=packagesettings.BuildSDist.tarball, + ), ], - ids=["legacy", "hook-source", "configured-tarball"], + ids=["legacy", "hook-source", "configured-tarball", "versionmap-git"], ) @pytest.mark.parametrize("has_override", [False, True]) def test_build_sdist_uses_configured_tarball_or_legacy_hook( source_resolver: ( packagesettings.HookSDistResolver | packagesettings.GitHubTagCloneResolver + | packagesettings.VersionMapGitResolver | None ), has_override: bool, diff --git a/tests/test_versionmap.py b/tests/test_versionmap.py index b0d75f1e6..ffc1e1946 100644 --- a/tests/test_versionmap.py +++ b/tests/test_versionmap.py @@ -8,9 +8,9 @@ def test_initialize() -> None: m = VersionMap( { - "1.2": "value for 1.2", - Version("1.3"): "value for 1.3", - "1.0": "value for 1.0", + "1.2": "refs/tags/v1.2", + Version("1.3"): "abad1dea", + "1.0": "refs/tags/v1.0", } ) assert list(m.versions()) == [Version("1.3"), Version("1.2"), Version("1.0")] @@ -19,81 +19,79 @@ def test_initialize() -> None: def test_lookup() -> None: m = VersionMap( { - "1.2": "value for 1.2", - Version("1.3"): "value for 1.3", - "1.0": "value for 1.0", + "1.2": "refs/tags/v1.2", + Version("1.3"): "abad1dea", + "1.0": "refs/tags/v1.0", } ) - assert m.lookup(Requirement("pkg")) == (Version("1.3"), "value for 1.3") - assert m.lookup(Requirement("pkg>1.0")) == (Version("1.3"), "value for 1.3") - assert m.lookup(Requirement("pkg<1.3")) == (Version("1.2"), "value for 1.2") + assert m.lookup(Requirement("pkg")) == (Version("1.3"), "abad1dea") + assert m.lookup(Requirement("pkg>1.0")) == (Version("1.3"), "abad1dea") + assert m.lookup(Requirement("pkg<1.3")) == (Version("1.2"), "refs/tags/v1.2") def test_prerelease() -> None: m = VersionMap( { - Version("0.4.1b0"): "value for 0.4.1b0", - "1.2": "value for 1.2", - Version("1.3"): "value for 1.3", - "1.0": "value for 1.0", - "1.5.0a0": "value for 1.5.0a0", + Version("0.4.1b0"): "refs/tags/v0.4.1b0", + "1.2": "refs/tags/v1.2", + Version("1.3"): "abad1dea", + "1.0": "refs/tags/v1.0", + "1.5.0a0": "refs/tags/v1.5.0a0", } ) - assert m.lookup(Requirement("pkg")) == (Version("1.3"), "value for 1.3") - assert m.lookup(Requirement("pkg>1.0")) == (Version("1.3"), "value for 1.3") - assert m.lookup(Requirement("pkg<1.3")) == (Version("1.2"), "value for 1.2") + assert m.lookup(Requirement("pkg")) == (Version("1.3"), "abad1dea") + assert m.lookup(Requirement("pkg>1.0")) == (Version("1.3"), "abad1dea") + assert m.lookup(Requirement("pkg<1.3")) == (Version("1.2"), "refs/tags/v1.2") assert m.lookup(Requirement("pkg"), allow_prerelease=True) == ( Version("1.5.0a0"), - "value for 1.5.0a0", + "refs/tags/v1.5.0a0", ) with pytest.raises(ValueError): - assert ( - m.lookup(Requirement("pkg"), Requirement("pkg<1.0")) == "value for 0.4.1b" - ) + m.lookup(Requirement("pkg"), Requirement("pkg<1.0")) assert m.lookup( Requirement("pkg"), Requirement("pkg<1.0"), allow_prerelease=True - ) == (Version("0.4.1b0"), "value for 0.4.1b0") + ) == (Version("0.4.1b0"), "refs/tags/v0.4.1b0") def test_only_prerelease() -> None: m = VersionMap( { - Version("0.4.1b0"): "value for 0.4.1b0", - Version("0.6b0"): "value for 0.6b0", + Version("0.4.1b0"): "refs/tags/v0.4.1b0", + Version("0.6b0"): "refs/tags/v0.6b0", } ) assert m.lookup( Requirement("pkg"), constraint=Requirement("pkg<0.6b"), allow_prerelease=True ) == ( Version("0.4.1b0"), - "value for 0.4.1b0", + "refs/tags/v0.4.1b0", ) def test_with_constraint() -> None: m = VersionMap( { - "1.2": "value for 1.2", - Version("1.3"): "value for 1.3", - "1.0": "value for 1.0", + "1.2": "refs/tags/v1.2", + Version("1.3"): "abad1dea", + "1.0": "refs/tags/v1.0", } ) assert m.lookup(Requirement("pkg"), Requirement("pkg<1.3")) == ( Version("1.2"), - "value for 1.2", + "refs/tags/v1.2", ) assert m.lookup(Requirement("pkg>1.0"), Requirement("pkg==1.2")) == ( Version("1.2"), - "value for 1.2", + "refs/tags/v1.2", ) def test_no_match() -> None: m = VersionMap( { - "1.2": "value for 1.2", - Version("1.3"): "value for 1.3", - "1.0": "value for 1.0", + "1.2": "refs/tags/v1.2", + Version("1.3"): "abad1dea", + "1.0": "refs/tags/v1.0", } ) with pytest.raises(ValueError): @@ -105,18 +103,18 @@ def test_no_match() -> None: def test_getitem() -> None: m = VersionMap( { - "1.2": "value for 1.2", - Version("1.3"): "value for 1.3", - "1.0": "value for 1.0", + "1.2": "refs/tags/v1.2", + Version("1.3"): "abad1dea", + "1.0": "refs/tags/v1.0", } ) # Access by Version object - assert m[Version("1.2")] == "value for 1.2" - assert m[Version("1.3")] == "value for 1.3" + assert m[Version("1.2")] == "refs/tags/v1.2" + assert m[Version("1.3")] == "abad1dea" # Access by string (auto-converted to Version) - assert m["1.2"] == "value for 1.2" - assert m["1.0"] == "value for 1.0" + assert m["1.2"] == "refs/tags/v1.2" + assert m["1.0"] == "refs/tags/v1.0" # Non-existent version raises KeyError with pytest.raises(KeyError):