Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions docs/concepts/resolver-architecture.rst
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,15 @@ their common ``find_matches()`` method.
- ``GitHubTagProvider`` and ``GitLabTagProvider`` query repository tags
and parse versions from tag names. GitLab also reports timestamps for
age filtering.
- ``VersionMapProvider`` uses an existing version-to-URL mapping, such
as results from an earlier resolution or a URL template.
- ``VersionMapProvider`` maps versions to Git commit IDs or ref paths,
such as ``abad1dea`` or ``refs/tags/v1.2``. Its separate ``clone_url``
identifies the repository. Candidate URLs combine both values as
``git+https://host.example/repo.git@refs/tags/v1.2``, which the Git
downloader uses to clone and check out the selected ref.

.. versionchanged:: 0.104.0
``VersionMap`` values are Git refs, and ``VersionMapProvider`` requires
a separate ``clone_url`` instead of accepting full URLs in the map.

Selecting a Provider
--------------------
Expand All @@ -96,6 +103,16 @@ index URLs are ignored. Without ``source:``, Fromager uses the legacy
resolver settings and the package's ``get_resolver_provider`` override
when present.

.. versionadded:: 0.104.0
The ``versionmap-git`` profile accepts a ``versionmap`` of version
strings to Git refs and a separate HTTPS or SSH ``clone_url``.

``versionmap-git`` selects the highest mapped version matching the
requirement and constraints, then clones the repository at its mapped ref.
The resolved URL includes the ref for build-order replay. Its
``build_sdist`` setting defaults to ``pep517`` and also accepts ``tarball``.
See :class:`~fromager.packagesettings.VersionMapGitResolver` for an example.

Hook Profiles
~~~~~~~~~~~~~

Expand Down Expand Up @@ -317,7 +334,8 @@ Per-Package Configuration
-------------------------

Package settings files (YAML) configure the resolver for each package
independently: resolver type (PyPI, GitHub tags, GitLab tags, hook-based,
or blocked), index URL, cooldown, tag pattern, and override plugins.
independently: resolver type (PyPI, GitHub tags, GitLab tags, mapped Git
refs, hook-based, or blocked), index URL, cooldown, tag pattern, and
override plugins.

See :doc:`package-settings` for how settings are loaded and merged.
6 changes: 6 additions & 0 deletions docs/reference/config-reference.rst
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,12 @@ Source Resolver
:inherited-members: AbstractPyPIResolver, CooldownMixin
:members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel

.. versionchanged:: 0.104.0
``pypi-git`` rejects host-only or root-only ``clone_url`` values.

.. autopydantic_model:: VersionMapGitResolver
:members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel

.. autopydantic_model:: GitHubTagDownloadResolver
:inherited-members: AbstractGitSourceResolver, CooldownMixin
:members: download_kinds, supports_override_hooks, resolves_prebuilt_wheel
Expand Down
2 changes: 2 additions & 0 deletions src/fromager/packagesettings/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
PyPIPrebuiltResolver,
PyPISDistResolver,
SourceResolver,
VersionMapGitResolver,
pep440_tag_matcher,
)
from ._settings import Settings, SettingsFile
Expand Down Expand Up @@ -93,6 +94,7 @@
"Variant",
"VariantChangelog",
"VariantInfo",
"VersionMapGitResolver",
"WheelSettings",
"default_update_extra_environ",
"get_extra_environ",
Expand Down
104 changes: 96 additions & 8 deletions src/fromager/packagesettings/_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,14 @@

import pydantic
from packaging.utils import canonicalize_name
from packaging.version import Version

from .. import downloads, gitutils, overrides, resolver
from ..versionmap import VersionMap
from ._typedefs import MODEL_CONFIG

if typing.TYPE_CHECKING:
from packaging.requirements import Requirement
from packaging.version import Version

from .. import context, requirements_file

Expand All @@ -27,6 +28,15 @@
_PYPI_INDEX_URL = pydantic.HttpUrl("https://pypi.org/simple/")


def _validate_git_clone_url(value: pydantic.AnyUrl) -> pydantic.AnyUrl:
"""Require an HTTPS or SSH repository URL with a non-root path."""
if value.scheme not in {"https", "ssh"}:
raise ValueError(f"invalid scheme in url {value}")
if not value.path or value.path == "/":
raise ValueError(f"url {value} has an empty path")
return value


class BuildSDist(enum.StrEnum):
pep517 = "pep517"
tarball = "tarball"
Expand Down Expand Up @@ -106,8 +116,8 @@ def download(
*download_url* must contain everything the download needs, such as
the git ref in ``git+https://host/repo.git@refs/tags/v1.0``. It is
passed to download hooks unchanged and is not validated, so it can be
a ref instead of a URL, such as a commit SHA from a
``VersionMapProvider``.
a ref instead of a URL when a custom resolver and download hook
use that convention.
"""
raise NotImplementedError

Expand Down Expand Up @@ -455,11 +465,8 @@ class PyPIGitResolver(AbstractPyPIResolver):
@pydantic.field_validator("clone_url", mode="after")
@classmethod
def validate_clone_url(cls, value: pydantic.AnyUrl) -> pydantic.AnyUrl:
if value.scheme not in {"https", "ssh"}:
raise ValueError(f"invalid scheme in url {value}")
if not value.path:
raise ValueError(f"url {value} has an empty path")
return value
"""Validate the repository's clone URL."""
return _validate_git_clone_url(value)

def resolver_provider(
self,
Expand Down Expand Up @@ -491,6 +498,86 @@ def download(
)


class VersionMapGitResolver(AbstractResolver):
"""Resolve mapped versions and clone the repository at their Git refs.

``versionmap`` associates PEP 440 versions with commit IDs or ref paths.
``clone_url`` identifies the repository. Build the sdist using
``build_sdist``, which defaults to PEP 517.

Example::

provider: versionmap-git
clone_url: https://git.example/project/repo.git
build_sdist: pep517
versionmap:
'1.0': abad1dea
'1.1': refs/tags/1.1

.. versionadded:: 0.104.0
"""

provider: typing.Literal["versionmap-git"]

download_kinds: typing.ClassVar[DownloadKindSet] = frozenset(
{DownloadKind.git_checkout}
)

clone_url: pydantic.AnyUrl
"""Repository clone URL using HTTPS or SSH."""

versionmap: dict[str, str] = pydantic.Field(min_length=1)
"""Nonempty mapping of version strings to Git refs, without download URLs."""

build_sdist: BuildSDist = BuildSDist.pep517
"""Source distribution build method."""

@pydantic.field_validator("clone_url", mode="after")
@classmethod
def validate_clone_url(cls, value: pydantic.AnyUrl) -> pydantic.AnyUrl:
"""Validate the repository's clone URL."""
return _validate_git_clone_url(value)

@pydantic.field_validator("versionmap", mode="after")
@classmethod
def validate_versionmap(cls, value: dict[str, str]) -> dict[str, str]:
"""Validate versions and refs before constructing checkout URLs."""
for version, ref in value.items():
Version(version)
if not ref or any(c.isspace() or c in "@?#:" for c in ref):
raise ValueError(f"invalid Git ref for version {version!r}: {ref!r}")
return value
Comment thread
jlarkin09 marked this conversation as resolved.
Resolved

def resolver_provider(
self,
ctx: context.WorkContext,
req: Requirement,
req_type: requirements_file.RequirementType | None,
) -> resolver.VersionMapProvider:
"""Return a provider using the mapped refs and current constraints."""
return resolver.VersionMapProvider(
version_map=VersionMap(
{Version(version): ref for version, ref in self.versionmap.items()}
),
package_name=req.name,
clone_url=str(self.clone_url),
constraints=ctx.constraints,
req_type=req_type,
)

def download(
self,
ctx: context.WorkContext,
req: Requirement,
version: Version,
download_url: str,
) -> DownloadedSource:
"""Clone the repository at the ref recorded in the resolved URL."""
return self._download(
ctx, req, version, download_url, DownloadKind.git_checkout
)


_PEP440_TAG_PATTERN = re.compile(r"^(v?\d.*)$")


Expand Down Expand Up @@ -1102,6 +1189,7 @@ def download(
| PyPIPrebuiltResolver
| PyPIDownloadResolver
| PyPIGitResolver
| VersionMapGitResolver
| GitHubTagCloneResolver
| GitHubTagDownloadResolver
| GitLabTagCloneResolver
Expand Down
30 changes: 18 additions & 12 deletions src/fromager/resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -1391,11 +1391,14 @@ def _find_tags(


class VersionMapProvider(BaseProvider):
"""Lookup package versions from a VersionMap
"""Resolve package versions to Git refs in ``clone_url``.

This provider wraps a VersionMap instance to provide versions and URLs
for package resolution. The VersionMap should contain Version keys mapped
to URL strings.
The ``VersionMap`` maps versions to commit IDs or ref paths. Candidate
URLs combine ``clone_url`` with each ref so the Git downloader can
clone the repository and check out the selected ref.

.. versionchanged:: 0.104.0
Require ``clone_url`` and map versions to Git refs instead of URLs.
"""

provider_description: typing.ClassVar[str] = (
Expand All @@ -1408,6 +1411,7 @@ def __init__(
package_name: str | None,
constraints: Constraints | None = None,
*,
clone_url: str,
req_type: RequirementType | None = None,
use_resolver_cache: bool = True,
) -> None:
Expand All @@ -1420,26 +1424,28 @@ def __init__(
)
self.version_map = version_map
self.package_name = package_name
self.clone_url = clone_url

@property
def cache_key(self) -> str:
"""Scope cached candidates to the package, repository, and mapped refs."""
if self.package_name is None:
raise ValueError("Cannot cache VersionMapProvider without package name")
return f"versionmap:{self.package_name}"
refs = tuple(
(str(version), self.version_map[version])
for version in self.version_map.versions()
)
return f"versionmap:{self.package_name}:{self.clone_url}:{refs!r}"

def find_candidates(self, identifier: str) -> Candidates:
"""Find candidates from the VersionMap

Iterates through all versions in the VersionMap and creates Candidate
objects with the associated URLs.
"""
"""Return candidates with Git URLs containing their mapped refs."""
candidates: list[Candidate] = []
for version in self.version_map.versions():
url = self.version_map[version]
ref = self.version_map[version]
candidate = Candidate(
name=identifier,
version=version,
url=url,
url=f"git+{self.clone_url}@{ref}",
)
candidates.append(candidate)

Expand Down
33 changes: 20 additions & 13 deletions src/fromager/versionmap.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""VersionMap interface for managing package settings in plugins."""
"""Map package versions to Git commit IDs or ref paths."""

import typing

Expand All @@ -7,21 +7,28 @@


class VersionMap:
def __init__(
self, initial_content: dict[Version | str, typing.Any] | None = None
) -> None:
"""Associate package versions with Git refs in a separate repository.

Values are commit IDs or ref paths, such as ``refs/tags/v1.2``.
The clone URL belongs to the resolver provider.

.. versionchanged:: 0.104.0
Values are Git ref strings instead of arbitrary data.
"""

def __init__(self, initial_content: dict[Version | str, str] | None = None) -> None:
"""Initialize the VersionMap

Stores the inputs associating versions and arbitrary data. If the
Store the inputs associating versions and Git refs. If the
versions are strings, they are converted to Version instances
internally. Any exceptions from the conversion are propagated.
"""
self._content: dict[Version, typing.Any] = {}
self._content: dict[Version, str] = {}
for k, v in (initial_content or {}).items():
self.add(k, v)

def add(self, key: Version | str, value: typing.Any) -> None:
"""Add a single value associated with a version
def add(self, key: Version | str, value: str) -> None:
"""Associate a Git ref with a version.

String keys are converted to Version instances. Any exceptions from the
conversion are propagated.
Expand All @@ -30,8 +37,8 @@ def add(self, key: Version | str, value: typing.Any) -> None:
key = Version(key)
self._content[key] = value

def __getitem__(self, key: Version | str) -> typing.Any:
"""Get the value associated with a version
def __getitem__(self, key: Version | str) -> str:
"""Get the Git ref associated with a version.

String keys are converted to Version instances. Raises KeyError if the
version is not found.
Expand All @@ -49,12 +56,12 @@ def lookup(
req: Requirement,
constraint: Requirement | None = None,
allow_prerelease: bool = False,
) -> tuple[Version, typing.Any]:
"""Return the matching version and associated value.
) -> tuple[Version, str]:
"""Return the matching version and associated Git ref.

Finds the known version that best matches the requirement and optional
constraint and returns a tuple containing that version and the
associated value.
associated Git ref.
"""
for version in self.versions():
if not req.specifier.contains(version, prereleases=allow_prerelease):
Expand Down
Loading
Loading