Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 15 additions & 8 deletions src/fromager/bootstrapper/_resolve.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ class Resolve(Phase):
Resolution runs in a background thread (via ``background_work``); ``run()``
blocks until the result is ready. In normal mode the highest matching
version is used; in ``multiple_versions`` mode every candidate version is
returned, minus any that previously failed or whose wheels are already
cached.
returned, minus any that previously failed. Cached dependency versions can
be skipped, but top-level versions always need dependency discovery.

Next phase: one ``Start`` item per version that still needs processing.
"""
Expand Down Expand Up @@ -77,13 +77,12 @@ def run(self, bt: Bootstrapper) -> list[Phase]:

Centralizes version resolution so all dependencies are expanded
uniformly. In multiple_versions mode, filters out versions that
already failed in this run and versions whose wheels are already
cached to avoid redundant builds and transitive dependency
processing.
already failed in this run and cached dependency versions. Retains
cached top-level versions so their existing graph nodes acquire build
edges before a later build encounters a cache miss.

Returns:
One Start item per resolved version that needs building.
Empty list if all versions are already cached.
One Start item per resolved version that needs dependency discovery.
"""
assert self.bg_future is not None
# bg_future.result() blocks until the background resolution completes,
Expand Down Expand Up @@ -115,12 +114,20 @@ def run(self, bt: Bootstrapper) -> list[Phase]:
cached_wheel, _ = _cache.find_cached_wheel(
bt.ctx, bt.cache_wheel_server_url, self.work_item.req, version
)
if cached_wheel:
if (
cached_wheel
and self.work_item.req_type != RequirementType.TOP_LEVEL
):
logger.info(
f"{self.work_item.req.name}=={version}: wheel already cached "
f"at {cached_wheel.name}, skipping"
)
else:
# Every top-level version was already added to graph.json.
# Leaving a cached version unprocessed would give it no
# build edges if its wheel disappears before build-parallel.
# PrepareSource reuses the wheel's saved build requirements,
# and Build reuses the cached wheel instead of rebuilding it.
filtered.append((source_url, version))
if not filtered:
# Always process the highest version (first in
Expand Down
124 changes: 117 additions & 7 deletions tests/test_bootstrapper_iterative.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import logging
import pathlib
import typing
import zipfile
from unittest.mock import Mock, call, patch

import pytest
Expand All @@ -31,11 +32,12 @@
from resolvelib.resolvers import ResolverException

from fromager import bootstrapper, build_environment
from fromager.bootstrapper import _cache
from fromager.bootstrapper._build import Build
from fromager.bootstrapper._complete import Complete
from fromager.bootstrapper._phase import Phase
from fromager.bootstrapper._prepare_build import PrepareBuild
from fromager.bootstrapper._prepare_source import PrepareSource
from fromager.bootstrapper._prepare_source import PrepareSource, _bg_prepare_source
from fromager.bootstrapper._process_install_deps import ProcessInstallDeps
from fromager.bootstrapper._resolve import Resolve
from fromager.bootstrapper._start import Start
Expand Down Expand Up @@ -353,6 +355,108 @@ def test_empty_deps(self, tmp_context: WorkContext) -> None:


class TestPhaseResolve:
def test_cached_top_level_versions_keep_build_edges(
self, tmp_context: WorkContext
) -> None:
"""Cache hits retain metadata needed to rebuild after a cache miss."""
bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True)
req = Requirement("testpkg")
versions = [(f"source-{v}", Version(v)) for v in ("2.0", "1.0")]
cached: dict[Version, tuple[pathlib.Path, pathlib.Path]] = {}
for _, version in versions:
wheel = tmp_context.wheels_downloads / f"testpkg-{version}-py3-none-any.whl"
with zipfile.ZipFile(wheel, "w") as archive:
for phase, contents in (
("system", f"backend=={version}\n"),
("backend", ""),
("sdist", ""),
):
archive.writestr(
f"testpkg-{version}.dist-info/fromager-build-{phase}-requirements.txt",
contents,
)
unpacked = _cache._extract_build_reqs_from_wheel(
tmp_context.work_dir, req, version, wheel
)
assert unpacked is not None
cached[version] = wheel, unpacked

with patch.object(bt, "resolve_versions", return_value=versions):
bt._resolve_and_add_top_level(req)
item = _make_resolve_item(req_type=RequirementType.TOP_LEVEL)
item.bg_future = _make_resolved_future(versions)

def find_cached(
ctx: WorkContext,
cache_url: str | None,
requirement: Requirement,
version: Version,
) -> tuple[pathlib.Path, pathlib.Path]:
return cached[version]

with patch(
"fromager.bootstrapper._cache.find_cached_wheel", side_effect=find_cached
):
starts = item.run(bt)
assert {start.work_item.resolved_version for start in starts} == {
Version("1.0"),
Version("2.0"),
}

with patch("fromager.sources.download_source") as download_source:
for start in starts:
prepare = start.run(bt)[0]
wi = prepare.work_item
assert wi.resolved_version is not None
assert wi.source_url is not None
with patch(
"fromager.bootstrapper._cache.find_cached_wheel",
side_effect=find_cached,
):
prepared = _bg_prepare_source(
tmp_context, None, req, wi.resolved_version, wi.source_url
)
prepare.bg_future = _make_resolved_future(prepared)
with patch(
"fromager.build_environment.BuildEnvironment", return_value=Mock()
):
phases = prepare.run(bt)
backend = next(phase for phase in phases if isinstance(phase, Resolve))
backend.bg_future = _make_resolved_future(
[("backend-source", wi.resolved_version)]
)
with patch(
"fromager.bootstrapper._cache.find_cached_wheel",
return_value=(None, None),
):
backend_start = backend.run(bt)[0]
backend_start.run(bt)
with patch.object(Build, "_build_wheel") as build_wheel:
assert Build(wi).do_build(tmp_context) == (
cached[wi.resolved_version][0],
None,
)
build_wheel.assert_not_called()
download_source.assert_not_called()

# The remote cache can now disappear: a source rebuild cannot run in
# round 1 before its backend, unlike a cache-skipped bare graph node.
graph = tmp_context.dependency_graph
for _, version in versions:
assert [
node.key
for node in graph.nodes[f"testpkg=={version}"].iter_build_requirements()
] == [f"backend=={version}"]
topology = graph.get_build_topology(tmp_context)
topology.prepare()
ready = topology.get_available()
assert {node.key for node in ready} == {"backend==1.0", "backend==2.0"}
topology.done(*ready)
assert {node.key for node in topology.get_available()} == {
"testpkg==1.0",
"testpkg==2.0",
}

def test_single_version(self, tmp_context: WorkContext) -> None:
bt = bootstrapper.Bootstrapper(tmp_context)
item = _make_resolve_item()
Expand Down Expand Up @@ -405,12 +509,19 @@ def test_preserves_why_snapshot(self, tmp_context: WorkContext) -> None:

assert result[0].work_item.why_snapshot == snapshot

def test_filters_cached_versions_in_multiple_versions_mode(
self, tmp_context: WorkContext
@pytest.mark.parametrize(
"req_type, expected",
[
(RequirementType.INSTALL, {"1.0", "3.0"}),
(RequirementType.TOP_LEVEL, {"1.0", "2.0", "3.0"}),
],
)
def test_filters_cached_dependency_versions_in_multiple_versions_mode(
self, tmp_context: WorkContext, req_type: RequirementType, expected: set[str]
) -> None:
"""Cached versions are filtered out before creating Starts."""
"""Skip cached dependency versions without stranding top-level nodes."""
bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True)
item = _make_resolve_item()
item = _make_resolve_item(req_type=req_type)
item.bg_future = _make_resolved_future(
[
("url-3.0", Version("3.0")),
Expand All @@ -431,9 +542,8 @@ def mock_cache(
):
result = item.run(bt)

assert len(result) == 2
versions = {str(it.work_item.resolved_version) for it in result}
assert versions == {"1.0", "3.0"}
assert versions == expected

def test_all_cached_keeps_highest_version(self, tmp_context: WorkContext) -> None:
"""If all versions are cached, keeps the highest for dependency discovery."""
Expand Down
Loading