Skip to content

Commit cbe25db

Browse files
Merge pull request #1433 from smoparth/codex/issue-1418-source-provider-check
fix(build-sequence): validate recorded source providers
2 parents ff8d768 + 3fc6de3 commit cbe25db

5 files changed

Lines changed: 239 additions & 65 deletions

File tree

‎docs/concepts/resolver-architecture.rst‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ settings are used.
132132

133133
A configured resolver's ``download()`` receives only the version and the
134134
download URL of the selected candidate. Resolution and download can run in
135-
different commands, and only these values are saved in ``graph.json`` and
135+
different commands, and these values are saved in ``graph.json`` and
136136
``build-order.json``. The URL must therefore contain everything the download
137137
needs, such as the git ref in ``git+https://host/repo.git@refs/tags/v1.0``.
138138

@@ -163,14 +163,15 @@ The separate ``step prepare-source`` invocation accepts ``--source-kind`` and
163163
actual path and kind without changing its default path-only output.
164164
Without ``--source-kind``, it keeps the legacy archive behavior.
165165
``download-sequence`` calls the configured resolver's normal ``download()``
166-
for source entries. Before downloading, it checks ``source_provider`` against
167-
the current provider for each source entry and each ``prebuilt: true`` wheel
168-
requested with ``--include-wheels``. A missing or different provider fails
169-
before any downloads start. Older build orders without ``source_provider``
170-
still work with legacy settings; regenerate them before replaying a configured
171-
source. This checks the provider name, so replay should still use the same
172-
package settings. A runtime ``prebuilt`` entry is skipped even when the
173-
package has source settings.
166+
for source entries. Both sequence commands check each entry's recorded
167+
``source_provider`` against current settings, including prebuilt entries whose
168+
URLs ``download-sequence`` will skip. A missing or different provider fails
169+
before any downloads or builds start. Older build orders without
170+
``source_provider`` still work with legacy settings; regenerate them before
171+
replaying a configured source.
172+
These checks compare provider names, so replay should still use the same
173+
package settings. ``download-sequence`` still skips downloading a runtime
174+
``prebuilt`` entry even when the package has source settings.
174175
Built-in archive resolvers cache the resolved file for a later build. Git
175176
checkout resolvers clone into the work directory instead: with cleanup enabled,
176177
a later build removes that checkout and clones again; without cleanup, the
@@ -182,7 +183,7 @@ effective configured ``source_provider`` for provenance; the
182183
the actual downloaded artifact kind.
183184

184185
.. versionchanged:: 0.103.0
185-
``download-sequence`` uses configured source resolvers for source entries.
186+
Sequence commands check recorded providers before replaying a build order.
186187

187188
Configured prebuilt resolvers select the prebuilt bootstrap path. Fresh
188189
external wheels use their ``download()`` contract and are staged in the

‎src/fromager/commands/build.py‎

Lines changed: 30 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
)
3636

3737
from ..log import VERBOSE_LOG_FMT, ThreadLogFilter, req_ctxvar_context
38+
from .build_order import check_source_providers
3839

3940
logger = logging.getLogger(__name__)
4041

@@ -177,36 +178,37 @@ def build_sequence(
177178

178179
logger.info("reading build order from %s", build_order_file)
179180
with read.open_file_or_url(build_order_file) as f:
180-
for entry in progress.progress(json.load(f)):
181-
dist_name = entry["dist"]
182-
resolved_version = Version(entry["version"])
183-
source_download_url = entry["source_url"]
184-
185-
req = Requirement(f"{dist_name}=={resolved_version}")
186-
187-
with req_ctxvar_context(req, resolved_version):
188-
logger.info("building %s", resolved_version)
189-
entry = _build(
190-
wkctx=wkctx,
191-
resolved_version=resolved_version,
192-
req=req,
193-
source_download_url=source_download_url,
194-
force=force,
195-
cache_wheel_server_url=cache_wheel_server_url,
181+
build_order = json.load(f)
182+
# Check provider consistency for every entry, including prebuilt wheels.
183+
check_source_providers(wkctx, build_order)
184+
for entry in progress.progress(build_order):
185+
dist_name = entry["dist"]
186+
resolved_version = Version(entry["version"])
187+
source_download_url = entry["source_url"]
188+
189+
req = Requirement(f"{dist_name}=={resolved_version}")
190+
191+
with req_ctxvar_context(req, resolved_version):
192+
logger.info("building %s", resolved_version)
193+
entry = _build(
194+
wkctx=wkctx,
195+
resolved_version=resolved_version,
196+
req=req,
197+
source_download_url=source_download_url,
198+
force=force,
199+
cache_wheel_server_url=cache_wheel_server_url,
200+
)
201+
if entry.prebuilt:
202+
logger.info("downloaded prebuilt wheel %s", entry.wheel_filename.name)
203+
elif entry.skipped:
204+
logger.info(
205+
"skipping building wheel since %s already exists",
206+
entry.wheel_filename.name,
196207
)
197-
if entry.prebuilt:
198-
logger.info(
199-
"downloaded prebuilt wheel %s", entry.wheel_filename.name
200-
)
201-
elif entry.skipped:
202-
logger.info(
203-
"skipping building wheel since %s already exists",
204-
entry.wheel_filename.name,
205-
)
206-
else:
207-
logger.info("built %s", entry.wheel_filename.name)
208+
else:
209+
logger.info("built %s", entry.wheel_filename.name)
208210

209-
entries.append(entry)
211+
entries.append(entry)
210212

211213
metrics.summarize(wkctx, "Building")
212214

‎src/fromager/commands/build_order.py‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,39 @@
33
import json
44
import pathlib
55
import sys
6+
import typing
67

78
import click
9+
from packaging.requirements import Requirement
810

9-
from .. import clickext, overrides
11+
from .. import clickext, context, overrides
12+
13+
14+
def check_source_providers(
15+
wkctx: context.WorkContext, entries: typing.Iterable[dict[str, typing.Any]]
16+
) -> None:
17+
"""Reject build-order entries whose source provider differs from settings.
18+
19+
Args:
20+
wkctx: Context containing the current package settings.
21+
entries: Build-order entries to validate.
22+
23+
Raises:
24+
ValueError: A recorded provider does not match current settings.
25+
"""
26+
for entry in entries:
27+
recorded_provider = entry.get("source_provider")
28+
req = Requirement(f"{entry['dist']}=={entry['version']}")
29+
source_resolver = wkctx.package_build_info(req).source_resolver
30+
current_provider = (
31+
source_resolver.provider if source_resolver is not None else None
32+
)
33+
if recorded_provider != current_provider:
34+
raise ValueError(
35+
f"{req}: build order records source provider {recorded_provider!r}, "
36+
f"current settings select {current_provider!r}; "
37+
"use matching settings or regenerate the build order"
38+
)
1039

1140

1241
@click.group()

‎src/fromager/commands/download_sequence.py‎

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
from .. import context, progress, read, sources, threading_utils, wheels
1111
from ..log import req_ctxvar_context
12+
from .build_order import check_source_providers
1213

1314
logger = logging.getLogger(__name__)
1415

@@ -55,10 +56,10 @@ def download_sequence(
5556
5657
SDIST_SERVER_URL is a PyPI-compatible index used for optional wheel lookup
5758
58-
Configured sources use their resolver's normal download operation. Entries
59-
that download a configured source or prebuilt wheel require the recorded
60-
provider to match current settings. Legacy entries retain the
61-
``source_url_type`` selection behavior.
59+
Configured sources use their resolver's normal download operation. Every
60+
entry requires its recorded provider to match current settings, even when
61+
this command skips its URL. Legacy entries retain the ``source_url_type``
62+
selection behavior.
6263
6364
"""
6465
# The local wheel server is a trusted cache (wheels built in this or
@@ -73,23 +74,8 @@ def download_sequence(
7374
with read.open_file_or_url(build_order_file) as f:
7475
build_order = json.load(f)
7576

76-
# Check recorded providers before workers can change the download cache
77-
# or a Git work directory. Skipped URLs do not use a source resolver.
78-
for entry in build_order:
79-
if _skip_recorded_url(entry, include_wheels):
80-
continue
81-
recorded_provider = entry.get("source_provider")
82-
req = Requirement(f"{entry['dist']}=={entry['version']}")
83-
source_resolver = wkctx.package_build_info(req).source_resolver
84-
current_provider = (
85-
source_resolver.provider if source_resolver is not None else None
86-
)
87-
if recorded_provider != current_provider:
88-
raise ValueError(
89-
f"{req}: build order records source provider {recorded_provider!r}, "
90-
f"current settings select {current_provider!r}; "
91-
"use matching settings or regenerate the build order"
92-
)
77+
# Validate the whole order before workers alter the cache or Git work dirs.
78+
check_source_providers(wkctx, build_order)
9379

9480
def download_one(entry: dict[str, typing.Any]) -> None:
9581
req = Requirement(f"{entry['dist']}=={entry['version']}")

0 commit comments

Comments
 (0)