Skip to content

Commit dbcd337

Browse files
authored
Merge pull request #1227 from tiran/tiran/downloads-module
refactor(downloads): centralize download helpers in new module
2 parents 3724915 + 1b790d8 commit dbcd337

10 files changed

Lines changed: 515 additions & 208 deletions

File tree

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ exclude = [
208208

209209
[[tool.mypy.overrides]]
210210
# packages without typing annotations and stubs
211-
module = ["hatchling", "hatchling.build", "license_expression", "pyproject_hooks", "requests_mock", "resolver", "spdx_tools.*", "stevedore"]
211+
module = ["hatchling", "hatchling.build", "license_expression", "pyproject_hooks", "requests_mock", "resolver", "spdx_tools.*", "stevedore", "wheel.*"]
212212
ignore_missing_imports = true
213213

214214
[tool.basedpyright]

‎src/fromager/commands/build.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
clickext,
2424
context,
2525
dependency_graph,
26+
downloads,
2627
hooks,
2728
metrics,
2829
overrides,
@@ -33,7 +34,6 @@
3334
wheels,
3435
)
3536

36-
from .. import resolver
3737
from ..log import VERBOSE_LOG_FMT, ThreadLogFilter, req_ctxvar_context
3838

3939
logger = logging.getLogger(__name__)
@@ -495,7 +495,7 @@ def _is_wheel_built(
495495
pbi = wkctx.package_build_info(req)
496496
build_tag_from_settings = pbi.build_tag(resolved_version)
497497
build_tag = build_tag_from_settings if build_tag_from_settings else (0, "")
498-
wheel_basename = resolver.extract_filename_from_url(url)
498+
wheel_basename = downloads.extract_filename_from_url(url)
499499
_, _, build_tag_from_name, _ = parse_wheel_filename(wheel_basename)
500500
existing_build_tag = build_tag_from_name if build_tag_from_name else (0, "")
501501
if (

‎src/fromager/downloads.py‎

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
from __future__ import annotations
2+
3+
import logging
4+
import os
5+
import pathlib
6+
import tarfile
7+
import tempfile
8+
import typing
9+
import zipfile
10+
from urllib.parse import unquote, urlparse
11+
12+
from packaging.utils import parse_sdist_filename, parse_wheel_filename
13+
from wheel import wheelfile
14+
15+
from . import gitutils
16+
from .http_retry import RETRYABLE_EXCEPTIONS, retry_on_exception
17+
from .request_session import session
18+
19+
logger = logging.getLogger(__name__)
20+
21+
22+
def extract_filename_from_url(url: str) -> str:
23+
"""Extract filename from URL and decode it."""
24+
path = urlparse(url).path
25+
filename = os.path.basename(path)
26+
return unquote(filename)
27+
28+
29+
def download_url(
30+
*,
31+
destination_dir: pathlib.Path,
32+
url: str,
33+
destination_filename: str | None = None,
34+
) -> pathlib.Path:
35+
"""Download a URL to destination_dir, returning the local path.
36+
37+
Returns immediately if the file already exists. Downloads to a
38+
temporary file in the same directory first and uses :func:`os.rename`
39+
for an atomic move on success, avoiding partial files. Retries on
40+
transient network errors.
41+
42+
.. versionadded:: 0.90.0
43+
"""
44+
basename = (
45+
destination_filename if destination_filename else extract_filename_from_url(url)
46+
)
47+
outfile = destination_dir / basename
48+
logger.debug(
49+
"looking for %s %s",
50+
outfile,
51+
"(exists)" if outfile.exists() else "(not there)",
52+
)
53+
if outfile.exists():
54+
logger.debug("already have %s", outfile)
55+
return outfile
56+
57+
destination_dir.mkdir(parents=True, exist_ok=True)
58+
59+
@retry_on_exception(
60+
exceptions=RETRYABLE_EXCEPTIONS,
61+
max_attempts=5,
62+
backoff_factor=1.5,
63+
max_backoff=120.0,
64+
)
65+
def _download_with_retry() -> pathlib.Path:
66+
"""Download to a temporary file, then atomically rename."""
67+
logger.debug("reading from %s", url)
68+
# NamedTemporaryFile in the same directory ensures os.rename() is
69+
# atomic (same filesystem). delete=False so we control cleanup.
70+
# Temp filename looks like ".tmp.foo-1.2.tar.gz.abc123def".
71+
tmp = tempfile.NamedTemporaryFile(
72+
dir=destination_dir,
73+
prefix=f".tmp.{basename}.",
74+
delete=False,
75+
)
76+
temp_path = pathlib.Path(tmp.name)
77+
try:
78+
with session.get(url, stream=True) as r:
79+
r.raise_for_status()
80+
logger.debug("writing to %s", temp_path)
81+
for chunk in r.iter_content(chunk_size=64 * 1024):
82+
if chunk:
83+
tmp.write(chunk)
84+
tmp.close()
85+
# Atomic rename on the same filesystem
86+
os.rename(temp_path, outfile)
87+
logger.info("saved %s", outfile)
88+
return outfile
89+
except BaseException:
90+
tmp.close()
91+
temp_path.unlink(missing_ok=True)
92+
raise
93+
94+
result: pathlib.Path = _download_with_retry()
95+
return result
96+
97+
98+
def download_sdist(
99+
*,
100+
destination_dir: pathlib.Path,
101+
url: str,
102+
destination_filename: str | None = None,
103+
) -> pathlib.Path:
104+
"""Download a source distribution and verify it.
105+
106+
Accepts ``.tar.gz`` and ``.zip`` sdists. Validates the filename
107+
with :func:`packaging.utils.parse_sdist_filename` before
108+
downloading and checks that the archive is non-empty afterwards.
109+
110+
.. note::
111+
112+
The function does not validate ``PKG-INFO`` metadata or the
113+
required top-level directory described in the `source distribution
114+
file format
115+
<https://packaging.python.org/en/latest/specifications/source-distribution-format/>`_
116+
specification.
117+
118+
.. versionadded:: 0.90.0
119+
"""
120+
basename = (
121+
destination_filename if destination_filename else extract_filename_from_url(url)
122+
)
123+
# Validates name, version, and extension (.tar.gz or .zip);
124+
# raises InvalidSdistFilename on malformed names.
125+
parse_sdist_filename(basename)
126+
127+
filepath = download_url(
128+
destination_dir=destination_dir,
129+
url=url,
130+
destination_filename=basename,
131+
)
132+
133+
if basename.endswith(".tar.gz"):
134+
with tarfile.open(filepath, mode="r:gz") as tar:
135+
if tar.next() is None:
136+
raise tarfile.TarError(f"empty tar file: {filepath}")
137+
elif basename.endswith(".zip"):
138+
with zipfile.ZipFile(filepath) as zf:
139+
if not zf.namelist():
140+
raise zipfile.BadZipFile(f"empty zip file: {filepath}")
141+
else:
142+
typing.assert_never(basename) # type: ignore[arg-type]
143+
144+
return filepath
145+
146+
147+
def download_wheel(
148+
*,
149+
destination_dir: pathlib.Path,
150+
url: str,
151+
destination_filename: str | None = None,
152+
) -> pathlib.Path:
153+
"""Download a wheel and verify it.
154+
155+
Validates that the filename is a valid wheel name (using
156+
:func:`packaging.utils.parse_wheel_filename`) before downloading,
157+
then checks that the zip archive is non-empty.
158+
159+
.. note::
160+
161+
The function does not validate dist-info ``METADATA`` and other
162+
files described in the `binary distribution format
163+
<https://packaging.python.org/en/latest/specifications/binary-distribution-format/>`_
164+
specification.
165+
166+
.. versionadded:: 0.90.0
167+
"""
168+
basename = (
169+
destination_filename if destination_filename else extract_filename_from_url(url)
170+
)
171+
# Validates name, version, build tag, tags, and .whl extension;
172+
# raises InvalidWheelFilename on malformed names.
173+
parse_wheel_filename(basename)
174+
175+
filepath = download_url(
176+
destination_dir=destination_dir,
177+
url=url,
178+
destination_filename=basename,
179+
)
180+
181+
# validate wheel file: WheelFile checks for dist-info dir and RECORD.
182+
# NOTE: Does not validate that METADATA file is valid or consistent with
183+
# wheel's distribution name and version.
184+
# https://packaging.python.org/en/latest/specifications/binary-distribution-format/
185+
with wheelfile.WheelFile(filepath) as wf:
186+
di_metadata = f"{wf.dist_info_path}/METADATA"
187+
if di_metadata not in wf.namelist():
188+
raise wheelfile.WheelError(f"{di_metadata!r} missing")
189+
190+
return filepath
191+
192+
193+
def download_git_source(
194+
*,
195+
destination_dir: pathlib.Path,
196+
vcs_url: str,
197+
require_ref: bool = True,
198+
) -> pathlib.Path:
199+
"""Clone a git repository from a pip VCS URL.
200+
201+
Parses *vcs_url* with :func:`~fromager.gitutils.parse_vcs_url`
202+
and clones using :func:`~fromager.gitutils.git_clone_fast`
203+
with recursive submodules.
204+
205+
Raises ``FileExistsError`` if the destination directory already
206+
exists.
207+
208+
.. versionadded:: 0.90.0
209+
"""
210+
repo_url, ref = gitutils.parse_vcs_url(vcs_url, require_ref=require_ref)
211+
212+
# git clone fails when the target directory is not empty.
213+
try:
214+
destination_dir.mkdir(parents=True, exist_ok=False)
215+
except FileExistsError:
216+
raise FileExistsError(
217+
f"destination directory {destination_dir} already exists, "
218+
f"cannot clone {vcs_url}"
219+
) from None
220+
gitutils.git_clone_fast(
221+
output_dir=destination_dir,
222+
repo_url=repo_url,
223+
ref=ref,
224+
)
225+
return destination_dir

‎src/fromager/gitutils.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,16 @@
1+
from __future__ import annotations
2+
13
import logging
24
import pathlib
35
import typing
46
from urllib.parse import urlparse
57

68
from packaging.requirements import Requirement
79

8-
from fromager import context, external_commands
10+
from fromager import external_commands
11+
12+
if typing.TYPE_CHECKING:
13+
from fromager import context
914

1015
logger = logging.getLogger(__name__)
1116

‎src/fromager/resolver.py‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from collections.abc import Iterable
1515
from operator import attrgetter
1616
from platform import python_version
17-
from urllib.parse import quote, unquote, urlparse
17+
from urllib.parse import quote, urlparse
1818

1919
import pypi_simple
2020
import resolvelib
@@ -33,6 +33,7 @@
3333
from . import overrides
3434
from .candidate import Candidate, Cooldown
3535
from .constraints import Constraints
36+
from .downloads import extract_filename_from_url as extract_filename_from_url
3637
from .extras_provider import ExtrasProvider
3738
from .http_retry import RETRYABLE_EXCEPTIONS, retry_on_exception
3839
from .request_session import session
@@ -204,13 +205,6 @@ def _compute_max_age_cutoff(
204205
return bootstrap_time - ctx.max_release_age
205206

206207

207-
def extract_filename_from_url(url: str) -> str:
208-
"""Extract filename from URL and decode it."""
209-
path = urlparse(url).path
210-
filename = os.path.basename(path)
211-
return unquote(filename)
212-
213-
214208
class LogReporter(resolvelib.BaseReporter):
215209
"""Report resolution events
216210

0 commit comments

Comments
 (0)