Skip to content

Commit 81d55de

Browse files
committed
ci(conda): move platform pins up and give them a review-by date (#1851)
- Move PLATFORM_PINS into conda/platform-pins.toml, so the weekly job can rewrite it; each pin carries a version, a review-by date and a reason. - Add util/conda_vendor.py --refresh-pins: it reads PyPI's JSON, finds the newest release up to the installed one with a wheel every conda Python installs on the pinned platform, abi3 and universal2 included, and moves the pin there, due for review REVIEW_HORIZON out. A pin that reaches the installed release, or already pins it, is dropped. A PyPI failure warns and keeps the pin. - Run it in conda-update before the requirements are written. - --check stays offline and now fails on a pin past its review-by date. - Keep the table out of BOT_MANAGED_PATHS, pinned by a test, and correct that comment's claim about which files tests read. - Read the pin from the table in the #1827 tests, and fix today in the #1814 --check tests, so a moved or overdue pin does not break them. Closes #1851
1 parent 06e16db commit 81d55de

7 files changed

Lines changed: 944 additions & 60 deletions

File tree

‎.github/workflows/cron-conda.yml‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,14 +161,30 @@ jobs:
161161
# extra, walks what they require through the installed metadata, and
162162
# pins each package at the version installed here, but for its
163163
# `PLATFORM_PINS` (#1827). So this install is the step that resolves the
164-
# newest releases, and the script itself reads nothing from the network.
164+
# newest releases, and of the script's runs only the next one reads the
165+
# network.
165166
- name: Install and Setup
166167
run: |
167168
set -x
168169
169170
python -m pip install -U pip setuptools wheel packaging
170171
python -m pip install -e .[all]
171172
173+
# #1851: asks PyPI for each `PLATFORM_PINS` package's releases and moves
174+
# its pin in conda/platform-pins.toml up to the newest release, up to the
175+
# one installed above, with a wheel for the pinned platform, renewing its
176+
# review-by date; a pin that reaches the installed release, or already
177+
# pins it, goes.
178+
# The step below then writes the moved pin into conda/requirements.txt,
179+
# and Commit Changes commits both. The table is not one of
180+
# util/verdict_covers_checkout.py's bot-managed paths, which says why:
181+
# the workflows chained off Unit Tests wait for such a commit's own run
182+
# (#1643). A package PyPI cannot be read for keeps its pin, with a
183+
# warning, and a pin past its review-by date gets one too; either way the
184+
# step passes.
185+
- name: Refresh the platform pins
186+
run: python util/conda_vendor.py --refresh-pins
187+
172188
- name: Setup Conda Distribution
173189
run: |
174190
export PCAPKIT_DEVMODE=1

‎conda/platform-pins.toml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# The packages conda/requirements.txt pins apart on some conda platforms, as a
2+
# table per package and conda subdir (#1827): the version that platform vendors
3+
# in place of the installed release, the date the pin is due for review by, and
4+
# why the platform cannot take the installed release.
5+
#
6+
# util/conda_vendor.py reads this file. Conda Update's `conda-update` job runs
7+
# its `--refresh-pins` first, which asks PyPI for the newest release, up to the
8+
# installed one, with a wheel each conda Python installs on the pinned platform.
9+
# It moves the pin up to that release with a fresh review-by date, and drops a
10+
# pin that reaches the installed release or already pins it (#1851). `--check`
11+
# fails on a pin past its review-by date: check by hand that the pin still
12+
# holds, then set a new date. The job writes this whole file, this comment
13+
# included, so anything a pin needs said goes in its `reason`.
14+
15+
[cryptography.osx-64]
16+
version = "48.0.1"
17+
review-by = 2027-01-15
18+
reason = "Its newest releases ship no Intel or universal2 macOS wheel, so the osx-64 build would compile them from source; 49.0.0 was the first (#1827)."

‎tests/project/test_conda_cryptography_pin_1827.py‎

Lines changed: 74 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,23 @@
11
# -*- coding: utf-8 -*-
22
"""cryptography is pinned apart on Intel macOS, which has no wheel of its newest releases (#1827).
33
4-
* :file:`conda/requirements.txt` carries a cryptography line per platform pin,
5-
and on each conda build exactly one of them applies: 48.0.1 on ``osx-64``,
6-
the installed release elsewhere.
4+
* :file:`conda/requirements.txt` carries a line per platform pin of each package
5+
:file:`conda/platform-pins.toml` pins, and of cryptography whether or not it
6+
is, and on each conda build exactly one of a package's lines applies: the
7+
table's pin on its platform, the installed release elsewhere. Read from the
8+
table rather than spelt here, since the weekly Conda Update moves and drops
9+
pins (#1851).
710
* :file:`util/conda_vendor.py` writes those lines from ``PLATFORM_PINS``, and
811
``--check`` passes on them and fails when the table entry or the file's pin
912
goes, when the pin moves, and when the table names a package or platform no
10-
build has. All offline, on stand-in metadata.
13+
build has. All offline, on stand-in metadata and a stand-in 48.0.1 pin.
1114
1215
"""
1316

1417
from __future__ import annotations
1518

1619
import contextlib
20+
import datetime
1721
import importlib.util
1822
import io
1923
import pathlib
@@ -33,6 +37,7 @@
3337

3438
ROOT = pathlib.Path(__file__).resolve().parents[2]
3539
REQUIREMENTS = ROOT / 'conda' / 'requirements.txt'
40+
#: The Intel macOS line :func:`stand_in`'s pin writes.
3641
INTEL = 'cryptography==48.0.1; sys_platform == "darwin" and platform_machine == "x86_64"'
3742
#: The other line, whose pin the weekly Conda Update moves.
3843
REST = re.compile(r'cryptography==(\S+); sys_platform != "darwin" or platform_machine != "x86_64"')
@@ -59,45 +64,74 @@ def committed() -> 'list[str]':
5964
if line.startswith('cryptography')]
6065

6166

67+
def pinned(*pins: 'tuple[str, str, str]') -> 'dict[str, dict[str, Any]]':
68+
"""A ``PLATFORM_PINS`` of (package, conda subdir, version), with a stand-in date and reason."""
69+
module, table = vendor(), {} # type: Any, dict[str, dict[str, Any]]
70+
for name, subdir, version in pins:
71+
table.setdefault(name, {})[subdir] = module.Pin(version, datetime.date(2027, 1, 15), 'a stand-in')
72+
return table
73+
74+
75+
def stand_in() -> 'Any':
76+
"""``PLATFORM_PINS`` patched to cryptography 48.0.1 on osx-64, as #1827 committed it."""
77+
return mock.patch.object(vendor(), 'PLATFORM_PINS', pinned(('cryptography', 'osx-64', '48.0.1')))
78+
79+
80+
def written() -> 'list[str]':
81+
"""The cryptography lines the writer gives :func:`stand_in`'s pin, with 50.0.2 installed."""
82+
module = vendor()
83+
envs = module.environments(module.conda_pythons())
84+
with stand_in():
85+
return module.lines(cryptography(module, envs), envs, lambda name: '50.0.2')
86+
87+
6288
def cryptography(module: 'Any', envs: 'list[dict[str, str]]') -> 'dict[str, Any]':
6389
"""What the conda builds vendor for ``pyproject.toml``'s cryptography line, on stand-in metadata."""
6490
return module.derive([req for req in module.direct() if req.name == 'cryptography'], envs, lambda name: [])
6591

6692

6793
class CommittedLinesTests(unittest.TestCase):
68-
"""The committed cryptography lines, evaluated for each conda build."""
94+
"""The committed lines of each pinned package, and of cryptography, evaluated for each conda build."""
6995

70-
def test_each_build_reads_one_cryptography_line(self) -> None:
96+
def test_each_build_reads_one_line_per_pinned_package(self) -> None:
7197
from packaging.requirements import Requirement
7298
module = vendor()
73-
lines = committed()
74-
self.assertIn(INTEL, lines)
75-
self.assertEqual(len([line for line in lines if REST.fullmatch(line)]), 1, lines)
76-
self.assertEqual(module.PLATFORM_PINS, {'cryptography': {'osx-64': '48.0.1'}})
77-
for env in module.environments(module.conda_pythons()):
78-
subdir = next(subdir for subdir, (sys_platform, _, machine) in module.PLATFORMS.items()
79-
if (sys_platform, machine) == (env['sys_platform'], env['platform_machine']))
80-
with self.subTest(subdir=subdir, python=env['python_version']):
81-
live = [line for line in lines if Requirement(line).marker.evaluate(env)]
82-
self.assertEqual(len(live), 1, live)
83-
self.assertEqual(live[0] == INTEL, subdir == 'osx-64', live)
99+
lines = [Requirement(line) for line in REQUIREMENTS.read_text(encoding='utf-8').splitlines()
100+
if line.split('#', 1)[0].strip()]
101+
for name in sorted({'cryptography', *module.PLATFORM_PINS}):
102+
pins = module.PLATFORM_PINS.get(name, {})
103+
reads = {} # type: dict[tuple[str, str], str]
104+
for env in module.environments(module.conda_pythons()):
105+
subdir = next(subdir for subdir, (sys_platform, _, machine) in module.PLATFORMS.items()
106+
if (sys_platform, machine) == (env['sys_platform'], env['platform_machine']))
107+
with self.subTest(package=name, subdir=subdir, python=env['python_version']):
108+
live = [str(req) for req in lines
109+
if req.name == name and (req.marker is None or req.marker.evaluate(env))]
110+
self.assertEqual(len(live), 1, live)
111+
reads[subdir, env['python_version']] = live[0]
112+
if subdir in pins:
113+
self.assertEqual(str(Requirement(live[0]).specifier), f'=={pins[subdir].version}')
114+
# The builds no pin names read none of the pinned lines.
115+
on_pinned = {line for (subdir, _), line in reads.items() if subdir in pins}
116+
self.assertFalse({line for (subdir, _), line in reads.items() if subdir not in pins} & on_pinned,
117+
reads)
84118

85119

86120
class WriterTests(unittest.TestCase):
87121
"""The writer emits a line per platform pin."""
88122

89123
def test_the_writer_emits_the_platform_lines(self) -> None:
90-
module = vendor()
91-
envs = module.environments(module.conda_pythons())
92-
self.assertEqual(module.lines(cryptography(module, envs), envs, lambda name: '50.0.2'),
124+
self.assertEqual(written(),
93125
[INTEL, 'cryptography==50.0.2; sys_platform != "darwin" or platform_machine != "x86_64"'])
126+
self.assertIsNotNone(REST.fullmatch(written()[1]))
94127

95128
def test_a_python_marker_is_kept_beside_the_platform_one(self) -> None:
96129
from packaging.requirements import Requirement
97130
module = vendor()
98131
envs = module.environments(['3.11', '3.12'])
99132
found = module.derive([Requirement('cryptography; python_version >= "3.12"')], envs, lambda name: [])
100-
lines = module.lines(found, envs, lambda name: '50.0.2')
133+
with stand_in():
134+
lines = module.lines(found, envs, lambda name: '50.0.2')
101135
self.assertEqual(lines, [
102136
'cryptography==48.0.1; (python_version == "3.12") and '
103137
'(sys_platform == "darwin" and platform_machine == "x86_64")',
@@ -111,38 +145,45 @@ def test_a_python_marker_is_kept_beside_the_platform_one(self) -> None:
111145

112146

113147
class CheckTests(unittest.TestCase):
114-
"""``--check`` against the committed cryptography lines."""
148+
"""``--check`` against the committed cryptography lines, and against :func:`stand_in`'s."""
115149

116150
def problems(self, text: 'str') -> 'list[str]':
117151
module = vendor()
118152
envs = module.environments(module.conda_pythons())
119153
return module.compare(text, cryptography(module, envs), envs)
120154

121155
def test_the_committed_lines_pass(self) -> None:
122-
lines = committed()
123-
self.assertIn(INTEL, lines)
124-
self.assertEqual(self.problems('\n'.join(lines)), [])
156+
self.assertEqual(self.problems('\n'.join(committed())), [])
157+
158+
def test_the_stand_in_lines_pass(self) -> None:
159+
lines = written()
160+
with stand_in():
161+
self.assertEqual(self.problems('\n'.join(lines)), [])
125162

126163
def test_without_the_override_the_platform_lines_fail(self) -> None:
164+
lines = written()
127165
with mock.patch.object(vendor(), 'PLATFORM_PINS', {}):
128-
problems = self.problems('\n'.join(committed()))
166+
problems = self.problems('\n'.join(lines))
129167
self.assertEqual(len(problems), 1, problems)
130168
self.assertTrue(problems[0].startswith('marker: cryptography has ['), problems)
131169

132170
def test_a_dropped_or_moved_platform_pin_fails(self) -> None:
133-
rest = [line for line in committed() if line != INTEL]
134-
dropped = self.problems('\n'.join(rest))
171+
rest = [line for line in written() if line != INTEL]
172+
with stand_in():
173+
dropped = self.problems('\n'.join(rest))
174+
unmarked = self.problems('\n'.join(rest + ['cryptography==48.0.1']))
175+
moved = self.problems('\n'.join(rest + [INTEL.replace('48.0.1', '49.0.0')]))
135176
self.assertEqual([problem.split(':')[0] for problem in dropped], ['marker'], dropped)
136-
unmarked = self.problems('\n'.join(rest + ['cryptography==48.0.1']))
137177
self.assertEqual([problem.split(':')[0] for problem in unmarked], ['marker'], unmarked)
138-
moved = self.problems('\n'.join(rest + [INTEL.replace('48.0.1', '49.0.0')]))
139178
self.assertEqual(moved, ["pin: cryptography pins '49.0.0' where sys_platform == \"darwin\" and "
140179
"platform_machine == \"x86_64\", and PLATFORM_PINS '48.0.1'"])
141180

142181
def test_an_override_no_build_can_use_fails(self) -> None:
143-
pins = {'cryptography': {'osx-64': '48.0.1', 'win-64': '1.0'}, 'gone': {'osx-64': '1.0'}}
182+
lines = written()
183+
pins = pinned(('cryptography', 'osx-64', '48.0.1'), ('cryptography', 'win-64', '1.0'),
184+
('gone', 'osx-64', '1.0'))
144185
with mock.patch.object(vendor(), 'PLATFORM_PINS', pins):
145-
problems = self.problems('\n'.join(committed()))
186+
problems = self.problems('\n'.join(lines))
146187
self.assertEqual(problems, ['override: cryptography is pinned for win-64, which no conda leg builds',
147188
'override: gone is pinned for osx-64, and no conda build vendors it'])
148189

@@ -155,7 +196,7 @@ def test_the_writer_refuses_an_override_no_build_can_use(self) -> None:
155196
module._requires = lambda name: []
156197
module._version = lambda name: '1.0'
157198
try:
158-
with mock.patch.object(module, 'PLATFORM_PINS', {'gone': {'osx-64': '1.0'}}), \
199+
with mock.patch.object(module, 'PLATFORM_PINS', pinned(('gone', 'osx-64', '1.0'))), \
159200
contextlib.redirect_stderr(io.StringIO()) as errors:
160201
self.assertEqual(module.main([]), 1)
161202
self.assertFalse(target.exists())

0 commit comments

Comments
 (0)