Skip to content

Commit fba6d92

Browse files
authored
ci(timing): drop step and leg patterns no workflow emits (#2230) (#2240)
Problem: STEPS_OF_INTEREST kept a `3\.10 cell` alternative that no step matches since #1660 renamed it, and non_blocking still treated any job ending in ` (experimental)` as non-blocking, a rule only the unit-tests.yml legs that #1732 moved to experimental.yml needed. Fix: drop the alternative, EXPERIMENTAL_SUFFIX and its branch; an experimental leg is non-blocking through NON_BLOCKING_WORKFLOWS alone. workflows.rst says so. Test: test_every_step_of_interest_names_a_real_step requires each alternative of each pattern to match a step in .github/workflows/; test_a_blocking_leg_stays_hard asserts an (experimental) name in unit-tests.yml is blocking. LEGACY is gone.
1 parent b0f731c commit fba6d92

3 files changed

Lines changed: 36 additions & 23 deletions

File tree

‎docs/source/contributing/workflows.rst‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -781,10 +781,9 @@ fire while the pool stays saturated, and the JUnit timings are what show whether
781781

782782
``H1``, ``H2`` and ``H4`` on a non-blocking job are warnings, not HARD, so they do
783783
not fail the run (:issue:`1732`): on every job of :file:`experimental.yml`, which no
784-
required context names, on every :ref:`KTLO <ktlo-tier>` leg, a job whose name ends in ``(KTLO)``
785-
(:issue:`1633`), and on a job whose name ends in ``(experimental)`` in any workflow. That
786-
last also covers older runs of :file:`unit-tests.yml`, whose experimental legs ran under
787-
``test``'s 45-minute cap until :issue:`1732` moved them to :file:`experimental.yml`.
784+
required context names, and on every :ref:`KTLO <ktlo-tier>` leg, a job whose name ends in ``(KTLO)``
785+
(:issue:`1633`). A job whose name ends in ``(experimental)`` counts by its workflow, not
786+
its name.
788787
The finding keeps its ``H`` id and says it is non-blocking. A
789788
KTLO leg past its job's ``timeout-minutes`` is still cancelled, and its check run
790789
reads red, though since :issue:`1738` nothing gating waits on it, so that warning is worth

‎tests/project/test_ci_timing.py‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,27 @@ def test_the_real_unit_tests_workflow(self) -> 'None':
276276
self.assertEqual((experimental['id'], experimental['timeout'],
277277
ci_timing.step_cap(experimental, ci_timing.TESTS_STEP)), ('experimental', 140, 120))
278278

279+
def test_every_step_of_interest_names_a_real_step(self) -> 'None':
280+
"""#2230: each alternative of :data:`STEPS_OF_INTEREST` matches a step some workflow has.
281+
282+
A pattern, or one alternative of a ``(?:...)`` group in it, that no step
283+
matches is dead, and makes the steps reported look wider than they are.
284+
285+
"""
286+
names = {step['name'] for path in sorted(WORKFLOWS.glob('*.yml'))
287+
for spec in ci_timing.parse_workflow(path.read_text(encoding='utf-8'))
288+
for step in spec['steps']}
289+
self.assertTrue(names)
290+
for pattern in ci_timing.STEPS_OF_INTEREST:
291+
groups = list(re.finditer(r'\(\?:([^()]*)\)', pattern))
292+
# Only a group's alternatives are split out, so a bare one would go unchecked.
293+
self.assertNotIn('|', re.sub(r'\(\?:[^()]*\)', '', pattern), pattern)
294+
variants = [pattern[:group.start()] + option + pattern[group.end():]
295+
for group in groups for option in group.group(1).split('|')] or [pattern]
296+
for variant in variants:
297+
with self.subTest(pattern=pattern, alternative=variant):
298+
self.assertTrue([name for name in names if re.fullmatch(variant, name)])
299+
279300
def test_a_cap_by_leg_reads_as_none(self) -> 'None':
280301
"""An expression is no one cap; a literal beside it still reads."""
281302
specs = ci_timing.parse_workflow(
@@ -1017,10 +1038,6 @@ class TestNonBlockingLegs(unittest.TestCase):
10171038
#: (workflow, job name, its cap in the real workflow file).
10181039
EXPERIMENTAL = ('experimental.yml', 'Python graalpy-25.4 (experimental)', 140)
10191040
KTLO = ('unit-tests.yml', 'Python 3.10 (KTLO)', 45)
1020-
#: Experimental legs from before the split, run under ``test``'s 45-minute cap and
1021-
#: still in the 7-day window for a week after it merges.
1022-
LEGACY = (('unit-tests.yml', 'Python graalpy-25.4 (experimental)', 45),
1023-
('unit-tests.yml', 'Python 3.14t (experimental)', 45))
10241041
BLOCKING = ('unit-tests.yml', 'Python 3.14', 45)
10251042

10261043
def levels(self, change: 'Callable[[dict], None]') -> 'list[tuple[str, str]]':
@@ -1029,7 +1046,7 @@ def levels(self, change: 'Callable[[dict], None]') -> 'list[tuple[str, str]]':
10291046
return [(item['id'], item['level']) for item in ci_timing.evaluate(report, baseline)]
10301047

10311048
def test_each_cap_rule_only_warns_on_a_non_blocking_leg(self) -> 'None':
1032-
for workflow, name, cap in (self.EXPERIMENTAL, self.KTLO, *self.LEGACY):
1049+
for workflow, name, cap in (self.EXPERIMENTAL, self.KTLO):
10331050
def leg(row: 'dict') -> 'None':
10341051
row.update(workflow=workflow, name=name, cap=cap)
10351052

@@ -1051,11 +1068,12 @@ def test_a_blocking_leg_stays_hard(self) -> 'None':
10511068
self.assertFalse(ci_timing.non_blocking(workflow, name))
10521069
self.assertEqual(self.levels(lambda r: r['jobs'][0].update(p90=cap / 2)), [('H1', 'hard')])
10531070
self.assertEqual(self.levels(lambda r: r['steps'][0].update(p90=10.0)), [('H2', 'hard')])
1054-
# The same name in experimental.yml would not be, and either suffix counts only
1055-
# at the end.
1071+
# The same name in experimental.yml would not be, and the KTLO suffix counts
1072+
# only at the end.
10561073
self.assertTrue(ci_timing.non_blocking('experimental.yml', name))
10571074
self.assertFalse(ci_timing.non_blocking(workflow, 'Python 3.10 (KTLO) extra'))
1058-
self.assertFalse(ci_timing.non_blocking(workflow, 'Python 3.14t (experimental) extra'))
1075+
# #2230: an experimental leg is non-blocking by its workflow, not by its name.
1076+
self.assertFalse(ci_timing.non_blocking(workflow, 'Python 3.14t (experimental)'))
10591077

10601078
def run_report(self, workflow: 'str', name: 'str', took: 'float') -> 'tuple[int, str]':
10611079
"""``_report`` on ``ci_timing.MIN_SAMPLES`` runs of one job taking ``took`` minutes, against the real file."""
@@ -1080,7 +1098,7 @@ def run_report(self, workflow: 'str', name: 'str', took: 'float') -> 'tuple[int,
10801098
return status, stdout.getvalue()
10811099

10821100
def test_the_run_stays_green_on_a_non_blocking_leg_past_its_share(self) -> 'None':
1083-
for workflow, name, cap in (self.EXPERIMENTAL, self.KTLO, *self.LEGACY):
1101+
for workflow, name, cap in (self.EXPERIMENTAL, self.KTLO):
10841102
with self.subTest(job=name):
10851103
status, stdout = self.run_report(workflow, name, cap / 2 + 1)
10861104
self.assertEqual(status, 0)
@@ -1098,7 +1116,8 @@ def test_the_run_fails_on_a_blocking_leg_past_its_share(self) -> 'None':
10981116
def test_the_rule_is_documented(self) -> 'None':
10991117
page = ' '.join(DOC.read_text(encoding='utf-8').split())
11001118
self.assertIn('``H1``, ``H2`` and ``H4`` on a non-blocking job are warnings', page)
1101-
self.assertIn('a job whose name ends in ``(experimental)``', page)
1119+
self.assertIn('a job whose name ends in ``(KTLO)``', page)
1120+
self.assertIn('``(experimental)`` counts by its workflow, not its name', page)
11021121

11031122

11041123
def step_blocks(text: 'str', name: 'str') -> 'list[str]':

‎util/ci_timing.py‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -116,9 +116,6 @@
116116
#: The suffix every KTLO leg's job name carries (#1633); such a leg's steps cannot
117117
#: fail it (#1672).
118118
KTLO_SUFFIX = ' (KTLO)'
119-
#: The suffix every experimental leg's job name carries (#1732); such a leg's
120-
#: steps cannot fail it either.
121-
EXPERIMENTAL_SUFFIX = ' (experimental)'
122119

123120
#: The aggregate job the ruleset requires; its completion is "mergeable".
124121
REQUIRED_JOB = 'Required checks passed'
@@ -157,8 +154,7 @@ def _coverage_legs() -> 'tuple[str, tuple[str, ...]]':
157154
r'Report coverage',
158155
r'Run full test suite',
159156
r'Run tests/.* under plain unittest',
160-
# engine-tests' KTLO step, named by its one version until #1660.
161-
r'Run the (?:3\.10 cell|KTLO cells).*',
157+
r'Run the KTLO cells.*',
162158
)
163159

164160
# Thresholds. Each is anchored to a value measured on 2026-10-05..09 for #1538,
@@ -1128,12 +1124,11 @@ def _fmt(value: 'Optional[float]', digits: 'int' = 1) -> 'str':
11281124
def non_blocking(workflow: 'str', job: 'str') -> 'bool':
11291125
"""Whether ``job``'s cap findings (H1, H2, H4) are warnings rather than HARD.
11301126
1131-
Every job of :data:`NON_BLOCKING_WORKFLOWS` is non-blocking, and so is a KTLO leg
1132-
or an experimental leg from before the split, wherever it ran (#1732).
1127+
Every job of :data:`NON_BLOCKING_WORKFLOWS` is non-blocking (#1732), and so is a
1128+
KTLO leg, wherever it runs.
11331129
11341130
"""
1135-
return (workflow in NON_BLOCKING_WORKFLOWS
1136-
or job.endswith(KTLO_SUFFIX) or job.endswith(EXPERIMENTAL_SUFFIX))
1131+
return workflow in NON_BLOCKING_WORKFLOWS or job.endswith(KTLO_SUFFIX)
11371132

11381133

11391134
def evaluate(report: 'dict[str, Any]', baseline: 'Optional[dict[str, Any]]') -> 'list[dict[str, str]]':

0 commit comments

Comments
 (0)