Repository navigation
Expand file tree
/
Copy pathcheck_e2e_coverage.py
More file actions
200 lines (168 loc) · 8.09 KB
/
Copy pathcheck_e2e_coverage.py
File metadata and controls
200 lines (168 loc) · 8.09 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
#!/usr/bin/env python3
"""Every e2e test runs somewhere, or says why it cannot (rule R5).
WHY THIS EXISTS
`run_all.sh` exits 0 on a skip, and a skip line reads the same whether the
host legitimately lacks a capability or the runner was set up wrong. Measured
on 2026-10-01, before this check: 24 of 564 tests ran on no host and were named
by no workflow. Seven declared `llvm`, which no line granted; three declared
`musl`, whose probe named a release (15.1.0) the runners no longer installed;
seven declared `mingw-cross`, which no shard installed. Every one of those runs
was green.
WHAT IT CHECKS
It reads the per-test reports the shards write (`E2E_REPORT` of run_all.sh, one
`<status>\\t<test>\\t<ms>\\t<detail>` line per test) and decides, for every test
under tests/e2e:
ran some report says pass, fail or timeout;
job no report ran it, but a workflow names it outside a comment, as a
whole token: by file name, by name without `.sh`, or by its number
as an `E2E_ONLY` pattern such as `239_*.sh`. A dedicated job runs it
and asserts its result itself. A name in a comment does not count,
and neither does a longer name that contains it;
excused tests/e2e/coverage-exceptions.tsv lists it with the reason no hosted
runner can run it;
uncovered none of these. The check fails.
An exception for a test that ran, or for a test that does not exist, also fails
the check, so that the list cannot outlive the reason it records.
Some claims need a test to pass on particular hosts, not on some host: docs/21
calls a target row `verified` when CI builds and runs its artefact, and the
`*-openkal` rows claim that for every system. tests/e2e/host-requirements.tsv
lists such a test with the hosts its result must come from; a listed test that
did not pass on one of them, or that does not exist, fails the check.
With --timings-out DIR it writes `<host>.tsv`, the measured duration of every
test that ran, merged across that host's shards, in the format run_all.sh reads
from E2E_TIMINGS; refreshing tests/e2e/timings/ is copying those files. A report
file is named `e2e-report-<host>-<shard>.tsv`.
Usage:
check_e2e_coverage.py --reports DIR [--root DIR] [--timings-out DIR]
"""
from __future__ import annotations
import argparse
import re
import sys
from collections import defaultdict
from pathlib import Path
RAN = {"pass", "fail", "timeout"}
REPORT_NAME = re.compile(r"e2e-report-(?P<host>[a-z0-9-]+?)-(?P<shard>\d+)\.tsv$")
def read_reports(directory: Path):
"""Yield (host, shard, status, test, ms, detail) for every report line."""
for path in sorted(directory.rglob("e2e-report-*.tsv")):
m = REPORT_NAME.search(path.name)
if not m:
continue
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
parts = line.split("\t")
if len(parts) < 3:
continue
ms = int(parts[2]) if parts[2].isdigit() else 0
yield m["host"], int(m["shard"]), parts[0], parts[1], ms, parts[3] if len(parts) > 3 else ""
def read_host_requirements(path: Path) -> dict[str, set[str]]:
"""`<test>\t<host> <host> ...\t<reason>` lines: the hosts a test must pass on."""
out: dict[str, set[str]] = {}
if not path.exists():
return out
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip() or line.startswith("#"):
continue
parts = line.split("\t")
out[parts[0].strip()] = set(parts[1].split()) if len(parts) > 1 else set()
return out
def read_exceptions(path: Path) -> dict[str, str]:
out: dict[str, str] = {}
if not path.exists():
return out
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip() or line.startswith("#"):
continue
test, _, reason = line.partition("\t")
out[test.strip()] = reason.strip()
return out
TOKEN = re.compile(r"(?<![A-Za-z0-9_.-])(\d+[a-z]?_(?:\*|[A-Za-z0-9_]+))(?:\.sh)?(?![A-Za-z0-9_])")
def workflow_tokens(texts: list[str]) -> set[str]:
"""Every test name or `<number>_*` pattern named outside a comment."""
tokens: set[str] = set()
for text in texts:
for line in text.splitlines():
code = line.split("#", 1)[0] if line.lstrip().startswith("#") else line
if not code.strip():
continue
tokens.update(m.group(1) for m in TOKEN.finditer(code))
return tokens
def named_by_a_workflow(test: str, tokens: set[str]) -> bool:
stem = test[:-3]
number = stem.split("_", 1)[0]
return stem in tokens or f"{number}_*" in tokens
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--reports", required=True)
parser.add_argument("--root", default=".")
parser.add_argument("--timings-out")
args = parser.parse_args()
root = Path(args.root).resolve()
tests = sorted(p.name for p in (root / "tests" / "e2e").glob("[0-9]*.sh"))
workflows = workflow_tokens([p.read_text(encoding="utf-8")
for p in sorted((root / ".github" / "workflows").glob("*.yml"))])
exceptions = read_exceptions(root / "tests" / "e2e" / "coverage-exceptions.tsv")
required = read_host_requirements(root / "tests" / "e2e" / "host-requirements.tsv")
ran: dict[str, set[str]] = defaultdict(set)
passed: dict[str, set[str]] = defaultdict(set)
skipped: dict[str, set[str]] = defaultdict(set)
shard_ms: dict[tuple[str, int], int] = defaultdict(int)
timings: dict[str, dict[str, int]] = defaultdict(dict)
reports = 0
for host, shard, status, test, ms, detail in read_reports(Path(args.reports)):
reports += 1
if status in RAN:
ran[test].add(host)
if status == "pass":
passed[test].add(host)
shard_ms[(host, shard)] += ms
timings[host][test] = ms
elif status == "skip":
skipped[test].add(f"{host}: {detail}")
if reports == 0:
print(f"no report under {args.reports}: nothing can be said about coverage")
return 1
uncovered, stale = [], []
counts = defaultdict(int)
for test in tests:
if test in ran:
counts["ran"] += 1
if test in exceptions:
stale.append(f"{test} is excused but ran on {', '.join(sorted(ran[test]))}")
elif named_by_a_workflow(test, workflows):
counts["job"] += 1
elif test in exceptions:
counts["excused"] += 1
else:
uncovered.append(f"{test}: {'; '.join(sorted(skipped[test])) or 'in no report'}")
for test in exceptions:
if test not in tests:
stale.append(f"{test} is excused but does not exist")
unmet = []
for test, hosts in sorted(required.items()):
if test not in tests:
stale.append(f"{test} is required on hosts but does not exist")
continue
for host in sorted(hosts - passed[test]):
why = [d for d in skipped[test] if d.startswith(f"{host}: ")]
unmet.append(f"{test} did not pass on {host}"
f"{' (' + '; '.join(why) + ')' if why else (' (it ran and did not pass)' if host in ran[test] else ' (no report of it there)')}")
print(f"{len(tests)} tests: {counts['ran']} ran on a shard, {counts['job']} run by a "
f"dedicated job, {counts['excused']} excused, {len(uncovered)} uncovered")
for (host, shard), ms in sorted(shard_ms.items()):
print(f" {host} shard {shard}: {ms / 60000:.1f} min of tests")
for line in uncovered:
print(f"UNCOVERED: {line}")
for line in stale:
print(f"STALE EXCEPTION: {line}")
for line in unmet:
print(f"REQUIRED ON HOST: {line}")
if args.timings_out:
out = Path(args.timings_out)
out.mkdir(parents=True, exist_ok=True)
for host, table in timings.items():
(out / f"{host}.tsv").write_text(
"".join(f"{t}\t{ms}\n" for t, ms in sorted(table.items())), encoding="utf-8")
return 1 if uncovered or stale or unmet else 0
if __name__ == "__main__":
sys.exit(main())