Skip to content

Commit 7715c92

Browse files
cmaloneyclaude
andauthored
gh-158895: Teach libregrtest to output github job summaries (GH-158896)
Goal is to make it faster to triage individual test failures while keeping the full existing regrtest fidelity. Fairly minimal integration to hopefully save some human eyeball time. Setup so only the test summary process should output the summary, subprocesses have the env var which will cause output unset. Includes a couple of fixes to existing tests that produced false-positives. --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 53b0004 commit 7715c92

11 files changed

Lines changed: 293 additions & 7 deletions

File tree

‎.github/workflows/reusable-test-lazy-imports-all.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ jobs:
6464
- name: Verify excluded tests still need exclusion
6565
run: |
6666
set -euo pipefail
67+
# Failures are expected here, don't report them to the job summary
68+
unset GITHUB_STEP_SUMMARY
6769
unexpected_passes=()
6870
while IFS= read -r test_name; do
6971
[ -n "$test_name" ] || continue

‎Lib/test/libregrtest/main.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from .findtests import findtests, split_test_packages, list_cases, collect_cases
1616
from .logger import Logger
1717
from .pgo import setup_pgo_tests
18+
from .reporter import get_reporter
1819
from .result import TestResult
1920
from .results import TestResults, EXITCODE_INTERRUPTED
2021
from .runtests import RunTests, HuntRefleak
@@ -68,6 +69,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False):
6869

6970
# Logger
7071
self.logger = Logger(self.results, self.quiet, self.pgo)
72+
self.reporter = get_reporter()
7173

7274
# Actions
7375
self.want_header: bool = ns.header
@@ -284,8 +286,10 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests:
284286
forever=False,
285287
fail_fast=False,
286288
match_tests_dict=match_tests_dict,
287-
output_on_failure=False)
289+
output_on_failure=False,
290+
final_run=True)
288291
self.logger.set_tests(runtests)
292+
self.reporter.run_started(runtests)
289293

290294
msg = f"Re-running {len(tests)} failed tests in verbose mode"
291295
if not self.single_process:
@@ -395,6 +399,7 @@ def run_test(
395399
result = run_single_test(test_name, runtests)
396400

397401
self.results.accumulate_result(result, runtests)
402+
self.reporter.test_result(result)
398403

399404
return result
400405

@@ -459,7 +464,8 @@ def get_state(self) -> str:
459464

460465
def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None:
461466
from .run_workers import RunWorkers
462-
RunWorkers(num_workers, runtests, self.logger, self.results).run()
467+
RunWorkers(num_workers, runtests, self.logger, self.results,
468+
self.reporter).run()
463469

464470
def finalize_tests(self, coverage: trace.CoverageResults | None) -> None:
465471
if self.next_single_filename:
@@ -480,6 +486,8 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None:
480486
if self.junit_filename:
481487
self.results.write_junit(self.junit_filename)
482488

489+
self.reporter.summary(self.results, self.get_state())
490+
483491
def display_summary(self) -> None:
484492
if self.first_runtests is None:
485493
raise ValueError(
@@ -517,6 +525,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests:
517525
hunt_refleak=self.hunt_refleak,
518526
test_dir=self.test_dir,
519527
use_junit=(self.junit_filename is not None),
528+
final_run=not self.want_rerun or bool(self.python_cmd),
520529
coverage=self.coverage,
521530
memory_limit=self.memory_limit,
522531
gc_threshold=self.gc_threshold,
@@ -568,6 +577,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
568577
runtests = runtests.copy(tests=case_ids, case_groups=case_groups)
569578
self.first_runtests = runtests
570579
self.logger.set_tests(runtests)
580+
self.reporter.run_started(runtests)
571581

572582
if (runtests.hunt_refleak is not None) and (not self.num_workers):
573583
# gh-109739: WindowsLoadTracker thread interferes with refleak check

‎Lib/test/libregrtest/reporter.py‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
import html
2+
import os
3+
import pathlib
4+
import re
5+
from _colorize import decolor # type: ignore[import-not-found]
6+
from typing import TextIO
7+
8+
from .findtests import findtestdir
9+
from .result import TestResult
10+
from .results import TestResults
11+
from .runtests import RunTests
12+
from .utils import count
13+
14+
15+
class Reporter:
16+
"""Report test failures, for example to a CI service. A no-op here."""
17+
18+
def run_started(self, runtests: RunTests) -> None:
19+
"""Called at the start of each run, and in the test process."""
20+
21+
def test_case_failed(self, test_case: str, traceback: str,
22+
stream: TextIO) -> None:
23+
"""Called in the test process after a test case failed."""
24+
25+
def test_result(self, result: TestResult) -> None:
26+
"""Called in the main process after a test file completed."""
27+
28+
def summary(self, results: TestResults, state: str) -> None:
29+
"""Called in the main process after all runs."""
30+
31+
@staticmethod
32+
def parse_traceback(traceback: str) -> tuple[str | None, str | None, str]:
33+
"""Return (filename, line, exception) of a failure."""
34+
# A frame is its "File" line and its source lines
35+
frames = list(re.finditer(
36+
r'^ *File "(.*)", line (\d+).*\n(?: .*\n)*', traceback,
37+
re.MULTILINE))
38+
filename = line = None
39+
exception = traceback
40+
if frames:
41+
filename, line = frames[0].groups()
42+
exception = traceback[frames[-1].end():]
43+
# Prefer the last frame in a test file, the assertion
44+
for frame in frames:
45+
if pathlib.Path(frame[1]).name.startswith("test_"):
46+
filename, line = frame.groups()
47+
return filename, line, exception.strip()
48+
49+
50+
def get_reporter() -> Reporter:
51+
if GitHubActionsReporter.enabled():
52+
return GitHubActionsReporter()
53+
return Reporter()
54+
55+
56+
class GitHubActionsReporter(Reporter):
57+
"""Annotate failures in the job log and list them in the job summary."""
58+
59+
ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"})
60+
ESCAPE_PROPERTY = ESCAPE_DATA | str.maketrans({":": "%3A", ",": "%2C"})
61+
62+
@staticmethod
63+
def enabled() -> bool:
64+
return bool(os.environ.get("GITHUB_STEP_SUMMARY"))
65+
66+
def run_started(self, runtests: RunTests) -> None:
67+
self.runtests = runtests
68+
69+
def test_case_failed(self, test_case: str, traceback: str,
70+
stream: TextIO) -> None:
71+
if not self.runtests.final_run:
72+
return
73+
filename, line, exception = self.parse_traceback(decolor(traceback))
74+
self.annotate(test_case, exception, filename, line, stream)
75+
76+
def test_result(self, result: TestResult) -> None:
77+
if not self.runtests.final_run:
78+
return
79+
if not result.is_failed(self.runtests.fail_env_changed):
80+
return
81+
# Already annotated by test_case_failed()
82+
if result.failed_test_cases:
83+
return
84+
# "test.test_x.test_y" is test_x/test_y.py or test_x/test_y/__init__.py
85+
name = result.test_name.removeprefix("test.")
86+
path = pathlib.Path(findtestdir(self.runtests.test_dir),
87+
*name.split("."))
88+
filename = None
89+
for candidate in (path.with_suffix(".py"), path / "__init__.py"):
90+
if candidate.exists():
91+
filename = str(candidate)
92+
message = "\n".join([str(result), *(result.env_changed_reasons or ())])
93+
self.annotate(result.test_name, decolor(message), filename)
94+
95+
def annotate(self, title: str, message: str, filename: str | None,
96+
line: str | None = None,
97+
stream: TextIO | None = None) -> None:
98+
if filename:
99+
# Make installed stdlib paths relative to the checkout
100+
path = pathlib.Path(filename)
101+
stdlib_dir = pathlib.Path(os.__file__).parent
102+
if path.is_relative_to(stdlib_dir):
103+
filename = ("Lib" / path.relative_to(stdlib_dir)).as_posix()
104+
# GitHub needs a line to show the annotation on the file
105+
line = line or "1"
106+
props = {"file": filename, "line": line, "title": title}
107+
props_text = ",".join(f"{key}={value.translate(self.ESCAPE_PROPERTY)}"
108+
for key, value in props.items() if value)
109+
print(f"::error {props_text}::{message.translate(self.ESCAPE_DATA)}",
110+
file=stream, flush=True)
111+
112+
def summary(self, results: TestResults, state: str) -> None:
113+
failed = [result for result in results.rerun_results
114+
if result.is_failed(self.runtests.fail_env_changed)]
115+
if not failed:
116+
return
117+
ncases = sum(len(result.failed_test_cases) for result in failed)
118+
blocks = [f"## {decolor(state)}: "
119+
f"{count(len(failed), 'test file')} and "
120+
f"{count(ncases, 'test case')} failed"]
121+
for result in failed:
122+
blocks.append(f"### {decolor(str(result))}")
123+
if result.env_changed_reasons:
124+
blocks.append("\n".join(
125+
f"- {reason}" for reason in result.env_changed_reasons))
126+
for name, traceback in result.failed_test_cases:
127+
# Expand short tracebacks if there are only a few
128+
if ncases <= 5 and traceback.count("\n") < 30:
129+
details = "<details open>"
130+
else:
131+
details = "<details>"
132+
blocks += [f"{details}<summary>{html.escape(name)}</summary>",
133+
f"```pytb\n{decolor(traceback).rstrip()}\n```",
134+
"</details>"]
135+
with open(os.environ["GITHUB_STEP_SUMMARY"], "a",
136+
encoding="utf-8") as fp:
137+
fp.write("".join(f"{block}\n\n" for block in blocks))

‎Lib/test/libregrtest/result.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,10 @@ def __str__(self) -> str:
178178
def has_meaningful_duration(self):
179179
return State.has_meaningful_duration(self.state)
180180

181+
@property
182+
def failed_test_cases(self) -> list[tuple[str, str]]:
183+
return (self.errors or []) + (self.failures or [])
184+
181185
def set_env_changed(self, *reasons):
182186
if self.state is None or self.state == State.PASSED:
183187
self.state = State.ENV_CHANGED

‎Lib/test/libregrtest/run_workers.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
from test.support import os_helper, MS_WINDOWS
1717

1818
from .logger import Logger
19+
from .reporter import Reporter
1920
from .result import TestResult, State
2021
from .results import TestResults
2122
from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType
@@ -499,13 +500,15 @@ def get_running(workers: list[WorkerThread]) -> str | None:
499500

500501
class RunWorkers:
501502
def __init__(self, num_workers: int, runtests: RunTests,
502-
logger: Logger, results: TestResults) -> None:
503+
logger: Logger, results: TestResults,
504+
reporter: Reporter) -> None:
503505
self.num_workers = num_workers
504506
self.runtests = runtests
505507
self.logger = logger
506508
self.log = logger.log
507509
self.display_progress = logger.display_progress
508510
self.results: TestResults = results
511+
self.reporter = reporter
509512
self.live_worker_count = 0
510513

511514
self.output: queue.Queue[QueueContent] = queue.Queue()
@@ -624,6 +627,8 @@ def _process_result(self, item: QueueOutput) -> TestResult:
624627
stdout = mp_result.worker_stdout
625628
if stdout:
626629
print(stdout, flush=True)
630+
# Report after the output: env changed warnings, crash traceback
631+
self.reporter.test_result(result)
627632

628633
return result
629634

‎Lib/test/libregrtest/runtests.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ class RunTests:
9393
hunt_refleak: HuntRefleak | None
9494
test_dir: StrPath | None
9595
use_junit: bool
96+
# Failed tests are not re-run after this run
97+
final_run: bool
9698
coverage: bool
9799
memory_limit: str | None
98100
gc_threshold: int | None

‎Lib/test/libregrtest/setup.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@
1111
from _colorize import can_colorize # type: ignore[import-not-found]
1212

1313
from .filter import set_match_tests
14+
from .reporter import get_reporter
1415
from .runtests import RunTests
16+
from .testresult import RegressionTestResult
1517
from .utils import (
1618
setup_unraisable_hook, setup_threading_excepthook,
1719
adjust_rlimit_nofile)
@@ -112,9 +114,11 @@ def setup_tests(runtests: RunTests) -> None:
112114

113115
set_match_tests(runtests.match_tests)
114116

117+
reporter = get_reporter()
118+
reporter.run_started(runtests)
119+
RegressionTestResult.reporter = reporter
115120
if runtests.use_junit:
116121
support.junit_xml_list = []
117-
from .testresult import RegressionTestResult
118122
RegressionTestResult.USE_XML = True
119123
else:
120124
support.junit_xml_list = None

‎Lib/test/libregrtest/testresult.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,12 @@
99
import traceback
1010
import unittest
1111
from test import support
12+
from test.libregrtest.reporter import Reporter
1213
from test.libregrtest.utils import sanitize_xml
1314

1415
class RegressionTestResult(unittest.TextTestResult):
1516
USE_XML = False
17+
reporter = Reporter()
1618

1719
def __init__(self, stream, descriptions, verbosity):
1820
super().__init__(stream=stream, descriptions=descriptions,
@@ -130,6 +132,14 @@ def addUnexpectedSuccess(self, test):
130132
self._add_result(test, outcome='UNEXPECTED_SUCCESS')
131133
super().addUnexpectedSuccess(test)
132134

135+
def printErrorList(self, flavour, errors):
136+
for test, err in errors:
137+
super().printErrorList(flavour, [(test, err)])
138+
self.report_failure(test, err)
139+
140+
def report_failure(self, test, err):
141+
self.reporter.test_case_failed(str(test), err, self.stream)
142+
133143
def get_xml_element(self):
134144
if not self.USE_XML:
135145
raise ValueError("USE_XML is false")
@@ -146,6 +156,9 @@ def __init__(self, stream, buffer=False):
146156

147157
def run(self, test):
148158
test(self.result)
159+
# Failures are not printed in quiet mode
160+
for test, err in self.result.errors + self.result.failures:
161+
self.result.report_failure(test, err)
149162
return self.result
150163

151164
def get_test_runner_class(verbosity, buffer=False):

‎Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import unittest
1313
from unittest import mock
1414
from test.support import requires, requires_remote_subprocess_debugging
15+
from test.support import os_helper
1516
from test.support.import_helper import import_module
1617

1718
# Only run these tests if curses is available
@@ -870,8 +871,11 @@ def test_run_failed_module_live(self):
870871
),
871872
mock.patch('curses.wrapper', side_effect=self.mock_curses_wrapper),
872873
mock.patch("sys.argv", args),
873-
mock.patch('sys.stderr', new=io.StringIO()) as fake_stderr
874+
mock.patch('sys.stderr', new=io.StringIO()) as fake_stderr,
875+
os_helper.EnvironmentVarGuard() as env,
874876
):
877+
# Don't report the crash to the CI running this test
878+
env.unset('GITHUB_STEP_SUMMARY')
875879
main()
876880
self.assertIn(
877881
'test test_asdasd crashed -- Traceback (most recent call last):',

0 commit comments

Comments
 (0)