|
| 1 | +import html |
| 2 | +import os |
| 3 | +import pathlib |
| 4 | +import re |
| 5 | +from _colorize import decolor # type: ignore[import-not-found] |
| 6 | +from typing import TextIO |
| 7 | + |
| 8 | +from .findtests import findtestdir |
| 9 | +from .result import TestResult |
| 10 | +from .results import TestResults |
| 11 | +from .runtests import RunTests |
| 12 | +from .utils import count |
| 13 | + |
| 14 | + |
| 15 | +class Reporter: |
| 16 | + """Report test failures, for example to a CI service. A no-op here.""" |
| 17 | + |
| 18 | + def run_started(self, runtests: RunTests) -> None: |
| 19 | + """Called at the start of each run, and in the test process.""" |
| 20 | + |
| 21 | + def test_case_failed(self, test_case: str, traceback: str, |
| 22 | + stream: TextIO) -> None: |
| 23 | + """Called in the test process after a test case failed.""" |
| 24 | + |
| 25 | + def test_result(self, result: TestResult) -> None: |
| 26 | + """Called in the main process after a test file completed.""" |
| 27 | + |
| 28 | + def summary(self, results: TestResults, state: str) -> None: |
| 29 | + """Called in the main process after all runs.""" |
| 30 | + |
| 31 | + @staticmethod |
| 32 | + def parse_traceback(traceback: str) -> tuple[str | None, str | None, str]: |
| 33 | + """Return (filename, line, exception) of a failure.""" |
| 34 | + # A frame is its "File" line and its source lines |
| 35 | + frames = list(re.finditer( |
| 36 | + r'^ *File "(.*)", line (\d+).*\n(?: .*\n)*', traceback, |
| 37 | + re.MULTILINE)) |
| 38 | + filename = line = None |
| 39 | + exception = traceback |
| 40 | + if frames: |
| 41 | + filename, line = frames[0].groups() |
| 42 | + exception = traceback[frames[-1].end():] |
| 43 | + # Prefer the last frame in a test file, the assertion |
| 44 | + for frame in frames: |
| 45 | + if pathlib.Path(frame[1]).name.startswith("test_"): |
| 46 | + filename, line = frame.groups() |
| 47 | + return filename, line, exception.strip() |
| 48 | + |
| 49 | + |
| 50 | +def get_reporter() -> Reporter: |
| 51 | + if GitHubActionsReporter.enabled(): |
| 52 | + return GitHubActionsReporter() |
| 53 | + return Reporter() |
| 54 | + |
| 55 | + |
| 56 | +class GitHubActionsReporter(Reporter): |
| 57 | + """Annotate failures in the job log and list them in the job summary.""" |
| 58 | + |
| 59 | + ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"}) |
| 60 | + ESCAPE_PROPERTY = ESCAPE_DATA | str.maketrans({":": "%3A", ",": "%2C"}) |
| 61 | + |
| 62 | + @staticmethod |
| 63 | + def enabled() -> bool: |
| 64 | + return bool(os.environ.get("GITHUB_STEP_SUMMARY")) |
| 65 | + |
| 66 | + def run_started(self, runtests: RunTests) -> None: |
| 67 | + self.runtests = runtests |
| 68 | + |
| 69 | + def test_case_failed(self, test_case: str, traceback: str, |
| 70 | + stream: TextIO) -> None: |
| 71 | + if not self.runtests.final_run: |
| 72 | + return |
| 73 | + filename, line, exception = self.parse_traceback(decolor(traceback)) |
| 74 | + self.annotate(test_case, exception, filename, line, stream) |
| 75 | + |
| 76 | + def test_result(self, result: TestResult) -> None: |
| 77 | + if not self.runtests.final_run: |
| 78 | + return |
| 79 | + if not result.is_failed(self.runtests.fail_env_changed): |
| 80 | + return |
| 81 | + # Already annotated by test_case_failed() |
| 82 | + if result.failed_test_cases: |
| 83 | + return |
| 84 | + # "test.test_x.test_y" is test_x/test_y.py or test_x/test_y/__init__.py |
| 85 | + name = result.test_name.removeprefix("test.") |
| 86 | + path = pathlib.Path(findtestdir(self.runtests.test_dir), |
| 87 | + *name.split(".")) |
| 88 | + filename = None |
| 89 | + for candidate in (path.with_suffix(".py"), path / "__init__.py"): |
| 90 | + if candidate.exists(): |
| 91 | + filename = str(candidate) |
| 92 | + message = "\n".join([str(result), *(result.env_changed_reasons or ())]) |
| 93 | + self.annotate(result.test_name, decolor(message), filename) |
| 94 | + |
| 95 | + def annotate(self, title: str, message: str, filename: str | None, |
| 96 | + line: str | None = None, |
| 97 | + stream: TextIO | None = None) -> None: |
| 98 | + if filename: |
| 99 | + # Make installed stdlib paths relative to the checkout |
| 100 | + path = pathlib.Path(filename) |
| 101 | + stdlib_dir = pathlib.Path(os.__file__).parent |
| 102 | + if path.is_relative_to(stdlib_dir): |
| 103 | + filename = ("Lib" / path.relative_to(stdlib_dir)).as_posix() |
| 104 | + # GitHub needs a line to show the annotation on the file |
| 105 | + line = line or "1" |
| 106 | + props = {"file": filename, "line": line, "title": title} |
| 107 | + props_text = ",".join(f"{key}={value.translate(self.ESCAPE_PROPERTY)}" |
| 108 | + for key, value in props.items() if value) |
| 109 | + print(f"::error {props_text}::{message.translate(self.ESCAPE_DATA)}", |
| 110 | + file=stream, flush=True) |
| 111 | + |
| 112 | + def summary(self, results: TestResults, state: str) -> None: |
| 113 | + failed = [result for result in results.rerun_results |
| 114 | + if result.is_failed(self.runtests.fail_env_changed)] |
| 115 | + if not failed: |
| 116 | + return |
| 117 | + ncases = sum(len(result.failed_test_cases) for result in failed) |
| 118 | + blocks = [f"## {decolor(state)}: " |
| 119 | + f"{count(len(failed), 'test file')} and " |
| 120 | + f"{count(ncases, 'test case')} failed"] |
| 121 | + for result in failed: |
| 122 | + blocks.append(f"### {decolor(str(result))}") |
| 123 | + if result.env_changed_reasons: |
| 124 | + blocks.append("\n".join( |
| 125 | + f"- {reason}" for reason in result.env_changed_reasons)) |
| 126 | + for name, traceback in result.failed_test_cases: |
| 127 | + # Expand short tracebacks if there are only a few |
| 128 | + if ncases <= 5 and traceback.count("\n") < 30: |
| 129 | + details = "<details open>" |
| 130 | + else: |
| 131 | + details = "<details>" |
| 132 | + blocks += [f"{details}<summary>{html.escape(name)}</summary>", |
| 133 | + f"```pytb\n{decolor(traceback).rstrip()}\n```", |
| 134 | + "</details>"] |
| 135 | + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", |
| 136 | + encoding="utf-8") as fp: |
| 137 | + fp.write("".join(f"{block}\n\n" for block in blocks)) |
0 commit comments