diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index b7707c6c66..d82f9eaa98 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -74,35 +74,53 @@ def run_helper(self): text=True, ) - def summary(self): + def run_report(self, *flags): result = subprocess.run( - [sys.executable, os.path.join(self.tmp, "util/genReport.py"), "-sv"], + [sys.executable, os.path.join(self.tmp, "util/genReport.py"), *flags], capture_output=True, text=True, ) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - with open(os.path.join(self.tmp, "reports/report-summary.log")) as f: + return result.stdout + + def read(self, path): + with open(os.path.join(self.tmp, path), encoding="utf-8") as f: return f.read() + def summary(self, flags="-sv"): + self.run_report(flags) + return self.read("reports/report-summary.log") + + def qor_fail(self, pdv, metrics, baseline="https://dashboard.example/builds/7"): + """A finished run with the CI QoR check verdict FAIL.""" + lines = [f"[ERROR] Dashboard QoR check: FAIL ({pdv})"] + lines.append(f"QoR regression detected for {pdv}:") + lines += [f" {m} 2 (base 1, limit 1.5)" for m in metrics] + lines.append(f" Baseline: {baseline}") + self.finished(pdv, "\n".join(lines) + "\n") + + def finished(self, pdv, check=""): + """A run that reached the last stage, with its metadata and DRC reports.""" + self.write_file(f"logs/{pdv}/6_report.log", "Finished\n") + self.write_file(f"reports/{pdv}/metadata-generate.log", "") + self.write_file(f"reports/{pdv}/metadata-check.log", check) + self.write_file(f"reports/{pdv}/5_route_drc.rpt", "") + def test_summary_includes_make_failure_before_the_first_stage(self): result = self.run_helper() self.assertEqual(result.returncode, 2, result.stdout + result.stderr) self.assertRegex(result.stdout, r"No rule to make target .missing\.v.") - for platform in ("nangate45", "ihp-sg13g2"): - self.write_file(f"logs/{platform}/gcd/base/6_report.log", "Finished\n") - self.write_file(f"reports/{platform}/gcd/base/metadata-check.log", "") - self.write_file( - "reports/nangate45/gcd/base/metadata-check.log", - "[ERROR] nangate45/gcd/base: FAIL\n", - ) + self.finished("ihp-sg13g2/gcd/base") + self.finished("nangate45/gcd/base", "[ERROR] nangate45/gcd/base: FAIL\n") summary = self.summary() - self.assertIn("Number of designs: 3", summary) - self.assertIn("nangate45 gcd (base)", summary) - self.assertIn("sky130hd gcd (base)\n Flow did not reach last stage.", summary) + self.assertIn("SUMMARY: 3 designs, 2 failing, 1 passing", summary) + self.assertIn("QOR nangate45/gcd/base", summary) + self.assertIn("STOP sky130hd/gcd/base", summary) + self.assertIn("sky130hd/gcd/base: stopped in 0_make.log", summary) self.assertRegex(summary, r"No rule to make target .missing\.v.") - self.assertNotIn("ihp-sg13g2 gcd (base)", summary) + self.assertNotIn("ihp-sg13g2/gcd/base", summary) def test_early_failure_uses_the_configured_variant_and_design_nickname(self): self.write_file( @@ -111,10 +129,7 @@ def test_early_failure_uses_the_configured_variant_and_design_nickname(self): ) result = self.run_helper() self.assertEqual(result.returncode, 2, result.stdout + result.stderr) - self.assertIn( - "sky130hd gcd_alias (alternate)\n Flow did not reach last stage.", - self.summary(), - ) + self.assertIn("STOP sky130hd/gcd_alias/alternate", self.summary()) def test_successful_rerun_does_not_keep_the_early_failure(self): result = self.run_helper() @@ -122,143 +137,306 @@ def test_successful_rerun_does_not_keep_the_early_failure(self): self.write_file("missing.v", "") result = self.run_helper() self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - summary = self.summary() - self.assertIn("Number of designs: 1", summary) - self.assertNotIn("Failing designs", summary) + self.assertIn("SUMMARY: 1 design, 0 failing, 1 passing", self.summary()) def test_finish_log_is_recognized_with_a_later_auxiliary_log(self): - self.write_file("logs/sky130hd/gcd/base/6_report.log", "Finished\n") + self.finished("sky130hd/gcd/base") self.write_file("logs/sky130hd/gcd/base/6_write_rc.log", "RC values\n") - summary = self.summary() - self.assertIn("Number of designs: 1", summary) - self.assertNotIn("Failing designs", summary) + self.assertIn("SUMMARY: 1 design, 0 failing, 1 passing", self.summary()) def test_empty_log_directory_is_reported_as_incomplete(self): os.makedirs(os.path.join(self.tmp, "logs/sky130hd/gcd/base")) summary = self.summary() - self.assertIn("Number of designs: 1", summary) - self.assertIn("sky130hd gcd (base)\n Flow did not reach last stage.", summary) - self.assertIn("No log file found.", summary) + self.assertIn("STOP sky130hd/gcd/base no log file found", summary) + self.assertIn( + "sky130hd/gcd/base: flow did not reach last stage, no log file found", + summary, + ) def test_dashboard_verdict_reaches_the_summary(self): - self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\n") - self.write_file( - "reports/nangate45/gcd/base/metadata-check.log", + self.finished( + "nangate45/gcd/base", "[ERROR] Dashboard QoR check: FAIL (nangate45/gcd/base)\n" "QoR regression detected for nangate45/gcd/base:\n" " finish__timing__setup__tns -12.0 (base -6.0, limit -8.0)\n" " Baseline: https://dashboard.example/builds/7\n", ) summary = self.summary() - self.assertIn("Failing designs", summary) - self.assertIn("nangate45 gcd (base)", summary) - self.assertIn("finish__timing__setup__tns -12.0", summary) - self.assertIn("limit -8.0", summary) - self.assertIn("https://dashboard.example/builds/7", summary) - - -class TestGenReport(unittest.TestCase): - def test_incomplete_flow_reports_that_it_did_not_reach_last_stage(self): - name = "nangate45 gcd (base)" - data = { - "status": genReport.STATUS_RED, - "finished": False, - "last_log": "5_route.log", - "log_errors": [], - "metrics_logs_errors": [], - "metrics_errors": [], - "metrics_regressions": [], - "calibre_errors": [], - "drcs": {}, - } - - report = genReport.gen_report(name, data) + self.assertIn("QoR regression 1", summary) + self.assertIn("QoR baseline: https://dashboard.example/builds/7", summary) + self.assertIn("QOR nangate45/gcd/base finish__timing__setup__tns", summary) + self.assertIn("nangate45/gcd/base: 1 metric over the limit", summary) + self.assertRegex(summary, r"finish__timing__setup__tns +-12\.0 +-6\.0 +-8\.0") - self.assertEqual( - report, - "\n".join( - [ - name, - " Flow did not reach last stage.", - " Last log file 5_route.log", - ] - ), - ) - - def test_parse_qor_regressions_omits_the_optional_heading(self): - with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as log: - log.write("Ignored UTF-8 text: µ\n") - log.write( - "QoR regression detected for nangate45/gcd:\n" - " cts__timing__setup__tns -12.2456 " - "(base -6.95522, limit -8.346264)\n" - ) - log.flush() - - self.assertEqual( - genReport.parse_qor_regressions(log.name), - [ - "cts__timing__setup__tns -12.2456 " - "(base -6.95522, limit -8.346264)" - ], - ) + def test_failures_are_ordered_by_criticality(self): + self.finished( + "asap7/a_one_metric/base", + "[ERROR] Dashboard QoR check: FAIL (asap7/a_one_metric/base)\n" + "QoR regression detected for asap7/a_one_metric/base:\n" + " m1 2 (base 1, limit 1.5)\n", + ) + self.finished( + "asap7/b_two_metrics/base", + "[ERROR] Dashboard QoR check: FAIL (asap7/b_two_metrics/base)\n" + "QoR regression detected for asap7/b_two_metrics/base:\n" + " m1 2 (base 1, limit 1.5)\n" + " m2 2 (base 1, limit 1.5)\n", + ) + self.finished("asap7/c_calibre/base") + self.write_file( + "logs/asap7/c_calibre/base/calibre/save-to-drc-db.log", + "[ERROR DRC-0001] violations\n", + ) + self.finished("asap7/d_error/base") + self.write_file("logs/asap7/d_error/base/5_1_grt.log", "[ERROR GRT-0116] x\n") + self.write_file( + "logs/asap7/e_killed/base/5_2_route.log", + "Command terminated by signal 9\n", + ) + self.finished( + "asap7/f_inconclusive/base", + "Dashboard QoR check: INCONCLUSIVE (asap7/f_inconclusive/base)\n" + "Reason: no baseline\n", + ) - def test_qor_metrics_are_nested_under_the_failing_run(self): - name = "nangate45 gcd (base)" - data = { - "status": genReport.STATUS_RED, - "finished": True, - "last_log": "", - "log_errors": [], - "metrics_logs_errors": [], - "metrics_errors": [ - "[ERROR] nangate45/gcd/base: FAIL — 19 passed, 6 failed, " - "0 missing (478 metrics sent)", - "[ERROR] Failing runs: nangate45/gcd/base", - ], - "metrics_regressions": [ - "cts__timing__setup__tns -12.2456 " "(base -6.95522, limit -8.346264)", - "cts__timing__setup__ws -0.261172 " "(base -0.156758, limit -0.169258)", - "finish__timing__setup__tns -12.4146 " - "(base -7.19445, limit -8.63334)", - "finish__timing__setup__ws -0.271991 " - "(base -0.163541, limit -0.176041)", - "globalroute__timing__setup__tns -12.7365 " - "(base -7.5407, limit -9.04884)", - "globalroute__timing__setup__ws -0.275208 " - "(base -0.171167, limit -0.183667)", + summary = self.summary() + + codes = [ + line.split()[:2] + for line in summary.split("-" * 60)[0].splitlines() + if line.split()[:1] in (["STOP"], ["ERR"], ["DRC"], ["QOR"], ["WARN"]) + ] + self.assertEqual( + codes, + [ + ["STOP", "asap7/e_killed/base"], + ["ERR", "asap7/d_error/base"], + ["DRC", "asap7/c_calibre/base"], + ["QOR", "asap7/b_two_metrics/base"], + ["QOR", "asap7/a_one_metric/base"], + ["WARN", "asap7/f_inconclusive/base"], ], - "calibre_errors": [], - "drcs": {}, - } + ) + self.assertIn("5_2_route.log: killed by signal 9 (out of memory?)", summary) + self.assertIn("5_1_grt.log: [ERROR GRT-0116] x", summary) + self.assertIn( + "asap7/f_inconclusive/base: QoR check INCONCLUSIVE: no baseline", summary + ) + + def test_missing_metadata_is_grouped_under_needs_attention(self): + for name in ("a", "b"): + self.write_file(f"logs/asap7/{name}/base/6_report.log", "Finished\n") + self.write_file(f"reports/asap7/{name}/base/5_route_drc.rpt", "") + summary = self.summary() + self.assertIn( + "missing metadata-generate.log, metadata-check.log (2 designs):\n" + " asap7/a/base, asap7/b/base\n", + summary, + ) + + def verbosity_fixture(self): + self.write_file( + "logs/asap7/stop/base/3_3_place_gp.log", + "[WARNING GPL-0302] Use a larger density\n" + "[ERROR GPL-0307] RePlAce divergence\n" + "[ERROR GPL-0307] RePlAce divergence\n", + ) + self.finished("asap7/pass/base") + self.write_file( + "logs/asap7/pass/base/2_1_floorplan.log", "[WARNING IFP-0028] Core area\n" + ) + self.finished( + "asap7/warn/base", + "Dashboard QoR check: INCONCLUSIVE (asap7/warn/base)\nReason: no baseline\n", + ) + + def test_level_0_shows_error_ids_only(self): + self.verbosity_fixture() + summary = self.summary("-s") + self.assertIn(" 3_3_place_gp.log: GPL-0307\n", summary) + self.assertNotIn("RePlAce divergence", summary) + self.assertNotIn("asap7/pass/base", summary) + + def test_level_1_shows_each_full_error_once_with_a_count(self): + self.verbosity_fixture() + summary = self.summary("-sv") + self.assertIn( + " 3_3_place_gp.log: [ERROR GPL-0307] RePlAce divergence (x2)\n", summary + ) + self.assertNotIn("GPL-0302", summary) + + def test_level_2_adds_passing_designs_and_warning_ids(self): + self.verbosity_fixture() + summary = self.summary("-svv") + self.assertIn("PASS asap7/pass/base", summary) + passing = summary.split("PASSING (2)")[1] + self.assertIn( + "asap7/pass/base: passed\n 2_1_floorplan.log: IFP-0028", passing + ) + # The note of a WARN design shows under "Needs attention" only. + self.assertIn("asap7/warn/base: passed, see NEEDS ATTENTION\n", passing) + self.assertNotIn("no baseline", passing) + self.assertIn(" 3_3_place_gp.log: GPL-0302", summary) + + def test_level_3_shows_full_warnings(self): + self.verbosity_fixture() + summary = self.summary("-svvv") + self.assertIn("2_1_floorplan.log: [WARNING IFP-0028] Core area", summary) + + def test_report_mode_writes_each_design_report(self): + self.verbosity_fixture() + # A run that stopped early may have no reports directory yet. + self.assertFalse(os.path.isdir(os.path.join(self.tmp, "reports/asap7/stop"))) + output = self.run_report("-v") + self.assertIn("asap7/stop/base: stopped in 3_3_place_gp.log", output) + self.assertIn( + "[ERROR GPL-0307] RePlAce divergence (x2)", + self.read("reports/asap7/stop/base/report.log"), + ) + self.assertIn( + "asap7/warn/base: QoR check INCONCLUSIVE: no baseline", + self.read("reports/asap7/warn/base/report.log"), + ) + self.assertIn("asap7/stop/base", self.read("reports/report.log")) + + def test_local_qor_check_format(self): + self.finished( + "nangate45/gcd/base", + "[ERROR] nangate45/gcd/base: FAIL \u2014 19 passed, 2 failed, 0 missing " + "(478 metrics sent)\n" + "[INFO] baseline build 7 (abc, latest)\n" + "Metric Baseline Current Limit Delta%\n" + "QoR regression detected for nangate45/gcd:\n" + " cts__timing__setup__tns -12.2456 (base -6.95522, limit -8.346264)\n" + " finish__timing__setup__ws -0.27 (base -0.16, limit -0.17)\n" + "[WARN] Rules not evaluated for 3 metric(s) absent from the baseline " + "build. Use --verbose to list them.\n" + "[ERROR] Failing runs: nangate45/gcd/base\n" + "QoR check: FAIL\n", + ) + summary = self.summary() + self.assertIn("nangate45/gcd/base: 2 metrics over the limit", summary) + self.assertRegex(summary, r"cts__timing__setup__tns +-12\.2456 +-6\.95522") + self.assertIn("Note: QoR: Rules not evaluated for 3 metric(s)", summary) + self.assertNotIn("Failing runs", summary) + + def test_each_design_shows_its_baseline_when_they_differ(self): + self.qor_fail("asap7/a/base", ["m1"], "https://dashboard.example/builds/1") + self.qor_fail("asap7/b/base", ["m1"], "https://dashboard.example/builds/2") + summary = self.summary() + self.assertNotIn("QoR baseline:", summary) + self.assertIn(" Baseline: https://dashboard.example/builds/1", summary) + self.assertIn(" Baseline: https://dashboard.example/builds/2", summary) + self.assertIn("Most regressed: m1 (2)", summary) + + def test_metric_count_orders_only_qor_rows(self): + for name in ("a", "b"): + self.write_file(f"logs/asap7/{name}/base/1_1_yosys.log", "ok\n") + self.write_file( + "reports/asap7/b/base/metadata-check.log", + "QoR regression detected for asap7/b/base:\n m1 2 (base 1, limit 1.5)\n", + ) + table = self.summary().split("-" * 60)[0] + self.assertLess( + table.index("STOP asap7/a/base"), table.index("STOP asap7/b/base") + ) + + def test_metadata_generation_error_text_reaches_the_summary(self): + self.finished("asap7/a/base") + self.write_file( + "reports/asap7/a/base/metadata-generate.log", "[ERROR] bad metric\n" + ) + summary = self.summary() + self.assertIn( + "asap7/a/base: metadata-generate.log: [ERROR] bad metric\n", summary + ) + + def test_failing_runs_line_alone_still_fails_the_design(self): + self.finished( + "nangate45/gcd/base", + "[INFO] nangate45/gcd/base: FAIL\n" + "[ERROR] Failing runs: nangate45/gcd/base\n", + ) + summary = self.summary() + self.assertIn("QOR nangate45/gcd/base [ERROR] Failing runs:", summary) + + def test_metadata_generation_error_of_a_passing_design_prints_once(self): + self.finished("asap7/a/base") + self.write_file( + "reports/asap7/a/base/metadata-generate.log", "[ERROR] bad metric\n" + ) + self.run_report("-v") + report = self.read("reports/asap7/a/base/report.log") + self.assertEqual(report.count("bad metric"), 1, report) + self.assertEqual( + self.summary("-svv").split("PASSING")[1].count("bad metric"), 0 + ) + + def test_log_that_is_not_utf8_is_read(self): + path = os.path.join(self.tmp, "logs/asap7/a/base/1_1_yosys.log") + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as f: + f.write(b"\xff\xfe \xc2\xb5\n[ERROR SYN-0001] bad input\n") + self.assertIn("1_1_yosys.log: SYN-0001", self.summary()) - report = genReport.gen_report(name, data) +class TestParseQorCheck(unittest.TestCase): + def test_ci_format(self): + metrics, baseline, notes = genReport.parse_qor_check( + [ + "[ERROR] Dashboard QoR check: FAIL (nangate45/gcd/base)\n", + "QoR regression detected for nangate45/gcd/base:\n", + " cts__timing__setup__tns -12.2456 (base -6.95522, limit -8.346264)\n", + " Baseline: https://dashboard.example/builds/7\n", + ] + ) + self.assertEqual( + metrics, [("cts__timing__setup__tns", "-12.2456", "-6.95522", "-8.346264")] + ) + self.assertEqual(baseline, "https://dashboard.example/builds/7") + self.assertEqual(notes, []) + + def test_local_check_warnings_become_notes(self): + _, _, notes = genReport.parse_qor_check( + [ + "[WARN] nangate45/gcd/base: no baseline to compare against " + "(strategy=none). Nothing was checked.\n", + "QoR check: INCONCLUSIVE\n", + ] + ) self.assertEqual( - report, - "\n".join( - [ - name, - " Flow reached last stage.", - " Found 2 metrics failures.", - " [ERROR] nangate45/gcd/base: FAIL — 19 passed, 6 failed, " - "0 missing (478 metrics sent)", - " cts__timing__setup__tns -12.2456 " - "(base -6.95522, limit -8.346264)", - " cts__timing__setup__ws -0.261172 " - "(base -0.156758, limit -0.169258)", - " finish__timing__setup__tns -12.4146 " - "(base -7.19445, limit -8.63334)", - " finish__timing__setup__ws -0.271991 " - "(base -0.163541, limit -0.176041)", - " globalroute__timing__setup__tns -12.7365 " - "(base -7.5407, limit -9.04884)", - " globalroute__timing__setup__ws -0.275208 " - "(base -0.171167, limit -0.183667)", - " [ERROR] Failing runs: nangate45/gcd/base", - ] - ), + notes, + [ + "QoR: no baseline to compare against (strategy=none). Nothing was checked." + ], + ) + + def test_reason_is_kept_only_for_a_check_without_a_verdict(self): + # The CI check writes these lines; only FAIL has the [ERROR] prefix. + for verdict in ( + "Dashboard QoR check: PASS", + "[ERROR] Dashboard QoR check: FAIL", + ): + _, _, notes = genReport.parse_qor_check( + [f"{verdict} (a/b/c)\n", "Reason: text\n"] + ) + self.assertEqual(notes, [], verdict) + _, _, notes = genReport.parse_qor_check( + [ + "Dashboard QoR check: INCONCLUSIVE (a/b/c)\n", + "[WARN] a/b/c: other\n", + "Reason: no data\n", + "[ERROR] Dashboard QoR check: FAIL (a/b/d)\n", + "Reason: ignored\n", + ] + ) + self.assertEqual(notes, ["QoR check INCONCLUSIVE: no data", "QoR: other"]) + + def test_unknown_metric_line_is_kept_verbatim(self): + metrics, _, _ = genReport.parse_qor_check( + ["QoR regression detected for x/y/z:\n", " odd line\n"] ) + self.assertEqual(metrics, [("odd line",)]) if __name__ == "__main__": diff --git a/flow/util/genReport.py b/flow/util/genReport.py index 52ec6115dd..3caac0a2bb 100755 --- a/flow/util/genReport.py +++ b/flow/util/genReport.py @@ -6,6 +6,8 @@ import argparse import os import re +import textwrap +from collections import Counter os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) @@ -16,13 +18,34 @@ SUMMARY_FILENAME = f"{REPORTS_FOLDER}/report-summary.log" DRC_FILENAME = "5_route_drc.rpt" LAST_EXPECTED_LOG = ["6_report.log", "generate_abstract.log"] -METRICS_LOG_FMT = "metadata-generate.log" -METRICS_CHECK_FMT = "{}/metadata-check.log" +METRICS_LOG = "metadata-generate.log" +METRICS_CHECK_LOG = "metadata-check.log" +CALIBRE_LOG = "calibre/save-to-drc-db.log" REGEX_ERROR = re.compile(r"^\[error ?(\w+-\d+)?\]", re.IGNORECASE) REGEX_WARNING = re.compile(r"^\[warning ?(\w+-\d+)?\]", re.IGNORECASE) +REGEX_SIGNAL = re.compile(r"\bsignal (\d+)|\bKilled\b") +# Metric line under "QoR regression detected for ...": +# (base , limit ) +REGEX_QOR_METRIC = re.compile(r"^(\S+) (\S+) \(base (\S+), limit (\S+)\)$") +REGEX_QOR_STATUS = re.compile(r"^(?:\[ERROR\] )?Dashboard QoR check: (\w+)") +# checkQorMetrics.py starts its lines with the run identity. +REGEX_RUN_PREFIX = re.compile(r"^[^\s/]+/[^\s/]+/[^\s/:]+: ") SKIPPED_FLOW_VARIANT_KEYWORDS = ["test", "tune"] STATUS_GREEN = "Passing" STATUS_RED = "Failing" +TAIL_LINES = 10 +RULE = "-" * 60 + +# Failure categories, most critical first. The summary sorts by this order. +CATEGORIES = { + "STOP": "flow stopped", + "ERR": "flow error", + "DRC": "Calibre DRC", + "QOR": "QoR regression", +} +CATEGORY_RANK = {code: rank for rank, code in enumerate(CATEGORIES)} +WARN = "WARN" +PASS = "PASS" HELP_TEXT = """ Scans "./logs" and "./reports" folders for errors and warnings. @@ -30,10 +53,10 @@ parser = argparse.ArgumentParser(description=HELP_TEXT) HELP_TEXT = """ -Level 0: report error message id and drc types. +Level 0: report error message ids. Level 1: report full error messages. -Level 2: report full error messages and warning messages id. -Level 3: report full error messages and full warning messages. +Level 2: also report passing designs and warning message ids. +Level 3: also report full warning messages. """ parser.add_argument( "--verbose", "-v", required=False, default=0, action="count", help=HELP_TEXT @@ -60,168 +83,333 @@ args = parser.parse_args() -def parse_messages(filename, print_missing=True): - """ - TODO: docs - """ - errors = list() - warnings = list() +def read_lines(filename): + """Return the lines of a file, or None if it cannot be read.""" try: - with open(filename, "r") as file: - lines = file.readlines() - for line in lines: - if re.search(REGEX_ERROR, line): - errors.append(line.strip()) - elif re.search(REGEX_WARNING, line): - warnings.append(line.strip()) - except BaseException: - if print_missing: - print(f"Failed to open {filename}.") + with open(filename, "r", encoding="utf-8", errors="replace") as file: + return file.readlines() + except OSError: + return None + + +def parse_messages(lines, source): + """Return ([ERROR] lines, [WARNING] lines) as (source, text) pairs.""" + errors = [] + warnings = [] + for line in lines or []: + if REGEX_ERROR.search(line): + errors.append((source, line.strip())) + elif REGEX_WARNING.search(line): + warnings.append((source, line.strip())) return errors, warnings -def parse_qor_regressions(filename): - """Return failed metric details from a QoR check log, if present.""" - regressions = [] +def parse_qor_check(lines): + """ + Read a metadata-check.log written by checkQorMetrics.py or the CI check. + + Returns (metrics, baseline, notes). Each metric is a tuple + (name, current, base, limit), or a 1-tuple with the raw line when the + format is unknown. Notes say why the QoR check did not give a verdict. + """ + metrics = [] + baseline = None + notes = [] in_regressions = False - try: - with open(filename, "r", encoding="utf-8") as file: - for line in file: - if line.startswith("QoR regression detected for "): - in_regressions = True - elif in_regressions and line.startswith(" ") and line.strip(): - regressions.append(line.strip()) - elif in_regressions: - break - except OSError: - pass - return regressions + status_note = None # index of the note for a check without a verdict + for line in lines or []: + text = line.strip() + if line.startswith("QoR regression detected for "): + in_regressions = True + continue + if text.startswith("Baseline: "): + baseline = text.split(" ", 1)[1] + continue + if in_regressions and line.startswith(" ") and text: + match = REGEX_QOR_METRIC.match(text) + metrics.append(match.groups() if match else (text,)) + continue + in_regressions = False + status = REGEX_QOR_STATUS.match(text) + if status: + status_note = None + if status.group(1) not in ("PASS", "FAIL"): + status_note = len(notes) + notes.append(f"QoR check {status.group(1)}") + elif text.startswith("Reason: ") and status_note is not None: + notes[status_note] += f": {text.split(' ', 1)[1]}" + elif text.startswith("[WARN] "): + message = REGEX_RUN_PREFIX.sub("", text[len("[WARN] ") :]) + notes.append(f"QoR: {message}") + return metrics, baseline, notes + + +def message_id(text, regex): + """Return the message id (e.g. GPL-0307), or the full text if it has none.""" + match = regex.match(text) + return match.group(1) if match and match.group(1) else text + + +def shorten(text, width=90): + return textwrap.shorten(text, width=width, placeholder=" ...") + + +def collect(log_dir, files, report_dir): + """Read the logs and reports of one design and classify it.""" + d = dict() + d["output_file"] = os.path.join(report_dir, REPORT_FILENAME) + d["log_dir"] = log_dir + + d["log_errors"] = [] + d["log_warnings"] = [] + d["last_log"] = "" + for name in sorted(files): + errors, warnings = parse_messages(read_lines(os.path.join(log_dir, name)), name) + d["log_errors"] += errors + d["log_warnings"] += warnings + if name.endswith(".log"): + d["last_log"] = name + d["finished"] = any(name in files for name in LAST_EXPECTED_LOG) + + missing = [] + + lines = read_lines(os.path.join(report_dir, METRICS_LOG)) + if lines is None and d["finished"]: + missing.append(METRICS_LOG) + d["metrics_logs_errors"], d["metrics_logs_warnings"] = parse_messages( + lines, METRICS_LOG + ) + lines = read_lines(os.path.join(report_dir, METRICS_CHECK_LOG)) + if lines is None and d["finished"]: + missing.append(METRICS_CHECK_LOG) + d["metrics_errors"], _ = parse_messages(lines, METRICS_CHECK_LOG) + d["qor_metrics"], d["qor_baseline"], qor_notes = parse_qor_check(lines) -def append_metrics_errors(errors, regressions, text): - if not errors: - return text + d["calibre_errors"], d["calibre_warnings"] = parse_messages( + read_lines(os.path.join(log_dir, CALIBRE_LOG)), CALIBRE_LOG + ) - text += f" Found {len(errors)} metrics failures.\n" - for error in errors: - text += f" {error}\n" - if error.startswith("[ERROR] ") and ": FAIL " in error: - for regression in regressions: - text += f" {regression}\n" - return text + d["drcs"] = Counter() + lines = read_lines(os.path.join(report_dir, DRC_FILENAME)) + if lines is None and d["finished"]: + missing.append(DRC_FILENAME) + for line in lines or []: + if "violation type:" in line: + d["drcs"][line.split("violation type:", 1)[1].strip()] += 1 + + d["tail"] = [] + d["cause"] = "" + if not d["finished"] and d["last_log"]: + tail = read_lines(os.path.join(log_dir, d["last_log"])) or [] + d["tail"] = [line.rstrip() for line in tail[-TAIL_LINES:]] + for line in reversed(d["tail"]): + match = REGEX_SIGNAL.search(line) + if match: + signal = match.group(1) or "9" + d["cause"] = f"killed by signal {signal}" + if signal == "9": + d["cause"] += " (out of memory?)" + break + + if not d["finished"]: + d["category"] = "STOP" + elif d["log_errors"]: + d["category"] = "ERR" + elif d["calibre_errors"]: + d["category"] = "DRC" + elif d["metrics_errors"]: + d["category"] = "QOR" + else: + d["category"] = None + d["status"] = STATUS_RED if d["category"] else STATUS_GREEN + notes = list(qor_notes) + if missing: + notes.append("missing " + ", ".join(missing)) + if d["status"] == STATUS_GREEN: + notes += [f"{source}: {text}" for source, text in d["metrics_logs_errors"]] + if d["drcs"]: + notes.append(f"router DRC under the limit: {format_drcs(d['drcs'])}") + d["notes"] = notes + return d -def append_text(list_, text, sup, regex, verbose): + +def qor_error_lines(data): """ - TODO + QoR check [ERROR] lines to show. checkQorMetrics.py repeats the failing + runs in a trailing line, so hide it unless it is the only one. """ - if len(list_) == 0: - return text - text += f" Found {len(list_)} {sup}.\n" - if verbose >= 1: # print full message - for item in list_: - text += f" {item}\n" - else: # only print the id - set_ = set() - long_set = set() - for item in list_: - message = re.match(regex, item).group(1) - if message is None: - message = item - if len(message) > 10: # considering a std msg id XYZ-0000 - long_set.add(message) - else: - set_.add(message) - set_ = sorted(set_, reverse=True) - if set_: - text += f" {', '.join(set_)}\n" - for item in long_set: - text += f" {item}\n" - return text - - -def gen_report(name, data): + texts = [text for _, text in data["metrics_errors"]] + shown = [t for t in texts if not t.startswith("[ERROR] Failing runs:")] + return shown or texts + + +def format_drcs(drcs): + return ", ".join(f"{name} ({count})" for name, count in drcs.items()) + + +def qor_names(data): + return [metric[0] for metric in data["qor_metrics"] if len(metric) == 4] + + +def row_code(data): + if data["category"]: + return data["category"] + return WARN if data["notes"] else PASS + + +def sort_key(item): + """Most critical first; more regressed metrics first within QoR.""" + name, data = item + code = row_code(data) + rank = CATEGORY_RANK.get(code, len(CATEGORY_RANK) + (code == PASS)) + metrics = len(data["qor_metrics"]) if code == "QOR" else 0 + return (rank, -metrics, name) + + +def last_error(errors): + """Short form of the last error: the one most likely to stop the flow.""" + source, text = errors[-1] + more = f" (+{len(errors) - 1} more)" if len(errors) > 1 else "" + return f"{source}: {message_id(text, REGEX_ERROR)}{more}" + + +def row_reason(data): + """One-line reason for the summary table.""" + code = row_code(data) + if code == "STOP": + if data["log_errors"]: + return last_error(data["log_errors"]) + if not data["last_log"]: + return "no log file found" + return f"{data['last_log']}: {data['cause'] or 'no error message'}" + if code == "ERR": + return last_error(data["log_errors"]) + if code == "DRC": + return f"Calibre: {len(data['calibre_errors'])} error(s)" + if code == "QOR": + names = qor_names(data) + if not names: + return message_id(qor_error_lines(data)[0], REGEX_ERROR) + if len(names) == 1: + return names[0] + return f"{len(names)} metrics: " + ", ".join(names) + if code == WARN: + more = len(data["notes"]) - 1 + return data["notes"][0] + (f" (+{more} more)" if more else "") + return "passed" + + +def headline(data): + """Short description after the design name in the detail block.""" + code = data["category"] + if code == "STOP": + if not data["last_log"]: + return "flow did not reach last stage, no log file found" + text = f"stopped in {data['last_log']}" + return f"{text}, {data['cause']}" if data["cause"] else text + if code == "ERR": + return f"{len(data['log_errors'])} error(s) in the logs" + if code == "DRC": + return f"{len(data['calibre_errors'])} Calibre error(s)" + if code == "QOR": + count = len(data["qor_metrics"]) + if count == 0: + return "QoR check failed" + return f"{count} metric{'s' if count > 1 else ''} over the limit" + return "passed" + + +def message_lines(messages, regex, full): """ - TODO: docs + Lines for a list of (source, text) messages. + Full mode shows each distinct message with a repeat count. + Otherwise one line per source with the distinct message ids. """ - - if args.verbose >= 2 or data["status"] != STATUS_GREEN: - output = f"{name}\n" - if data["finished"]: - output += " Flow reached last stage.\n" + lines = [] + if full: + for (source, text), count in Counter(messages).items(): + repeat = f" (x{count})" if count > 1 else "" + lines.append(f"{source}: {text}{repeat}") + return lines + by_source = {} + for source, text in messages: + by_source.setdefault(source, {})[message_id(text, regex)] = None + for source, ids in by_source.items(): + lines.append(f"{source}: {', '.join(ids)}") + return lines + + +def qor_table(metrics): + rows = [("metric", "current", "base", "limit")] + rows += [m for m in metrics if len(m) == 4] + lines = [] + if len(rows) > 1: + width = max(len(row[0]) for row in rows) + for row in rows: + lines.append( + f"{row[0]:<{width}} {row[1]:<12} {row[2]:<12} {row[3]}".rstrip() + ) + lines += [m[0] for m in metrics if len(m) == 1] + return lines + + +def gen_report(name, data, shared_baseline=None, with_notes=True): + """ + Detail block for one design. Empty for a passing design without notes. + with_notes=False leaves out the notes of a passing design, for a summary + that already shows them under "Needs attention". + """ + verbose = args.verbose + if data["status"] == STATUS_GREEN and not data["notes"] and verbose < 2: + return "" + + body = [] + if data["category"] == "STOP" and data["tail"] and not data["log_errors"]: + body.append(f"Last {len(data['tail'])} lines of {data['last_log']}:") + body += [f" {line}" for line in data["tail"]] + full = verbose >= 1 + body += message_lines(data["log_errors"], REGEX_ERROR, full) + if data["status"] == STATUS_RED: + body += message_lines(data["metrics_logs_errors"], REGEX_ERROR, full) + body += message_lines(data["calibre_errors"], REGEX_ERROR, full) + if data["metrics_errors"]: + if data["qor_metrics"]: + body += qor_table(data["qor_metrics"]) else: - output += " Flow did not reach last stage.\n" - if data["last_log"]: - output += f" Last log file {data['last_log']}\n" - else: - output += " No log file found.\n" - else: - output = "" - - output = append_text( - data["log_errors"], output, "errors in the logs", REGEX_ERROR, args.verbose - ) - output = append_text( - data["metrics_logs_errors"], - output, - "errors in the metrics logs", - REGEX_ERROR, - args.verbose, - ) - output = append_metrics_errors( - data["metrics_errors"], data["metrics_regressions"], output - ) - output = append_text( - data["calibre_errors"], output, "calibre failures", REGEX_ERROR, args.verbose - ) - - if args.verbose >= 2: - output = append_text( - data["log_warnings"], - output, - "warnings in the logs", - REGEX_WARNING, - args.verbose - 2, - ) - output = append_text( - data["metrics_logs_warnings"], - output, - "warnings in the metrics logs", - REGEX_WARNING, - args.verbose - 2, - ) - output = append_text( - data["metrics_warnings"], - output, - "metrics warnings", - REGEX_WARNING, - args.verbose - 2, - ) - output = append_text( - data["calibre_warnings"], - output, - "calibre warnings", - REGEX_WARNING, - args.verbose - 2, + body += qor_error_lines(data) + if data["qor_baseline"] and data["qor_baseline"] != shared_baseline: + body.append(f"Baseline: {data['qor_baseline']}") + if data["drcs"] and data["status"] == STATUS_RED: + body.append(f"Router DRC: {format_drcs(data['drcs'])}") + if data["status"] == STATUS_RED: + body += [f"Note: {note}" for note in data["notes"]] + if verbose >= 2: + warnings = ( + data["log_warnings"] + + data["metrics_logs_warnings"] + + data["calibre_warnings"] ) + body += message_lines(warnings, REGEX_WARNING, verbose >= 3) - if len(data["drcs"].keys()) > 0: - if data["status"] == STATUS_GREEN: - output += " Design has the violations under the allowed limit: " + if data["status"] == STATUS_GREEN: + if with_notes: + lines = [f"{name}: {note}" for note in data["notes"]] else: - output += " Design has the violations over the allowed limit: " - for drc, count in data["drcs"].items(): - output += f"{drc} ({count}) " - output += "\n" - - return output.strip() + lines = [f"{name}: passed, see NEEDS ATTENTION"] if data["notes"] else [] + lines = lines or [f"{name}: passed"] + else: + lines = [f"{name}: {headline(data)}"] + lines += [f" {line}" for line in body] + return "\n".join(lines) def write_report(path, content): - """ - TODO: docs - """ + """Write one design report and append it to the combined report.""" content = content.strip() + # A flow that stops before its first stage has no reports directory. + os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w") as file: file.write(content) if args.verbose: @@ -233,41 +421,95 @@ def write_report(path, content): print(content) -def get_summary(status, text): - """ - TODO: docs - """ - tmp = "" - for name, data in design_list.items(): - if data["status"] == status: - content = gen_report(name, data) - if content != "": - tmp += content + "\n" - # add empty line for readability - if args.verbose >= 2 or status == STATUS_RED: - tmp += "\n" - if tmp != "": - text += "-" * 30 + "\n" - text += " " * 7 + f"{status} designs\n" - text += "-" * 30 + "\n" - text += tmp - - return text +def section(title): + return f"{RULE}\n{title}\n{RULE}\n" -def write_summary(): - """ - Write summary and individual report files with data on design_list - """ - summary = "\n" - summary += "=" * 60 + "\n" - summary += " " * 26 + "SUMMARY\n" - summary += "=" * 60 + "\n" - summary += f"\nNumber of designs: {len(design_list.keys())}\n\n" +def summary_header(designs): + failing = [d for d in designs.values() if d["status"] == STATUS_RED] + lines = ["=" * 60] + lines.append( + f"SUMMARY: {len(designs)} design{'s' if len(designs) != 1 else ''}, {len(failing)} failing, " + f"{len(designs) - len(failing)} passing" + ) + if failing: + counts = Counter(d["category"] for d in failing) + lines.append( + " " + + " | ".join( + f"{label} {counts[code]}" for code, label in CATEGORIES.items() + ) + ) + regressed = Counter(name for d in failing for name in qor_names(d)) + # A metric in one design only repeats its table row, so list shared ones. + common = [(name, n) for name, n in regressed.most_common(3) if n > 1] + if common: + lines.append( + " Most regressed: " + ", ".join(f"{name} ({n})" for name, n in common) + ) + baselines = {d["qor_baseline"] for d in failing if d["metrics_errors"]} + shared_baseline = baselines.pop() if len(baselines) == 1 else None + if shared_baseline: + lines.append(f" QoR baseline: {shared_baseline}") + lines.append("=" * 60) + return "\n".join(lines) + "\n", shared_baseline + + +def summary_table(rows): + if not rows: + return "" + width = max(len(name) for name, _ in rows) + lines = [ + f"{row_code(data):<4} {name:<{width}} {shorten(row_reason(data))}" + for name, data in rows + ] + return "\n".join(lines) + "\n" + + +def attention_section(rows): + """Notes of passing designs; a note shared by several designs prints once.""" + by_note = {} + for name, data in rows: + for note in data["notes"]: + by_note.setdefault(note, []).append(name) + lines = [] + for note, names in by_note.items(): + if len(names) == 1: + lines.append(f"{names[0]}: {note}") + else: + lines.append(f"{note} ({len(names)} designs):") + wrapped = textwrap.wrap(", ".join(names), width=76, break_on_hyphens=False) + lines += [f" {line}" for line in wrapped] + return "\n".join(lines) + "\n" + - summary = get_summary(STATUS_RED, summary) - summary += "\n" - summary = get_summary(STATUS_GREEN, summary) +def write_summary(): + """Write the summary: header, one row per design, then details.""" + rows = sorted(design_list.items(), key=sort_key) + header, shared_baseline = summary_header(design_list) + listed = [ + (name, data) + for name, data in rows + if row_code(data) != PASS or args.verbose >= 2 + ] + + summary = "\n" + header + "\n" + summary_table(listed) + for code, label in CATEGORIES.items(): + group = [(n, d) for n, d in rows if d["category"] == code] + if group: + summary += "\n" + section(f"{label.upper()} ({len(group)})") + summary += "\n\n".join(gen_report(n, d, shared_baseline) for n, d in group) + summary += "\n" + noted = [(n, d) for n, d in rows if row_code(d) == WARN] + if noted: + summary += "\n" + section("NEEDS ATTENTION (not failing)") + summary += attention_section(noted) + if args.verbose >= 2: + passing = [(n, d) for n, d in rows if d["status"] == STATUS_GREEN] + details = [gen_report(n, d, with_notes=False) for n, d in passing] + if passing: + summary += "\n" + section(f"PASSING ({len(passing)})") + summary += "\n".join(details) + "\n" if not args.quiet: if os.path.isfile(SUMMARY_FILENAME): @@ -281,95 +523,30 @@ def write_summary(): design_list = dict() -for log_dir, dirs, files in sorted(os.walk(LOGS_FOLDER, topdown=False)): - dir_list = log_dir.split(os.sep) - # Handles autotuner folders, which do not have `report.log` natively. - # TODO: Can we log something for autotuner? - if len(dir_list) != 4 or any( - word in dir_list[-1] for word in SKIPPED_FLOW_VARIANT_KEYWORDS - ): - continue - report_dir = log_dir.replace(LOGS_FOLDER, REPORTS_FOLDER) - - # basic info about current design - platform, design, variant = dir_list[1:] - d = dict() - d["output_file"] = os.path.join(report_dir, REPORT_FILENAME) - - # check if design ran to completion without errors or warnings - d["log_errors"] = list() - d["log_warnings"] = list() - d["last_log"] = "" - for name_ in sorted(files): - temp_e, temp_w = parse_messages(os.path.join(log_dir, name_)) - d["log_errors"] += temp_e - d["log_warnings"] += temp_w - if name_.endswith(".log"): - d["last_log"] = name_ - d["finished"] = any(name in files for name in LAST_EXPECTED_LOG) - - # check if metrics generation had issues - d["metrics_logs_errors"], d["metrics_logs_warnings"] = parse_messages( - os.path.join(report_dir, METRICS_LOG_FMT), - print_missing=d["finished"], - ) - - # check if metrics passed - d["metrics_errors"], d["metrics_warnings"] = parse_messages( - METRICS_CHECK_FMT.format(report_dir), print_missing=d["finished"] - ) - d["metrics_regressions"] = parse_qor_regressions( - METRICS_CHECK_FMT.format(report_dir) - ) +if __name__ == "__main__": + for log_dir, dirs, files in sorted(os.walk(LOGS_FOLDER, topdown=False)): + dir_list = log_dir.split(os.sep) + # Handles autotuner folders, which do not have `report.log` natively. + # TODO: Can we log something for autotuner? + if len(dir_list) != 4 or any( + word in dir_list[-1] for word in SKIPPED_FLOW_VARIANT_KEYWORDS + ): + continue + platform, design, variant = dir_list[1:] + report_dir = log_dir.replace(LOGS_FOLDER, REPORTS_FOLDER, 1) + design_list[f"{platform}/{design}/{variant}"] = collect( + log_dir, files, report_dir + ) - # check if calibre was run and if drc check passed - d["calibre_errors"], d["calibre_warnings"] = parse_messages( - os.path.join(log_dir, "calibre/save-to-drc-db.log"), print_missing=False - ) + if not os.path.isdir(REPORTS_FOLDER): + os.mkdir(REPORTS_FOLDER) - # check if there were drc violations - d["drcs"] = dict() - try: - drc_report_file = os.path.join(report_dir, DRC_FILENAME) - with open(drc_report_file, "r") as file_: - for line_ in file_.readlines(): - if "violation type:" in line_: - type_ = line_.split("violation type:", 1)[1].strip() - if type_ in d["drcs"].keys(): - d["drcs"][type_] += 1 - else: - d["drcs"][type_] = 1 - except BaseException: - if d["finished"]: - print(f"Failed to open {DRC_FILENAME}.") - - if d["log_errors"] or d["metrics_errors"] or d["calibre_errors"]: - d["status"] = STATUS_RED + if args.summary: + write_summary() else: - if not d["finished"]: - d["status"] = STATUS_RED - last_lines = "Could not find last lines of the log file." - if d["last_log"]: - with open(os.path.join(log_dir, d["last_log"]), "r") as last_file: - last_lines = last_file.readlines()[-10:] - last_lines = " ".join(last_lines) - message = "[ERROR CI-0000] No error message found. " - message += "Here are the last 10 lines of the log.\n" - d["log_errors"] = [message + last_lines] - else: - d["status"] = STATUS_GREEN - - design_list[f"{platform} {design} ({variant})"] = d - -if not os.path.isdir(REPORTS_FOLDER): - os.mkdir(REPORTS_FOLDER) - -if args.summary: - write_summary() -else: - if os.path.isfile(SINGLE_REPORT_FILENAME): - if not args.quiet: - print(f"Overwriting report {SINGLE_REPORT_FILENAME}.") - os.remove(SINGLE_REPORT_FILENAME) - for name_, data_ in design_list.items(): - write_report(data_["output_file"], gen_report(name_, data_)) + if os.path.isfile(SINGLE_REPORT_FILENAME): + if not args.quiet: + print(f"Overwriting report {SINGLE_REPORT_FILENAME}.") + os.remove(SINGLE_REPORT_FILENAME) + for name_, data_ in design_list.items(): + write_report(data_["output_file"], gen_report(name_, data_))