From 1255bc898b0c6827fcf6eb4d551f63118b16a3af Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Wed, 4 Nov 2026 11:48:05 -0300 Subject: [PATCH 1/3] feat(genReport): sort the summary by criticality The CI summary listed each failing design as a separate block. It counted error lines instead of regressed metrics and printed the same baseline URL for each design. Readers could not see which failures were most severe, or which metric regressed in many designs. The summary now starts with one row per design, sorted by severity: flow stopped, flow error, Calibre DRC, QoR regression, then warnings. Grouped details follow. QoR results that did not give a verdict were not shown before. The same applies to missing metadata logs. These items now show under "Needs attention" and do not change the pass or fail status. Signed-off-by: Vitor Bandeira --- flow/test/test_genReport.py | 249 +++++++------ flow/util/genReport.py | 691 ++++++++++++++++++++++-------------- 2 files changed, 549 insertions(+), 391 deletions(-) diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index b7707c6c66..0d49110f03 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -84,25 +84,28 @@ def summary(self): with open(os.path.join(self.tmp, "reports/report-summary.log")) as f: return f.read() + def finished(self, pdv, check=""): + """A run that reached the last stage, with its metadata and DRC reports.""" + self.write_file(f"logs/{pdv}/6_report.log", "Finished\n") + self.write_file(f"reports/{pdv}/metadata-generate.log", "") + self.write_file(f"reports/{pdv}/metadata-check.log", check) + self.write_file(f"reports/{pdv}/5_route_drc.rpt", "") + def test_summary_includes_make_failure_before_the_first_stage(self): result = self.run_helper() self.assertEqual(result.returncode, 2, result.stdout + result.stderr) self.assertRegex(result.stdout, r"No rule to make target .missing\.v.") - for platform in ("nangate45", "ihp-sg13g2"): - self.write_file(f"logs/{platform}/gcd/base/6_report.log", "Finished\n") - self.write_file(f"reports/{platform}/gcd/base/metadata-check.log", "") - self.write_file( - "reports/nangate45/gcd/base/metadata-check.log", - "[ERROR] nangate45/gcd/base: FAIL\n", - ) + self.finished("ihp-sg13g2/gcd/base") + self.finished("nangate45/gcd/base", "[ERROR] nangate45/gcd/base: FAIL\n") summary = self.summary() - self.assertIn("Number of designs: 3", summary) - self.assertIn("nangate45 gcd (base)", summary) - self.assertIn("sky130hd gcd (base)\n Flow did not reach last stage.", summary) + self.assertIn("SUMMARY: 3 designs, 2 failing, 1 passing", summary) + self.assertIn("QOR nangate45/gcd/base", summary) + self.assertIn("STOP sky130hd/gcd/base", summary) + self.assertIn("sky130hd/gcd/base: stopped in 0_make.log", summary) self.assertRegex(summary, r"No rule to make target .missing\.v.") - self.assertNotIn("ihp-sg13g2 gcd (base)", summary) + self.assertNotIn("ihp-sg13g2/gcd/base", summary) def test_early_failure_uses_the_configured_variant_and_design_nickname(self): self.write_file( @@ -111,10 +114,7 @@ def test_early_failure_uses_the_configured_variant_and_design_nickname(self): ) result = self.run_helper() self.assertEqual(result.returncode, 2, result.stdout + result.stderr) - self.assertIn( - "sky130hd gcd_alias (alternate)\n Flow did not reach last stage.", - self.summary(), - ) + self.assertIn("STOP sky130hd/gcd_alias/alternate", self.summary()) def test_successful_rerun_does_not_keep_the_early_failure(self): result = self.run_helper() @@ -122,143 +122,140 @@ def test_successful_rerun_does_not_keep_the_early_failure(self): self.write_file("missing.v", "") result = self.run_helper() self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - summary = self.summary() - self.assertIn("Number of designs: 1", summary) - self.assertNotIn("Failing designs", summary) + self.assertIn("SUMMARY: 1 design, 0 failing, 1 passing", self.summary()) def test_finish_log_is_recognized_with_a_later_auxiliary_log(self): - self.write_file("logs/sky130hd/gcd/base/6_report.log", "Finished\n") + self.finished("sky130hd/gcd/base") self.write_file("logs/sky130hd/gcd/base/6_write_rc.log", "RC values\n") - summary = self.summary() - self.assertIn("Number of designs: 1", summary) - self.assertNotIn("Failing designs", summary) + self.assertIn("SUMMARY: 1 design, 0 failing, 1 passing", self.summary()) def test_empty_log_directory_is_reported_as_incomplete(self): os.makedirs(os.path.join(self.tmp, "logs/sky130hd/gcd/base")) summary = self.summary() - self.assertIn("Number of designs: 1", summary) - self.assertIn("sky130hd gcd (base)\n Flow did not reach last stage.", summary) - self.assertIn("No log file found.", summary) + self.assertIn("STOP sky130hd/gcd/base no log file found", summary) + self.assertIn( + "sky130hd/gcd/base: flow did not reach last stage, no log file found", + summary, + ) def test_dashboard_verdict_reaches_the_summary(self): - self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\n") - self.write_file( - "reports/nangate45/gcd/base/metadata-check.log", + self.finished( + "nangate45/gcd/base", "[ERROR] Dashboard QoR check: FAIL (nangate45/gcd/base)\n" "QoR regression detected for nangate45/gcd/base:\n" " finish__timing__setup__tns -12.0 (base -6.0, limit -8.0)\n" " Baseline: https://dashboard.example/builds/7\n", ) summary = self.summary() - self.assertIn("Failing designs", summary) - self.assertIn("nangate45 gcd (base)", summary) - self.assertIn("finish__timing__setup__tns -12.0", summary) - self.assertIn("limit -8.0", summary) - self.assertIn("https://dashboard.example/builds/7", summary) - + self.assertIn("QoR regression 1", summary) + self.assertIn("QoR baseline: https://dashboard.example/builds/7", summary) + self.assertIn("QOR nangate45/gcd/base finish__timing__setup__tns", summary) + self.assertIn("nangate45/gcd/base: 1 metric over the limit", summary) + self.assertRegex(summary, r"finish__timing__setup__tns +-12\.0 +-6\.0 +-8\.0") -class TestGenReport(unittest.TestCase): - def test_incomplete_flow_reports_that_it_did_not_reach_last_stage(self): - name = "nangate45 gcd (base)" - data = { - "status": genReport.STATUS_RED, - "finished": False, - "last_log": "5_route.log", - "log_errors": [], - "metrics_logs_errors": [], - "metrics_errors": [], - "metrics_regressions": [], - "calibre_errors": [], - "drcs": {}, - } + def test_failures_are_ordered_by_criticality(self): + self.finished( + "asap7/a_one_metric/base", + "[ERROR] Dashboard QoR check: FAIL (asap7/a_one_metric/base)\n" + "QoR regression detected for asap7/a_one_metric/base:\n" + " m1 2 (base 1, limit 1.5)\n", + ) + self.finished( + "asap7/b_two_metrics/base", + "[ERROR] Dashboard QoR check: FAIL (asap7/b_two_metrics/base)\n" + "QoR regression detected for asap7/b_two_metrics/base:\n" + " m1 2 (base 1, limit 1.5)\n" + " m2 2 (base 1, limit 1.5)\n", + ) + self.finished("asap7/c_calibre/base") + self.write_file( + "logs/asap7/c_calibre/base/calibre/save-to-drc-db.log", + "[ERROR DRC-0001] violations\n", + ) + self.finished("asap7/d_error/base") + self.write_file("logs/asap7/d_error/base/5_1_grt.log", "[ERROR GRT-0116] x\n") + self.write_file( + "logs/asap7/e_killed/base/5_2_route.log", + "Command terminated by signal 9\n", + ) + self.finished( + "asap7/f_inconclusive/base", + "Dashboard QoR check: INCONCLUSIVE (asap7/f_inconclusive/base)\n" + "Reason: no baseline\n", + ) - report = genReport.gen_report(name, data) + summary = self.summary() + codes = [ + line.split()[:2] + for line in summary.split("-" * 60)[0].splitlines() + if line.split()[:1] in (["STOP"], ["ERR"], ["DRC"], ["QOR"], ["WARN"]) + ] self.assertEqual( - report, - "\n".join( - [ - name, - " Flow did not reach last stage.", - " Last log file 5_route.log", - ] - ), + codes, + [ + ["STOP", "asap7/e_killed/base"], + ["ERR", "asap7/d_error/base"], + ["DRC", "asap7/c_calibre/base"], + ["QOR", "asap7/b_two_metrics/base"], + ["QOR", "asap7/a_one_metric/base"], + ["WARN", "asap7/f_inconclusive/base"], + ], + ) + self.assertIn("5_2_route.log: killed by signal 9 (out of memory?)", summary) + self.assertIn("5_1_grt.log: [ERROR GRT-0116] x", summary) + self.assertIn( + "asap7/f_inconclusive/base: QoR check INCONCLUSIVE: no baseline", summary ) - def test_parse_qor_regressions_omits_the_optional_heading(self): - with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as log: - log.write("Ignored UTF-8 text: µ\n") - log.write( - "QoR regression detected for nangate45/gcd:\n" - " cts__timing__setup__tns -12.2456 " - "(base -6.95522, limit -8.346264)\n" - ) - log.flush() + def test_missing_metadata_is_grouped_under_needs_attention(self): + for name in ("a", "b"): + self.write_file(f"logs/asap7/{name}/base/6_report.log", "Finished\n") + self.write_file(f"reports/asap7/{name}/base/5_route_drc.rpt", "") + summary = self.summary() + self.assertIn( + "missing metadata-generate.log, metadata-check.log (2 designs):\n" + " asap7/a/base, asap7/b/base\n", + summary, + ) - self.assertEqual( - genReport.parse_qor_regressions(log.name), - [ - "cts__timing__setup__tns -12.2456 " - "(base -6.95522, limit -8.346264)" - ], - ) - def test_qor_metrics_are_nested_under_the_failing_run(self): - name = "nangate45 gcd (base)" - data = { - "status": genReport.STATUS_RED, - "finished": True, - "last_log": "", - "log_errors": [], - "metrics_logs_errors": [], - "metrics_errors": [ - "[ERROR] nangate45/gcd/base: FAIL — 19 passed, 6 failed, " - "0 missing (478 metrics sent)", - "[ERROR] Failing runs: nangate45/gcd/base", - ], - "metrics_regressions": [ - "cts__timing__setup__tns -12.2456 " "(base -6.95522, limit -8.346264)", - "cts__timing__setup__ws -0.261172 " "(base -0.156758, limit -0.169258)", - "finish__timing__setup__tns -12.4146 " - "(base -7.19445, limit -8.63334)", - "finish__timing__setup__ws -0.271991 " - "(base -0.163541, limit -0.176041)", - "globalroute__timing__setup__tns -12.7365 " - "(base -7.5407, limit -9.04884)", - "globalroute__timing__setup__ws -0.275208 " - "(base -0.171167, limit -0.183667)", - ], - "calibre_errors": [], - "drcs": {}, - } - - report = genReport.gen_report(name, data) +class TestParseQorCheck(unittest.TestCase): + def test_ci_format(self): + metrics, baseline, notes = genReport.parse_qor_check( + [ + "[ERROR] Dashboard QoR check: FAIL (nangate45/gcd/base)\n", + "QoR regression detected for nangate45/gcd/base:\n", + " cts__timing__setup__tns -12.2456 (base -6.95522, limit -8.346264)\n", + " Baseline: https://dashboard.example/builds/7\n", + ] + ) + self.assertEqual( + metrics, [("cts__timing__setup__tns", "-12.2456", "-6.95522", "-8.346264")] + ) + self.assertEqual(baseline, "https://dashboard.example/builds/7") + self.assertEqual(notes, []) + def test_local_check_warnings_become_notes(self): + _, _, notes = genReport.parse_qor_check( + [ + "[WARN] nangate45/gcd/base: no baseline to compare against " + "(strategy=none). Nothing was checked.\n", + "QoR check: INCONCLUSIVE\n", + ] + ) self.assertEqual( - report, - "\n".join( - [ - name, - " Flow reached last stage.", - " Found 2 metrics failures.", - " [ERROR] nangate45/gcd/base: FAIL — 19 passed, 6 failed, " - "0 missing (478 metrics sent)", - " cts__timing__setup__tns -12.2456 " - "(base -6.95522, limit -8.346264)", - " cts__timing__setup__ws -0.261172 " - "(base -0.156758, limit -0.169258)", - " finish__timing__setup__tns -12.4146 " - "(base -7.19445, limit -8.63334)", - " finish__timing__setup__ws -0.271991 " - "(base -0.163541, limit -0.176041)", - " globalroute__timing__setup__tns -12.7365 " - "(base -7.5407, limit -9.04884)", - " globalroute__timing__setup__ws -0.275208 " - "(base -0.171167, limit -0.183667)", - " [ERROR] Failing runs: nangate45/gcd/base", - ] - ), + notes, + [ + "QoR: no baseline to compare against (strategy=none). Nothing was checked." + ], + ) + + def test_unknown_metric_line_is_kept_verbatim(self): + metrics, _, _ = genReport.parse_qor_check( + ["QoR regression detected for x/y/z:\n", " odd line\n"] ) + self.assertEqual(metrics, [("odd line",)]) if __name__ == "__main__": diff --git a/flow/util/genReport.py b/flow/util/genReport.py index 52ec6115dd..db1628ec4c 100755 --- a/flow/util/genReport.py +++ b/flow/util/genReport.py @@ -6,6 +6,8 @@ import argparse import os import re +import textwrap +from collections import Counter os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) @@ -16,13 +18,34 @@ SUMMARY_FILENAME = f"{REPORTS_FOLDER}/report-summary.log" DRC_FILENAME = "5_route_drc.rpt" LAST_EXPECTED_LOG = ["6_report.log", "generate_abstract.log"] -METRICS_LOG_FMT = "metadata-generate.log" -METRICS_CHECK_FMT = "{}/metadata-check.log" +METRICS_LOG = "metadata-generate.log" +METRICS_CHECK_LOG = "metadata-check.log" +CALIBRE_LOG = "calibre/save-to-drc-db.log" REGEX_ERROR = re.compile(r"^\[error ?(\w+-\d+)?\]", re.IGNORECASE) REGEX_WARNING = re.compile(r"^\[warning ?(\w+-\d+)?\]", re.IGNORECASE) +REGEX_SIGNAL = re.compile(r"\bsignal (\d+)|\bKilled\b") +# Metric line under "QoR regression detected for ...": +# (base , limit ) +REGEX_QOR_METRIC = re.compile(r"^(\S+) (\S+) \(base (\S+), limit (\S+)\)$") +REGEX_QOR_STATUS = re.compile(r"^Dashboard QoR check: (\w+)") +# checkQorMetrics.py starts its lines with the run identity. +REGEX_RUN_PREFIX = re.compile(r"^[^\s/]+/[^\s/]+/[^\s/:]+: ") SKIPPED_FLOW_VARIANT_KEYWORDS = ["test", "tune"] STATUS_GREEN = "Passing" STATUS_RED = "Failing" +TAIL_LINES = 10 +RULE = "-" * 60 + +# Failure categories, most critical first. The summary sorts by this order. +CATEGORIES = { + "STOP": "flow stopped", + "ERR": "flow error", + "DRC": "Calibre DRC", + "QOR": "QoR regression", +} +CATEGORY_RANK = {code: rank for rank, code in enumerate(CATEGORIES)} +WARN = "WARN" +PASS = "PASS" HELP_TEXT = """ Scans "./logs" and "./reports" folders for errors and warnings. @@ -30,10 +53,10 @@ parser = argparse.ArgumentParser(description=HELP_TEXT) HELP_TEXT = """ -Level 0: report error message id and drc types. +Level 0: report error message ids. Level 1: report full error messages. -Level 2: report full error messages and warning messages id. -Level 3: report full error messages and full warning messages. +Level 2: also report passing designs and warning message ids. +Level 3: also report full warning messages. """ parser.add_argument( "--verbose", "-v", required=False, default=0, action="count", help=HELP_TEXT @@ -60,168 +83,318 @@ args = parser.parse_args() -def parse_messages(filename, print_missing=True): - """ - TODO: docs - """ - errors = list() - warnings = list() +def read_lines(filename): + """Return the lines of a file, or None if it cannot be read.""" try: - with open(filename, "r") as file: - lines = file.readlines() - for line in lines: - if re.search(REGEX_ERROR, line): - errors.append(line.strip()) - elif re.search(REGEX_WARNING, line): - warnings.append(line.strip()) - except BaseException: - if print_missing: - print(f"Failed to open {filename}.") + with open(filename, "r", encoding="utf-8", errors="replace") as file: + return file.readlines() + except OSError: + return None + + +def parse_messages(lines, source): + """Return ([ERROR] lines, [WARNING] lines) as (source, text) pairs.""" + errors = [] + warnings = [] + for line in lines or []: + if REGEX_ERROR.search(line): + errors.append((source, line.strip())) + elif REGEX_WARNING.search(line): + warnings.append((source, line.strip())) return errors, warnings -def parse_qor_regressions(filename): - """Return failed metric details from a QoR check log, if present.""" - regressions = [] +def parse_qor_check(lines): + """ + Read a metadata-check.log written by checkQorMetrics.py or the CI check. + + Returns (metrics, baseline, notes). Each metric is a tuple + (name, current, base, limit), or a 1-tuple with the raw line when the + format is unknown. Notes say why the QoR check did not give a verdict. + """ + metrics = [] + baseline = None + notes = [] in_regressions = False - try: - with open(filename, "r", encoding="utf-8") as file: - for line in file: - if line.startswith("QoR regression detected for "): - in_regressions = True - elif in_regressions and line.startswith(" ") and line.strip(): - regressions.append(line.strip()) - elif in_regressions: - break - except OSError: - pass - return regressions + for line in lines or []: + text = line.strip() + if line.startswith("QoR regression detected for "): + in_regressions = True + continue + if text.startswith("Baseline: "): + baseline = text.split(" ", 1)[1] + continue + if in_regressions and line.startswith(" ") and text: + match = REGEX_QOR_METRIC.match(text) + metrics.append(match.groups() if match else (text,)) + continue + in_regressions = False + status = REGEX_QOR_STATUS.match(text) + if status and status.group(1) not in ("PASS", "FAIL"): + notes.append(f"QoR check {status.group(1)}") + elif text.startswith("Reason: "): + reason = text.split(" ", 1)[1] + if notes and notes[-1].startswith("QoR check "): + notes[-1] += f": {reason}" + else: + notes.append(f"QoR check: {reason}") + elif text.startswith("[WARN] "): + message = REGEX_RUN_PREFIX.sub("", text[len("[WARN] ") :]) + notes.append(f"QoR: {message}") + return metrics, baseline, notes -def append_metrics_errors(errors, regressions, text): - if not errors: - return text +def message_id(text, regex): + """Return the message id (e.g. GPL-0307), or the full text if it has none.""" + match = regex.match(text) + return match.group(1) if match and match.group(1) else text - text += f" Found {len(errors)} metrics failures.\n" - for error in errors: - text += f" {error}\n" - if error.startswith("[ERROR] ") and ": FAIL " in error: - for regression in regressions: - text += f" {regression}\n" - return text +def shorten(text, width=90): + return textwrap.shorten(text, width=width, placeholder=" ...") -def append_text(list_, text, sup, regex, verbose): - """ - TODO - """ - if len(list_) == 0: - return text - text += f" Found {len(list_)} {sup}.\n" - if verbose >= 1: # print full message - for item in list_: - text += f" {item}\n" - else: # only print the id - set_ = set() - long_set = set() - for item in list_: - message = re.match(regex, item).group(1) - if message is None: - message = item - if len(message) > 10: # considering a std msg id XYZ-0000 - long_set.add(message) - else: - set_.add(message) - set_ = sorted(set_, reverse=True) - if set_: - text += f" {', '.join(set_)}\n" - for item in long_set: - text += f" {item}\n" - return text +def collect(log_dir, files, report_dir): + """Read the logs and reports of one design and classify it.""" + d = dict() + d["output_file"] = os.path.join(report_dir, REPORT_FILENAME) + d["log_dir"] = log_dir -def gen_report(name, data): - """ - TODO: docs - """ + d["log_errors"] = [] + d["log_warnings"] = [] + d["last_log"] = "" + for name in sorted(files): + errors, warnings = parse_messages(read_lines(os.path.join(log_dir, name)), name) + d["log_errors"] += errors + d["log_warnings"] += warnings + if name.endswith(".log"): + d["last_log"] = name + d["finished"] = any(name in files for name in LAST_EXPECTED_LOG) - if args.verbose >= 2 or data["status"] != STATUS_GREEN: - output = f"{name}\n" - if data["finished"]: - output += " Flow reached last stage.\n" - else: - output += " Flow did not reach last stage.\n" - if data["last_log"]: - output += f" Last log file {data['last_log']}\n" - else: - output += " No log file found.\n" - else: - output = "" + missing = [] - output = append_text( - data["log_errors"], output, "errors in the logs", REGEX_ERROR, args.verbose - ) - output = append_text( - data["metrics_logs_errors"], - output, - "errors in the metrics logs", - REGEX_ERROR, - args.verbose, - ) - output = append_metrics_errors( - data["metrics_errors"], data["metrics_regressions"], output - ) - output = append_text( - data["calibre_errors"], output, "calibre failures", REGEX_ERROR, args.verbose + lines = read_lines(os.path.join(report_dir, METRICS_LOG)) + if lines is None and d["finished"]: + missing.append(METRICS_LOG) + d["metrics_logs_errors"], d["metrics_logs_warnings"] = parse_messages( + lines, METRICS_LOG ) - if args.verbose >= 2: - output = append_text( - data["log_warnings"], - output, - "warnings in the logs", - REGEX_WARNING, - args.verbose - 2, - ) - output = append_text( - data["metrics_logs_warnings"], - output, - "warnings in the metrics logs", - REGEX_WARNING, - args.verbose - 2, - ) - output = append_text( - data["metrics_warnings"], - output, - "metrics warnings", - REGEX_WARNING, - args.verbose - 2, - ) - output = append_text( - data["calibre_warnings"], - output, - "calibre warnings", - REGEX_WARNING, - args.verbose - 2, - ) + lines = read_lines(os.path.join(report_dir, METRICS_CHECK_LOG)) + if lines is None and d["finished"]: + missing.append(METRICS_CHECK_LOG) + errors, _ = parse_messages(lines, METRICS_CHECK_LOG) + # checkQorMetrics.py repeats the failing runs in a trailing line. + d["metrics_errors"] = [ + e for e in errors if not e[1].startswith("[ERROR] Failing runs:") + ] + d["qor_metrics"], d["qor_baseline"], qor_notes = parse_qor_check(lines) - if len(data["drcs"].keys()) > 0: - if data["status"] == STATUS_GREEN: - output += " Design has the violations under the allowed limit: " + d["calibre_errors"], d["calibre_warnings"] = parse_messages( + read_lines(os.path.join(log_dir, CALIBRE_LOG)), CALIBRE_LOG + ) + + d["drcs"] = Counter() + lines = read_lines(os.path.join(report_dir, DRC_FILENAME)) + if lines is None and d["finished"]: + missing.append(DRC_FILENAME) + for line in lines or []: + if "violation type:" in line: + d["drcs"][line.split("violation type:", 1)[1].strip()] += 1 + + d["tail"] = [] + d["cause"] = "" + if not d["finished"] and d["last_log"]: + tail = read_lines(os.path.join(log_dir, d["last_log"])) or [] + d["tail"] = [line.rstrip() for line in tail[-TAIL_LINES:]] + for line in reversed(d["tail"]): + match = REGEX_SIGNAL.search(line) + if match: + signal = match.group(1) or "9" + d["cause"] = f"killed by signal {signal}" + if signal == "9": + d["cause"] += " (out of memory?)" + break + + if not d["finished"]: + d["category"] = "STOP" + elif d["log_errors"]: + d["category"] = "ERR" + elif d["calibre_errors"]: + d["category"] = "DRC" + elif d["metrics_errors"]: + d["category"] = "QOR" + else: + d["category"] = None + d["status"] = STATUS_RED if d["category"] else STATUS_GREEN + + notes = list(qor_notes) + if missing: + notes.append("missing " + ", ".join(missing)) + if d["status"] == STATUS_GREEN: + if d["metrics_logs_errors"]: + notes.append(f"{len(d['metrics_logs_errors'])} error(s) in {METRICS_LOG}") + if d["drcs"]: + notes.append(f"router DRC under the limit: {format_drcs(d['drcs'])}") + d["notes"] = notes + return d + + +def format_drcs(drcs): + return ", ".join(f"{name} ({count})" for name, count in drcs.items()) + + +def qor_names(data): + return [metric[0] for metric in data["qor_metrics"] if len(metric) == 4] + + +def row_code(data): + if data["category"]: + return data["category"] + return WARN if data["notes"] else PASS + + +def sort_key(item): + """Most critical first; more regressed metrics first within QoR.""" + name, data = item + code = row_code(data) + rank = CATEGORY_RANK.get(code, len(CATEGORY_RANK) + (code == PASS)) + return (rank, -len(data["qor_metrics"]), name) + + +def first_error(errors): + """Short form of the last error: the one most likely to stop the flow.""" + source, text = errors[-1] + more = f" (+{len(errors) - 1} more)" if len(errors) > 1 else "" + return f"{source}: {message_id(text, REGEX_ERROR)}{more}" + + +def row_reason(data): + """One-line reason for the summary table.""" + code = row_code(data) + if code == "STOP": + if data["log_errors"]: + return first_error(data["log_errors"]) + if not data["last_log"]: + return "no log file found" + return f"{data['last_log']}: {data['cause'] or 'no error message'}" + if code == "ERR": + return first_error(data["log_errors"]) + if code == "DRC": + return f"Calibre: {len(data['calibre_errors'])} error(s)" + if code == "QOR": + names = qor_names(data) + if not names: + return message_id(data["metrics_errors"][0][1], REGEX_ERROR) + if len(names) == 1: + return names[0] + return f"{len(names)} metrics: " + ", ".join(names) + if code == WARN: + more = len(data["notes"]) - 1 + return data["notes"][0] + (f" (+{more} more)" if more else "") + return "passed" + + +def headline(data): + """Short description after the design name in the detail block.""" + code = data["category"] + if code == "STOP": + if not data["last_log"]: + return "flow did not reach last stage, no log file found" + text = f"stopped in {data['last_log']}" + return f"{text}, {data['cause']}" if data["cause"] else text + if code == "ERR": + return f"{len(data['log_errors'])} error(s) in the logs" + if code == "DRC": + return f"{len(data['calibre_errors'])} Calibre error(s)" + if code == "QOR": + count = len(data["qor_metrics"]) + if count == 0: + return "QoR check failed" + return f"{count} metric{'s' if count > 1 else ''} over the limit" + return "passed" + + +def message_lines(messages, regex, full): + """ + Lines for a list of (source, text) messages. + Full mode shows each distinct message with a repeat count. + Otherwise one line per source with the distinct message ids. + """ + lines = [] + if full: + for (source, text), count in Counter(messages).items(): + repeat = f" (x{count})" if count > 1 else "" + lines.append(f"{source}: {text}{repeat}") + return lines + by_source = {} + for source, text in messages: + by_source.setdefault(source, {})[message_id(text, regex)] = None + for source, ids in by_source.items(): + lines.append(f"{source}: {', '.join(ids)}") + return lines + + +def qor_table(metrics): + rows = [("metric", "current", "base", "limit")] + rows += [m for m in metrics if len(m) == 4] + lines = [] + if len(rows) > 1: + width = max(len(row[0]) for row in rows) + for row in rows: + lines.append( + f"{row[0]:<{width}} {row[1]:<12} {row[2]:<12} {row[3]}".rstrip() + ) + lines += [m[0] for m in metrics if len(m) == 1] + return lines + + +def gen_report(name, data, shared_baseline=None): + """Detail block for one design. Empty for a passing design without notes.""" + verbose = args.verbose + if data["status"] == STATUS_GREEN and not data["notes"] and verbose < 2: + return "" + + body = [] + if data["category"] == "STOP" and data["tail"] and not data["log_errors"]: + body.append(f"Last {len(data['tail'])} lines of {data['last_log']}:") + body += [f" {line}" for line in data["tail"]] + full = verbose >= 1 + body += message_lines(data["log_errors"], REGEX_ERROR, full) + body += message_lines(data["metrics_logs_errors"], REGEX_ERROR, full) + body += message_lines(data["calibre_errors"], REGEX_ERROR, full) + if data["metrics_errors"]: + if data["qor_metrics"]: + body += qor_table(data["qor_metrics"]) else: - output += " Design has the violations over the allowed limit: " - for drc, count in data["drcs"].items(): - output += f"{drc} ({count}) " - output += "\n" + body += [text for _, text in data["metrics_errors"]] + if data["qor_baseline"] and data["qor_baseline"] != shared_baseline: + body.append(f"Baseline: {data['qor_baseline']}") + if data["drcs"] and data["status"] == STATUS_RED: + body.append(f"Router DRC: {format_drcs(data['drcs'])}") + if data["status"] == STATUS_RED: + body += [f"Note: {note}" for note in data["notes"]] + if verbose >= 2: + warnings = ( + data["log_warnings"] + + data["metrics_logs_warnings"] + + data["calibre_warnings"] + ) + body += message_lines(warnings, REGEX_WARNING, verbose >= 3) - return output.strip() + if data["status"] == STATUS_GREEN: + lines = [f"{name}: {note}" for note in data["notes"]] or [f"{name}: passed"] + else: + lines = [f"{name}: {headline(data)}"] + lines += [f" {line}" for line in body] + return "\n".join(lines) def write_report(path, content): - """ - TODO: docs - """ + """Write one design report and append it to the combined report.""" content = content.strip() + # A flow that stops before its first stage has no reports directory. + os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w") as file: file.write(content) if args.verbose: @@ -233,41 +406,94 @@ def write_report(path, content): print(content) -def get_summary(status, text): - """ - TODO: docs - """ - tmp = "" - for name, data in design_list.items(): - if data["status"] == status: - content = gen_report(name, data) - if content != "": - tmp += content + "\n" - # add empty line for readability - if args.verbose >= 2 or status == STATUS_RED: - tmp += "\n" - if tmp != "": - text += "-" * 30 + "\n" - text += " " * 7 + f"{status} designs\n" - text += "-" * 30 + "\n" - text += tmp - - return text +def section(title): + return f"{RULE}\n{title}\n{RULE}\n" -def write_summary(): - """ - Write summary and individual report files with data on design_list - """ - summary = "\n" - summary += "=" * 60 + "\n" - summary += " " * 26 + "SUMMARY\n" - summary += "=" * 60 + "\n" - summary += f"\nNumber of designs: {len(design_list.keys())}\n\n" +def summary_header(designs): + failing = [d for d in designs.values() if d["status"] == STATUS_RED] + lines = ["=" * 60] + lines.append( + f"SUMMARY: {len(designs)} design{'s' if len(designs) != 1 else ''}, {len(failing)} failing, " + f"{len(designs) - len(failing)} passing" + ) + if failing: + counts = Counter(d["category"] for d in failing) + lines.append( + " " + + " | ".join( + f"{label} {counts[code]}" for code, label in CATEGORIES.items() + ) + ) + regressed = Counter(name for d in failing for name in qor_names(d)) + common = [(name, n) for name, n in regressed.most_common(3) if n > 1] + if common: + lines.append( + " Most regressed: " + ", ".join(f"{name} ({n})" for name, n in common) + ) + baselines = {d["qor_baseline"] for d in failing if d["metrics_errors"]} + shared_baseline = baselines.pop() if len(baselines) == 1 else None + if shared_baseline: + lines.append(f" QoR baseline: {shared_baseline}") + lines.append("=" * 60) + return "\n".join(lines) + "\n", shared_baseline + + +def summary_table(rows): + if not rows: + return "" + width = max(len(name) for name, _ in rows) + lines = [ + f"{row_code(data):<4} {name:<{width}} {shorten(row_reason(data))}" + for name, data in rows + ] + return "\n".join(lines) + "\n" + + +def attention_section(rows): + """Notes of passing designs; a note shared by several designs prints once.""" + by_note = {} + for name, data in rows: + for note in data["notes"]: + by_note.setdefault(note, []).append(name) + lines = [] + for note, names in by_note.items(): + if len(names) == 1: + lines.append(f"{names[0]}: {note}") + else: + lines.append(f"{note} ({len(names)} designs):") + wrapped = textwrap.wrap(", ".join(names), width=76, break_on_hyphens=False) + lines += [f" {line}" for line in wrapped] + return "\n".join(lines) + "\n" - summary = get_summary(STATUS_RED, summary) - summary += "\n" - summary = get_summary(STATUS_GREEN, summary) + +def write_summary(): + """Write the summary: header, one row per design, then details.""" + rows = sorted(design_list.items(), key=sort_key) + header, shared_baseline = summary_header(design_list) + listed = [ + (name, data) + for name, data in rows + if row_code(data) != PASS or args.verbose >= 2 + ] + + summary = "\n" + header + "\n" + summary_table(listed) + for code, label in CATEGORIES.items(): + group = [(n, d) for n, d in rows if d["category"] == code] + if group: + summary += "\n" + section(f"{label.upper()} ({len(group)})") + summary += "\n\n".join(gen_report(n, d, shared_baseline) for n, d in group) + summary += "\n" + noted = [(n, d) for n, d in rows if row_code(d) == WARN] + if noted: + summary += "\n" + section("NEEDS ATTENTION (not failing)") + summary += attention_section(noted) + if args.verbose >= 2: + passing = [(n, d) for n, d in rows if d["status"] == STATUS_GREEN] + details = [gen_report(n, d) for n, d in passing] + if passing: + summary += "\n" + section(f"PASSING ({len(passing)})") + summary += "\n".join(details) + "\n" if not args.quiet: if os.path.isfile(SUMMARY_FILENAME): @@ -281,95 +507,30 @@ def write_summary(): design_list = dict() -for log_dir, dirs, files in sorted(os.walk(LOGS_FOLDER, topdown=False)): - dir_list = log_dir.split(os.sep) - # Handles autotuner folders, which do not have `report.log` natively. - # TODO: Can we log something for autotuner? - if len(dir_list) != 4 or any( - word in dir_list[-1] for word in SKIPPED_FLOW_VARIANT_KEYWORDS - ): - continue - report_dir = log_dir.replace(LOGS_FOLDER, REPORTS_FOLDER) - - # basic info about current design - platform, design, variant = dir_list[1:] - d = dict() - d["output_file"] = os.path.join(report_dir, REPORT_FILENAME) - - # check if design ran to completion without errors or warnings - d["log_errors"] = list() - d["log_warnings"] = list() - d["last_log"] = "" - for name_ in sorted(files): - temp_e, temp_w = parse_messages(os.path.join(log_dir, name_)) - d["log_errors"] += temp_e - d["log_warnings"] += temp_w - if name_.endswith(".log"): - d["last_log"] = name_ - d["finished"] = any(name in files for name in LAST_EXPECTED_LOG) - - # check if metrics generation had issues - d["metrics_logs_errors"], d["metrics_logs_warnings"] = parse_messages( - os.path.join(report_dir, METRICS_LOG_FMT), - print_missing=d["finished"], - ) - - # check if metrics passed - d["metrics_errors"], d["metrics_warnings"] = parse_messages( - METRICS_CHECK_FMT.format(report_dir), print_missing=d["finished"] - ) - d["metrics_regressions"] = parse_qor_regressions( - METRICS_CHECK_FMT.format(report_dir) - ) +if __name__ == "__main__": + for log_dir, dirs, files in sorted(os.walk(LOGS_FOLDER, topdown=False)): + dir_list = log_dir.split(os.sep) + # Handles autotuner folders, which do not have `report.log` natively. + # TODO: Can we log something for autotuner? + if len(dir_list) != 4 or any( + word in dir_list[-1] for word in SKIPPED_FLOW_VARIANT_KEYWORDS + ): + continue + platform, design, variant = dir_list[1:] + report_dir = log_dir.replace(LOGS_FOLDER, REPORTS_FOLDER, 1) + design_list[f"{platform}/{design}/{variant}"] = collect( + log_dir, files, report_dir + ) - # check if calibre was run and if drc check passed - d["calibre_errors"], d["calibre_warnings"] = parse_messages( - os.path.join(log_dir, "calibre/save-to-drc-db.log"), print_missing=False - ) + if not os.path.isdir(REPORTS_FOLDER): + os.mkdir(REPORTS_FOLDER) - # check if there were drc violations - d["drcs"] = dict() - try: - drc_report_file = os.path.join(report_dir, DRC_FILENAME) - with open(drc_report_file, "r") as file_: - for line_ in file_.readlines(): - if "violation type:" in line_: - type_ = line_.split("violation type:", 1)[1].strip() - if type_ in d["drcs"].keys(): - d["drcs"][type_] += 1 - else: - d["drcs"][type_] = 1 - except BaseException: - if d["finished"]: - print(f"Failed to open {DRC_FILENAME}.") - - if d["log_errors"] or d["metrics_errors"] or d["calibre_errors"]: - d["status"] = STATUS_RED + if args.summary: + write_summary() else: - if not d["finished"]: - d["status"] = STATUS_RED - last_lines = "Could not find last lines of the log file." - if d["last_log"]: - with open(os.path.join(log_dir, d["last_log"]), "r") as last_file: - last_lines = last_file.readlines()[-10:] - last_lines = " ".join(last_lines) - message = "[ERROR CI-0000] No error message found. " - message += "Here are the last 10 lines of the log.\n" - d["log_errors"] = [message + last_lines] - else: - d["status"] = STATUS_GREEN - - design_list[f"{platform} {design} ({variant})"] = d - -if not os.path.isdir(REPORTS_FOLDER): - os.mkdir(REPORTS_FOLDER) - -if args.summary: - write_summary() -else: - if os.path.isfile(SINGLE_REPORT_FILENAME): - if not args.quiet: - print(f"Overwriting report {SINGLE_REPORT_FILENAME}.") - os.remove(SINGLE_REPORT_FILENAME) - for name_, data_ in design_list.items(): - write_report(data_["output_file"], gen_report(name_, data_)) + if os.path.isfile(SINGLE_REPORT_FILENAME): + if not args.quiet: + print(f"Overwriting report {SINGLE_REPORT_FILENAME}.") + os.remove(SINGLE_REPORT_FILENAME) + for name_, data_ in design_list.items(): + write_report(data_["output_file"], gen_report(name_, data_)) From 17783411d1481109cc8876bfe8fa898dded9dafb Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Wed, 4 Nov 2026 11:53:28 -0300 Subject: [PATCH 2/3] fix(genReport): correct report details found in review At -vv, a design with notes printed twice. Its notes now show only under "Needs attention". A "Reason:" line after a PASS or FAIL verdict created a note. It is now kept only for a check without a verdict. The metric count changed the order of non-QoR rows. It now orders only QoR rows. At -v, errors in metadata-generate.log for a passing design gave only a count. The summary now shows the error text. Tests now cover each verbose level, the per-design reports, the local QoR check format, and different baselines. Signed-off-by: Vitor Bandeira --- flow/test/test_genReport.py | 156 +++++++++++++++++++++++++++++++++++- flow/util/genReport.py | 41 +++++----- 2 files changed, 176 insertions(+), 21 deletions(-) diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index 0d49110f03..6061f96320 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -74,16 +74,31 @@ def run_helper(self): text=True, ) - def summary(self): + def run_report(self, *flags): result = subprocess.run( - [sys.executable, os.path.join(self.tmp, "util/genReport.py"), "-sv"], + [sys.executable, os.path.join(self.tmp, "util/genReport.py"), *flags], capture_output=True, text=True, ) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - with open(os.path.join(self.tmp, "reports/report-summary.log")) as f: + return result.stdout + + def read(self, path): + with open(os.path.join(self.tmp, path), encoding="utf-8") as f: return f.read() + def summary(self, flags="-sv"): + self.run_report(flags) + return self.read("reports/report-summary.log") + + def qor_fail(self, pdv, metrics, baseline="https://dashboard.example/builds/7"): + """A finished run with the CI QoR check verdict FAIL.""" + lines = [f"[ERROR] Dashboard QoR check: FAIL ({pdv})"] + lines.append(f"QoR regression detected for {pdv}:") + lines += [f" {m} 2 (base 1, limit 1.5)" for m in metrics] + lines.append(f" Baseline: {baseline}") + self.finished(pdv, "\n".join(lines) + "\n") + def finished(self, pdv, check=""): """A run that reached the last stage, with its metadata and DRC reports.""" self.write_file(f"logs/{pdv}/6_report.log", "Finished\n") @@ -219,6 +234,130 @@ def test_missing_metadata_is_grouped_under_needs_attention(self): summary, ) + def verbosity_fixture(self): + self.write_file( + "logs/asap7/stop/base/3_3_place_gp.log", + "[WARNING GPL-0302] Use a larger density\n" + "[ERROR GPL-0307] RePlAce divergence\n" + "[ERROR GPL-0307] RePlAce divergence\n", + ) + self.finished("asap7/pass/base") + self.write_file( + "logs/asap7/pass/base/2_1_floorplan.log", "[WARNING IFP-0028] Core area\n" + ) + self.finished( + "asap7/warn/base", + "Dashboard QoR check: INCONCLUSIVE (asap7/warn/base)\nReason: no baseline\n", + ) + + def test_level_0_shows_error_ids_only(self): + self.verbosity_fixture() + summary = self.summary("-s") + self.assertIn(" 3_3_place_gp.log: GPL-0307\n", summary) + self.assertNotIn("RePlAce divergence", summary) + self.assertNotIn("asap7/pass/base", summary) + + def test_level_1_shows_each_full_error_once_with_a_count(self): + self.verbosity_fixture() + summary = self.summary("-sv") + self.assertIn( + " 3_3_place_gp.log: [ERROR GPL-0307] RePlAce divergence (x2)\n", summary + ) + self.assertNotIn("GPL-0302", summary) + + def test_level_2_adds_passing_designs_and_warning_ids(self): + self.verbosity_fixture() + summary = self.summary("-svv") + self.assertIn("PASS asap7/pass/base", summary) + passing = summary.split("PASSING (2)")[1] + self.assertIn( + "asap7/pass/base: passed\n 2_1_floorplan.log: IFP-0028", passing + ) + # The note of a WARN design shows under "Needs attention" only. + self.assertIn("asap7/warn/base: passed\n", passing) + self.assertNotIn("no baseline", passing) + self.assertIn(" 3_3_place_gp.log: GPL-0302", summary) + + def test_level_3_shows_full_warnings(self): + self.verbosity_fixture() + summary = self.summary("-svvv") + self.assertIn("2_1_floorplan.log: [WARNING IFP-0028] Core area", summary) + + def test_report_mode_writes_each_design_report(self): + self.verbosity_fixture() + # A run that stopped early may have no reports directory yet. + self.assertFalse(os.path.isdir(os.path.join(self.tmp, "reports/asap7/stop"))) + output = self.run_report("-v") + self.assertIn("asap7/stop/base: stopped in 3_3_place_gp.log", output) + self.assertIn( + "[ERROR GPL-0307] RePlAce divergence (x2)", + self.read("reports/asap7/stop/base/report.log"), + ) + self.assertIn( + "asap7/warn/base: QoR check INCONCLUSIVE: no baseline", + self.read("reports/asap7/warn/base/report.log"), + ) + self.assertIn("asap7/stop/base", self.read("reports/report.log")) + + def test_local_qor_check_format(self): + self.finished( + "nangate45/gcd/base", + "[ERROR] nangate45/gcd/base: FAIL \u2014 19 passed, 2 failed, 0 missing " + "(478 metrics sent)\n" + "[INFO] baseline build 7 (abc, latest)\n" + "Metric Baseline Current Limit Delta%\n" + "QoR regression detected for nangate45/gcd:\n" + " cts__timing__setup__tns -12.2456 (base -6.95522, limit -8.346264)\n" + " finish__timing__setup__ws -0.27 (base -0.16, limit -0.17)\n" + "[WARN] Rules not evaluated for 3 metric(s) absent from the baseline " + "build. Use --verbose to list them.\n" + "[ERROR] Failing runs: nangate45/gcd/base\n" + "QoR check: FAIL\n", + ) + summary = self.summary() + self.assertIn("nangate45/gcd/base: 2 metrics over the limit", summary) + self.assertRegex(summary, r"cts__timing__setup__tns +-12\.2456 +-6\.95522") + self.assertIn("Note: QoR: Rules not evaluated for 3 metric(s)", summary) + self.assertNotIn("Failing runs", summary) + + def test_each_design_shows_its_baseline_when_they_differ(self): + self.qor_fail("asap7/a/base", ["m1"], "https://dashboard.example/builds/1") + self.qor_fail("asap7/b/base", ["m1"], "https://dashboard.example/builds/2") + summary = self.summary() + self.assertNotIn("QoR baseline:", summary) + self.assertIn(" Baseline: https://dashboard.example/builds/1", summary) + self.assertIn(" Baseline: https://dashboard.example/builds/2", summary) + self.assertIn("Most regressed: m1 (2)", summary) + + def test_metric_count_orders_only_qor_rows(self): + for name in ("a", "b"): + self.write_file(f"logs/asap7/{name}/base/1_1_yosys.log", "ok\n") + self.write_file( + "reports/asap7/b/base/metadata-check.log", + "QoR regression detected for asap7/b/base:\n m1 2 (base 1, limit 1.5)\n", + ) + table = self.summary().split("-" * 60)[0] + self.assertLess( + table.index("STOP asap7/a/base"), table.index("STOP asap7/b/base") + ) + + def test_metadata_generation_error_text_reaches_the_summary(self): + self.finished("asap7/a/base") + self.write_file( + "reports/asap7/a/base/metadata-generate.log", "[ERROR] bad metric\n" + ) + summary = self.summary() + self.assertIn( + "asap7/a/base: metadata-generate.log: [ERROR] bad metric\n", summary + ) + + def test_log_that_is_not_utf8_is_read(self): + path = os.path.join(self.tmp, "logs/asap7/a/base/1_1_yosys.log") + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as f: + f.write(b"\xff\xfe \xc2\xb5\n[ERROR SYN-0001] bad input\n") + self.assertIn("1_1_yosys.log: SYN-0001", self.summary()) + class TestParseQorCheck(unittest.TestCase): def test_ci_format(self): @@ -251,6 +390,17 @@ def test_local_check_warnings_become_notes(self): ], ) + def test_reason_is_kept_only_for_a_check_without_a_verdict(self): + for status in ("PASS", "FAIL"): + _, _, notes = genReport.parse_qor_check( + [f"Dashboard QoR check: {status} (a/b/c)\n", "Reason: text\n"] + ) + self.assertEqual(notes, [], status) + _, _, notes = genReport.parse_qor_check( + ["Dashboard QoR check: INCONCLUSIVE (a/b/c)\n", "Reason: no data\n"] + ) + self.assertEqual(notes, ["QoR check INCONCLUSIVE: no data"]) + def test_unknown_metric_line_is_kept_verbatim(self): metrics, _, _ = genReport.parse_qor_check( ["QoR regression detected for x/y/z:\n", " odd line\n"] diff --git a/flow/util/genReport.py b/flow/util/genReport.py index db1628ec4c..9107389d6f 100755 --- a/flow/util/genReport.py +++ b/flow/util/genReport.py @@ -116,6 +116,7 @@ def parse_qor_check(lines): baseline = None notes = [] in_regressions = False + no_verdict = False for line in lines or []: text = line.strip() if line.startswith("QoR regression detected for "): @@ -130,14 +131,12 @@ def parse_qor_check(lines): continue in_regressions = False status = REGEX_QOR_STATUS.match(text) - if status and status.group(1) not in ("PASS", "FAIL"): - notes.append(f"QoR check {status.group(1)}") - elif text.startswith("Reason: "): - reason = text.split(" ", 1)[1] - if notes and notes[-1].startswith("QoR check "): - notes[-1] += f": {reason}" - else: - notes.append(f"QoR check: {reason}") + if status: + no_verdict = status.group(1) not in ("PASS", "FAIL") + if no_verdict: + notes.append(f"QoR check {status.group(1)}") + elif text.startswith("Reason: ") and no_verdict: + notes[-1] += f": {text.split(' ', 1)[1]}" elif text.startswith("[WARN] "): message = REGEX_RUN_PREFIX.sub("", text[len("[WARN] ") :]) notes.append(f"QoR: {message}") @@ -232,8 +231,7 @@ def collect(log_dir, files, report_dir): if missing: notes.append("missing " + ", ".join(missing)) if d["status"] == STATUS_GREEN: - if d["metrics_logs_errors"]: - notes.append(f"{len(d['metrics_logs_errors'])} error(s) in {METRICS_LOG}") + notes += [f"{source}: {text}" for source, text in d["metrics_logs_errors"]] if d["drcs"]: notes.append(f"router DRC under the limit: {format_drcs(d['drcs'])}") d["notes"] = notes @@ -259,10 +257,11 @@ def sort_key(item): name, data = item code = row_code(data) rank = CATEGORY_RANK.get(code, len(CATEGORY_RANK) + (code == PASS)) - return (rank, -len(data["qor_metrics"]), name) + metrics = len(data["qor_metrics"]) if code == "QOR" else 0 + return (rank, -metrics, name) -def first_error(errors): +def last_error(errors): """Short form of the last error: the one most likely to stop the flow.""" source, text = errors[-1] more = f" (+{len(errors) - 1} more)" if len(errors) > 1 else "" @@ -274,12 +273,12 @@ def row_reason(data): code = row_code(data) if code == "STOP": if data["log_errors"]: - return first_error(data["log_errors"]) + return last_error(data["log_errors"]) if not data["last_log"]: return "no log file found" return f"{data['last_log']}: {data['cause'] or 'no error message'}" if code == "ERR": - return first_error(data["log_errors"]) + return last_error(data["log_errors"]) if code == "DRC": return f"Calibre: {len(data['calibre_errors'])} error(s)" if code == "QOR": @@ -349,8 +348,12 @@ def qor_table(metrics): return lines -def gen_report(name, data, shared_baseline=None): - """Detail block for one design. Empty for a passing design without notes.""" +def gen_report(name, data, shared_baseline=None, with_notes=True): + """ + Detail block for one design. Empty for a passing design without notes. + with_notes=False leaves out the notes of a passing design, for a summary + that already shows them under "Needs attention". + """ verbose = args.verbose if data["status"] == STATUS_GREEN and not data["notes"] and verbose < 2: return "" @@ -383,7 +386,8 @@ def gen_report(name, data, shared_baseline=None): body += message_lines(warnings, REGEX_WARNING, verbose >= 3) if data["status"] == STATUS_GREEN: - lines = [f"{name}: {note}" for note in data["notes"]] or [f"{name}: passed"] + notes = data["notes"] if with_notes else [] + lines = [f"{name}: {note}" for note in notes] or [f"{name}: passed"] else: lines = [f"{name}: {headline(data)}"] lines += [f" {line}" for line in body] @@ -426,6 +430,7 @@ def summary_header(designs): ) ) regressed = Counter(name for d in failing for name in qor_names(d)) + # A metric in one design only repeats its table row, so list shared ones. common = [(name, n) for name, n in regressed.most_common(3) if n > 1] if common: lines.append( @@ -490,7 +495,7 @@ def write_summary(): summary += attention_section(noted) if args.verbose >= 2: passing = [(n, d) for n, d in rows if d["status"] == STATUS_GREEN] - details = [gen_report(n, d) for n, d in passing] + details = [gen_report(n, d, with_notes=False) for n, d in passing] if passing: summary += "\n" + section(f"PASSING ({len(passing)})") summary += "\n".join(details) + "\n" From d2f98b1df12b5ae13fe44304dbbcd1219dc197f4 Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Wed, 4 Nov 2026 11:55:54 -0300 Subject: [PATCH 3/3] fix(genReport): keep the QoR verdict rules and avoid repeats The script dropped the "Failing runs" line before it set the design status. A design with only that [ERROR] line then passed. The script now hides that line only in the text, so the old status rule applies. The CI writes FAIL as "[ERROR] Dashboard QoR check: FAIL". The parser did not match that prefix, so a later "Reason:" line could join an earlier INCONCLUSIVE note. The script printed the metadata-generate errors of a passing design two times: once as a note and once in its details. They now print once. Signed-off-by: Vitor Bandeira --- flow/test/test_genReport.py | 43 +++++++++++++++++++++++++++++++------ flow/util/genReport.py | 43 +++++++++++++++++++++++-------------- 2 files changed, 64 insertions(+), 22 deletions(-) diff --git a/flow/test/test_genReport.py b/flow/test/test_genReport.py index 6061f96320..d82f9eaa98 100644 --- a/flow/test/test_genReport.py +++ b/flow/test/test_genReport.py @@ -274,7 +274,7 @@ def test_level_2_adds_passing_designs_and_warning_ids(self): "asap7/pass/base: passed\n 2_1_floorplan.log: IFP-0028", passing ) # The note of a WARN design shows under "Needs attention" only. - self.assertIn("asap7/warn/base: passed\n", passing) + self.assertIn("asap7/warn/base: passed, see NEEDS ATTENTION\n", passing) self.assertNotIn("no baseline", passing) self.assertIn(" 3_3_place_gp.log: GPL-0302", summary) @@ -351,6 +351,27 @@ def test_metadata_generation_error_text_reaches_the_summary(self): "asap7/a/base: metadata-generate.log: [ERROR] bad metric\n", summary ) + def test_failing_runs_line_alone_still_fails_the_design(self): + self.finished( + "nangate45/gcd/base", + "[INFO] nangate45/gcd/base: FAIL\n" + "[ERROR] Failing runs: nangate45/gcd/base\n", + ) + summary = self.summary() + self.assertIn("QOR nangate45/gcd/base [ERROR] Failing runs:", summary) + + def test_metadata_generation_error_of_a_passing_design_prints_once(self): + self.finished("asap7/a/base") + self.write_file( + "reports/asap7/a/base/metadata-generate.log", "[ERROR] bad metric\n" + ) + self.run_report("-v") + report = self.read("reports/asap7/a/base/report.log") + self.assertEqual(report.count("bad metric"), 1, report) + self.assertEqual( + self.summary("-svv").split("PASSING")[1].count("bad metric"), 0 + ) + def test_log_that_is_not_utf8_is_read(self): path = os.path.join(self.tmp, "logs/asap7/a/base/1_1_yosys.log") os.makedirs(os.path.dirname(path)) @@ -391,15 +412,25 @@ def test_local_check_warnings_become_notes(self): ) def test_reason_is_kept_only_for_a_check_without_a_verdict(self): - for status in ("PASS", "FAIL"): + # The CI check writes these lines; only FAIL has the [ERROR] prefix. + for verdict in ( + "Dashboard QoR check: PASS", + "[ERROR] Dashboard QoR check: FAIL", + ): _, _, notes = genReport.parse_qor_check( - [f"Dashboard QoR check: {status} (a/b/c)\n", "Reason: text\n"] + [f"{verdict} (a/b/c)\n", "Reason: text\n"] ) - self.assertEqual(notes, [], status) + self.assertEqual(notes, [], verdict) _, _, notes = genReport.parse_qor_check( - ["Dashboard QoR check: INCONCLUSIVE (a/b/c)\n", "Reason: no data\n"] + [ + "Dashboard QoR check: INCONCLUSIVE (a/b/c)\n", + "[WARN] a/b/c: other\n", + "Reason: no data\n", + "[ERROR] Dashboard QoR check: FAIL (a/b/d)\n", + "Reason: ignored\n", + ] ) - self.assertEqual(notes, ["QoR check INCONCLUSIVE: no data"]) + self.assertEqual(notes, ["QoR check INCONCLUSIVE: no data", "QoR: other"]) def test_unknown_metric_line_is_kept_verbatim(self): metrics, _, _ = genReport.parse_qor_check( diff --git a/flow/util/genReport.py b/flow/util/genReport.py index 9107389d6f..3caac0a2bb 100755 --- a/flow/util/genReport.py +++ b/flow/util/genReport.py @@ -27,7 +27,7 @@ # Metric line under "QoR regression detected for ...": # (base , limit ) REGEX_QOR_METRIC = re.compile(r"^(\S+) (\S+) \(base (\S+), limit (\S+)\)$") -REGEX_QOR_STATUS = re.compile(r"^Dashboard QoR check: (\w+)") +REGEX_QOR_STATUS = re.compile(r"^(?:\[ERROR\] )?Dashboard QoR check: (\w+)") # checkQorMetrics.py starts its lines with the run identity. REGEX_RUN_PREFIX = re.compile(r"^[^\s/]+/[^\s/]+/[^\s/:]+: ") SKIPPED_FLOW_VARIANT_KEYWORDS = ["test", "tune"] @@ -116,7 +116,7 @@ def parse_qor_check(lines): baseline = None notes = [] in_regressions = False - no_verdict = False + status_note = None # index of the note for a check without a verdict for line in lines or []: text = line.strip() if line.startswith("QoR regression detected for "): @@ -132,11 +132,12 @@ def parse_qor_check(lines): in_regressions = False status = REGEX_QOR_STATUS.match(text) if status: - no_verdict = status.group(1) not in ("PASS", "FAIL") - if no_verdict: + status_note = None + if status.group(1) not in ("PASS", "FAIL"): + status_note = len(notes) notes.append(f"QoR check {status.group(1)}") - elif text.startswith("Reason: ") and no_verdict: - notes[-1] += f": {text.split(' ', 1)[1]}" + elif text.startswith("Reason: ") and status_note is not None: + notes[status_note] += f": {text.split(' ', 1)[1]}" elif text.startswith("[WARN] "): message = REGEX_RUN_PREFIX.sub("", text[len("[WARN] ") :]) notes.append(f"QoR: {message}") @@ -182,11 +183,7 @@ def collect(log_dir, files, report_dir): lines = read_lines(os.path.join(report_dir, METRICS_CHECK_LOG)) if lines is None and d["finished"]: missing.append(METRICS_CHECK_LOG) - errors, _ = parse_messages(lines, METRICS_CHECK_LOG) - # checkQorMetrics.py repeats the failing runs in a trailing line. - d["metrics_errors"] = [ - e for e in errors if not e[1].startswith("[ERROR] Failing runs:") - ] + d["metrics_errors"], _ = parse_messages(lines, METRICS_CHECK_LOG) d["qor_metrics"], d["qor_baseline"], qor_notes = parse_qor_check(lines) d["calibre_errors"], d["calibre_warnings"] = parse_messages( @@ -238,6 +235,16 @@ def collect(log_dir, files, report_dir): return d +def qor_error_lines(data): + """ + QoR check [ERROR] lines to show. checkQorMetrics.py repeats the failing + runs in a trailing line, so hide it unless it is the only one. + """ + texts = [text for _, text in data["metrics_errors"]] + shown = [t for t in texts if not t.startswith("[ERROR] Failing runs:")] + return shown or texts + + def format_drcs(drcs): return ", ".join(f"{name} ({count})" for name, count in drcs.items()) @@ -284,7 +291,7 @@ def row_reason(data): if code == "QOR": names = qor_names(data) if not names: - return message_id(data["metrics_errors"][0][1], REGEX_ERROR) + return message_id(qor_error_lines(data)[0], REGEX_ERROR) if len(names) == 1: return names[0] return f"{len(names)} metrics: " + ", ".join(names) @@ -364,13 +371,14 @@ def gen_report(name, data, shared_baseline=None, with_notes=True): body += [f" {line}" for line in data["tail"]] full = verbose >= 1 body += message_lines(data["log_errors"], REGEX_ERROR, full) - body += message_lines(data["metrics_logs_errors"], REGEX_ERROR, full) + if data["status"] == STATUS_RED: + body += message_lines(data["metrics_logs_errors"], REGEX_ERROR, full) body += message_lines(data["calibre_errors"], REGEX_ERROR, full) if data["metrics_errors"]: if data["qor_metrics"]: body += qor_table(data["qor_metrics"]) else: - body += [text for _, text in data["metrics_errors"]] + body += qor_error_lines(data) if data["qor_baseline"] and data["qor_baseline"] != shared_baseline: body.append(f"Baseline: {data['qor_baseline']}") if data["drcs"] and data["status"] == STATUS_RED: @@ -386,8 +394,11 @@ def gen_report(name, data, shared_baseline=None, with_notes=True): body += message_lines(warnings, REGEX_WARNING, verbose >= 3) if data["status"] == STATUS_GREEN: - notes = data["notes"] if with_notes else [] - lines = [f"{name}: {note}" for note in notes] or [f"{name}: passed"] + if with_notes: + lines = [f"{name}: {note}" for note in data["notes"]] + else: + lines = [f"{name}: passed, see NEEDS ATTENTION"] if data["notes"] else [] + lines = lines or [f"{name}: passed"] else: lines = [f"{name}: {headline(data)}"] lines += [f" {line}" for line in body]