From 0dd9bddc0ea6bece36093ef8b7974638ad779548 Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Mon, 5 Oct 2026 12:27:38 -0300 Subject: [PATCH] fix(qor-check): name each failed rule in the report summary The report summary that CI prints at the end of a build showed only counts for a failed QoR check, such as "2 failed". genReport.py copies only the [ERROR] lines of metadata-check.log into the summary, and the failed rules were in a table without that prefix. To find the values, a reader had to open the log of each design. Now each failed rule has its own [ERROR] line with the current value, the baseline value, the limit, and the delta. The run headline and the "Failing runs" line are now [INFO], so "Found N metrics failures" in the summary is the number of failed rules. The headline stays [ERROR] when the dashboard fails a run but names no metric. The table now shows only with --verbose, because the [ERROR] lines hold the same values. Signed-off-by: Vitor Bandeira --- docs/contrib/Metrics.md | 9 +- flow/test/test_checkQorMetrics_report.py | 155 +++++++++++++++++++++++ flow/util/checkQorMetrics.py | 31 ++++- 3 files changed, 185 insertions(+), 10 deletions(-) create mode 100644 flow/test/test_checkQorMetrics_report.py diff --git a/docs/contrib/Metrics.md b/docs/contrib/Metrics.md index 5bef0658d8..d16d1c7174 100644 --- a/docs/contrib/Metrics.md +++ b/docs/contrib/Metrics.md @@ -35,9 +35,12 @@ make [clean_metadata] metadata `make metadata` runs three steps in order: `finish` (the full flow), `metadata-generate` (write `metadata.json`), and `metadata-check` (send it to the dashboard). The check log is written to -`$(REPORTS_DIR)/metadata-check.log`. It lists every failed rule with the -baseline value, the current value, the limit, and the delta, and ends with a -`QoR check: PASS`, `FAIL`, `INCONCLUSIVE`, or `ERROR` line. +`$(REPORTS_DIR)/metadata-check.log`. It has one `[ERROR]` line for each +failed rule, with the current value, the baseline value, the limit, and the +delta. It ends with a `QoR check: PASS`, `FAIL`, `INCONCLUSIVE`, or `ERROR` +line. `util/genReport.py` copies the `[ERROR]` lines into +`reports/report-summary.log`, so the summary shows the failed rules of all +designs in one place. A `FAIL` verdict fails the target. So does `ERROR`, which means `metadata.json` was missing, malformed, or held no numeric metric. An diff --git a/flow/test/test_checkQorMetrics_report.py b/flow/test/test_checkQorMetrics_report.py new file mode 100644 index 0000000000..8d20259954 --- /dev/null +++ b/flow/test/test_checkQorMetrics_report.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Tests that failed QoR rules reach the report summary. + +genReport.py copies only the [ERROR] lines of metadata-check.log into +reports/report-summary.log, which CI prints at the end of a build. These +tests check that checkQorMetrics.py writes one such line per failed rule, +with its values, and that the summary shows them. +""" + +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + +UTIL_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "util") +sys.path.append(UTIL_DIR) + +import checkQorMetrics + + +def metric(name, status, target, base, threshold, delta): + return { + "metricName": name, + "status": status, + "targetValue": target, + "baseValue": base, + "thresholdValue": threshold, + "deltaPct": delta, + "ruleSource": "rule_config", + } + + +def result(status, metrics, failed): + return { + "platform": "sky130hd", + "design": "riscv32i", + "variant": "base", + "status": status, + "metric_count": 499, + "response": { + "status": status, + "baselineStrategy": "latest-master", + "baseBuildId": 33029, + "baseCommitSha": "55687ece2024cc7682ec56b34109ab0490cf2ee4", + "totals": { + "passed": len(metrics) - failed, + "failed": failed, + "missingFromBaseline": 0, + }, + "metrics": metrics, + }, + } + + +FAILING = result( + "fail", + [ + metric("cts__timing__setup__ws", "pass", -52.623, -60.0663, -80.0663, -12.39), + metric( + "globalroute__timing__setup__tns", + "fail", + -8899.95, + -4617.66, + -5541.192, + 92.74, + ), + metric( + "finish__timing__setup__tns", "fail", -8900.79, -4650.05, -5580.06, 91.41 + ), + ], + failed=2, +) + + +def errors(lines): + return [line for line in lines if line.startswith("[ERROR]")] + + +class RenderRunTest(unittest.TestCase): + def test_one_error_line_per_failed_rule(self): + self.assertEqual( + errors(checkQorMetrics.render_run(FAILING, verbose=False)), + [ + "[ERROR] sky130hd/riscv32i/base: finish__timing__setup__tns: " + "current -8900.7900, baseline -4650.0500, limit -5580.0600, " + "delta +91.41%", + "[ERROR] sky130hd/riscv32i/base: globalroute__timing__setup__tns: " + "current -8899.9500, baseline -4617.6600, limit -5541.1920, " + "delta +92.74%", + ], + ) + + def test_summary_adds_no_error_line(self): + self.assertEqual(errors(checkQorMetrics.summarize([FAILING])), []) + self.assertEqual(checkQorMetrics.summarize([FAILING])[-1], "QoR check: FAIL") + + def test_table_only_when_verbose(self): + quiet = "\n".join(checkQorMetrics.render_run(FAILING, verbose=False)) + loud = "\n".join(checkQorMetrics.render_run(FAILING, verbose=True)) + self.assertNotIn("| Metric", quiet) + self.assertIn("| Metric", loud) + self.assertIn("cts__timing__setup__ws", loud) + + def test_fail_without_named_metric_still_errors(self): + lines = checkQorMetrics.render_run(result("fail", [], failed=1), False) + self.assertEqual(len(errors(lines)), 1) + self.assertIn("FAIL", errors(lines)[0]) + + def test_pass_has_no_error_line(self): + passing = result( + "pass", [metric("cts__timing__setup__ws", "pass", 1, 1, 0, 0)], failed=0 + ) + self.assertEqual(errors(checkQorMetrics.render_run(passing, False)), []) + + +class ReportSummaryTest(unittest.TestCase): + """Runs genReport.py the way CI does, on a minimal flow tree.""" + + def test_summary_lists_failed_rules_with_values(self): + with tempfile.TemporaryDirectory() as root: + # genReport.py works from the parent of its own directory. + os.mkdir(os.path.join(root, "util")) + shutil.copy(os.path.join(UTIL_DIR, "genReport.py"), f"{root}/util") + run = os.path.join("sky130hd", "riscv32i", "base") + for folder in ("logs", "reports"): + os.makedirs(os.path.join(root, folder, run)) + with open(os.path.join(root, "logs", run, "6_report.log"), "w") as f: + f.write("done\n") + lines = checkQorMetrics.render_run(FAILING, False) + lines += checkQorMetrics.summarize([FAILING]) + check_log = os.path.join(root, "reports", run, "metadata-check.log") + with open(check_log, "w") as f: + f.write("\n".join(lines) + "\n") + + subprocess.run( + [sys.executable, f"{root}/util/genReport.py", "-sv", "-q"], + check=True, + capture_output=True, + ) + with open(os.path.join(root, "reports", "report-summary.log")) as f: + summary = f.read() + + self.assertIn("Found 2 metrics failures.", summary) + self.assertIn( + "finish__timing__setup__tns: current -8900.7900, " + "baseline -4650.0500, limit -5580.0600, delta +91.41%", + summary, + ) + self.assertIn("globalroute__timing__setup__tns: current -8899.9500", summary) + + +if __name__ == "__main__": + unittest.main() diff --git a/flow/util/checkQorMetrics.py b/flow/util/checkQorMetrics.py index 7146c53e50..4e9712c87f 100755 --- a/flow/util/checkQorMetrics.py +++ b/flow/util/checkQorMetrics.py @@ -327,7 +327,17 @@ def render_run(result, verbose): passed = totals.get("passed", 0) failed = totals.get("failed", 0) missing = totals.get("missingFromBaseline", 0) - prefix = "[ERROR]" if failed else "[INFO]" + + metrics = response.get("metrics", []) + failing = sorted( + (m for m in metrics if m.get("status") == "fail"), + key=lambda m: m.get("metricName", ""), + ) + # genReport.py copies only the [ERROR] lines of metadata-check.log into the + # report summary, so each failed rule gets its own [ERROR] line below. The + # headline is [ERROR] only when the dashboard failed the run without naming + # a metric; otherwise the summary count would not match the failed rules. + prefix = "[ERROR]" if failed and not failing else "[INFO]" lines.append( f"{prefix} {identity}: {result['status'].upper()} — " @@ -345,11 +355,17 @@ def render_run(result, verbose): "or the platform is private and no DASHBOARD_API_KEY was set." ) - metrics = response.get("metrics", []) - shown = metrics if verbose else [m for m in metrics if m.get("status") == "fail"] - shown = [m for m in shown if m.get("status") != "missing_from_baseline"] + for metric in failing: + lines.append( + f"[ERROR] {identity}: {metric.get('metricName', '?')}: " + f"current {fmt_value(metric.get('targetValue'))}, " + f"baseline {fmt_value(metric.get('baseValue'))}, " + f"limit {fmt_value(metric.get('thresholdValue'))}, " + f"delta {fmt_delta(metric.get('deltaPct'))}%" + ) - if shown: + shown = [m for m in metrics if m.get("status") != "missing_from_baseline"] + if verbose and shown: lines.append(ROW_FMT.format("Metric", "Baseline", "Current", "Limit", "Delta%")) lines.append( ROW_FMT.format("-" * NAME_W, "-" * 12, "-" * 12, "-" * 12, "-" * 7) @@ -423,7 +439,8 @@ def summarize(results): f"{counts.get('invalid', 0)} with unusable metadata" ) if failing: - lines.append(f"[ERROR] Failing runs: {', '.join(sorted(failing))}") + # [INFO]: each run already printed an [ERROR] line per failed rule. + lines.append(f"[INFO] Failing runs: {', '.join(sorted(failing))}") else: lines.append("[INFO] No run failed a QoR dashboard rule.") @@ -556,7 +573,7 @@ def parse_args(argv): parser.add_argument( "--verbose", action="store_true", - help="Report every rule-checked metric, not only the failing ones, " + help="Also print a table of every rule-checked metric, " "and name the metrics absent from the baseline build.", ) args = parser.parse_args(argv)