Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions docs/contrib/Metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,12 @@ make [clean_metadata] metadata
`make metadata` runs three steps in order: `finish` (the full flow),
`metadata-generate` (write `metadata.json`), and `metadata-check` (send it to
the dashboard). The check log is written to
`$(REPORTS_DIR)/metadata-check.log`. It lists every failed rule with the
baseline value, the current value, the limit, and the delta, and ends with a
`QoR check: PASS`, `FAIL`, `INCONCLUSIVE`, or `ERROR` line.
`$(REPORTS_DIR)/metadata-check.log`. It has one `[ERROR]` line for each
failed rule, with the current value, the baseline value, the limit, and the
delta. It ends with a `QoR check: PASS`, `FAIL`, `INCONCLUSIVE`, or `ERROR`
line. `util/genReport.py` copies the `[ERROR]` lines into
`reports/report-summary.log`, so the summary shows the failed rules of all
designs in one place.

A `FAIL` verdict fails the target. So does `ERROR`, which means
`metadata.json` was missing, malformed, or held no numeric metric. An
Expand Down
155 changes: 155 additions & 0 deletions flow/test/test_checkQorMetrics_report.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""Tests that failed QoR rules reach the report summary.

genReport.py copies only the [ERROR] lines of metadata-check.log into
reports/report-summary.log, which CI prints at the end of a build. These
tests check that checkQorMetrics.py writes one such line per failed rule,
with its values, and that the summary shows them.
"""

import os
import shutil
import subprocess
import sys
import tempfile
import unittest

UTIL_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "util")
sys.path.append(UTIL_DIR)

import checkQorMetrics


def metric(name, status, target, base, threshold, delta):
return {
"metricName": name,
"status": status,
"targetValue": target,
"baseValue": base,
"thresholdValue": threshold,
"deltaPct": delta,
"ruleSource": "rule_config",
}


def result(status, metrics, failed):
return {
"platform": "sky130hd",
"design": "riscv32i",
"variant": "base",
"status": status,
"metric_count": 499,
"response": {
"status": status,
"baselineStrategy": "latest-master",
"baseBuildId": 33029,
"baseCommitSha": "55687ece2024cc7682ec56b34109ab0490cf2ee4",
"totals": {
"passed": len(metrics) - failed,
"failed": failed,
"missingFromBaseline": 0,
},
"metrics": metrics,
},
}


FAILING = result(
"fail",
[
metric("cts__timing__setup__ws", "pass", -52.623, -60.0663, -80.0663, -12.39),
metric(
"globalroute__timing__setup__tns",
"fail",
-8899.95,
-4617.66,
-5541.192,
92.74,
),
metric(
"finish__timing__setup__tns", "fail", -8900.79, -4650.05, -5580.06, 91.41
),
],
failed=2,
)


def errors(lines):
return [line for line in lines if line.startswith("[ERROR]")]


class RenderRunTest(unittest.TestCase):
def test_one_error_line_per_failed_rule(self):
self.assertEqual(
errors(checkQorMetrics.render_run(FAILING, verbose=False)),
[
"[ERROR] sky130hd/riscv32i/base: finish__timing__setup__tns: "
"current -8900.7900, baseline -4650.0500, limit -5580.0600, "
"delta +91.41%",
"[ERROR] sky130hd/riscv32i/base: globalroute__timing__setup__tns: "
"current -8899.9500, baseline -4617.6600, limit -5541.1920, "
"delta +92.74%",
],
)

def test_summary_adds_no_error_line(self):
self.assertEqual(errors(checkQorMetrics.summarize([FAILING])), [])
self.assertEqual(checkQorMetrics.summarize([FAILING])[-1], "QoR check: FAIL")

def test_table_only_when_verbose(self):
quiet = "\n".join(checkQorMetrics.render_run(FAILING, verbose=False))
loud = "\n".join(checkQorMetrics.render_run(FAILING, verbose=True))
self.assertNotIn("| Metric", quiet)
self.assertIn("| Metric", loud)
self.assertIn("cts__timing__setup__ws", loud)

def test_fail_without_named_metric_still_errors(self):
lines = checkQorMetrics.render_run(result("fail", [], failed=1), False)
self.assertEqual(len(errors(lines)), 1)
self.assertIn("FAIL", errors(lines)[0])

def test_pass_has_no_error_line(self):
passing = result(
"pass", [metric("cts__timing__setup__ws", "pass", 1, 1, 0, 0)], failed=0
)
self.assertEqual(errors(checkQorMetrics.render_run(passing, False)), [])


class ReportSummaryTest(unittest.TestCase):
"""Runs genReport.py the way CI does, on a minimal flow tree."""

def test_summary_lists_failed_rules_with_values(self):
with tempfile.TemporaryDirectory() as root:
# genReport.py works from the parent of its own directory.
os.mkdir(os.path.join(root, "util"))
shutil.copy(os.path.join(UTIL_DIR, "genReport.py"), f"{root}/util")
run = os.path.join("sky130hd", "riscv32i", "base")
for folder in ("logs", "reports"):
os.makedirs(os.path.join(root, folder, run))
with open(os.path.join(root, "logs", run, "6_report.log"), "w") as f:
f.write("done\n")
lines = checkQorMetrics.render_run(FAILING, False)
lines += checkQorMetrics.summarize([FAILING])
check_log = os.path.join(root, "reports", run, "metadata-check.log")
with open(check_log, "w") as f:
f.write("\n".join(lines) + "\n")

subprocess.run(
[sys.executable, f"{root}/util/genReport.py", "-sv", "-q"],
check=True,
capture_output=True,
)
with open(os.path.join(root, "reports", "report-summary.log")) as f:
summary = f.read()

self.assertIn("Found 2 metrics failures.", summary)
self.assertIn(
"finish__timing__setup__tns: current -8900.7900, "
"baseline -4650.0500, limit -5580.0600, delta +91.41%",
summary,
)
self.assertIn("globalroute__timing__setup__tns: current -8899.9500", summary)


if __name__ == "__main__":
unittest.main()
31 changes: 24 additions & 7 deletions flow/util/checkQorMetrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,17 @@ def render_run(result, verbose):
passed = totals.get("passed", 0)
failed = totals.get("failed", 0)
missing = totals.get("missingFromBaseline", 0)
prefix = "[ERROR]" if failed else "[INFO]"

metrics = response.get("metrics", [])
failing = sorted(
(m for m in metrics if m.get("status") == "fail"),
key=lambda m: m.get("metricName", ""),
)
# genReport.py copies only the [ERROR] lines of metadata-check.log into the
# report summary, so each failed rule gets its own [ERROR] line below. The
# headline is [ERROR] only when the dashboard failed the run without naming
# a metric; otherwise the summary count would not match the failed rules.
prefix = "[ERROR]" if failed and not failing else "[INFO]"

lines.append(
f"{prefix} {identity}: {result['status'].upper()} — "
Expand All @@ -345,11 +355,17 @@ def render_run(result, verbose):
"or the platform is private and no DASHBOARD_API_KEY was set."
)

metrics = response.get("metrics", [])
shown = metrics if verbose else [m for m in metrics if m.get("status") == "fail"]
shown = [m for m in shown if m.get("status") != "missing_from_baseline"]
for metric in failing:
lines.append(
f"[ERROR] {identity}: {metric.get('metricName', '?')}: "
f"current {fmt_value(metric.get('targetValue'))}, "
f"baseline {fmt_value(metric.get('baseValue'))}, "
f"limit {fmt_value(metric.get('thresholdValue'))}, "
f"delta {fmt_delta(metric.get('deltaPct'))}%"
)

if shown:
shown = [m for m in metrics if m.get("status") != "missing_from_baseline"]

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

To avoid performing an unnecessary list comprehension over all metrics when verbose is False (which is the default in typical CI runs), we can conditionally evaluate the list comprehension only when verbose is True.

Suggested change
shown = [m for m in metrics if m.get("status") != "missing_from_baseline"]
shown = [m for m in metrics if m.get("status") != "missing_from_baseline"] if verbose else []

if verbose and shown:
lines.append(ROW_FMT.format("Metric", "Baseline", "Current", "Limit", "Delta%"))
lines.append(
ROW_FMT.format("-" * NAME_W, "-" * 12, "-" * 12, "-" * 12, "-" * 7)
Expand Down Expand Up @@ -423,7 +439,8 @@ def summarize(results):
f"{counts.get('invalid', 0)} with unusable metadata"
)
if failing:
lines.append(f"[ERROR] Failing runs: {', '.join(sorted(failing))}")
# [INFO]: each run already printed an [ERROR] line per failed rule.
lines.append(f"[INFO] Failing runs: {', '.join(sorted(failing))}")
else:
lines.append("[INFO] No run failed a QoR dashboard rule.")

Expand Down Expand Up @@ -556,7 +573,7 @@ def parse_args(argv):
parser.add_argument(
"--verbose",
action="store_true",
help="Report every rule-checked metric, not only the failing ones, "
help="Also print a table of every rule-checked metric, "
"and name the metrics absent from the baseline build.",
)
args = parser.parse_args(argv)
Expand Down
Loading