From e1cd61ca4201c9d9b65e455e86015349a8abdba4 Mon Sep 17 00:00:00 2001 From: Miguel Dalberto Pedro Date: Tue, 27 Oct 2026 14:45:42 -0300 Subject: [PATCH] Report how each design's flow run ended in the metadata upload uploadMetadata.py now reads flow-status.json, which the CI job writes next to each design's reports with the run's exit code, whether its time limit stopped it, and the elapsed and limit minutes. Each design record gains an "outcome" object: passed, error, timeout or crash, with the stage the run ended in and the first error line found in its stage logs. A crash is a failed run whose last stage log shows the tool died on a signal. A design that failed before writing metadata.json used to be skipped with a warning. When its run left a status file it is now reported with only its outcome, so a design that keeps timing out shows up instead of silently going missing. On the per-design fallback for oversized payloads, such a record names itself with the run__flow__ keys that metadata.json would have held. Designs without a status file are reported exactly as before. Signed-off-by: Miguel Dalberto Pedro --- flow/test/test_uploadMetadata.py | 144 +++++++++++++++++++++++ flow/util/uploadMetadata.py | 196 ++++++++++++++++++++++++++++--- 2 files changed, 325 insertions(+), 15 deletions(-) diff --git a/flow/test/test_uploadMetadata.py b/flow/test/test_uploadMetadata.py index 92555ed5a5..be94f63757 100755 --- a/flow/test/test_uploadMetadata.py +++ b/flow/test/test_uploadMetadata.py @@ -102,6 +102,23 @@ def add_design(self, platform, design, variant="base", metrics=None): with open(os.path.join(report_dir, "metadata.json"), "w") as f: json.dump(metrics or {"finish:area": 1.0}, f) + def add_flow_status(self, platform, design, status, variant="base", logs=None): + """Record how a design's flow run ended, as the CI job does, with stage + logs given as {file name: content}.""" + report_dir = os.path.join(self.flow, "reports", platform, design, variant) + os.makedirs(report_dir, exist_ok=True) + with open(os.path.join(report_dir, "flow-status.json"), "w") as f: + f.write(status if isinstance(status, str) else json.dumps(status)) + log_dir = os.path.join(self.flow, "logs", platform, design, variant) + os.makedirs(log_dir, exist_ok=True) + for name, content in (logs or {}).items(): + with open(os.path.join(log_dir, name), "w") as f: + f.write(content) + + def designs(self, messages): + """The pipeline payload's design records, by design name.""" + return {d["design"]: d for d in messages[0]["data"]["designs"]} + def run_script(self, *extra, fail=False, tz=None, extra_env=None): env = dict(os.environ) for name in REQUESTER_ENV: @@ -537,6 +554,133 @@ def test_no_publisher_publishes_nothing(self): self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertFalse(os.path.exists(self.out)) + def test_timed_out_design_without_metrics_reports_its_outcome(self): + self.add_design("nangate45", "gcd") + self.add_flow_status( + "nangate45", + "ibex", + { + "exit_code": 124, + "timed_out": True, + "elapsed_minutes": 240.2, + "limit_minutes": 240, + }, + logs={"3_3_place_gp.log": "placing\n", "5_1_grt.log": "routing\n"}, + ) + result, messages = self.run_script("--jobName", "ORFS") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + designs = self.designs(messages) + self.assertEqual( + designs["ibex"], + { + "platform": "nangate45", + "design": "ibex", + "variant": "base", + "outcome": { + "status": "timeout", + "stage": "grt", + "elapsed_minutes": 240.2, + "limit_minutes": 240, + }, + }, + ) + self.assertNotIn("outcome", designs["gcd"]) + self.assertIn("reporting its timeout outcome only", result.stdout) + + def test_failed_design_reports_stage_and_first_error_line(self): + self.add_flow_status( + "asap7", + "aes", + {"exit_code": 2, "timed_out": False}, + logs={ + "4_1_cts.log": "[ERROR CTS-0001] not this one\n", + "5_1_grt.log": "info\n[ERROR GRT-0116] Global routing failed.\n" + "[ERROR GRT-0001] later\n", + }, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["aes"]["outcome"] + self.assertEqual(outcome["status"], "error") + self.assertEqual(outcome["stage"], "grt") + self.assertEqual(outcome["message"], "[ERROR GRT-0116] Global routing failed.") + + def test_yosys_error_line_is_reported(self): + self.add_flow_status( + "asap7", + "aes", + {"exit_code": 2}, + logs={"1_2_yosys.log": "ERROR: Module `foo' referenced but not found.\n"}, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["aes"]["outcome"] + self.assertEqual((outcome["status"], outcome["stage"]), ("error", "synth")) + self.assertEqual( + outcome["message"], "ERROR: Module `foo' referenced but not found." + ) + + def test_signal_in_last_stage_log_is_a_crash(self): + self.add_flow_status( + "sky130hd", + "jpeg", + {"exit_code": 2}, + logs={"3_3_place_gp.log": "Signal 11 received\nStack trace:\n"}, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["jpeg"]["outcome"] + self.assertEqual((outcome["status"], outcome["stage"]), ("crash", "place")) + self.assertNotIn("message", outcome) + + def test_passing_design_keeps_its_metrics_and_reports_passed(self): + self.add_design("nangate45", "gcd", metrics={"finish:area": 5.0}) + self.add_flow_status( + "nangate45", "gcd", {"exit_code": 0, "elapsed_minutes": 3.5} + ) + _, messages = self.run_script() + record = self.designs(messages)["gcd"] + self.assertEqual(record["metrics"], {"finish__area": 5.0}) + self.assertEqual( + record["outcome"], {"status": "passed", "elapsed_minutes": 3.5} + ) + + def test_long_error_line_is_truncated(self): + self.add_flow_status( + "asap7", + "aes", + {"exit_code": 2}, + logs={"5_2_route.log": "[ERROR DRT-0001] " + "x" * 2000 + "\n"}, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["aes"]["outcome"] + self.assertEqual(outcome["stage"], "route") + self.assertEqual(len(outcome["message"]), 1024) + + def test_unreadable_status_without_metrics_skips_the_design(self): + self.add_design("nangate45", "gcd") + self.add_flow_status("nangate45", "ibex", "not json") + self.add_flow_status("nangate45", "jpeg", {"timed_out": False}) + result, messages = self.run_script() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(set(self.designs(messages)), {"gcd"}) + self.assertIn("Reporting no outcome", result.stdout) + self.assertIn("[WARN] No data file for nangate45 ibex base.", result.stdout) + + def test_outcome_only_design_names_itself_on_the_v1_fallback(self): + self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS) + self.add_flow_status( + "nangate45", + "ibex", + {"exit_code": 124, "timed_out": True, "limit_minutes": 60}, + ) + result, messages = self.run_script("--buildTime", "2026-10-05 12:34") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + by_design = {m["data"].get("run__flow__design"): m["data"] for m in messages} + ibex = by_design["ibex"] + self.assertEqual(ibex["run__flow__platform"], "nangate45") + self.assertEqual(ibex["run__flow__variant"], "base") + self.assertEqual(ibex["run__flow__generate_date"], "2026-10-05 12:34") + self.assertEqual(ibex["outcome"], {"status": "timeout", "limit_minutes": 60}) + self.assertNotIn("outcome", by_design.get(None, {})) + if __name__ == "__main__": unittest.main() diff --git a/flow/util/uploadMetadata.py b/flow/util/uploadMetadata.py index 5aff02839a..4a7419d4a8 100755 --- a/flow/util/uploadMetadata.py +++ b/flow/util/uploadMetadata.py @@ -3,6 +3,7 @@ import json import argparse import os +import re import sys from datetime import datetime, timezone @@ -160,17 +161,154 @@ def _non_negative_int(val): MAX_PUBSUB_BYTES = 8 * 1024 * 1024 -def build_design_record(dataFile, platform, design, variant): - """Return a dict for one design to be included in the pipeline-level payload.""" - with open(dataFile) as f: - data = json.load(f) - metrics = {k.replace(":", "__"): v for k, v in data.items()} +def build_design_record(dataFile, platform, design, variant, outcome=None): + """Return a dict for one design to be included in the pipeline-level payload. + + ``dataFile`` is the design's metadata.json, or ``None`` for a run that ended + before writing one; such a record carries only its ``outcome``. + """ + record = {"platform": platform, "design": design, "variant": variant} + if dataFile: + with open(dataFile) as f: + data = json.load(f) + record["metrics"] = {k.replace(":", "__"): v for k, v in data.items()} + if outcome: + record["outcome"] = outcome + return record + + +# How a flow run ended, as the CI job records it next to the design's reports: +# {"exit_code": int, "timed_out": bool, "elapsed_minutes": float, +# "limit_minutes": int}. Every key is optional. +STATUS_FILENAME = "flow-status.json" +MAX_OUTCOME_MESSAGE = 1024 +# Stage log names start with the step number: 1_2_yosys.log, 5_1_grt.log. +STAGE_LOG = re.compile(r"^(\d+)_(?:(\d+)_)?.*\.log$") +# The first matching line of a failing stage's log is the error reported. The +# OpenROAD form carries the tool's message ID; Yosys prints "ERROR: ". +ERROR_LINE = re.compile( + r"\[(?:ERROR|CRITICAL|FATAL) [A-Z][A-Z0-9]*-\d+\].*|^ERROR: .*|^Error: .*" +) +# What a tool that died on a signal leaves in its log, as opposed to an error it +# reported itself. +CRASH_LINE = re.compile(r"Signal \d+ received|Segmentation fault|core dumped") + + +def _stage_name(log_name): + """The flow stage a stage log belongs to, e.g. 5_1_grt.log -> "grt".""" + match = STAGE_LOG.match(log_name) + if not match: + return None + step, sub = match.group(1), match.group(2) + if step == "5": + return "grt" if sub == "1" else "route" return { - "platform": platform, - "design": design, - "variant": variant, - "metrics": metrics, - } + "1": "synth", + "2": "floorplan", + "3": "place", + "4": "cts", + "6": "finish", + }.get(step) + + +def _stage_logs(log_dir): + """The design's stage logs, last step first.""" + try: + names = os.listdir(log_dir) + except OSError: + return [] + + def key(name): + match = STAGE_LOG.match(name) + return (int(match.group(1)), int(match.group(2) or 0), name) + + return sorted((n for n in names if STAGE_LOG.match(n)), key=key, reverse=True) + + +def _scan_log(path): + """The first error line of a log and whether it shows a crash.""" + error_line = None + crashed = False + try: + with open(path, errors="replace") as f: + for line in f: + if error_line is None: + match = ERROR_LINE.search(line) + if match: + error_line = match.group(0).strip() + if not crashed and CRASH_LINE.search(line): + crashed = True + if error_line is not None and crashed: + break + except OSError: + pass + return error_line, crashed + + +def read_flow_outcome(report_dir, log_dir): + """The design's outcome from its flow-status.json and stage logs, or ``None``. + + ``status`` is ``timeout`` when the time limit stopped the run, ``passed`` on + exit code 0, ``crash`` when the last stage log shows the tool died on a + signal, and ``error`` otherwise. A failed run also names the stage it ended + in and the first error line found in its logs, last stage first. + """ + path = os.path.join(report_dir, STATUS_FILENAME) + if not os.path.exists(path): + return None + try: + with open(path) as f: + status = json.load(f) + except (OSError, ValueError) as e: + print(f"[WARN] Could not read {path}: {e}. Reporting no outcome.") + return None + if not isinstance(status, dict): + print(f"[WARN] {path} does not hold a JSON object. Reporting no outcome.") + return None + + exit_code = status.get("exit_code") + timed_out = status.get("timed_out") is True + if not timed_out and not isinstance(exit_code, int): + print(f"[WARN] {path} has no exit_code. Reporting no outcome.") + return None + + outcome = {} + for key in ("elapsed_minutes", "limit_minutes"): + value = status.get(key) + if ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and value >= 0 + ): + outcome[key] = value + + if not timed_out and exit_code == 0: + outcome["status"] = "passed" + return outcome + + logs = _stage_logs(log_dir) + if logs: + stage = _stage_name(logs[0]) + if stage: + outcome["stage"] = stage + + message = None + crashed = False + for i, name in enumerate(logs): + error_line, log_crashed = _scan_log(os.path.join(log_dir, name)) + if i == 0: + crashed = log_crashed + if error_line: + message = error_line + break + if message: + outcome["message"] = message[:MAX_OUTCOME_MESSAGE] + + if timed_out: + outcome["status"] = "timeout" + else: + outcome["status"] = "crash" if crashed else "error" + return outcome def _submodule_defect(entry): @@ -426,6 +564,10 @@ def publish_v1_per_design( The requester keys go on after the metrics, so a metric that happened to share their name could not replace who asked for the build. + + A design's outcome rides at the root as ``outcome``. A design that has only + an outcome carries no metrics to name it, so its message names the design + with the same run__flow__ keys metadata.json would hold, dated by the build. """ futures = [] failed = 0 @@ -441,7 +583,19 @@ def publish_v1_per_design( } if provenance: payload.update(provenance) - payload.update(d["metrics"]) + if "metrics" in d: + payload.update(d["metrics"]) + else: + payload.update( + { + "run__flow__platform": d["platform"], + "run__flow__design": d["design"], + "run__flow__variant": d["variant"], + "run__flow__generate_date": resolve_build_time(args), + } + ) + if "outcome" in d: + payload["outcome"] = d["outcome"] if requester: payload.update(requester) @@ -513,16 +667,28 @@ def publish_v1_per_design( platform = dirList[1] design = dirList[2] variant = dirList[3] + if platform == "sky130hd_fakestack" or platform == "src": + print(f"[WARN] Skiping upload {platform} {design} {variant}.") + continue dataFile = os.path.join(reportDir, RUN_FILENAME) if not os.path.exists(dataFile): + dataFile = None + outcome = read_flow_outcome( + reportDir, os.path.join("logs", platform, design, variant) + ) + if dataFile is None and outcome is None: print(f"[WARN] No data file for {platform} {design} {variant}.") continue - if platform == "sky130hd_fakestack" or platform == "src": - print(f"[WARN] Skiping upload {platform} {design} {variant}.") - continue + if dataFile is None: + print( + f"[INFO] No data file for {platform} {design} {variant}; " + f"reporting its {outcome['status']} outcome only." + ) # --- PUBSUB --- if publisher: - design_records.append(build_design_record(dataFile, platform, design, variant)) + design_records.append( + build_design_record(dataFile, platform, design, variant, outcome) + ) # --- END PUBSUB --- # --- PUBSUB ---