diff --git a/flow/test/test_uploadMetadata.py b/flow/test/test_uploadMetadata.py index 92555ed5a5..be94f63757 100755 --- a/flow/test/test_uploadMetadata.py +++ b/flow/test/test_uploadMetadata.py @@ -102,6 +102,23 @@ def add_design(self, platform, design, variant="base", metrics=None): with open(os.path.join(report_dir, "metadata.json"), "w") as f: json.dump(metrics or {"finish:area": 1.0}, f) + def add_flow_status(self, platform, design, status, variant="base", logs=None): + """Record how a design's flow run ended, as the CI job does, with stage + logs given as {file name: content}.""" + report_dir = os.path.join(self.flow, "reports", platform, design, variant) + os.makedirs(report_dir, exist_ok=True) + with open(os.path.join(report_dir, "flow-status.json"), "w") as f: + f.write(status if isinstance(status, str) else json.dumps(status)) + log_dir = os.path.join(self.flow, "logs", platform, design, variant) + os.makedirs(log_dir, exist_ok=True) + for name, content in (logs or {}).items(): + with open(os.path.join(log_dir, name), "w") as f: + f.write(content) + + def designs(self, messages): + """The pipeline payload's design records, by design name.""" + return {d["design"]: d for d in messages[0]["data"]["designs"]} + def run_script(self, *extra, fail=False, tz=None, extra_env=None): env = dict(os.environ) for name in REQUESTER_ENV: @@ -537,6 +554,133 @@ def test_no_publisher_publishes_nothing(self): self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertFalse(os.path.exists(self.out)) + def test_timed_out_design_without_metrics_reports_its_outcome(self): + self.add_design("nangate45", "gcd") + self.add_flow_status( + "nangate45", + "ibex", + { + "exit_code": 124, + "timed_out": True, + "elapsed_minutes": 240.2, + "limit_minutes": 240, + }, + logs={"3_3_place_gp.log": "placing\n", "5_1_grt.log": "routing\n"}, + ) + result, messages = self.run_script("--jobName", "ORFS") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + designs = self.designs(messages) + self.assertEqual( + designs["ibex"], + { + "platform": "nangate45", + "design": "ibex", + "variant": "base", + "outcome": { + "status": "timeout", + "stage": "grt", + "elapsed_minutes": 240.2, + "limit_minutes": 240, + }, + }, + ) + self.assertNotIn("outcome", designs["gcd"]) + self.assertIn("reporting its timeout outcome only", result.stdout) + + def test_failed_design_reports_stage_and_first_error_line(self): + self.add_flow_status( + "asap7", + "aes", + {"exit_code": 2, "timed_out": False}, + logs={ + "4_1_cts.log": "[ERROR CTS-0001] not this one\n", + "5_1_grt.log": "info\n[ERROR GRT-0116] Global routing failed.\n" + "[ERROR GRT-0001] later\n", + }, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["aes"]["outcome"] + self.assertEqual(outcome["status"], "error") + self.assertEqual(outcome["stage"], "grt") + self.assertEqual(outcome["message"], "[ERROR GRT-0116] Global routing failed.") + + def test_yosys_error_line_is_reported(self): + self.add_flow_status( + "asap7", + "aes", + {"exit_code": 2}, + logs={"1_2_yosys.log": "ERROR: Module `foo' referenced but not found.\n"}, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["aes"]["outcome"] + self.assertEqual((outcome["status"], outcome["stage"]), ("error", "synth")) + self.assertEqual( + outcome["message"], "ERROR: Module `foo' referenced but not found." + ) + + def test_signal_in_last_stage_log_is_a_crash(self): + self.add_flow_status( + "sky130hd", + "jpeg", + {"exit_code": 2}, + logs={"3_3_place_gp.log": "Signal 11 received\nStack trace:\n"}, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["jpeg"]["outcome"] + self.assertEqual((outcome["status"], outcome["stage"]), ("crash", "place")) + self.assertNotIn("message", outcome) + + def test_passing_design_keeps_its_metrics_and_reports_passed(self): + self.add_design("nangate45", "gcd", metrics={"finish:area": 5.0}) + self.add_flow_status( + "nangate45", "gcd", {"exit_code": 0, "elapsed_minutes": 3.5} + ) + _, messages = self.run_script() + record = self.designs(messages)["gcd"] + self.assertEqual(record["metrics"], {"finish__area": 5.0}) + self.assertEqual( + record["outcome"], {"status": "passed", "elapsed_minutes": 3.5} + ) + + def test_long_error_line_is_truncated(self): + self.add_flow_status( + "asap7", + "aes", + {"exit_code": 2}, + logs={"5_2_route.log": "[ERROR DRT-0001] " + "x" * 2000 + "\n"}, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["aes"]["outcome"] + self.assertEqual(outcome["stage"], "route") + self.assertEqual(len(outcome["message"]), 1024) + + def test_unreadable_status_without_metrics_skips_the_design(self): + self.add_design("nangate45", "gcd") + self.add_flow_status("nangate45", "ibex", "not json") + self.add_flow_status("nangate45", "jpeg", {"timed_out": False}) + result, messages = self.run_script() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(set(self.designs(messages)), {"gcd"}) + self.assertIn("Reporting no outcome", result.stdout) + self.assertIn("[WARN] No data file for nangate45 ibex base.", result.stdout) + + def test_outcome_only_design_names_itself_on_the_v1_fallback(self): + self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS) + self.add_flow_status( + "nangate45", + "ibex", + {"exit_code": 124, "timed_out": True, "limit_minutes": 60}, + ) + result, messages = self.run_script("--buildTime", "2026-10-05 12:34") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + by_design = {m["data"].get("run__flow__design"): m["data"] for m in messages} + ibex = by_design["ibex"] + self.assertEqual(ibex["run__flow__platform"], "nangate45") + self.assertEqual(ibex["run__flow__variant"], "base") + self.assertEqual(ibex["run__flow__generate_date"], "2026-10-05 12:34") + self.assertEqual(ibex["outcome"], {"status": "timeout", "limit_minutes": 60}) + self.assertNotIn("outcome", by_design.get(None, {})) + if __name__ == "__main__": unittest.main() diff --git a/flow/util/uploadMetadata.py b/flow/util/uploadMetadata.py index 5aff02839a..4a7419d4a8 100755 --- a/flow/util/uploadMetadata.py +++ b/flow/util/uploadMetadata.py @@ -3,6 +3,7 @@ import json import argparse import os +import re import sys from datetime import datetime, timezone @@ -160,17 +161,154 @@ def _non_negative_int(val): MAX_PUBSUB_BYTES = 8 * 1024 * 1024 -def build_design_record(dataFile, platform, design, variant): - """Return a dict for one design to be included in the pipeline-level payload.""" - with open(dataFile) as f: - data = json.load(f) - metrics = {k.replace(":", "__"): v for k, v in data.items()} +def build_design_record(dataFile, platform, design, variant, outcome=None): + """Return a dict for one design to be included in the pipeline-level payload. + + ``dataFile`` is the design's metadata.json, or ``None`` for a run that ended + before writing one; such a record carries only its ``outcome``. + """ + record = {"platform": platform, "design": design, "variant": variant} + if dataFile: + with open(dataFile) as f: + data = json.load(f) + record["metrics"] = {k.replace(":", "__"): v for k, v in data.items()} + if outcome: + record["outcome"] = outcome + return record + + +# How a flow run ended, as the CI job records it next to the design's reports: +# {"exit_code": int, "timed_out": bool, "elapsed_minutes": float, +# "limit_minutes": int}. Every key is optional. +STATUS_FILENAME = "flow-status.json" +MAX_OUTCOME_MESSAGE = 1024 +# Stage log names start with the step number: 1_2_yosys.log, 5_1_grt.log. +STAGE_LOG = re.compile(r"^(\d+)_(?:(\d+)_)?.*\.log$") +# The first matching line of a failing stage's log is the error reported. The +# OpenROAD form carries the tool's message ID; Yosys prints "ERROR: ". +ERROR_LINE = re.compile( + r"\[(?:ERROR|CRITICAL|FATAL) [A-Z][A-Z0-9]*-\d+\].*|^ERROR: .*|^Error: .*" +) +# What a tool that died on a signal leaves in its log, as opposed to an error it +# reported itself. +CRASH_LINE = re.compile(r"Signal \d+ received|Segmentation fault|core dumped") + + +def _stage_name(log_name): + """The flow stage a stage log belongs to, e.g. 5_1_grt.log -> "grt".""" + match = STAGE_LOG.match(log_name) + if not match: + return None + step, sub = match.group(1), match.group(2) + if step == "5": + return "grt" if sub == "1" else "route" return { - "platform": platform, - "design": design, - "variant": variant, - "metrics": metrics, - } + "1": "synth", + "2": "floorplan", + "3": "place", + "4": "cts", + "6": "finish", + }.get(step) + + +def _stage_logs(log_dir): + """The design's stage logs, last step first.""" + try: + names = os.listdir(log_dir) + except OSError: + return [] + + def key(name): + match = STAGE_LOG.match(name) + return (int(match.group(1)), int(match.group(2) or 0), name) + + return sorted((n for n in names if STAGE_LOG.match(n)), key=key, reverse=True) + + +def _scan_log(path): + """The first error line of a log and whether it shows a crash.""" + error_line = None + crashed = False + try: + with open(path, errors="replace") as f: + for line in f: + if error_line is None: + match = ERROR_LINE.search(line) + if match: + error_line = match.group(0).strip() + if not crashed and CRASH_LINE.search(line): + crashed = True + if error_line is not None and crashed: + break + except OSError: + pass + return error_line, crashed + + +def read_flow_outcome(report_dir, log_dir): + """The design's outcome from its flow-status.json and stage logs, or ``None``. + + ``status`` is ``timeout`` when the time limit stopped the run, ``passed`` on + exit code 0, ``crash`` when the last stage log shows the tool died on a + signal, and ``error`` otherwise. A failed run also names the stage it ended + in and the first error line found in its logs, last stage first. + """ + path = os.path.join(report_dir, STATUS_FILENAME) + if not os.path.exists(path): + return None + try: + with open(path) as f: + status = json.load(f) + except (OSError, ValueError) as e: + print(f"[WARN] Could not read {path}: {e}. Reporting no outcome.") + return None + if not isinstance(status, dict): + print(f"[WARN] {path} does not hold a JSON object. Reporting no outcome.") + return None + + exit_code = status.get("exit_code") + timed_out = status.get("timed_out") is True + if not timed_out and not isinstance(exit_code, int): + print(f"[WARN] {path} has no exit_code. Reporting no outcome.") + return None + + outcome = {} + for key in ("elapsed_minutes", "limit_minutes"): + value = status.get(key) + if ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and value >= 0 + ): + outcome[key] = value + + if not timed_out and exit_code == 0: + outcome["status"] = "passed" + return outcome + + logs = _stage_logs(log_dir) + if logs: + stage = _stage_name(logs[0]) + if stage: + outcome["stage"] = stage + + message = None + crashed = False + for i, name in enumerate(logs): + error_line, log_crashed = _scan_log(os.path.join(log_dir, name)) + if i == 0: + crashed = log_crashed + if error_line: + message = error_line + break + if message: + outcome["message"] = message[:MAX_OUTCOME_MESSAGE] + + if timed_out: + outcome["status"] = "timeout" + else: + outcome["status"] = "crash" if crashed else "error" + return outcome def _submodule_defect(entry): @@ -426,6 +564,10 @@ def publish_v1_per_design( The requester keys go on after the metrics, so a metric that happened to share their name could not replace who asked for the build. + + A design's outcome rides at the root as ``outcome``. A design that has only + an outcome carries no metrics to name it, so its message names the design + with the same run__flow__ keys metadata.json would hold, dated by the build. """ futures = [] failed = 0 @@ -441,7 +583,19 @@ def publish_v1_per_design( } if provenance: payload.update(provenance) - payload.update(d["metrics"]) + if "metrics" in d: + payload.update(d["metrics"]) + else: + payload.update( + { + "run__flow__platform": d["platform"], + "run__flow__design": d["design"], + "run__flow__variant": d["variant"], + "run__flow__generate_date": resolve_build_time(args), + } + ) + if "outcome" in d: + payload["outcome"] = d["outcome"] if requester: payload.update(requester) @@ -513,16 +667,28 @@ def publish_v1_per_design( platform = dirList[1] design = dirList[2] variant = dirList[3] + if platform == "sky130hd_fakestack" or platform == "src": + print(f"[WARN] Skiping upload {platform} {design} {variant}.") + continue dataFile = os.path.join(reportDir, RUN_FILENAME) if not os.path.exists(dataFile): + dataFile = None + outcome = read_flow_outcome( + reportDir, os.path.join("logs", platform, design, variant) + ) + if dataFile is None and outcome is None: print(f"[WARN] No data file for {platform} {design} {variant}.") continue - if platform == "sky130hd_fakestack" or platform == "src": - print(f"[WARN] Skiping upload {platform} {design} {variant}.") - continue + if dataFile is None: + print( + f"[INFO] No data file for {platform} {design} {variant}; " + f"reporting its {outcome['status']} outcome only." + ) # --- PUBSUB --- if publisher: - design_records.append(build_design_record(dataFile, platform, design, variant)) + design_records.append( + build_design_record(dataFile, platform, design, variant, outcome) + ) # --- END PUBSUB --- # --- PUBSUB ---