From e1cd61ca4201c9d9b65e455e86015349a8abdba4 Mon Sep 17 00:00:00 2001 From: Miguel Dalberto Pedro Date: Tue, 27 Oct 2026 14:45:42 -0300 Subject: [PATCH 1/3] Report how each design's flow run ended in the metadata upload uploadMetadata.py now reads flow-status.json, which the CI job writes next to each design's reports with the run's exit code, whether its time limit stopped it, and the elapsed and limit minutes. Each design record gains an "outcome" object: passed, error, timeout or crash, with the stage the run ended in and the first error line found in its stage logs. A crash is a failed run whose last stage log shows the tool died on a signal. A design that failed before writing metadata.json used to be skipped with a warning. When its run left a status file it is now reported with only its outcome, so a design that keeps timing out shows up instead of silently going missing. On the per-design fallback for oversized payloads, such a record names itself with the run__flow__ keys that metadata.json would have held. Designs without a status file are reported exactly as before. Signed-off-by: Miguel Dalberto Pedro --- flow/test/test_uploadMetadata.py | 144 +++++++++++++++++++++++ flow/util/uploadMetadata.py | 196 ++++++++++++++++++++++++++++--- 2 files changed, 325 insertions(+), 15 deletions(-) diff --git a/flow/test/test_uploadMetadata.py b/flow/test/test_uploadMetadata.py index 92555ed5a5..be94f63757 100755 --- a/flow/test/test_uploadMetadata.py +++ b/flow/test/test_uploadMetadata.py @@ -102,6 +102,23 @@ def add_design(self, platform, design, variant="base", metrics=None): with open(os.path.join(report_dir, "metadata.json"), "w") as f: json.dump(metrics or {"finish:area": 1.0}, f) + def add_flow_status(self, platform, design, status, variant="base", logs=None): + """Record how a design's flow run ended, as the CI job does, with stage + logs given as {file name: content}.""" + report_dir = os.path.join(self.flow, "reports", platform, design, variant) + os.makedirs(report_dir, exist_ok=True) + with open(os.path.join(report_dir, "flow-status.json"), "w") as f: + f.write(status if isinstance(status, str) else json.dumps(status)) + log_dir = os.path.join(self.flow, "logs", platform, design, variant) + os.makedirs(log_dir, exist_ok=True) + for name, content in (logs or {}).items(): + with open(os.path.join(log_dir, name), "w") as f: + f.write(content) + + def designs(self, messages): + """The pipeline payload's design records, by design name.""" + return {d["design"]: d for d in messages[0]["data"]["designs"]} + def run_script(self, *extra, fail=False, tz=None, extra_env=None): env = dict(os.environ) for name in REQUESTER_ENV: @@ -537,6 +554,133 @@ def test_no_publisher_publishes_nothing(self): self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertFalse(os.path.exists(self.out)) + def test_timed_out_design_without_metrics_reports_its_outcome(self): + self.add_design("nangate45", "gcd") + self.add_flow_status( + "nangate45", + "ibex", + { + "exit_code": 124, + "timed_out": True, + "elapsed_minutes": 240.2, + "limit_minutes": 240, + }, + logs={"3_3_place_gp.log": "placing\n", "5_1_grt.log": "routing\n"}, + ) + result, messages = self.run_script("--jobName", "ORFS") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + designs = self.designs(messages) + self.assertEqual( + designs["ibex"], + { + "platform": "nangate45", + "design": "ibex", + "variant": "base", + "outcome": { + "status": "timeout", + "stage": "grt", + "elapsed_minutes": 240.2, + "limit_minutes": 240, + }, + }, + ) + self.assertNotIn("outcome", designs["gcd"]) + self.assertIn("reporting its timeout outcome only", result.stdout) + + def test_failed_design_reports_stage_and_first_error_line(self): + self.add_flow_status( + "asap7", + "aes", + {"exit_code": 2, "timed_out": False}, + logs={ + "4_1_cts.log": "[ERROR CTS-0001] not this one\n", + "5_1_grt.log": "info\n[ERROR GRT-0116] Global routing failed.\n" + "[ERROR GRT-0001] later\n", + }, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["aes"]["outcome"] + self.assertEqual(outcome["status"], "error") + self.assertEqual(outcome["stage"], "grt") + self.assertEqual(outcome["message"], "[ERROR GRT-0116] Global routing failed.") + + def test_yosys_error_line_is_reported(self): + self.add_flow_status( + "asap7", + "aes", + {"exit_code": 2}, + logs={"1_2_yosys.log": "ERROR: Module `foo' referenced but not found.\n"}, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["aes"]["outcome"] + self.assertEqual((outcome["status"], outcome["stage"]), ("error", "synth")) + self.assertEqual( + outcome["message"], "ERROR: Module `foo' referenced but not found." + ) + + def test_signal_in_last_stage_log_is_a_crash(self): + self.add_flow_status( + "sky130hd", + "jpeg", + {"exit_code": 2}, + logs={"3_3_place_gp.log": "Signal 11 received\nStack trace:\n"}, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["jpeg"]["outcome"] + self.assertEqual((outcome["status"], outcome["stage"]), ("crash", "place")) + self.assertNotIn("message", outcome) + + def test_passing_design_keeps_its_metrics_and_reports_passed(self): + self.add_design("nangate45", "gcd", metrics={"finish:area": 5.0}) + self.add_flow_status( + "nangate45", "gcd", {"exit_code": 0, "elapsed_minutes": 3.5} + ) + _, messages = self.run_script() + record = self.designs(messages)["gcd"] + self.assertEqual(record["metrics"], {"finish__area": 5.0}) + self.assertEqual( + record["outcome"], {"status": "passed", "elapsed_minutes": 3.5} + ) + + def test_long_error_line_is_truncated(self): + self.add_flow_status( + "asap7", + "aes", + {"exit_code": 2}, + logs={"5_2_route.log": "[ERROR DRT-0001] " + "x" * 2000 + "\n"}, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["aes"]["outcome"] + self.assertEqual(outcome["stage"], "route") + self.assertEqual(len(outcome["message"]), 1024) + + def test_unreadable_status_without_metrics_skips_the_design(self): + self.add_design("nangate45", "gcd") + self.add_flow_status("nangate45", "ibex", "not json") + self.add_flow_status("nangate45", "jpeg", {"timed_out": False}) + result, messages = self.run_script() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(set(self.designs(messages)), {"gcd"}) + self.assertIn("Reporting no outcome", result.stdout) + self.assertIn("[WARN] No data file for nangate45 ibex base.", result.stdout) + + def test_outcome_only_design_names_itself_on_the_v1_fallback(self): + self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS) + self.add_flow_status( + "nangate45", + "ibex", + {"exit_code": 124, "timed_out": True, "limit_minutes": 60}, + ) + result, messages = self.run_script("--buildTime", "2026-10-05 12:34") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + by_design = {m["data"].get("run__flow__design"): m["data"] for m in messages} + ibex = by_design["ibex"] + self.assertEqual(ibex["run__flow__platform"], "nangate45") + self.assertEqual(ibex["run__flow__variant"], "base") + self.assertEqual(ibex["run__flow__generate_date"], "2026-10-05 12:34") + self.assertEqual(ibex["outcome"], {"status": "timeout", "limit_minutes": 60}) + self.assertNotIn("outcome", by_design.get(None, {})) + if __name__ == "__main__": unittest.main() diff --git a/flow/util/uploadMetadata.py b/flow/util/uploadMetadata.py index 5aff02839a..4a7419d4a8 100755 --- a/flow/util/uploadMetadata.py +++ b/flow/util/uploadMetadata.py @@ -3,6 +3,7 @@ import json import argparse import os +import re import sys from datetime import datetime, timezone @@ -160,17 +161,154 @@ def _non_negative_int(val): MAX_PUBSUB_BYTES = 8 * 1024 * 1024 -def build_design_record(dataFile, platform, design, variant): - """Return a dict for one design to be included in the pipeline-level payload.""" - with open(dataFile) as f: - data = json.load(f) - metrics = {k.replace(":", "__"): v for k, v in data.items()} +def build_design_record(dataFile, platform, design, variant, outcome=None): + """Return a dict for one design to be included in the pipeline-level payload. + + ``dataFile`` is the design's metadata.json, or ``None`` for a run that ended + before writing one; such a record carries only its ``outcome``. + """ + record = {"platform": platform, "design": design, "variant": variant} + if dataFile: + with open(dataFile) as f: + data = json.load(f) + record["metrics"] = {k.replace(":", "__"): v for k, v in data.items()} + if outcome: + record["outcome"] = outcome + return record + + +# How a flow run ended, as the CI job records it next to the design's reports: +# {"exit_code": int, "timed_out": bool, "elapsed_minutes": float, +# "limit_minutes": int}. Every key is optional. +STATUS_FILENAME = "flow-status.json" +MAX_OUTCOME_MESSAGE = 1024 +# Stage log names start with the step number: 1_2_yosys.log, 5_1_grt.log. +STAGE_LOG = re.compile(r"^(\d+)_(?:(\d+)_)?.*\.log$") +# The first matching line of a failing stage's log is the error reported. The +# OpenROAD form carries the tool's message ID; Yosys prints "ERROR: ". +ERROR_LINE = re.compile( + r"\[(?:ERROR|CRITICAL|FATAL) [A-Z][A-Z0-9]*-\d+\].*|^ERROR: .*|^Error: .*" +) +# What a tool that died on a signal leaves in its log, as opposed to an error it +# reported itself. +CRASH_LINE = re.compile(r"Signal \d+ received|Segmentation fault|core dumped") + + +def _stage_name(log_name): + """The flow stage a stage log belongs to, e.g. 5_1_grt.log -> "grt".""" + match = STAGE_LOG.match(log_name) + if not match: + return None + step, sub = match.group(1), match.group(2) + if step == "5": + return "grt" if sub == "1" else "route" return { - "platform": platform, - "design": design, - "variant": variant, - "metrics": metrics, - } + "1": "synth", + "2": "floorplan", + "3": "place", + "4": "cts", + "6": "finish", + }.get(step) + + +def _stage_logs(log_dir): + """The design's stage logs, last step first.""" + try: + names = os.listdir(log_dir) + except OSError: + return [] + + def key(name): + match = STAGE_LOG.match(name) + return (int(match.group(1)), int(match.group(2) or 0), name) + + return sorted((n for n in names if STAGE_LOG.match(n)), key=key, reverse=True) + + +def _scan_log(path): + """The first error line of a log and whether it shows a crash.""" + error_line = None + crashed = False + try: + with open(path, errors="replace") as f: + for line in f: + if error_line is None: + match = ERROR_LINE.search(line) + if match: + error_line = match.group(0).strip() + if not crashed and CRASH_LINE.search(line): + crashed = True + if error_line is not None and crashed: + break + except OSError: + pass + return error_line, crashed + + +def read_flow_outcome(report_dir, log_dir): + """The design's outcome from its flow-status.json and stage logs, or ``None``. + + ``status`` is ``timeout`` when the time limit stopped the run, ``passed`` on + exit code 0, ``crash`` when the last stage log shows the tool died on a + signal, and ``error`` otherwise. A failed run also names the stage it ended + in and the first error line found in its logs, last stage first. + """ + path = os.path.join(report_dir, STATUS_FILENAME) + if not os.path.exists(path): + return None + try: + with open(path) as f: + status = json.load(f) + except (OSError, ValueError) as e: + print(f"[WARN] Could not read {path}: {e}. Reporting no outcome.") + return None + if not isinstance(status, dict): + print(f"[WARN] {path} does not hold a JSON object. Reporting no outcome.") + return None + + exit_code = status.get("exit_code") + timed_out = status.get("timed_out") is True + if not timed_out and not isinstance(exit_code, int): + print(f"[WARN] {path} has no exit_code. Reporting no outcome.") + return None + + outcome = {} + for key in ("elapsed_minutes", "limit_minutes"): + value = status.get(key) + if ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and value >= 0 + ): + outcome[key] = value + + if not timed_out and exit_code == 0: + outcome["status"] = "passed" + return outcome + + logs = _stage_logs(log_dir) + if logs: + stage = _stage_name(logs[0]) + if stage: + outcome["stage"] = stage + + message = None + crashed = False + for i, name in enumerate(logs): + error_line, log_crashed = _scan_log(os.path.join(log_dir, name)) + if i == 0: + crashed = log_crashed + if error_line: + message = error_line + break + if message: + outcome["message"] = message[:MAX_OUTCOME_MESSAGE] + + if timed_out: + outcome["status"] = "timeout" + else: + outcome["status"] = "crash" if crashed else "error" + return outcome def _submodule_defect(entry): @@ -426,6 +564,10 @@ def publish_v1_per_design( The requester keys go on after the metrics, so a metric that happened to share their name could not replace who asked for the build. + + A design's outcome rides at the root as ``outcome``. A design that has only + an outcome carries no metrics to name it, so its message names the design + with the same run__flow__ keys metadata.json would hold, dated by the build. """ futures = [] failed = 0 @@ -441,7 +583,19 @@ def publish_v1_per_design( } if provenance: payload.update(provenance) - payload.update(d["metrics"]) + if "metrics" in d: + payload.update(d["metrics"]) + else: + payload.update( + { + "run__flow__platform": d["platform"], + "run__flow__design": d["design"], + "run__flow__variant": d["variant"], + "run__flow__generate_date": resolve_build_time(args), + } + ) + if "outcome" in d: + payload["outcome"] = d["outcome"] if requester: payload.update(requester) @@ -513,16 +667,28 @@ def publish_v1_per_design( platform = dirList[1] design = dirList[2] variant = dirList[3] + if platform == "sky130hd_fakestack" or platform == "src": + print(f"[WARN] Skiping upload {platform} {design} {variant}.") + continue dataFile = os.path.join(reportDir, RUN_FILENAME) if not os.path.exists(dataFile): + dataFile = None + outcome = read_flow_outcome( + reportDir, os.path.join("logs", platform, design, variant) + ) + if dataFile is None and outcome is None: print(f"[WARN] No data file for {platform} {design} {variant}.") continue - if platform == "sky130hd_fakestack" or platform == "src": - print(f"[WARN] Skiping upload {platform} {design} {variant}.") - continue + if dataFile is None: + print( + f"[INFO] No data file for {platform} {design} {variant}; " + f"reporting its {outcome['status']} outcome only." + ) # --- PUBSUB --- if publisher: - design_records.append(build_design_record(dataFile, platform, design, variant)) + design_records.append( + build_design_record(dataFile, platform, design, variant, outcome) + ) # --- END PUBSUB --- # --- PUBSUB --- From 37e361708479c362a0f7ee2f0010b76b06782f76 Mon Sep 17 00:00:00 2001 From: Jonas Gava Date: Tue, 3 Nov 2026 23:02:34 +0000 Subject: [PATCH 2/3] set CUGR as default routing engine for ten designs Signed-off-by: Jonas Gava --- flow/designs/asap7/ibex/config.mk | 2 ++ flow/designs/asap7/jpeg_lvt/config.mk | 1 + flow/designs/gf180/ibex/config.mk | 2 ++ flow/designs/ihp-sg13g2/i2c-gpio-expander/config.mk | 1 + flow/designs/ihp-sg13g2/riscv32i/config.mk | 2 ++ flow/designs/nangate45/ariane136/config.mk | 2 ++ flow/designs/nangate45/bp_be_top/config.mk | 1 + flow/designs/nangate45/dynamic_node/config.mk | 3 ++- flow/designs/sky130hs/aes/config.mk | 2 ++ flow/designs/sky130hs/gcd/config.mk | 1 + 10 files changed, 16 insertions(+), 1 deletion(-) diff --git a/flow/designs/asap7/ibex/config.mk b/flow/designs/asap7/ibex/config.mk index c66b82f84d..6518ee8018 100644 --- a/flow/designs/asap7/ibex/config.mk +++ b/flow/designs/asap7/ibex/config.mk @@ -32,3 +32,5 @@ export SETUP_REPAIR_PHASES = PARALLEL_EVAL export SWAP_ARITH_OPERATORS = 1 export OPENROAD_HIERARCHICAL = 1 +export GLOBAL_ROUTE_USE_CUGR = 1 + diff --git a/flow/designs/asap7/jpeg_lvt/config.mk b/flow/designs/asap7/jpeg_lvt/config.mk index f15962861d..c75a448446 100644 --- a/flow/designs/asap7/jpeg_lvt/config.mk +++ b/flow/designs/asap7/jpeg_lvt/config.mk @@ -20,3 +20,4 @@ export RECOVER_POWER = 100 export ASAP7_USE_VT = LVT export SYNTH_USE_SYN = 1 +export GLOBAL_ROUTE_USE_CUGR = 1 diff --git a/flow/designs/gf180/ibex/config.mk b/flow/designs/gf180/ibex/config.mk index dfb4d78e38..2350e73f7a 100644 --- a/flow/designs/gf180/ibex/config.mk +++ b/flow/designs/gf180/ibex/config.mk @@ -19,3 +19,5 @@ export SWAP_ARITH_OPERATORS = 1 export OPENROAD_HIERARCHICAL = 1 export SETUP_REPAIR_PHASES = PARALLEL_EVAL + +export GLOBAL_ROUTE_USE_CUGR = 1 diff --git a/flow/designs/ihp-sg13g2/i2c-gpio-expander/config.mk b/flow/designs/ihp-sg13g2/i2c-gpio-expander/config.mk index 53f618cb07..78a92658ff 100644 --- a/flow/designs/ihp-sg13g2/i2c-gpio-expander/config.mk +++ b/flow/designs/ihp-sg13g2/i2c-gpio-expander/config.mk @@ -55,3 +55,4 @@ export PDN_TCL = $(DESIGN_HOME)/$(PLATFORM)/$(DESIGN_NICKNAME)/pdn.tcl export BLOCKS = I2cDeviceCtrl +export GLOBAL_ROUTE_USE_CUGR = 1 diff --git a/flow/designs/ihp-sg13g2/riscv32i/config.mk b/flow/designs/ihp-sg13g2/riscv32i/config.mk index 4aa59f484d..9f9408ba31 100644 --- a/flow/designs/ihp-sg13g2/riscv32i/config.mk +++ b/flow/designs/ihp-sg13g2/riscv32i/config.mk @@ -15,3 +15,5 @@ export CTS_BUF_DISTANCE = 60 export SWAP_ARITH_OPERATORS = 1 export OPENROAD_HIERARCHICAL = 1 + +export GLOBAL_ROUTE_USE_CUGR = 1 \ No newline at end of file diff --git a/flow/designs/nangate45/ariane136/config.mk b/flow/designs/nangate45/ariane136/config.mk index 3954bf78d8..4d4f8f4177 100644 --- a/flow/designs/nangate45/ariane136/config.mk +++ b/flow/designs/nangate45/ariane136/config.mk @@ -22,3 +22,5 @@ export RTLMP_MIN_CHANNEL_SIZE = 24 24 export TNS_END_PERCENT = 100 export PLACE_DENSITY = 0.35 + +export GLOBAL_ROUTE_USE_CUGR = 1 diff --git a/flow/designs/nangate45/bp_be_top/config.mk b/flow/designs/nangate45/bp_be_top/config.mk index 3b623cce36..5b09d26b57 100644 --- a/flow/designs/nangate45/bp_be_top/config.mk +++ b/flow/designs/nangate45/bp_be_top/config.mk @@ -31,3 +31,4 @@ export CTS_CLUSTER_DIAMETER = 50 export SYNTH_MINIMUM_KEEP_SIZE = 3000 export FASTROUTE_TCL = $(DESIGN_HOME)/$(PLATFORM)/$(DESIGN_NAME)/fastroute.tcl +export GLOBAL_ROUTE_USE_CUGR = 1 diff --git a/flow/designs/nangate45/dynamic_node/config.mk b/flow/designs/nangate45/dynamic_node/config.mk index 493001c412..4e0ab89abe 100644 --- a/flow/designs/nangate45/dynamic_node/config.mk +++ b/flow/designs/nangate45/dynamic_node/config.mk @@ -5,7 +5,6 @@ export PLATFORM = nangate45 export VERILOG_FILES = $(DESIGN_HOME)/src/$(DESIGN_NICKNAME)/dynamic_node.pickle.v export SDC_FILE = $(DESIGN_HOME)/$(PLATFORM)/$(DESIGN_NICKNAME)/constraint.sdc - export CORE_UTILIZATION = 40 export CORE_ASPECT_RATIO = 1 export CORE_MARGIN = 5 @@ -18,3 +17,5 @@ export GPL_PLACE_IOS = 1 export SWAP_ARITH_OPERATORS = 1 export OPENROAD_HIERARCHICAL = 1 export SYNTH_USE_SYN = 1 + +export GLOBAL_ROUTE_USE_CUGR = 1 \ No newline at end of file diff --git a/flow/designs/sky130hs/aes/config.mk b/flow/designs/sky130hs/aes/config.mk index 6989c90124..c63f0b2328 100644 --- a/flow/designs/sky130hs/aes/config.mk +++ b/flow/designs/sky130hs/aes/config.mk @@ -20,3 +20,5 @@ export CTS_CLUSTER_DIAMETER = 50 export SWAP_ARITH_OPERATORS = 1 export OPENROAD_HIERARCHICAL = 1 +export GLOBAL_ROUTE_USE_CUGR = 1 + diff --git a/flow/designs/sky130hs/gcd/config.mk b/flow/designs/sky130hs/gcd/config.mk index 2491313ff8..3be65cd1c2 100644 --- a/flow/designs/sky130hs/gcd/config.mk +++ b/flow/designs/sky130hs/gcd/config.mk @@ -13,3 +13,4 @@ export PLACE_DENSITY_LB_ADDON = 0.1 export TNS_END_PERCENT = 100 #export SYNTH_HDL_FRONTEND ?= slang #export OPENROAD_HIERARCHICAL ?= 1 +export GLOBAL_ROUTE_USE_CUGR = 1 From 64fee0b4f78ca4121f1eff951f1cd25e40a002b5 Mon Sep 17 00:00:00 2001 From: Jonas Gava Date: Thu, 5 Nov 2026 15:41:20 +0000 Subject: [PATCH 3/3] fix no newline at the end Signed-off-by: Jonas Gava --- flow/designs/ihp-sg13g2/riscv32i/config.mk | 2 +- flow/designs/nangate45/dynamic_node/config.mk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flow/designs/ihp-sg13g2/riscv32i/config.mk b/flow/designs/ihp-sg13g2/riscv32i/config.mk index 9f9408ba31..b49ddd98fb 100644 --- a/flow/designs/ihp-sg13g2/riscv32i/config.mk +++ b/flow/designs/ihp-sg13g2/riscv32i/config.mk @@ -16,4 +16,4 @@ export CTS_BUF_DISTANCE = 60 export SWAP_ARITH_OPERATORS = 1 export OPENROAD_HIERARCHICAL = 1 -export GLOBAL_ROUTE_USE_CUGR = 1 \ No newline at end of file +export GLOBAL_ROUTE_USE_CUGR = 1 diff --git a/flow/designs/nangate45/dynamic_node/config.mk b/flow/designs/nangate45/dynamic_node/config.mk index 4e0ab89abe..6697c831a3 100644 --- a/flow/designs/nangate45/dynamic_node/config.mk +++ b/flow/designs/nangate45/dynamic_node/config.mk @@ -18,4 +18,4 @@ export SWAP_ARITH_OPERATORS = 1 export OPENROAD_HIERARCHICAL = 1 export SYNTH_USE_SYN = 1 -export GLOBAL_ROUTE_USE_CUGR = 1 \ No newline at end of file +export GLOBAL_ROUTE_USE_CUGR = 1