From e1cd61ca4201c9d9b65e455e86015349a8abdba4 Mon Sep 17 00:00:00 2001 From: Miguel Dalberto Pedro Date: Tue, 27 Oct 2026 14:45:42 -0300 Subject: [PATCH 1/6] Report how each design's flow run ended in the metadata upload uploadMetadata.py now reads flow-status.json, which the CI job writes next to each design's reports with the run's exit code, whether its time limit stopped it, and the elapsed and limit minutes. Each design record gains an "outcome" object: passed, error, timeout or crash, with the stage the run ended in and the first error line found in its stage logs. A crash is a failed run whose last stage log shows the tool died on a signal. A design that failed before writing metadata.json used to be skipped with a warning. When its run left a status file it is now reported with only its outcome, so a design that keeps timing out shows up instead of silently going missing. On the per-design fallback for oversized payloads, such a record names itself with the run__flow__ keys that metadata.json would have held. Designs without a status file are reported exactly as before. Signed-off-by: Miguel Dalberto Pedro --- flow/test/test_uploadMetadata.py | 144 +++++++++++++++++++++++ flow/util/uploadMetadata.py | 196 ++++++++++++++++++++++++++++--- 2 files changed, 325 insertions(+), 15 deletions(-) diff --git a/flow/test/test_uploadMetadata.py b/flow/test/test_uploadMetadata.py index 92555ed5a5..be94f63757 100755 --- a/flow/test/test_uploadMetadata.py +++ b/flow/test/test_uploadMetadata.py @@ -102,6 +102,23 @@ def add_design(self, platform, design, variant="base", metrics=None): with open(os.path.join(report_dir, "metadata.json"), "w") as f: json.dump(metrics or {"finish:area": 1.0}, f) + def add_flow_status(self, platform, design, status, variant="base", logs=None): + """Record how a design's flow run ended, as the CI job does, with stage + logs given as {file name: content}.""" + report_dir = os.path.join(self.flow, "reports", platform, design, variant) + os.makedirs(report_dir, exist_ok=True) + with open(os.path.join(report_dir, "flow-status.json"), "w") as f: + f.write(status if isinstance(status, str) else json.dumps(status)) + log_dir = os.path.join(self.flow, "logs", platform, design, variant) + os.makedirs(log_dir, exist_ok=True) + for name, content in (logs or {}).items(): + with open(os.path.join(log_dir, name), "w") as f: + f.write(content) + + def designs(self, messages): + """The pipeline payload's design records, by design name.""" + return {d["design"]: d for d in messages[0]["data"]["designs"]} + def run_script(self, *extra, fail=False, tz=None, extra_env=None): env = dict(os.environ) for name in REQUESTER_ENV: @@ -537,6 +554,133 @@ def test_no_publisher_publishes_nothing(self): self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertFalse(os.path.exists(self.out)) + def test_timed_out_design_without_metrics_reports_its_outcome(self): + self.add_design("nangate45", "gcd") + self.add_flow_status( + "nangate45", + "ibex", + { + "exit_code": 124, + "timed_out": True, + "elapsed_minutes": 240.2, + "limit_minutes": 240, + }, + logs={"3_3_place_gp.log": "placing\n", "5_1_grt.log": "routing\n"}, + ) + result, messages = self.run_script("--jobName", "ORFS") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + designs = self.designs(messages) + self.assertEqual( + designs["ibex"], + { + "platform": "nangate45", + "design": "ibex", + "variant": "base", + "outcome": { + "status": "timeout", + "stage": "grt", + "elapsed_minutes": 240.2, + "limit_minutes": 240, + }, + }, + ) + self.assertNotIn("outcome", designs["gcd"]) + self.assertIn("reporting its timeout outcome only", result.stdout) + + def test_failed_design_reports_stage_and_first_error_line(self): + self.add_flow_status( + "asap7", + "aes", + {"exit_code": 2, "timed_out": False}, + logs={ + "4_1_cts.log": "[ERROR CTS-0001] not this one\n", + "5_1_grt.log": "info\n[ERROR GRT-0116] Global routing failed.\n" + "[ERROR GRT-0001] later\n", + }, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["aes"]["outcome"] + self.assertEqual(outcome["status"], "error") + self.assertEqual(outcome["stage"], "grt") + self.assertEqual(outcome["message"], "[ERROR GRT-0116] Global routing failed.") + + def test_yosys_error_line_is_reported(self): + self.add_flow_status( + "asap7", + "aes", + {"exit_code": 2}, + logs={"1_2_yosys.log": "ERROR: Module `foo' referenced but not found.\n"}, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["aes"]["outcome"] + self.assertEqual((outcome["status"], outcome["stage"]), ("error", "synth")) + self.assertEqual( + outcome["message"], "ERROR: Module `foo' referenced but not found." + ) + + def test_signal_in_last_stage_log_is_a_crash(self): + self.add_flow_status( + "sky130hd", + "jpeg", + {"exit_code": 2}, + logs={"3_3_place_gp.log": "Signal 11 received\nStack trace:\n"}, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["jpeg"]["outcome"] + self.assertEqual((outcome["status"], outcome["stage"]), ("crash", "place")) + self.assertNotIn("message", outcome) + + def test_passing_design_keeps_its_metrics_and_reports_passed(self): + self.add_design("nangate45", "gcd", metrics={"finish:area": 5.0}) + self.add_flow_status( + "nangate45", "gcd", {"exit_code": 0, "elapsed_minutes": 3.5} + ) + _, messages = self.run_script() + record = self.designs(messages)["gcd"] + self.assertEqual(record["metrics"], {"finish__area": 5.0}) + self.assertEqual( + record["outcome"], {"status": "passed", "elapsed_minutes": 3.5} + ) + + def test_long_error_line_is_truncated(self): + self.add_flow_status( + "asap7", + "aes", + {"exit_code": 2}, + logs={"5_2_route.log": "[ERROR DRT-0001] " + "x" * 2000 + "\n"}, + ) + _, messages = self.run_script() + outcome = self.designs(messages)["aes"]["outcome"] + self.assertEqual(outcome["stage"], "route") + self.assertEqual(len(outcome["message"]), 1024) + + def test_unreadable_status_without_metrics_skips_the_design(self): + self.add_design("nangate45", "gcd") + self.add_flow_status("nangate45", "ibex", "not json") + self.add_flow_status("nangate45", "jpeg", {"timed_out": False}) + result, messages = self.run_script() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(set(self.designs(messages)), {"gcd"}) + self.assertIn("Reporting no outcome", result.stdout) + self.assertIn("[WARN] No data file for nangate45 ibex base.", result.stdout) + + def test_outcome_only_design_names_itself_on_the_v1_fallback(self): + self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS) + self.add_flow_status( + "nangate45", + "ibex", + {"exit_code": 124, "timed_out": True, "limit_minutes": 60}, + ) + result, messages = self.run_script("--buildTime", "2026-10-05 12:34") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + by_design = {m["data"].get("run__flow__design"): m["data"] for m in messages} + ibex = by_design["ibex"] + self.assertEqual(ibex["run__flow__platform"], "nangate45") + self.assertEqual(ibex["run__flow__variant"], "base") + self.assertEqual(ibex["run__flow__generate_date"], "2026-10-05 12:34") + self.assertEqual(ibex["outcome"], {"status": "timeout", "limit_minutes": 60}) + self.assertNotIn("outcome", by_design.get(None, {})) + if __name__ == "__main__": unittest.main() diff --git a/flow/util/uploadMetadata.py b/flow/util/uploadMetadata.py index 5aff02839a..4a7419d4a8 100755 --- a/flow/util/uploadMetadata.py +++ b/flow/util/uploadMetadata.py @@ -3,6 +3,7 @@ import json import argparse import os +import re import sys from datetime import datetime, timezone @@ -160,17 +161,154 @@ def _non_negative_int(val): MAX_PUBSUB_BYTES = 8 * 1024 * 1024 -def build_design_record(dataFile, platform, design, variant): - """Return a dict for one design to be included in the pipeline-level payload.""" - with open(dataFile) as f: - data = json.load(f) - metrics = {k.replace(":", "__"): v for k, v in data.items()} +def build_design_record(dataFile, platform, design, variant, outcome=None): + """Return a dict for one design to be included in the pipeline-level payload. + + ``dataFile`` is the design's metadata.json, or ``None`` for a run that ended + before writing one; such a record carries only its ``outcome``. + """ + record = {"platform": platform, "design": design, "variant": variant} + if dataFile: + with open(dataFile) as f: + data = json.load(f) + record["metrics"] = {k.replace(":", "__"): v for k, v in data.items()} + if outcome: + record["outcome"] = outcome + return record + + +# How a flow run ended, as the CI job records it next to the design's reports: +# {"exit_code": int, "timed_out": bool, "elapsed_minutes": float, +# "limit_minutes": int}. Every key is optional. +STATUS_FILENAME = "flow-status.json" +MAX_OUTCOME_MESSAGE = 1024 +# Stage log names start with the step number: 1_2_yosys.log, 5_1_grt.log. +STAGE_LOG = re.compile(r"^(\d+)_(?:(\d+)_)?.*\.log$") +# The first matching line of a failing stage's log is the error reported. The +# OpenROAD form carries the tool's message ID; Yosys prints "ERROR: ". +ERROR_LINE = re.compile( + r"\[(?:ERROR|CRITICAL|FATAL) [A-Z][A-Z0-9]*-\d+\].*|^ERROR: .*|^Error: .*" +) +# What a tool that died on a signal leaves in its log, as opposed to an error it +# reported itself. +CRASH_LINE = re.compile(r"Signal \d+ received|Segmentation fault|core dumped") + + +def _stage_name(log_name): + """The flow stage a stage log belongs to, e.g. 5_1_grt.log -> "grt".""" + match = STAGE_LOG.match(log_name) + if not match: + return None + step, sub = match.group(1), match.group(2) + if step == "5": + return "grt" if sub == "1" else "route" return { - "platform": platform, - "design": design, - "variant": variant, - "metrics": metrics, - } + "1": "synth", + "2": "floorplan", + "3": "place", + "4": "cts", + "6": "finish", + }.get(step) + + +def _stage_logs(log_dir): + """The design's stage logs, last step first.""" + try: + names = os.listdir(log_dir) + except OSError: + return [] + + def key(name): + match = STAGE_LOG.match(name) + return (int(match.group(1)), int(match.group(2) or 0), name) + + return sorted((n for n in names if STAGE_LOG.match(n)), key=key, reverse=True) + + +def _scan_log(path): + """The first error line of a log and whether it shows a crash.""" + error_line = None + crashed = False + try: + with open(path, errors="replace") as f: + for line in f: + if error_line is None: + match = ERROR_LINE.search(line) + if match: + error_line = match.group(0).strip() + if not crashed and CRASH_LINE.search(line): + crashed = True + if error_line is not None and crashed: + break + except OSError: + pass + return error_line, crashed + + +def read_flow_outcome(report_dir, log_dir): + """The design's outcome from its flow-status.json and stage logs, or ``None``. + + ``status`` is ``timeout`` when the time limit stopped the run, ``passed`` on + exit code 0, ``crash`` when the last stage log shows the tool died on a + signal, and ``error`` otherwise. A failed run also names the stage it ended + in and the first error line found in its logs, last stage first. + """ + path = os.path.join(report_dir, STATUS_FILENAME) + if not os.path.exists(path): + return None + try: + with open(path) as f: + status = json.load(f) + except (OSError, ValueError) as e: + print(f"[WARN] Could not read {path}: {e}. Reporting no outcome.") + return None + if not isinstance(status, dict): + print(f"[WARN] {path} does not hold a JSON object. Reporting no outcome.") + return None + + exit_code = status.get("exit_code") + timed_out = status.get("timed_out") is True + if not timed_out and not isinstance(exit_code, int): + print(f"[WARN] {path} has no exit_code. Reporting no outcome.") + return None + + outcome = {} + for key in ("elapsed_minutes", "limit_minutes"): + value = status.get(key) + if ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and value >= 0 + ): + outcome[key] = value + + if not timed_out and exit_code == 0: + outcome["status"] = "passed" + return outcome + + logs = _stage_logs(log_dir) + if logs: + stage = _stage_name(logs[0]) + if stage: + outcome["stage"] = stage + + message = None + crashed = False + for i, name in enumerate(logs): + error_line, log_crashed = _scan_log(os.path.join(log_dir, name)) + if i == 0: + crashed = log_crashed + if error_line: + message = error_line + break + if message: + outcome["message"] = message[:MAX_OUTCOME_MESSAGE] + + if timed_out: + outcome["status"] = "timeout" + else: + outcome["status"] = "crash" if crashed else "error" + return outcome def _submodule_defect(entry): @@ -426,6 +564,10 @@ def publish_v1_per_design( The requester keys go on after the metrics, so a metric that happened to share their name could not replace who asked for the build. + + A design's outcome rides at the root as ``outcome``. A design that has only + an outcome carries no metrics to name it, so its message names the design + with the same run__flow__ keys metadata.json would hold, dated by the build. """ futures = [] failed = 0 @@ -441,7 +583,19 @@ def publish_v1_per_design( } if provenance: payload.update(provenance) - payload.update(d["metrics"]) + if "metrics" in d: + payload.update(d["metrics"]) + else: + payload.update( + { + "run__flow__platform": d["platform"], + "run__flow__design": d["design"], + "run__flow__variant": d["variant"], + "run__flow__generate_date": resolve_build_time(args), + } + ) + if "outcome" in d: + payload["outcome"] = d["outcome"] if requester: payload.update(requester) @@ -513,16 +667,28 @@ def publish_v1_per_design( platform = dirList[1] design = dirList[2] variant = dirList[3] + if platform == "sky130hd_fakestack" or platform == "src": + print(f"[WARN] Skiping upload {platform} {design} {variant}.") + continue dataFile = os.path.join(reportDir, RUN_FILENAME) if not os.path.exists(dataFile): + dataFile = None + outcome = read_flow_outcome( + reportDir, os.path.join("logs", platform, design, variant) + ) + if dataFile is None and outcome is None: print(f"[WARN] No data file for {platform} {design} {variant}.") continue - if platform == "sky130hd_fakestack" or platform == "src": - print(f"[WARN] Skiping upload {platform} {design} {variant}.") - continue + if dataFile is None: + print( + f"[INFO] No data file for {platform} {design} {variant}; " + f"reporting its {outcome['status']} outcome only." + ) # --- PUBSUB --- if publisher: - design_records.append(build_design_record(dataFile, platform, design, variant)) + design_records.append( + build_design_record(dataFile, platform, design, variant, outcome) + ) # --- END PUBSUB --- # --- PUBSUB --- From 85bcd28e4f5d4c889ddba88fb3a53a6bbc391c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Harboe?= Date: Sun, 8 Nov 2026 08:00:06 +0100 Subject: [PATCH 2/6] cts.tcl: drop CTS_SNAPSHOTS and save_progress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CTS_SNAPSHOTS wrote the design before repair_timing so it could be opened in the GUI, and save_progress printed the make target to type to open it. Debugging today is ad-hoc edits in a fast loop, where a checkpoint is one orfs_write_db line added and removed again, so the script need not carry it. No CI design sets the variable, which the Makefile contract rules out for a default-off toggle, and with SKIP_CTS_REPAIR_TIMING=1 4_1_cts.odb is the same pre-repair state. The failure path still writes 4_1_error.odb and .sdc, now with orfs_write_db and orfs_write_sdc directly. Nothing else changes. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Øyvind Harboe --- docs/contrib/EndpointSlackTracking.md | 4 ++-- docs/user/FlowVariables.md | 2 -- flow/scripts/cts.tcl | 17 ++++------------- flow/scripts/variables.json | 7 ------- flow/scripts/variables.yaml | 6 ------ 5 files changed, 6 insertions(+), 30 deletions(-) diff --git a/docs/contrib/EndpointSlackTracking.md b/docs/contrib/EndpointSlackTracking.md index 1bab138900..6af5736e43 100644 --- a/docs/contrib/EndpointSlackTracking.md +++ b/docs/contrib/EndpointSlackTracking.md @@ -24,8 +24,8 @@ Every step then writes `$(REPORTS_DIR)/_endpoint_slack.json`, e.g. It works in the single-process `flow/scripts/flow.tcl` flow as well. Steps that just copy their input (`orfs_copy_db`, e.g. `5_3_fillcell` without fill cells) re-time the design if it is loaded, or else copy the previous step's -snapshot. Intermediate `.odb` files written through `orfs_write_db` (e.g. CTS -`save_progress`) also get a snapshot; `*-failed` ones don't. +snapshot. Intermediate `.odb` files written through `orfs_write_db` (e.g. CTS's +`4_1_error.odb`) also get a snapshot; `*-failed` ones don't. A placed step is timed with the parasitics OpenROAD has for it: a bare `estimate_parasitics` estimates from the global routes when there are any, diff --git a/docs/user/FlowVariables.md b/docs/user/FlowVariables.md index 0df148d71d..6e415790b8 100644 --- a/docs/user/FlowVariables.md +++ b/docs/user/FlowVariables.md @@ -130,7 +130,6 @@ configuration file. | CTS_CLUSTER_DIAMETER| Maximum diameter (in microns) of sink cluster.| | | CTS_CLUSTER_SIZE| Maximum number of sinks per cluster.| | | CTS_LIB_NAME| Name of the Liberty library to use in selecting the clock buffers.| | -| CTS_SNAPSHOTS| Create ODB/SDC files at different stages of CTS.| 0| | DESIGN_NAME| The name of the top-level module of the design.| | | DESIGN_NICKNAME| DESIGN_NICKNAME just changes the directory name that ORFS outputs to be DESIGN_NICKNAME instead of DESIGN_NAME in case DESIGN_NAME is unwieldy or conflicts with a different design.| | | DETAILED_METRICS| If set, then calls report_metrics prior to repair operations in the CTS and global route stages| 0| @@ -542,7 +541,6 @@ configuration file. - [CTS_CLUSTER_DIAMETER](#CTS_CLUSTER_DIAMETER) - [CTS_CLUSTER_SIZE](#CTS_CLUSTER_SIZE) - [CTS_LIB_NAME](#CTS_LIB_NAME) -- [CTS_SNAPSHOTS](#CTS_SNAPSHOTS) - [DETAILED_METRICS](#DETAILED_METRICS) - [DETAIL_PLACEMENT_ARGS](#DETAIL_PLACEMENT_ARGS) - [HOLD_SLACK_MARGIN](#HOLD_SLACK_MARGIN) diff --git a/flow/scripts/cts.tcl b/flow/scripts/cts.tcl index 7439569d12..22cee47e16 100644 --- a/flow/scripts/cts.tcl +++ b/flow/scripts/cts.tcl @@ -9,12 +9,6 @@ source_step_tcl PRE CTS # so cts does not try to buffer the inverted clocks. repair_clock_inverters -proc save_progress { stage } { - puts "Run 'make gui_$stage.odb' to load progress snapshot" - orfs_write_db $::env(RESULTS_DIR)/$stage.odb - orfs_write_sdc $::env(RESULTS_DIR)/$stage.sdc -} - # Run CTS set cts_args [list \ -sink_clustering_enable \ @@ -26,7 +20,6 @@ append_env_var cts_args CTS_CLUSTER_DIAMETER -sink_clustering_max_diameter 1 append_env_var cts_args CTS_BUF_LIST -buf_list 1 append_env_var cts_args CTS_LIB_NAME -library 1 - if { [env_var_exists_and_non_empty CTS_ARGS] } { set cts_args $::env(CTS_ARGS) } @@ -48,16 +41,13 @@ set_placement_padding -global \ set result [catch { detailed_placement_helper -quiet } msg] if { $result != 0 } { - save_progress 4_1_error + orfs_write_db $::env(RESULTS_DIR)/4_1_error.odb + orfs_write_sdc $::env(RESULTS_DIR)/4_1_error.sdc error "Detailed placement failed in CTS: $msg" } log_cmd estimate_parasitics -if { $::env(CTS_SNAPSHOTS) } { - save_progress 4_1_pre_repair_setup_hold -} - if { !$::env(SKIP_CTS_REPAIR_TIMING) } { set lec_enabled [lec_check_enabled] if { $lec_enabled } { @@ -73,7 +63,8 @@ if { !$::env(SKIP_CTS_REPAIR_TIMING) } { set result [catch { detailed_placement_helper -quiet } msg] if { $result != 0 } { - save_progress 4_1_error + orfs_write_db $::env(RESULTS_DIR)/4_1_error.odb + orfs_write_sdc $::env(RESULTS_DIR)/4_1_error.sdc error "Detailed placement failed in CTS: $msg" } diff --git a/flow/scripts/variables.json b/flow/scripts/variables.json index 8bebf70c48..81fadc562f 100644 --- a/flow/scripts/variables.json +++ b/flow/scripts/variables.json @@ -250,13 +250,6 @@ "cts" ] }, - "CTS_SNAPSHOTS": { - "default": 0, - "description": "Create ODB/SDC files at different stages of CTS.\n", - "stages": [ - "cts" - ] - }, "DESIGN_NAME": { "description": "The name of the top-level module of the design.\n", "required": true diff --git a/flow/scripts/variables.yaml b/flow/scripts/variables.yaml index ac9d582494..8c1fb94727 100644 --- a/flow/scripts/variables.yaml +++ b/flow/scripts/variables.yaml @@ -822,12 +822,6 @@ CTS_ARGS: Override `clock_tree_synthesis` arguments. stages: - cts -CTS_SNAPSHOTS: - description: | - Create ODB/SDC files at different stages of CTS. - stages: - - cts - default: 0 HOLD_SLACK_MARGIN: description: > Specifies a time margin for the slack when fixing hold violations. From d10ca2f6bd7bdda60d09ad67a75aafda7dd29b67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Harboe?= Date: Sun, 8 Nov 2026 08:04:16 +0100 Subject: [PATCH 3/6] cts.tcl: leave variables.json to its own removal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit variables.json is no longer read since #4800, so it is deleted in a pull request of its own rather than kept in step here. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Øyvind Harboe --- flow/scripts/variables.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flow/scripts/variables.json b/flow/scripts/variables.json index 81fadc562f..8bebf70c48 100644 --- a/flow/scripts/variables.json +++ b/flow/scripts/variables.json @@ -250,6 +250,13 @@ "cts" ] }, + "CTS_SNAPSHOTS": { + "default": 0, + "description": "Create ODB/SDC files at different stages of CTS.\n", + "stages": [ + "cts" + ] + }, "DESIGN_NAME": { "description": "The name of the top-level module of the design.\n", "required": true From 07649bc7490283d1b7fb329a6fcdba25402a1a9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Harboe?= Date: Sun, 8 Nov 2026 08:05:12 +0100 Subject: [PATCH 4/6] cts.tcl: estimate pre-repair parasitics only for the detailed metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The estimate before detailed placement was added in 2020 for a repair_clock_nets call that the script then made itself. clock_tree_synthesis -repair_clock_nets does that repair now, and the estimate after detailed placement replaces this one before anything is timed, so it only feeds the DETAILED_METRICS report. It moves inside that if, with the metrics stage it is reported under. On asap7 gcd, 4_1_cts.odb is byte-identical before and after, with DETAILED_METRICS=0 and =1, and so are the 4_1_cts.json metrics apart from runtimes. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Øyvind Harboe --- flow/scripts/cts.tcl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flow/scripts/cts.tcl b/flow/scripts/cts.tcl index 22cee47e16..0a42b47964 100644 --- a/flow/scripts/cts.tcl +++ b/flow/scripts/cts.tcl @@ -28,12 +28,12 @@ set_dont_use $::env(DONT_USE_CELLS) log_cmd clock_tree_synthesis {*}$cts_args -utl::push_metrics_stage "cts__{}__pre_repair_timing" -log_cmd estimate_parasitics if { $::env(DETAILED_METRICS) } { + utl::push_metrics_stage "cts__{}__pre_repair_timing" + log_cmd estimate_parasitics report_metrics 4 "cts pre-repair-timing" + utl::pop_metrics_stage } -utl::pop_metrics_stage set_placement_padding -global \ -left $::env(CELL_PAD_IN_SITES_DETAIL_PLACEMENT) \ From 5d81dd195daf1aa1a6dd3e59e9a32d3be4306b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Harboe?= Date: Wed, 11 Nov 2026 05:14:27 +0100 Subject: [PATCH 5/6] cts.tcl: load_design takes the .odb alone, as on master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge of master took this branch's cts.tcl whole and brought back two lines master had removed: load_design's second argument (it now takes only the .odb, its .sdc travels with it) and the trailing orfs_write_sdc (orfs_write_db writes the .sdc next to every .odb). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Øyvind Harboe --- flow/scripts/cts.tcl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flow/scripts/cts.tcl b/flow/scripts/cts.tcl index 67b6bb82e0..1010cedeed 100644 --- a/flow/scripts/cts.tcl +++ b/flow/scripts/cts.tcl @@ -2,7 +2,7 @@ utl::set_metrics_stage "cts__{}" source $::env(SCRIPTS_DIR)/load.tcl source $::env(SCRIPTS_DIR)/formal_check.tcl erase_non_stage_variables cts -load_design 3_place.odb 3_place.sdc +load_design 3_place.odb source_step_tcl PRE CTS # Clone clock tree inverters next to register loads @@ -76,4 +76,3 @@ report_metrics 4 "cts final" source_step_tcl POST CTS orfs_write_db $::env(RESULTS_DIR)/4_1_cts.odb -orfs_write_sdc $::env(RESULTS_DIR)/4_cts.sdc From e54d0af42372a3034e4101aaa6b53314c48cfc4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Harboe?= Date: Wed, 11 Nov 2026 05:14:32 +0100 Subject: [PATCH 6/6] Leave the metadata upload outcome change to its own pull request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit uploadMetadata.py and its test came onto this branch with a merge; the change is unrelated to cts.tcl and is under review on its own as #4819. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Øyvind Harboe --- flow/test/test_uploadMetadata.py | 144 ----------------------- flow/util/uploadMetadata.py | 196 +++---------------------------- 2 files changed, 15 insertions(+), 325 deletions(-) diff --git a/flow/test/test_uploadMetadata.py b/flow/test/test_uploadMetadata.py index be94f63757..92555ed5a5 100755 --- a/flow/test/test_uploadMetadata.py +++ b/flow/test/test_uploadMetadata.py @@ -102,23 +102,6 @@ def add_design(self, platform, design, variant="base", metrics=None): with open(os.path.join(report_dir, "metadata.json"), "w") as f: json.dump(metrics or {"finish:area": 1.0}, f) - def add_flow_status(self, platform, design, status, variant="base", logs=None): - """Record how a design's flow run ended, as the CI job does, with stage - logs given as {file name: content}.""" - report_dir = os.path.join(self.flow, "reports", platform, design, variant) - os.makedirs(report_dir, exist_ok=True) - with open(os.path.join(report_dir, "flow-status.json"), "w") as f: - f.write(status if isinstance(status, str) else json.dumps(status)) - log_dir = os.path.join(self.flow, "logs", platform, design, variant) - os.makedirs(log_dir, exist_ok=True) - for name, content in (logs or {}).items(): - with open(os.path.join(log_dir, name), "w") as f: - f.write(content) - - def designs(self, messages): - """The pipeline payload's design records, by design name.""" - return {d["design"]: d for d in messages[0]["data"]["designs"]} - def run_script(self, *extra, fail=False, tz=None, extra_env=None): env = dict(os.environ) for name in REQUESTER_ENV: @@ -554,133 +537,6 @@ def test_no_publisher_publishes_nothing(self): self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertFalse(os.path.exists(self.out)) - def test_timed_out_design_without_metrics_reports_its_outcome(self): - self.add_design("nangate45", "gcd") - self.add_flow_status( - "nangate45", - "ibex", - { - "exit_code": 124, - "timed_out": True, - "elapsed_minutes": 240.2, - "limit_minutes": 240, - }, - logs={"3_3_place_gp.log": "placing\n", "5_1_grt.log": "routing\n"}, - ) - result, messages = self.run_script("--jobName", "ORFS") - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - designs = self.designs(messages) - self.assertEqual( - designs["ibex"], - { - "platform": "nangate45", - "design": "ibex", - "variant": "base", - "outcome": { - "status": "timeout", - "stage": "grt", - "elapsed_minutes": 240.2, - "limit_minutes": 240, - }, - }, - ) - self.assertNotIn("outcome", designs["gcd"]) - self.assertIn("reporting its timeout outcome only", result.stdout) - - def test_failed_design_reports_stage_and_first_error_line(self): - self.add_flow_status( - "asap7", - "aes", - {"exit_code": 2, "timed_out": False}, - logs={ - "4_1_cts.log": "[ERROR CTS-0001] not this one\n", - "5_1_grt.log": "info\n[ERROR GRT-0116] Global routing failed.\n" - "[ERROR GRT-0001] later\n", - }, - ) - _, messages = self.run_script() - outcome = self.designs(messages)["aes"]["outcome"] - self.assertEqual(outcome["status"], "error") - self.assertEqual(outcome["stage"], "grt") - self.assertEqual(outcome["message"], "[ERROR GRT-0116] Global routing failed.") - - def test_yosys_error_line_is_reported(self): - self.add_flow_status( - "asap7", - "aes", - {"exit_code": 2}, - logs={"1_2_yosys.log": "ERROR: Module `foo' referenced but not found.\n"}, - ) - _, messages = self.run_script() - outcome = self.designs(messages)["aes"]["outcome"] - self.assertEqual((outcome["status"], outcome["stage"]), ("error", "synth")) - self.assertEqual( - outcome["message"], "ERROR: Module `foo' referenced but not found." - ) - - def test_signal_in_last_stage_log_is_a_crash(self): - self.add_flow_status( - "sky130hd", - "jpeg", - {"exit_code": 2}, - logs={"3_3_place_gp.log": "Signal 11 received\nStack trace:\n"}, - ) - _, messages = self.run_script() - outcome = self.designs(messages)["jpeg"]["outcome"] - self.assertEqual((outcome["status"], outcome["stage"]), ("crash", "place")) - self.assertNotIn("message", outcome) - - def test_passing_design_keeps_its_metrics_and_reports_passed(self): - self.add_design("nangate45", "gcd", metrics={"finish:area": 5.0}) - self.add_flow_status( - "nangate45", "gcd", {"exit_code": 0, "elapsed_minutes": 3.5} - ) - _, messages = self.run_script() - record = self.designs(messages)["gcd"] - self.assertEqual(record["metrics"], {"finish__area": 5.0}) - self.assertEqual( - record["outcome"], {"status": "passed", "elapsed_minutes": 3.5} - ) - - def test_long_error_line_is_truncated(self): - self.add_flow_status( - "asap7", - "aes", - {"exit_code": 2}, - logs={"5_2_route.log": "[ERROR DRT-0001] " + "x" * 2000 + "\n"}, - ) - _, messages = self.run_script() - outcome = self.designs(messages)["aes"]["outcome"] - self.assertEqual(outcome["stage"], "route") - self.assertEqual(len(outcome["message"]), 1024) - - def test_unreadable_status_without_metrics_skips_the_design(self): - self.add_design("nangate45", "gcd") - self.add_flow_status("nangate45", "ibex", "not json") - self.add_flow_status("nangate45", "jpeg", {"timed_out": False}) - result, messages = self.run_script() - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertEqual(set(self.designs(messages)), {"gcd"}) - self.assertIn("Reporting no outcome", result.stdout) - self.assertIn("[WARN] No data file for nangate45 ibex base.", result.stdout) - - def test_outcome_only_design_names_itself_on_the_v1_fallback(self): - self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS) - self.add_flow_status( - "nangate45", - "ibex", - {"exit_code": 124, "timed_out": True, "limit_minutes": 60}, - ) - result, messages = self.run_script("--buildTime", "2026-10-05 12:34") - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - by_design = {m["data"].get("run__flow__design"): m["data"] for m in messages} - ibex = by_design["ibex"] - self.assertEqual(ibex["run__flow__platform"], "nangate45") - self.assertEqual(ibex["run__flow__variant"], "base") - self.assertEqual(ibex["run__flow__generate_date"], "2026-10-05 12:34") - self.assertEqual(ibex["outcome"], {"status": "timeout", "limit_minutes": 60}) - self.assertNotIn("outcome", by_design.get(None, {})) - if __name__ == "__main__": unittest.main() diff --git a/flow/util/uploadMetadata.py b/flow/util/uploadMetadata.py index 4a7419d4a8..5aff02839a 100755 --- a/flow/util/uploadMetadata.py +++ b/flow/util/uploadMetadata.py @@ -3,7 +3,6 @@ import json import argparse import os -import re import sys from datetime import datetime, timezone @@ -161,154 +160,17 @@ def _non_negative_int(val): MAX_PUBSUB_BYTES = 8 * 1024 * 1024 -def build_design_record(dataFile, platform, design, variant, outcome=None): - """Return a dict for one design to be included in the pipeline-level payload. - - ``dataFile`` is the design's metadata.json, or ``None`` for a run that ended - before writing one; such a record carries only its ``outcome``. - """ - record = {"platform": platform, "design": design, "variant": variant} - if dataFile: - with open(dataFile) as f: - data = json.load(f) - record["metrics"] = {k.replace(":", "__"): v for k, v in data.items()} - if outcome: - record["outcome"] = outcome - return record - - -# How a flow run ended, as the CI job records it next to the design's reports: -# {"exit_code": int, "timed_out": bool, "elapsed_minutes": float, -# "limit_minutes": int}. Every key is optional. -STATUS_FILENAME = "flow-status.json" -MAX_OUTCOME_MESSAGE = 1024 -# Stage log names start with the step number: 1_2_yosys.log, 5_1_grt.log. -STAGE_LOG = re.compile(r"^(\d+)_(?:(\d+)_)?.*\.log$") -# The first matching line of a failing stage's log is the error reported. The -# OpenROAD form carries the tool's message ID; Yosys prints "ERROR: ". -ERROR_LINE = re.compile( - r"\[(?:ERROR|CRITICAL|FATAL) [A-Z][A-Z0-9]*-\d+\].*|^ERROR: .*|^Error: .*" -) -# What a tool that died on a signal leaves in its log, as opposed to an error it -# reported itself. -CRASH_LINE = re.compile(r"Signal \d+ received|Segmentation fault|core dumped") - - -def _stage_name(log_name): - """The flow stage a stage log belongs to, e.g. 5_1_grt.log -> "grt".""" - match = STAGE_LOG.match(log_name) - if not match: - return None - step, sub = match.group(1), match.group(2) - if step == "5": - return "grt" if sub == "1" else "route" +def build_design_record(dataFile, platform, design, variant): + """Return a dict for one design to be included in the pipeline-level payload.""" + with open(dataFile) as f: + data = json.load(f) + metrics = {k.replace(":", "__"): v for k, v in data.items()} return { - "1": "synth", - "2": "floorplan", - "3": "place", - "4": "cts", - "6": "finish", - }.get(step) - - -def _stage_logs(log_dir): - """The design's stage logs, last step first.""" - try: - names = os.listdir(log_dir) - except OSError: - return [] - - def key(name): - match = STAGE_LOG.match(name) - return (int(match.group(1)), int(match.group(2) or 0), name) - - return sorted((n for n in names if STAGE_LOG.match(n)), key=key, reverse=True) - - -def _scan_log(path): - """The first error line of a log and whether it shows a crash.""" - error_line = None - crashed = False - try: - with open(path, errors="replace") as f: - for line in f: - if error_line is None: - match = ERROR_LINE.search(line) - if match: - error_line = match.group(0).strip() - if not crashed and CRASH_LINE.search(line): - crashed = True - if error_line is not None and crashed: - break - except OSError: - pass - return error_line, crashed - - -def read_flow_outcome(report_dir, log_dir): - """The design's outcome from its flow-status.json and stage logs, or ``None``. - - ``status`` is ``timeout`` when the time limit stopped the run, ``passed`` on - exit code 0, ``crash`` when the last stage log shows the tool died on a - signal, and ``error`` otherwise. A failed run also names the stage it ended - in and the first error line found in its logs, last stage first. - """ - path = os.path.join(report_dir, STATUS_FILENAME) - if not os.path.exists(path): - return None - try: - with open(path) as f: - status = json.load(f) - except (OSError, ValueError) as e: - print(f"[WARN] Could not read {path}: {e}. Reporting no outcome.") - return None - if not isinstance(status, dict): - print(f"[WARN] {path} does not hold a JSON object. Reporting no outcome.") - return None - - exit_code = status.get("exit_code") - timed_out = status.get("timed_out") is True - if not timed_out and not isinstance(exit_code, int): - print(f"[WARN] {path} has no exit_code. Reporting no outcome.") - return None - - outcome = {} - for key in ("elapsed_minutes", "limit_minutes"): - value = status.get(key) - if ( - isinstance(value, (int, float)) - and not isinstance(value, bool) - and value >= 0 - ): - outcome[key] = value - - if not timed_out and exit_code == 0: - outcome["status"] = "passed" - return outcome - - logs = _stage_logs(log_dir) - if logs: - stage = _stage_name(logs[0]) - if stage: - outcome["stage"] = stage - - message = None - crashed = False - for i, name in enumerate(logs): - error_line, log_crashed = _scan_log(os.path.join(log_dir, name)) - if i == 0: - crashed = log_crashed - if error_line: - message = error_line - break - if message: - outcome["message"] = message[:MAX_OUTCOME_MESSAGE] - - if timed_out: - outcome["status"] = "timeout" - else: - outcome["status"] = "crash" if crashed else "error" - return outcome + "platform": platform, + "design": design, + "variant": variant, + "metrics": metrics, + } def _submodule_defect(entry): @@ -564,10 +426,6 @@ def publish_v1_per_design( The requester keys go on after the metrics, so a metric that happened to share their name could not replace who asked for the build. - - A design's outcome rides at the root as ``outcome``. A design that has only - an outcome carries no metrics to name it, so its message names the design - with the same run__flow__ keys metadata.json would hold, dated by the build. """ futures = [] failed = 0 @@ -583,19 +441,7 @@ def publish_v1_per_design( } if provenance: payload.update(provenance) - if "metrics" in d: - payload.update(d["metrics"]) - else: - payload.update( - { - "run__flow__platform": d["platform"], - "run__flow__design": d["design"], - "run__flow__variant": d["variant"], - "run__flow__generate_date": resolve_build_time(args), - } - ) - if "outcome" in d: - payload["outcome"] = d["outcome"] + payload.update(d["metrics"]) if requester: payload.update(requester) @@ -667,28 +513,16 @@ def publish_v1_per_design( platform = dirList[1] design = dirList[2] variant = dirList[3] - if platform == "sky130hd_fakestack" or platform == "src": - print(f"[WARN] Skiping upload {platform} {design} {variant}.") - continue dataFile = os.path.join(reportDir, RUN_FILENAME) if not os.path.exists(dataFile): - dataFile = None - outcome = read_flow_outcome( - reportDir, os.path.join("logs", platform, design, variant) - ) - if dataFile is None and outcome is None: print(f"[WARN] No data file for {platform} {design} {variant}.") continue - if dataFile is None: - print( - f"[INFO] No data file for {platform} {design} {variant}; " - f"reporting its {outcome['status']} outcome only." - ) + if platform == "sky130hd_fakestack" or platform == "src": + print(f"[WARN] Skiping upload {platform} {design} {variant}.") + continue # --- PUBSUB --- if publisher: - design_records.append( - build_design_record(dataFile, platform, design, variant, outcome) - ) + design_records.append(build_design_record(dataFile, platform, design, variant)) # --- END PUBSUB --- # --- PUBSUB ---