Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 144 additions & 0 deletions flow/test/test_uploadMetadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,23 @@ def add_design(self, platform, design, variant="base", metrics=None):
with open(os.path.join(report_dir, "metadata.json"), "w") as f:
json.dump(metrics or {"finish:area": 1.0}, f)

def add_flow_status(self, platform, design, status, variant="base", logs=None):
"""Record how a design's flow run ended, as the CI job does, with stage
logs given as {file name: content}."""
report_dir = os.path.join(self.flow, "reports", platform, design, variant)
os.makedirs(report_dir, exist_ok=True)
with open(os.path.join(report_dir, "flow-status.json"), "w") as f:
f.write(status if isinstance(status, str) else json.dumps(status))
log_dir = os.path.join(self.flow, "logs", platform, design, variant)
os.makedirs(log_dir, exist_ok=True)
for name, content in (logs or {}).items():
with open(os.path.join(log_dir, name), "w") as f:
f.write(content)

def designs(self, messages):
"""The pipeline payload's design records, by design name."""
return {d["design"]: d for d in messages[0]["data"]["designs"]}

def run_script(self, *extra, fail=False, tz=None, extra_env=None):
env = dict(os.environ)
for name in REQUESTER_ENV:
Expand Down Expand Up @@ -537,6 +554,133 @@ def test_no_publisher_publishes_nothing(self):
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertFalse(os.path.exists(self.out))

def test_timed_out_design_without_metrics_reports_its_outcome(self):
self.add_design("nangate45", "gcd")
self.add_flow_status(
"nangate45",
"ibex",
{
"exit_code": 124,
"timed_out": True,
"elapsed_minutes": 240.2,
"limit_minutes": 240,
},
logs={"3_3_place_gp.log": "placing\n", "5_1_grt.log": "routing\n"},
)
result, messages = self.run_script("--jobName", "ORFS")
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
designs = self.designs(messages)
self.assertEqual(
designs["ibex"],
{
"platform": "nangate45",
"design": "ibex",
"variant": "base",
"outcome": {
"status": "timeout",
"stage": "grt",
"elapsed_minutes": 240.2,
"limit_minutes": 240,
},
},
)
self.assertNotIn("outcome", designs["gcd"])
self.assertIn("reporting its timeout outcome only", result.stdout)

def test_failed_design_reports_stage_and_first_error_line(self):
self.add_flow_status(
"asap7",
"aes",
{"exit_code": 2, "timed_out": False},
logs={
"4_1_cts.log": "[ERROR CTS-0001] not this one\n",
"5_1_grt.log": "info\n[ERROR GRT-0116] Global routing failed.\n"
"[ERROR GRT-0001] later\n",
},
)
_, messages = self.run_script()
outcome = self.designs(messages)["aes"]["outcome"]
self.assertEqual(outcome["status"], "error")
self.assertEqual(outcome["stage"], "grt")
self.assertEqual(outcome["message"], "[ERROR GRT-0116] Global routing failed.")

def test_yosys_error_line_is_reported(self):
self.add_flow_status(
"asap7",
"aes",
{"exit_code": 2},
logs={"1_2_yosys.log": "ERROR: Module `foo' referenced but not found.\n"},
)
_, messages = self.run_script()
outcome = self.designs(messages)["aes"]["outcome"]
self.assertEqual((outcome["status"], outcome["stage"]), ("error", "synth"))
self.assertEqual(
outcome["message"], "ERROR: Module `foo' referenced but not found."
)

def test_signal_in_last_stage_log_is_a_crash(self):
self.add_flow_status(
"sky130hd",
"jpeg",
{"exit_code": 2},
logs={"3_3_place_gp.log": "Signal 11 received\nStack trace:\n"},
)
_, messages = self.run_script()
outcome = self.designs(messages)["jpeg"]["outcome"]
self.assertEqual((outcome["status"], outcome["stage"]), ("crash", "place"))
self.assertNotIn("message", outcome)

def test_passing_design_keeps_its_metrics_and_reports_passed(self):
self.add_design("nangate45", "gcd", metrics={"finish:area": 5.0})
self.add_flow_status(
"nangate45", "gcd", {"exit_code": 0, "elapsed_minutes": 3.5}
)
_, messages = self.run_script()
record = self.designs(messages)["gcd"]
self.assertEqual(record["metrics"], {"finish__area": 5.0})
self.assertEqual(
record["outcome"], {"status": "passed", "elapsed_minutes": 3.5}
)

def test_long_error_line_is_truncated(self):
self.add_flow_status(
"asap7",
"aes",
{"exit_code": 2},
logs={"5_2_route.log": "[ERROR DRT-0001] " + "x" * 2000 + "\n"},
)
_, messages = self.run_script()
outcome = self.designs(messages)["aes"]["outcome"]
self.assertEqual(outcome["stage"], "route")
self.assertEqual(len(outcome["message"]), 1024)

def test_unreadable_status_without_metrics_skips_the_design(self):
self.add_design("nangate45", "gcd")
self.add_flow_status("nangate45", "ibex", "not json")
self.add_flow_status("nangate45", "jpeg", {"timed_out": False})
result, messages = self.run_script()
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual(set(self.designs(messages)), {"gcd"})
self.assertIn("Reporting no outcome", result.stdout)
self.assertIn("[WARN] No data file for nangate45 ibex base.", result.stdout)

def test_outcome_only_design_names_itself_on_the_v1_fallback(self):
self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS)
self.add_flow_status(
"nangate45",
"ibex",
{"exit_code": 124, "timed_out": True, "limit_minutes": 60},
)
result, messages = self.run_script("--buildTime", "2026-10-05 12:34")
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
by_design = {m["data"].get("run__flow__design"): m["data"] for m in messages}
ibex = by_design["ibex"]
self.assertEqual(ibex["run__flow__platform"], "nangate45")
self.assertEqual(ibex["run__flow__variant"], "base")
self.assertEqual(ibex["run__flow__generate_date"], "2026-10-05 12:34")
self.assertEqual(ibex["outcome"], {"status": "timeout", "limit_minutes": 60})
self.assertNotIn("outcome", by_design.get(None, {}))


if __name__ == "__main__":
unittest.main()
196 changes: 181 additions & 15 deletions flow/util/uploadMetadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import json
import argparse
import os
import re
import sys
from datetime import datetime, timezone

Expand Down Expand Up @@ -160,17 +161,154 @@ def _non_negative_int(val):
MAX_PUBSUB_BYTES = 8 * 1024 * 1024


def build_design_record(dataFile, platform, design, variant):
"""Return a dict for one design to be included in the pipeline-level payload."""
with open(dataFile) as f:
data = json.load(f)
metrics = {k.replace(":", "__"): v for k, v in data.items()}
def build_design_record(dataFile, platform, design, variant, outcome=None):
"""Return a dict for one design to be included in the pipeline-level payload.

``dataFile`` is the design's metadata.json, or ``None`` for a run that ended
before writing one; such a record carries only its ``outcome``.
"""
record = {"platform": platform, "design": design, "variant": variant}
if dataFile:
with open(dataFile) as f:
data = json.load(f)
record["metrics"] = {k.replace(":", "__"): v for k, v in data.items()}
if outcome:
record["outcome"] = outcome
return record


# How a flow run ended, as the CI job records it next to the design's reports:
# {"exit_code": int, "timed_out": bool, "elapsed_minutes": float,
# "limit_minutes": int}. Every key is optional.
STATUS_FILENAME = "flow-status.json"
MAX_OUTCOME_MESSAGE = 1024
# Stage log names start with the step number: 1_2_yosys.log, 5_1_grt.log.
STAGE_LOG = re.compile(r"^(\d+)_(?:(\d+)_)?.*\.log$")
# The first matching line of a failing stage's log is the error reported. The
# OpenROAD form carries the tool's message ID; Yosys prints "ERROR: ".
ERROR_LINE = re.compile(
r"\[(?:ERROR|CRITICAL|FATAL) [A-Z][A-Z0-9]*-\d+\].*|^ERROR: .*|^Error: .*"
)
# What a tool that died on a signal leaves in its log, as opposed to an error it
# reported itself.
CRASH_LINE = re.compile(r"Signal \d+ received|Segmentation fault|core dumped")


def _stage_name(log_name):
"""The flow stage a stage log belongs to, e.g. 5_1_grt.log -> "grt"."""
match = STAGE_LOG.match(log_name)
if not match:
return None
step, sub = match.group(1), match.group(2)
if step == "5":
return "grt" if sub == "1" else "route"
return {
"platform": platform,
"design": design,
"variant": variant,
"metrics": metrics,
}
"1": "synth",
"2": "floorplan",
"3": "place",
"4": "cts",
"6": "finish",
}.get(step)


def _stage_logs(log_dir):
"""The design's stage logs, last step first."""
try:
names = os.listdir(log_dir)
except OSError:
return []

def key(name):
match = STAGE_LOG.match(name)
return (int(match.group(1)), int(match.group(2) or 0), name)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Respect execution order when selecting the crash-detection log

When final reporting crashes, 6_report.log is ordered behind the successful 6_1_fill.log because its absent substep is converted to zero, even though flow/Makefile:685 makes final reporting depend on fill. read_flow_outcome() only takes the crash flag from logs[0], so a Signal 11 received in 6_report.log is published as error, not crash. The same ordering problem affects 1_synth.log after 1_2_yosys.log. Account for these unnumbered terminal logs when selecting the log that determines the failure type.

Useful? React with 👍 / 👎.


return sorted((n for n in names if STAGE_LOG.match(n)), key=key, reverse=True)
Comment on lines +214 to +225

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The current implementation of _stage_logs is slightly inefficient because it calls STAGE_LOG.match(n) twice for each matching log name (once in the generator filter and once inside the key function). Additionally, defining the nested key function inside _stage_logs recreates it on every call.\n\nWe can simplify and optimize this by performing a single pass over names, matching once, and sorting the resulting tuples directly.

def _stage_logs(log_dir):\n    \"\"\"The design's stage logs, last step first.\"\"\"\n    try:\n        names = os.listdir(log_dir)\n    except OSError:\n        return []\n\n    matched_logs = []\n    for name in names:\n        match = STAGE_LOG.match(name)\n        if match:\n            matched_logs.append((int(match.group(1)), int(match.group(2) or 0), name))\n\n    matched_logs.sort(reverse=True)\n    return [item[2] for item in matched_logs]



def _scan_log(path):
"""The first error line of a log and whether it shows a crash."""
error_line = None
crashed = False
try:
with open(path, errors="replace") as f:
for line in f:
if error_line is None:
match = ERROR_LINE.search(line)
if match:
error_line = match.group(0).strip()
if not crashed and CRASH_LINE.search(line):
crashed = True
if error_line is not None and crashed:
break
except OSError:
pass
return error_line, crashed
Comment on lines +228 to +245

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Scanning the entire log file line-by-line can be very slow for large log files (which can be tens or hundreds of megabytes in VLSI flows). Since we only care about detecting a crash in the last stage log (i == 0), we can avoid scanning for crash lines in older logs (i > 0).\n\nBy adding a scan_for_crash parameter (defaulting to True), we can break out of the loop immediately once the first error line is found when scan_for_crash is False.

def _scan_log(path, scan_for_crash=True):\n    \"\"\"The first error line of a log and whether it shows a crash.\"\"\"\n    error_line = None\n    crashed = False\n    try:\n        with open(path, errors=\"replace\") as f:\n            for line in f:\n                if error_line is None:\n                    match = ERROR_LINE.search(line)\n                    if match:\n                        error_line = match.group(0).strip()\n                if scan_for_crash and not crashed and CRASH_LINE.search(line):\n                    crashed = True\n                if error_line is not None and (not scan_for_crash or crashed):\n                    break\n    except OSError:\n        pass\n    return error_line, crashed



def read_flow_outcome(report_dir, log_dir):
"""The design's outcome from its flow-status.json and stage logs, or ``None``.

``status`` is ``timeout`` when the time limit stopped the run, ``passed`` on
exit code 0, ``crash`` when the last stage log shows the tool died on a
signal, and ``error`` otherwise. A failed run also names the stage it ended
in and the first error line found in its logs, last stage first.
"""
path = os.path.join(report_dir, STATUS_FILENAME)
if not os.path.exists(path):
return None
try:
with open(path) as f:
status = json.load(f)
except (OSError, ValueError) as e:
print(f"[WARN] Could not read {path}: {e}. Reporting no outcome.")
return None
if not isinstance(status, dict):
print(f"[WARN] {path} does not hold a JSON object. Reporting no outcome.")
return None

exit_code = status.get("exit_code")
timed_out = status.get("timed_out") is True
if not timed_out and not isinstance(exit_code, int):
print(f"[WARN] {path} has no exit_code. Reporting no outcome.")
return None

outcome = {}
for key in ("elapsed_minutes", "limit_minutes"):
value = status.get(key)
if (
isinstance(value, (int, float))
and not isinstance(value, bool)
and value >= 0
):
outcome[key] = value

if not timed_out and exit_code == 0:
outcome["status"] = "passed"
return outcome

logs = _stage_logs(log_dir)
if logs:
stage = _stage_name(logs[0])
if stage:
outcome["stage"] = stage

message = None
crashed = False
for i, name in enumerate(logs):
error_line, log_crashed = _scan_log(os.path.join(log_dir, name))
if i == 0:
crashed = log_crashed
if error_line:
message = error_line
break
Comment on lines +297 to +303

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Pass scan_for_crash=(i == 0) to _scan_log so that we only scan for crash lines in the last stage log. For older logs (i > 0), this allows _scan_log to stop reading the file immediately once the first error line is found, significantly improving performance.

    for i, name in enumerate(logs):\n        error_line, log_crashed = _scan_log(os.path.join(log_dir, name), scan_for_crash=(i == 0))\n        if i == 0:\n            crashed = log_crashed\n        if error_line:\n            message = error_line\n            break

if message:
outcome["message"] = message[:MAX_OUTCOME_MESSAGE]

if timed_out:
outcome["status"] = "timeout"
else:
outcome["status"] = "crash" if crashed else "error"
return outcome


def _submodule_defect(entry):
Expand Down Expand Up @@ -426,6 +564,10 @@ def publish_v1_per_design(

The requester keys go on after the metrics, so a metric that happened to
share their name could not replace who asked for the build.

A design's outcome rides at the root as ``outcome``. A design that has only
an outcome carries no metrics to name it, so its message names the design
with the same run__flow__ keys metadata.json would hold, dated by the build.
"""
futures = []
failed = 0
Expand All @@ -441,7 +583,19 @@ def publish_v1_per_design(
}
if provenance:
payload.update(provenance)
payload.update(d["metrics"])
if "metrics" in d:
payload.update(d["metrics"])
else:
payload.update(
{
"run__flow__platform": d["platform"],
"run__flow__design": d["design"],
"run__flow__variant": d["variant"],
"run__flow__generate_date": resolve_build_time(args),
}
)
if "outcome" in d:
payload["outcome"] = d["outcome"]
if requester:
payload.update(requester)

Expand Down Expand Up @@ -513,16 +667,28 @@ def publish_v1_per_design(
platform = dirList[1]
design = dirList[2]
variant = dirList[3]
if platform == "sky130hd_fakestack" or platform == "src":
print(f"[WARN] Skiping upload {platform} {design} {variant}.")
continue
dataFile = os.path.join(reportDir, RUN_FILENAME)
if not os.path.exists(dataFile):
dataFile = None
outcome = read_flow_outcome(
reportDir, os.path.join("logs", platform, design, variant)
)
if dataFile is None and outcome is None:
print(f"[WARN] No data file for {platform} {design} {variant}.")
continue
if platform == "sky130hd_fakestack" or platform == "src":
print(f"[WARN] Skiping upload {platform} {design} {variant}.")
continue
if dataFile is None:
print(
f"[INFO] No data file for {platform} {design} {variant}; "
f"reporting its {outcome['status']} outcome only."
)
# --- PUBSUB ---
if publisher:
design_records.append(build_design_record(dataFile, platform, design, variant))
design_records.append(
build_design_record(dataFile, platform, design, variant, outcome)
)
# --- END PUBSUB ---

# --- PUBSUB ---
Expand Down