Repository navigation
Report how each design's flow run ended in the metadata upload #4819
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,6 +3,7 @@ | |
| import json | ||
| import argparse | ||
| import os | ||
| import re | ||
| import sys | ||
| from datetime import datetime, timezone | ||
|
|
||
|
|
@@ -160,17 +161,154 @@ def _non_negative_int(val): | |
| MAX_PUBSUB_BYTES = 8 * 1024 * 1024 | ||
|
|
||
|
|
||
| def build_design_record(dataFile, platform, design, variant): | ||
| """Return a dict for one design to be included in the pipeline-level payload.""" | ||
| with open(dataFile) as f: | ||
| data = json.load(f) | ||
| metrics = {k.replace(":", "__"): v for k, v in data.items()} | ||
| def build_design_record(dataFile, platform, design, variant, outcome=None): | ||
| """Return a dict for one design to be included in the pipeline-level payload. | ||
|
|
||
| ``dataFile`` is the design's metadata.json, or ``None`` for a run that ended | ||
| before writing one; such a record carries only its ``outcome``. | ||
| """ | ||
| record = {"platform": platform, "design": design, "variant": variant} | ||
| if dataFile: | ||
| with open(dataFile) as f: | ||
| data = json.load(f) | ||
| record["metrics"] = {k.replace(":", "__"): v for k, v in data.items()} | ||
| if outcome: | ||
| record["outcome"] = outcome | ||
| return record | ||
|
|
||
|
|
||
| # How a flow run ended, as the CI job records it next to the design's reports: | ||
| # {"exit_code": int, "timed_out": bool, "elapsed_minutes": float, | ||
| # "limit_minutes": int}. Every key is optional. | ||
| STATUS_FILENAME = "flow-status.json" | ||
| MAX_OUTCOME_MESSAGE = 1024 | ||
| # Stage log names start with the step number: 1_2_yosys.log, 5_1_grt.log. | ||
| STAGE_LOG = re.compile(r"^(\d+)_(?:(\d+)_)?.*\.log$") | ||
| # The first matching line of a failing stage's log is the error reported. The | ||
| # OpenROAD form carries the tool's message ID; Yosys prints "ERROR: ". | ||
| ERROR_LINE = re.compile( | ||
| r"\[(?:ERROR|CRITICAL|FATAL) [A-Z][A-Z0-9]*-\d+\].*|^ERROR: .*|^Error: .*" | ||
| ) | ||
| # What a tool that died on a signal leaves in its log, as opposed to an error it | ||
| # reported itself. | ||
| CRASH_LINE = re.compile(r"Signal \d+ received|Segmentation fault|core dumped") | ||
|
|
||
|
|
||
| def _stage_name(log_name): | ||
| """The flow stage a stage log belongs to, e.g. 5_1_grt.log -> "grt".""" | ||
| match = STAGE_LOG.match(log_name) | ||
| if not match: | ||
| return None | ||
| step, sub = match.group(1), match.group(2) | ||
| if step == "5": | ||
| return "grt" if sub == "1" else "route" | ||
| return { | ||
| "platform": platform, | ||
| "design": design, | ||
| "variant": variant, | ||
| "metrics": metrics, | ||
| } | ||
| "1": "synth", | ||
| "2": "floorplan", | ||
| "3": "place", | ||
| "4": "cts", | ||
| "6": "finish", | ||
| }.get(step) | ||
|
|
||
|
|
||
| def _stage_logs(log_dir): | ||
| """The design's stage logs, last step first.""" | ||
| try: | ||
| names = os.listdir(log_dir) | ||
| except OSError: | ||
| return [] | ||
|
|
||
| def key(name): | ||
| match = STAGE_LOG.match(name) | ||
| return (int(match.group(1)), int(match.group(2) or 0), name) | ||
|
|
||
| return sorted((n for n in names if STAGE_LOG.match(n)), key=key, reverse=True) | ||
|
Comment on lines
+214
to
+225
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The current implementation of def _stage_logs(log_dir):\n \"\"\"The design's stage logs, last step first.\"\"\"\n try:\n names = os.listdir(log_dir)\n except OSError:\n return []\n\n matched_logs = []\n for name in names:\n match = STAGE_LOG.match(name)\n if match:\n matched_logs.append((int(match.group(1)), int(match.group(2) or 0), name))\n\n matched_logs.sort(reverse=True)\n return [item[2] for item in matched_logs] |
||
|
|
||
|
|
||
| def _scan_log(path): | ||
| """The first error line of a log and whether it shows a crash.""" | ||
| error_line = None | ||
| crashed = False | ||
| try: | ||
| with open(path, errors="replace") as f: | ||
| for line in f: | ||
| if error_line is None: | ||
| match = ERROR_LINE.search(line) | ||
| if match: | ||
| error_line = match.group(0).strip() | ||
| if not crashed and CRASH_LINE.search(line): | ||
| crashed = True | ||
| if error_line is not None and crashed: | ||
| break | ||
| except OSError: | ||
| pass | ||
| return error_line, crashed | ||
|
Comment on lines
+228
to
+245
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Scanning the entire log file line-by-line can be very slow for large log files (which can be tens or hundreds of megabytes in VLSI flows). Since we only care about detecting a crash in the last stage log ( def _scan_log(path, scan_for_crash=True):\n \"\"\"The first error line of a log and whether it shows a crash.\"\"\"\n error_line = None\n crashed = False\n try:\n with open(path, errors=\"replace\") as f:\n for line in f:\n if error_line is None:\n match = ERROR_LINE.search(line)\n if match:\n error_line = match.group(0).strip()\n if scan_for_crash and not crashed and CRASH_LINE.search(line):\n crashed = True\n if error_line is not None and (not scan_for_crash or crashed):\n break\n except OSError:\n pass\n return error_line, crashed |
||
|
|
||
|
|
||
| def read_flow_outcome(report_dir, log_dir): | ||
| """The design's outcome from its flow-status.json and stage logs, or ``None``. | ||
|
|
||
| ``status`` is ``timeout`` when the time limit stopped the run, ``passed`` on | ||
| exit code 0, ``crash`` when the last stage log shows the tool died on a | ||
| signal, and ``error`` otherwise. A failed run also names the stage it ended | ||
| in and the first error line found in its logs, last stage first. | ||
| """ | ||
| path = os.path.join(report_dir, STATUS_FILENAME) | ||
| if not os.path.exists(path): | ||
| return None | ||
| try: | ||
| with open(path) as f: | ||
| status = json.load(f) | ||
| except (OSError, ValueError) as e: | ||
| print(f"[WARN] Could not read {path}: {e}. Reporting no outcome.") | ||
| return None | ||
| if not isinstance(status, dict): | ||
| print(f"[WARN] {path} does not hold a JSON object. Reporting no outcome.") | ||
| return None | ||
|
|
||
| exit_code = status.get("exit_code") | ||
| timed_out = status.get("timed_out") is True | ||
| if not timed_out and not isinstance(exit_code, int): | ||
| print(f"[WARN] {path} has no exit_code. Reporting no outcome.") | ||
| return None | ||
|
|
||
| outcome = {} | ||
| for key in ("elapsed_minutes", "limit_minutes"): | ||
| value = status.get(key) | ||
| if ( | ||
| isinstance(value, (int, float)) | ||
| and not isinstance(value, bool) | ||
| and value >= 0 | ||
| ): | ||
| outcome[key] = value | ||
|
|
||
| if not timed_out and exit_code == 0: | ||
| outcome["status"] = "passed" | ||
| return outcome | ||
|
|
||
| logs = _stage_logs(log_dir) | ||
| if logs: | ||
| stage = _stage_name(logs[0]) | ||
| if stage: | ||
| outcome["stage"] = stage | ||
|
|
||
| message = None | ||
| crashed = False | ||
| for i, name in enumerate(logs): | ||
| error_line, log_crashed = _scan_log(os.path.join(log_dir, name)) | ||
| if i == 0: | ||
| crashed = log_crashed | ||
| if error_line: | ||
| message = error_line | ||
| break | ||
|
Comment on lines
+297
to
+303
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Pass for i, name in enumerate(logs):\n error_line, log_crashed = _scan_log(os.path.join(log_dir, name), scan_for_crash=(i == 0))\n if i == 0:\n crashed = log_crashed\n if error_line:\n message = error_line\n break |
||
| if message: | ||
| outcome["message"] = message[:MAX_OUTCOME_MESSAGE] | ||
|
|
||
| if timed_out: | ||
| outcome["status"] = "timeout" | ||
| else: | ||
| outcome["status"] = "crash" if crashed else "error" | ||
| return outcome | ||
|
|
||
|
|
||
| def _submodule_defect(entry): | ||
|
|
@@ -426,6 +564,10 @@ def publish_v1_per_design( | |
|
|
||
| The requester keys go on after the metrics, so a metric that happened to | ||
| share their name could not replace who asked for the build. | ||
|
|
||
| A design's outcome rides at the root as ``outcome``. A design that has only | ||
| an outcome carries no metrics to name it, so its message names the design | ||
| with the same run__flow__ keys metadata.json would hold, dated by the build. | ||
| """ | ||
| futures = [] | ||
| failed = 0 | ||
|
|
@@ -441,7 +583,19 @@ def publish_v1_per_design( | |
| } | ||
| if provenance: | ||
| payload.update(provenance) | ||
| payload.update(d["metrics"]) | ||
| if "metrics" in d: | ||
| payload.update(d["metrics"]) | ||
| else: | ||
| payload.update( | ||
| { | ||
| "run__flow__platform": d["platform"], | ||
| "run__flow__design": d["design"], | ||
| "run__flow__variant": d["variant"], | ||
| "run__flow__generate_date": resolve_build_time(args), | ||
| } | ||
| ) | ||
| if "outcome" in d: | ||
| payload["outcome"] = d["outcome"] | ||
| if requester: | ||
| payload.update(requester) | ||
|
|
||
|
|
@@ -513,16 +667,28 @@ def publish_v1_per_design( | |
| platform = dirList[1] | ||
| design = dirList[2] | ||
| variant = dirList[3] | ||
| if platform == "sky130hd_fakestack" or platform == "src": | ||
| print(f"[WARN] Skiping upload {platform} {design} {variant}.") | ||
| continue | ||
| dataFile = os.path.join(reportDir, RUN_FILENAME) | ||
| if not os.path.exists(dataFile): | ||
| dataFile = None | ||
| outcome = read_flow_outcome( | ||
| reportDir, os.path.join("logs", platform, design, variant) | ||
| ) | ||
| if dataFile is None and outcome is None: | ||
| print(f"[WARN] No data file for {platform} {design} {variant}.") | ||
| continue | ||
| if platform == "sky130hd_fakestack" or platform == "src": | ||
| print(f"[WARN] Skiping upload {platform} {design} {variant}.") | ||
| continue | ||
| if dataFile is None: | ||
| print( | ||
| f"[INFO] No data file for {platform} {design} {variant}; " | ||
| f"reporting its {outcome['status']} outcome only." | ||
| ) | ||
| # --- PUBSUB --- | ||
| if publisher: | ||
| design_records.append(build_design_record(dataFile, platform, design, variant)) | ||
| design_records.append( | ||
| build_design_record(dataFile, platform, design, variant, outcome) | ||
| ) | ||
| # --- END PUBSUB --- | ||
|
|
||
| # --- PUBSUB --- | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When final reporting crashes,
6_report.logis ordered behind the successful6_1_fill.logbecause its absent substep is converted to zero, even thoughflow/Makefile:685makes final reporting depend on fill.read_flow_outcome()only takes the crash flag fromlogs[0], so aSignal 11 receivedin6_report.logis published aserror, notcrash. The same ordering problem affects1_synth.logafter1_2_yosys.log. Account for these unnumbered terminal logs when selecting the log that determines the failure type.Useful? React with 👍 / 👎.