Skip to content

Commit e54d0af

Browse files
oharboeclaude
andcommitted
Leave the metadata upload outcome change to its own pull request
uploadMetadata.py and its test came onto this branch with a merge; the change is unrelated to cts.tcl and is under review on its own as #4819. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Øyvind Harboe <oyvind.harboe@zylin.com>
1 parent 5d81dd1 commit e54d0af

2 files changed

Lines changed: 15 additions & 325 deletions

File tree

‎flow/test/test_uploadMetadata.py‎

Lines changed: 0 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -102,23 +102,6 @@ def add_design(self, platform, design, variant="base", metrics=None):
102102
with open(os.path.join(report_dir, "metadata.json"), "w") as f:
103103
json.dump(metrics or {"finish:area": 1.0}, f)
104104

105-
def add_flow_status(self, platform, design, status, variant="base", logs=None):
106-
"""Record how a design's flow run ended, as the CI job does, with stage
107-
logs given as {file name: content}."""
108-
report_dir = os.path.join(self.flow, "reports", platform, design, variant)
109-
os.makedirs(report_dir, exist_ok=True)
110-
with open(os.path.join(report_dir, "flow-status.json"), "w") as f:
111-
f.write(status if isinstance(status, str) else json.dumps(status))
112-
log_dir = os.path.join(self.flow, "logs", platform, design, variant)
113-
os.makedirs(log_dir, exist_ok=True)
114-
for name, content in (logs or {}).items():
115-
with open(os.path.join(log_dir, name), "w") as f:
116-
f.write(content)
117-
118-
def designs(self, messages):
119-
"""The pipeline payload's design records, by design name."""
120-
return {d["design"]: d for d in messages[0]["data"]["designs"]}
121-
122105
def run_script(self, *extra, fail=False, tz=None, extra_env=None):
123106
env = dict(os.environ)
124107
for name in REQUESTER_ENV:
@@ -554,133 +537,6 @@ def test_no_publisher_publishes_nothing(self):
554537
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
555538
self.assertFalse(os.path.exists(self.out))
556539

557-
def test_timed_out_design_without_metrics_reports_its_outcome(self):
558-
self.add_design("nangate45", "gcd")
559-
self.add_flow_status(
560-
"nangate45",
561-
"ibex",
562-
{
563-
"exit_code": 124,
564-
"timed_out": True,
565-
"elapsed_minutes": 240.2,
566-
"limit_minutes": 240,
567-
},
568-
logs={"3_3_place_gp.log": "placing\n", "5_1_grt.log": "routing\n"},
569-
)
570-
result, messages = self.run_script("--jobName", "ORFS")
571-
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
572-
designs = self.designs(messages)
573-
self.assertEqual(
574-
designs["ibex"],
575-
{
576-
"platform": "nangate45",
577-
"design": "ibex",
578-
"variant": "base",
579-
"outcome": {
580-
"status": "timeout",
581-
"stage": "grt",
582-
"elapsed_minutes": 240.2,
583-
"limit_minutes": 240,
584-
},
585-
},
586-
)
587-
self.assertNotIn("outcome", designs["gcd"])
588-
self.assertIn("reporting its timeout outcome only", result.stdout)
589-
590-
def test_failed_design_reports_stage_and_first_error_line(self):
591-
self.add_flow_status(
592-
"asap7",
593-
"aes",
594-
{"exit_code": 2, "timed_out": False},
595-
logs={
596-
"4_1_cts.log": "[ERROR CTS-0001] not this one\n",
597-
"5_1_grt.log": "info\n[ERROR GRT-0116] Global routing failed.\n"
598-
"[ERROR GRT-0001] later\n",
599-
},
600-
)
601-
_, messages = self.run_script()
602-
outcome = self.designs(messages)["aes"]["outcome"]
603-
self.assertEqual(outcome["status"], "error")
604-
self.assertEqual(outcome["stage"], "grt")
605-
self.assertEqual(outcome["message"], "[ERROR GRT-0116] Global routing failed.")
606-
607-
def test_yosys_error_line_is_reported(self):
608-
self.add_flow_status(
609-
"asap7",
610-
"aes",
611-
{"exit_code": 2},
612-
logs={"1_2_yosys.log": "ERROR: Module `foo' referenced but not found.\n"},
613-
)
614-
_, messages = self.run_script()
615-
outcome = self.designs(messages)["aes"]["outcome"]
616-
self.assertEqual((outcome["status"], outcome["stage"]), ("error", "synth"))
617-
self.assertEqual(
618-
outcome["message"], "ERROR: Module `foo' referenced but not found."
619-
)
620-
621-
def test_signal_in_last_stage_log_is_a_crash(self):
622-
self.add_flow_status(
623-
"sky130hd",
624-
"jpeg",
625-
{"exit_code": 2},
626-
logs={"3_3_place_gp.log": "Signal 11 received\nStack trace:\n"},
627-
)
628-
_, messages = self.run_script()
629-
outcome = self.designs(messages)["jpeg"]["outcome"]
630-
self.assertEqual((outcome["status"], outcome["stage"]), ("crash", "place"))
631-
self.assertNotIn("message", outcome)
632-
633-
def test_passing_design_keeps_its_metrics_and_reports_passed(self):
634-
self.add_design("nangate45", "gcd", metrics={"finish:area": 5.0})
635-
self.add_flow_status(
636-
"nangate45", "gcd", {"exit_code": 0, "elapsed_minutes": 3.5}
637-
)
638-
_, messages = self.run_script()
639-
record = self.designs(messages)["gcd"]
640-
self.assertEqual(record["metrics"], {"finish__area": 5.0})
641-
self.assertEqual(
642-
record["outcome"], {"status": "passed", "elapsed_minutes": 3.5}
643-
)
644-
645-
def test_long_error_line_is_truncated(self):
646-
self.add_flow_status(
647-
"asap7",
648-
"aes",
649-
{"exit_code": 2},
650-
logs={"5_2_route.log": "[ERROR DRT-0001] " + "x" * 2000 + "\n"},
651-
)
652-
_, messages = self.run_script()
653-
outcome = self.designs(messages)["aes"]["outcome"]
654-
self.assertEqual(outcome["stage"], "route")
655-
self.assertEqual(len(outcome["message"]), 1024)
656-
657-
def test_unreadable_status_without_metrics_skips_the_design(self):
658-
self.add_design("nangate45", "gcd")
659-
self.add_flow_status("nangate45", "ibex", "not json")
660-
self.add_flow_status("nangate45", "jpeg", {"timed_out": False})
661-
result, messages = self.run_script()
662-
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
663-
self.assertEqual(set(self.designs(messages)), {"gcd"})
664-
self.assertIn("Reporting no outcome", result.stdout)
665-
self.assertIn("[WARN] No data file for nangate45 ibex base.", result.stdout)
666-
667-
def test_outcome_only_design_names_itself_on_the_v1_fallback(self):
668-
self.add_design("nangate45", "gcd", metrics=OVERSIZED_METRICS)
669-
self.add_flow_status(
670-
"nangate45",
671-
"ibex",
672-
{"exit_code": 124, "timed_out": True, "limit_minutes": 60},
673-
)
674-
result, messages = self.run_script("--buildTime", "2026-10-05 12:34")
675-
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
676-
by_design = {m["data"].get("run__flow__design"): m["data"] for m in messages}
677-
ibex = by_design["ibex"]
678-
self.assertEqual(ibex["run__flow__platform"], "nangate45")
679-
self.assertEqual(ibex["run__flow__variant"], "base")
680-
self.assertEqual(ibex["run__flow__generate_date"], "2026-10-05 12:34")
681-
self.assertEqual(ibex["outcome"], {"status": "timeout", "limit_minutes": 60})
682-
self.assertNotIn("outcome", by_design.get(None, {}))
683-
684540

685541
if __name__ == "__main__":
686542
unittest.main()

‎flow/util/uploadMetadata.py‎

Lines changed: 15 additions & 181 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import json
44
import argparse
55
import os
6-
import re
76
import sys
87
from datetime import datetime, timezone
98

@@ -161,154 +160,17 @@ def _non_negative_int(val):
161160
MAX_PUBSUB_BYTES = 8 * 1024 * 1024
162161

163162

164-
def build_design_record(dataFile, platform, design, variant, outcome=None):
165-
"""Return a dict for one design to be included in the pipeline-level payload.
166-
167-
``dataFile`` is the design's metadata.json, or ``None`` for a run that ended
168-
before writing one; such a record carries only its ``outcome``.
169-
"""
170-
record = {"platform": platform, "design": design, "variant": variant}
171-
if dataFile:
172-
with open(dataFile) as f:
173-
data = json.load(f)
174-
record["metrics"] = {k.replace(":", "__"): v for k, v in data.items()}
175-
if outcome:
176-
record["outcome"] = outcome
177-
return record
178-
179-
180-
# How a flow run ended, as the CI job records it next to the design's reports:
181-
# {"exit_code": int, "timed_out": bool, "elapsed_minutes": float,
182-
# "limit_minutes": int}. Every key is optional.
183-
STATUS_FILENAME = "flow-status.json"
184-
MAX_OUTCOME_MESSAGE = 1024
185-
# Stage log names start with the step number: 1_2_yosys.log, 5_1_grt.log.
186-
STAGE_LOG = re.compile(r"^(\d+)_(?:(\d+)_)?.*\.log$")
187-
# The first matching line of a failing stage's log is the error reported. The
188-
# OpenROAD form carries the tool's message ID; Yosys prints "ERROR: ".
189-
ERROR_LINE = re.compile(
190-
r"\[(?:ERROR|CRITICAL|FATAL) [A-Z][A-Z0-9]*-\d+\].*|^ERROR: .*|^Error: .*"
191-
)
192-
# What a tool that died on a signal leaves in its log, as opposed to an error it
193-
# reported itself.
194-
CRASH_LINE = re.compile(r"Signal \d+ received|Segmentation fault|core dumped")
195-
196-
197-
def _stage_name(log_name):
198-
"""The flow stage a stage log belongs to, e.g. 5_1_grt.log -> "grt"."""
199-
match = STAGE_LOG.match(log_name)
200-
if not match:
201-
return None
202-
step, sub = match.group(1), match.group(2)
203-
if step == "5":
204-
return "grt" if sub == "1" else "route"
163+
def build_design_record(dataFile, platform, design, variant):
164+
"""Return a dict for one design to be included in the pipeline-level payload."""
165+
with open(dataFile) as f:
166+
data = json.load(f)
167+
metrics = {k.replace(":", "__"): v for k, v in data.items()}
205168
return {
206-
"1": "synth",
207-
"2": "floorplan",
208-
"3": "place",
209-
"4": "cts",
210-
"6": "finish",
211-
}.get(step)
212-
213-
214-
def _stage_logs(log_dir):
215-
"""The design's stage logs, last step first."""
216-
try:
217-
names = os.listdir(log_dir)
218-
except OSError:
219-
return []
220-
221-
def key(name):
222-
match = STAGE_LOG.match(name)
223-
return (int(match.group(1)), int(match.group(2) or 0), name)
224-
225-
return sorted((n for n in names if STAGE_LOG.match(n)), key=key, reverse=True)
226-
227-
228-
def _scan_log(path):
229-
"""The first error line of a log and whether it shows a crash."""
230-
error_line = None
231-
crashed = False
232-
try:
233-
with open(path, errors="replace") as f:
234-
for line in f:
235-
if error_line is None:
236-
match = ERROR_LINE.search(line)
237-
if match:
238-
error_line = match.group(0).strip()
239-
if not crashed and CRASH_LINE.search(line):
240-
crashed = True
241-
if error_line is not None and crashed:
242-
break
243-
except OSError:
244-
pass
245-
return error_line, crashed
246-
247-
248-
def read_flow_outcome(report_dir, log_dir):
249-
"""The design's outcome from its flow-status.json and stage logs, or ``None``.
250-
251-
``status`` is ``timeout`` when the time limit stopped the run, ``passed`` on
252-
exit code 0, ``crash`` when the last stage log shows the tool died on a
253-
signal, and ``error`` otherwise. A failed run also names the stage it ended
254-
in and the first error line found in its logs, last stage first.
255-
"""
256-
path = os.path.join(report_dir, STATUS_FILENAME)
257-
if not os.path.exists(path):
258-
return None
259-
try:
260-
with open(path) as f:
261-
status = json.load(f)
262-
except (OSError, ValueError) as e:
263-
print(f"[WARN] Could not read {path}: {e}. Reporting no outcome.")
264-
return None
265-
if not isinstance(status, dict):
266-
print(f"[WARN] {path} does not hold a JSON object. Reporting no outcome.")
267-
return None
268-
269-
exit_code = status.get("exit_code")
270-
timed_out = status.get("timed_out") is True
271-
if not timed_out and not isinstance(exit_code, int):
272-
print(f"[WARN] {path} has no exit_code. Reporting no outcome.")
273-
return None
274-
275-
outcome = {}
276-
for key in ("elapsed_minutes", "limit_minutes"):
277-
value = status.get(key)
278-
if (
279-
isinstance(value, (int, float))
280-
and not isinstance(value, bool)
281-
and value >= 0
282-
):
283-
outcome[key] = value
284-
285-
if not timed_out and exit_code == 0:
286-
outcome["status"] = "passed"
287-
return outcome
288-
289-
logs = _stage_logs(log_dir)
290-
if logs:
291-
stage = _stage_name(logs[0])
292-
if stage:
293-
outcome["stage"] = stage
294-
295-
message = None
296-
crashed = False
297-
for i, name in enumerate(logs):
298-
error_line, log_crashed = _scan_log(os.path.join(log_dir, name))
299-
if i == 0:
300-
crashed = log_crashed
301-
if error_line:
302-
message = error_line
303-
break
304-
if message:
305-
outcome["message"] = message[:MAX_OUTCOME_MESSAGE]
306-
307-
if timed_out:
308-
outcome["status"] = "timeout"
309-
else:
310-
outcome["status"] = "crash" if crashed else "error"
311-
return outcome
169+
"platform": platform,
170+
"design": design,
171+
"variant": variant,
172+
"metrics": metrics,
173+
}
312174

313175

314176
def _submodule_defect(entry):
@@ -564,10 +426,6 @@ def publish_v1_per_design(
564426
565427
The requester keys go on after the metrics, so a metric that happened to
566428
share their name could not replace who asked for the build.
567-
568-
A design's outcome rides at the root as ``outcome``. A design that has only
569-
an outcome carries no metrics to name it, so its message names the design
570-
with the same run__flow__ keys metadata.json would hold, dated by the build.
571429
"""
572430
futures = []
573431
failed = 0
@@ -583,19 +441,7 @@ def publish_v1_per_design(
583441
}
584442
if provenance:
585443
payload.update(provenance)
586-
if "metrics" in d:
587-
payload.update(d["metrics"])
588-
else:
589-
payload.update(
590-
{
591-
"run__flow__platform": d["platform"],
592-
"run__flow__design": d["design"],
593-
"run__flow__variant": d["variant"],
594-
"run__flow__generate_date": resolve_build_time(args),
595-
}
596-
)
597-
if "outcome" in d:
598-
payload["outcome"] = d["outcome"]
444+
payload.update(d["metrics"])
599445
if requester:
600446
payload.update(requester)
601447

@@ -667,28 +513,16 @@ def publish_v1_per_design(
667513
platform = dirList[1]
668514
design = dirList[2]
669515
variant = dirList[3]
670-
if platform == "sky130hd_fakestack" or platform == "src":
671-
print(f"[WARN] Skiping upload {platform} {design} {variant}.")
672-
continue
673516
dataFile = os.path.join(reportDir, RUN_FILENAME)
674517
if not os.path.exists(dataFile):
675-
dataFile = None
676-
outcome = read_flow_outcome(
677-
reportDir, os.path.join("logs", platform, design, variant)
678-
)
679-
if dataFile is None and outcome is None:
680518
print(f"[WARN] No data file for {platform} {design} {variant}.")
681519
continue
682-
if dataFile is None:
683-
print(
684-
f"[INFO] No data file for {platform} {design} {variant}; "
685-
f"reporting its {outcome['status']} outcome only."
686-
)
520+
if platform == "sky130hd_fakestack" or platform == "src":
521+
print(f"[WARN] Skiping upload {platform} {design} {variant}.")
522+
continue
687523
# --- PUBSUB ---
688524
if publisher:
689-
design_records.append(
690-
build_design_record(dataFile, platform, design, variant, outcome)
691-
)
525+
design_records.append(build_design_record(dataFile, platform, design, variant))
692526
# --- END PUBSUB ---
693527

694528
# --- PUBSUB ---

0 commit comments

Comments
 (0)