Skip to content

Commit 6880de8

Browse files
authored
Keep full-flow steps when generating metadata without QoR checks (#4683)
* fix(test): keep full-flow steps with metadata generation Allow an external QoR checker to own the verdict without skipping simulation, power, or issue packaging after metadata generation. Signed-off-by: Vitor Bandeira <vvbandeira@precisioninno.com> * test(report): use public synthetic dashboard data Use a public platform and synthetic timing values in the report fixture. Keep the same assertions for the failed metric, limit, and baseline link. Signed-off-by: Vitor Bandeira <vvbandeira@precisioninno.com> --------- Signed-off-by: Vitor Bandeira <vvbandeira@precisioninno.com>
1 parent 2ea71df commit 6880de8

3 files changed

Lines changed: 101 additions & 1 deletion

File tree

‎flow/test/test_genReport.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,22 @@ def test_empty_log_directory_is_reported_as_incomplete(self):
140140
self.assertIn("sky130hd gcd (base)\n Flow did not reach last stage.", summary)
141141
self.assertIn("No log file found.", summary)
142142

143+
def test_dashboard_verdict_reaches_the_summary(self):
144+
self.write_file("logs/nangate45/gcd/base/6_report.log", "Finished\n")
145+
self.write_file(
146+
"reports/nangate45/gcd/base/metadata-check.log",
147+
"[ERROR] Dashboard QoR check: FAIL (nangate45/gcd/base)\n"
148+
"QoR regression detected for nangate45/gcd/base:\n"
149+
" finish__timing__setup__tns -12.0 (base -6.0, limit -8.0)\n"
150+
" Baseline: https://dashboard.example/builds/7\n",
151+
)
152+
summary = self.summary()
153+
self.assertIn("Failing designs", summary)
154+
self.assertIn("nangate45 gcd (base)", summary)
155+
self.assertIn("finish__timing__setup__tns -12.0", summary)
156+
self.assertIn("limit -8.0", summary)
157+
self.assertIn("https://dashboard.example/builds/7", summary)
158+
143159

144160
class TestGenReport(unittest.TestCase):
145161
def test_incomplete_flow_reports_that_it_did_not_reach_last_stage(self):

‎flow/test/test_helper.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ Options:
1717
--design-path DIR Root path containing 'designs/' (default: ./)
1818
--variant NAME Flow variant (default: unset)
1919
--target STR Make target(s), space-separated (default: 'finish metadata')
20+
'finish metadata-generate' keeps all full-flow follow-up steps
21+
and leaves QoR checking to an external caller
2022
--work-home DIR Work home (default: .)
2123
--private-dir DIR Private tool scripts dir (default: ../../private_tool_scripts)
2224
--save-to-db Save metrics to DB (requires private.mk)
@@ -106,7 +108,7 @@ if [ "$ret" -ne 0 ] && ! compgen -G "$REPORT_LOG_DIR/*.log" > /dev/null; then
106108
command cp "$LOG_FILE" "$REPORT_LOG_DIR/0_make.log"
107109
fi
108110

109-
if [ "${TARGET}" != "finish metadata" ]; then
111+
if [ "${TARGET}" != "finish metadata" ] && [ "${TARGET}" != "finish metadata-generate" ]; then
110112
exit $ret
111113
fi
112114

‎flow/test/test_test_helper.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
"""Run the real flow helper against a disposable Makefile."""
2+
3+
import os
4+
from pathlib import Path
5+
import shutil
6+
import subprocess
7+
import tempfile
8+
import unittest
9+
10+
HELPER = Path(__file__).with_name("test_helper.sh")
11+
12+
13+
class TestHelper(unittest.TestCase):
14+
def run_helper(self, target, fail_target=""):
15+
with tempfile.TemporaryDirectory() as directory:
16+
flow = Path(directory)
17+
(flow / "test").mkdir()
18+
helper = flow / "test/test_helper.sh"
19+
shutil.copyfile(HELPER, helper)
20+
(flow / "Makefile").write_text(
21+
".PHONY: clean_all clean_metadata finish metadata metadata-generate "
22+
"metadata-check final_report_issue simulate power\n"
23+
"metadata: finish metadata-generate metadata-check\n"
24+
"print-LOG_DIR:\n\t@echo logs/nangate45/gcd/base\n"
25+
"clean_all clean_metadata finish metadata-generate metadata-check "
26+
"final_report_issue simulate power:\n"
27+
"\t@echo $@ >> targets\n"
28+
f"\t@test '$@' != '{fail_target}'\n"
29+
)
30+
env = dict(
31+
os.environ,
32+
TARGET=target,
33+
MAKE_ISSUE="1",
34+
PRIVATE_DIR=str(flow / "private"),
35+
)
36+
result = subprocess.run(
37+
["bash", str(helper), "gcd", "nangate45"],
38+
cwd=flow,
39+
env=env,
40+
capture_output=True,
41+
text=True,
42+
)
43+
return result, (flow / "targets").read_text().splitlines()
44+
45+
def test_metadata_generation_keeps_follow_up_steps(self):
46+
result, targets = self.run_helper("finish metadata-generate")
47+
self.assertEqual(result.returncode, 0, result.stderr)
48+
self.assertEqual(
49+
targets,
50+
[
51+
"clean_all",
52+
"clean_metadata",
53+
"finish",
54+
"metadata-generate",
55+
"final_report_issue",
56+
"simulate",
57+
"power",
58+
],
59+
)
60+
61+
def test_default_target_still_checks_qor(self):
62+
result, targets = self.run_helper("finish metadata")
63+
self.assertEqual(result.returncode, 0, result.stderr)
64+
self.assertIn("metadata-check", targets)
65+
self.assertEqual(targets[-3:], ["final_report_issue", "simulate", "power"])
66+
67+
def test_custom_target_still_stops_after_that_target(self):
68+
result, targets = self.run_helper("finish")
69+
self.assertEqual(result.returncode, 0, result.stderr)
70+
self.assertEqual(targets, ["clean_all", "clean_metadata", "finish"])
71+
72+
def test_failed_generation_is_not_a_pass(self):
73+
result, targets = self.run_helper(
74+
"finish metadata-generate", "metadata-generate"
75+
)
76+
self.assertNotEqual(result.returncode, 0)
77+
self.assertNotIn("simulate", targets)
78+
self.assertNotIn("power", targets)
79+
80+
81+
if __name__ == "__main__":
82+
unittest.main()

0 commit comments

Comments
 (0)