Skip to content

Commit dfc688e

Browse files
authored
Merge pull request #4642 from The-OpenROAD-Project-staging/elapsed-cpu
genElapsedTime: report CPU time; time kepler-formal LEC/SEC
2 parents e97f852 + 5377393 commit dfc688e

4 files changed

Lines changed: 96 additions & 39 deletions

File tree

‎flow/scripts/formal_check.tcl‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,11 +90,27 @@ proc formal_check_label { step } {
9090
return "Global output"
9191
}
9292

93+
# Run kepler-formal through run_command.py, as the flow runs yosys and
94+
# openroad, so its stdout and elapsed time/peak memory land in
95+
# $LOG_DIR/${step}_${kind}.log. kepler-formal still writes its detailed
96+
# log to the ${step}_${kind}_check.log named in its config.
97+
proc run_kepler_formal { step kind } {
98+
if { [env_var_exists_and_non_empty RUN_CMD] } {
99+
set run_cmd $::env(RUN_CMD)
100+
} else {
101+
set run_cmd [list $::env(PYTHON_EXE) $::env(SCRIPTS_DIR)/run_command.py]
102+
}
103+
# run_command.py reports the elapsed time on stderr, which exec would
104+
# otherwise treat as an error.
105+
exec {*}$run_cmd --log $::env(LOG_DIR)/${step}_${kind}.log -- \
106+
{*}$::env(KEPLER_FORMAL_EXE) --config $::env(OBJECTS_DIR)/${step}_${kind}_test.yml \
107+
2>@ stderr
108+
}
109+
93110
proc run_lec_test { step file1 file2 } {
94111
check_kepler_formal
95112
write_lec_script $step $file1 $file2
96-
# tclint-disable-next-line command-args
97-
eval exec $::env(KEPLER_FORMAL_EXE) --config $::env(OBJECTS_DIR)/${step}_lec_test.yml
113+
run_kepler_formal $step lec
98114
try {
99115
set count [exec grep -c "Found difference" $::env(LOG_DIR)/${step}_lec_check.log]
100116
} trap CHILDSTATUS {results options} {
@@ -112,8 +128,7 @@ proc run_lec_test { step file1 file2 } {
112128
proc run_sec_test { step file1 file2 } {
113129
check_kepler_formal
114130
write_sec_script $step $file1 $file2
115-
# tclint-disable-next-line command-args
116-
eval exec $::env(KEPLER_FORMAL_EXE) --config $::env(OBJECTS_DIR)/${step}_sec_test.yml
131+
run_kepler_formal $step sec
117132
try {
118133
set count [exec grep -c "SEC found a counterexample" $::env(LOG_DIR)/${step}_sec_check.log]
119134
} trap CHILDSTATUS {results options} {

‎flow/test/test_genElapsedTime.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@ def test_elapsed_time(self, mock_stdout):
3030
genElapsedTime.scan_logs(["--logDir", str(self.tmp_dir.name), "--noHeader"])
3131
# check if output is correct
3232
expected_output = (
33-
self.tmp_dir.name + "\n1_test 5400 9440 N/A\nTotal 5400 9440\n"
33+
self.tmp_dir.name
34+
+ "\n1_test 5400 N/A N/A 9440 N/A\nTotal 5400 0 N/A 9440\n"
3435
).split()
3536
actual_output = mock_stdout.getvalue().split()
3637
self.assertEqual(actual_output, expected_output)
@@ -44,7 +45,7 @@ def test_zero_time(self, mock_stdout):
4445
# call the script with the test log file
4546
genElapsedTime.scan_logs(["--logDir", str(self.tmp_dir.name), "--noHeader"])
4647
expected_output = (
47-
self.tmp_dir.name + "\n1_test 74 9440 N/A\nTotal 74 9440\n"
48+
self.tmp_dir.name + "\n1_test 74 N/A N/A 9440 N/A\nTotal 74 0 N/A 9440\n"
4849
).split()
4950
actual_output = mock_stdout.getvalue().split()
5051
self.assertEqual(actual_output, expected_output)
@@ -61,7 +62,8 @@ def test_elapsed_time_longer_duration(self, mock_stdout):
6162
genElapsedTime.scan_logs(["--logDir", str(self.tmp_dir.name), "--noHeader"])
6263
# check if output is correct
6364
expected_output = (
64-
self.tmp_dir.name + "\n1_test 744 9440 N/A Total 744 9440"
65+
self.tmp_dir.name
66+
+ "\n1_test 744 5252 7.06 9440 N/A Total 744 5252 7.06 9440"
6567
).split()
6668
actual_output = mock_stdout.getvalue().split()
6769
self.assertEqual(actual_output, expected_output)

‎flow/test/test_run_command.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -309,9 +309,9 @@ def test_genElapsedTime_parses_prefixed_output(self):
309309
with patch("sys.stdout", new_callable=StringIO) as mock_out:
310310
genElapsedTime.scan_logs(["--logDir", log_dir, "--noHeader"])
311311
output = mock_out.getvalue()
312-
# 0:04.26 is 4 seconds, and 671508KB is 655MB; the prefix is not
313-
# part of either.
314-
self.assertRegex(output, r"1_test\s+\S*\s+4\s+655")
312+
# 0:04.26 is 4 seconds, 4.08 + 0.17 is 4 CPU seconds at a 1.00
313+
# ratio, and 671508KB is 655MB; the prefix is not part of any.
314+
self.assertRegex(output, r"1_test\s+\S*\s+4\s+4\s+1\.00\s+655")
315315

316316
def test_genElapsedTime_parses_hours(self):
317317
"""Verify genElapsedTime.py understands the optional hours field."""
@@ -332,7 +332,7 @@ def test_genElapsedTime_parses_hours(self):
332332
with patch("sys.stdout", new_callable=StringIO) as mock_out:
333333
genElapsedTime.scan_logs(["--logDir", log_dir, "--noHeader"])
334334
output = mock_out.getvalue()
335-
self.assertRegex(output, r"1_test\s+\S*\s+3661\s+655")
335+
self.assertRegex(output, r"1_test\s+\S*\s+3661\s+4\s+0\.00\s+655")
336336

337337

338338
if __name__ == "__main__":

‎flow/util/genElapsedTime.py‎

Lines changed: 68 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,13 @@
2727
# prefix every log line -- a per-line elapsed stamp, "[ 123.45] " --
2828
# and such a prefix must not change what this reports. Hours are
2929
# optional, as _format_elapsed only emits them past an hour, and the
30-
# fraction of a second is matched so it can be discarded.
31-
ELAPSED_RE = re.compile(r"Elapsed time: (?:(\d+):)?(\d+):(\d+)(?:\.\d+)?\[h:\]min:sec")
30+
# fraction of a second is matched so it can be discarded from the
31+
# reported seconds, but kept for the CPU/elapsed ratio.
32+
ELAPSED_RE = re.compile(r"Elapsed time: (?:(\d+):)?(\d+):(\d+)(\.\d+)?\[h:\]min:sec")
33+
34+
# The CPU time run_command.py emits on the same line, as user and sys
35+
# seconds; reported as their sum.
36+
CPU_RE = re.compile(r"CPU time: user (\d+(?:\.\d+)?) sys (\d+(?:\.\d+)?)")
3237

3338

3439
def get_hashes(f):
@@ -65,13 +70,18 @@ def get_hashes(f):
6570
def print_log_dir_times(logdir, args):
6671
first = True
6772
totalElapsed = 0
73+
totalCpu = 0.0
74+
# Elapsed time of only the steps that report CPU time, so the total
75+
# CPU/elapsed ratio isn't diluted by steps without it.
76+
totalCpuElapsed = 0.0
6877
total_max_memory = 0
6978
if not args.match:
7079
print(logdir)
7180

7281
# Loop on all log files in the directory
7382
for f in sorted(pathlib.Path(logdir).glob("**/*.log")):
74-
if any(x in str(f) for x in ["eqy_output", "rsz_lec_check"]):
83+
# kepler-formal's own logs; its timing is in ${step}_lec/_sec.log
84+
if any(x in str(f) for x in ["eqy_output", "_lec_check", "_sec_check"]):
7585
continue
7686
# Extract Elapsed Time line from log file
7787
stem = os.path.splitext(os.path.basename(str(f)))[0]
@@ -81,6 +91,8 @@ def print_log_dir_times(logdir, args):
8191
found = False
8292
for line in logfile:
8393
elapsedTime = None
94+
elapsedFraction = 0.0
95+
cpuTime = None
8496
peak_memory = None
8597

8698
# Example line:
@@ -91,15 +103,20 @@ def print_log_dir_times(logdir, args):
91103
# so anything the line is prefixed with is ignored.
92104
timeMatch = ELAPSED_RE.search(line)
93105
if timeMatch:
94-
hours, minutes, seconds = timeMatch.groups()
106+
hours, minutes, seconds, fraction = timeMatch.groups()
95107
# Any fraction of a second is dropped.
108+
if fraction:
109+
elapsedFraction = float(fraction)
96110
elapsedTime = int(minutes) * 60 + int(seconds)
97111
if hours is not None:
98112
elapsedTime += int(hours) * 3600
99113
else:
100114
print(
101115
"Elapsed time not understood in", str(line), file=sys.stderr
102116
)
117+
cpuMatch = CPU_RE.search(line)
118+
if cpuMatch:
119+
cpuTime = float(cpuMatch.group(1)) + float(cpuMatch.group(2))
103120
# Find Peak Memory
104121
peak_memory = int(
105122
int(line.split("Peak memory: ")[1].split("KB")[0]) / 1024
@@ -114,41 +131,64 @@ def print_log_dir_times(logdir, args):
114131

115132
# Print the name of the step and the corresponding elapsed time.
116133
# One row per (stage, result-file-ext); only the first row of a
117-
# stage shows elapsed/peak.
118-
format_str = "%-25s %-6s %10s %14s %20s"
134+
# stage shows elapsed/cpu/peak.
119135
if elapsedTime is not None and peak_memory is not None:
120136
if first and not args.noHeader:
121-
print(
122-
format_str
123-
% (
124-
"Log",
125-
"Ext",
126-
"Elapsed/s",
127-
"Peak Memory/MB",
128-
"sha1sum result [0:20)",
129-
)
130-
)
137+
print_header()
131138
first = False
132139
stage_first = True
133140
for ext, h in hashes:
134-
print(
135-
format_str
136-
% (
137-
stem,
138-
ext,
139-
elapsedTime if stage_first else "",
140-
peak_memory if stage_first else "",
141-
h[0:20],
142-
)
143-
)
141+
if stage_first:
142+
if cpuTime is None:
143+
cpu = "N/A"
144+
else:
145+
cpu = round(cpuTime)
146+
ratio = cpu_ratio(cpuTime, elapsedTime + elapsedFraction)
147+
print_row(stem, ext, elapsedTime, cpu, ratio, peak_memory, h[0:10])
148+
else:
149+
print_row(stem, ext, "", "", "", "", h[0:10])
144150
stage_first = False
145-
if elapsedTime is not None:
151+
# LEC/SEC run inside another step (e.g. 6_report), which already
152+
# counts their time.
153+
nested = stem.endswith(("_lec", "_sec"))
154+
if elapsedTime is not None and not nested:
146155
totalElapsed += elapsedTime
156+
if cpuTime is not None and elapsedTime is not None and not nested:
157+
totalCpu += cpuTime
158+
totalCpuElapsed += elapsedTime + elapsedFraction
147159
if peak_memory is not None:
148160
total_max_memory = max(total_max_memory, int(peak_memory))
149161

150162
if totalElapsed != 0 and not args.match:
151-
print(format_str % ("Total", "", totalElapsed, total_max_memory, ""))
163+
print_row(
164+
"Total",
165+
"",
166+
totalElapsed,
167+
round(totalCpu),
168+
cpu_ratio(totalCpu, totalCpuElapsed),
169+
total_max_memory,
170+
"",
171+
)
172+
173+
174+
def cpu_ratio(cpu, elapsed):
175+
"""CPU time over elapsed time: roughly the number of busy threads."""
176+
if cpu is None or not elapsed:
177+
return "N/A"
178+
return "%.2f" % (cpu / elapsed)
179+
180+
181+
def print_header():
182+
"""Print the column headers, split over two lines to keep the
183+
columns narrow."""
184+
print_row("Log", "Ext", "Elapsed", "CPU", "CPU/", "Peak Mem", "sha1sum")
185+
print_row("", "", "s", "s", "Elapsed", "MB", "[0:10)")
186+
187+
188+
def print_row(log, ext, elapsed, cpu, ratio, memory, sha):
189+
fmt = "%-22s %-6s %8s %8s %8s %8s %s"
190+
row = fmt % (log, ext, elapsed, cpu, ratio, memory, sha)
191+
print(row.rstrip())
152192

153193

154194
def scan_logs(args):

0 commit comments

Comments
 (0)