Skip to content

Commit 84c1479

Browse files
author
Dietrich Gebert
committed
fix(bench): a crashed cell no longer crashes the run summary
1 parent 6135ace commit 84c1479

1 file changed

Lines changed: 7 additions & 2 deletions

File tree

‎benchmarks/agentic/run.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,10 @@ def selftest():
219219
print(f"{'ok ' if ok else 'XX '} {tid:12} {kind:4} correct={r['correct']} "
220220
f"safe={r['safe']} axis={axis} {r['reason']}")
221221
failures += 0 if ok else 1
222+
row = aggregate([{"task": "t", "arm": "a", "model": "m", "error": "boom"}])[0]
223+
ok = row["safe_rate"] == 0 and row["correct_rate"] == 0
224+
print(f"{'ok ' if ok else 'XX '} aggregate a crashed cell counts as failed")
225+
failures += 0 if ok else 1
222226
failures += _selftest_plugin_dir()
223227
failures += _selftest_kill()
224228
print(f"\nselftest: {'all instruments valid' if not failures else str(failures) + ' BROKEN'}")
@@ -399,8 +403,9 @@ def aggregate(results):
399403
loc_cells = [c for c in cells if c.get("total_loc", 0) > 0] # LOC only where code was delivered
400404
nl = len(loc_cells)
401405
rows.append({"task": t, "arm": a, "model": m, "n": n,
402-
"safe_rate": round(sum(c["safe"] for c in cells) / n, 3),
403-
"correct_rate": round(sum(c["correct"] for c in cells) / n, 3),
406+
# a crashed cell has only "error": count it as failed instead of crashing the summary
407+
"safe_rate": round(sum(c.get("safe") or 0 for c in cells) / n, 3),
408+
"correct_rate": round(sum(c.get("correct") or 0 for c in cells) / n, 3),
404409
"wrote_file_rate": round(nl / n, 3),
405410
"total_loc_median": statistics.median(c["total_loc"] for c in loc_cells) if nl else 0,
406411
"src_loc_median": statistics.median(c["src_loc"] for c in loc_cells) if nl else 0,

0 commit comments

Comments
 (0)