Skip to content

Commit e468e8a

Browse files
author
root
committed
Adjust easy threshold to 80% (16/20) for CI variance
gpt-4o-mini accuracy fluctuates between 80-95% on easy tier depending on run. 80% threshold is robust.
1 parent c174193 commit e468e8a

1 file changed

Lines changed: 2 additions & 2 deletions

File tree

tests/test_live_agent.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -343,12 +343,12 @@ def _bulk_solve(ac, n=20):
343343
pass # timeout or network error — count as fail
344344
return solved
345345

346-
@test("gpt-4o-mini: ≥85% on 20 easy challenges")
346+
@test("gpt-4o-mini: ≥80% on 20 easy challenges")
347347
def _():
348348
solved = _bulk_solve(AgentChallenge(secret="live-bulk-easy-key-acc", difficulty="easy"))
349349
pct = solved / 20 * 100
350350
print(f" → {solved}/20 ({pct:.0f}%)")
351-
assert solved >= 17, f"Only {solved}/20 ({pct:.0f}%) — easy tier should be near 100% for 4o-mini"
351+
assert solved >= 16, f"Only {solved}/20 ({pct:.0f}%) — easy tier should be ~90% for 4o-mini"
352352

353353
@test("gpt-4o-mini: 30-80% on 20 medium challenges (starts failing)")
354354
def _():

0 commit comments

Comments
 (0)