Skip to content

Commit ebf0fb3

Browse files
JuliaEdomclaude
andcommitted
fix: gold-fail empty-empty false positive (Codex audit 2026-05-25 #1)
Mirror of the qid 518 pred-side bug already fixed in c74b46c, but on the gold side. `_execute_gold` historically returned `([], [])` when BIRD's gold SQL crashed (both `execute_readonly` and the raw-connection fallback raised SQLAlchemyError/MemoryError — ~1% of BIRD gold SQLs). If a pred happened to also return zero rows (e.g. `SELECT * WHERE 1=0`, or a wrong filter that filtered out everything), `compare_results([], [])` blessed match=True under BIRD set-equality scoring. Changes: - `_execute_gold_with_status` returns `(rows, cols, gold_failed: bool)`; the legacy 2-tuple `_execute_gold` is retained as a thin wrapper so the dozen+ scripts that import it keep working (their migration is Codex audit #2-4, separate commit). - `_compare_outcome` accepts a `gold_failed` kwarg and short-circuits to `match=False, reason='gold execution failed'` when set. - `safe_compare_pred` gains a symmetric `gold_failed: bool = False` kwarg for voting/rescoring scripts that bypass the runner. - All 3 runner-internal callsites (run_config_a / run_config_c-via-pipeline / self-consistency voter) now route via `_execute_gold_with_status` and pass the flag through to `_compare_outcome`. - Regression tests: - `tests/eval/test_metrics.py::TestSafeComparePred::test_gold_failed_*` (2) - `tests/eval/test_runner.py::test_run_config_a_gold_fail_with_empty_pred_does_not_false_positive` Gates: ruff check ✓ · ruff format --check ✓ · mypy strict ✓ · 347 tests pass (was 344, +3 new). No behaviour change for the 99% of BIRD gold SQLs that execute cleanly; only the gold-fail + empty-pred quirk path closes. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 4a79ecb commit ebf0fb3

4 files changed

Lines changed: 138 additions & 20 deletions

File tree

‎src/nl_sql/eval/metrics/execution_accuracy.py‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -110,26 +110,41 @@ def safe_compare_pred(
110110
*,
111111
gold_sql: str | None = None,
112112
pred_failed: bool,
113+
gold_failed: bool = False,
113114
) -> ResultComparison:
114-
"""Comparison wrapper that short-circuits pred execution failures.
115+
"""Comparison wrapper that short-circuits pred OR gold execution failures.
115116
116117
Plain `compare_results` is row-level: it treats `pred_rows=[]` identically
117118
whether pred returned zero rows or pred raised before producing any. When
118119
gold also returns zero rows (BIRD quirks: empty filter results, missing
119120
Banned legalities, etc.), `compare_results([], [])` returns match=True —
120121
a silent false positive for malformed pred SQL.
121122
122-
The runner's `_run_one` already handles this (see eval/runner.py:662 —
123-
explicit ResultComparison(match=False) when result.outcome is None).
124-
Voting and rescoring scripts that bypass the runner must use this helper
125-
instead of calling compare_results directly.
123+
Symmetric defect on the gold side: `_execute_gold` historically returned
124+
`([], [])` when BIRD's gold SQL crashed (~1% of cases), and any pred that
125+
happened to also return zero rows would then be blessed as match=True.
126+
127+
The runner's `_run_one` and `_run_one_via_pipeline` paths already route
128+
pred-failure and gold-failure through `_compare_outcome` / direct
129+
`ResultComparison(match=False)`. Voting and rescoring scripts that bypass
130+
the runner must use this helper instead of calling `compare_results`
131+
directly. Pass `pred_failed=True` when pred SQL raised, `gold_failed=True`
132+
when gold SQL raised.
126133
127134
Discovered via Codex review of c74b46c → qid 518 (card_games moderate
128135
"format with most banned cards"): pred CTE missing the WITH prefix,
129136
SyntaxError on every execution, gold returns 0 rows for that DB, scoring
130137
blessed it as match=True since v13 (helallao grok-4.1-reasoning rescue).
131138
Re-merge v22-v29 + 2026-05-25 EOD fix lands the correction.
139+
Gold-side mirror is Codex audit 2026-05-25 #1 (`runner.py:960`).
132140
"""
141+
if gold_failed:
142+
return ResultComparison(
143+
match=False,
144+
reason="gold execution failed",
145+
gold_rows=0,
146+
pred_rows=len(pred_rows),
147+
)
133148
if pred_failed:
134149
return ResultComparison(
135150
match=False,

‎src/nl_sql/eval/runner.py‎

Lines changed: 72 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -558,13 +558,15 @@ def _run_one_config_a(
558558
statement_timeout_ms=statement_timeout_ms,
559559
row_cap=row_cap,
560560
)
561-
gold_rows, _gold_columns = _execute_gold(
561+
gold_rows, _gold_columns, gold_failed = _execute_gold_with_status(
562562
engine,
563563
example.sql,
564564
statement_timeout_ms=statement_timeout_ms,
565565
row_cap=row_cap,
566566
)
567-
comparison = _compare_outcome(outcome, gold_rows, gold_sql=example.sql)
567+
comparison = _compare_outcome(
568+
outcome, gold_rows, gold_sql=example.sql, gold_failed=gold_failed
569+
)
568570
gold_tables = tuple(extract_gold_tables(example.sql))
569571
retrieved = tuple(c.table_name for c in chunks)
570572
recall = schema_recall_at_k(gold_tables, retrieved)
@@ -650,7 +652,7 @@ def _run_one_via_pipeline(
650652
gold_row_count=0,
651653
comparison_reason=f"pipeline raised: {exc!r}",
652654
)
653-
gold_rows, _ = _execute_gold(
655+
gold_rows, _, gold_failed = _execute_gold_with_status(
654656
gold_engine,
655657
example.sql,
656658
statement_timeout_ms=statement_timeout_ms,
@@ -659,7 +661,18 @@ def _run_one_via_pipeline(
659661
# The pipeline's outcome is what `match` should reflect — but the
660662
# comparison runs against the gold rows we just fetched. Build a
661663
# synthetic outcome view for `_compare_outcome`, or pull rows out.
662-
if result.outcome is not None and result.outcome.result is not None:
664+
if gold_failed:
665+
comparison = ResultComparison(
666+
match=False,
667+
reason="gold execution failed",
668+
gold_rows=0,
669+
pred_rows=(
670+
len(result.outcome.result.rows)
671+
if result.outcome is not None and result.outcome.result is not None
672+
else 0
673+
),
674+
)
675+
elif result.outcome is not None and result.outcome.result is not None:
663676
comparison = compare_results(
664677
gold_rows,
665678
result.outcome.result.rows,
@@ -772,13 +785,24 @@ def _run_one_self_consistency(
772785

773786
winner = vote(candidates)
774787
result = winner.result
775-
gold_rows, _ = _execute_gold(
788+
gold_rows, _, gold_failed = _execute_gold_with_status(
776789
gold_engine,
777790
example.sql,
778791
statement_timeout_ms=statement_timeout_ms,
779792
row_cap=row_cap,
780793
)
781-
if result.outcome is not None and result.outcome.result is not None:
794+
if gold_failed:
795+
comparison = ResultComparison(
796+
match=False,
797+
reason="gold execution failed",
798+
gold_rows=0,
799+
pred_rows=(
800+
len(result.outcome.result.rows)
801+
if result.outcome is not None and result.outcome.result is not None
802+
else 0
803+
),
804+
)
805+
elif result.outcome is not None and result.outcome.result is not None:
782806
comparison = compare_results(
783807
gold_rows, result.outcome.result.rows, gold_sql=example.sql
784808
)
@@ -926,23 +950,26 @@ def _compose_question(example: BirdExample) -> str:
926950
return f"{example.question}\n\nHint: {example.evidence}"
927951

928952

929-
def _execute_gold(
953+
def _execute_gold_with_status(
930954
engine: Engine,
931955
sql: str,
932956
*,
933957
statement_timeout_ms: int,
934958
row_cap: int,
935-
) -> tuple[list[tuple[Any, ...]], list[str]]:
936-
"""Run gold SQL with the same row cap / timeout as predictions.
937-
938-
Bypasses the validator (gold is trusted, BIRD ships it). Errors propagate
939-
as empty result + sentinel — the EA comparison will then fail naturally.
959+
) -> tuple[list[tuple[Any, ...]], list[str], bool]:
960+
"""Run gold SQL and return `(rows, columns, gold_failed)`.
961+
962+
Mirror of `_execute_gold` that surfaces the failure flag. Used by the
963+
runner internals so `_compare_outcome` can short-circuit gold-failure
964+
instead of letting `compare_results([], [])` bless an empty pred as
965+
match=True (Codex audit 2026-05-25 #1, same defect class as the qid 518
966+
pred-side bug already fixed in `safe_compare_pred`).
940967
"""
941968
try:
942969
with execute_readonly(
943970
engine, sql, statement_timeout_ms=statement_timeout_ms, row_cap=row_cap
944971
) as result:
945-
return list(result.rows), list(result.columns)
972+
return list(result.rows), list(result.columns), False
946973
except (SQLAlchemyError, MemoryError):
947974
# Last-resort: try the raw connection to surface gold-SQL bugs in
948975
# logs without crashing the runner. BIRD ships ~1% gold SQLs that
@@ -955,17 +982,47 @@ def _execute_gold(
955982
cols = list(cursor.keys())
956983
rows = [tuple(r) for r in cursor.fetchmany(row_cap)]
957984
cursor.close()
958-
return rows, cols
985+
return rows, cols, False
959986
except (SQLAlchemyError, MemoryError):
960-
return [], []
987+
return [], [], True
988+
989+
990+
def _execute_gold(
991+
engine: Engine,
992+
sql: str,
993+
*,
994+
statement_timeout_ms: int,
995+
row_cap: int,
996+
) -> tuple[list[tuple[Any, ...]], list[str]]:
997+
"""Run gold SQL with the same row cap / timeout as predictions.
998+
999+
Bypasses the validator (gold is trusted, BIRD ships it). Errors propagate
1000+
as empty result + sentinel — the EA comparison will then fail naturally.
1001+
1002+
Legacy 2-tuple wrapper retained for the dozen+ scripts that import this
1003+
name; new runner-internal callsites should use `_execute_gold_with_status`
1004+
so the gold-failure flag can route to `safe_compare_pred(gold_failed=True)`.
1005+
"""
1006+
rows, cols, _gold_failed = _execute_gold_with_status(
1007+
engine, sql, statement_timeout_ms=statement_timeout_ms, row_cap=row_cap
1008+
)
1009+
return rows, cols
9611010

9621011

9631012
def _compare_outcome(
9641013
outcome: ExecutionOutcome,
9651014
gold_rows: list[tuple[Any, ...]],
9661015
*,
9671016
gold_sql: str,
1017+
gold_failed: bool = False,
9681018
) -> ResultComparison:
1019+
if gold_failed:
1020+
return ResultComparison(
1021+
match=False,
1022+
reason="gold execution failed",
1023+
gold_rows=0,
1024+
pred_rows=0 if outcome.result is None else len(outcome.result.rows),
1025+
)
9691026
if outcome.result is None:
9701027
return ResultComparison(
9711028
match=False,

‎tests/eval/test_metrics.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,25 @@ def test_pred_failed_but_gold_nonempty_still_false(self) -> None:
4141
assert cmp.gold_rows == 2
4242
assert cmp.pred_rows == 0
4343

44+
def test_gold_failed_short_circuits_to_false_even_when_both_empty(self) -> None:
45+
# Codex audit 2026-05-25 #1 — gold-side mirror of the qid 518 bug.
46+
# _execute_gold returned ([], []) when BIRD gold crashed; if pred also
47+
# happened to return zero rows, compare_results([], []) blessed match=True.
48+
cmp = safe_compare_pred([], [], pred_failed=False, gold_failed=True)
49+
assert not cmp.match
50+
assert cmp.reason == "gold execution failed"
51+
# gold_failed takes precedence over pred_failed in the reason field
52+
# because gold-side failure invalidates the whole scoring attempt.
53+
cmp2 = safe_compare_pred([], [], pred_failed=True, gold_failed=True)
54+
assert not cmp2.match
55+
assert cmp2.reason == "gold execution failed"
56+
57+
def test_gold_failed_with_nonempty_pred_still_false(self) -> None:
58+
cmp = safe_compare_pred([], [(1, "a")], pred_failed=False, gold_failed=True)
59+
assert not cmp.match
60+
assert cmp.gold_rows == 0
61+
assert cmp.pred_rows == 1
62+
4463

4564
class TestCompareResults:
4665
def test_identical_rows_match_set_eq(self) -> None:

‎tests/eval/test_runner.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,33 @@ def test_run_config_a_handles_broken_gold_sql(
191191
assert rec.gold_row_count == 0
192192

193193

194+
def test_run_config_a_gold_fail_with_empty_pred_does_not_false_positive(
195+
chinook_registry: DatabaseRegistry,
196+
) -> None:
197+
"""Regression: gold-fail + empty-pred must NOT be blessed as match=True.
198+
199+
Codex audit 2026-05-25 #1, mirror of the qid 518 pred-side bug fixed in
200+
c74b46c. Pre-fix `_execute_gold` returned ([], []) on gold SQL crash, and
201+
if pred happened to also return zero rows (e.g. `SELECT * WHERE 0=1`),
202+
`compare_results([], [])` blessed match=True under BIRD set-equality.
203+
"""
204+
pred = json.dumps(
205+
{
206+
"sql": "SELECT AlbumId FROM Album WHERE 1=0",
207+
"tables_used": ["Album"],
208+
}
209+
)
210+
llm = ScriptedLLM([pred])
211+
# Bogus gold — both `execute_readonly` and the raw-connection fallback
212+
# raise SQLAlchemyError → `_execute_gold_with_status` returns gold_failed=True.
213+
examples = [_example(99, "q", "SELECT * FROM TableThatDoesNotExist")]
214+
run = run_config_a(examples, sql_provider=llm, registry=chinook_registry)
215+
216+
rec = run.records[0]
217+
assert rec.match is False
218+
assert rec.comparison_reason == "gold execution failed"
219+
220+
194221
def test_progress_callback_invoked(chinook_registry: DatabaseRegistry) -> None:
195222
correct = json.dumps({"sql": "SELECT COUNT(*) FROM Album"})
196223
llm = ScriptedLLM([correct, correct])

0 commit comments

Comments
 (0)