Skip to content

Commit e40e4da

Browse files
JuliaEdomclaude
andcommitted
fix: route voting/rescore through safe_compare_pred (Codex audit #2-4)
Three same-pattern findings: pred/gold execution exceptions get swallowed into empty rows, then `compare_results` direct-call blesses empty-empty as match=True. Mirrors of the qid 518 bug surface area: #2 `scripts/merge_voting_rescues.py:73` — flipped baseline match=True from stored alt_match booleans without re-executing SQL. Pre-fix voting JSONs (with stale empty-empty match=True) could silently inflate EA. Fix: default `--reverify` re-executes pred+gold via safe_compare_pred (pred_failed + gold_failed flags) and rejects unverified candidates. `--no-reverify` escape hatch retained for trusted legacy merges. #3 `scripts/run_helallao_voting.py:189` — pred execution exceptions sank alt_rows to [], then compare_results called directly. Fix: track pred_failed + gold_failed via _execute_gold_with_status, route through safe_compare_pred. #4 `scripts/rescore_arcwise.py:127` — corrected-gold execution exceptions sank gold_rows to [] then safe_compare_pred(pred_failed=...) — but gold_failed wasn't tracked. Fix: switch to _execute_gold_with_status, pass gold_failed through to safe_compare_pred. Tests: - tests/scripts/test_merge_voting_rescues.py: 4 new tests (reverify rejects pred-fail / accepts real match / rejects gold-fail / no-reverify flag skips path) - tests/scripts/test_rescore_arcwise.py: 2 existing tests updated to monkeypatch _execute_gold_with_status (3-tuple return) Gates: ruff check ✓ · ruff format --check ✓ · mypy strict ✓ · 351 tests pass (was 347, +4 new). Existing voting JSONs on disk are not retroactively re-scored by this commit — historical baselines stay as-committed; only new merges through merge_voting_rescues benefit from the gate. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent ebf0fb3 commit e40e4da

5 files changed

Lines changed: 317 additions & 14 deletions

File tree

‎scripts/merge_voting_rescues.py‎

Lines changed: 96 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,82 @@
2929
from pathlib import Path
3030
from typing import Any
3131

32+
from nl_sql.db.registry import get_default_registry
33+
from nl_sql.eval.metrics.execution_accuracy import safe_compare_pred
34+
from nl_sql.eval.runner import _execute_gold_with_status
35+
from nl_sql.execution.runner import execute_validated
36+
37+
38+
def _reverify_candidate(
39+
baseline_record: dict[str, Any],
40+
alt_pred: str,
41+
registry: Any,
42+
) -> tuple[bool, str]:
43+
"""Re-execute alt pred + gold; return (verified_match, reason).
44+
45+
Voting reports produced before commit c74b46c (`safe_compare_pred` fix)
46+
can carry stale `alt_match=True` for empty-pred + empty-gold cases.
47+
Re-running closes that loophole — Codex audit 2026-05-25 #2.
48+
"""
49+
db_id = baseline_record.get("db_id")
50+
gold_sql = baseline_record.get("gold_sql")
51+
if not db_id or not gold_sql:
52+
return False, "baseline record missing db_id or gold_sql"
53+
try:
54+
engine = registry.engine_for(db_id)
55+
except Exception as exc:
56+
return False, f"engine unavailable for db_id={db_id}: {exc}"
57+
pred_rows: list[tuple[Any, ...]] = []
58+
pred_failed = False
59+
if alt_pred.strip():
60+
try:
61+
outcome = execute_validated(
62+
engine,
63+
alt_pred,
64+
dialect="sqlite",
65+
statement_timeout_ms=30_000,
66+
row_cap=10_000,
67+
)
68+
if outcome.result:
69+
pred_rows = list(outcome.result.rows)
70+
else:
71+
pred_failed = True
72+
except Exception:
73+
pred_failed = True
74+
else:
75+
pred_failed = True
76+
try:
77+
gold_rows, _, gold_failed = _execute_gold_with_status(
78+
engine, gold_sql, statement_timeout_ms=30_000, row_cap=10_000
79+
)
80+
except Exception:
81+
gold_rows = []
82+
gold_failed = True
83+
cmp = safe_compare_pred(
84+
gold_rows,
85+
pred_rows,
86+
gold_sql=gold_sql,
87+
pred_failed=pred_failed,
88+
gold_failed=gold_failed,
89+
)
90+
return bool(cmp.match), cmp.reason
91+
3292

3393
def main() -> int:
3494
p = argparse.ArgumentParser(description=__doc__)
3595
p.add_argument("--baseline", type=Path, required=True)
3696
p.add_argument("--voting", type=Path, nargs="+", required=True)
3797
p.add_argument("--out", type=Path, required=True)
98+
p.add_argument(
99+
"--no-reverify",
100+
action="store_true",
101+
help=(
102+
"Trust the stored alt_match flag from voting reports without "
103+
"re-executing pred+gold via safe_compare_pred. Default is to "
104+
"reverify so pre-fix voting JSONs (empty-empty false positives) "
105+
"are rejected at merge time. Codex audit 2026-05-25 #2."
106+
),
107+
)
38108
args = p.parse_args()
39109

40110
base = json.loads(args.baseline.read_text(encoding="utf-8"))
@@ -62,13 +132,33 @@ def main() -> int:
62132
candidates[qid].append({"alt_model": alt_model, "alt_pred": vr["alt_pred"]})
63133

64134
# Apply: first valid candidate wins (we iterate in CLI order).
135+
# Default behavior re-verifies each candidate via safe_compare_pred so
136+
# stale empty-empty false positives in pre-fix voting JSONs cannot
137+
# silently inflate baseline EA (Codex audit 2026-05-25 #2).
65138
rescues = 0
139+
rejected_stale = 0
66140
rescue_models: Counter[str] = Counter()
141+
registry = None if args.no_reverify else get_default_registry()
67142
for qid, cands in candidates.items():
68143
br = by_qid[qid]
69144
if br.get("match"):
70145
continue
71-
winner = cands[0]
146+
winner = None
147+
for cand in cands:
148+
if args.no_reverify:
149+
winner = cand
150+
break
151+
verified, reason = _reverify_candidate(br, cand["alt_pred"], registry)
152+
if verified:
153+
winner = cand
154+
break
155+
rejected_stale += 1
156+
print(
157+
f" reject qid={qid} alt_model={cand['alt_model']}: {reason}",
158+
file=sys.stderr,
159+
)
160+
if winner is None:
161+
continue
72162
br["pred_sql"] = winner["alt_pred"]
73163
br["match"] = True
74164
br["voted_by"] = winner["alt_model"]
@@ -113,6 +203,11 @@ def main() -> int:
113203
args.out.write_text(json.dumps(base, indent=2, default=str), encoding="utf-8")
114204

115205
print(f"Rescues applied: {rescues}", file=sys.stderr)
206+
if rejected_stale and not args.no_reverify:
207+
print(
208+
f"Stale-voting rejections (failed reverify via safe_compare_pred): {rejected_stale}",
209+
file=sys.stderr,
210+
)
116211
for model_name, count in rescue_models.most_common():
117212
print(f" by {model_name}: {count}", file=sys.stderr)
118213
print(f"\nEA: {matched}/{n} = {matched / n * 100:.1f}%", file=sys.stderr)

‎scripts/rescore_arcwise.py‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
from nl_sql.db.connection import execute_readonly
3636
from nl_sql.db.registry import get_default_registry
3737
from nl_sql.eval.metrics.execution_accuracy import safe_compare_pred
38-
from nl_sql.eval.runner import _execute_gold
38+
from nl_sql.eval.runner import _execute_gold_with_status
3939

4040

4141
def _load_arcwise(path: Path) -> dict[int, dict[str, Any]]:
@@ -117,15 +117,25 @@ def main() -> int:
117117
):
118118
if not source:
119119
continue
120+
gold_failed = False
120121
try:
121-
gold_rows, _ = _execute_gold(
122+
gold_rows, _, gold_failed = _execute_gold_with_status(
122123
engine, source, statement_timeout_ms=30_000, row_cap=10_000
123124
)
125+
if gold_failed:
126+
out_entry[f"{variant}_gold_exec_error"] = (
127+
"gold SQL crashed in both execute_readonly and raw-connection paths"
128+
)
124129
except Exception as exc:
125130
gold_rows = []
131+
gold_failed = True
126132
out_entry[f"{variant}_gold_exec_error"] = str(exc)
127133
cmp = safe_compare_pred(
128-
gold_rows, pred_rows, gold_sql=source, pred_failed=pred_failed
134+
gold_rows,
135+
pred_rows,
136+
gold_sql=source,
137+
pred_failed=pred_failed,
138+
gold_failed=gold_failed,
129139
)
130140
is_match = bool(cmp.match)
131141
out_entry[f"{variant}_match"] = is_match

‎scripts/run_helallao_voting.py‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@
3030
from nl_sql.config import get_settings
3131
from nl_sql.db.registry import get_default_registry
3232
from nl_sql.eval.dataset import load_bird_mini_dev
33-
from nl_sql.eval.metrics.execution_accuracy import compare_results
34-
from nl_sql.eval.runner import _compose_question, _execute_gold
33+
from nl_sql.eval.metrics.execution_accuracy import safe_compare_pred
34+
from nl_sql.eval.runner import _compose_question, _execute_gold_with_status
3535
from nl_sql.execution.runner import execute_validated
3636
from nl_sql.llm.cache import CachingEmbeddingProvider
3737
from nl_sql.llm.providers.helallao_perplexity import HelallaoPerplexityProvider
@@ -180,6 +180,7 @@ def main() -> int:
180180

181181
alt_sql = alt.sql or ""
182182
alt_rows: list[Any] = []
183+
pred_failed = False
183184
try:
184185
outcome = execute_validated(
185186
engine,
@@ -190,15 +191,25 @@ def main() -> int:
190191
)
191192
if outcome.result:
192193
alt_rows = list(outcome.result.rows)
194+
else:
195+
pred_failed = True
193196
except Exception:
194-
pass
197+
pred_failed = True
198+
gold_failed = False
195199
try:
196-
gold_rows, _ = _execute_gold(
200+
gold_rows, _, gold_failed = _execute_gold_with_status(
197201
engine, ex.sql, statement_timeout_ms=30_000, row_cap=10_000
198202
)
199203
except Exception:
200204
gold_rows = []
201-
alt_cmp = compare_results(gold_rows, alt_rows, gold_sql=ex.sql)
205+
gold_failed = True
206+
alt_cmp = safe_compare_pred(
207+
gold_rows,
208+
alt_rows,
209+
gold_sql=ex.sql,
210+
pred_failed=pred_failed,
211+
gold_failed=gold_failed,
212+
)
202213
alt_match = bool(alt_cmp.match)
203214

204215
if alt_match and not br.get("match"):

0 commit comments

Comments
 (0)