Add 14 safe_solve exact-answer enforcement tests (217 total)

Tests cover: code fence stripping, quote stripping, multi-line
first-line extraction, explanation marker extraction, import/require/
__proto__ blocking, clean passthrough. All 217 pass.

Author: root

run_tests.py

@@ -1368,6 +1368,91 @@ def mock_llm(system, user):
     answer = safe_solve("What is 6 * 7?", llm_fn=mock_llm)
     assert answer == "42"
 
+# ── safe_solve exact-answer enforcement ───────────────
+print("\n── safe_solve exact-answer enforcement ─────────")
+
+@test("safe_solve: strips markdown code fences")
+def _():
+    def mock_llm(s, u): return "```\n42\n```"
+    assert safe_solve("What is 6*7?", llm_fn=mock_llm) == "42"
+
+@test("safe_solve: strips code fence with language tag")
+def _():
+    def mock_llm(s, u): return "```text\nHELLO\n```"
+    assert safe_solve("Reverse OLLEH", llm_fn=mock_llm) == "HELLO"
+
+@test("safe_solve: strips surrounding double quotes")
+def _():
+    def mock_llm(s, u): return '"42"'
+    assert safe_solve("What is 6*7?", llm_fn=mock_llm) == "42"
+
+@test("safe_solve: strips surrounding single quotes")
+def _():
+    def mock_llm(s, u): return "'HELLO'"
+    assert safe_solve("Reverse OLLEH", llm_fn=mock_llm) == "HELLO"
+
+@test("safe_solve: strips surrounding backticks")
+def _():
+    def mock_llm(s, u): return "`42`"
+    assert safe_solve("What is 6*7?", llm_fn=mock_llm) == "42"
+
+@test("safe_solve: multi-line takes first non-empty line")
+def _():
+    def mock_llm(s, u): return "42\nThis is my explanation"
+    assert safe_solve("What is 6*7?", llm_fn=mock_llm) == "42"
+
+@test("safe_solve: extracts answer from 'the answer is X'")
+def _():
+    def mock_llm(s, u): return "the answer is 42"
+    assert safe_solve("What is 6*7?", llm_fn=mock_llm) == "42"
+
+@test("safe_solve: extracts answer from 'the result is: X'")
+def _():
+    def mock_llm(s, u): return "the result is: HELLO"
+    assert safe_solve("Reverse OLLEH", llm_fn=mock_llm) == "HELLO"
+
+@test("safe_solve: extracts answer from 'therefore X'")
+def _():
+    def mock_llm(s, u): return "therefore 42"
+    assert safe_solve("What is 6*7?", llm_fn=mock_llm) == "42"
+
+@test("safe_solve: strips quotes after explanation extraction")
+def _():
+    def mock_llm(s, u): return 'the answer is "42"'
+    assert safe_solve("What is 6*7?", llm_fn=mock_llm) == "42"
+
+@test("safe_solve: blocks import statement in answer")
+def _():
+    def mock_llm(s, u): return "import os"
+    try:
+        safe_solve("What is 2+2?", llm_fn=mock_llm)
+        assert False, "Should have raised"
+    except ValueError as e:
+        assert "suspicious" in str(e).lower()
+
+@test("safe_solve: blocks require() in answer")
+def _():
+    def mock_llm(s, u): return "require('fs')"
+    try:
+        safe_solve("What is 2+2?", llm_fn=mock_llm)
+        assert False, "Should have raised"
+    except ValueError as e:
+        assert "suspicious" in str(e).lower()
+
+@test("safe_solve: blocks __proto__ in answer")
+def _():
+    def mock_llm(s, u): return "__proto__"
+    try:
+        safe_solve("What is 2+2?", llm_fn=mock_llm)
+        assert False, "Should have raised"
+    except ValueError as e:
+        assert "suspicious" in str(e).lower()
+
+@test("safe_solve: clean short answer passes through unchanged")
+def _():
+    def mock_llm(s, u): return "KRPS"
+    assert safe_solve("Reverse SPARK then remove vowels", llm_fn=mock_llm) == "KRPS"
+
 
 # ── prompt_builder Tests ──────────────────────────────
 print("\n── prompt_builder ──────────────────────────────")

tests/calibrate_5_2.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Quick calibration of gpt-5.2 against all 25 challenge types (10 attempts each)."""
+import os, sys, json, time, subprocess
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
+
+from agentchallenge import AgentChallenge
+from agentchallenge.types import CHALLENGE_TYPES, DIFFICULTY_MAP
+
+OPENAI_KEY = os.environ.get("OPENAI_API_KEY", "")
+ATTEMPTS = 10
+MODEL = "gpt-5.2"
+ALL_TYPES = sorted(CHALLENGE_TYPES.keys())
+
+def call_openai(model, prompt):
+    payload = json.dumps({
+        "model": model,
+        "messages": [
+            {"role": "system", "content": "You are solving a challenge. Reply with ONLY the answer, nothing else. No explanation, no quotes, no formatting."},
+            {"role": "user", "content": prompt}
+        ],
+        "temperature": 0, "max_tokens": 200,
+    })
+    try:
+        r = subprocess.run(
+            ["curl", "-s", "-m", "20", "https://api.openai.com/v1/chat/completions",
+             "-H", f"Authorization: Bearer {OPENAI_KEY}",
+             "-H", "Content-Type: application/json", "-d", payload],
+            capture_output=True, text=True, timeout=25
+        )
+        data = json.loads(r.stdout)
+        return data["choices"][0]["message"]["content"].strip()
+    except Exception as e:
+        return f"ERROR: {e}"
+
+print(f"\n{'='*60}")
+print(f"  MODEL: {MODEL} ({ATTEMPTS} attempts per type)")
+print(f"{'='*60}")
+
+results = {}
+for ctype in ALL_TYPES:
+    correct = 0
+    for i in range(ATTEMPTS):
+        ac = AgentChallenge(secret=f"cal52-{ctype}-{i}-key", types=[ctype])
+        ch = ac.create()
+        answer = call_openai(MODEL, ch.prompt)
+        result = ac.verify(ch.token, answer)
+        if result.valid:
+            correct += 1
+        time.sleep(0.2)
+
+    pct = correct / ATTEMPTS * 100
+    bar = "█" * int(pct / 5) + "░" * (20 - int(pct / 5))
+    status = "✅" if pct == 100 else "⚠️" if pct >= 80 else "❌"
+    print(f"  {status} {ctype:25s} {bar} {correct}/{ATTEMPTS} ({pct:.0f}%)")
+    results[ctype] = {"correct": correct, "total": ATTEMPTS, "pct": pct}
+
+# Summary by tier
+print(f"\n{'='*60}")
+print(f"  TIER SUMMARY")
+print(f"{'='*60}")
+for tier_name, tier_types in DIFFICULTY_MAP.items():
+    tier_results = [results.get(t, {}).get("pct", 0) for t in tier_types if t in results]
+    avg = sum(tier_results) / len(tier_results) if tier_results else 0
+    all_100 = all(p == 100 for p in tier_results)
+    print(f"  {tier_name:10s}: avg {avg:.0f}%  {'✅ ALL 100%' if all_100 else ''}")
+
+# Write results
+out = os.path.join(os.path.dirname(__file__), "calibration_gpt52.json")
+with open(out, "w") as f:
+    json.dump(results, f, indent=2)
+print(f"\n  Saved to {out}")