backtest.py

#!/usr/bin/env python3
"""Walk-forward backtest — re-price every settled game with the CURRENT model and compare to what
was submitted, scored against the real outcomes and the crowd.

This is the gate for every model change: run it before/after a change and confirm mean Brier does
not worsen and RBP-vs-crowd does not drop. It reuses the historical tracker (results/games/*.json)
as the out-of-sample set — each question carries the submitted forecast (`you`), the `crowd`, the
`outcome`, and the realized `rbp`.

Usage:
    python scripts/backtest.py            # full report
    python scripts/backtest.py --json      # machine-readable summary
    python scripts/backtest.py --by-category

Note: "current model" re-prices from live providers (StatsJsonProvider + odds/context/referee).
Because team rates/odds evolve, this is an approximate replay, not a perfect point-in-time
snapshot — treat it as a guardrail against regressions, read alongside results/ (the true record).
"""

from __future__ import annotations

import glob
import json
import sys
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent
RESULTS_DIR = REPO_ROOT / "results" / "games"

sys.path.insert(0, str(REPO_ROOT))


def _brier(p_pct: float, outcome_pct: float) -> float:
    return ((p_pct - outcome_pct) / 100.0) ** 2


def _load_question_index() -> dict:
    """Map (fixture_slug, question_text) -> match name, from games/*/questions.json."""
    idx = {}
    for f in glob.glob(str(REPO_ROOT / "games" / "*" / "questions.json")):
        try:
            d = json.load(open(f, encoding="utf-8"))
        except (OSError, json.JSONDecodeError):
            continue
        # Skip questions.json reconstructed from results/: they carry ABBREVIATED question text
        # ("Haaland score") that the model cannot re-price meaningfully, which would pollute the
        # walk-forward backtest. Only live-fetched boards (full question text) are re-priced.
        if d.get("reconstructed_from"):
            continue
        slug = Path(f).parent.name.split("-", 1)[-1]
        name = d.get("match", {}).get("name")
        for m in d.get("markets", []):
            idx[(slug, m.get("question", ""))] = name
    return idx


def run() -> dict:
    from bot.cli import _price_market, _provider
    from bot.data.context import ContextProvider
    from bot.data.odds import OddsProvider
    from bot.data.referees import RefereeProvider

    prov, op, cp, rp = _provider(), OddsProvider(), ContextProvider(), RefereeProvider()
    qidx = _load_question_index()

    rows = []
    for f in sorted(glob.glob(str(RESULTS_DIR / "*.json"))):
        g = json.load(open(f, encoding="utf-8"))
        if g.get("status") != "settled":
            continue
        fixture = g.get("fixture")
        for q in g.get("questions", []):
            o = q["outcome"]
            you = q["you"]
            crowd = q.get("crowd")
            # Skip voided/unscored markets (null you/outcome) e.g. a player prop removed when the
            # named player isn't in the final squad.
            if you is None or o is None:
                continue
            name = qidx.get((fixture, q["question"]))
            model_pct = None
            if name is not None:
                rec = _price_market(prov, q["question"], name, op, cp, rp)
                model_pct = rec["probability"]
            rows.append({
                "game": g.get("game"), "category": q.get("category"), "question": q["question"],
                "you": you, "crowd": crowd, "outcome": o, "model": model_pct,
            })

    def agg(key_pct):
        briers = [_brier(r[key_pct], r["outcome"]) for r in rows if r.get(key_pct) is not None]
        return sum(briers) / len(briers) if briers else None

    # RBP-vs-crowd proxy = (crowd_brier - your_brier) * 100 (the deviation-from-crowd component)
    def rbp_vs_crowd(key_pct):
        vals = [(_brier(r["crowd"], r["outcome"]) - _brier(r[key_pct], r["outcome"])) * 100
                for r in rows if r.get(key_pct) is not None and r.get("crowd") is not None]
        return sum(vals) if vals else None

    cats = {}
    for r in rows:
        if r.get("model") is None or r.get("crowd") is None:
            continue
        c = cats.setdefault(r["category"], {"n": 0, "sub_rbp": 0.0, "mod_rbp": 0.0})
        c["n"] += 1
        c["sub_rbp"] += (_brier(r["crowd"], r["outcome"]) - _brier(r["you"], r["outcome"])) * 100
        c["mod_rbp"] += (_brier(r["crowd"], r["outcome"]) - _brier(r["model"], r["outcome"])) * 100

    from scripts.robust import robust_summary
    return {
        "questions": len(rows),
        "repriced": sum(1 for r in rows if r.get("model") is not None),
        "submitted_mean_brier": round(agg("you"), 4) if agg("you") is not None else None,
        "model_mean_brier": round(agg("model"), 4) if agg("model") is not None else None,
        "crowd_mean_brier": round(agg("crowd"), 4) if agg("crowd") is not None else None,
        "submitted_rbp_vs_crowd": round(rbp_vs_crowd("you"), 2) if rbp_vs_crowd("you") is not None else None,
        "model_rbp_vs_crowd": round(rbp_vs_crowd("model"), 2) if rbp_vs_crowd("model") is not None else None,
        "by_category": {c: {"n": v["n"], "submitted_rbp": round(v["sub_rbp"], 2),
                            "model_rbp": round(v["mod_rbp"], 2)} for c, v in cats.items()},
        # Robust real-RBP block (E1): per-game median + bootstrap CI + winsorized total.
        "robust_model": robust_summary([r for r in rows if r.get("model") is not None], "model"),
        "robust_submitted": robust_summary(rows, "you"),
        "rows": rows,
    }


def main() -> None:
    s = run()
    if not s["questions"]:
        print("No settled games in results/games/. Nothing to backtest.")
        return
    if "--json" in sys.argv:
        s.pop("rows", None)
        print(json.dumps(s, indent=2))
        return
    print(f"== Walk-forward backtest: {s['repriced']}/{s['questions']} questions re-priced ==")
    print(f"Mean Brier  — submitted {s['submitted_mean_brier']} | current model {s['model_mean_brier']} | crowd {s['crowd_mean_brier']}")
    if s["model_mean_brier"] is not None and s["submitted_mean_brier"] is not None:
        d = s["submitted_mean_brier"] - s["model_mean_brier"]
        print(f"            current model is {'BETTER' if d>0 else 'worse'} than submitted by {abs(d):.4f} (lower Brier = better)")
    print(f"RBP-vs-crowd — submitted {s['submitted_rbp_vs_crowd']} | current model {s['model_rbp_vs_crowd']}")
    rm = s.get("robust_model", {})
    if rm.get("n_games"):
        print(f"Robust real-RBP (model) — total {rm['total']} | winsor {rm['winsor_total']} | "
              f"median/game {rm['median_per_game']} | mean {rm['mean_per_game']} "
              f"95%CI [{rm['ci95_lo']}, {rm['ci95_hi']}] | worst game {rm['worst_game']}")
    if "--by-category" in sys.argv or True:
        print("\nBy category (RBP-vs-crowd, submitted -> current model):")
        for c, v in sorted(s["by_category"].items(), key=lambda kv: kv[1]["model_rbp"]):
            arrow = "UP" if v["model_rbp"] > v["submitted_rbp"] + 0.01 else ("DOWN" if v["model_rbp"] < v["submitted_rbp"] - 0.01 else "==")
            print(f"  {c:14} n={v['n']:>2}  {v['submitted_rbp']:+7.2f} -> {v['model_rbp']:+7.2f}  {arrow}")
    if "--leak-check" in sys.argv:
        _leak_check()


def _leak_check() -> None:
    """Flag settled games whose priced teams' stats.json `as_of` postdates the game date.

    The backtest re-prices from *current* stats.json, so any team whose rates were updated AFTER a
    game leaks that game's information into its own re-price. For GATING (experiment vs baseline,
    both on identical current data) this is common-mode and cancels; this check quantifies the
    absolute optimism so we know which rows to discount when reading the absolute level.
    """
    as_of = {}
    for f in glob.glob(str(REPO_ROOT / "teams" / "*" / "*" / "stats.json")):
        try:
            d = json.load(open(f, encoding="utf-8"))
        except (OSError, json.JSONDecodeError):
            continue
        as_of[Path(f).parent.name] = str(d.get("as_of", ""))[:10]
    flagged = []
    for f in sorted(glob.glob(str(RESULTS_DIR / "*.json"))):
        g = json.load(open(f, encoding="utf-8"))
        if g.get("status") != "settled":
            continue
        gdate = str(g.get("date", ""))[:10]
        fixture = g.get("fixture", "")
        teams = fixture.split("-vs-") if "-vs-" in fixture else []
        for t in teams:
            a = as_of.get(t)
            if a and gdate and a > gdate:
                flagged.append((g.get("game"), t, gdate, a))
    print(f"\n-- leak-check: {len(flagged)} (team, game) rows where stats as_of > game date --")
    for game, t, gdate, a in flagged:
        print(f"  game {game}: {t} stats as_of {a} > game {gdate}  (LEAKY re-price)")
    if not flagged:
        print("  none — no team's stats postdate a settled game it appears in.")


if __name__ == "__main__":
    main()