set_pieces_audit.py

#!/usr/bin/env python3
"""Set Pieces & Possessions audit + board lint — fix our ONLY negative category.

Official SportsPredict category stats (1077 forecasts) put **Set Pieces & Possessions at -0.2 vs
crowd** — the only negative category, while our MODEL re-prices these markets strongly positive
(backtest: fouls submitted -170 -> model -5; territorial -40 -> +89; offsides +1 -> +72). The leak
is therefore our **manual hand-leans**, not the model.

This tool has two modes:

  # 1) ATTRIBUTION (default) — reproduce, from the settled record, WHICH set-piece sub-category leaks
  python scripts/set_pieces_audit.py

  # 2) LINT a staged board — flag set-piece markets whose MANUAL deviates from the MODEL against the
  #    measured per-sub-category rule, and print the corrected number to submit.
  python scripts/set_pieces_audit.py --lint predictions/baselines/ARGSUI_FINAL_bytext.json

Measured per-sub-category rule (brier-deviation A/B over all settled set-piece markets — edge-vs-crowd
of each policy, higher is better; submit-crowd == 0 by construction):

  sub-category        our leans   submit-50   submit-crowd   -> RULE
  comparing-fouls       -236.6      -28.5          0.0        SUBMIT THE MODEL (never lean; ~coin-flip)
  offsides              -133.7      -13.1          0.0        ANCHOR TO MODEL/CROWD (do not deviate)
  comparing-corners      -19.0     -165.2          0.0        REAL signal but we OVER-lean; shrink ~50%
  corners-total          +49.5       +1.8          0.0        KEEP leaning (our proven-good sub-cat)

Industry consensus agrees: corners are predictable (expected-corners Poisson) and worth modelling;
fouls & offsides are low-signal / near-random and high-vig — sharps don't lean them.
"""

from __future__ import annotations

import glob
import json
import sys
from collections import defaultdict
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent
RESULTS_DIR = REPO_ROOT / "results" / "games"


def _brier(p_pct: float, outcome_pct: float) -> float:
    return ((p_pct - outcome_pct) / 100.0) ** 2


def subcat(question: str) -> str | None:
    """Classify a market's TEXT into one of the 4 official Set-Pieces sub-categories."""
    t = (question or "").lower()
    if "offside" in t:
        return "offsides"
    has_corner = "corner" in t
    has_foul = "foul" in t
    # A COMPARISON market pits two teams ("X ... than Y"). "N or more corner kicks" is a TOTAL, not a
    # comparison, so match on " than " only (the bare "more corner"/"more foul" substring is a
    # threshold trap: "4 or more corner kicks" contains "more corner").
    is_cmp = " than " in t
    if has_foul:
        # This contest has no "team fouls total" sub-category — every foul market is a comparison.
        return "comparing-fouls"
    if has_corner and is_cmp:
        return "comparing-corners"
    if has_corner:
        return "corners-total"
    return None


# Per-sub-category lint policy: max allowed |manual - model| deviation, and whether the manual is
# allowed to be MORE extreme (further from 50) than the model. Derived from the A/B table above.
_LINT_POLICY = {
    "comparing-fouls":   {"max_dev": 3,  "allow_more_extreme": False},
    "offsides":          {"max_dev": 5,  "allow_more_extreme": False},
    "comparing-corners": {"max_dev": 6,  "allow_more_extreme": False},
    "corners-total":     {"max_dev": 15, "allow_more_extreme": True},
}


def run_attribution() -> None:
    buckets: dict = defaultdict(lambda: {"n": 0, "rbp": 0.0, "you": 0, "crowd": 0, "yes": 0})
    policies: dict = defaultdict(list)  # sub -> [(you, crowd, outcome), ...]

    for f in sorted(glob.glob(str(RESULTS_DIR / "*.json"))):
        g = json.load(open(f, encoding="utf-8"))
        if g.get("status") != "settled":
            continue
        for q in g.get("questions", []):
            sc = subcat(q.get("question", ""))
            if sc is None:
                continue
            you, crowd, out = q.get("you"), q.get("crowd"), q.get("outcome")
            if you is None or out is None:
                continue
            b = buckets[sc]
            b["n"] += 1
            b["you"] += you
            b["yes"] += 1 if out >= 50 else 0
            if q.get("rbp") is not None:
                b["rbp"] += q["rbp"]
            if crowd is not None:
                b["crowd"] += crowd
                policies[sc].append((you, crowd, out))

    def edge(rows, fn):
        return sum((_brier(c, o) - _brier(fn(y, c), o)) * 100 for y, c, o in rows)

    print("=== SET-PIECES sub-category attribution (settled record) ===")
    print(f"{'sub-category':18s}{'n':>4}{'realRBP':>10}{'avgYou':>8}{'avgCrowd':>9}{'yes%':>6}"
          f"{'|  leans':>10}{'submit50':>10}{'crowd':>7}")
    order = ["comparing-fouls", "offsides", "comparing-corners", "corners-total"]
    for sc in order:
        b = buckets.get(sc)
        if not b or not b["n"]:
            continue
        n = b["n"]
        rows = policies.get(sc, [])
        leans = edge(rows, lambda y, c: y) if rows else 0.0
        f50 = edge(rows, lambda y, c: 50) if rows else 0.0
        print(f"{sc:18s}{n:>4}{b['rbp']:>10.1f}{b['you']/n:>8.1f}"
              f"{(b['crowd']/n if n else 0):>9.1f}{100*b['yes']/n:>6.0f}"
              f"{leans:>10.1f}{f50:>10.1f}{0.0:>7.1f}")
    print("\nRULE  fouls/offsides comparisons = SUBMIT THE MODEL (near-random, high-vig, sharps don't")
    print("      lean).  comparing-corners = real signal but we OVER-lean -> shrink ~50% toward model.")
    print("      corners-TOTAL = our proven-good bucket -> keep leaning.")


def run_lint(board_path: str) -> int:
    d = json.load(open(board_path, encoding="utf-8"))
    mapping = d.get("mapping") or d.get("markets") or []
    print(f"=== SET-PIECES lint: {board_path} ===")
    flags = 0
    for e in mapping:
        q = e.get("q") or e.get("question") or ""
        sc = subcat(q)
        if sc is None:
            continue
        model = e.get("model")
        manual = e.get("manual")
        if model is None or manual is None:
            continue
        pol = _LINT_POLICY[sc]
        dev = manual - model
        more_extreme = abs(manual - 50) > abs(model - 50) + 0.5
        bad = abs(dev) > pol["max_dev"] or (not pol["allow_more_extreme"] and more_extreme)
        if bad:
            flags += 1
            # recommended: for no-lean cats snap to model; for shrink cat pull halfway to model
            if sc == "comparing-corners":
                rec = round((manual + model) / 2)
            else:
                rec = model
            why = "more-extreme-than-model" if more_extreme else f"|manual-model|={abs(dev):.0f}>{pol['max_dev']}"
            print(f"  [FLAG {sc:16s}] manual={manual} model={model}  {why}")
            print(f"            -> {q[:70]}")
            print(f"            -> RECOMMEND {rec}  ({'shrink halfway to model' if sc=='comparing-corners' else 'submit the model'})")
        else:
            print(f"  [ok   {sc:16s}] manual={manual} model={model}")
    if flags == 0:
        print("  No set-piece violations — board is disciplined. ✅")
    else:
        print(f"\n  {flags} set-piece market(s) flagged — apply the recommendations before firing.")
    return flags


def main() -> None:
    args = sys.argv[1:]
    if "--lint" in args:
        i = args.index("--lint")
        if i + 1 >= len(args):
            print("usage: set_pieces_audit.py --lint <board.json>")
            sys.exit(2)
        sys.exit(1 if run_lint(args[i + 1]) else 0)
    run_attribution()


if __name__ == "__main__":
    main()