make_portfolio_charts.py

#!/usr/bin/env python3
"""Generate the portfolio/README charts from results/ data.

Outputs dark-navy themed PNGs to docs/plots/ (palette matches the sibling
IMC-Prosperity-4 repo: navy #0B1B33, gold #F4C430, blue #4FB6FF).

Run:  PYTHONPATH=. python3 scripts/make_portfolio_charts.py
Every number is derived from results/games/*.json + results/leaderboard.json —
no hand-entered stats.
"""
from __future__ import annotations

import glob
import json
import os

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np

# ---- theme -----------------------------------------------------------------
NAVY = "#0B1B33"
PANEL = "#12243f"
GOLD = "#F4C430"
BLUE = "#4FB6FF"
GREEN = "#3FB950"
RED = "#F85149"
TEXT = "#E6EDF3"
MUTED = "#8B98A9"
GRID = "#24344f"
OUT = "docs/plots"

plt.rcParams.update(
    {
        "figure.facecolor": NAVY,
        "axes.facecolor": PANEL,
        "savefig.facecolor": NAVY,
        "text.color": TEXT,
        "axes.labelcolor": TEXT,
        "axes.edgecolor": GRID,
        "xtick.color": MUTED,
        "ytick.color": MUTED,
        "grid.color": GRID,
        "font.size": 12,
        "axes.titlesize": 15,
        "axes.titleweight": "bold",
        "figure.dpi": 140,
    }
)


def _load_games():
    games = [json.load(open(f)) for f in glob.glob("results/games/*.json")]
    games.sort(key=lambda d: d["game"])
    return games


def _stage(g: int) -> str:
    if g <= 72:
        return "Group"
    if g <= 88:
        return "R32"
    if g <= 96:
        return "R16"
    if g <= 100:
        return "QF"
    if g <= 102:
        return "SF (2x)"
    return "3rd/Final (3x)"


def _finish(fig, name):
    os.makedirs(OUT, exist_ok=True)
    fig.tight_layout()
    p = os.path.join(OUT, name)
    fig.savefig(p, bbox_inches="tight")
    plt.close(fig)
    print("wrote", p)


# ---- 1. rank trajectory ----------------------------------------------------
def rank_trajectory():
    lb = json.load(open("results/leaderboard.json"))["history"]
    pts = [(e["game"], e["position"]) for e in lb if e.get("position")]
    gs = [g for g, _ in pts]
    rk = [p for _, p in pts]
    fig, ax = plt.subplots(figsize=(9, 5))
    ax.plot(gs, rk, color=BLUE, lw=2.4, marker="o", ms=4, mfc=GOLD, mec=GOLD, zorder=3)
    ax.invert_yaxis()  # rank 1 at top
    # peak + final annotations
    pk_g, pk_r = min(pts, key=lambda x: x[1])
    fn_g, fn_r = pts[-1]
    for g, r, label, col in [(pk_g, pk_r, f"peak #{pk_r}", GOLD), (fn_g, fn_r, f"final #{fn_r}", GREEN)]:
        ax.scatter([g], [r], s=120, color=col, zorder=4, edgecolor=NAVY, linewidth=1.5)
        ax.annotate(label, (g, r), textcoords="offset points", xytext=(0, -18 if col == GOLD else 16),
                    ha="center", color=col, fontweight="bold", fontsize=12)
    ax.set_title("Leaderboard rank over the tournament  (lower = better)")
    ax.set_xlabel("game #")
    ax.set_ylabel("rank")
    ax.grid(True, alpha=0.35)
    ax.text(0.5, 0.06, "peaked #11 after the group stage \u2192 field consolidated \u2192 climbed back to 89th on the \u00d73 final",
            transform=ax.transAxes, ha="center", color=MUTED, fontsize=10, style="italic")
    _finish(fig, "rank_trajectory.png")


# ---- 2. cumulative RBP -----------------------------------------------------
def cumulative_rbp():
    games = _load_games()
    gs, cum, run = [], [], 0.0
    for d in games:
        r = d.get("match_rbp")
        if r is None:
            r = sum((q.get("rbp") or 0) for q in d.get("questions", []))
        run += float(r)
        gs.append(d["game"])
        cum.append(run)
    fig, ax = plt.subplots(figsize=(9, 5))
    ax.axhline(0, color=MUTED, lw=1, ls="--")
    ax.plot(gs, cum, color=GOLD, lw=2.6, zorder=3)
    ax.fill_between(gs, 0, cum, color=GOLD, alpha=0.10)
    ax.set_title("Cumulative Relative Brier Points vs the crowd")
    ax.set_xlabel("game #")
    ax.set_ylabel("cumulative RBP")
    ax.grid(True, alpha=0.35)
    ax.annotate(f"+{cum[-1]:,.0f} RBP", (gs[-1], cum[-1]), textcoords="offset points",
                xytext=(-8, 8), ha="right", color=GOLD, fontweight="bold", fontsize=13)
    ax.text(0.5, 0.05, "crowd baseline = 0 (dashed) \u00b7 every point above the line is a market we forecast better than the field",
            transform=ax.transAxes, ha="center", color=MUTED, fontsize=9.5, style="italic")
    _finish(fig, "cumulative_rbp.png")


# ---- 3. per-category performance -------------------------------------------
def category_performance():
    from collections import defaultdict
    cat = defaultdict(lambda: [0, 0.0, 0])
    for d in _load_games():
        for q in d.get("questions", []):
            c = q.get("category", "?")
            r = q.get("rbp") or 0
            try:
                r = float(r)
            except Exception:
                r = 0
            cat[c][0] += 1
            cat[c][1] += r
            cat[c][2] += 1 if r > 0 else 0
    rows = sorted(cat.items(), key=lambda x: x[1][1])
    labels = [f"{c}  ({n})" for c, (n, _, _) in rows]
    vals = [v[1] for _, v in rows]
    beat = [100 * v[2] / v[0] for _, v in rows]
    colors = [GREEN if v >= 0 else RED for v in vals]
    fig, ax = plt.subplots(figsize=(9.5, 7.5))
    ax.barh(labels, vals, color=colors, alpha=0.9)
    ax.axvline(0, color=MUTED, lw=1)
    for i, (v, b) in enumerate(zip(vals, beat)):
        ax.text(v + (12 if v >= 0 else -12), i, f"{v:+.0f}  \u00b7 {b:.0f}% beat",
                va="center", ha="left" if v >= 0 else "right", color=TEXT, fontsize=9)
    ax.set_title("Net RBP by market family  (all 1,196 markets)")
    ax.set_xlabel("net RBP  (green = edge, red = leak)")
    ax.margins(x=0.18)
    ax.grid(True, axis="x", alpha=0.3)
    _finish(fig, "category_performance.png")


# ---- 4. calibration / reliability ------------------------------------------
def calibration():
    you, out, crowd, rbps = [], [], [], []
    total_q = beat_q = 0
    total_rbp = 0.0
    for d in _load_games():
        for q in d.get("questions", []):
            r = q.get("rbp")
            if r is not None:
                total_q += 1
                total_rbp += float(r)
                if float(r) > 0:
                    beat_q += 1
            y, o, c = q.get("you"), q.get("outcome"), q.get("crowd")
            if y is None or o is None:
                continue
            you.append(y / 100.0)
            out.append(1.0 if o >= 50 else 0.0)
            crowd.append((c / 100.0) if c is not None else np.nan)
    you = np.array(you)
    out = np.array(out)
    crowd = np.array(crowd)
    bins = np.linspace(0, 1, 11)
    idx = np.clip(np.digitize(you, bins) - 1, 0, 9)
    xs, ys, ns = [], [], []
    ece = 0.0
    for b in range(10):
        m = idx == b
        if m.sum() >= 5:
            xs.append(you[m].mean())
            ys.append(out[m].mean())
            ns.append(m.sum())
            ece += (m.sum() / len(you)) * abs(you[m].mean() - out[m].mean())
    brier = float(np.mean((you - out) ** 2))
    cm = ~np.isnan(crowd)
    brier_c = float(np.mean((crowd[cm] - out[cm]) ** 2))
    beat_pct = 100 * beat_q / total_q
    fig, ax = plt.subplots(figsize=(6.8, 6.6))
    ax.plot([0, 1], [0, 1], color=MUTED, ls="--", lw=1.2, label="perfect calibration")
    sizes = 60 + 900 * (np.array(ns) / max(ns))
    ax.scatter(xs, ys, s=sizes, color=GOLD, edgecolor=NAVY, linewidth=1.2, zorder=3, alpha=0.9)
    ax.plot(xs, ys, color=BLUE, lw=1.8, alpha=0.7, zorder=2)
    ax.set_title("Calibration  \u2014  honest probabilities, relative edge")
    ax.set_xlabel("forecast probability")
    ax.set_ylabel("observed frequency")
    ax.set_xlim(0, 1)
    ax.set_ylim(0, 1)
    ax.grid(True, alpha=0.35)
    ax.legend(loc="upper left", facecolor=PANEL, edgecolor=GRID, labelcolor=TEXT, fontsize=10)
    ax.text(0.97, 0.06,
            f"beat crowd:  {beat_pct:.0f}% of {total_q:,}\ncumulative:  +{total_rbp:,.0f} RBP\nBrier: {brier:.3f}  (crowd {brier_c:.3f})\ncalib. error (ECE): {ece:.3f}",
            transform=ax.transAxes, ha="right", va="bottom", color=TEXT, fontsize=10.5,
            family="monospace", bbox=dict(boxstyle="round", fc=PANEL, ec=GRID))
    ax.text(0.5, -0.13,
            "The crowd is a near-efficient baseline (Brier \u2248 ours). The edge is in RELATIVE scoring:\n"
            "we out-forecast the field on the majority of markets \u2014 that is what RBP rewards.",
            transform=ax.transAxes, ha="center", va="top", color=MUTED, fontsize=9, style="italic")
    _finish(fig, "calibration.png")


# ---- 5. stage x multiplier -------------------------------------------------
def stage_multiplier():
    from collections import defaultdict
    st = defaultdict(float)
    order = ["Group", "R32", "R16", "QF", "SF (2x)", "3rd/Final (3x)"]
    for d in _load_games():
        r = d.get("match_rbp")
        if r is None:
            r = sum((q.get("rbp") or 0) for q in d.get("questions", []))
        st[_stage(d["game"])] += float(r)
    vals = [st[s] for s in order]
    fig, ax = plt.subplots(figsize=(8.5, 5))
    bars = ax.bar(order, vals, color=[BLUE, BLUE, BLUE, BLUE, GOLD, GOLD])
    ax.axhline(0, color=MUTED, lw=1)
    for b, v in zip(bars, vals):
        ax.text(b.get_x() + b.get_width() / 2, v + (18 if v >= 0 else -26),
                f"{v:+.0f}", ha="center", color=TEXT, fontweight="bold")
    ax.set_title("RBP by tournament stage")
    ax.set_ylabel("net RBP")
    ax.grid(True, axis="y", alpha=0.3)
    plt.setp(ax.get_xticklabels(), rotation=15, ha="right")
    _finish(fig, "stage_multiplier.png")


# ---- 6. 3-lane divergence on the x3 final ----------------------------------
def bot_divergence_final():
    d = json.load(open("results/games/104-spain-vs-argentina.json"))
    qs = d["questions"]
    x = [q["q"] for q in qs]

    def cum(key):
        run, acc = 0.0, []
        for q in qs:
            run += float(q.get(key) or 0)
            acc.append(run)
        return acc

    man = cum("rbp")
    a = cum("bot_a_rbp")
    b = cum("bot_b_rbp")
    fig, ax = plt.subplots(figsize=(9, 5))
    ax.plot(x, a, color=GOLD, lw=2.4, marker="o", ms=3, label=f"Bot A \u2014 Spain-control ({a[-1]:+.0f})")
    ax.plot(x, man, color=BLUE, lw=2.4, marker="o", ms=3, label=f"Manual \u2014 You ({man[-1]:+.0f})")
    ax.plot(x, b, color=GREEN, lw=2.4, marker="o", ms=3, label=f"Bot B \u2014 Argentina-magic ({b[-1]:+.0f})")
    ax.axhline(0, color=MUTED, lw=1, ls="--")
    ax.set_title("The \u00d73 final: three de-correlated lanes  (cumulative RBP)")
    ax.set_xlabel("question #")
    ax.set_ylabel("cumulative RBP")
    ax.grid(True, alpha=0.35)
    ax.legend(loc="upper left", facecolor=PANEL, edgecolor=GRID, labelcolor=TEXT, fontsize=10)
    ax.text(0.5, 0.05, "Spain won 1-0 (a.e.t.): the control/under lane hit the scenario \u2014 best single-lane result of the run",
            transform=ax.transAxes, ha="center", color=MUTED, fontsize=9.5, style="italic")
    _finish(fig, "bot_divergence_final.png")


if __name__ == "__main__":
    rank_trajectory()
    cumulative_rbp()
    category_performance()
    calibration()
    stage_multiplier()
    bot_divergence_final()
    print("done \u2014 charts in docs/plots/")