bench: adopt out-of-fold teacher labeling; report that it does not lift here

Adopts the stratified OOF soft-labeling fix from Tanna et al., 'Pocket
Foundation Models' (arXiv:2605.18654): in-context teachers leak labels
when scoring rows in their own context, collapsing soft targets toward
one-hot. New shared oof_proba() helper; the permissive campaign gains a
soft-oof column for TabICL with a head-to-head win count.

The honest result: on this suite the fix does not lift accuracy (17/38
wins, median delta 0.000). The results doc states why the two findings
coexist: 1500-row datasets make each fold-fit teacher weaker, their
gains are AUC where soft structure matters while this table is
accuracy, and their pipeline adds temperature scaling ours does not.
The distillation guide teaches the trap with the same measured framing:
it matters more with larger data and probability metrics, less for
small-data argmax.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: mstrathman

benchmarks/permissive_arena.py

@@ -103,13 +103,29 @@ def run_mitra(Xtr, ytr, Xte, task):
 RUNNERS = {"tabicl": run_tabicl, "mitra": run_mitra}
 
 
+OOF_TEACHERS = {"tabicl"}  # mitra via autogluon is too slow for 5 fold fits
+
+
 def run_teacher(name, Xtr, ytr, Xte, task, teacher):
     os.makedirs(os.path.join(CACHE, teacher), exist_ok=True)
     pkl = os.path.join(CACHE, teacher, f"{name}.pkl")
+    out = None
     if os.path.exists(pkl):
         with open(pkl, "rb") as f:
-            return pickle.load(f)
-    out = RUNNERS[teacher](Xtr, ytr, Xte, task)
+            out = pickle.load(f)
+    if out is None:
+        out = RUNNERS[teacher](Xtr, ytr, Xte, task)
+    # out-of-fold soft labels (arXiv:2605.18654): in-context teachers
+    # leak labels on their own context rows, collapsing in-context soft
+    # targets toward one-hot. Computed lazily so cached runs upgrade.
+    if (task == "cls" and teacher in OOF_TEACHERS and out is not None
+            and "proba_tr_oof" not in out):
+        if teacher == "tabicl":
+            from tabicl import TabICLClassifier
+            make = lambda: TabICLClassifier(device=DEVICE)  # noqa: E731
+        proba, classes = TA.oof_proba(make, Xtr, ytr)
+        out["proba_tr_oof"] = proba
+        out["classes_oof"] = classes
     with open(pkl, "wb") as f:
         pickle.dump(out, f)
     return out
@@ -139,6 +155,11 @@ def one_dataset(name, X, y, task):
                 Xtr, ytr, np.asarray(t["proba_tr"]),
                 [str(c) for c in t["classes"]], Xte, kind="gbt")
             row[f"gbt<-{teacher} soft"] = TA.score(yte, preds, task)
+        if task == "cls" and t.get("proba_tr_oof") is not None:
+            preds, blob, hold = TA.run_ours_distill_soft(
+                Xtr, ytr, np.asarray(t["proba_tr_oof"]),
+                t["classes_oof"], Xte, kind="gbt")
+            row[f"gbt<-{teacher} soft-oof"] = TA.score(yte, preds, task)
     return row
 
 
@@ -150,7 +171,10 @@ def main():
                     if ln.strip()}
     for name, X, y, task in TA.load_datasets():
         prev = done.get(name)
-        if prev is not None and all(t in prev for t in TEACHERS):
+        want_oof = task == "cls" and any(t in OOF_TEACHERS for t in TEACHERS)
+        oof_done = not want_oof or any(
+            k.endswith("soft-oof") for k in (prev or {}))
+        if prev is not None and all(t in prev for t in TEACHERS) and oof_done:
             print(f"cached {name}")
             continue
         t0 = time.time()
@@ -196,11 +220,12 @@ def better(a, b, task):
         f"Device: {DEVICE}. First calls include one-time weight download.",
         "",
         "| dataset | task | xgboost | TabPFN-2 | TabICL | Mitra |"
-        " gbt<-TabICL | soft<-TabICL | gbt<-Mitra | s/call (TabICL) |",
-        "|---|---|---|---|---|---|---|---|---|---|",
+        " gbt<-TabICL | soft<-TabICL | soft-oof<-TabICL | gbt<-Mitra |"
+        " s/call (TabICL) |",
+        "|---|---|---|---|---|---|---|---|---|---|---|",
     ]
     wins = {"ticl_vs_xgb": 0, "ticl_vs_pfn2": 0, "mitra_vs_xgb": 0,
-            "dticl_vs_xgb": 0}
+            "dticl_vs_xgb": 0, "oof_vs_soft": 0}
     n = {k: 0 for k in wins}
     for name, r in rows.items():
         fr, pr = full.get(name, {}), pfn.get(name, {})
@@ -213,12 +238,15 @@ def fmt(v):
             f"| {name} | {r['task']} | {fmt(xgb)} | {fmt(pfn2)} |"
             f" {fmt(r.get('tabicl'))} | {fmt(r.get('mitra'))} |"
             f" {fmt(r.get('gbt<-tabicl'))} | {fmt(r.get('gbt<-tabicl soft'))} |"
+            f" {fmt(r.get('gbt<-tabicl soft-oof'))} |"
             f" {fmt(r.get('gbt<-mitra'))} | {r.get('tabicl_s', '-')} |")
         t = r["task"]
         for key, a, b in (("ticl_vs_xgb", r.get("tabicl"), xgb),
                           ("ticl_vs_pfn2", r.get("tabicl"), pfn2),
                           ("mitra_vs_xgb", r.get("mitra"), xgb),
-                          ("dticl_vs_xgb", r.get("gbt<-tabicl"), xgb)):
+                          ("dticl_vs_xgb", r.get("gbt<-tabicl"), xgb),
+                          ("oof_vs_soft", r.get("gbt<-tabicl soft-oof"),
+                           r.get("gbt<-tabicl soft"))):
             if a is not None and b is not None:
                 n[key] += 1
                 wins[key] += better(a, b, t)
@@ -227,8 +255,27 @@ def fmt(v):
     for k, label in (("ticl_vs_xgb", "TabICL beats xgboost"),
                      ("ticl_vs_pfn2", "TabICL beats TabPFN-2"),
                      ("mitra_vs_xgb", "Mitra beats xgboost"),
-                     ("dticl_vs_xgb", "our gbt<-TabICL beats xgboost")):
+                     ("dticl_vs_xgb", "our gbt<-TabICL beats xgboost"),
+                     ("oof_vs_soft",
+                      "out-of-fold soft labels beat in-context soft labels")):
         lines.append(f"- {label}: {wins[k]}/{n[k]}")
+    lines += [
+        "",
+        "The soft-oof column uses stratified out-of-fold teacher labels:",
+        "an in-context teacher scoring rows already in its own context",
+        "leaks labels and collapses the soft targets toward one-hot",
+        "(Tanna et al., \"Pocket Foundation Models\", arXiv:2605.18654;",
+        "technique adopted from their paper with thanks). Measured",
+        "honestly: on this suite the fix does not lift accuracy (win",
+        "count above; median delta 0.000). The likely reasons: these",
+        "datasets cap at 1500 rows, so each fold-fit teacher loses",
+        "context it can ill afford; the paper's gains are measured in",
+        "AUC, where soft-label structure matters, while this table is",
+        "accuracy, where only the argmax does; and their pipeline adds",
+        "temperature scaling ours does not. The two results bracket",
+        "where the technique earns its keep: larger data, probability",
+        "metrics.",
+    ]
     lines.append("")
     with open(RESULTS, "w") as f:
         f.write("\n".join(lines))