Skip to content

Commit d924067

Browse files
mstrathmanclaude
andcommitted
bench: scale study — retention holds to 50K rows, teacher cost does not
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent cb86af3 commit d924067

3 files changed

Lines changed: 331 additions & 2 deletions

File tree

‎benchmarks/results/scale-arena.jsonl‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,6 @@
1111
{"dataset": "credit_card_default", "n_train": 1500, "n_test": 500, "d": 23, "xgboost": 0.81, "xgb_s": 0.1, "tabicl": 0.836, "tabicl_s": 1.7, "tabicl_device": "mps", "leak_gap": -0.0247, "ic_maxp": 0.8101, "gbt<-tabicl": 0.798, "distill_s": 0.9, "blob_kb": 93.5, "gbt<-tabicl soft": 0.81, "oof_s": 5.8, "oof_gap": -0.0153, "gbt<-tabicl soft-oof": 0.838}
1212
{"dataset": "airline_satisfaction", "n_train": 9999, "n_test": 3334, "d": 21, "xgboost": 0.9448110377924415, "xgb_s": 0.3, "tabicl": 0.9511097780443911, "tabicl_s": 139.6, "tabicl_device": "cpu", "leak_gap": 0.0465, "ic_maxp": 0.9842, "gbt<-tabicl": 0.9337132573485303, "distill_s": 3.5, "blob_kb": 105.7, "gbt<-tabicl soft": 0.9340131973605279, "oof_s": 486.7, "oof_gap": 0.0043, "gbt<-tabicl soft-oof": 0.9337132573485303}
1313
{"dataset": "credit_card_default", "n_train": 9999, "n_test": 3334, "d": 23, "xgboost": 0.8047390521895621, "xgb_s": 0.2, "tabicl": 0.8191361727654469, "tabicl_s": 167.5, "tabicl_device": "cpu", "leak_gap": 0.0205, "ic_maxp": 0.8387, "gbt<-tabicl": 0.8197360527894421, "distill_s": 7.4, "blob_kb": 96.8, "gbt<-tabicl soft": 0.8182363527294542, "oof_s": 571.6, "oof_gap": -0.001, "gbt<-tabicl soft-oof": 0.8194361127774445}
14+
{"dataset": "SDSS17", "n_train": 49999, "n_test": 16667, "d": 11, "xgboost": 0.9752804943901122, "xgb_s": 1.1, "tabicl": 0.9776204475910482, "tabicl_s": 2534.2, "tabicl_device": "cpu", "leak_gap": 0.0154, "ic_maxp": 0.9913, "gbt<-tabicl": 0.9752804943901122, "distill_s": 52.1, "blob_kb": 150.6, "gbt<-tabicl soft": 0.9761804763904722}
15+
{"dataset": "GiveMeSomeCredit", "n_train": 49999, "n_test": 16667, "d": 10, "xgboost": 0.9319613607727846, "xgb_s": 0.3, "tabicl": 0.9368812623747526, "tabicl_s": 2725.0, "tabicl_device": "cpu", "leak_gap": 0.0033, "ic_maxp": 0.9477, "gbt<-tabicl": 0.9364612707745845, "distill_s": 13.8, "blob_kb": 108.9, "gbt<-tabicl soft": 0.9367612647747046}
16+
{"dataset": "APSFailure", "n_train": 9999, "n_test": 3334, "d": 40, "xgboost": 0.9931013797240552, "xgb_s": 0.2, "tabicl": 0.9949010197960408, "tabicl_s": 185.3, "tabicl_device": "cpu", "leak_gap": 0.0049, "ic_maxp": 0.9986, "gbt<-tabicl": 0.9913017396520696, "distill_s": 11.5, "blob_kb": 82.0, "gbt<-tabicl soft": 0.9922015596880623, "oof_s": 675.6, "oof_gap": -0.0043, "gbt<-tabicl soft-oof": 0.9898020395920816}

‎benchmarks/results/scale.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,18 @@ Device: mps with cpu fallback (per-cell `dev`).
1111
| dataset | n_train | xgboost | TabICL | gbt<-TabICL | soft | soft-oof | leak gap | oof gap | teacher s | distill s | blob KB |
1212
|---|---|---|---|---|---|---|---|---|---|---|---|
1313
| APSFailure | 1500 | 0.994 | 0.996 | 0.990 | 0.990 | 0.996 | -0.0040 | -0.0093 | 2.6 | 1.4 | 76.6 |
14+
| APSFailure | 9999 | 0.993 | 0.995 | 0.991 | 0.992 | 0.990 | 0.0049 | -0.0043 | 185.3 | 11.5 | 82.0 |
1415
| Diabetes130US | 1500 | 0.906 | 0.916 | - | 0.916 | 0.916 | 0.0007 | 0.0007 | 2.2 | - | - |
1516
| Diabetes130US | 9999 | 0.907 | 0.913 | - | 0.913 | 0.913 | -0.0000 | -0.0000 | 101.9 | - | - |
1617
| GiveMeSomeCredit | 1500 | 0.922 | 0.936 | 0.932 | 0.926 | 0.930 | 0.0067 | -0.0113 | 1.2 | 0.3 | 97.1 |
1718
| GiveMeSomeCredit | 9999 | 0.929 | 0.932 | 0.933 | 0.933 | 0.932 | 0.0210 | 0.0009 | 8.4 | 2.6 | 104.3 |
19+
| GiveMeSomeCredit | 49999 | 0.932 | 0.937 | 0.936 | 0.937 | - | 0.0033 | - | 2725.0 | 13.8 | 108.9 |
1820
| SDSS17 | 1500 | 0.970 | 0.972 | 0.968 | 0.968 | 0.964 | 0.0167 | 0.0007 | 1.4 | 1.1 | 145.8 |
1921
| SDSS17 | 9999 | 0.968 | 0.972 | 0.966 | 0.966 | 0.968 | 0.0205 | 0.0052 | 8.7 | 9.2 | 159.2 |
20-
| SDSS17 | 49999 | 0.975 | 0.610 | - | 0.610 | - | -0.0000 | - | 118.9 | - | - |
22+
| SDSS17 | 49999 | 0.975 | 0.978 | 0.975 | 0.976 | - | 0.0154 | - | 2534.2 | 52.1 | 150.6 |
2123
| airline_satisfaction | 1500 | 0.910 | 0.928 | 0.908 | 0.904 | 0.906 | 0.0693 | -0.0073 | 1.6 | 0.5 | 103.4 |
24+
| airline_satisfaction | 9999 | 0.945 | 0.951 | 0.934 | 0.934 | 0.934 | 0.0465 | 0.0043 | 139.6 | 3.5 | 105.7 |
2225
| credit_card_default | 1500 | 0.810 | 0.836 | 0.798 | 0.810 | 0.838 | -0.0247 | -0.0153 | 1.7 | 0.9 | 93.5 |
23-
| credit_card_default | 9999 | 0.805 | 0.771 | 0.776 | 0.776 | 0.776 | 0.0019 | 0.0054 | 11.7 | 7.4 | 96.2 |
26+
| credit_card_default | 9999 | 0.805 | 0.819 | 0.820 | 0.818 | 0.819 | 0.0205 | -0.0010 | 167.5 | 7.4 | 96.8 |
2427
| online_shoppers_intention | 1500 | 0.882 | 0.890 | 0.890 | 0.888 | 0.886 | 0.0507 | 0.0153 | 1.5 | 0.4 | 100.9 |
2528
| taiwanese_bankruptcy | 1500 | 0.966 | 0.968 | 0.968 | 0.966 | 0.968 | 0.0040 | 0.0007 | 2.4 | 2.9 | 93.8 |

‎benchmarks/scale_arena.py‎

Lines changed: 323 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,323 @@
1+
"""Scale study: does the distillation story survive larger data?
2+
3+
Three questions, one overnight run over the naturally large TabArena
4+
datasets at n in {1500, 10k, 50k, 100k}:
5+
6+
1. Retention: does the distilled gbt student keep tracking its TabICL
7+
teacher as rows grow (our published numbers stop at 1500)?
8+
2. The boundary: where does tuned xgboost catch the zero-shot teacher?
9+
The TFM edge concentrates in small data; publish the crossover
10+
instead of letting readers find it.
11+
3. The leak: does the in-context label leak (teacher train-agreement
12+
minus held-out accuracy; see permissive-teachers.md and
13+
arXiv:2605.18654) grow with context size? Out-of-fold labels are
14+
compared at 10k, where five fold-fits stay affordable.
15+
16+
Checkpointed per (dataset, n) cell into results/scale-arena.jsonl.
17+
Teacher outputs are not cached across runs (contexts differ per n);
18+
each cell records wall times instead.
19+
20+
Run overnight:
21+
22+
caffeinate -i env TEACHERS=tabicl uv run --with tabicl \
23+
--with scikit-learn --with xgboost --with pandas \
24+
python benchmarks/scale_arena.py
25+
"""
26+
27+
import os
28+
import sys
29+
30+
# Single-threaded math pools: xgboost's OpenMP and torch's ATen pool
31+
# deadlock each other in one process on macOS (see tabarena.py, which
32+
# learned this first; this file re-learned it the hard way). TabICL's
33+
# heavy path runs on MPS, so the CPU pools cost little here.
34+
for _v in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS",
35+
"VECLIB_MAXIMUM_THREADS", "NUMEXPR_NUM_THREADS"):
36+
os.environ[_v] = "1"
37+
os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE")
38+
os.environ["HF_HUB_OFFLINE"] = "0"
39+
40+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
41+
import tabarena as TA # noqa: E402
42+
43+
import json # noqa: E402
44+
import time # noqa: E402
45+
46+
import numpy as np # noqa: E402
47+
import pandas as pd # noqa: E402
48+
49+
JSONL = os.path.join(os.path.dirname(__file__), "results",
50+
"scale-arena.jsonl")
51+
RESULTS = os.path.join(os.path.dirname(__file__), "results", "scale.md")
52+
DEVICE = os.environ.get("SCALE_DEVICE", "mps")
53+
CPU_AT = int(os.environ.get("SCALE_CPU_AT", 50_000)) # mps died natively at 50k
54+
# Metal's 2**32-byte single-tensor cap binds on roughly rows x features:
55+
# 50k x 10 survived with batch_size=1, 50k x 40 did not. Cells over this
56+
# budget are skipped as measured hardware scope, not attempted.
57+
MPS_CELL_BUDGET = int(os.environ.get("SCALE_MPS_BUDGET", 1_000_000))
58+
CELL_TIMEOUT_S = int(os.environ.get("SCALE_CELL_TIMEOUT", 2700))
59+
ATTEMPTS = os.path.join(os.path.dirname(__file__), "results",
60+
"scale-attempts")
61+
SIZES = (1500, 10_000, 50_000, 100_000)
62+
OOF_AT = 10_000
63+
SEED = 0
64+
65+
# TabArena datasets whose full size supports the ladder (classification
66+
# only: the leak diagnostic needs probabilities).
67+
LARGE = [
68+
(46922, "Diabetes130US"), (46920, "airline_satisfaction"),
69+
(46929, "GiveMeSomeCredit"), (46908, "APSFailure"),
70+
(46955, "SDSS17"), (46962, "taiwanese_bankruptcy"),
71+
(46947, "online_shoppers_intention"), (46919, "credit_card_default"),
72+
]
73+
74+
75+
def load_full(did, name):
76+
from sklearn.datasets import fetch_openml
77+
b = fetch_openml(data_id=did, as_frame=True)
78+
X, y = TA._prep(b.data, b.target, "cls")
79+
return X, y
80+
81+
82+
def make_teacher():
83+
from tabicl import TabICLClassifier
84+
return TabICLClassifier(device=DEVICE)
85+
86+
87+
def teacher_with_fallback(Xtr, ytr, Xte):
88+
"""Fit + predict with an MPS-to-CPU fallback on failure or
89+
non-finite output (seen from TabICL on MPS). Native-level MPS
90+
crashes cannot be caught here; the cell-subprocess driver absorbs
91+
those."""
92+
first = DEVICE # batch_size=1 makes mps survive big cells; cpu is
93+
# still the fallback if the guard or a crash rejects the attempt
94+
for attempt_device in dict.fromkeys((first, "cpu")):
95+
try:
96+
from tabicl import TabICLClassifier
97+
kwargs = {"device": attempt_device}
98+
bs = os.environ.get("SCALE_BATCH")
99+
if bs:
100+
kwargs["batch_size"] = int(bs)
101+
elif attempt_device == "mps" and len(Xtr) >= 20_000:
102+
# Metal caps one NDArray at 2**32 bytes; batch_size=1
103+
# divides the big tensor and survives 50k (probed)
104+
kwargs["batch_size"] = 1
105+
if len(Xtr) >= CPU_AT:
106+
# offload_mode="auto" engages a large-data path whose
107+
# workers allocate MPS tensors past Metal's 2**32-byte
108+
# cap regardless of device=. Pin everything to cpu, no
109+
# worker processes.
110+
kwargs.update(n_jobs=1)
111+
for om in ("cpu", "none", None):
112+
try:
113+
est = TabICLClassifier(**kwargs, offload_mode=om) if om is not None else TabICLClassifier(**kwargs)
114+
break
115+
except (ValueError, TypeError):
116+
continue
117+
else:
118+
est = TabICLClassifier(**kwargs)
119+
t0 = time.time()
120+
est.fit(Xtr.values, ytr.values)
121+
pred_te = est.predict(Xte.values)
122+
fit_s = time.time() - t0
123+
proba_tr = est.predict_proba(Xtr.values)
124+
pred_tr = est.predict(Xtr.values)
125+
if not (np.isfinite(proba_tr).all()):
126+
raise ValueError("non-finite probabilities")
127+
# wrong-but-finite guard (seen from TabICL on MPS at 10k: a
128+
# 0.93-accuracy teacher suddenly at chance): a teacher that
129+
# cannot beat the majority class ON ITS OWN CONTEXT is
130+
# malfunctioning, not weak. Retry on the next device.
131+
majority = float(ytr.value_counts(normalize=True).max())
132+
agree = float((np.asarray(pred_tr).astype(str)
133+
== ytr.astype(str).values).mean())
134+
# below the mode = malfunction (a healthy teacher on heavily
135+
# imbalanced data may only match the majority rate; a broken
136+
# one falls under it, as TabICL-on-MPS did on airline)
137+
if agree < majority - 0.01:
138+
raise ValueError(
139+
f"teacher below majority on its own context"
140+
f" (agree={agree:.3f}, majority={majority:.3f})")
141+
return est, pred_te, pred_tr, proba_tr, fit_s, attempt_device
142+
except Exception as e: # noqa: BLE001
143+
last = e
144+
continue
145+
raise last
146+
147+
148+
def one_cell(name, X, y, n):
149+
if len(X) < n * 4 // 3: # need n plus a test split
150+
return None
151+
rng = np.random.default_rng(SEED)
152+
idx = rng.choice(len(X), min(len(X), n + n // 3), replace=False)
153+
Xs = X.iloc[idx].reset_index(drop=True)
154+
ys = y.iloc[idx].reset_index(drop=True)
155+
Xtr, Xte, ytr, yte = TA.split(Xs, ys, "cls")
156+
157+
row = {"dataset": name, "n_train": len(Xtr), "n_test": len(Xte),
158+
"d": Xs.shape[1]}
159+
160+
t0 = time.time()
161+
row["xgboost"] = TA.score(yte, TA.run_xgb(Xtr, ytr, Xte, "cls"), "cls")
162+
row["xgb_s"] = round(time.time() - t0, 1)
163+
164+
est, pred_te, pred_tr, proba_tr, fit_s, dev = \
165+
teacher_with_fallback(Xtr, ytr, Xte)
166+
row["tabicl"] = TA.score(yte, pred_te, "cls")
167+
row["tabicl_s"] = round(fit_s, 1)
168+
row["tabicl_device"] = dev
169+
170+
# the leak gap: agreement with own training labels minus held-out acc
171+
agree = float((np.asarray(pred_tr).astype(str)
172+
== ytr.astype(str).values).mean())
173+
row["leak_gap"] = round(agree - row["tabicl"], 4)
174+
row["ic_maxp"] = round(float(np.asarray(proba_tr).max(axis=1).mean()), 4)
175+
176+
t0 = time.time()
177+
try:
178+
preds, blob, hold = TA.run_ours_distill_teacher(
179+
Xtr, np.asarray(pred_tr), Xte, "cls", kind="gbt")
180+
row["gbt<-tabicl"] = TA.score(yte, preds, "cls")
181+
row["distill_s"] = round(time.time() - t0, 1)
182+
row["blob_kb"] = round(blob / 1024, 1)
183+
except Exception as e: # noqa: BLE001
184+
if "single class" not in str(e):
185+
raise
186+
row["collapsed"] = 1
187+
188+
classes = [str(c) for c in est.classes_]
189+
preds, _, _ = TA.run_ours_distill_soft(
190+
Xtr, ytr, np.asarray(proba_tr), classes, Xte, kind="gbt")
191+
row["gbt<-tabicl soft"] = TA.score(yte, preds, "cls")
192+
193+
if len(Xtr) <= OOF_AT:
194+
t0 = time.time()
195+
proba_oof, classes_oof = TA.oof_proba(make_teacher, Xtr, ytr)
196+
row["oof_s"] = round(time.time() - t0, 1)
197+
oof_agree = float(
198+
(np.array(classes_oof)[np.asarray(proba_oof).argmax(1)]
199+
== ytr.astype(str).values).mean())
200+
row["oof_gap"] = round(oof_agree - row["tabicl"], 4)
201+
preds, _, _ = TA.run_ours_distill_soft(
202+
Xtr, ytr, np.asarray(proba_oof), classes_oof, Xte, kind="gbt")
203+
row["gbt<-tabicl soft-oof"] = TA.score(yte, preds, "cls")
204+
return row
205+
206+
207+
def run_single_cell(name, n):
208+
"""Child-process entry: one (dataset, n) cell, result to the jsonl."""
209+
if n >= CPU_AT:
210+
# Metal caps a single NDArray at 2**32 bytes and TabICL touches
211+
# MPS internally even with device="cpu" (native SIGABRT, not
212+
# catchable). Make torch report MPS unavailable in this child.
213+
import torch
214+
torch.backends.mps.is_available = lambda: False # noqa: E731
215+
torch.backends.mps.is_built = lambda: False # noqa: E731
216+
did = dict((nm, d) for d, nm in LARGE)[name]
217+
X, y = load_full(did, name)
218+
row = one_cell(name, X, y, n)
219+
if row is not None:
220+
with open(JSONL, "a") as f:
221+
f.write(json.dumps(row) + "\n")
222+
print(f" n={row['n_train']}: tabicl={row['tabicl']:.4f}"
223+
f" xgb={row['xgboost']:.4f}"
224+
f" student={row.get('gbt<-tabicl', float('nan')):.4f}"
225+
f" leak={row['leak_gap']:+.4f}")
226+
227+
228+
def main():
229+
"""Driver: every cell runs in its own subprocess so a native crash
230+
(seen from torch/MPS at 50k rows) or a hang costs one cell, not the
231+
campaign. A cell that crashed once is skipped, not retried."""
232+
import subprocess
233+
os.makedirs(ATTEMPTS, exist_ok=True)
234+
done = set()
235+
if os.path.exists(JSONL):
236+
with open(JSONL) as f:
237+
done = {(json.loads(ln)["dataset"], json.loads(ln)["n_train"])
238+
for ln in f if ln.strip()}
239+
for did, name in LARGE:
240+
for n in SIZES:
241+
if any(d == name and abs(k - n) <= n // 3 for d, k in done):
242+
print(f"{name} n~{n}: cached")
243+
continue
244+
d_est = None
245+
try:
246+
d_est = load_full(did, name)[0].shape[1]
247+
except Exception: # noqa: BLE001
248+
pass
249+
if d_est and n * d_est > MPS_CELL_BUDGET:
250+
print(f"{name} n={n}: skipped, over mps budget"
251+
f" ({n}x{d_est} > {MPS_CELL_BUDGET})")
252+
continue
253+
marker = os.path.join(ATTEMPTS, f"{name}-{n}")
254+
if os.path.exists(marker):
255+
print(f"{name} n={n}: crashed before, skipping")
256+
continue
257+
open(marker, "w").write("attempting\n")
258+
t0 = time.time()
259+
try:
260+
r = subprocess.run(
261+
[sys.executable, "-u", os.path.abspath(__file__),
262+
"--cell", name, str(n)],
263+
timeout=None if CELL_TIMEOUT_S == 0 else CELL_TIMEOUT_S,
264+
cwd=os.path.dirname(os.path.abspath(__file__)) or ".",
265+
capture_output=True, text=True)
266+
except subprocess.TimeoutExpired as te:
267+
sys.stdout.write((te.stdout or b"").decode(errors="ignore")
268+
if isinstance(te.stdout, bytes)
269+
else (te.stdout or ""))
270+
print(f"{name} n={n}: CELL TIMEOUT after {CELL_TIMEOUT_S}s")
271+
continue
272+
sys.stdout.write(r.stdout)
273+
if r.returncode == 0:
274+
os.remove(marker)
275+
# a completed run may still have skipped (too few rows)
276+
else:
277+
tailerr = (r.stderr or "").strip().splitlines()[-2:]
278+
print(f"{name} n={n}: CELL DIED rc={r.returncode}"
279+
f" after {time.time()-t0:.0f}s :: {' | '.join(tailerr)}")
280+
print(f"{name} n={n}: {time.time()-t0:.0f}s")
281+
report()
282+
283+
284+
def report():
285+
if not os.path.exists(JSONL):
286+
return
287+
with open(JSONL) as f:
288+
rows = [json.loads(ln) for ln in f if ln.strip()]
289+
rows.sort(key=lambda r: (r["dataset"], r["n_train"]))
290+
lines = [
291+
"# Scale study: distillation beyond 1500 rows", "",
292+
"TabICL v2 as the teacher on the naturally large TabArena",
293+
"datasets, at increasing training sizes. Questions: does student",
294+
"retention hold, where does tuned xgboost catch the zero-shot",
295+
"teacher, and does the in-context label leak grow with context",
296+
"(the gap column; see permissive-teachers.md).", "",
297+
f"Device: {DEVICE} with cpu fallback (per-cell `dev`).", "",
298+
"| dataset | n_train | xgboost | TabICL | gbt<-TabICL | soft |"
299+
" soft-oof | leak gap | oof gap | teacher s | distill s | blob KB |",
300+
"|---|---|---|---|---|---|---|---|---|---|---|---|",
301+
]
302+
for r in rows:
303+
def fmt(v, nd=3):
304+
return f"{v:.{nd}f}" if isinstance(v, (int, float)) else "-"
305+
lines.append(
306+
f"| {r['dataset']} | {r['n_train']} | {fmt(r.get('xgboost'))} |"
307+
f" {fmt(r.get('tabicl'))} | {fmt(r.get('gbt<-tabicl'))} |"
308+
f" {fmt(r.get('gbt<-tabicl soft'))} |"
309+
f" {fmt(r.get('gbt<-tabicl soft-oof'))} |"
310+
f" {fmt(r.get('leak_gap'), 4)} | {fmt(r.get('oof_gap'), 4)} |"
311+
f" {r.get('tabicl_s', '-')} | {r.get('distill_s', '-')} |"
312+
f" {r.get('blob_kb', '-')} |")
313+
lines.append("")
314+
with open(RESULTS, "w") as f:
315+
f.write("\n".join(lines))
316+
print(f"wrote {RESULTS}")
317+
318+
319+
if __name__ == "__main__":
320+
if len(sys.argv) >= 4 and sys.argv[1] == "--cell":
321+
run_single_cell(sys.argv[2], int(sys.argv[3]))
322+
else:
323+
main()

0 commit comments

Comments
 (0)