Skip to content

Commit f81c4e9

Browse files
mstrathmanclaude
andcommitted
bench: TabPFN harness (v2 + v3) and de-receipt benchmark scripts
tabpfn_arena.py runs TabPFN-2 (Prior Labs License: distillation permitted, the commercially usable teacher) and TabPFN-3 (the latest, non-commercial: evaluation ceiling) on the TabArena subset, distills both into gbt students hard and soft, and merges the TabFM campaign numbers at report time. Checkpointed per dataset; weights need a one-time Prior Labs license acceptance (TABPFN_TOKEN). Also strips the removed 'receipt' option from every benchmark script (they predated the receipts removal) and tightens the TabPFN license note in the docs: the permissive Prior Labs terms cover the TabPFN-2 weights; 2.5/2.6/3 are non-commercial. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent d42d342 commit f81c4e9

14 files changed

Lines changed: 245 additions & 25 deletions

‎README.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -211,10 +211,12 @@ derived from its teacher, and some model licenses restrict that: TabFM's
211211
non-commercial license, for example, permits testing and evaluation (which
212212
is what our benchmarks are) but expressly excludes distilling it for
213213
commercial use or deploying its outputs in production. Chronos is
214-
Apache-2.0; TabPFN's Prior Labs License permits distillation, including
214+
Apache-2.0. TabPFN is version-dependent: the TabPFN-2 weights are under
215+
the Prior Labs License, which permits distillation, including
215216
commercially, if you attribute ("Built with PriorLabs-TabPFN" and a
216-
TabPFN-prefixed model name) when you distribute the student; your own
217-
labels or models carry no such limits. The
217+
TabPFN-prefixed model name) when you distribute the student, while the
218+
later TabPFN-2.5/2.6/3 weights are non-commercial. Your own labels or
219+
models carry no such limits. The
218220
extension's license gate (`accept_license`) makes a restrictively-licensed
219221
teacher an explicit opt-in rather than an accident, but checking that your
220222
use fits the teacher's license is on you.

‎benchmarks/anomaly_tsb_ad.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ def score(vals):
5454
opts = "'model','%s'," % MODEL if MODEL else ""
5555
rows = db.execute(
5656
"SELECT anomaly_probability FROM detect_anomalies("
57-
"'SELECT ts,value FROM a ORDER BY rn', json_object(%s'receipt',0))" % opts
57+
"'SELECT ts,value FROM a ORDER BY rn', json_object(%s))" % opts
5858
).fetchall()
5959
return np.array([r[0] if r[0] is not None else 0.0 for r in rows], float)
6060

‎benchmarks/compare.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ def run_extension_model(model, train, horizon):
5555
db.enable_load_extension(True)
5656
db.load_extension(EXT)
5757
syn.load_into(db, train)
58-
opts = json.dumps({"model": model, "receipt": 0})
58+
opts = json.dumps({"model": model})
5959
t0 = time.perf_counter()
6060
rows = db.execute(
6161
"SELECT forecast, lower_bound, upper_bound FROM"

‎benchmarks/forecast_bench.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ def run_ext(model, hist, start, freq, h):
8181
idx = pd.period_range(start=start, periods=len(hist), freq=freq).to_timestamp()
8282
db.executemany("INSERT INTO series VALUES (?,?)",
8383
[(t.isoformat(), float(v)) for t, v in zip(idx, hist)])
84-
opts = json.dumps({"model": model, "confidence_level": 0.95, "receipt": 0})
84+
opts = json.dumps({"model": model, "confidence_level": 0.95})
8585
rows = db.execute(
8686
"SELECT forecast, lower_bound, upper_bound FROM forecast('SELECT ts,"
8787
" value FROM series', ?, ?)", (h, opts)).fetchall()

‎benchmarks/forecast_distill.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ def distill_one(rows, feats, base, target_idx, std, nfeat):
137137
[[k] + list(map(float, f)) for k, f in enumerate(feats)])
138138
out = db.execute(
139139
f"SELECT row_ref, prediction FROM predict(NULL, 'SELECT id, {fl} FROM"
140-
f" te', json_object('model','s','receipt',0)) ORDER BY row_ref").fetchall()
140+
f" te', json_object('model','s')) ORDER BY row_ref").fetchall()
141141
db.close()
142142
return np.array([float(r[1]) for r in out]) * std + base, blob
143143

‎benchmarks/forecast_native.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ def main():
7474
db.executemany(f"INSERT INTO w VALUES ({','.join('?' * ncol)})",
7575
[list(map(float, r)) for r in rows])
7676
cols = ",".join(f"c{i}" for i in range(ncol))
77-
opts = json.dumps({"context": L, "horizon": H, "student_id": "f", "receipt": 0})
77+
opts = json.dumps({"context": L, "horizon": H, "student_id": "f"})
7878
t0 = time.time()
7979
mid, trows, rmse = db.execute(
8080
f"SELECT model_id, train_rows, train_rmse FROM distill_forecast("
@@ -84,7 +84,7 @@ def main():
8484
print(f"distilled '{mid}' on {trows} windows in {time.time()-t0:.0f}s "
8585
f"(holdout rmse {rmse:.3f}, blob {blob} B)")
8686

87-
sopts = json.dumps({"model": "f", "receipt": 0})
87+
sopts = json.dumps({"model": "f"})
8888
native, snaive, chronos = [], [], []
8989
for tb in tests:
9090
hist, truth = tb["hist"], tb["truth"]

‎benchmarks/forecast_onnx.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ def main():
7474
r = np.array(db.execute(
7575
"SELECT forecast, lower_bound, upper_bound FROM forecast("
7676
"'SELECT ts, value FROM s', ?, json_object('model','chronos-onnx',"
77-
"'confidence_level',0.8,'receipt',0))", (H,)).fetchall())
77+
"'confidence_level',0.8))", (H,)).fetchall())
7878
onx["mase"].append(mase(r[:, 0], truth, hist, M))
7979
c, wk = cov_winkler(r[:, 1], r[:, 2], truth, alpha=0.2)
8080
onx["cov"].append(c)

‎benchmarks/forecast_quantile.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ def main():
8686
[list(map(float, r)) for r in rows])
8787
cols = ",".join(f"c{i}" for i in range(ncol))
8888
opts = json.dumps({"context": L, "horizon": H, "student_id": "qf",
89-
"receipt": 0, "quantiles": QLEV})
89+
"quantiles": QLEV})
9090
t0 = time.time()
9191
mid, tr, rmse = db.execute(
9292
f"SELECT model_id, train_rows, train_rmse FROM distill_forecast("
@@ -100,7 +100,7 @@ def serve(hist, conf):
100100
idx = pd.date_range("2020-01-01", periods=len(hist), freq="h")
101101
db.executemany("INSERT INTO s VALUES (?,?)",
102102
[(t.isoformat(), float(v)) for t, v in zip(idx, hist)])
103-
o = json.dumps({"model": "qf", "confidence_level": conf, "receipt": 0})
103+
o = json.dumps({"model": "qf", "confidence_level": conf})
104104
return np.array(db.execute(
105105
"SELECT forecast, lower_bound, upper_bound FROM forecast("
106106
"'SELECT ts, value FROM s', ?, ?)", (H, o)).fetchall())

‎benchmarks/forecast_student_arch.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ def serve(model):
7676
[(t.isoformat(), float(v)) for t, v in zip(idx, hist)])
7777
fc = [r[0] for r in db.execute(
7878
"SELECT forecast FROM forecast('SELECT ts,value FROM q',?,"
79-
"json_object('model',?,'confidence_level',0.8,'receipt',0))",
79+
"json_object('model',?,'confidence_level',0.8))",
8080
(H, model)).fetchall()]
8181
out.append(mase(np.array(fc), truth[:len(fc)], hist, M))
8282
return float(np.mean(out))
@@ -90,7 +90,7 @@ def serve(model):
9090
sid = f"h{hid}"
9191
t0 = time.time()
9292
opts = json.dumps({"teacher": "chronos-onnx", "context": 512, "horizon": H,
93-
"student_id": sid, "hidden": hid, "epochs": EPOCHS, "receipt": 0})
93+
"student_id": sid, "hidden": hid, "epochs": EPOCHS})
9494
db.execute("SELECT model_id FROM distill_forecast("
9595
"'SELECT series_key, value FROM s ORDER BY series_key, t', json(?))",
9696
(opts,)).fetchone()

‎benchmarks/gift_eval_bench.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ def train_student(db, name, ys, H, m, L, si_eval):
113113
except sqlite3.OperationalError:
114114
pass
115115
opts = json.dumps({"context": L, "horizon": H, "student_id": "qf",
116-
"receipt": 0, "quantiles": QLEV, "epochs": 800})
116+
"quantiles": QLEV, "epochs": 800})
117117
db.execute(f"SELECT model_id FROM distill_forecast('SELECT {cols} FROM w',"
118118
" json(?))", (opts,)).fetchone()
119119
return True
@@ -128,7 +128,7 @@ def serve_student(db, hist, H):
128128
dec = np.zeros((H, Q))
129129
for conf, (a, b) in [(0.8, (0, 8)), (0.6, (1, 7)), (0.4, (2, 6)),
130130
(0.2, (3, 5))]:
131-
o = json.dumps({"model": "qf", "confidence_level": conf, "receipt": 0})
131+
o = json.dumps({"model": "qf", "confidence_level": conf})
132132
r = np.array(db.execute("SELECT forecast, lower_bound, upper_bound FROM"
133133
" forecast('SELECT ts, value FROM s', ?, ?)",
134134
(H, o)).fetchall())

0 commit comments

Comments
 (0)