Skip to content

Commit 2da027a

Browse files
author
Kenneth Pernyer
committed
Add pinned TabPFN-3 GPU comparison with isolated test-row inference
1 parent be95b0b commit 2da027a

5 files changed

Lines changed: 312 additions & 0 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ A simple model that wins is a successful research result. A shared model valid f
6969
| [Research plan](docs/research-plan.md) | Hypotheses, comparisons, evaluation, and stop conditions |
7070
| [Data contract](docs/data-contract.md) | What to collect, label, connect, and keep out |
7171
| [Random Forest demo](docs/random-forest-demo.md) | A small CPU-only classifier, historical base-rate comparison, and temporal checks on synthetic data |
72+
| [TabPFN-3 comparison](docs/tabpfn-comparison.md) | An optional GPU experiment using the same rows and a pinned pretrained classifier |
7273
| [Application integration](docs/application-integration.md) | How apps contribute to the Organisation Core |
7374
| [First 90 days](docs/first-90-days.md) | A bounded starting project and collaboration questions |
7475
| [Reading list](docs/reading-list.md) | Intellectual lineage and the limits of the evidence |

‎com_jepa/tabpfn_demo.py‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
"""Optional TabPFN-3 GPU comparison using the forest demo's exact synthetic split.
2+
3+
Requires a separately downloaded, hash-verified checkpoint. Does not serve a model.
4+
"""
5+
6+
import argparse
7+
from hashlib import file_digest
8+
from importlib.metadata import version
9+
import json
10+
from pathlib import Path
11+
import platform
12+
from time import perf_counter
13+
14+
from .forest_demo import SETTLED, feature_matrix, generate_rows, run_demo, temporal_split
15+
16+
17+
MODEL_REPO = "Prior-Labs/tabpfn_3"
18+
MODEL_REVISION = "24a16a89d245878b846555110985634aa2e656d7"
19+
MODEL_FILENAME = "tabpfn-v3-classifier-v3_default.ckpt"
20+
MODEL_SHA256 = "d0d865d54dfbc524f5703104be90620182dca7e5fb2c16de72e9959ea18f3988"
21+
MODEL_LICENCE = "https://huggingface.co/Prior-Labs/tabpfn_3/blob/" + MODEL_REVISION + "/LICENSE"
22+
23+
24+
def verify_checkpoint(path, expected_sha256):
25+
if len(expected_sha256) != 64 or any(c not in "0123456789abcdef" for c in expected_sha256):
26+
raise ValueError("checkpoint SHA-256 must be 64 lowercase hexadecimal characters")
27+
with Path(path).open("rb") as stream:
28+
actual = file_digest(stream, "sha256").hexdigest()
29+
if actual != expected_sha256:
30+
raise ValueError("checkpoint hash mismatch; refusing to load model")
31+
return actual
32+
33+
34+
def predict_independently(estimator, rows):
35+
"""A future test row must not enter an earlier row's preprocessing/context."""
36+
import numpy as np
37+
38+
classes = list(estimator.classes_)
39+
if set(classes) != {0, 1}:
40+
raise ValueError("expected classes 0=not_met and 1=met")
41+
positive_column = classes.index(1)
42+
predictions = []
43+
for row in rows:
44+
output = np.asarray(estimator.predict_proba(np.asarray([row], dtype=float)))
45+
if output.shape != (1, 2) or not np.isfinite(output).all():
46+
raise ValueError("invalid probability response")
47+
if (output < 0).any() or (output > 1).any() or not np.allclose(output.sum(axis=1), 1):
48+
raise ValueError("invalid probability distribution")
49+
predictions.append(float(output[0, positive_column]))
50+
return np.asarray(predictions)
51+
52+
53+
def comparison(checkpoint, expected_sha256, samples=300, seed=42, gpu_memory_gib=4):
54+
checkpoint = Path(checkpoint).resolve()
55+
if expected_sha256 != MODEL_SHA256:
56+
raise ValueError("this adapter pins one checkpoint; update its provenance before changing weights")
57+
checkpoint_hash = verify_checkpoint(checkpoint, expected_sha256)
58+
if checkpoint.name != MODEL_FILENAME:
59+
raise ValueError("use the explicitly documented TabPFN-3 checkpoint filename")
60+
if not 0 < gpu_memory_gib <= 8:
61+
raise ValueError("GPU allocation budget must be greater than 0 and at most 8 GiB")
62+
63+
import torch
64+
from sklearn.metrics import accuracy_score, brier_score_loss, log_loss
65+
from tabpfn import TabPFNClassifier
66+
67+
if not torch.cuda.is_available():
68+
raise ValueError("this experiment requires a working CUDA GPU; no silent CPU fallback")
69+
properties = torch.cuda.get_device_properties(0)
70+
budget_bytes = int(gpu_memory_gib * 1024**3)
71+
torch.cuda.set_per_process_memory_fraction(min(1.0, budget_bytes / properties.total_memory), 0)
72+
torch.set_num_threads(2)
73+
torch.manual_seed(seed)
74+
torch.cuda.reset_peak_memory_stats(0)
75+
76+
baseline_start = perf_counter()
77+
report = run_demo(samples, seed)
78+
baseline_seconds = perf_counter() - baseline_start
79+
rows = generate_rows(samples, seed)
80+
train, test, excluded = temporal_split(rows, report["training_cutoff"], report["test_start"], report["evaluation_as_of"])
81+
if [r["initiative_id"] for r in train] != report["training_initiatives"]:
82+
raise ValueError("training split differs from baseline")
83+
if [r["initiative_id"] for r in test] != [r["initiative_id"] for r in report["test_predictions"]]:
84+
raise ValueError("test split differs from baseline")
85+
if excluded != report["counts"]["excluded"]:
86+
raise ValueError("exclusion policy differs from baseline")
87+
88+
import numpy as np
89+
90+
x_train, x_test = feature_matrix(train), feature_matrix(test)
91+
y_train = np.asarray([SETTLED[r["label_status"]] for r in train])
92+
y_test = np.asarray([SETTLED[r["label_status"]] for r in test])
93+
estimator = TabPFNClassifier(
94+
model_path=checkpoint, device="cuda", random_state=seed,
95+
n_estimators=4, auto_scale_n_estimators=False, n_preprocessing_jobs=1,
96+
fit_mode="fit_preprocessors", softmax_temperature=0.9,
97+
)
98+
torch.cuda.synchronize()
99+
start = perf_counter()
100+
estimator.fit(np.asarray(x_train, dtype=float), y_train)
101+
torch.cuda.synchronize()
102+
fit_seconds = perf_counter() - start
103+
start = perf_counter()
104+
predicted = predict_independently(estimator, x_test)
105+
torch.cuda.synchronize()
106+
predict_seconds = perf_counter() - start
107+
report["metrics"]["tabpfn_3"] = {
108+
"brier_score": float(brier_score_loss(y_test, predicted)),
109+
"log_loss": float(log_loss(y_test, predicted, labels=[0, 1])),
110+
"accuracy_at_0_5": float(accuracy_score(y_test, predicted >= 0.5)),
111+
}
112+
for row, probability in zip(report["test_predictions"], predicted):
113+
row["tabpfn_3"] = float(probability)
114+
report["environment"].update({p: version(p) for p in ("tabpfn", "torch", "huggingface-hub")})
115+
report["environment"].update({"python": platform.python_version(), "architecture": platform.machine(),
116+
"gpu": properties.name, "cuda_runtime": torch.version.cuda})
117+
report["candidate"] = {
118+
"name": "TabPFN-3", "repository": MODEL_REPO, "revision": MODEL_REVISION,
119+
"filename": checkpoint.name, "sha256": checkpoint_hash, "licence": MODEL_LICENCE,
120+
"n_estimators": 4, "random_state": seed, "device": "cuda", "n_preprocessing_jobs": 1,
121+
"auto_scale_n_estimators": False, "fit_mode": "fit_preprocessors", "softmax_temperature": 0.9,
122+
"prediction_protocol": "one_test_row_per_call_with_fixed_training_context",
123+
"gpu_allocator_budget_gib": gpu_memory_gib,
124+
"peak_torch_allocated_bytes": torch.cuda.max_memory_allocated(0),
125+
"peak_torch_reserved_bytes": torch.cuda.max_memory_reserved(0),
126+
}
127+
report["timing_seconds"] = {
128+
"complete_baseline_demo": baseline_seconds,
129+
"tabpfn_fit_including_model_loading": fit_seconds,
130+
"tabpfn_predict_all_test_rows_individually": predict_seconds,
131+
}
132+
report["implementation_sha256"] = {}
133+
for name in ("forest_demo.py", "tabpfn_demo.py"):
134+
with Path(__file__).with_name(name).open("rb") as stream:
135+
report["implementation_sha256"][name] = file_digest(stream, "sha256").hexdigest()
136+
report["limitations"].extend([
137+
"One pretrained checkpoint, one synthetic generator and seed; no claim of model superiority or organisational validity.",
138+
"TabPFN has external pretraining; this comparison does not isolate architecture from pretraining.",
139+
"GPU timing is from a shared machine and single-row prediction; not a controlled hardware speed benchmark.",
140+
"CUDA seeds do not guarantee bitwise reproducibility across platforms or library versions.",
141+
"Allocator budget is not a hard bound on all process or unified-memory use.",
142+
"TabPFN-3 weights and outputs have their own non-commercial licence; the repository MIT licence does not replace it.",
143+
])
144+
return report
145+
146+
147+
def main():
148+
parser = argparse.ArgumentParser(description=__doc__)
149+
parser.add_argument("--checkpoint", type=Path, required=True)
150+
parser.add_argument("--checkpoint-sha256", default=MODEL_SHA256)
151+
parser.add_argument("--samples", type=int, default=300)
152+
parser.add_argument("--seed", type=int, default=42)
153+
parser.add_argument("--gpu-memory-gib", type=float, default=4)
154+
parser.add_argument("--output", type=Path, required=True)
155+
args = parser.parse_args()
156+
try:
157+
if args.output.exists():
158+
raise ValueError("output already exists; choose a new report filename")
159+
report = comparison(args.checkpoint, args.checkpoint_sha256, args.samples, args.seed, args.gpu_memory_gib)
160+
args.output.parent.mkdir(parents=True, exist_ok=True)
161+
with args.output.open("x") as stream:
162+
json.dump(report, stream, indent=2, allow_nan=False)
163+
stream.write("\n")
164+
except ModuleNotFoundError as error:
165+
parser.exit(1, f"Missing optional dependency: {error.name}. See docs/tabpfn-comparison.md.\n")
166+
except (ValueError, OSError) as error:
167+
parser.exit(1, f"error: {error}\n")
168+
print("SYNTHETIC GPU COMPARISON — not evidence of organisational prediction quality")
169+
print(f"Train: {report['counts']['train']}; test: {report['counts']['test']}; generator seed: {report['seed']}")
170+
print("Model Brier (lower) Log loss (lower) Accuracy @ 0.5")
171+
for name, metrics in report["metrics"].items():
172+
print(f"{name:25} {metrics['brier_score']:.4f} {metrics['log_loss']:.4f} {metrics['accuracy_at_0_5']:.4f}")
173+
print(f"TabPFN-3: checkpoint {report['candidate']['sha256']}")
174+
print(f"Report: {args.output}")
175+
176+
177+
if __name__ == "__main__":
178+
main()

‎docs/tabpfn-comparison.md‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# TabPFN-3 GPU comparison
2+
3+
This optional adapter compares a pretrained TabPFN-3 classifier with the Random Forest and historical base rate on the **same fictional data, temporal split, feature allowlist, and scoring targets**. It does not change the generator, label the discussion cases, or introduce real organisational data.
4+
5+
## What is pinned
6+
7+
| Component | Selection |
8+
| --- | --- |
9+
| TabPFN package | `8.5.0` |
10+
| PyTorch | `2.11.0`, official CUDA 13.0 build for the GPU experiment |
11+
| Model repository | `Prior-Labs/tabpfn_3` |
12+
| Model revision | `24a16a89d245878b846555110985634aa2e656d7` |
13+
| Checkpoint | `tabpfn-v3-classifier-v3_default.ckpt` |
14+
| Checkpoint SHA-256 | `d0d865d54dfbc524f5703104be90620182dca7e5fb2c16de72e9959ea18f3988` |
15+
| Candidate ensemble | Four estimators; automatic ensemble scaling disabled |
16+
| Test protocol | One test row per prediction call, fixed historical training context |
17+
18+
The adapter verifies the checkpoint hash before loading it. It requires CUDA and does not silently fall back to another model or device. A future checkpoint comparison must update provenance explicitly.
19+
20+
## Environment and model access
21+
22+
The GPU setup uses Linux ARM64, Python 3.12, and an NVIDIA GB10. The pinned tabular dependencies also support this Python version. Create an isolated environment on the GPU machine; keep its existing inference service separate.
23+
24+
```sh
25+
python3.12 -m venv .venv
26+
.venv/bin/python -m pip install torch==2.11.0 --index-url https://download.pytorch.org/whl/cu130
27+
.venv/bin/python -m pip install -r requirements-tabpfn.txt
28+
.venv/bin/python -m pip check
29+
```
30+
31+
The [upstream package](https://github.com/PriorLabs/TabPFN) documents local inference. The model weights and outputs have a [separate non-commercial licence](https://huggingface.co/Prior-Labs/tabpfn_3/blob/24a16a89d245878b846555110985634aa2e656d7/LICENSE). The repository's MIT licence does not grant production rights for TabPFN. This example is a non-commercial synthetic evaluation; it does not distribute weights, distil their outputs into another model, or deploy a service.
32+
33+
After reviewing the applicable model terms, download the fixed revision:
34+
35+
```sh
36+
.venv/bin/python - <<'PY'
37+
from huggingface_hub import hf_hub_download
38+
from com_jepa.tabpfn_demo import MODEL_REPO, MODEL_REVISION, MODEL_FILENAME
39+
for filename in ('LICENSE', MODEL_FILENAME):
40+
hf_hub_download(
41+
repo_id=MODEL_REPO, revision=MODEL_REVISION, filename=filename,
42+
local_dir='artifacts/tabpfn-3',
43+
)
44+
PY
45+
```
46+
47+
At initial inspection this revision was publicly downloadable without authentication. If access requirements change, use the publisher's authorised access process. The adapter never accepts a licence, logs into an account, or downloads an unspecified replacement model.
48+
49+
## Run the experiment
50+
51+
```sh
52+
.venv/bin/python -m unittest discover -s tests -v
53+
.venv/bin/python -m com_jepa validate examples/fictional-trajectory.jsonl
54+
.venv/bin/python -m com_jepa.tabpfn_demo \
55+
--checkpoint artifacts/tabpfn-3/tabpfn-v3-classifier-v3_default.ckpt \
56+
--output artifacts/tabpfn-comparison.json
57+
.venv/bin/python -m pip freeze > artifacts/tabpfn-environment.txt
58+
```
59+
60+
The output file must be new. The model and reports stay in ignored `artifacts/`. To initiate an already prepared run from another machine, use SSH to the experiment checkout and invoke the same module. Host aliases and paths are operator configuration, not embedded in the adapter.
61+
62+
The default PyTorch allocator budget is 4 GiB, adjustable with `--gpu-memory-gib` up to 8 GiB. This is not a hard bound on all process or unified-memory use. Check available memory and other workloads before running. The adapter does not stop, replace, or reconfigure an existing model service, and no GPU runner is added to CI.
63+
64+
## What is compared
65+
66+
The adapter first runs the original forest demonstration, then reconstructs and verifies identical training/test identities and exclusions. The pretrained candidate gets only the selected training features and labels. It processes each test row separately; it never receives future test rows or any test labels as context. Four feature values and the fixed historical examples are its entire task input.
67+
68+
TabPFN uses external pretraining, so equal task data does not imply equal total training information or compute. This is a practical estimator comparison, not an experiment isolating model architecture from pretraining. It is also not a JEPA experiment or a test of organisational transfer.
69+
70+
The JSON report extends the baseline report with candidate metrics, per-row probabilities, checkpoint provenance, code hashes, device/library versions, timing, and peak PyTorch memory. `fit` includes checkpoint loading and preparation of the training context; it is not fine-tuning the neural weights. Prediction time covers individual calls and must not be advertised as maximum batched throughput. The machine may be sharing its GPU with another workload.
71+
72+
GPU timings and floating-point outputs need not be bitwise reproducible across hardware and versions. Preserve the environment capture alongside the report. A single generator/seed cannot establish broad model superiority, even if one model scores better on this run. Probability calibration, uncertainty intervals, multiple reviewed generators, and real-data evaluation remain separate work.
73+
74+
## Testing boundaries
75+
76+
Unit tests verify checkpoint rejection, positive-class mapping, probability validity, and one-row-per-call isolation without downloading a model. The existing temporal and missing-label tests still apply. A real checkpoint execution is a separate, explicitly invoked GPU experiment, not a stubbed test result.

‎requirements-tabpfn.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Install CUDA PyTorch from its official index first; see the comparison guide.
2+
-r requirements-ml.txt
3+
torch==2.11.0
4+
tabpfn==8.5.0

‎tests/test_tabpfn_demo.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
from hashlib import sha256
2+
from importlib.util import find_spec
3+
from pathlib import Path
4+
import tempfile
5+
import unittest
6+
7+
from com_jepa.tabpfn_demo import predict_independently, verify_checkpoint
8+
9+
10+
class CheckpointTests(unittest.TestCase):
11+
def test_changed_checkpoint_is_rejected_before_loading(self):
12+
with tempfile.TemporaryDirectory() as directory:
13+
path = Path(directory) / "checkpoint.ckpt"
14+
path.write_bytes(b"fictional checkpoint for hash test")
15+
expected = sha256(path.read_bytes()).hexdigest()
16+
self.assertEqual(verify_checkpoint(path, expected), expected)
17+
path.write_bytes(b"changed")
18+
with self.assertRaisesRegex(ValueError, "hash mismatch"):
19+
verify_checkpoint(path, expected)
20+
with self.assertRaisesRegex(ValueError, "64 lowercase"):
21+
verify_checkpoint(path, "invalid")
22+
23+
24+
@unittest.skipUnless(find_spec("numpy"), "optional ML adapter tests: install requirements-ml.txt")
25+
class PredictionIsolationTests(unittest.TestCase):
26+
def test_each_call_sees_one_row_and_positive_class_is_looked_up(self):
27+
class Estimator:
28+
classes_ = [1, 0]
29+
30+
def __init__(self):
31+
self.seen = []
32+
33+
def predict_proba(self, rows):
34+
self.seen.append(rows.tolist())
35+
return [[float(rows[0][0]), 1 - float(rows[0][0])]]
36+
37+
estimator = Estimator()
38+
self.assertEqual(predict_independently(estimator, [[0.2], [0.8]]).tolist(), [0.2, 0.8])
39+
self.assertEqual(estimator.seen, [[[0.2]], [[0.8]]])
40+
41+
def test_invalid_probabilities_fail_instead_of_becoming_metrics(self):
42+
class Estimator:
43+
classes_ = [0, 1]
44+
45+
def predict_proba(self, rows):
46+
return [[0.4, 0.8]]
47+
48+
with self.assertRaisesRegex(ValueError, "distribution"):
49+
predict_independently(Estimator(), [[0.2]])
50+
51+
52+
if __name__ == "__main__":
53+
unittest.main()

0 commit comments

Comments
 (0)