fix(ci): track new packages and shipped weights

The repo-root gitignore had data/ (unanchored) which silently
matched the new defaultplusplus.data subpackage and excluded it
from git. Also drop the legacy pretrained/weights/*.pt /
*.pkl rules that pre-dated the 0.4.0 decision to ship trained
weights inside the wheel.

Stages the previously-ignored files:
  - defaultplusplus.data (data download + CLI)
  - pretrained/weights/{encoder,decoder}.pt

Updates test_phase0_gate.test_pretrained_weights_are_tracked so the
suite enforces the inverted invariant: the weights MUST be tracked
or the next wheel build silently breaks.

Author: SigmaJahan

.gitignore

@@ -46,16 +46,23 @@ results/
 *.fls
 *.synctex.gz
 
-# Pretrained weights (downloaded on first use)
-src/defaultplusplus/pretrained/weights/*.pt
-src/defaultplusplus/pretrained/weights/*.pkl
+# Pretrained weights are tracked under
+# defaultplusplus/src/defaultplusplus/pretrained/weights/ — they ship
+# with the wheel (~5 MB total). The old "weights are downloaded on
+# first use" rules below are intentionally NOT used for the in-tree
+# pretrained checkpoints; they remain only in case some research-side
+# script writes a separate cache to a project root location.
 
 # Temp staging
 .tmp/
-.0_DEFault-Previous-Work/ 
+.0_DEFault-Previous-Work/
 evaluation/
-2_Raw-to-Standardized/ 
-data/ 
+2_Raw-to-Standardized/
+# Top-level only — DO NOT use bare ``data/`` here; that pattern would
+# also match the installable ``defaultplusplus/src/defaultplusplus/data/``
+# subpackage and cause CI checkouts to be missing the data download
+# module.
+/data/
 remaining-tasks.md
 
 

defaultplusplus/.gitignore

@@ -1,5 +1,7 @@
 .venv/
 .pytest_cache/
 results/
-src/defaultplusplus/pretrained/weights/*.pt
-src/defaultplusplus/pretrained/weights/*.pkl
+# Pretrained weights ARE tracked (they ship in the wheel). If any
+# research-side script writes ad-hoc .pkl checkpoints, prefer naming
+# them with a clearly-temporary suffix (e.g. ``*.tmp.pkl``) and
+# adding that pattern instead.

defaultplusplus/src/defaultplusplus/data/__init__.py

@@ -0,0 +1,30 @@
+"""Public bench-dataset distribution.
+
+The training data CSVs are too large to ship in the wheel, so the
+package downloads them on demand.
+
+    from defaultplusplus.data import download_bench
+    path = download_bench(version="v1")          # ~/.cache/defaultplusplus/bench/v1
+    # or
+    defaultpp-bench-download                     # console script
+
+Verifies SHA256 before extraction and short-circuits when an existing
+download already matches the expected checksum, so re-running is free.
+"""
+from .download import (
+    BENCH_VERSIONS,
+    BenchVersion,
+    DatasetNotPublishedError,
+    DownloadError,
+    bench_dir,
+    download_bench,
+)
+
+__all__ = [
+    "BENCH_VERSIONS",
+    "BenchVersion",
+    "DatasetNotPublishedError",
+    "DownloadError",
+    "bench_dir",
+    "download_bench",
+]

defaultplusplus/src/defaultplusplus/data/cli.py

@@ -0,0 +1,98 @@
+"""``defaultpp-bench-download`` console script.
+
+Downloads + verifies + extracts the published DEFault++ benchmark to
+the user's cache. Idempotent — re-running on a verified cache is a
+no-op.
+
+    defaultpp-bench-download              # latest version
+    defaultpp-bench-download --version v1
+    defaultpp-bench-download --force      # bypass cache and re-fetch
+    defaultpp-bench-download --list       # show known versions
+
+Override the source for testing or mirroring:
+
+    defaultpp-bench-download --url file:///path/to/bundle.tar.gz \\
+                             --sha256 <expected sha>
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from .download import (
+    BENCH_VERSIONS,
+    DEFAULT_VERSION,
+    DatasetNotPublishedError,
+    DownloadError,
+    bench_dir,
+    download_bench,
+)
+
+
+def _format_versions() -> str:
+    rows = []
+    for name, info in BENCH_VERSIONS.items():
+        status = "PUBLISHED" if info.url else "NOT YET PUBLISHED"
+        rows.append(f"  {name:6s} [{status}]  {info.description}")
+    return "\n".join(rows)
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--version", default=DEFAULT_VERSION,
+                   help=f"bench version (default: {DEFAULT_VERSION})")
+    p.add_argument("--cache-dir", type=Path, default=None,
+                   help="override the cache root (default: "
+                        "$DEFAULTPP_CACHE_DIR / $XDG_CACHE_HOME / platform default)")
+    p.add_argument("--force", action="store_true",
+                   help="wipe cached extract and re-download")
+    p.add_argument("--url", default=None,
+                   help="custom source URL (file:// or http(s)://); requires --sha256")
+    p.add_argument("--sha256", default=None,
+                   help="expected SHA256 for --url")
+    p.add_argument("--no-verify", action="store_true",
+                   help="skip per-file MANIFEST verification (faster on known-good cache)")
+    p.add_argument("--list", action="store_true",
+                   help="print the known bench versions and exit")
+    args = p.parse_args(argv)
+
+    if args.list:
+        print("Available bench versions:")
+        print(_format_versions())
+        return 0
+
+    if (args.url is None) ^ (args.sha256 is None):
+        print("ERROR: --url and --sha256 must be used together",
+              file=sys.stderr)
+        return 2
+
+    try:
+        path = download_bench(
+            version=args.version,
+            cache_dir=args.cache_dir,
+            force=args.force,
+            url_override=args.url,
+            sha256_override=args.sha256,
+            verify_manifest=not args.no_verify,
+        )
+    except DatasetNotPublishedError as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        return 3
+    except DownloadError as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        return 1
+
+    print(f"[ok] benchmark {args.version} ready at:")
+    print(f"  {path}")
+    print()
+    print(f"Trainer entry point:")
+    print(f"  python defaultplusplus/scripts/train_diagnoser.py \\\\")
+    print(f"      --arch encoder \\\\")
+    print(f"      --csv {path / 'encoder_merged.csv'} \\\\")
+    print(f"      --output encoder.pt")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())