feat(autotune): S7 cycle orchestrator + verifier seam

run_cycle.py drives dataset → train → eval → markdown report from one TOML
config, so a cycle stops being four hand-typed commands whose flags drifted
between runs. Two flags mattered enough to pin in tests: `--fewshot-top-k 0`
(the tuning replaces few-shot; anything else is a train/inference mismatch)
and unique `--report-suffix` per target (two runs sharing one write the same
file and erase each other).

The report refuses to quote raw EA alone. The v10 base run has 47 transport
holes, so its raw EA reads 37.0% against 48.4% on the 153 questions it really
answered; the markdown carries answered/exceptions next to every number and,
for two targets, a paired comparison over the questions both answered. That
paired block reproduces the hand-rolled scoring exactly — tuned 55.6% vs base
48.4%, +7.2 pp, fixed 24 / broke 13, McNemar exact p = 0.099 — which is an
independent second implementation agreeing, not a copy.

`train` is honest about not being local: mode="remote" prints the runbook
pointer instead of pretending a Colab/Kaggle session can be started here.

nl_sql.autotune.verifiers is the answer → pass/fail seam the track needs for
the S6 levers (execution-filter of gold, teacher normalisation). It mirrors
the eval runner deliberately: candidate SQL through the AST guards, gold
unguarded, comparison through safe_compare_pred so a candidate that never
executed cannot be blessed by an empty gold (the qid-518 class, pinned by a
mutation-checked test).

Config is TOML, not the YAML the plan sketched: tomllib is stdlib, so this
adds no dependency and no type stubs, and comments still survive.

Also un-breaks CI: colab_serve.ipynb landed in efcf46e with an unused import
and an unguarded open(), so `ruff check`/`ruff format --check` have been red
at HEAD since. Fixed in place, plus a note that the vLLM path in that cell is
dead and should not be retried.

Gate §7.3 green: ruff, format, mypy strict, pytest 550 passed / 14 skipped
cov 91.05%, hygiene, no_raw_compare.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: JuliaEdom

configs/autotune/cycle.toml

@@ -0,0 +1,55 @@
+# Autotune cycle config — plan_autotune S7.
+#
+#   .venv/Scripts/python.exe scripts/autotune/run_cycle.py --dry-run
+#   .venv/Scripts/python.exe scripts/autotune/run_cycle.py --only eval,report
+#
+# TOML rather than YAML: tomllib is stdlib (no new dependency, no type stubs)
+# and still allows the comments that make this file readable.
+
+name = "v10"
+
+[dataset]
+# Rebuilds data/autotune/{train,val}.jsonl from data/bird_train.parquet.
+# Off by default: the v10 adapter was trained on the dataset already on disk,
+# and rebuilding it silently invalidates that provenance.
+enabled = false
+
+[train]
+# local  — run train_qlora.py on this machine (needs CUDA; bnb-4bit is CUDA-only)
+# remote — the real path of this track: Kaggle/Colab notebook, driven by hand
+# skip   — reuse an adapter that already exists
+mode = "remote"
+adapter = ".tmp/v10_adapter"
+runbook = "plan_autotune.md → 'RUNBOOK добивки 47 вопросов'"
+# Used only when mode = "local".
+args = ["--epochs", "1", "--val-rows", "64"]
+
+[eval]
+config = "E"
+n = 200
+seed = 0
+provider = "local_vllm"
+# The tuning replaces few-shot; keep it at 0 so train and inference agree.
+fewshot_top_k = 0
+explain_provider = "mistral"
+reports_root = "eval/reports"
+
+[[eval.targets]]
+label = "tuned"
+model = "sqltuned"
+suffix = "student-tuned"
+
+[[eval.targets]]
+label = "base"
+# The training base, 4-bit — NOT Qwen/Qwen2.5-Coder-7B-Instruct.
+model = "unsloth/qwen2.5-coder-7b-instruct-bnb-4bit"
+suffix = "student-base"
+
+[report]
+out = ".tmp/autotune_cycle_report.md"
+
+# Reference points the verdict is read against (EA %, n=200, config E, seed 0).
+[report.anchors]
+"codestral (product)" = 61.5
+"mimo-v2.5-free" = 68.5
+"Opus 4.8 max" = 79.5

scripts/autotune/colab/colab_serve.ipynb

@@ -1,22 +1,8 @@
 {
- "nbformat": 4,
- "nbformat_minor": 5,
- "metadata": {
-  "colab": {
-   "provenance": []
-  },
-  "kernelspec": {
-   "display_name": "Python 3",
-   "name": "python3"
-  },
-  "language_info": {
-   "name": "python"
-  },
-  "accelerator": "GPU"
- },
  "cells": [
   {
    "cell_type": "markdown",
+   "id": "7fb27b941602401d91542211134fc71a",
    "metadata": {},
    "source": [
     "# NL_SQL autotune — Colab SERVE (tuned student behind a public URL)\n",
@@ -38,6 +24,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "acae54e37e7d407bbb7b55eff062a284",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -47,12 +34,13 @@
     "DRIVE_DIR = \"/content/drive/MyDrive/nlsql_autotune\"  # kaggle.json persists here\n",
     "ADAPTER_LOCAL = \"/content/nlsql_adapter\"\n",
     "SERVE_HOURS = 8.0\n",
-    "PORT = 8000\n"
+    "PORT = 8000"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "9a63283cbaf04dbcab1f6479b197f3a8",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -78,12 +66,13 @@
     "    up = files.upload()\n",
     "    cred.write_bytes(next(iter(up.values())))\n",
     "    drive_cred.write_bytes(cred.read_bytes())\n",
-    "cred.chmod(0o600)\n"
+    "cred.chmod(0o600)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "8dd0d8092fe74a7c96281538738b07e2",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -106,25 +95,36 @@
     "adapter_dir = str(hits[0].parent)\n",
     "if not Path(adapter_dir, \"adapter_model.safetensors\").exists():\n",
     "    raise RuntimeError(f\"adapter_model.safetensors missing next to {hits[0]}\")\n",
-    "print(f\"adapter_dir resolved: {adapter_dir}\", flush=True)\n"
+    "print(f\"adapter_dir resolved: {adapter_dir}\", flush=True)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "72eea5119410473aa328ad9291626812",
    "metadata": {},
    "outputs": [],
    "source": [
     "import json\n",
-    "import os\n",
     "import re\n",
     "import subprocess\n",
     "import sys\n",
     "import time\n",
     "import urllib.request\n",
     "from pathlib import Path\n",
     "\n",
-    "subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"vllm>=0.6.3\"], check=True)\n",
+    "# DEAD PATH (2026-07-21): vLLM does not run on the Colab image at all — cu13\n",
+    "# wheel linkage, an aimv2 collision on 0.9.2, and triton PassManager failures\n",
+    "# on SM 7.5 LoRA kernels. Serving now goes through the transformers+peft+bnb\n",
+    "# shim instead (plan_autotune.md). Kept for the record; do not retry.\n",
+    "# Colab ships torch 2.11+cu128; the latest vLLM PyPI wheel is built for CUDA 13\n",
+    "# and dies on `libcudart.so.13`. Pin vLLM to the cu128 build via uv's torch-backend\n",
+    "# selector so it matches the preinstalled torch instead of the driver's CUDA 13.\n",
+    "subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"uv\"], check=True)\n",
+    "subprocess.run(\n",
+    "    [\"uv\", \"pip\", \"install\", \"-q\", \"--system\", \"--torch-backend=cu128\", \"vllm\"],\n",
+    "    check=True,\n",
+    ")\n",
     "cf = \"/content/cloudflared\"\n",
     "if not Path(cf).exists():\n",
     "    urllib.request.urlretrieve(\n",
@@ -154,7 +154,7 @@
     "    f\"sqltuned={adapter_dir}\",\n",
     "]\n",
     "print(\" \".join(vllm_cmd), flush=True)\n",
-    "vllm_log = open(\"/content/vllm.log\", \"w\")\n",
+    "vllm_log = open(\"/content/vllm.log\", \"w\")  # noqa: SIM115 - handle is the child's stdout, must outlive this cell\n",
     "vllm_proc = subprocess.Popen(vllm_cmd, stdout=vllm_log, stderr=subprocess.STDOUT)\n",
     "\n",
     "print(\"waiting for vllm (первый старт качает ~5.5GB весов)...\", flush=True)\n",
@@ -226,8 +226,23 @@
     "while time.time() < deadline and vllm_proc.poll() is None:\n",
     "    time.sleep(300)\n",
     "    print(f\"alive, url={url}, {int((deadline - time.time()) / 60)} min left\", flush=True)\n",
-    "print(\"serve window over\")\n"
+    "print(\"serve window over\")"
    ]
   }
- ]
-}
+ ],
+ "metadata": {
+  "accelerator": "GPU",
+  "colab": {
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}