feat(autotune): Colab serve notebook — bnb-4bit base + LoRA on a single T4

fp16 7B does not fit Colab's single T4, but the adapter was trained on the
bnb-4bit base, so 4-bit serving is the training-time base, not a confound —
and base-vs-tuned on the same quantization is the clean ablation. vLLM
bitsandbytes + LoRA (sqltuned), smoke of both served names before exposing
the cloudflared URL, relay via the private nlsql-tunnel dataset. Unblocks
the student eval without waiting for the Kaggle weekly quota reset.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: JuliaEdom

scripts/autotune/colab/colab_serve.ipynb

@@ -0,0 +1,233 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 5,
+ "metadata": {
+  "colab": {
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python"
+  },
+  "accelerator": "GPU"
+ },
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# NL_SQL autotune — Colab SERVE (tuned student behind a public URL)\n",
+    "\n",
+    "Runs vLLM with the QLoRA adapter on a single T4 and exposes it via a cloudflared\n",
+    "quick tunnel, so the Windows-side harness can eval `sqltuned` and the base model.\n",
+    "\n",
+    "Why bnb-4bit: fp16 7B (~15GB) does not fit a 16GB T4; the adapter was trained on\n",
+    "`unsloth/qwen2.5-coder-7b-instruct-bnb-4bit`, so 4-bit serving matches the\n",
+    "training-time base exactly.\n",
+    "\n",
+    "Steps: Runtime -> Change runtime type -> **T4 GPU** -> Run all. Two prompts:\n",
+    "Drive authorisation, and (first time only) `kaggle.json` upload\n",
+    "(Kaggle -> Settings -> Create New API Token). Then leave the tab open.\n",
+    "The tunnel URL is printed below the last cell AND pushed to the private\n",
+    "Kaggle dataset `liovinajo/nlsql-tunnel` for the Windows side to poll.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "BASE_MODEL = \"unsloth/qwen2.5-coder-7b-instruct-bnb-4bit\"  # training-time base, pre-quantized\n",
+    "ADAPTER_DATASET = \"liovinajo/nlsql-adapter\"\n",
+    "\n",
+    "DRIVE_DIR = \"/content/drive/MyDrive/nlsql_autotune\"  # kaggle.json persists here\n",
+    "ADAPTER_LOCAL = \"/content/nlsql_adapter\"\n",
+    "SERVE_HOURS = 8.0\n",
+    "PORT = 8000\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "from pathlib import Path\n",
+    "\n",
+    "from google.colab import drive, files\n",
+    "\n",
+    "print(subprocess.run([\"nvidia-smi\"], capture_output=True, text=True).stdout, flush=True)\n",
+    "\n",
+    "drive.mount(\"/content/drive\")\n",
+    "Path(DRIVE_DIR).mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# kaggle.json: reuse the copy on Drive if a previous session left one, else ask once.\n",
+    "cred = Path(\"/root/.kaggle/kaggle.json\")\n",
+    "cred.parent.mkdir(parents=True, exist_ok=True)\n",
+    "drive_cred = Path(DRIVE_DIR) / \"kaggle.json\"\n",
+    "if drive_cred.exists():\n",
+    "    cred.write_bytes(drive_cred.read_bytes())\n",
+    "    print(\"kaggle.json taken from Drive\", flush=True)\n",
+    "else:\n",
+    "    print(\"upload kaggle.json (Kaggle -> Settings -> Create New API Token)\", flush=True)\n",
+    "    up = files.upload()\n",
+    "    cred.write_bytes(next(iter(up.values())))\n",
+    "    drive_cred.write_bytes(cred.read_bytes())\n",
+    "cred.chmod(0o600)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import shutil\n",
+    "import subprocess\n",
+    "from pathlib import Path\n",
+    "\n",
+    "dst = Path(ADAPTER_LOCAL)\n",
+    "if dst.exists():\n",
+    "    shutil.rmtree(dst)\n",
+    "dst.mkdir(parents=True)\n",
+    "subprocess.run(\n",
+    "    [\"kaggle\", \"datasets\", \"download\", ADAPTER_DATASET, \"-p\", str(dst), \"--unzip\"],\n",
+    "    check=True,\n",
+    ")\n",
+    "# Kaggle unpacks uploaded zips flat (v10 lesson) -- self-home on the config.\n",
+    "hits = sorted(dst.rglob(\"adapter_config.json\"))\n",
+    "if not hits:\n",
+    "    raise RuntimeError(f\"no adapter_config.json under {dst} -- wrong dataset?\")\n",
+    "adapter_dir = str(hits[0].parent)\n",
+    "if not Path(adapter_dir, \"adapter_model.safetensors\").exists():\n",
+    "    raise RuntimeError(f\"adapter_model.safetensors missing next to {hits[0]}\")\n",
+    "print(f\"adapter_dir resolved: {adapter_dir}\", flush=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import os\n",
+    "import re\n",
+    "import subprocess\n",
+    "import sys\n",
+    "import time\n",
+    "import urllib.request\n",
+    "from pathlib import Path\n",
+    "\n",
+    "subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"vllm>=0.6.3\"], check=True)\n",
+    "cf = \"/content/cloudflared\"\n",
+    "if not Path(cf).exists():\n",
+    "    urllib.request.urlretrieve(\n",
+    "        \"https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64\",\n",
+    "        cf,\n",
+    "    )\n",
+    "    subprocess.run([\"chmod\", \"+x\", cf], check=True)\n",
+    "\n",
+    "vllm_cmd = [\n",
+    "    \"vllm\",\n",
+    "    \"serve\",\n",
+    "    BASE_MODEL,\n",
+    "    \"--port\",\n",
+    "    str(PORT),\n",
+    "    \"--quantization\",\n",
+    "    \"bitsandbytes\",\n",
+    "    \"--load-format\",\n",
+    "    \"bitsandbytes\",\n",
+    "    \"--gpu-memory-utilization\",\n",
+    "    \"0.92\",\n",
+    "    \"--max-model-len\",\n",
+    "    \"12288\",\n",
+    "    \"--enable-lora\",\n",
+    "    \"--max-lora-rank\",\n",
+    "    \"16\",\n",
+    "    \"--lora-modules\",\n",
+    "    f\"sqltuned={adapter_dir}\",\n",
+    "]\n",
+    "print(\" \".join(vllm_cmd), flush=True)\n",
+    "vllm_log = open(\"/content/vllm.log\", \"w\")\n",
+    "vllm_proc = subprocess.Popen(vllm_cmd, stdout=vllm_log, stderr=subprocess.STDOUT)\n",
+    "\n",
+    "print(\"waiting for vllm (первый старт качает ~5.5GB весов)...\", flush=True)\n",
+    "for _ in range(240):  # up to 40 min\n",
+    "    time.sleep(10)\n",
+    "    try:\n",
+    "        urllib.request.urlopen(f\"http://127.0.0.1:{PORT}/v1/models\", timeout=5)\n",
+    "        break\n",
+    "    except Exception:\n",
+    "        if vllm_proc.poll() is not None:\n",
+    "            print(Path(\"/content/vllm.log\").read_text()[-4000:])\n",
+    "            raise RuntimeError(\"vllm died during startup\") from None\n",
+    "else:\n",
+    "    raise RuntimeError(\"vllm never became ready\")\n",
+    "\n",
+    "# smoke BOTH served names before exposing the URL\n",
+    "for name in (\"sqltuned\", BASE_MODEL):\n",
+    "    body = json.dumps(\n",
+    "        {\n",
+    "            \"model\": name,\n",
+    "            \"messages\": [{\"role\": \"user\", \"content\": \"Reply with the word ok.\"}],\n",
+    "            \"max_tokens\": 5,\n",
+    "        }\n",
+    "    ).encode()\n",
+    "    req = urllib.request.Request(\n",
+    "        f\"http://127.0.0.1:{PORT}/v1/chat/completions\",\n",
+    "        data=body,\n",
+    "        headers={\"Content-Type\": \"application/json\"},\n",
+    "    )\n",
+    "    with urllib.request.urlopen(req, timeout=120) as r:\n",
+    "        out = json.loads(r.read())[\"choices\"][0][\"message\"][\"content\"]\n",
+    "    print(f\"smoke {name}: {out!r}\", flush=True)\n",
+    "\n",
+    "tun = subprocess.Popen(\n",
+    "    [cf, \"tunnel\", \"--url\", f\"http://127.0.0.1:{PORT}\"],\n",
+    "    stdout=subprocess.PIPE,\n",
+    "    stderr=subprocess.STDOUT,\n",
+    "    text=True,\n",
+    ")\n",
+    "url = None\n",
+    "assert tun.stdout is not None\n",
+    "for line in tun.stdout:\n",
+    "    m = re.search(r\"https://[a-z0-9-]+\\.trycloudflare\\.com\", line)\n",
+    "    if m:\n",
+    "        url = m.group(0)\n",
+    "        break\n",
+    "print(f\"\\n{'=' * 60}\\nTUNNEL: {url}\\nbase_url for .env: {url}/v1\\n{'=' * 60}\\n\", flush=True)\n",
+    "\n",
+    "# push the URL out through the private nlsql-tunnel dataset (Windows polls it)\n",
+    "relay = Path(\"/content/relay\")\n",
+    "relay.mkdir(exist_ok=True)\n",
+    "(relay / \"tunnel_url.txt\").write_text(url or \"NONE\")\n",
+    "(relay / \"dataset-metadata.json\").write_text(\n",
+    "    json.dumps(\n",
+    "        {\"title\": \"nlsql-tunnel\", \"id\": \"liovinajo/nlsql-tunnel\", \"licenses\": [{\"name\": \"CC0-1.0\"}]}\n",
+    "    )\n",
+    ")\n",
+    "for verb in ([\"version\", \"-m\", \"tunnel-url\"], [\"create\"]):\n",
+    "    rr = subprocess.run(\n",
+    "        [\"kaggle\", \"datasets\", *verb, \"-p\", str(relay)], capture_output=True, text=True\n",
+    "    )\n",
+    "    print(f\"relay {verb[0]}: rc={rr.returncode}\", flush=True)\n",
+    "    if rr.returncode == 0:\n",
+    "        break\n",
+    "else:\n",
+    "    print(\"relay FAILED; copy the URL above manually\", flush=True)\n",
+    "\n",
+    "deadline = time.time() + SERVE_HOURS * 3600\n",
+    "while time.time() < deadline and vllm_proc.poll() is None:\n",
+    "    time.sleep(300)\n",
+    "    print(f\"alive, url={url}, {int((deadline - time.time()) / 60)} min left\", flush=True)\n",
+    "print(\"serve window over\")\n"
+   ]
+  }
+ ]
+}