fix(autotune): resume interrupted Kaggle run from rescued checkpoint

v8 hit the 12h session cut at step 336/500 (steady ~127 s/step); a cancelled
kernel keeps no attachable output, but ListKernelSessionOutput still serves
the session files — checkpoint-300 rescued that way and re-uploaded as the
private dataset nlsql-autotune-prev. Kaggle flattens the uploaded zip, so the
notebook now rebuilds checkpoint-<global_step>/ around any trainer_state.json
(+optimizer.pt guard) found under /kaggle/input, failing loudly when resume
is requested but no checkpoint is present. Smoke off for v9: the full path
was already proven on v8.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: JuliaEdom

scripts/autotune/kaggle/kaggle_autotune.ipynb

@@ -20,111 +20,15 @@
    "id": "acae54e37e7d407bbb7b55eff062a284",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "MODE = \"train\"  # \"train\" | \"serve\"\n",
-    "\n",
-    "BASE_MODEL = \"Qwen/Qwen2.5-Coder-7B-Instruct\"\n",
-    "DATA_DIR = \"/kaggle/input/nlsql-autotune-data\"  # train.jsonl, val.jsonl, train_qlora.py\n",
-    "OUT = \"/kaggle/working/qlora_out\"\n",
-    "\n",
-    "# train mode\n",
-    "EPOCHS = 1.0\n",
-    "MAX_SEQ = 4096\n",
-    "SMOKE_STEPS = 5  # >0: run this many steps end-to-end first, then the real run (0 = straight in)\n",
-    "RESUME_INPUT = \"\"  # e.g. \"/kaggle/input/nlsql-autotune-prev\" (previous run's output) or \"\"\n",
-    "\n",
-    "# serve mode\n",
-    "ADAPTER_DIR = \"/kaggle/input/nlsql-adapter/adapter\"  # absent path => serve plain base model\n",
-    "SERVE_HOURS = 5.0\n",
-    "PORT = 8000"
-   ]
+   "source": "MODE = \"train\"  # \"train\" | \"serve\"\n\nBASE_MODEL = \"Qwen/Qwen2.5-Coder-7B-Instruct\"\nDATA_DIR = \"/kaggle/input/nlsql-autotune-data\"  # train.jsonl, val.jsonl, train_qlora.py\nOUT = \"/kaggle/working/qlora_out\"\n\n# train mode\nEPOCHS = 1.0\nMAX_SEQ = 4096\nSMOKE_STEPS = 0  # >0: run this many steps end-to-end first, then the real run (0 = straight in)\nRESUME_INPUT = \"auto\"  # non-empty: rglob /kaggle/input for checkpoint-* and resume; \"\" = fresh\n\n# serve mode\nADAPTER_DIR = \"/kaggle/input/nlsql-adapter/adapter\"  # absent path => serve plain base model\nSERVE_HOURS = 5.0\nPORT = 8000"
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "9a63283cbaf04dbcab1f6479b197f3a8",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "import shutil\n",
-    "import subprocess\n",
-    "import sys\n",
-    "from pathlib import Path\n",
-    "\n",
-    "if MODE == \"train\":\n",
-    "    tree = subprocess.run(\n",
-    "        [\"find\", \"/kaggle/input\", \"-maxdepth\", \"4\"], capture_output=True, text=True\n",
-    "    ).stdout\n",
-    "    print(tree, flush=True)\n",
-    "    hits = sorted(Path(\"/kaggle/input\").rglob(\"train_qlora.py\"))\n",
-    "    if not hits:\n",
-    "        raise RuntimeError(\"train_qlora.py not found anywhere under /kaggle/input\")\n",
-    "    data_dir = hits[0].parent\n",
-    "    print(f\"data_dir resolved: {data_dir}\", flush=True)\n",
-    "    subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"unsloth\"], check=True)\n",
-    "    subprocess.run(\n",
-    "        [\n",
-    "            sys.executable,\n",
-    "            \"-m\",\n",
-    "            \"pip\",\n",
-    "            \"install\",\n",
-    "            \"-q\",\n",
-    "            \"datasets>=3.0\",\n",
-    "            \"trl>=0.12\",\n",
-    "            \"peft>=0.13\",\n",
-    "            \"bitsandbytes>=0.44\",\n",
-    "            \"accelerate>=1.0\",\n",
-    "        ],\n",
-    "        check=True,\n",
-    "    )\n",
-    "\n",
-    "    def train_cmd(out, extra):\n",
-    "        return [\n",
-    "            sys.executable,\n",
-    "            str(data_dir / \"train_qlora.py\"),\n",
-    "            \"--model\",\n",
-    "            BASE_MODEL,\n",
-    "            \"--train\",\n",
-    "            str(data_dir / \"train.jsonl\"),\n",
-    "            \"--val\",\n",
-    "            str(data_dir / \"val.jsonl\"),\n",
-    "            \"--out\",\n",
-    "            out,\n",
-    "            \"--max-seq-len\",\n",
-    "            str(MAX_SEQ),\n",
-    "            \"--batch-size\",\n",
-    "            \"1\",\n",
-    "            \"--grad-accum\",\n",
-    "            \"16\",\n",
-    "            \"--no-merge\",\n",
-    "            *extra,\n",
-    "        ]\n",
-    "\n",
-    "    if SMOKE_STEPS:\n",
-    "        # Both earlier sessions died in their first minutes on a trl kwarg rename,\n",
-    "        # and each failure cost a day because nobody was watching. Prove the whole\n",
-    "        # path -- config, trainer, a real step, evaluate, adapter save -- on a few\n",
-    "        # steps before committing the 6-12h run.\n",
-    "        smoke_out = \"/kaggle/working/smoke_out\"\n",
-    "        smoke = train_cmd(smoke_out, [\"--epochs\", \"1\", \"--max-steps\", str(SMOKE_STEPS)])\n",
-    "        print(\" \".join(smoke), flush=True)\n",
-    "        subprocess.run(smoke, check=True)\n",
-    "        if not Path(smoke_out, \"adapter\", \"adapter_config.json\").exists():\n",
-    "            raise RuntimeError(\"smoke run finished but saved no adapter\")\n",
-    "        shutil.rmtree(smoke_out)\n",
-    "        print(\"SMOKE OK -- starting the real run\", flush=True)\n",
-    "\n",
-    "    ckpt_dst = Path(OUT) / \"checkpoints\"\n",
-    "    prev = Path(RESUME_INPUT) / \"qlora_out\" / \"checkpoints\" if RESUME_INPUT else None\n",
-    "    if prev and prev.is_dir() and not ckpt_dst.exists():\n",
-    "        shutil.copytree(prev, ckpt_dst)\n",
-    "        print(f\"resume: copied checkpoints from {prev}\")\n",
-    "    cmd = train_cmd(OUT, [\"--epochs\", str(EPOCHS), \"--resume\"])\n",
-    "    print(\" \".join(cmd), flush=True)\n",
-    "    subprocess.run(cmd, check=True)\n",
-    "    shutil.make_archive(\"/kaggle/working/adapter\", \"zip\", OUT, \"adapter\")\n",
-    "    print(\"DONE: /kaggle/working/adapter.zip\")"
-   ]
+   "source": "import json\nimport shutil\nimport subprocess\nimport sys\nfrom pathlib import Path\n\nif MODE == \"train\":\n    tree = subprocess.run(\n        [\"find\", \"/kaggle/input\", \"-maxdepth\", \"4\"], capture_output=True, text=True\n    ).stdout\n    print(tree, flush=True)\n    hits = sorted(Path(\"/kaggle/input\").rglob(\"train_qlora.py\"))\n    if not hits:\n        raise RuntimeError(\"train_qlora.py not found anywhere under /kaggle/input\")\n    data_dir = hits[0].parent\n    print(f\"data_dir resolved: {data_dir}\", flush=True)\n    subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"unsloth\"], check=True)\n    subprocess.run(\n        [\n            sys.executable,\n            \"-m\",\n            \"pip\",\n            \"install\",\n            \"-q\",\n            \"datasets>=3.0\",\n            \"trl>=0.12\",\n            \"peft>=0.13\",\n            \"bitsandbytes>=0.44\",\n            \"accelerate>=1.0\",\n        ],\n        check=True,\n    )\n\n    def train_cmd(out, extra):\n        return [\n            sys.executable,\n            str(data_dir / \"train_qlora.py\"),\n            \"--model\",\n            BASE_MODEL,\n            \"--train\",\n            str(data_dir / \"train.jsonl\"),\n            \"--val\",\n            str(data_dir / \"val.jsonl\"),\n            \"--out\",\n            out,\n            \"--max-seq-len\",\n            str(MAX_SEQ),\n            \"--batch-size\",\n            \"1\",\n            \"--grad-accum\",\n            \"16\",\n            \"--no-merge\",\n            *extra,\n        ]\n\n    if SMOKE_STEPS:\n        # Both earlier sessions died in their first minutes on a trl kwarg rename,\n        # and each failure cost a day because nobody was watching. Prove the whole\n        # path -- config, trainer, a real step, evaluate, adapter save -- on a few\n        # steps before committing the 6-12h run.\n        smoke_out = \"/kaggle/working/smoke_out\"\n        smoke = train_cmd(smoke_out, [\"--epochs\", \"1\", \"--max-steps\", str(SMOKE_STEPS)])\n        print(\" \".join(smoke), flush=True)\n        subprocess.run(smoke, check=True)\n        if not Path(smoke_out, \"adapter\", \"adapter_config.json\").exists():\n            raise RuntimeError(\"smoke run finished but saved no adapter\")\n        shutil.rmtree(smoke_out)\n        print(\"SMOKE OK -- starting the real run\", flush=True)\n\n    ckpt_dst = Path(OUT) / \"checkpoints\"\n    if RESUME_INPUT and not ckpt_dst.exists():\n        # A cancelled session leaves no kernel output to attach, so checkpoints\n        # ride in via a dataset -- and Kaggle flattens the uploaded zip, dropping\n        # the checkpoint-N/ folder. Locate trainer_state.json anywhere under\n        # /kaggle/input and rebuild checkpoint-<global_step>/ around it (HF\n        # resume needs that exact dir-name pattern).\n        states = [\n            p\n            for p in Path(\"/kaggle/input\").rglob(\"trainer_state.json\")\n            if (p.parent / \"optimizer.pt\").exists()\n        ]\n        if not states:\n            raise RuntimeError(\n                f\"RESUME_INPUT={RESUME_INPUT!r} but no resumable checkpoint \"\n                \"(trainer_state.json + optimizer.pt) under /kaggle/input \"\n                \"-- refusing to silently train from scratch\"\n            )\n        ckpt_dst.mkdir(parents=True)\n        for state in states:\n            step = json.loads(state.read_text())[\"global_step\"]\n            dst = ckpt_dst / f\"checkpoint-{step}\"\n            shutil.copytree(state.parent, dst)\n            print(f\"resume: {state.parent} -> {dst}\", flush=True)\n    cmd = train_cmd(OUT, [\"--epochs\", str(EPOCHS), \"--resume\"])\n    print(\" \".join(cmd), flush=True)\n    subprocess.run(cmd, check=True)\n    shutil.make_archive(\"/kaggle/working/adapter\", \"zip\", OUT, \"adapter\")\n    print(\"DONE: /kaggle/working/adapter.zip\")"
   },
   {
    "cell_type": "code",
@@ -230,4 +134,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

scripts/autotune/kaggle/kernel-metadata.json

@@ -8,7 +8,7 @@
   "enable_gpu": true,
   "machine_shape": "NvidiaTeslaT4",
   "enable_internet": true,
-  "dataset_sources": ["liovinajo/nlsql-autotune-data"],
+  "dataset_sources": ["liovinajo/nlsql-autotune-data", "liovinajo/nlsql-autotune-prev"],
   "competition_sources": [],
   "kernel_sources": []
 }