Repair v33 Debian service launch, NLTK resources and uv sandbox interpreter mounts

Author: Mantiee

.github/workflows/v100-cpu-validation.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Install isolated test dependencies and CPU Torch
         run: |
           python -m pip install --index-url https://download.pytorch.org/whl/cpu torch==2.6.0
-          python -m pip install . pytest==8.4.2 numpy==2.2.6 psutil==7.0.0 safetensors==0.8.0 tomli-w==1.2.0 transformers==5.17.0 peft==0.21.2 accelerate==1.15.0
+          python -m pip install . pytest==8.4.2 numpy==2.2.6 psutil==7.0.0 safetensors==0.8.0 tomli-w==1.2.0 transformers==5.17.0 peft==0.21.2 accelerate==1.15.0 nltk==3.10.3
       - name: Exercise controller, DPO gradients, tensor breeding and tiny model training
         env:
           PYTEST_DISABLE_PLUGIN_AUTOLOAD: '1'

CAMPAIGN.md

@@ -1,4 +1,6 @@
-# V100 campaign v32
+# V100 campaign v33
+
+v33 repairs the deployment failures observed on Debian: systemd receives an unquoted absolute `WorkingDirectory` and validates the generated unit before starting it; the first supervisor start keeps the newly prepared configuration instead of reverting to an old failed run. NLTK downloads into an explicitly created, owner-private resource directory without disabling its security enforcement or changing the user's home permissions. Canonical helper migration checks the newly requested 32768 context and retains the old metadata snapshot; actual template, value or manifest changes still fail. Sandbox mounts include intermediate `uv` interpreter symlinks without exposing the whole project/home, and failed hardware probes preserve the full child diagnostic in `sandbox-probe.log`.
 
 Use `tools/upgrade-v100-campaign.sh RELEASE_COMMIT` on Debian after any existing training/calibration/MTP sweep completes. It updates only the existing isolated continual venv, repairs the launcher atomically, retains checkpoints, prepares optional components, runs mandatory short CUDA/master acceptance probes, and starts the owned supervisor only after those probes pass. Failed acceptance retains a report and leaves the mission stopped. Preparation prints each stage and records failures as `deferred`; a deferred feature is not operational.
 

pyproject.toml

@@ -3,7 +3,7 @@ name = "rlms"
 authors = [
     {name = "Alex Zhang", email = "altzhang@mit.edu"},
 ]
-version = "0.1.3+v100.32"
+version = "0.1.3+v100.33"
 description = "Recursive Language Models."
 readme = "README.md"
 license = "MIT"

rlm/v100/benchmark_worker.py

@@ -2,6 +2,8 @@
 
 import hashlib
 import json
+import os
+import stat
 import sys
 from pathlib import Path
 
@@ -10,6 +12,21 @@ def key(row):
     return "/".join(str(row[k]) for k in ("category", "task", "question_id"))
 
 
+def prepare_nltk(folder):
+    import nltk
+
+    # Authorize an existing private data root, rather than asking NLTK to trust
+    # the snapshot's possibly group-writable parent before creating it.
+    resources = folder / "nltk"
+    resources.mkdir(mode=0o700, exist_ok=True)
+    info = resources.lstat()
+    if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o022:
+        raise ValueError("Official grader resources must use an owned private directory")
+    for package in ("punkt", "punkt_tab", "averaged_perceptron_tagger_eng"):
+        if not nltk.download(package, download_dir=str(resources), quiet=True):
+            raise RuntimeError("Official grader resource download failed: " + package)
+
+
 def prepare(folder, limit, coding=False):
     from datasets import load_dataset
     from huggingface_hub import HfApi
@@ -91,11 +108,7 @@ def prepare(folder, limit, coding=False):
             indent=2,
         )
     )
-    import nltk
-
-    for package in ("punkt", "punkt_tab", "averaged_perceptron_tagger_eng"):
-        if not nltk.download(package, download_dir=str(folder / "nltk"), quiet=True):
-            raise RuntimeError("Official grader resource download failed: " + package)
+    prepare_nltk(folder)
     # Import during preparation so missing official dependencies fail before GPU work.
     from livebench.gen_ground_truth_judgment import play_a_match_gt  # noqa: F401
 

rlm/v100/campaign.py

@@ -104,7 +104,7 @@ def stage(name, function):
         print(json.dumps({"campaign": name, **stages[name]}), flush=True)
 
     def helper():
-        evidence = canonicalize_remote(root)
+        evidence = canonicalize_remote(root, context_window=32768)
         from rlm.v100.competition import helper_client
 
         helper_path = root / "research/researcher-rtx3090.json"

rlm/v100/code_lab.py

@@ -15,6 +15,7 @@
 # change its algorithms; the host prompt/evaluator/controller stay outside it.
 PROMPT_FILES = {
     "rlm/v100/hardware_acceptance.py",
+    "rlm/v100/sandbox_python.py",
     "rlm/v100/scratch_master.py",
     "rlm/v100/market_adapters.py",
     "rlm/v100/architecture_promotion.py",
@@ -226,10 +227,10 @@ def sandbox_command(
             "bubblewrap is required; untrusted code never falls back to host execution"
         )
     args = [bwrap, "--unshare-all", "--die-with-parent", "--new-session", "--cap-drop", "ALL"]
-    prefixes = [Path("/usr"), Path("/lib"), Path("/lib64"), Path(sys.prefix), Path(sys.base_prefix)]
-    for path in dict.fromkeys(prefixes):
-        if path.exists():
-            args += ["--ro-bind", str(path), str(path)]
+    from rlm.v100.sandbox_python import runtime_mounts
+
+    for path in runtime_mounts():
+        args += ["--ro-bind", str(path), str(path)]
     args += [
         "--proc",
         "/proc",

rlm/v100/hardware_acceptance.py

@@ -9,6 +9,29 @@
 from rlm.v100.common import atomic_json, load_profile
 
 
+def sandbox_probe(folder: Path) -> dict:
+    from rlm.v100.code_lab import sandbox_command, sandbox_environment
+
+    (folder / "source").mkdir()
+    (folder / "checks").mkdir()
+    log = folder / "sandbox-probe.log"
+    with log.open("w") as stream:
+        child = subprocess.run(
+            sandbox_command(folder, [sys.executable, "-I", "-c", "print('sandbox-ok')"]),
+            env=sandbox_environment(),
+            timeout=30,
+            stdout=stream,
+            stderr=subprocess.STDOUT,
+            text=True,
+        )
+    output = log.read_text()
+    if child.returncode or output.strip() != "sandbox-ok":
+        raise RuntimeError(
+            f"Sandbox probe exit={child.returncode}; log={log}; detail={output[-1000:]}"
+        )
+    return {"namespace_execution": True, "network_unshared": True, "log": str(log)}
+
+
 def run(root: Path, profile_path: Path, desktop: bool = False) -> Path:
     from rlm.v100.competition import helper_client, managed_server, require_idle_gpu
     from rlm.v100.mission import status
@@ -35,7 +58,7 @@ def check(name, action, required=False):
             result["checks"][name] = {
                 "state": "failed",
                 "error": type(error).__name__,
-                "detail": str(error)[:500],
+                "detail": str(error)[:1500],
             }
             if required:
                 result["required_passed"] = False
@@ -94,24 +117,7 @@ def master():
 
     check("master-inference", master, required=True)
 
-    def sandbox():
-        from rlm.v100.code_lab import sandbox_command, sandbox_environment
-
-        (folder / "source").mkdir()
-        (folder / "checks").mkdir()
-        child = subprocess.run(
-            sandbox_command(folder, [sys.executable, "-I", "-c", "print('sandbox-ok')"]),
-            env=sandbox_environment(),
-            timeout=30,
-            capture_output=True,
-            text=True,
-            check=True,
-        )
-        if child.stdout.strip() != "sandbox-ok":
-            raise ValueError("Sandbox probe returned another result")
-        return {"namespace_execution": True, "network_unshared": True}
-
-    check("private-code-sandbox", sandbox)
+    check("private-code-sandbox", lambda: sandbox_probe(folder))
 
     def rtx():
         path = root / "research/researcher-rtx3090.json"

rlm/v100/public_benchmarks.py

@@ -109,11 +109,10 @@ def grade(root: Path, snapshot: Path, question: dict, answer: dict, folder: Path
         "--cap-drop",
         "ALL",
     ]
-    for path in dict.fromkeys(
-        [Path("/usr"), Path("/lib"), Path("/lib64"), Path(sys.base_prefix), python.parent.parent]
-    ):
-        if path.exists():
-            args += ["--ro-bind", str(path), str(path)]
+    from rlm.v100.sandbox_python import runtime_mounts
+
+    for path in runtime_mounts(python):
+        args += ["--ro-bind", str(path), str(path)]
     args += [
         "--proc",
         "/proc",

rlm/v100/remote_helper.py

@@ -538,7 +538,7 @@ def selected_helper(root: Path) -> Path:
     return path
 
 
-def canonicalize_remote(root: Path) -> dict:
+def canonicalize_remote(root: Path, context_window: int | None = None) -> dict:
     """Rebind only proven parameter-order/date changes; never silently trust new templates."""
     from rlm.v100.common import load_profile
     from rlm.v100.competition import helper_client
@@ -549,9 +549,15 @@ def canonicalize_remote(root: Path) -> dict:
     path = root / "research/researcher-rtx3090.json"
     original = json.loads(path.read_text())
     profile = load_profile(path, root)
+    if context_window is not None:
+        if type(context_window) is not int or not 4096 <= context_window <= 262144:
+            raise ValueError("Invalid migration context budget")
+        profile["runtime"]["context_window"] = context_window
+        profile["server"]["context_per_slot"] = context_window
     client = helper_client(profile, root)
     if client.metadata_hash_scheme == ORDERED_METADATA:
         client.identity()
+        client.loaded()
         return {"status": "canonical identity verified"}
     snapshot = Path(profile["resources"]["metadata_snapshot"])
     if not snapshot.is_absolute():
@@ -572,6 +578,9 @@ def canonicalize_remote(root: Path) -> dict:
         metadata_sha256=metadata_sha(info, ORDERED_METADATA),
         metadata_snapshot=str((audit / "show.json").relative_to(root)),
     )
+    if context_window is not None:
+        original["runtime"]["context_window"] = context_window
+        original["server"]["context_per_slot"] = context_window
     validate_remote(original)
     atomic_json(path, original)
     return {

rlm/v100/research_sandbox.py

@@ -51,11 +51,10 @@ def command(root: Path, source: Path, work: Path, broker: Path) -> list[str]:
         "ALL",
         "--clearenv",
     ]
-    for path in dict.fromkeys(
-        [Path("/usr"), Path("/lib"), Path("/lib64"), Path(sys.prefix), Path(sys.base_prefix)]
-    ):
-        if path.exists():
-            args += ["--ro-bind", str(path), str(path)]
+    from rlm.v100.sandbox_python import runtime_mounts
+
+    for path in runtime_mounts():
+        args += ["--ro-bind", str(path), str(path)]
     return args + [
         "--proc",
         "/proc",