Repair v34 managed Python stdlib aliases and isolate official grader setup

Author: Mantiee

CAMPAIGN.md

@@ -1,4 +1,6 @@
-# V100 campaign v33
+# V100 campaign v34
+
+v34 addresses the follow-up deployment report: the sandbox exposes a managed Python runtime's complete standard library at its executable alias, not only `bin/`; official grader setup explicitly targets its own virtual environment for spaCy language-model installation. NLTK resources include stopwords and are verified locally during offline grading without a downloader/index request. Hardware acceptance retries strict helper preparation after the Windows startup opportunity, applies batch 16 and 15% request pacing only to a verified model, and preserves exact metadata differences if a semantic change is rejected. These repairs do not establish RTX reachability or hardware acceptance until the on-device checks run.
 
 v33 repairs the deployment failures observed on Debian: systemd receives an unquoted absolute `WorkingDirectory` and validates the generated unit before starting it; the first supervisor start keeps the newly prepared configuration instead of reverting to an old failed run. NLTK downloads into an explicitly created, owner-private resource directory without disabling its security enforcement or changing the user's home permissions. Canonical helper migration checks the newly requested 32768 context and retains the old metadata snapshot; actual template, value or manifest changes still fail. Sandbox mounts include intermediate `uv` interpreter symlinks without exposing the whole project/home, and failed hardware probes preserve the full child diagnostic in `sandbox-probe.log`.
 

pyproject.toml

@@ -3,7 +3,7 @@ name = "rlms"
 authors = [
     {name = "Alex Zhang", email = "altzhang@mit.edu"},
 ]
-version = "0.1.3+v100.33"
+version = "0.1.3+v100.34"
 description = "Recursive Language Models."
 readme = "README.md"
 license = "MIT"

rlm/v100/benchmark_worker.py

@@ -7,6 +7,13 @@
 import sys
 from pathlib import Path
 
+NLTK_RESOURCES = {
+    "punkt": "tokenizers/punkt",
+    "punkt_tab": "tokenizers/punkt_tab",
+    "averaged_perceptron_tagger_eng": "taggers/averaged_perceptron_tagger_eng",
+    "stopwords": "corpora/stopwords",
+}
+
 
 def key(row):
     return "/".join(str(row[k]) for k in ("category", "task", "question_id"))
@@ -22,11 +29,25 @@ def prepare_nltk(folder):
     info = resources.lstat()
     if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o022:
         raise ValueError("Official grader resources must use an owned private directory")
-    for package in ("punkt", "punkt_tab", "averaged_perceptron_tagger_eng"):
+    for package in NLTK_RESOURCES:
         if not nltk.download(package, download_dir=str(resources), quiet=True):
             raise RuntimeError("Official grader resource download failed: " + package)
 
 
+def offline_nltk():
+    import nltk
+
+    def present(package, *args, **kwargs):
+        # IFBench calls nltk.download('stopwords') even while grading. Check
+        # the frozen local resource instead of fetching an index without network.
+        if package not in NLTK_RESOURCES:
+            raise ValueError("Unprepared official grader resource: " + str(package))
+        nltk.data.find(NLTK_RESOURCES[package])
+        return True
+
+    nltk.download = present
+
+
 def prepare(folder, limit, coding=False):
     from datasets import load_dataset
     from huggingface_hub import HfApi
@@ -114,6 +135,7 @@ def prepare(folder, limit, coding=False):
 
 
 def grade(request, output):
+    offline_nltk()
     from livebench.common import MatchSingle
     from livebench.gen_ground_truth_judgment import play_a_match_gt
 

rlm/v100/campaign.py

@@ -67,6 +67,36 @@ def choose_profile(root: Path) -> Path:
     return max(candidates, key=lambda path: path.stat().st_mtime_ns)
 
 
+def configure_helper(root: Path) -> dict:
+    from rlm.v100.competition import helper_client
+    from rlm.v100.remote_helper import canonicalize_remote
+
+    evidence = canonicalize_remote(root, context_window=32768)
+    helper_path = root / "research/researcher-rtx3090.json"
+    selected = load_profile(helper_path, root)
+    selected["runtime"]["context_window"] = 32768
+    selected["server"]["context_per_slot"] = 32768
+    selected["resources"].update(helper_batch_tokens=16, helper_duty_percent=15)
+    client = helper_client(selected, root)
+    client.identity()
+    measured = client.loaded()
+    atomic_json(
+        helper_path.with_name(
+            "researcher-rtx3090.before-campaign-" + str(time.time_ns()) + ".json"
+        ),
+        json.loads(helper_path.read_text()),
+    )
+    atomic_json(helper_path, selected)
+    return {
+        **evidence,
+        "context": 32768,
+        "batch": 16,
+        "active_request_target_percent": 15,
+        "loaded": measured,
+        "scope": "Helper request pacing; no hard GPU peak/power cap",
+    }
+
+
 def prepare(
     root: Path,
     path: Path | None = None,
@@ -77,7 +107,6 @@ def prepare(
     from rlm.v100.mission import status
     from rlm.v100.public_benchmarks import current
     from rlm.v100.public_benchmarks import prepare as prepare_benchmarks
-    from rlm.v100.remote_helper import canonicalize_remote
     from rlm.v100.self_code import prepare as prepare_code
 
     if status(root)["running"]:
@@ -103,34 +132,7 @@ def stage(name, function):
         atomic_json(folder / "stages.json", stages)
         print(json.dumps({"campaign": name, **stages[name]}), flush=True)
 
-    def helper():
-        evidence = canonicalize_remote(root, context_window=32768)
-        from rlm.v100.competition import helper_client
-
-        helper_path = root / "research/researcher-rtx3090.json"
-        selected = load_profile(helper_path, root)
-        selected["runtime"]["context_window"] = 32768
-        selected["server"]["context_per_slot"] = 32768
-        selected["resources"].update(helper_batch_tokens=16, helper_duty_percent=15)
-        client = helper_client(selected, root)
-        client.identity()
-        client.loaded()
-        atomic_json(
-            helper_path.with_name(
-                "researcher-rtx3090.before-campaign-" + str(time.time_ns()) + ".json"
-            ),
-            json.loads(helper_path.read_text()),
-        )
-        atomic_json(helper_path, selected)
-        return {
-            **evidence,
-            "context": 32768,
-            "batch": 16,
-            "active_request_target_percent": 15,
-            "scope": "Helper request pacing; no hard GPU peak/power cap",
-        }
-
-    stage("helper-identity", helper)
+    stage("helper-identity", lambda: configure_helper(root))
     stage("own-source-and-cpu-sandbox", lambda: prepare_code(root))
     if benchmarks:
         stage("official-public-benchmarks", lambda: current(root) or prepare_benchmarks(root, 20))

rlm/v100/hardware_acceptance.py

@@ -120,11 +120,11 @@ def master():
     check("private-code-sandbox", lambda: sandbox_probe(folder))
 
     def rtx():
-        path = root / "research/researcher-rtx3090.json"
-        selected = load_profile(path, root)
-        client = helper_client(selected, root)
-        client.identity()
-        return client.loaded()
+        from rlm.v100.campaign import configure_helper
+
+        # Windows may finish loading after the first preparation attempt.
+        # Retry the strict migration now, with the mission still stopped.
+        return configure_helper(root)
 
     check("rtx-helper-identity-and-loaded-model", rtx)
     if desktop:

rlm/v100/public_benchmarks.py

@@ -2,6 +2,7 @@
 
 import json
 import math
+import os
 import statistics
 import subprocess
 import sys
@@ -50,9 +51,12 @@ def prepare(root: Path, limit: int = 20, coding: bool = False) -> Path:
         check=True,
     )
     worker = Path(__file__).with_name("benchmark_worker.py")
+    resources = folder / "nltk"
+    resources.mkdir(mode=0o700)
     subprocess.run(
         [str(python), str(worker), "prepare", str(folder), str(limit)]
         + (["--coding"] if coding else []),
+        env=preparation_environment(environment, resources),
         check=True,
     )
     frozen = subprocess.check_output(
@@ -76,6 +80,18 @@ def prepare(root: Path, limit: int = 20, coding: bool = False) -> Path:
     return folder
 
 
+def preparation_environment(environment: Path, resources: Path | None = None) -> dict:
+    # LiveBench imports IFBench, whose official spaCy grader downloads its
+    # compatible language model if absent. uv must target this isolated venv.
+    selected = dict(os.environ)
+    selected["VIRTUAL_ENV"] = str(environment)
+    selected["UV_PYTHON"] = str(environment / "bin/python")
+    selected["PATH"] = str(environment / "bin") + os.pathsep + selected.get("PATH", "")
+    if resources is not None:
+        selected["NLTK_DATA"] = str(resources)
+    return selected
+
+
 def current(root: Path) -> Path | None:
     path = root / "research/public-benchmarks/current.json"
     return Path(json.loads(path.read_text())["snapshot"]) if path.exists() else None

rlm/v100/remote_helper.py

@@ -569,7 +569,20 @@ def canonicalize_remote(root: Path, context_window: int | None = None) -> dict:
     info = client.identity()
     client.loaded()
     if metadata_sha(info, ORDERED_METADATA) != metadata_sha(previous, ORDERED_METADATA):
-        raise ValueError("Metadata changed beyond parameter order/date; inspect the exact diff")
+        changed = [
+            key
+            for key in sorted(set(info) | set(previous))
+            if key != "modified_at"
+            and metadata_sha({key: info.get(key)}, ORDERED_METADATA)
+            != metadata_sha({key: previous.get(key)}, ORDERED_METADATA)
+        ]
+        audit = root / "research/helper-metadata" / f"rejected-{time.time_ns()}"
+        atomic_json(audit / "profile-before.json", original)
+        atomic_json(audit / "show-before.json", previous)
+        atomic_json(audit / "show-current.json", info)
+        raise ValueError(
+            f"Metadata changed beyond parameter order/date; fields={changed}; audit={audit}"
+        )
     audit = root / "research/helper-metadata" / f"canonical-{time.time_ns()}"
     atomic_json(audit / "profile-before.json", original)
     atomic_json(audit / "show.json", info)

rlm/v100/sandbox_python.py

@@ -24,9 +24,15 @@ def runtime_mounts(python: Path | None = None) -> list[Path]:
         seen.add(current)
         target = Path(os.readlink(current))
         current = Path(os.path.abspath(target if target.is_absolute() else current.parent / target))
-        # Bind only the interpreter's bin directories, never the whole home or
-        # project root just to resolve an intermediate train-venv Python link.
-        prefixes.append(current.parent)
+        # Intermediate venv links need only bin/. A distribution alias also
+        # needs lib/ to discover encodings instead of its compiled /install path.
+        # Neither case exposes the whole home or project root.
+        runtime_alias = current.parent.parent
+        prefixes.append(
+            runtime_alias
+            if runtime_alias.resolve() == Path(sys.base_prefix).resolve()
+            else current.parent
+        )
     if not current.is_file():
         raise ValueError("Sandbox interpreter symlink target is missing")
     return [path for path in dict.fromkeys(prefixes) if path.exists()]

tests/test_v100_deployment.py

@@ -12,6 +12,7 @@
     code_lab,
     hardware_acceptance,
     mission,
+    public_benchmarks,
     sandbox_python,
     supervisor,
 )
@@ -32,7 +33,7 @@ def download(package, download_dir, quiet):
 
     monkeypatch.setitem(sys.modules, "nltk", SimpleNamespace(download=download))
     benchmark_worker.prepare_nltk(tmp_path)
-    assert len(calls) == 3 and tmp_path.stat().st_mode & 0o777 == 0o775
+    assert len(calls) == 4 and tmp_path.stat().st_mode & 0o777 == 0o775
 
 
 @pytest.mark.parametrize("unsafe", ["writable", "link"])
@@ -159,6 +160,62 @@ def test_uv_python_chain_mounts_intermediate_bin_without_exposing_project_root(t
     assert tmp_path not in mounts and tmp_path / "venvs" not in mounts
 
 
+def test_managed_python_runtime_alias_mounts_its_stdlib_not_just_bin(tmp_path, monkeypatch):
+    distribution = tmp_path / "python/cpython-3.11.15-linux-x86_64-gnu"
+    alias = distribution.with_name("cpython-3.11-linux-x86_64-gnu")
+    python = distribution / "bin/python3.11"
+    python.parent.mkdir(parents=True)
+    python.write_text("managed interpreter fixture")
+    stdlib = distribution / "lib/python3.11/encodings"
+    stdlib.mkdir(parents=True)
+    (stdlib / "__init__.py").write_text("codec fixture")
+    alias.symlink_to(distribution, target_is_directory=True)
+    venv_python = tmp_path / "venvs/continual/bin/python"
+    venv_python.parent.mkdir(parents=True)
+    venv_python.symlink_to(alias / "bin/python3.11")
+    monkeypatch.setattr(sandbox_python.sys, "base_prefix", str(distribution))
+    mounts = sandbox_python.runtime_mounts(venv_python)
+    assert alias in mounts and (alias / "lib/python3.11/encodings").is_dir()
+    assert alias / "bin" not in mounts
+    assert tmp_path not in mounts and distribution.parent not in mounts
+
+
+def test_official_grader_installation_targets_its_own_environment(tmp_path, monkeypatch):
+    monkeypatch.setenv("VIRTUAL_ENV", "/wrong/train-venv")
+    monkeypatch.setenv("UV_PYTHON", "/wrong/system-python")
+    monkeypatch.setenv("PATH", "/usr/bin")
+    environment = tmp_path / "venvs/official-benchmarks"
+    selected = public_benchmarks.preparation_environment(environment)
+    assert selected["VIRTUAL_ENV"] == str(environment)
+    assert selected["UV_PYTHON"] == str(environment / "bin/python")
+    assert selected["PATH"] == str(environment / "bin") + ":/usr/bin"
+    assert os.environ["VIRTUAL_ENV"] == "/wrong/train-venv"
+    resources = tmp_path / "snapshot/nltk"
+    assert public_benchmarks.preparation_environment(environment, resources)["NLTK_DATA"] == str(
+        resources
+    )
+
+
+def test_offline_grader_requires_actual_frozen_nltk_resources_without_network(monkeypatch):
+    found = []
+
+    def find(path):
+        found.append(path)
+        if path == "tokenizers/punkt":
+            raise LookupError("Missing frozen punkt")
+        return path
+
+    nltk = SimpleNamespace(data=SimpleNamespace(find=find))
+    monkeypatch.setitem(sys.modules, "nltk", nltk)
+    benchmark_worker.offline_nltk()
+    assert nltk.download("stopwords")
+    assert found == ["corpora/stopwords"]
+    with pytest.raises(LookupError, match="frozen punkt"):
+        nltk.download("punkt")
+    with pytest.raises(ValueError, match="Unprepared"):
+        nltk.download("unapproved")
+
+
 def test_sandbox_probe_failure_preserves_actual_child_error_and_log(tmp_path, monkeypatch):
     monkeypatch.setattr(code_lab, "sandbox_command", lambda *args: ["isolated-fixture"])
 

tests/test_v100_remote_helper.py

@@ -141,6 +141,24 @@ def test_canonical_migration_still_rejects_semantic_changes_or_unloaded_model(tr
     assert path.read_bytes() == original
 
 
+def test_late_helper_preparation_applies_low_workload_only_after_verified_identity(
+    transport, tmp_path
+):
+    from rlm.v100.campaign import configure_helper
+
+    path, _ = prepared(tmp_path)
+    result = configure_helper(tmp_path)
+    actual = load_profile(path, tmp_path)
+    assert actual["resources"]["helper_batch_tokens"] == 16
+    assert actual["resources"]["helper_duty_percent"] == 15
+    assert result["loaded"]["context_length"] == 32768
+    transport[1]["info"]["template"] = "unauthorized template"
+    previous = path.read_bytes()
+    with pytest.raises(ValueError, match="metadata"):
+        configure_helper(tmp_path)
+    assert path.read_bytes() == previous
+
+
 def test_stable_metadata_ignores_only_date_and_keeps_legacy_semantics(transport):
     first = {**INFO, "modified_at": "before"}
     second = {**INFO, "modified_at": "after"}