Allow native server restarts after TIME_WAIT while rejecting live listeners (v100.36)

Author: Mantiee

V100_AUDIT.md

@@ -124,3 +124,11 @@ The updater includes this check before installing, so recovery also works with
 older installed packages. Live command/session ownership mismatches still abort
 without killing the process or replacing packages. Checkpoints and failure reports
 are retained. A successful update is not evidence of a successful weight promotion.
+
+## v100.36 native server restart ports
+
+Native and MTP availability probes now use SO_REUSEADDR and actually listen before
+closing the probe. Linux TIME_WAIT after a completed acceptance request no longer
+looks like a live server occupying the port. An existing loopback or wildcard
+listener still blocks launch, with the port included in the error. No listener is
+killed or reused, and no second model is started against an occupied endpoint.

pyproject.toml

@@ -3,7 +3,7 @@ name = "rlms"
 authors = [
     {name = "Alex Zhang", email = "altzhang@mit.edu"},
 ]
-version = "0.1.3+v100.35"
+version = "0.1.3+v100.36"
 description = "Recursive Language Models."
 readme = "README.md"
 license = "MIT"

rlm/v100/competition.py

@@ -3,7 +3,6 @@
 import copy
 import json
 import os
-import socket
 import subprocess
 import sys
 import time
@@ -19,6 +18,7 @@
 from rlm.v100.common import atomic_json, load_profile
 from rlm.v100.experiments import SharedLab, judge_duel, load_duel, plan_duel
 from rlm.v100.researchers import available_ram_gib, research_task, review_research
+from rlm.v100.serving import ensure_local_port_available
 
 
 def command(root: Path, profile: Path, *arguments: str) -> list[str]:
@@ -85,8 +85,7 @@ def managed_server(profile_path: Path, root: Path, log_path: Path):
         yield profile
         return
     origin = urlparse(profile["runtime"]["base_url"])
-    with socket.socket() as probe:
-        probe.bind(("127.0.0.1", origin.port))
+    ensure_local_port_available(origin.port)
     cpu = profile.get("resources", {}).get("device") == "cpu"
     minimum = profile.get("resources", {}).get("min_available_ram_gib", 6)
     if cpu and available_ram_gib() < minimum:

rlm/v100/serving.py

@@ -2,13 +2,27 @@
 
 import json
 import os
+import socket
 from pathlib import Path
 from urllib.parse import urlparse
 
 from rlm.v100.common import atomic_json
 from rlm.v100.protection import execution_hash, file_hash
 
 
+def ensure_local_port_available(port: int) -> None:
+    """Reject live listeners, while allowing a restarted server's TIME_WAIT sockets."""
+    with socket.socket() as probe:
+        probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        try:
+            probe.bind(("127.0.0.1", port))
+            probe.listen(1)
+        except OSError as error:
+            raise OSError(
+                error.errno, f"Local server port {port} unavailable: {error.strerror}"
+            ) from error
+
+
 def process_identity(pid: int) -> str:
     # Linux start time prevents a reused PID being accepted as the protected server.
     fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()

rlm/v100/speculative.py

@@ -5,7 +5,6 @@
 import os
 import re
 import shutil
-import socket
 import statistics
 import subprocess
 import sys
@@ -18,6 +17,7 @@
 import requests
 
 from rlm.v100.common import atomic_json, load_profile
+from rlm.v100.serving import ensure_local_port_available
 
 ASSISTANT_MODEL = "google/gemma-4-12B-it-assistant"
 ASSISTANT_REVISION = "46d4c6f13f0ac0ad827b915669b8df9b81c64c51"
@@ -216,8 +216,7 @@ def test_mtp(
             value = profile["server"][key]
             if value and not Path(value).is_file():
                 raise FileNotFoundError(value)
-    with socket.socket() as probe:
-        probe.bind(("127.0.0.1", 8089))
+    ensure_local_port_available(8089)
     memory = (
         subprocess.run(
             ["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"],

tests/test_v100_server_ports.py

@@ -0,0 +1,47 @@
+"""Local restart probes must distinguish TIME_WAIT from an occupied listener."""
+
+import errno
+import socket
+import sys
+
+import pytest
+
+from rlm.v100.serving import ensure_local_port_available
+
+
+@pytest.mark.parametrize("host", ["127.0.0.1", "0.0.0.0"])
+def test_live_listener_still_blocks_reusable_probe(host):
+    with socket.socket() as listener:
+        listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        listener.bind((host, 0))
+        listener.listen(1)
+        port = listener.getsockname()[1]
+        with pytest.raises(OSError, match=f"port {port} unavailable") as failure:
+            ensure_local_port_available(port)
+        assert failure.value.errno == errno.EADDRINUSE
+
+
+@pytest.mark.skipif(sys.platform != "linux", reason="Linux TIME_WAIT restart regression")
+def test_restart_after_server_closes_connection_does_not_need_kernel_timeout():
+    with socket.socket() as listener, socket.socket() as client:
+        listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        listener.bind(("127.0.0.1", 0))
+        listener.listen(1)
+        port = listener.getsockname()[1]
+        client.connect(("127.0.0.1", port))
+        accepted, _ = listener.accept()
+        # Server closes first, so its local endpoint enters TIME_WAIT.
+        accepted.shutdown(socket.SHUT_WR)
+        assert client.recv(1) == b""
+        client.close()
+        accepted.close()
+    with socket.socket() as old_probe:
+        with pytest.raises(OSError) as failure:
+            old_probe.bind(("127.0.0.1", port))
+        assert failure.value.errno == errno.EADDRINUSE
+    ensure_local_port_available(port)
+    # The probe itself leaves no resident listener behind.
+    with socket.socket() as restarted:
+        restarted.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        restarted.bind(("127.0.0.1", port))
+        restarted.listen(1)

tools/upgrade-v100-campaign.sh

@@ -55,7 +55,7 @@ PY_STOP
 uv --no-config pip install --python "$PY" --no-deps --reinstall-package rlms \
   "rlms @ git+https://github.com/Mantiee/rlm.git@$REV"
 uv --no-config pip check --python "$PY"
-uv --no-config pip freeze --python "$PY" > "$ROOT/research/requirements.continual.v10035.txt"
+uv --no-config pip freeze --python "$PY" > "$ROOT/research/requirements.continual.v10036.txt"
 LAB="$ROOT/bin/v100-continual"
 if [[ -e "$LAB" ]]; then cp -p "$LAB" "$LAB.backup-$(date -u +%Y%m%dT%H%M%SZ)"; fi
 TMP="$(mktemp "$ROOT/bin/.v100-continual.XXXXXX")"