Skip to content

Commit a840170

Browse files
committed
v87: add isolated auto-updating Debian RTX 3060 Ti worker
Register the explicitly owned 192.168.0.9 peer with scoped text inference, independent fixed-kernel CPU transport and local safe-tensor validation. Add a non-login account, protected system services, RAM/temperature budgets, boot autostart, SHA-verified runtime/source and deferred root-managed updates. Qualify advisory models independently of CPU transport; preserve accepted V100 training and existing Windows limits. Route available advisory work and serving fallbacks to the node; report failures and telemetry in the dashboard. Validation: 1389 passed, 65 skipped; additional release/updater checks 13 passed. Ruff, release SHA256 and shell syntax passed. Physical GPU residency, networking and boot execution are checked by operator-run installers, not claimed from mocked tests.
1 parent f8c3413 commit a840170

18 files changed

Lines changed: 2008 additions & 34 deletions

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name = "rlms"
33
authors = [
44
{name = "Alex Zhang", email = "altzhang@mit.edu"},
55
]
6-
version = "0.1.3+v100.86"
6+
version = "0.1.3+v100.87"
77
description = "Recursive Language Models."
88
readme = "README.md"
99
license = "MIT"

‎rlm/v100/competition.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,7 @@ def helper_client(
411411
"max_vram_gib": profile["resources"]["max_vram_gib"],
412412
"helper_batch_tokens": profile["resources"].get("helper_batch_tokens", 64),
413413
"helper_duty_percent": profile["resources"].get("helper_duty_percent", 65),
414+
"dedicated_host": profile["resources"].get("dedicated_host", False),
414415
"on_demand": profile["resources"].get("on_demand", False),
415416
"allow_cpu_offload": profile["resources"].get("allow_cpu_offload", False),
416417
"require_adaptive_windows_guard": profile["resources"].get(
@@ -436,7 +437,9 @@ def helper_client(
436437
if profile.get("resources", {}).get("device") in ("cpu", "remote")
437438
else "model",
438439
activity_context={
439-
"device": profile.get("resources", {}).get("device", "local GPU"),
440+
"device": profile.get("resources", {}).get(
441+
"worker_name", profile.get("resources", {}).get("device", "local GPU")
442+
),
440443
"endpoint": settings["base_url"],
441444
"model_version": settings["model_version"],
442445
"target": profile["server"]["model"],

‎rlm/v100/drones.py‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
from rlm.v100.activity import ActivityLog
1818
from rlm.v100.common import atomic_json, load_profile
19+
from rlm.v100.remote_nodes import selected_node
1920

2021

2122
@contextmanager
@@ -174,9 +175,8 @@ def execute(root: Path, job: dict) -> dict:
174175
if preferences(root).get("remote_helper_enabled", True) is False:
175176
client = accepted_master(root, job["branch"])
176177
else:
177-
profile = load_profile(
178-
selected_helper(root, "planner" if job["kind"] == "critic" else "fast"), root
179-
)
178+
chosen_path = selected_helper(root, "planner" if job["kind"] == "critic" else "fast")
179+
profile = load_profile(chosen_path, root)
180180
if not remote_profile(profile):
181181
raise ValueError(
182182
"Resident research requires an explicitly selected remote or accepted master"
@@ -186,15 +186,25 @@ def execute(root: Path, job: dict) -> dict:
186186
try:
187187
client.identity()
188188
except requests.RequestException as error:
189+
node = selected_node(root)
190+
alternative = None
191+
if node is not None and node.resolve() != chosen_path.resolve():
192+
try:
193+
alternative = helper_client(load_profile(node, root), root, job["branch"])
194+
alternative.identity()
195+
except requests.RequestException:
196+
alternative = None
197+
client = (
198+
alternative if alternative is not None else accepted_master(root, job["branch"])
199+
)
189200
ActivityLog(root, job["branch"], "resident-" + job["kind"]).write(
190201
"steps",
191-
"local-master-fallback",
202+
"owned-node-fallback" if alternative is not None else "local-master-fallback",
192203
{
193204
"reason": str(error)[:300],
194-
"scope": "Remote transport unavailable; accepted local master only, no changed-identity fallback",
205+
"scope": "Unavailable transport replaced before generation by a qualified owned helper or the protected accepted master; no changed-identity fallback",
195206
},
196207
)
197-
client = accepted_master(root, job["branch"])
198208
client.research_tool_steps = 6
199209
client.activity_actor = "resident-" + job["kind"]
200210
client.research_tool_names = {
@@ -358,7 +368,10 @@ def service(root: Path, stop: threading.Event) -> None:
358368
(kind in ("researcher", "critic", "benchmark", "income")) == llm
359369
for _, kind in active.values()
360370
)
361-
if used >= (min(1, budget["cpu_slots"]) if llm else budget["cpu_slots"]):
371+
llm_slots = min(1, budget["cpu_slots"]) + int(
372+
not gpu_disabled(root) and selected_node(root) is not None
373+
)
374+
if used >= (llm_slots if llm else budget["cpu_slots"]):
362375
continue
363376
job = dict(row)
364377
if job["kind"] == "python":
@@ -385,7 +398,9 @@ def service(root: Path, stop: threading.Event) -> None:
385398
"cpu_slots": budget["cpu_slots"],
386399
"budget": budget,
387400
"cpu_affinities": affinities,
388-
"rtx_slots": 0 if gpu_disabled(root) else 1,
401+
"rtx_slots": 0
402+
if gpu_disabled(root)
403+
else 1 + int(selected_node(root) is not None),
389404
"local_model_slots": 1 if gpu_disabled(root) else 0,
390405
},
391406
)

‎rlm/v100/income_work.py‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,31 @@ class MasterNotReady(RuntimeError):
3131
"""Transient serving startup, shutdown or unavailable protected receipt."""
3232

3333

34+
def owned_advisor(root: Path, branch: str, reason: str):
35+
"""Use an independently qualified advisory node while protected serving is unavailable."""
36+
from rlm.v100.competition import helper_client
37+
from rlm.v100.mission_chat import preferences
38+
from rlm.v100.remote_nodes import selected_node
39+
40+
path = selected_node(root) if preferences(root).get("remote_helper_enabled", True) else None
41+
if path is None:
42+
raise MasterNotReady(reason)
43+
client = helper_client(load_profile(path, root), root, branch)
44+
client.timeout = 5
45+
try:
46+
client.identity()
47+
except requests.RequestException as error:
48+
raise MasterNotReady(reason) from error
49+
client.timeout = 120
50+
client.request_deadline = time.monotonic() + 150
51+
client.enable_thinking = False
52+
client.sampling_args = dict(client.sampling_args) | {"max_tokens": 2048}
53+
client.research_token_ceiling = 2048
54+
client.research_thinking_allowed = False
55+
client.activity_actor = "income-owned-advisor"
56+
return client
57+
58+
3459
def accepted_master(root: Path, branch: str):
3560
"""Reuse a protected serving master, never launch a model or use a candidate."""
3661
from rlm.v100.competition import helper_client
@@ -65,7 +90,9 @@ def accepted_master(root: Path, branch: str):
6590
paths.append(run / f"profile-{state.get('context_window')}.json")
6691
path = next((candidate for candidate in paths if candidate.is_file()), None)
6792
if path is None:
68-
raise MasterNotReady("Accepted master profile not ready; collected evidence retained")
93+
return owned_advisor(
94+
root, branch, "Accepted master profile not ready; collected evidence retained"
95+
)
6996
startup = run / f"server-{state.get('context_window')}.startup.json"
7097
# The initial server is stopped when continuous learning starts its own
7198
# protected server. Its old startup file must not block the live profile.
@@ -74,16 +101,18 @@ def accepted_master(root: Path, branch: str):
74101
and startup.exists()
75102
and json.loads(startup.read_text()).get("stage") != "ready"
76103
):
77-
raise MasterNotReady("Accepted V100 serving startup not ready; evidence retained")
104+
return owned_advisor(
105+
root, branch, "Accepted V100 serving startup not ready; evidence retained"
106+
)
78107
profile = load_profile(path, root)
79108
client = helper_client(profile, root, branch)
80109
client.timeout = min(10, client.timeout)
81110
# Full protected receipt and endpoint check precedes any generation. A
82111
# candidate temporarily occupying the same endpoint is never used.
83112
try:
84113
assert_served_expert(client, profile, root)
85-
except (FileNotFoundError, ProcessLookupError, requests.RequestException) as error:
86-
raise MasterNotReady("Accepted serving process or endpoint not ready") from error
114+
except (FileNotFoundError, ProcessLookupError, requests.RequestException):
115+
return owned_advisor(root, branch, "Accepted serving process or endpoint not ready")
87116
client.timeout = 40
88117
client.request_deadline = time.monotonic() + 60
89118
client.enable_thinking = False

‎rlm/v100/live_status.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,47 @@ def snapshot(root: Path, mission: dict, report: dict, gpu: dict) -> dict:
423423
if remaining and not rtx["stale"]:
424424
rtx["state"] = "cooldown"
425425
actors.append(rtx)
426+
owned = root / "research/owned-node/status.json"
427+
if (root / "research/owned-node.json").exists():
428+
try:
429+
value = (
430+
json.loads(owned.read_text())
431+
if owned.exists()
432+
else {"phase": "waiting for controller service"}
433+
)
434+
age = time.time() - value.get("updated", 0)
435+
actors.append(
436+
{
437+
"id": "debian-3060ti",
438+
"label": "Debian RTX 3060 Ti / CPU",
439+
"state": value.get("phase", "unknown"),
440+
"detail": value.get(
441+
"reason",
442+
"Dedicated owned advisory inference and fixed-kernel CPU trials; remote metrics require local validation",
443+
)
444+
+ (
445+
"; CPU: " + json.dumps(value["cpu_errors"])
446+
if value.get("cpu_errors")
447+
else ""
448+
),
449+
"metrics": value.get("resources", {}),
450+
"stale": age > 90,
451+
"model": value.get("model"),
452+
"receipt": str(owned),
453+
"jobs": [],
454+
}
455+
)
456+
except (OSError, ValueError) as error:
457+
actors.append(
458+
{
459+
"id": "debian-3060ti",
460+
"label": "Debian RTX 3060 Ti / CPU",
461+
"state": "unavailable",
462+
"detail": str(error)[:300],
463+
"stale": True,
464+
"jobs": [],
465+
}
466+
)
426467
for identity, label, relative in (
427468
("knowledge", "Wikipedia / RAG", "research/knowledge/import-status.json"),
428469
("knowledge-index", "Live Wikipedia index", "research/knowledge/index-progress.json"),

0 commit comments

Comments
 (0)