Skip to content

Commit e4ab08d

Browse files
fix(review): apply remaining findings #6, #7, #10 -- hang backstop, fail-fast, env stripping
SIGALRM hard-deadline backstop around each governed run slice so a candidate callable that hangs mid-call terminates, persists as RESOURCE_TERMINATED, and is charged by the breaker (#6). Host-side materialization failures after subprocess vetting now raise loudly instead of masquerading as candidate rejections (#7). The gate subprocess runs candidate code under an allowlisted environment and re-scrubs dotenv-loaded secrets after imports (#10). Claude-Session: https://claude.ai/code/session_01QfVmUyopDjKxK3b4nWxKWM
1 parent e5f01b3 commit e4ab08d

6 files changed

Lines changed: 447 additions & 19 deletions

File tree

‎docs/harness-proposal-interface.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,12 @@ candidate, including the ones that never ran.
162162
| 7 | `harness_check` | `shrlm.runner.check_harness`: invariant probes (I1 boundedness, S9 signature/return type, S6 field vocabulary, plumbing, stated limits) — same subprocess | Subprocess only |
163163
| 8 | `round_trip` | The materialized harness re-serializes byte-identically to the envelope | Subprocess only |
164164

165+
The gate subprocess runs with a stripped environment: the child inherits only a
166+
small allowlist (`PATH`, `PYTHONPATH`, `HOME`, `TMPDIR`, and locale variables),
167+
and it re-scrubs itself after import so values re-loaded from a `.env` file are
168+
dropped too — API keys and every other host secret are gone before candidate
169+
code runs.
170+
165171
A candidate that survives is returned as a `LoadedCandidate`: the live
166172
`Harness` (edited surface materialized, every unchanged surface the incumbent's
167173
own object), the edited surface id, the envelope hash, and the parsed proposal

‎shrlm/optimization/candidates.py‎

Lines changed: 47 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -40,17 +40,23 @@
4040
defense-in-depth, not a security boundary — the REPL already executes
4141
model-written code by design.
4242
43-
Rejections are values, never exceptions: expected-invalid input comes back as a
44-
``CandidateRejection`` naming the gate and the violation, in the style of
45-
``attribution.py``'s validate-with-named-violation loop, so the promotion
46-
ledger can record every candidate including the ones that never ran.
43+
Rejections are values, never exceptions — for expected-invalid candidate
44+
input. Every gate failure comes back as a ``CandidateRejection`` naming the
45+
gate and the violation, in the style of ``attribution.py``'s
46+
validate-with-named-violation loop, so the promotion ledger can record every
47+
candidate including the ones that never ran. The boundary is precise: once the
48+
subprocess has vetted a candidate, a host-side failure while rebuilding the
49+
same harness is a bug in this module, not stage-2 input, and it raises loudly
50+
(the round-trip hash comparison after it stays a structured rejection because
51+
it checks the candidate's envelope, not our own code).
4752
"""
4853

4954
import ast
5055
import hashlib
5156
import importlib.util
5257
import json
5358
import math
59+
import os
5460
import subprocess
5561
import sys
5662
from dataclasses import dataclass, replace
@@ -621,11 +627,34 @@ def run_subprocess_gate(proposal_path: str) -> dict[str, Any]:
621627
return {"ok": True}
622628

623629

630+
# The entire environment the gate subprocess inherits (plus any ``LC_*``
631+
# locale variables). The child runs candidate (model-authored) code and needs
632+
# no credentials — only what it takes to launch ``sys.executable -m
633+
# shrlm.optimization.candidates`` survives; every API key, token, and secret
634+
# in the host environment is dropped.
635+
_GATE_ENV_ALLOWLIST = ("PATH", "PYTHONPATH", "HOME", "TMPDIR", "LANG", "SYSTEMROOT")
636+
637+
638+
def _gate_environment() -> dict[str, str]:
639+
"""The curated environment for the gate subprocess: allowlist plus LC_*."""
640+
return {
641+
key: value
642+
for key, value in os.environ.items()
643+
if key in _GATE_ENV_ALLOWLIST or key.startswith("LC_")
644+
}
645+
646+
624647
def _run_gate_subprocess(proposal_path: Path, timeout_seconds: float) -> dict[str, Any]:
625648
"""Host-side wrapper: run the gate in a child under a wall-clock timeout."""
626649
command = [sys.executable, "-m", "shrlm.optimization.candidates", str(proposal_path)]
627650
try:
628-
process = subprocess.run(command, capture_output=True, text=True, timeout=timeout_seconds)
651+
process = subprocess.run(
652+
command,
653+
capture_output=True,
654+
text=True,
655+
timeout=timeout_seconds,
656+
env=_gate_environment(),
657+
)
629658
except subprocess.TimeoutExpired:
630659
return {
631660
"ok": False,
@@ -778,15 +807,13 @@ def rejection(candidate_id: str, gate: str, reason: str) -> CandidateRejection:
778807

779808
# The subprocess vetted materialization and invariants; now build the live
780809
# harness the evaluation will run: edited surface from the envelope,
781-
# unchanged surfaces the incumbent's own objects (KTD2).
810+
# unchanged surfaces the incumbent's own objects (KTD2). A failure here is
811+
# a host-side bug, not expected-invalid input, so it raises (fail fast).
782812
module_path = proposal_path.parent / MODULE_FILENAME
783-
try:
784-
module = import_surface_module(module_path) if surface_id in ("S7", "S8", "S9") else None
785-
fields = surface_field_values(serialization, surface_id, module)
786-
harness = replace(incumbent, name=serialization["name"], **fields)
787-
materialized_hash = hash_of_serialization(serialize_harness(harness))
788-
except Exception as error: # defensive: the subprocess already vetted this path
789-
return rejection(candidate_id, GATE_MATERIALIZATION, f"{type(error).__name__}: {error}")
813+
module = import_surface_module(module_path) if surface_id in ("S7", "S8", "S9") else None
814+
fields = surface_field_values(serialization, surface_id, module)
815+
harness = replace(incumbent, name=serialization["name"], **fields)
816+
materialized_hash = hash_of_serialization(serialize_harness(harness))
790817
if materialized_hash != envelope["hash"]:
791818
return rejection(
792819
candidate_id,
@@ -907,6 +934,13 @@ def load_candidates(
907934

908935

909936
if __name__ == "__main__":
937+
# The host already launches this child with the curated environment, but
938+
# the imports above re-populate secrets from any ``.env`` file
939+
# (``rlm.clients`` calls ``load_dotenv`` at import time). Scrub back down
940+
# to the allowlist before a byte of candidate code runs.
941+
for _key in list(os.environ):
942+
if _key not in _GATE_ENV_ALLOWLIST and not _key.startswith("LC_"):
943+
del os.environ[_key]
910944
print(json.dumps(run_subprocess_gate(sys.argv[1])))
911945

912946

‎shrlm/optimization/costs.py‎

Lines changed: 154 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,34 @@
2929
from the manifest alone, and the candidate comes back marked
3030
``over_budget`` -- rejected, never silently dropped.
3131
32+
One more experiment-owned containment, for the same KTD3 reason the caps are:
33+
the *hard wall-clock backstop*. Candidate callables (the S7/S8/S9 surfaces)
34+
run synchronously inside the host loop, and the runtime's own ``max_timeout``
35+
is only checked between iterations -- so a candidate call that never returns
36+
(``while True`` in a middleware, a hang in harness construction) would block
37+
the host forever: nothing persisted, nothing charged, ``validate_round`` never
38+
returning. The harness cannot be trusted to bound itself (its runtime policy
39+
is an editable surface), so ``run_governed_round`` arms a SIGALRM timer around
40+
each single-run slice at ``hard_deadline_seconds`` -- the per-run
41+
``max_timeout`` times ``HARD_DEADLINE_FACTOR`` plus
42+
``HARD_DEADLINE_GRACE_SECONDS``. The factor leaves room for a final iteration
43+
that started just under the limit (the runtime's between-iteration check
44+
remains the primary, precise mechanism); the fixed grace absorbs the
45+
per-slice overhead outside the RLM loop (harness build, verification,
46+
persistence). When the alarm fires, ``HardDeadlineExceeded`` is raised at the
47+
hung bytecode; as a ``TimeoutExceededError`` subclass it flows through the
48+
driver's per-run limit handler and persists as an ordinary
49+
RESOURCE_TERMINATED run (persist-first, KTD5), which the breaker then charges
50+
at the per-run ceiling. If the interrupt escapes ``run_round`` entirely (a
51+
hang outside the per-run try, e.g. in harness construction), the governed
52+
round synthesizes and persists the in-flight run's terminated manifest line
53+
itself (``driver.persist_interrupted_run``) before continuing. The backstop
54+
is skipped -- documented, not silently -- where SIGALRM cannot bind (no
55+
SIGALRM on the platform, or not the main thread); it closes the in-host hang
56+
hole at the v1 level, and a full subprocess-per-run boundary is the v2
57+
escalation. A candidate that swallows ``BaseException`` inside its hang can
58+
still defeat a signal-raised exception -- that adversarial case is also v2's.
59+
3260
Pricing one persisted run (``breaker_run_cost``) has one policy worth pinning:
3361
a budget-terminated run carries the limit exception's actual ``spent`` figure
3462
as its persisted cost (see ``driver._partial_completion``) and is charged
@@ -40,18 +68,95 @@
4068
costume (the same stance ``runner.acceptance_inputs`` takes).
4169
"""
4270

71+
import signal
72+
import threading
73+
from collections.abc import Callable
4374
from dataclasses import dataclass, field
44-
from typing import Any
75+
from typing import Any, TypeVar
4576

77+
from rlm.utils.exceptions import TimeoutExceededError
4678
from shrlm.optimization.bundle import round_dir
4779
from shrlm.optimization.candidates import GATE_CAPS, CandidateRejection, cap_violations
48-
from shrlm.optimization.driver import RoundConfig, run_id_for, run_round
80+
from shrlm.optimization.driver import (
81+
RoundConfig,
82+
load_manifest,
83+
persist_interrupted_run,
84+
run_id_for,
85+
run_round,
86+
)
4987
from shrlm.optimization.taxonomy import VerifierCause
5088

5189
# Candidate outcomes the promotion ledger records for a governed round.
5290
OUTCOME_COMPLETED = "completed"
5391
OUTCOME_OVER_BUDGET = "over_budget"
5492

93+
# The hard wall-clock deadline for one run slice, derived from the per-run
94+
# ``max_timeout`` cap: ``max_timeout * HARD_DEADLINE_FACTOR +
95+
# HARD_DEADLINE_GRACE_SECONDS`` (see the module docstring for the rationale).
96+
HARD_DEADLINE_FACTOR = 1.5
97+
HARD_DEADLINE_GRACE_SECONDS = 30.0
98+
99+
_T = TypeVar("_T")
100+
101+
102+
class HardDeadlineExceeded(TimeoutExceededError):
103+
"""The hard wall-clock backstop fired: a run slice never returned control.
104+
105+
Subclasses ``TimeoutExceededError`` deliberately: the driver's per-run
106+
limit handler (``driver.ROOT_LIMIT_EXCEPTIONS``) then persists it exactly
107+
like the runtime's own timeout -- a failing RESOURCE_TERMINATED run whose
108+
error string names this class -- with no driver change required.
109+
"""
110+
111+
def __init__(self, deadline: float):
112+
super().__init__(
113+
elapsed=deadline,
114+
timeout=deadline,
115+
message=(
116+
f"hard wall-clock deadline exceeded: the run slice did not return "
117+
f"within {deadline:.1f}s; candidate code likely hung inside a live call"
118+
),
119+
)
120+
121+
122+
def hard_deadline_seconds(max_timeout: float | None) -> float | None:
123+
"""The backstop deadline for one run slice; ``None`` disables the backstop.
124+
125+
``None`` in means ``None`` out: governed rounds always carry a
126+
``max_timeout`` (``ValidationCaps`` makes it mandatory), so an unbounded
127+
slice only occurs for a hand-built config that opted out of the cap.
128+
"""
129+
if max_timeout is None:
130+
return None
131+
return max_timeout * HARD_DEADLINE_FACTOR + HARD_DEADLINE_GRACE_SECONDS
132+
133+
134+
def _alarm_available() -> bool:
135+
"""Whether SIGALRM can bind here: POSIX only, main thread only."""
136+
return hasattr(signal, "SIGALRM") and threading.current_thread() is threading.main_thread()
137+
138+
139+
def _call_with_hard_deadline(fn: Callable[[], _T], deadline: float | None) -> _T:
140+
"""Run ``fn`` under a SIGALRM hard deadline, restoring alarm state after.
141+
142+
With no deadline, or where SIGALRM cannot bind (``_alarm_available``), the
143+
call runs unguarded -- the documented no-backstop mode; the runtime's own
144+
between-iteration timeout remains in force either way.
145+
"""
146+
if deadline is None or not _alarm_available():
147+
return fn()
148+
149+
def _on_alarm(signum: int, frame: Any) -> None:
150+
raise HardDeadlineExceeded(deadline)
151+
152+
previous = signal.signal(signal.SIGALRM, _on_alarm)
153+
signal.setitimer(signal.ITIMER_REAL, deadline)
154+
try:
155+
return fn()
156+
finally:
157+
signal.setitimer(signal.ITIMER_REAL, 0.0)
158+
signal.signal(signal.SIGALRM, previous)
159+
55160

56161
@dataclass(frozen=True)
57162
class ValidationCaps:
@@ -195,6 +300,33 @@ def over_budget(self) -> bool:
195300
return self.outcome == OUTCOME_OVER_BUDGET
196301

197302

303+
def _run_slice(
304+
config: RoundConfig, *, stop_after: int, deadline: float | None, known: int
305+
) -> list[dict[str, Any]]:
306+
"""One ``run_round`` slice under the hard-deadline backstop.
307+
308+
The common hang -- inside a candidate call during ``harnessed.completion``
309+
-- never reaches this function's handler: the driver's per-run limit
310+
handler catches ``HardDeadlineExceeded`` (a ``TimeoutExceededError``) and
311+
persists the terminated run itself, so the slice returns normally. What is
312+
handled here is the interrupt that *escaped* ``run_round`` -- a hang in
313+
harness construction, or an alarm landing between a run's completion and
314+
its manifest append. ``known`` (the manifest lines already accounted for)
315+
decides recovery: if nothing new persisted, the in-flight run's terminated
316+
line is synthesized (``persist_interrupted_run``); if the run did persist
317+
before the alarm landed, the reloaded manifest is simply returned.
318+
"""
319+
try:
320+
return _call_with_hard_deadline(lambda: run_round(config, stop_after=stop_after), deadline)
321+
except HardDeadlineExceeded as error:
322+
persisted = load_manifest(config.out_dir, config.round_index)
323+
if len(persisted) == known:
324+
entry = persist_interrupted_run(config, error)
325+
if entry is not None:
326+
persisted.append(entry)
327+
return persisted
328+
329+
198330
def run_governed_round(config: RoundConfig, breaker: CandidateSpendBreaker) -> GovernedRoundResult:
199331
"""Run one round under the breaker: persist-first, one run at a time.
200332
@@ -205,6 +337,12 @@ def run_governed_round(config: RoundConfig, breaker: CandidateSpendBreaker) -> G
205337
moment its manifest line lands; once the breaker trips, the remaining runs
206338
are skipped and reported, never silently dropped.
207339
340+
Each slice runs under the hard wall-clock backstop (module docstring): a
341+
candidate call that hangs is interrupted at ``hard_deadline_seconds``,
342+
persisted as a RESOURCE_TERMINATED run, charged (at the per-run ceiling
343+
when the termination persisted no cost), and the round continues -- the
344+
host never blocks indefinitely on candidate code.
345+
208346
Args:
209347
config: The round to run, its limits already merged by
210348
``governed_limits``.
@@ -216,13 +354,21 @@ def run_governed_round(config: RoundConfig, breaker: CandidateSpendBreaker) -> G
216354
breaker's cumulative spend, and the run ids the breaker skipped.
217355
"""
218356
namespace = str(round_dir(config.out_dir, config.round_index))
219-
entries = run_round(config, stop_after=0)
357+
deadline = hard_deadline_seconds(config.max_timeout)
358+
# The stop_after=0 slice executes no runs but does build the harness when
359+
# runs are pending -- candidate code that can itself hang, hence the guard.
360+
entries = _run_slice(
361+
config,
362+
stop_after=0,
363+
deadline=deadline,
364+
known=len(load_manifest(config.out_dir, config.round_index)),
365+
)
220366
for entry in entries:
221367
breaker.charge(entry, namespace=namespace)
222368

223369
while not breaker.tripped:
224370
known = len(entries)
225-
entries = run_round(config, stop_after=1)
371+
entries = _run_slice(config, stop_after=1, deadline=deadline, known=known)
226372
if len(entries) == known:
227373
break # nothing pending: the round is complete
228374
for entry in entries[known:]:
@@ -244,12 +390,16 @@ def run_governed_round(config: RoundConfig, breaker: CandidateSpendBreaker) -> G
244390

245391

246392
__all__ = [
393+
"HARD_DEADLINE_FACTOR",
394+
"HARD_DEADLINE_GRACE_SECONDS",
247395
"OUTCOME_COMPLETED",
248396
"OUTCOME_OVER_BUDGET",
249397
"CandidateSpendBreaker",
250398
"GovernedRoundResult",
399+
"HardDeadlineExceeded",
251400
"ValidationCaps",
252401
"breaker_run_cost",
253402
"governed_limits",
403+
"hard_deadline_seconds",
254404
"run_governed_round",
255405
]

0 commit comments

Comments
 (0)