2929 from the manifest alone, and the candidate comes back marked
3030 ``over_budget`` -- rejected, never silently dropped.
3131
32+ One more experiment-owned containment, for the same KTD3 reason the caps are:
33+ the *hard wall-clock backstop*. Candidate callables (the S7/S8/S9 surfaces)
34+ run synchronously inside the host loop, and the runtime's own ``max_timeout``
35+ is only checked between iterations -- so a candidate call that never returns
36+ (``while True`` in a middleware, a hang in harness construction) would block
37+ the host forever: nothing persisted, nothing charged, ``validate_round`` never
38+ returning. The harness cannot be trusted to bound itself (its runtime policy
39+ is an editable surface), so ``run_governed_round`` arms a SIGALRM timer around
40+ each single-run slice at ``hard_deadline_seconds`` -- the per-run
41+ ``max_timeout`` times ``HARD_DEADLINE_FACTOR`` plus
42+ ``HARD_DEADLINE_GRACE_SECONDS``. The factor leaves room for a final iteration
43+ that started just under the limit (the runtime's between-iteration check
44+ remains the primary, precise mechanism); the fixed grace absorbs the
45+ per-slice overhead outside the RLM loop (harness build, verification,
46+ persistence). When the alarm fires, ``HardDeadlineExceeded`` is raised at the
47+ hung bytecode; as a ``TimeoutExceededError`` subclass it flows through the
48+ driver's per-run limit handler and persists as an ordinary
49+ RESOURCE_TERMINATED run (persist-first, KTD5), which the breaker then charges
50+ at the per-run ceiling. If the interrupt escapes ``run_round`` entirely (a
51+ hang outside the per-run try, e.g. in harness construction), the governed
52+ round synthesizes and persists the in-flight run's terminated manifest line
53+ itself (``driver.persist_interrupted_run``) before continuing. The backstop
54+ is skipped -- documented, not silently -- where SIGALRM cannot bind (no
55+ SIGALRM on the platform, or not the main thread); it closes the in-host hang
56+ hole at the v1 level, and a full subprocess-per-run boundary is the v2
57+ escalation. A candidate that swallows ``BaseException`` inside its hang can
58+ still defeat a signal-raised exception -- that adversarial case is also v2's.
59+
3260Pricing one persisted run (``breaker_run_cost``) has one policy worth pinning:
3361a budget-terminated run carries the limit exception's actual ``spent`` figure
3462as its persisted cost (see ``driver._partial_completion``) and is charged
4068costume (the same stance ``runner.acceptance_inputs`` takes).
4169"""
4270
71+ import signal
72+ import threading
73+ from collections .abc import Callable
4374from dataclasses import dataclass , field
44- from typing import Any
75+ from typing import Any , TypeVar
4576
77+ from rlm .utils .exceptions import TimeoutExceededError
4678from shrlm .optimization .bundle import round_dir
4779from shrlm .optimization .candidates import GATE_CAPS , CandidateRejection , cap_violations
48- from shrlm .optimization .driver import RoundConfig , run_id_for , run_round
80+ from shrlm .optimization .driver import (
81+ RoundConfig ,
82+ load_manifest ,
83+ persist_interrupted_run ,
84+ run_id_for ,
85+ run_round ,
86+ )
4987from shrlm .optimization .taxonomy import VerifierCause
5088
5189# Candidate outcomes the promotion ledger records for a governed round.
5290OUTCOME_COMPLETED = "completed"
5391OUTCOME_OVER_BUDGET = "over_budget"
5492
93+ # The hard wall-clock deadline for one run slice, derived from the per-run
94+ # ``max_timeout`` cap: ``max_timeout * HARD_DEADLINE_FACTOR +
95+ # HARD_DEADLINE_GRACE_SECONDS`` (see the module docstring for the rationale).
96+ HARD_DEADLINE_FACTOR = 1.5
97+ HARD_DEADLINE_GRACE_SECONDS = 30.0
98+
99+ _T = TypeVar ("_T" )
100+
101+
102+ class HardDeadlineExceeded (TimeoutExceededError ):
103+ """The hard wall-clock backstop fired: a run slice never returned control.
104+
105+ Subclasses ``TimeoutExceededError`` deliberately: the driver's per-run
106+ limit handler (``driver.ROOT_LIMIT_EXCEPTIONS``) then persists it exactly
107+ like the runtime's own timeout -- a failing RESOURCE_TERMINATED run whose
108+ error string names this class -- with no driver change required.
109+ """
110+
111+ def __init__ (self , deadline : float ):
112+ super ().__init__ (
113+ elapsed = deadline ,
114+ timeout = deadline ,
115+ message = (
116+ f"hard wall-clock deadline exceeded: the run slice did not return "
117+ f"within { deadline :.1f} s; candidate code likely hung inside a live call"
118+ ),
119+ )
120+
121+
122+ def hard_deadline_seconds (max_timeout : float | None ) -> float | None :
123+ """The backstop deadline for one run slice; ``None`` disables the backstop.
124+
125+ ``None`` in means ``None`` out: governed rounds always carry a
126+ ``max_timeout`` (``ValidationCaps`` makes it mandatory), so an unbounded
127+ slice only occurs for a hand-built config that opted out of the cap.
128+ """
129+ if max_timeout is None :
130+ return None
131+ return max_timeout * HARD_DEADLINE_FACTOR + HARD_DEADLINE_GRACE_SECONDS
132+
133+
134+ def _alarm_available () -> bool :
135+ """Whether SIGALRM can bind here: POSIX only, main thread only."""
136+ return hasattr (signal , "SIGALRM" ) and threading .current_thread () is threading .main_thread ()
137+
138+
139+ def _call_with_hard_deadline (fn : Callable [[], _T ], deadline : float | None ) -> _T :
140+ """Run ``fn`` under a SIGALRM hard deadline, restoring alarm state after.
141+
142+ With no deadline, or where SIGALRM cannot bind (``_alarm_available``), the
143+ call runs unguarded -- the documented no-backstop mode; the runtime's own
144+ between-iteration timeout remains in force either way.
145+ """
146+ if deadline is None or not _alarm_available ():
147+ return fn ()
148+
149+ def _on_alarm (signum : int , frame : Any ) -> None :
150+ raise HardDeadlineExceeded (deadline )
151+
152+ previous = signal .signal (signal .SIGALRM , _on_alarm )
153+ signal .setitimer (signal .ITIMER_REAL , deadline )
154+ try :
155+ return fn ()
156+ finally :
157+ signal .setitimer (signal .ITIMER_REAL , 0.0 )
158+ signal .signal (signal .SIGALRM , previous )
159+
55160
56161@dataclass (frozen = True )
57162class ValidationCaps :
@@ -195,6 +300,33 @@ def over_budget(self) -> bool:
195300 return self .outcome == OUTCOME_OVER_BUDGET
196301
197302
303+ def _run_slice (
304+ config : RoundConfig , * , stop_after : int , deadline : float | None , known : int
305+ ) -> list [dict [str , Any ]]:
306+ """One ``run_round`` slice under the hard-deadline backstop.
307+
308+ The common hang -- inside a candidate call during ``harnessed.completion``
309+ -- never reaches this function's handler: the driver's per-run limit
310+ handler catches ``HardDeadlineExceeded`` (a ``TimeoutExceededError``) and
311+ persists the terminated run itself, so the slice returns normally. What is
312+ handled here is the interrupt that *escaped* ``run_round`` -- a hang in
313+ harness construction, or an alarm landing between a run's completion and
314+ its manifest append. ``known`` (the manifest lines already accounted for)
315+ decides recovery: if nothing new persisted, the in-flight run's terminated
316+ line is synthesized (``persist_interrupted_run``); if the run did persist
317+ before the alarm landed, the reloaded manifest is simply returned.
318+ """
319+ try :
320+ return _call_with_hard_deadline (lambda : run_round (config , stop_after = stop_after ), deadline )
321+ except HardDeadlineExceeded as error :
322+ persisted = load_manifest (config .out_dir , config .round_index )
323+ if len (persisted ) == known :
324+ entry = persist_interrupted_run (config , error )
325+ if entry is not None :
326+ persisted .append (entry )
327+ return persisted
328+
329+
198330def run_governed_round (config : RoundConfig , breaker : CandidateSpendBreaker ) -> GovernedRoundResult :
199331 """Run one round under the breaker: persist-first, one run at a time.
200332
@@ -205,6 +337,12 @@ def run_governed_round(config: RoundConfig, breaker: CandidateSpendBreaker) -> G
205337 moment its manifest line lands; once the breaker trips, the remaining runs
206338 are skipped and reported, never silently dropped.
207339
340+ Each slice runs under the hard wall-clock backstop (module docstring): a
341+ candidate call that hangs is interrupted at ``hard_deadline_seconds``,
342+ persisted as a RESOURCE_TERMINATED run, charged (at the per-run ceiling
343+ when the termination persisted no cost), and the round continues -- the
344+ host never blocks indefinitely on candidate code.
345+
208346 Args:
209347 config: The round to run, its limits already merged by
210348 ``governed_limits``.
@@ -216,13 +354,21 @@ def run_governed_round(config: RoundConfig, breaker: CandidateSpendBreaker) -> G
216354 breaker's cumulative spend, and the run ids the breaker skipped.
217355 """
218356 namespace = str (round_dir (config .out_dir , config .round_index ))
219- entries = run_round (config , stop_after = 0 )
357+ deadline = hard_deadline_seconds (config .max_timeout )
358+ # The stop_after=0 slice executes no runs but does build the harness when
359+ # runs are pending -- candidate code that can itself hang, hence the guard.
360+ entries = _run_slice (
361+ config ,
362+ stop_after = 0 ,
363+ deadline = deadline ,
364+ known = len (load_manifest (config .out_dir , config .round_index )),
365+ )
220366 for entry in entries :
221367 breaker .charge (entry , namespace = namespace )
222368
223369 while not breaker .tripped :
224370 known = len (entries )
225- entries = run_round (config , stop_after = 1 )
371+ entries = _run_slice (config , stop_after = 1 , deadline = deadline , known = known )
226372 if len (entries ) == known :
227373 break # nothing pending: the round is complete
228374 for entry in entries [known :]:
@@ -244,12 +390,16 @@ def run_governed_round(config: RoundConfig, breaker: CandidateSpendBreaker) -> G
244390
245391
246392__all__ = [
393+ "HARD_DEADLINE_FACTOR" ,
394+ "HARD_DEADLINE_GRACE_SECONDS" ,
247395 "OUTCOME_COMPLETED" ,
248396 "OUTCOME_OVER_BUDGET" ,
249397 "CandidateSpendBreaker" ,
250398 "GovernedRoundResult" ,
399+ "HardDeadlineExceeded" ,
251400 "ValidationCaps" ,
252401 "breaker_run_cost" ,
253402 "governed_limits" ,
403+ "hard_deadline_seconds" ,
254404 "run_governed_round" ,
255405]
0 commit comments