@@ -77,11 +77,22 @@ def guard_cpu_process(
7777def managed_server (
7878 profile_path : Path , root : Path , log_path : Path , cancel : threading .Event | None = None
7979):
80+ log_path .parent .mkdir (parents = True , exist_ok = True )
81+ startup_path = log_path .with_suffix (".startup.json" )
82+
83+ def startup (stage : str , ** details ) -> None :
84+ value = {"stage" : stage , "updated" : time .time (), "log" : str (log_path ), ** details }
85+ atomic_json (startup_path , value )
86+ print ("Server startup:" , json .dumps (value ), flush = True )
87+
88+ startup ("validating-profile" )
8089 profile = load_profile (profile_path , root )
8190 from rlm .v100 .remote_helper import remote_profile
82- from rlm .v100 .scratch_master import is_scratch , verify
8391
84- if is_scratch (profile ):
92+ if profile ["runtime" ].get ("backend" ) == "isolated-architecture" :
93+ from rlm .v100 .scratch_master import verify
94+
95+ startup ("verifying-isolated-architecture" )
8596 verify (profile )
8697 log_path .parent .mkdir (parents = True , exist_ok = True )
8798 with log_path .open ("a" ) as log :
@@ -95,6 +106,7 @@ def managed_server(
95106 return
96107
97108 if remote_profile (profile ):
109+ startup ("checking-remote-backend" , endpoint = profile ["runtime" ]["base_url" ])
98110 client = helper_client (profile , root )
99111 from rlm .v100 .remote_helper import OllamaResearchClient
100112
@@ -125,16 +137,31 @@ def managed_server(
125137 }
126138 if cpu :
127139 environment ["CUDA_VISIBLE_DEVICES" ] = ""
128- log_path .parent .mkdir (parents = True , exist_ok = True )
140+ if profile ["server" ].get ("library_path" ):
141+ environment ["LD_LIBRARY_PATH" ] = (
142+ profile ["server" ]["library_path" ] + ":" + environment .get ("LD_LIBRARY_PATH" , "" )
143+ )
144+ from rlm .v100 .cli import server_command
145+ from rlm .v100 .serving import write_receipt
146+
147+ argv = server_command (profile )
148+ for key in ("binary" , "model" , "draft_model" ):
149+ value = profile ["server" ][key ]
150+ if value and not Path (value ).is_file ():
151+ raise FileNotFoundError (value )
152+ startup ("launching-native-process" )
129153 with log_path .open ("a" ) as log :
154+ log .write ("Starting native server directly: " + json .dumps (argv ) + "\n " )
155+ log .flush ()
130156 process = subprocess .Popen (
131- command ( root , profile_path , "serve" ) ,
157+ argv ,
132158 stdout = log ,
133159 stderr = subprocess .STDOUT ,
134160 env = environment ,
135161 )
136162 stop_guard = threading .Event ()
137163 guard = None
164+ ready = False
138165 if cpu and profile .get ("resources" , {}).get ("cpu_guard" ):
139166 guard = threading .Thread (
140167 target = guard_cpu_process ,
@@ -143,9 +170,21 @@ def managed_server(
143170 )
144171 guard .start ()
145172 try :
173+ deadline = time .monotonic () + min (180 , profile ["runtime" ]["max_timeout" ])
174+ startup ("verifying-artifacts" , pid = process .pid )
175+ # The native child loads CUDA concurrently with verification. No second
176+ # controller CLI import, registry initialization or Python launcher.
177+ if process .poll () is not None :
178+ raise RuntimeError (f"Native server exited before verification; inspect { log_path } " )
179+ try :
180+ write_receipt (profile , root , pid = process .pid )
181+ except ProcessLookupError as error :
182+ raise RuntimeError (
183+ f"Native server exited during verification; inspect { log_path } "
184+ ) from error
185+ startup ("waiting-for-health" , pid = process .pid )
146186 if cpu :
147187 os .setpriority (os .PRIO_PROCESS , process .pid , 10 )
148- deadline = time .monotonic () + profile ["runtime" ]["max_timeout" ]
149188 with requests .Session () as session :
150189 session .trust_env = False
151190 while True :
@@ -165,7 +204,12 @@ def managed_server(
165204 if response .status_code == 200 :
166205 break
167206 time .sleep (0.25 )
207+ startup ("ready" , pid = process .pid )
208+ ready = True
168209 yield profile
210+ except Exception as error :
211+ startup ("failed" , pid = process .pid , error = str (error )[:500 ])
212+ raise
169213 finally :
170214 stop_guard .set ()
171215 if guard is not None :
@@ -177,6 +221,8 @@ def managed_server(
177221 except subprocess .TimeoutExpired :
178222 process .kill ()
179223 process .wait ()
224+ if ready :
225+ startup ("stopped" , pid = process .pid )
180226
181227
182228@contextmanager
0 commit comments