@@ -32,15 +32,29 @@ class StreamedLogBase:
3232 duration of the run (Impit currently maps it to an effective 24-hour cap) and mirrors the JS client.
3333 """
3434
35+ _empty_stream_retry_s : ClassVar [float ] = 0.5
36+ """Pause before the first reopen of a log stream that ended before the run logged anything.
37+
38+ The API serves the log of a run that has not logged anything yet as an empty stream that ends at once. Each further
39+ reopen doubles the pause up to `_empty_stream_max_retry_s`, which bounds the request rate while a run waits long to
40+ start, for example for free memory.
41+ """
42+
43+ _empty_stream_max_retry_s : ClassVar [float ] = 5
44+ """Upper bound on the pause between reopens of an empty log stream."""
45+
3546 def __init__ (self , to_logger : logging .Logger , * , from_start : bool = True ) -> None :
3647 if self ._force_propagate :
3748 to_logger .propagate = True
3849 self ._to_logger = to_logger
3950 self ._stream_buffer = list [bytes ]()
4051 self ._split_marker = re .compile (rb'(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)' )
4152 self ._relevancy_time_limit : datetime | None = None if from_start else datetime .now (tz = UTC )
53+ self ._received_data = False
4254
4355 def _process_new_data (self , data : bytes ) -> None :
56+ if data :
57+ self ._received_data = True
4458 new_chunk = data
4559 self ._stream_buffer .append (new_chunk )
4660 if re .findall (self ._split_marker , new_chunk ):
@@ -75,6 +89,12 @@ def _log_buffer_content(self, *, include_last_part: bool = False) -> None:
7589 message = decoded_marker + decoded_content
7690 self ._to_logger .log (level = self ._guess_log_level_from_message (message ), msg = message .strip ())
7791
92+ def _process_whole_log (self , log : bytes | None ) -> None :
93+ """Redirect a whole log read in one request, including its last part."""
94+ if log :
95+ self ._process_new_data (log )
96+ self ._log_buffer_content (include_last_part = True )
97+
7898 @staticmethod
7999 def _guess_log_level_from_message (message : str ) -> int :
80100 """Guess the log level from the message."""
@@ -120,7 +140,7 @@ def __init__(self, log_client: LogClient, *, to_logger: logging.Logger, from_sta
120140 self ._log_client = log_client
121141 self ._streaming_thread : Thread | None = None
122142 self ._log_stream : HttpResponse | None = None
123- self ._stop_logging = False
143+ self ._stop_event = threading . Event ()
124144
125145 def start (self ) -> Thread :
126146 """Start the streaming thread.
@@ -129,7 +149,8 @@ def start(self) -> Thread:
129149 """
130150 if self ._streaming_thread and self ._streaming_thread .is_alive ():
131151 raise RuntimeError ('Streaming thread already active' )
132- self ._stop_logging = False
152+ self ._stop_event .clear ()
153+ self ._received_data = False
133154 # A daemon thread so a stream still blocked on a read can never hold up interpreter shutdown.
134155 self ._streaming_thread = threading .Thread (target = self ._stream_log , daemon = True )
135156 self ._streaming_thread .start ()
@@ -138,13 +159,14 @@ def start(self) -> Thread:
138159 def stop (self ) -> None :
139160 """Signal the streaming thread to stop logging and wait up to `_stop_timeout_s` for it to finish.
140161
141- A thread that outlives the wait is a daemon with `_stop_logging ` set, so it exits after at most one more chunk,
162+ A thread that outlives the wait is a daemon with `_stop_event ` set, so it exits after at most one more chunk,
142163 and only then does its buffered tail reach the logger. Its handle is kept while it is alive, so `start` cannot
143- revive it beside a second thread on the same buffer.
164+ revive it beside a second thread on the same buffer. If no stream has delivered anything yet, the thread reads
165+ the whole log in one request before it ends.
144166 """
145167 if not self ._streaming_thread :
146168 raise RuntimeError ('Streaming thread is not active' )
147- self ._stop_logging = True
169+ self ._stop_event . set ()
148170 # Read once; the streaming thread clears the attribute as soon as the stream ends.
149171 log_stream = self ._log_stream
150172 if log_stream is not None :
@@ -173,39 +195,60 @@ def __exit__(
173195
174196 def _stream_log (self ) -> None :
175197 try :
176- with self ._log_client .stream (raw = True , timeout = self ._stream_timeout ) as log_stream :
177- if not log_stream :
198+ # An empty stream means the run has not logged anything yet, so reopen it until the first bytes arrive.
199+ retry_s = self ._empty_stream_retry_s
200+ while not self ._stop_event .is_set ():
201+ if not self ._stream_log_once () or self ._received_data :
178202 return
179- # Published so `stop` can close the response.
180- self ._log_stream = log_stream
181- try :
182- # `stop` may have run before the response existed for it to close.
183- if self ._stop_logging :
184- return
185- for data in log_stream .iter_bytes ():
186- self ._process_new_data (data )
187- if self ._stop_logging :
188- break
189- finally :
190- self ._log_stream = None
191- try :
192- # Flush the last buffered part even if the read timed out or was stopped.
193- self ._log_buffer_content (include_last_part = True )
194- except Exception :
195- # A truncated stream leaves an undecodable tail, which is worth a traceback even while a stop
196- # is in progress.
197- self ._to_logger .exception ('Log redirection stopped due to unexpected error:' )
203+ self ._stop_event .wait (retry_s )
204+ retry_s = min (retry_s * 2 , self ._empty_stream_max_retry_s )
198205 except Exception as exc :
199- if self ._stop_logging :
206+ if self ._stop_event . is_set () :
200207 # `stop` closed the stream out from under the read, so the failure is expected.
201208 self ._to_logger .debug ('Log streaming stopped while `stop` was in progress: %r' , exc )
202- return
203- if self ._log_client ._http_client .is_timeout_error (exc ): # noqa: SLF001
209+ elif self ._log_client ._http_client .is_timeout_error (exc ): # noqa: SLF001
204210 # The stream cannot continue, so warn and let the thread end instead of leaking a traceback.
205211 self ._to_logger .warning ('Log streaming stopped: the log stream request timed out.' )
212+ return
206213 else :
207214 # Any other failure in log redirection must not escape the background thread; log it instead.
208215 self ._to_logger .exception ('Log redirection stopped due to unexpected error:' )
216+ return
217+ if self ._received_data :
218+ return
219+ # Stopped before any stream delivered a byte, which a run that finishes quickly can cause.
220+ try :
221+ self ._process_whole_log (self ._log_client .get_as_bytes (raw = True ))
222+ except Exception :
223+ self ._to_logger .exception ('Log redirection stopped due to unexpected error:' )
224+
225+ def _stream_log_once (self ) -> bool :
226+ """Redirect one log stream until it ends or `stop` is called. Return `False` when the log does not exist."""
227+ with self ._log_client .stream (raw = True , timeout = self ._stream_timeout ) as log_stream :
228+ if not log_stream :
229+ return False
230+ # Published so `stop` can close the response.
231+ self ._log_stream = log_stream
232+ try :
233+ # `stop` may have run before the response existed for it to close. No earlier stream delivered
234+ # anything, and one opened this late would end after its first chunk, so read the whole log in one
235+ # request.
236+ if self ._stop_event .is_set ():
237+ return True
238+ for data in log_stream .iter_bytes ():
239+ self ._process_new_data (data )
240+ if self ._stop_event .is_set ():
241+ break
242+ finally :
243+ self ._log_stream = None
244+ try :
245+ # Flush the last buffered part even if the read timed out or was stopped.
246+ self ._log_buffer_content (include_last_part = True )
247+ except Exception :
248+ # A truncated stream leaves an undecodable tail, which is worth a traceback even while a stop is
249+ # in progress.
250+ self ._to_logger .exception ('Log redirection stopped due to unexpected error:' )
251+ return True
209252
210253
211254@docs_group ('Other' )
@@ -240,21 +283,33 @@ def start(self) -> Task:
240283 """
241284 if self ._streaming_task and not self ._streaming_task .done ():
242285 raise RuntimeError ('Streaming task already active' )
286+ self ._received_data = False
243287 self ._streaming_task = asyncio .create_task (self ._stream_log ())
244288 return self ._streaming_task
245289
246290 async def stop (self ) -> None :
247- """Stop the streaming task."""
291+ """Stop the streaming task.
292+
293+ If no stream has delivered anything yet, read the whole log in one request instead.
294+ """
248295 if not self ._streaming_task :
249296 raise RuntimeError ('Streaming task is not active' )
250297
298+ was_streaming = not self ._streaming_task .done ()
251299 self ._streaming_task .cancel ()
252300 try :
253301 await self ._streaming_task
254302 except asyncio .CancelledError :
255303 pass
256304 finally :
257305 self ._streaming_task = None
306+ if not was_streaming or self ._received_data :
307+ return
308+ # Stopped before any stream delivered a byte, which a run that finishes quickly can cause.
309+ try :
310+ self ._process_whole_log (await self ._log_client .get_as_bytes (raw = True ))
311+ except Exception :
312+ self ._to_logger .exception ('Log redirection stopped due to unexpected error:' )
258313
259314 async def __aenter__ (self ) -> Self :
260315 """Start the streaming task within the context. Exiting the context will cancel the streaming task."""
@@ -269,20 +324,27 @@ async def __aexit__(
269324
270325 async def _stream_log (self ) -> None :
271326 try :
272- async with self ._log_client .stream (raw = True , timeout = self ._stream_timeout ) as log_stream :
273- if not log_stream :
274- return
275- try :
276- async for data in log_stream .aiter_bytes ():
277- self ._process_new_data (data )
278- finally :
327+ # An empty stream means the run has not logged anything yet, so reopen it until the first bytes arrive.
328+ retry_s = self ._empty_stream_retry_s
329+ while True :
330+ async with self ._log_client .stream (raw = True , timeout = self ._stream_timeout ) as log_stream :
331+ if not log_stream :
332+ return
279333 try :
280- # Flush the last buffered part even if the task is cancelled by `stop()`.
281- self ._log_buffer_content (include_last_part = True )
282- except Exception :
283- # A truncated stream leaves an undecodable tail. Keeping the failure here also keeps the
284- # cancellation `stop` raised propagating, so the task ends up cancelled as asyncio expects.
285- self ._to_logger .exception ('Log redirection stopped due to unexpected error:' )
334+ async for data in log_stream .aiter_bytes ():
335+ self ._process_new_data (data )
336+ finally :
337+ try :
338+ # Flush the last buffered part even if the task is cancelled by `stop()`.
339+ self ._log_buffer_content (include_last_part = True )
340+ except Exception :
341+ # A truncated stream leaves an undecodable tail. Keeping the failure here also keeps the
342+ # cancellation `stop` raised propagating, so the task ends up cancelled as asyncio expects.
343+ self ._to_logger .exception ('Log redirection stopped due to unexpected error:' )
344+ if self ._received_data :
345+ return
346+ await asyncio .sleep (retry_s )
347+ retry_s = min (retry_s * 2 , self ._empty_stream_max_retry_s )
286348 except Exception as exc :
287349 if self ._log_client ._http_client .is_timeout_error (exc ): # noqa: SLF001
288350 # A timeout on the long-lived stream is an expected terminal condition, not an error.
0 commit comments