Skip to content

Commit a370e7a

Browse files
authored
fix: redirect the run log when the first log stream comes back empty (#1084)
Python twin of apify/apify-client-js#1105. `call(logger=...)` and `get_streamed_log()` sometimes redirected no log lines. Before a run writes its first line, the API answers `GET /logs/:id?stream=1` with an empty 200 that closes at once, and `StreamedLog` / `StreamedLogAsync` took that as the whole log. Both now reopen an empty stream after a pause that starts at 0.5 s and doubles up to 5 s. If `stop()` comes before anything has arrived, they read the log once with `get_as_bytes()`. `start()` resets that state, so a restarted instance behaves the same. Tests cover the same cases as #1105 for both classes and both HTTP clients, plus the backoff and a restart. *✍️ Drafted by Claude Code*
1 parent e3ace1c commit a370e7a

2 files changed

Lines changed: 436 additions & 45 deletions

File tree

‎src/apify_client/_streamed_log.py‎

Lines changed: 105 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -32,15 +32,29 @@ class StreamedLogBase:
3232
duration of the run (Impit currently maps it to an effective 24-hour cap) and mirrors the JS client.
3333
"""
3434

35+
_empty_stream_retry_s: ClassVar[float] = 0.5
36+
"""Pause before the first reopen of a log stream that ended before the run logged anything.
37+
38+
The API serves the log of a run that has not logged anything yet as an empty stream that ends at once. Each further
39+
reopen doubles the pause up to `_empty_stream_max_retry_s`, which bounds the request rate while a run waits long to
40+
start, for example for free memory.
41+
"""
42+
43+
_empty_stream_max_retry_s: ClassVar[float] = 5
44+
"""Upper bound on the pause between reopens of an empty log stream."""
45+
3546
def __init__(self, to_logger: logging.Logger, *, from_start: bool = True) -> None:
3647
if self._force_propagate:
3748
to_logger.propagate = True
3849
self._to_logger = to_logger
3950
self._stream_buffer = list[bytes]()
4051
self._split_marker = re.compile(rb'(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)')
4152
self._relevancy_time_limit: datetime | None = None if from_start else datetime.now(tz=UTC)
53+
self._received_data = False
4254

4355
def _process_new_data(self, data: bytes) -> None:
56+
if data:
57+
self._received_data = True
4458
new_chunk = data
4559
self._stream_buffer.append(new_chunk)
4660
if re.findall(self._split_marker, new_chunk):
@@ -75,6 +89,12 @@ def _log_buffer_content(self, *, include_last_part: bool = False) -> None:
7589
message = decoded_marker + decoded_content
7690
self._to_logger.log(level=self._guess_log_level_from_message(message), msg=message.strip())
7791

92+
def _process_whole_log(self, log: bytes | None) -> None:
93+
"""Redirect a whole log read in one request, including its last part."""
94+
if log:
95+
self._process_new_data(log)
96+
self._log_buffer_content(include_last_part=True)
97+
7898
@staticmethod
7999
def _guess_log_level_from_message(message: str) -> int:
80100
"""Guess the log level from the message."""
@@ -120,7 +140,7 @@ def __init__(self, log_client: LogClient, *, to_logger: logging.Logger, from_sta
120140
self._log_client = log_client
121141
self._streaming_thread: Thread | None = None
122142
self._log_stream: HttpResponse | None = None
123-
self._stop_logging = False
143+
self._stop_event = threading.Event()
124144

125145
def start(self) -> Thread:
126146
"""Start the streaming thread.
@@ -129,7 +149,8 @@ def start(self) -> Thread:
129149
"""
130150
if self._streaming_thread and self._streaming_thread.is_alive():
131151
raise RuntimeError('Streaming thread already active')
132-
self._stop_logging = False
152+
self._stop_event.clear()
153+
self._received_data = False
133154
# A daemon thread so a stream still blocked on a read can never hold up interpreter shutdown.
134155
self._streaming_thread = threading.Thread(target=self._stream_log, daemon=True)
135156
self._streaming_thread.start()
@@ -138,13 +159,14 @@ def start(self) -> Thread:
138159
def stop(self) -> None:
139160
"""Signal the streaming thread to stop logging and wait up to `_stop_timeout_s` for it to finish.
140161
141-
A thread that outlives the wait is a daemon with `_stop_logging` set, so it exits after at most one more chunk,
162+
A thread that outlives the wait is a daemon with `_stop_event` set, so it exits after at most one more chunk,
142163
and only then does its buffered tail reach the logger. Its handle is kept while it is alive, so `start` cannot
143-
revive it beside a second thread on the same buffer.
164+
revive it beside a second thread on the same buffer. If no stream has delivered anything yet, the thread reads
165+
the whole log in one request before it ends.
144166
"""
145167
if not self._streaming_thread:
146168
raise RuntimeError('Streaming thread is not active')
147-
self._stop_logging = True
169+
self._stop_event.set()
148170
# Read once; the streaming thread clears the attribute as soon as the stream ends.
149171
log_stream = self._log_stream
150172
if log_stream is not None:
@@ -173,39 +195,60 @@ def __exit__(
173195

174196
def _stream_log(self) -> None:
175197
try:
176-
with self._log_client.stream(raw=True, timeout=self._stream_timeout) as log_stream:
177-
if not log_stream:
198+
# An empty stream means the run has not logged anything yet, so reopen it until the first bytes arrive.
199+
retry_s = self._empty_stream_retry_s
200+
while not self._stop_event.is_set():
201+
if not self._stream_log_once() or self._received_data:
178202
return
179-
# Published so `stop` can close the response.
180-
self._log_stream = log_stream
181-
try:
182-
# `stop` may have run before the response existed for it to close.
183-
if self._stop_logging:
184-
return
185-
for data in log_stream.iter_bytes():
186-
self._process_new_data(data)
187-
if self._stop_logging:
188-
break
189-
finally:
190-
self._log_stream = None
191-
try:
192-
# Flush the last buffered part even if the read timed out or was stopped.
193-
self._log_buffer_content(include_last_part=True)
194-
except Exception:
195-
# A truncated stream leaves an undecodable tail, which is worth a traceback even while a stop
196-
# is in progress.
197-
self._to_logger.exception('Log redirection stopped due to unexpected error:')
203+
self._stop_event.wait(retry_s)
204+
retry_s = min(retry_s * 2, self._empty_stream_max_retry_s)
198205
except Exception as exc:
199-
if self._stop_logging:
206+
if self._stop_event.is_set():
200207
# `stop` closed the stream out from under the read, so the failure is expected.
201208
self._to_logger.debug('Log streaming stopped while `stop` was in progress: %r', exc)
202-
return
203-
if self._log_client._http_client.is_timeout_error(exc): # noqa: SLF001
209+
elif self._log_client._http_client.is_timeout_error(exc): # noqa: SLF001
204210
# The stream cannot continue, so warn and let the thread end instead of leaking a traceback.
205211
self._to_logger.warning('Log streaming stopped: the log stream request timed out.')
212+
return
206213
else:
207214
# Any other failure in log redirection must not escape the background thread; log it instead.
208215
self._to_logger.exception('Log redirection stopped due to unexpected error:')
216+
return
217+
if self._received_data:
218+
return
219+
# Stopped before any stream delivered a byte, which a run that finishes quickly can cause.
220+
try:
221+
self._process_whole_log(self._log_client.get_as_bytes(raw=True))
222+
except Exception:
223+
self._to_logger.exception('Log redirection stopped due to unexpected error:')
224+
225+
def _stream_log_once(self) -> bool:
226+
"""Redirect one log stream until it ends or `stop` is called. Return `False` when the log does not exist."""
227+
with self._log_client.stream(raw=True, timeout=self._stream_timeout) as log_stream:
228+
if not log_stream:
229+
return False
230+
# Published so `stop` can close the response.
231+
self._log_stream = log_stream
232+
try:
233+
# `stop` may have run before the response existed for it to close. No earlier stream delivered
234+
# anything, and one opened this late would end after its first chunk, so read the whole log in one
235+
# request.
236+
if self._stop_event.is_set():
237+
return True
238+
for data in log_stream.iter_bytes():
239+
self._process_new_data(data)
240+
if self._stop_event.is_set():
241+
break
242+
finally:
243+
self._log_stream = None
244+
try:
245+
# Flush the last buffered part even if the read timed out or was stopped.
246+
self._log_buffer_content(include_last_part=True)
247+
except Exception:
248+
# A truncated stream leaves an undecodable tail, which is worth a traceback even while a stop is
249+
# in progress.
250+
self._to_logger.exception('Log redirection stopped due to unexpected error:')
251+
return True
209252

210253

211254
@docs_group('Other')
@@ -240,21 +283,33 @@ def start(self) -> Task:
240283
"""
241284
if self._streaming_task and not self._streaming_task.done():
242285
raise RuntimeError('Streaming task already active')
286+
self._received_data = False
243287
self._streaming_task = asyncio.create_task(self._stream_log())
244288
return self._streaming_task
245289

246290
async def stop(self) -> None:
247-
"""Stop the streaming task."""
291+
"""Stop the streaming task.
292+
293+
If no stream has delivered anything yet, read the whole log in one request instead.
294+
"""
248295
if not self._streaming_task:
249296
raise RuntimeError('Streaming task is not active')
250297

298+
was_streaming = not self._streaming_task.done()
251299
self._streaming_task.cancel()
252300
try:
253301
await self._streaming_task
254302
except asyncio.CancelledError:
255303
pass
256304
finally:
257305
self._streaming_task = None
306+
if not was_streaming or self._received_data:
307+
return
308+
# Stopped before any stream delivered a byte, which a run that finishes quickly can cause.
309+
try:
310+
self._process_whole_log(await self._log_client.get_as_bytes(raw=True))
311+
except Exception:
312+
self._to_logger.exception('Log redirection stopped due to unexpected error:')
258313

259314
async def __aenter__(self) -> Self:
260315
"""Start the streaming task within the context. Exiting the context will cancel the streaming task."""
@@ -269,20 +324,27 @@ async def __aexit__(
269324

270325
async def _stream_log(self) -> None:
271326
try:
272-
async with self._log_client.stream(raw=True, timeout=self._stream_timeout) as log_stream:
273-
if not log_stream:
274-
return
275-
try:
276-
async for data in log_stream.aiter_bytes():
277-
self._process_new_data(data)
278-
finally:
327+
# An empty stream means the run has not logged anything yet, so reopen it until the first bytes arrive.
328+
retry_s = self._empty_stream_retry_s
329+
while True:
330+
async with self._log_client.stream(raw=True, timeout=self._stream_timeout) as log_stream:
331+
if not log_stream:
332+
return
279333
try:
280-
# Flush the last buffered part even if the task is cancelled by `stop()`.
281-
self._log_buffer_content(include_last_part=True)
282-
except Exception:
283-
# A truncated stream leaves an undecodable tail. Keeping the failure here also keeps the
284-
# cancellation `stop` raised propagating, so the task ends up cancelled as asyncio expects.
285-
self._to_logger.exception('Log redirection stopped due to unexpected error:')
334+
async for data in log_stream.aiter_bytes():
335+
self._process_new_data(data)
336+
finally:
337+
try:
338+
# Flush the last buffered part even if the task is cancelled by `stop()`.
339+
self._log_buffer_content(include_last_part=True)
340+
except Exception:
341+
# A truncated stream leaves an undecodable tail. Keeping the failure here also keeps the
342+
# cancellation `stop` raised propagating, so the task ends up cancelled as asyncio expects.
343+
self._to_logger.exception('Log redirection stopped due to unexpected error:')
344+
if self._received_data:
345+
return
346+
await asyncio.sleep(retry_s)
347+
retry_s = min(retry_s * 2, self._empty_stream_max_retry_s)
286348
except Exception as exc:
287349
if self._log_client._http_client.is_timeout_error(exc): # noqa: SLF001
288350
# A timeout on the long-lived stream is an expected terminal condition, not an error.

0 commit comments

Comments
 (0)