Skip to content

Commit dba6aba

Browse files
authored
fix: Prevent losing a status message when redirecting Actor run logs (#1036)
### What `ActorClient.call` built its two log redirectors inside a single `with` statement: ```python with run_client.get_status_message_watcher(to_logger=logger), run_client.get_streamed_log(to_logger=logger): ``` `with A(), B():` enters `A` before constructing `B`, so the status watcher's polling thread was already running - and already logging into `apify.<actor> runId:<id>` - when `get_streamed_log` called `create_redirect_logger` on that same logger. That call reconfigures the logger from scratch (`propagate = False`, `removeHandler`, `addHandler`), so a status message emitted in that window went to a logger that was mid-rebuild and was dropped. Both redirectors are now constructed before either one starts polling, which is what the async `call` already did. ### Why This surfaced as a flaky unit test - [CI run 32853344220](https://github.com/apify/apify-client-python/actions/runs/32853344220/job/97819276512), `test_actor_call_redirect_logs_to_default_logger_sync` on `ubuntu-latest` / 3.13. `Status: RUNNING, Message: Initial message` appeared in captured stderr but not in `caplog`, i.e. the record reached the logger's own handler while `propagate` was `False`. Only the sync test ever flaked, because the async `call` constructs both redirectors up front and so has no such window. The nondeterminism is in the client, not in the test: under real usage the same window can silently drop a status message. ### Test The new test pins the damaging interleaving with two events instead of relying on thread scheduling, so it is deterministic in both directions - it fails on `master` and passes with this change, at the same runtime either way. Verification: the new test fails without the `actor.py` change and passes with it; full unit suite (911 passed), lint, type check and docstring check all pass. The original flake does not reproduce naturally on an 8-core machine (0 failures in 40 x `tests/unit/test_logging.py` under `--numprocesses=16` before the change), which is why the regression test forces the interleaving rather than hammering for it. *✍️ Drafted by Claude Code*
1 parent 9876382 commit dba6aba

2 files changed

Lines changed: 71 additions & 2 deletions

File tree

‎src/apify_client/_resource_clients/actor.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -371,7 +371,13 @@ def call(
371371
if logger == 'default':
372372
logger = None
373373

374-
with run_client.get_status_message_watcher(to_logger=logger), run_client.get_streamed_log(to_logger=logger):
374+
# With the default logger, each helper rebuilds the same redirect logger from scratch, so both must exist before
375+
# either starts polling; otherwise the streamed log reconfigures a logger the status watcher thread is already
376+
# writing into.
377+
status_redirector = run_client.get_status_message_watcher(to_logger=logger)
378+
streamed_log = run_client.get_streamed_log(to_logger=logger)
379+
380+
with status_redirector, streamed_log:
375381
return run_client.wait_for_finish(wait_duration=wait_duration)
376382

377383
def build(
@@ -868,6 +874,9 @@ async def call(
868874
if logger == 'default':
869875
logger = None
870876

877+
# With the default logger, each helper rebuilds the same redirect logger from scratch, so both must exist before
878+
# either starts polling; otherwise the streamed log reconfigures a logger the status watcher task is already
879+
# writing into.
871880
status_redirector = await run_client.get_status_message_watcher(to_logger=logger)
872881
streamed_log = await run_client.get_streamed_log(to_logger=logger)
873882

‎tests/unit/test_logging.py‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from __future__ import annotations
22

33
import asyncio
4+
import itertools
45
import json
56
import logging
67
import threading
@@ -14,7 +15,8 @@
1415

1516
from apify_client import ApifyClient, ApifyClientAsync
1617
from apify_client._logging import LoggerOnce, RedirectLogFormatter
17-
from apify_client._status_message_watcher import StatusMessageWatcherBase
18+
from apify_client._resource_clients import run as run_module
19+
from apify_client._status_message_watcher import StatusMessageWatcher, StatusMessageWatcherBase
1820
from apify_client._streamed_log import StreamedLog, StreamedLogAsync, StreamedLogBase
1921

2022
if TYPE_CHECKING:
@@ -24,6 +26,7 @@
2426
from pytest_httpserver import HTTPServer
2527

2628
from apify_client._literals import ActorJobStatus
29+
from apify_client._models import Run
2730
from apify_client.http_clients import HttpClient, HttpClientAsync
2831

2932
_MOCKED_RUN_ID = 'mocked_run_id'
@@ -372,6 +375,63 @@ def test_actor_call_redirect_logs_to_default_logger_sync(
372375
)
373376

374377

378+
@pytest.mark.usefixtures('mock_api', 'propagate_stream_logs', 'reduce_final_timeout_for_status_message_redirector')
379+
def test_actor_call_sync_does_not_reconfigure_logger_used_by_running_watcher(
380+
caplog: LogCaptureFixture,
381+
httpserver: HTTPServer,
382+
monkeypatch: pytest.MonkeyPatch,
383+
) -> None:
384+
"""No status message is lost when `call` builds the streamed log while the status watcher already runs."""
385+
# The events pin the damaging interleaving instead of relying on thread scheduling: the watcher holds its first
386+
# message until the logger has been rebuilt, and a rebuild that finds the watcher already running returns only
387+
# after that message has been logged - i.e. before `StreamedLog.__init__` re-enables propagation.
388+
watcher_started = threading.Event()
389+
logger_rebuilt = threading.Event()
390+
watcher_logged = threading.Event()
391+
392+
original_start = StatusMessageWatcher.start
393+
394+
def recording_start(self: StatusMessageWatcher) -> threading.Thread:
395+
thread = original_start(self)
396+
watcher_started.set()
397+
return thread
398+
399+
original_create_redirect_logger = run_module.create_redirect_logger
400+
create_calls = itertools.count(1)
401+
402+
def instrumented_create_redirect_logger(name: str) -> logging.Logger:
403+
to_logger = original_create_redirect_logger(name)
404+
if next(create_calls) >= 2:
405+
logger_rebuilt.set()
406+
if watcher_started.is_set():
407+
assert watcher_logged.wait(timeout=5)
408+
return to_logger
409+
410+
original_log_run_data = StatusMessageWatcherBase._log_run_data
411+
412+
def gated_log_run_data(self: StatusMessageWatcherBase, run_data: Run | None) -> bool:
413+
logger_rebuilt.wait(timeout=5)
414+
more_data_expected = original_log_run_data(self, run_data)
415+
watcher_logged.set()
416+
return more_data_expected
417+
418+
monkeypatch.setattr(StatusMessageWatcher, 'start', recording_start)
419+
monkeypatch.setattr(run_module, 'create_redirect_logger', instrumented_create_redirect_logger)
420+
monkeypatch.setattr(StatusMessageWatcherBase, '_log_run_data', gated_log_run_data)
421+
422+
api_url = httpserver.url_for('/').removesuffix('/')
423+
424+
logger_name = f'apify.{_MOCKED_ACTOR_NAME} runId:{_MOCKED_RUN_ID}'
425+
actor_client = ApifyClient(token='mocked_token', api_url=api_url).actor(actor_id=_MOCKED_ACTOR_ID)
426+
427+
with caplog.at_level(logging.DEBUG, logger=logger_name):
428+
actor_client.call()
429+
430+
assert ('Status: RUNNING, Message: Initial message', logging.INFO) in {
431+
(record.message, record.levelno) for record in caplog.records
432+
}
433+
434+
375435
@pytest.mark.usefixtures('mock_api', 'propagate_stream_logs')
376436
async def test_actor_call_no_redirect_logs_async(
377437
caplog: LogCaptureFixture,

0 commit comments

Comments
 (0)