Skip to content

Commit c8bb281

Browse files
committed
test: generalize integration polling helpers
Make call_with_exp_backoff and poll_until_condition generic and consistent so the same two helpers can be shared across apify-sdk-python, crawlee-python and apify-client-python: - call_with_exp_backoff no longer takes rq_access_mode. It now takes an optional condition (default: truthy) and retries fn until the condition holds, returning the last result. The Apify RQ single/shared distinction moves to the call sites via max_retries (0 = call once / single mode, 5 = retry / shared mode). - poll_until_condition gains the same condition default and now accepts both sync and async callables (so it can poll plain values too). - Both helpers accept sync or async fn via a small _maybe_await helper, typed with overloads so the return type is preserved. All ~36 call sites in test_request_queue.py are refactored accordingly.
1 parent f26a010 commit c8bb281

2 files changed

Lines changed: 139 additions & 75 deletions

File tree

‎tests/integration/_utils.py‎

Lines changed: 78 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,77 +1,120 @@
11
from __future__ import annotations
22

33
import asyncio
4+
import inspect
5+
import logging
46
import time
5-
from typing import TYPE_CHECKING, Literal, TypeVar
7+
from typing import TYPE_CHECKING, TypeVar, cast, overload
68

79
from crawlee._utils.crypto import crypto_random_object_id
810

9-
from apify import Actor
10-
1111
if TYPE_CHECKING:
1212
from collections.abc import Awaitable, Callable
1313

14-
T = TypeVar('T')
14+
logger = logging.getLogger(__name__)
1515

16+
T = TypeVar('T')
1617

17-
async def call_with_exp_backoff(
18-
fn: Callable[[], Awaitable[T]],
19-
*,
20-
rq_access_mode: Literal['single', 'shared'],
21-
max_retries: int = 5,
22-
) -> T | None:
23-
"""Call an async callable with exponential backoff retries until it returns a truthy value.
2418

25-
In shared request queue mode, there is a propagation delay before newly added, reclaimed, or handled requests
26-
become visible in the API (see https://github.com/apify/apify-sdk-python/issues/808). This helper retries with
27-
exponential backoff to handle that delay in integration tests.
19+
async def _maybe_await(value: Awaitable[T] | T) -> T:
20+
"""Await `value` if it is awaitable, otherwise return it unchanged.
2821
29-
When `rq_access_mode` is `'single'`, the function is called once without retries.
22+
Lets `call_with_exp_backoff` and `poll_until_condition` accept both sync and async callables.
3023
"""
31-
if rq_access_mode == 'single':
32-
return await fn()
33-
34-
if rq_access_mode == 'shared':
35-
result = None
24+
if inspect.isawaitable(value):
25+
return await cast('Awaitable[T]', value)
26+
return cast('T', value)
3627

37-
for attempt in range(max_retries):
38-
result = await fn()
3928

40-
if result:
41-
return result
29+
@overload
30+
async def call_with_exp_backoff(
31+
fn: Callable[[], Awaitable[T]],
32+
condition: Callable[[T], bool] = ...,
33+
*,
34+
max_retries: int = ...,
35+
base_delay: float = ...,
36+
) -> T: ...
37+
@overload
38+
async def call_with_exp_backoff(
39+
fn: Callable[[], T],
40+
condition: Callable[[T], bool] = ...,
41+
*,
42+
max_retries: int = ...,
43+
base_delay: float = ...,
44+
) -> T: ...
45+
async def call_with_exp_backoff(
46+
fn: Callable[[], Awaitable[T] | T],
47+
condition: Callable[[T], bool] = bool,
48+
*,
49+
max_retries: int = 5,
50+
base_delay: float = 1.0,
51+
) -> T:
52+
"""Call `fn`, retrying with exponential backoff until `condition(result)` is True.
4253
43-
delay = 2**attempt
44-
Actor.log.info(f'{fn} returned {result!r}, retrying in {delay}s (attempt {attempt + 1}/{max_retries})')
45-
await asyncio.sleep(delay)
54+
Calls `fn` and checks whether `condition` holds for its result. If it does not, `fn` is retried up to
55+
`max_retries` times, sleeping `base_delay * 2 ** attempt` seconds before each retry. The last result is
56+
returned regardless of whether the condition was ever satisfied, so the caller can run its own assertion.
4657
47-
return result
58+
This is useful for eventually-consistent APIs where a freshly added, reclaimed, or handled item may take a
59+
moment to become visible (see https://github.com/apify/apify-sdk-python/issues/808). The default condition
60+
checks for a truthy result. Pass `max_retries=0` to call `fn` exactly once without any retries.
4861
49-
raise ValueError(f'Invalid rq_access_mode: {rq_access_mode}')
62+
Unlike `poll_until_condition`, the delay between attempts grows exponentially rather than staying constant.
63+
"""
64+
result = await _maybe_await(fn())
65+
for attempt in range(max_retries):
66+
if condition(result):
67+
return result
68+
delay = base_delay * 2**attempt
69+
logger.info(
70+
'Condition not met for %r, retrying in %ss (attempt %d/%d).', result, delay, attempt + 1, max_retries
71+
)
72+
await asyncio.sleep(delay)
73+
result = await _maybe_await(fn())
74+
return result
5075

5176

77+
@overload
5278
async def poll_until_condition(
5379
fn: Callable[[], Awaitable[T]],
54-
condition: Callable[[T], bool],
80+
condition: Callable[[T], bool] = ...,
81+
*,
82+
timeout: float = ...,
83+
poll_interval: float = ...,
84+
) -> T: ...
85+
@overload
86+
async def poll_until_condition(
87+
fn: Callable[[], T],
88+
condition: Callable[[T], bool] = ...,
89+
*,
90+
timeout: float = ...,
91+
poll_interval: float = ...,
92+
) -> T: ...
93+
async def poll_until_condition(
94+
fn: Callable[[], Awaitable[T] | T],
95+
condition: Callable[[T], bool] = bool,
5596
*,
5697
timeout: float = 60,
5798
poll_interval: float = 5,
5899
) -> T:
59100
"""Poll `fn` until `condition(result)` is True or the timeout expires.
60101
61102
Polls `fn` at `poll_interval`-second intervals until `condition` is satisfied or `timeout` seconds have elapsed.
62-
Returns the last polled result regardless of whether the condition was met.
103+
Returns the last polled result regardless of whether the condition was met, so the caller can run its own
104+
assertion. The default condition checks for a truthy result.
63105
64-
Use this instead of a fixed `asyncio.sleep` when waiting for eventually-consistent API state (e.g. request queue
65-
stats) that may take a variable amount of time to propagate.
106+
Use this instead of a fixed `asyncio.sleep` when waiting for eventually-consistent state (e.g. request queue
107+
stats) that may take a variable amount of time to propagate. Unlike `call_with_exp_backoff`, the interval
108+
between polls stays constant.
66109
"""
67110
deadline = time.monotonic() + timeout
68-
result = await fn()
111+
result = await _maybe_await(fn())
69112
while not condition(result):
70113
remaining = deadline - time.monotonic()
71114
if remaining <= 0:
72115
break
73116
await asyncio.sleep(min(poll_interval, remaining))
74-
result = await fn()
117+
result = await _maybe_await(fn())
75118
return result
76119

77120

0 commit comments

Comments
 (0)