Skip to content

Commit 1952df5

Browse files
committed
test: unify polling helpers with apify-sdk-python
Add generic call_with_exp_backoff and poll_until_condition test helpers, matching the signatures shared with apify-sdk-python and apify-client-python: - poll_until_condition replaces wait_for_condition. It polls fn until an optional condition (default: truthy) holds, then returns the last result instead of raising, so call sites assert explicitly. It accepts both sync and async callables, typed with overloads. - call_with_exp_backoff is added for parity (same retry-until-condition helper as the sibling repos). The five wait_for_condition call sites in test_autoscaled_pool.py are migrated to `assert await poll_until_condition(...)`.
1 parent 6a9f6f4 commit 1952df5

2 files changed

Lines changed: 110 additions & 24 deletions

File tree

‎tests/unit/_autoscaling/test_autoscaled_pool.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from crawlee._autoscaling._types import LoadRatioInfo, SystemInfo
1616
from crawlee._types import ConcurrencySettings
1717
from crawlee._utils.time import measure_time
18-
from tests.unit.utils import wait_for_condition
18+
from tests.unit.utils import poll_until_condition
1919

2020
if TYPE_CHECKING:
2121
from collections.abc import Awaitable
@@ -192,20 +192,20 @@ def get_historical_system_info() -> SystemInfo:
192192

193193
try:
194194
# Wait until concurrency scales up above 1.
195-
await wait_for_condition(lambda: pool.desired_concurrency > 1, timeout=5.0)
195+
assert await poll_until_condition(lambda: pool.desired_concurrency > 1, timeout=5.0)
196196

197197
# Wait until concurrency reaches maximum.
198-
await wait_for_condition(lambda: pool.desired_concurrency == 4, timeout=5.0)
198+
assert await poll_until_condition(lambda: pool.desired_concurrency == 4, timeout=5.0)
199199

200200
# Multiple concurrent workers should have completed more tasks than a single worker could.
201-
await wait_for_condition(lambda: done_count > 10, timeout=5.0)
201+
assert await poll_until_condition(lambda: done_count > 10, timeout=5.0)
202202

203203
# Simulate CPU overload and wait for the pool to scale down.
204204
overload_active = True
205-
await wait_for_condition(lambda: pool.desired_concurrency < 4, timeout=5.0)
205+
assert await poll_until_condition(lambda: pool.desired_concurrency < 4, timeout=5.0)
206206

207207
# Wait until the pool scales all the way down to minimum.
208-
await wait_for_condition(lambda: pool.desired_concurrency == 1, timeout=5.0)
208+
assert await poll_until_condition(lambda: pool.desired_concurrency == 1, timeout=5.0)
209209
finally:
210210
pool_run_task.cancel()
211211
with suppress(asyncio.CancelledError):

‎tests/unit/utils.py‎

Lines changed: 104 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,120 @@
11
from __future__ import annotations
22

33
import asyncio
4+
import inspect
5+
import logging
46
import sys
5-
from typing import TYPE_CHECKING
7+
import time
8+
from typing import TYPE_CHECKING, TypeVar, cast, overload
69

710
import pytest
811

912
if TYPE_CHECKING:
10-
from collections.abc import Callable
13+
from collections.abc import Awaitable, Callable
14+
15+
logger = logging.getLogger(__name__)
16+
17+
T = TypeVar('T')
1118

1219
run_alone_on_mac = pytest.mark.run_alone if sys.platform == 'darwin' else lambda x: x
1320

1421

15-
async def wait_for_condition(
16-
condition: Callable[[], bool],
22+
async def _maybe_await(value: Awaitable[T] | T) -> T:
23+
"""Await `value` if it is awaitable, otherwise return it unchanged.
24+
25+
Lets `call_with_exp_backoff` and `poll_until_condition` accept both sync and async callables.
26+
"""
27+
if inspect.isawaitable(value):
28+
return await cast('Awaitable[T]', value)
29+
return cast('T', value)
30+
31+
32+
@overload
33+
async def call_with_exp_backoff(
34+
fn: Callable[[], Awaitable[T]],
35+
condition: Callable[[T], bool] = ...,
36+
*,
37+
max_retries: int = ...,
38+
base_delay: float = ...,
39+
) -> T: ...
40+
@overload
41+
async def call_with_exp_backoff(
42+
fn: Callable[[], T],
43+
condition: Callable[[T], bool] = ...,
44+
*,
45+
max_retries: int = ...,
46+
base_delay: float = ...,
47+
) -> T: ...
48+
async def call_with_exp_backoff(
49+
fn: Callable[[], Awaitable[T] | T],
50+
condition: Callable[[T], bool] = bool,
51+
*,
52+
max_retries: int = 5,
53+
base_delay: float = 1.0,
54+
) -> T:
55+
"""Call `fn`, retrying with exponential backoff until `condition(result)` is True.
56+
57+
Calls `fn` and checks whether `condition` holds for its result. If it does not, `fn` is retried up to
58+
`max_retries` times, sleeping `base_delay * 2 ** attempt` seconds before each retry. The last result is
59+
returned regardless of whether the condition was ever satisfied, so the caller can run its own assertion.
60+
61+
This is useful for eventually-consistent state where the expected value may take a moment to appear. The
62+
default condition checks for a truthy result. Pass `max_retries=0` to call `fn` exactly once without retries.
63+
64+
Unlike `poll_until_condition`, the delay between attempts grows exponentially rather than staying constant.
65+
"""
66+
result = await _maybe_await(fn())
67+
for attempt in range(max_retries):
68+
if condition(result):
69+
return result
70+
delay = base_delay * 2**attempt
71+
logger.info(
72+
'Condition not met for %r, retrying in %ss (attempt %d/%d).', result, delay, attempt + 1, max_retries
73+
)
74+
await asyncio.sleep(delay)
75+
result = await _maybe_await(fn())
76+
return result
77+
78+
79+
@overload
80+
async def poll_until_condition(
81+
fn: Callable[[], Awaitable[T]],
82+
condition: Callable[[T], bool] = ...,
83+
*,
84+
timeout: float = ...,
85+
poll_interval: float = ...,
86+
) -> T: ...
87+
@overload
88+
async def poll_until_condition(
89+
fn: Callable[[], T],
90+
condition: Callable[[T], bool] = ...,
1791
*,
18-
timeout: float = 5.0,
92+
timeout: float = ...,
93+
poll_interval: float = ...,
94+
) -> T: ...
95+
async def poll_until_condition(
96+
fn: Callable[[], Awaitable[T] | T],
97+
condition: Callable[[T], bool] = bool,
98+
*,
99+
timeout: float = 5,
19100
poll_interval: float = 0.05,
20-
) -> None:
21-
"""Poll `condition` until it returns True, or raise `AssertionError` on timeout.
101+
) -> T:
102+
"""Poll `fn` until `condition(result)` is True or the timeout expires.
103+
104+
Polls `fn` at `poll_interval`-second intervals until `condition` is satisfied or `timeout` seconds have elapsed.
105+
Returns the last polled result regardless of whether the condition was met, so the caller can run its own
106+
assertion. The default condition checks for a truthy result.
22107
23-
Args:
24-
condition: A callable that returns True when the desired state is reached.
25-
timeout: Maximum time in seconds to wait before raising.
26-
poll_interval: Time in seconds between condition checks.
108+
Use this instead of a fixed `asyncio.sleep` when waiting for some state to settle (e.g. autoscaling
109+
concurrency) that may take a variable amount of time. Unlike `call_with_exp_backoff`, the interval between
110+
polls stays constant.
27111
"""
28-
loop = asyncio.get_running_loop()
29-
deadline = loop.time() + timeout
30-
while loop.time() < deadline:
31-
if condition():
32-
return
33-
await asyncio.sleep(poll_interval)
34-
raise AssertionError(f'Condition not met within {timeout}s')
112+
deadline = time.monotonic() + timeout
113+
result = await _maybe_await(fn())
114+
while not condition(result):
115+
remaining = deadline - time.monotonic()
116+
if remaining <= 0:
117+
break
118+
await asyncio.sleep(min(poll_interval, remaining))
119+
result = await _maybe_await(fn())
120+
return result

0 commit comments

Comments
 (0)