Skip to content

Commit cbf3de4

Browse files
committed
test: add polling helpers for eventually-consistent integration tests
Add a wall-clock-deadline `poll_until_condition` helper, generalize `maybe_await` to any awaitable, and refactor `collect_iterate_until_present` to reuse a shared drain step.
1 parent d2ab2ae commit cbf3de4

1 file changed

Lines changed: 64 additions & 22 deletions

File tree

‎tests/integration/_utils.py‎

Lines changed: 64 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
from __future__ import annotations
22

33
import asyncio
4+
import inspect
45
import secrets
56
import string
67
import time
78
from collections.abc import AsyncIterator, Iterator
89
from dataclasses import dataclass
9-
from typing import TYPE_CHECKING, Any, Protocol, TypeVar, overload
10+
from typing import TYPE_CHECKING, Any, Protocol, TypeVar, cast, overload
1011

1112
import pytest
1213

1314
if TYPE_CHECKING:
14-
from collections.abc import Callable, Coroutine
15+
from collections.abc import Awaitable, Callable
1516

1617
# Environment variable names for test configuration
1718
TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN'
@@ -92,22 +93,14 @@ def get_random_resource_name(label: str) -> str:
9293
return name_template.format(label, get_crypto_random_object_id(random_id_length))
9394

9495

95-
@overload
96-
async def maybe_await(value: Coroutine[Any, Any, T]) -> T: ...
97-
98-
99-
@overload
100-
async def maybe_await(value: T) -> T: ...
101-
102-
103-
async def maybe_await(value: T | Coroutine[Any, Any, T]) -> T:
104-
"""Await coroutines, pass through other values.
96+
async def maybe_await(value: Awaitable[T] | T) -> T:
97+
"""Await `value` if it is awaitable, otherwise return it unchanged.
10598
10699
Enables unified test code for both sync and async clients:
107100
result = await maybe_await(client.datasets().list())
108101
"""
109-
if hasattr(value, '__await__'):
110-
return await value # ty: ignore[invalid-await]
102+
if inspect.isawaitable(value):
103+
return await cast('Awaitable[T]', value)
111104
return value
112105

113106

@@ -119,6 +112,49 @@ async def maybe_sleep(seconds: float, *, is_async: bool) -> None:
119112
time.sleep(seconds) # noqa: ASYNC251
120113

121114

115+
@overload
116+
async def poll_until_condition(
117+
fn: Callable[[], Awaitable[T]],
118+
condition: Callable[[T], bool] = ...,
119+
*,
120+
timeout: float = ...,
121+
poll_interval: float = ...,
122+
) -> T: ...
123+
@overload
124+
async def poll_until_condition(
125+
fn: Callable[[], T],
126+
condition: Callable[[T], bool] = ...,
127+
*,
128+
timeout: float = ...,
129+
poll_interval: float = ...,
130+
) -> T: ...
131+
async def poll_until_condition(
132+
fn: Callable[[], Awaitable[T] | T],
133+
condition: Callable[[T], bool] = bool,
134+
*,
135+
timeout: float = 5,
136+
poll_interval: float = 1,
137+
) -> T:
138+
"""Poll `fn` until `condition(result)` is True or the timeout expires.
139+
140+
Polls `fn` at `poll_interval`-second intervals until `condition` is satisfied or `timeout` seconds have elapsed.
141+
Returns the last polled result regardless of whether the condition was met, so the caller can run its own
142+
assertion. The default condition checks for a truthy result.
143+
144+
Use this instead of a fixed `asyncio.sleep` when waiting for eventually-consistent state (e.g. a freshly
145+
created resource appearing in a listing) that may take a variable amount of time to propagate.
146+
"""
147+
deadline = time.monotonic() + timeout
148+
result = await maybe_await(fn())
149+
while not condition(result):
150+
remaining = deadline - time.monotonic()
151+
if remaining <= 0:
152+
break
153+
await asyncio.sleep(min(poll_interval, remaining))
154+
result = await maybe_await(fn())
155+
return result
156+
157+
122158
async def collect_iterate_until_present(
123159
iterator_factory: Callable[[], Iterator[_HasIdT] | AsyncIterator[_HasIdT]],
124160
expected_ids: set[str],
@@ -132,7 +168,7 @@ async def collect_iterate_until_present(
132168
133169
Handles eventual consistency on listing endpoints: under parallel load a freshly
134170
created resource may not appear in the listing for a short window. Each attempt
135-
builds a fresh iterator via `iterator_factory`, drains it, and breaks early once
171+
builds a fresh iterator via `iterator_factory`, drains it, and stops early once
136172
`expected_ids` is a subset of the collected items' `.id` values. The most recent
137173
collection is returned regardless of whether the condition was met, so the caller
138174
can run its own assertion with a helpful failure message.
@@ -142,18 +178,16 @@ async def collect_iterate_until_present(
142178
expected_ids: IDs that must all appear in the collected items.
143179
item_type: Asserted to match the runtime type of each yielded item.
144180
is_async: Whether the iterator is async (and so are sleeps).
145-
max_attempts: Maximum number of polling rounds.
146-
interval: Seconds to sleep before each attempt.
181+
max_attempts: Maximum number of polling rounds, guaranteed regardless of how long each drain takes.
182+
interval: Seconds to sleep between attempts.
147183
148184
Returns:
149185
The most recently collected items.
150186
"""
151-
collected: list[_HasIdT] = []
152-
for attempt in range(max_attempts):
153-
if attempt > 0:
154-
await maybe_sleep(interval, is_async=is_async)
187+
188+
async def drain() -> list[_HasIdT]:
155189
iterator = iterator_factory()
156-
collected = []
190+
collected: list[_HasIdT] = []
157191
if is_async:
158192
assert isinstance(iterator, AsyncIterator)
159193
async for item in iterator:
@@ -164,8 +198,16 @@ async def collect_iterate_until_present(
164198
for item in iterator:
165199
assert isinstance(item, item_type)
166200
collected.append(item)
201+
return collected
202+
203+
# Loop on attempt count rather than a wall-clock deadline: drains take HTTP time, and charging it
204+
# against a deadline would mean fewer retries under load — exactly when they are needed most.
205+
collected = await drain()
206+
for _ in range(max_attempts - 1):
167207
if expected_ids.issubset(item.id for item in collected):
168208
break
209+
await maybe_sleep(interval, is_async=is_async)
210+
collected = await drain()
169211
return collected
170212

171213

0 commit comments

Comments
 (0)