11from __future__ import annotations
22
33import asyncio
4+ import inspect
45import secrets
56import string
67import time
78from collections .abc import AsyncIterator , Iterator
89from dataclasses import dataclass
9- from typing import TYPE_CHECKING , Any , Protocol , TypeVar , overload
10+ from typing import TYPE_CHECKING , Any , Protocol , TypeVar , cast , overload
1011
1112import pytest
1213
1314if TYPE_CHECKING :
14- from collections .abc import Callable , Coroutine
15+ from collections .abc import Awaitable , Callable
1516
1617# Environment variable names for test configuration
1718TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN'
@@ -92,22 +93,14 @@ def get_random_resource_name(label: str) -> str:
9293 return name_template .format (label , get_crypto_random_object_id (random_id_length ))
9394
9495
95- @overload
96- async def maybe_await (value : Coroutine [Any , Any , T ]) -> T : ...
97-
98-
99- @overload
100- async def maybe_await (value : T ) -> T : ...
101-
102-
103- async def maybe_await (value : T | Coroutine [Any , Any , T ]) -> T :
104- """Await coroutines, pass through other values.
96+ async def maybe_await (value : Awaitable [T ] | T ) -> T :
97+ """Await `value` if it is awaitable, otherwise return it unchanged.
10598
10699 Enables unified test code for both sync and async clients:
107100 result = await maybe_await(client.datasets().list())
108101 """
109- if hasattr (value , '__await__' ):
110- return await value # ty: ignore[invalid-await]
102+ if inspect . isawaitable (value ):
103+ return await cast ( 'Awaitable[T]' , value )
111104 return value
112105
113106
@@ -119,6 +112,49 @@ async def maybe_sleep(seconds: float, *, is_async: bool) -> None:
119112 time .sleep (seconds ) # noqa: ASYNC251
120113
121114
115+ @overload
116+ async def poll_until_condition (
117+ fn : Callable [[], Awaitable [T ]],
118+ condition : Callable [[T ], bool ] = ...,
119+ * ,
120+ timeout : float = ...,
121+ poll_interval : float = ...,
122+ ) -> T : ...
123+ @overload
124+ async def poll_until_condition (
125+ fn : Callable [[], T ],
126+ condition : Callable [[T ], bool ] = ...,
127+ * ,
128+ timeout : float = ...,
129+ poll_interval : float = ...,
130+ ) -> T : ...
131+ async def poll_until_condition (
132+ fn : Callable [[], Awaitable [T ] | T ],
133+ condition : Callable [[T ], bool ] = bool ,
134+ * ,
135+ timeout : float = 5 ,
136+ poll_interval : float = 1 ,
137+ ) -> T :
138+ """Poll `fn` until `condition(result)` is True or the timeout expires.
139+
140+ Polls `fn` at `poll_interval`-second intervals until `condition` is satisfied or `timeout` seconds have elapsed.
141+ Returns the last polled result regardless of whether the condition was met, so the caller can run its own
142+ assertion. The default condition checks for a truthy result.
143+
144+ Use this instead of a fixed `asyncio.sleep` when waiting for eventually-consistent state (e.g. a freshly
145+ created resource appearing in a listing) that may take a variable amount of time to propagate.
146+ """
147+ deadline = time .monotonic () + timeout
148+ result = await maybe_await (fn ())
149+ while not condition (result ):
150+ remaining = deadline - time .monotonic ()
151+ if remaining <= 0 :
152+ break
153+ await asyncio .sleep (min (poll_interval , remaining ))
154+ result = await maybe_await (fn ())
155+ return result
156+
157+
122158async def collect_iterate_until_present (
123159 iterator_factory : Callable [[], Iterator [_HasIdT ] | AsyncIterator [_HasIdT ]],
124160 expected_ids : set [str ],
@@ -132,7 +168,7 @@ async def collect_iterate_until_present(
132168
133169 Handles eventual consistency on listing endpoints: under parallel load a freshly
134170 created resource may not appear in the listing for a short window. Each attempt
135- builds a fresh iterator via `iterator_factory`, drains it, and breaks early once
171+ builds a fresh iterator via `iterator_factory`, drains it, and stops early once
136172 `expected_ids` is a subset of the collected items' `.id` values. The most recent
137173 collection is returned regardless of whether the condition was met, so the caller
138174 can run its own assertion with a helpful failure message.
@@ -142,18 +178,16 @@ async def collect_iterate_until_present(
142178 expected_ids: IDs that must all appear in the collected items.
143179 item_type: Asserted to match the runtime type of each yielded item.
144180 is_async: Whether the iterator is async (and so are sleeps).
145- max_attempts: Maximum number of polling rounds.
146- interval: Seconds to sleep before each attempt .
181+ max_attempts: Maximum number of polling rounds, guaranteed regardless of how long each drain takes .
182+ interval: Seconds to sleep between attempts .
147183
148184 Returns:
149185 The most recently collected items.
150186 """
151- collected : list [_HasIdT ] = []
152- for attempt in range (max_attempts ):
153- if attempt > 0 :
154- await maybe_sleep (interval , is_async = is_async )
187+
188+ async def drain () -> list [_HasIdT ]:
155189 iterator = iterator_factory ()
156- collected = []
190+ collected : list [ _HasIdT ] = []
157191 if is_async :
158192 assert isinstance (iterator , AsyncIterator )
159193 async for item in iterator :
@@ -164,8 +198,16 @@ async def collect_iterate_until_present(
164198 for item in iterator :
165199 assert isinstance (item , item_type )
166200 collected .append (item )
201+ return collected
202+
203+ # Loop on attempt count rather than a wall-clock deadline: drains take HTTP time, and charging it
204+ # against a deadline would mean fewer retries under load — exactly when they are needed most.
205+ collected = await drain ()
206+ for _ in range (max_attempts - 1 ):
167207 if expected_ids .issubset (item .id for item in collected ):
168208 break
209+ await maybe_sleep (interval , is_async = is_async )
210+ collected = await drain ()
169211 return collected
170212
171213
0 commit comments