11from __future__ import annotations
22
33import asyncio
4+ import inspect
5+ import logging
46import secrets
57import string
68import time
79from collections .abc import AsyncIterator , Iterator
810from dataclasses import dataclass
9- from typing import TYPE_CHECKING , Any , Protocol , TypeVar , overload
11+ from typing import TYPE_CHECKING , Any , Protocol , TypeVar , cast , overload
1012
1113import pytest
1214
1315if TYPE_CHECKING :
14- from collections .abc import Callable , Coroutine
16+ from collections .abc import Awaitable , Callable , Coroutine
17+
18+ logger = logging .getLogger (__name__ )
1519
1620# Environment variable names for test configuration
1721TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN'
@@ -119,6 +123,107 @@ async def maybe_sleep(seconds: float, *, is_async: bool) -> None:
119123 time .sleep (seconds ) # noqa: ASYNC251
120124
121125
126+ async def _maybe_await (value : Awaitable [T ] | T ) -> T :
127+ """Await `value` if it is awaitable, otherwise return it unchanged.
128+
129+ Lets `call_with_exp_backoff` and `poll_until_condition` accept both sync and async callables.
130+ """
131+ if inspect .isawaitable (value ):
132+ return await cast ('Awaitable[T]' , value )
133+ return cast ('T' , value )
134+
135+
136+ @overload
137+ async def call_with_exp_backoff (
138+ fn : Callable [[], Awaitable [T ]],
139+ condition : Callable [[T ], bool ] = ...,
140+ * ,
141+ max_retries : int = ...,
142+ base_delay : float = ...,
143+ ) -> T : ...
144+ @overload
145+ async def call_with_exp_backoff (
146+ fn : Callable [[], T ],
147+ condition : Callable [[T ], bool ] = ...,
148+ * ,
149+ max_retries : int = ...,
150+ base_delay : float = ...,
151+ ) -> T : ...
152+ async def call_with_exp_backoff (
153+ fn : Callable [[], Awaitable [T ] | T ],
154+ condition : Callable [[T ], bool ] = bool ,
155+ * ,
156+ max_retries : int = 5 ,
157+ base_delay : float = 1.0 ,
158+ ) -> T :
159+ """Call `fn`, retrying with exponential backoff until `condition(result)` is True.
160+
161+ Calls `fn` and checks whether `condition` holds for its result. If it does not, `fn` is retried up to
162+ `max_retries` times, sleeping `base_delay * 2 ** attempt` seconds before each retry. The last result is
163+ returned regardless of whether the condition was ever satisfied, so the caller can run its own assertion.
164+
165+ This is useful for eventually-consistent APIs where a freshly created resource may take a moment to become
166+ visible. The default condition checks for a truthy result. Pass `max_retries=0` to call `fn` exactly once.
167+
168+ Unlike `poll_until_condition`, the delay between attempts grows exponentially rather than staying constant.
169+ """
170+ result = await _maybe_await (fn ())
171+ for attempt in range (max_retries ):
172+ if condition (result ):
173+ return result
174+ delay = base_delay * 2 ** attempt
175+ logger .info (
176+ 'Condition not met for %r, retrying in %ss (attempt %d/%d).' , result , delay , attempt + 1 , max_retries
177+ )
178+ await asyncio .sleep (delay )
179+ result = await _maybe_await (fn ())
180+ return result
181+
182+
183+ @overload
184+ async def poll_until_condition (
185+ fn : Callable [[], Awaitable [T ]],
186+ condition : Callable [[T ], bool ] = ...,
187+ * ,
188+ timeout : float = ...,
189+ poll_interval : float = ...,
190+ ) -> T : ...
191+ @overload
192+ async def poll_until_condition (
193+ fn : Callable [[], T ],
194+ condition : Callable [[T ], bool ] = ...,
195+ * ,
196+ timeout : float = ...,
197+ poll_interval : float = ...,
198+ ) -> T : ...
199+ async def poll_until_condition (
200+ fn : Callable [[], Awaitable [T ] | T ],
201+ condition : Callable [[T ], bool ] = bool ,
202+ * ,
203+ timeout : float = 5 ,
204+ poll_interval : float = 1 ,
205+ ) -> T :
206+ """Poll `fn` until `condition(result)` is True or the timeout expires.
207+
208+ Polls `fn` at `poll_interval`-second intervals until `condition` is satisfied or `timeout` seconds have elapsed.
209+ Returns the last polled result regardless of whether the condition was met, so the caller can run its own
210+ assertion. The default condition checks for a truthy result.
211+
212+ Use this instead of a fixed `asyncio.sleep` when waiting for eventually-consistent state (e.g. a freshly
213+ created resource appearing in a listing) that may take a variable amount of time to propagate. Unlike
214+ `call_with_exp_backoff`, the interval between polls stays constant.
215+ """
216+ deadline = time .monotonic () + timeout
217+ result = await _maybe_await (fn ())
218+ while not condition (result ):
219+ remaining = deadline - time .monotonic ()
220+ if remaining <= 0 :
221+ break
222+ await asyncio .sleep (min (poll_interval , remaining ))
223+ result = await _maybe_await (fn ())
224+ return result
225+
226+
122227async def collect_iterate_until_present (
123228 iterator_factory : Callable [[], Iterator [_HasIdT ] | AsyncIterator [_HasIdT ]],
124229 expected_ids : set [str ],
@@ -132,7 +237,7 @@ async def collect_iterate_until_present(
132237
133238 Handles eventual consistency on listing endpoints: under parallel load a freshly
134239 created resource may not appear in the listing for a short window. Each attempt
135- builds a fresh iterator via `iterator_factory`, drains it, and breaks early once
240+ builds a fresh iterator via `iterator_factory`, drains it, and stops early once
136241 `expected_ids` is a subset of the collected items' `.id` values. The most recent
137242 collection is returned regardless of whether the condition was met, so the caller
138243 can run its own assertion with a helpful failure message.
@@ -141,19 +246,17 @@ async def collect_iterate_until_present(
141246 iterator_factory: No-arg callable returning a fresh iterator on each call.
142247 expected_ids: IDs that must all appear in the collected items.
143248 item_type: Asserted to match the runtime type of each yielded item.
144- is_async: Whether the iterator is async (and so are sleeps) .
249+ is_async: Whether the iterator is async.
145250 max_attempts: Maximum number of polling rounds.
146- interval: Seconds to sleep before each attempt .
251+ interval: Seconds to sleep between attempts .
147252
148253 Returns:
149254 The most recently collected items.
150255 """
151- collected : list [_HasIdT ] = []
152- for attempt in range (max_attempts ):
153- if attempt > 0 :
154- await maybe_sleep (interval , is_async = is_async )
256+
257+ async def drain () -> list [_HasIdT ]:
155258 iterator = iterator_factory ()
156- collected = []
259+ collected : list [ _HasIdT ] = []
157260 if is_async :
158261 assert isinstance (iterator , AsyncIterator )
159262 async for item in iterator :
@@ -164,9 +267,14 @@ async def collect_iterate_until_present(
164267 for item in iterator :
165268 assert isinstance (item , item_type )
166269 collected .append (item )
167- if expected_ids .issubset (item .id for item in collected ):
168- break
169- return collected
270+ return collected
271+
272+ return await poll_until_condition (
273+ drain ,
274+ lambda collected : expected_ids .issubset (item .id for item in collected ),
275+ timeout = max_attempts * interval ,
276+ poll_interval = interval ,
277+ )
170278
171279
172280# ============================================================================
0 commit comments