|
1 | 1 | from __future__ import annotations |
2 | 2 |
|
3 | 3 | import asyncio |
| 4 | +import inspect |
| 5 | +import logging |
4 | 6 | import time |
5 | | -from typing import TYPE_CHECKING, Literal, TypeVar |
| 7 | +from typing import TYPE_CHECKING, TypeVar, cast, overload |
6 | 8 |
|
7 | 9 | from crawlee._utils.crypto import crypto_random_object_id |
8 | 10 |
|
9 | | -from apify import Actor |
10 | | - |
11 | 11 | if TYPE_CHECKING: |
12 | 12 | from collections.abc import Awaitable, Callable |
13 | 13 |
|
14 | | -T = TypeVar('T') |
| 14 | +logger = logging.getLogger(__name__) |
15 | 15 |
|
| 16 | +T = TypeVar('T') |
16 | 17 |
|
17 | | -async def call_with_exp_backoff( |
18 | | - fn: Callable[[], Awaitable[T]], |
19 | | - *, |
20 | | - rq_access_mode: Literal['single', 'shared'], |
21 | | - max_retries: int = 5, |
22 | | -) -> T | None: |
23 | | - """Call an async callable with exponential backoff retries until it returns a truthy value. |
24 | 18 |
|
25 | | - In shared request queue mode, there is a propagation delay before newly added, reclaimed, or handled requests |
26 | | - become visible in the API (see https://github.com/apify/apify-sdk-python/issues/808). This helper retries with |
27 | | - exponential backoff to handle that delay in integration tests. |
| 19 | +async def _maybe_await(value: Awaitable[T] | T) -> T: |
| 20 | + """Await `value` if it is awaitable, otherwise return it unchanged. |
28 | 21 |
|
29 | | - When `rq_access_mode` is `'single'`, the function is called once without retries. |
| 22 | + Lets `call_with_exp_backoff` and `poll_until_condition` accept both sync and async callables. |
30 | 23 | """ |
31 | | - if rq_access_mode == 'single': |
32 | | - return await fn() |
33 | | - |
34 | | - if rq_access_mode == 'shared': |
35 | | - result = None |
| 24 | + if inspect.isawaitable(value): |
| 25 | + return await cast('Awaitable[T]', value) |
| 26 | + return cast('T', value) |
36 | 27 |
|
37 | | - for attempt in range(max_retries): |
38 | | - result = await fn() |
39 | 28 |
|
40 | | - if result: |
41 | | - return result |
| 29 | +@overload |
| 30 | +async def call_with_exp_backoff( |
| 31 | + fn: Callable[[], Awaitable[T]], |
| 32 | + condition: Callable[[T], bool] = ..., |
| 33 | + *, |
| 34 | + max_retries: int = ..., |
| 35 | + base_delay: float = ..., |
| 36 | +) -> T: ... |
| 37 | +@overload |
| 38 | +async def call_with_exp_backoff( |
| 39 | + fn: Callable[[], T], |
| 40 | + condition: Callable[[T], bool] = ..., |
| 41 | + *, |
| 42 | + max_retries: int = ..., |
| 43 | + base_delay: float = ..., |
| 44 | +) -> T: ... |
| 45 | +async def call_with_exp_backoff( |
| 46 | + fn: Callable[[], Awaitable[T] | T], |
| 47 | + condition: Callable[[T], bool] = bool, |
| 48 | + *, |
| 49 | + max_retries: int = 5, |
| 50 | + base_delay: float = 1.0, |
| 51 | +) -> T: |
| 52 | + """Call `fn`, retrying with exponential backoff until `condition(result)` is True. |
42 | 53 |
|
43 | | - delay = 2**attempt |
44 | | - Actor.log.info(f'{fn} returned {result!r}, retrying in {delay}s (attempt {attempt + 1}/{max_retries})') |
45 | | - await asyncio.sleep(delay) |
| 54 | + Calls `fn` and checks whether `condition` holds for its result. If it does not, `fn` is retried up to |
| 55 | + `max_retries` times, sleeping `base_delay * 2 ** attempt` seconds before each retry. The last result is |
| 56 | + returned regardless of whether the condition was ever satisfied, so the caller can run its own assertion. |
46 | 57 |
|
47 | | - return result |
| 58 | + This is useful for eventually-consistent APIs where a freshly added, reclaimed, or handled item may take a |
| 59 | + moment to become visible (see https://github.com/apify/apify-sdk-python/issues/808). The default condition |
| 60 | + checks for a truthy result. Pass `max_retries=0` to call `fn` exactly once without any retries. |
48 | 61 |
|
49 | | - raise ValueError(f'Invalid rq_access_mode: {rq_access_mode}') |
| 62 | + Unlike `poll_until_condition`, the delay between attempts grows exponentially rather than staying constant. |
| 63 | + """ |
| 64 | + result = await _maybe_await(fn()) |
| 65 | + for attempt in range(max_retries): |
| 66 | + if condition(result): |
| 67 | + return result |
| 68 | + delay = base_delay * 2**attempt |
| 69 | + logger.info( |
| 70 | + 'Condition not met for %r, retrying in %ss (attempt %d/%d).', result, delay, attempt + 1, max_retries |
| 71 | + ) |
| 72 | + await asyncio.sleep(delay) |
| 73 | + result = await _maybe_await(fn()) |
| 74 | + return result |
50 | 75 |
|
51 | 76 |
|
| 77 | +@overload |
52 | 78 | async def poll_until_condition( |
53 | 79 | fn: Callable[[], Awaitable[T]], |
54 | | - condition: Callable[[T], bool], |
| 80 | + condition: Callable[[T], bool] = ..., |
| 81 | + *, |
| 82 | + timeout: float = ..., |
| 83 | + poll_interval: float = ..., |
| 84 | +) -> T: ... |
| 85 | +@overload |
| 86 | +async def poll_until_condition( |
| 87 | + fn: Callable[[], T], |
| 88 | + condition: Callable[[T], bool] = ..., |
| 89 | + *, |
| 90 | + timeout: float = ..., |
| 91 | + poll_interval: float = ..., |
| 92 | +) -> T: ... |
| 93 | +async def poll_until_condition( |
| 94 | + fn: Callable[[], Awaitable[T] | T], |
| 95 | + condition: Callable[[T], bool] = bool, |
55 | 96 | *, |
56 | 97 | timeout: float = 60, |
57 | 98 | poll_interval: float = 5, |
58 | 99 | ) -> T: |
59 | 100 | """Poll `fn` until `condition(result)` is True or the timeout expires. |
60 | 101 |
|
61 | 102 | Polls `fn` at `poll_interval`-second intervals until `condition` is satisfied or `timeout` seconds have elapsed. |
62 | | - Returns the last polled result regardless of whether the condition was met. |
| 103 | + Returns the last polled result regardless of whether the condition was met, so the caller can run its own |
| 104 | + assertion. The default condition checks for a truthy result. |
63 | 105 |
|
64 | | - Use this instead of a fixed `asyncio.sleep` when waiting for eventually-consistent API state (e.g. request queue |
65 | | - stats) that may take a variable amount of time to propagate. |
| 106 | + Use this instead of a fixed `asyncio.sleep` when waiting for eventually-consistent state (e.g. request queue |
| 107 | + stats) that may take a variable amount of time to propagate. Unlike `call_with_exp_backoff`, the interval |
| 108 | + between polls stays constant. |
66 | 109 | """ |
67 | 110 | deadline = time.monotonic() + timeout |
68 | | - result = await fn() |
| 111 | + result = await _maybe_await(fn()) |
69 | 112 | while not condition(result): |
70 | 113 | remaining = deadline - time.monotonic() |
71 | 114 | if remaining <= 0: |
72 | 115 | break |
73 | 116 | await asyncio.sleep(min(poll_interval, remaining)) |
74 | | - result = await fn() |
| 117 | + result = await _maybe_await(fn()) |
75 | 118 | return result |
76 | 119 |
|
77 | 120 |
|
|
0 commit comments