|
1 | 1 | from __future__ import annotations |
2 | 2 |
|
3 | 3 | import asyncio |
| 4 | +import inspect |
| 5 | +import logging |
4 | 6 | import sys |
5 | | -from typing import TYPE_CHECKING |
| 7 | +import time |
| 8 | +from typing import TYPE_CHECKING, TypeVar, cast, overload |
6 | 9 |
|
7 | 10 | import pytest |
8 | 11 |
|
9 | 12 | if TYPE_CHECKING: |
10 | | - from collections.abc import Callable |
| 13 | + from collections.abc import Awaitable, Callable |
| 14 | + |
| 15 | +logger = logging.getLogger(__name__) |
| 16 | + |
| 17 | +T = TypeVar('T') |
11 | 18 |
|
12 | 19 | run_alone_on_mac = pytest.mark.run_alone if sys.platform == 'darwin' else lambda x: x |
13 | 20 |
|
14 | 21 |
|
15 | | -async def wait_for_condition( |
16 | | - condition: Callable[[], bool], |
| 22 | +async def _maybe_await(value: Awaitable[T] | T) -> T: |
| 23 | + """Await `value` if it is awaitable, otherwise return it unchanged. |
| 24 | +
|
| 25 | + Lets `call_with_exp_backoff` and `poll_until_condition` accept both sync and async callables. |
| 26 | + """ |
| 27 | + if inspect.isawaitable(value): |
| 28 | + return await cast('Awaitable[T]', value) |
| 29 | + return cast('T', value) |
| 30 | + |
| 31 | + |
| 32 | +@overload |
| 33 | +async def call_with_exp_backoff( |
| 34 | + fn: Callable[[], Awaitable[T]], |
| 35 | + condition: Callable[[T], bool] = ..., |
| 36 | + *, |
| 37 | + max_retries: int = ..., |
| 38 | + base_delay: float = ..., |
| 39 | +) -> T: ... |
| 40 | +@overload |
| 41 | +async def call_with_exp_backoff( |
| 42 | + fn: Callable[[], T], |
| 43 | + condition: Callable[[T], bool] = ..., |
| 44 | + *, |
| 45 | + max_retries: int = ..., |
| 46 | + base_delay: float = ..., |
| 47 | +) -> T: ... |
| 48 | +async def call_with_exp_backoff( |
| 49 | + fn: Callable[[], Awaitable[T] | T], |
| 50 | + condition: Callable[[T], bool] = bool, |
| 51 | + *, |
| 52 | + max_retries: int = 5, |
| 53 | + base_delay: float = 1.0, |
| 54 | +) -> T: |
| 55 | + """Call `fn`, retrying with exponential backoff until `condition(result)` is True. |
| 56 | +
|
| 57 | + Calls `fn` and checks whether `condition` holds for its result. If it does not, `fn` is retried up to |
| 58 | + `max_retries` times, sleeping `base_delay * 2 ** attempt` seconds before each retry. The last result is |
| 59 | + returned regardless of whether the condition was ever satisfied, so the caller can run its own assertion. |
| 60 | +
|
| 61 | + This is useful for eventually-consistent state where the expected value may take a moment to appear. The |
| 62 | + default condition checks for a truthy result. Pass `max_retries=0` to call `fn` exactly once without retries. |
| 63 | +
|
| 64 | + Unlike `poll_until_condition`, the delay between attempts grows exponentially rather than staying constant. |
| 65 | + """ |
| 66 | + result = await _maybe_await(fn()) |
| 67 | + for attempt in range(max_retries): |
| 68 | + if condition(result): |
| 69 | + return result |
| 70 | + delay = base_delay * 2**attempt |
| 71 | + logger.info( |
| 72 | + 'Condition not met for %r, retrying in %ss (attempt %d/%d).', result, delay, attempt + 1, max_retries |
| 73 | + ) |
| 74 | + await asyncio.sleep(delay) |
| 75 | + result = await _maybe_await(fn()) |
| 76 | + return result |
| 77 | + |
| 78 | + |
| 79 | +@overload |
| 80 | +async def poll_until_condition( |
| 81 | + fn: Callable[[], Awaitable[T]], |
| 82 | + condition: Callable[[T], bool] = ..., |
| 83 | + *, |
| 84 | + timeout: float = ..., |
| 85 | + poll_interval: float = ..., |
| 86 | +) -> T: ... |
| 87 | +@overload |
| 88 | +async def poll_until_condition( |
| 89 | + fn: Callable[[], T], |
| 90 | + condition: Callable[[T], bool] = ..., |
17 | 91 | *, |
18 | | - timeout: float = 5.0, |
| 92 | + timeout: float = ..., |
| 93 | + poll_interval: float = ..., |
| 94 | +) -> T: ... |
| 95 | +async def poll_until_condition( |
| 96 | + fn: Callable[[], Awaitable[T] | T], |
| 97 | + condition: Callable[[T], bool] = bool, |
| 98 | + *, |
| 99 | + timeout: float = 5, |
19 | 100 | poll_interval: float = 0.05, |
20 | | -) -> None: |
21 | | - """Poll `condition` until it returns True, or raise `AssertionError` on timeout. |
| 101 | +) -> T: |
| 102 | + """Poll `fn` until `condition(result)` is True or the timeout expires. |
| 103 | +
|
| 104 | + Polls `fn` at `poll_interval`-second intervals until `condition` is satisfied or `timeout` seconds have elapsed. |
| 105 | + Returns the last polled result regardless of whether the condition was met, so the caller can run its own |
| 106 | + assertion. The default condition checks for a truthy result. |
22 | 107 |
|
23 | | - Args: |
24 | | - condition: A callable that returns True when the desired state is reached. |
25 | | - timeout: Maximum time in seconds to wait before raising. |
26 | | - poll_interval: Time in seconds between condition checks. |
| 108 | + Use this instead of a fixed `asyncio.sleep` when waiting for some state to settle (e.g. autoscaling |
| 109 | + concurrency) that may take a variable amount of time. Unlike `call_with_exp_backoff`, the interval between |
| 110 | + polls stays constant. |
27 | 111 | """ |
28 | | - loop = asyncio.get_running_loop() |
29 | | - deadline = loop.time() + timeout |
30 | | - while loop.time() < deadline: |
31 | | - if condition(): |
32 | | - return |
33 | | - await asyncio.sleep(poll_interval) |
34 | | - raise AssertionError(f'Condition not met within {timeout}s') |
| 112 | + deadline = time.monotonic() + timeout |
| 113 | + result = await _maybe_await(fn()) |
| 114 | + while not condition(result): |
| 115 | + remaining = deadline - time.monotonic() |
| 116 | + if remaining <= 0: |
| 117 | + break |
| 118 | + await asyncio.sleep(min(poll_interval, remaining)) |
| 119 | + result = await _maybe_await(fn()) |
| 120 | + return result |
0 commit comments