Repository navigation
Expand file tree
/
Copy path_utils.py
More file actions
223 lines (177 loc) · 7.57 KB
/
Copy path_utils.py
File metadata and controls
223 lines (177 loc) · 7.57 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
from __future__ import annotations
import asyncio
import inspect
import secrets
import string
import time
from collections.abc import AsyncIterator, Iterator
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Protocol, TypeVar, cast, overload
import pytest
if TYPE_CHECKING:
from collections.abc import Awaitable, Callable
# Environment variable names for test configuration
TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN'
TOKEN_ENV_VAR_2 = 'APIFY_TEST_USER_2_API_TOKEN'
API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL'
T = TypeVar('T')
class _HasId(Protocol):
"""Items returned by collection `iterate()` endpoints all expose `.id`."""
@property
def id(self) -> str: ...
_HasIdT = TypeVar('_HasIdT', bound=_HasId)
@dataclass
class StorageFixture:
"""Base storage fixture with ID and signature."""
id: str
signature: str
@dataclass
class DatasetFixture(StorageFixture):
"""Dataset fixture with expected content."""
expected_content: list
@dataclass
class KvsFixture(StorageFixture):
"""Key-value store fixture with expected content and key signatures."""
expected_content: dict[str, Any]
keys_signature: dict[str, str]
def get_crypto_random_object_id(length: int = 17) -> str:
"""Generate a cryptographically secure random object ID."""
chars = 'abcdefghijklmnopqrstuvwxyzABCEDFGHIJKLMNOPQRSTUVWXYZ0123456789'
return ''.join(secrets.choice(chars) for _ in range(length))
def get_random_string(length: int = 10) -> str:
"""Generate a random alphabetic string."""
return ''.join(secrets.choice(string.ascii_letters) for _ in range(length))
def get_random_resource_name(label: str) -> str:
"""Generate a unique resource name containing the given label.
Ensures the generated name does not exceed the API limit of 63 characters.
"""
name_template = 'python-client-test-{}-{}'
template_length = len(name_template.format('', ''))
api_name_limit = 63
random_id_length = 8
label_length_limit = api_name_limit - template_length - random_id_length
label = label.replace('_', '-')
assert len(label) <= label_length_limit, f'Max label length is {label_length_limit}, but got {len(label)}'
return name_template.format(label, get_crypto_random_object_id(random_id_length))
async def maybe_await(value: Awaitable[T] | T) -> T:
"""Await `value` if it is awaitable, otherwise return it unchanged.
Enables unified test code for both sync and async clients:
result = await maybe_await(client.datasets().list())
"""
if inspect.isawaitable(value):
return await cast('Awaitable[T]', value)
return value
async def maybe_sleep(seconds: float, *, is_async: bool) -> None:
"""Sleep using asyncio or time.sleep based on client type."""
if is_async:
await asyncio.sleep(seconds)
else:
time.sleep(seconds) # noqa: ASYNC251
@overload
async def poll_until_condition(
fn: Callable[[], Awaitable[T]],
condition: Callable[[T], bool] = ...,
*,
timeout: float = ...,
poll_interval: float = ...,
backoff_factor: float = ...,
) -> T: ...
@overload
async def poll_until_condition(
fn: Callable[[], T],
condition: Callable[[T], bool] = ...,
*,
timeout: float = ...,
poll_interval: float = ...,
backoff_factor: float = ...,
) -> T: ...
async def poll_until_condition(
fn: Callable[[], Awaitable[T] | T],
condition: Callable[[T], bool] = bool,
*,
timeout: float = 30,
poll_interval: float = 1,
backoff_factor: float = 1,
) -> T:
"""Poll `fn` until `condition(result)` is True or the timeout expires.
Polls `fn` at `poll_interval`-second intervals until `condition` is satisfied or `timeout` seconds have elapsed.
Returns the last polled result regardless of whether the condition was met, so the caller can run its own
assertion. The default condition checks for a truthy result.
Use this instead of a fixed `asyncio.sleep` when waiting for eventually-consistent state (e.g. a freshly
created resource appearing in a listing) that may take a variable amount of time to propagate. For highly
variable wait times (e.g. an Actor run container starting up), pass `backoff_factor` > 1 to multiply the
interval after each poll, covering a long timeout with few calls.
"""
deadline = time.monotonic() + timeout
delay = poll_interval
result = await maybe_await(fn())
while not condition(result):
remaining = deadline - time.monotonic()
if remaining <= 0:
break
await asyncio.sleep(min(delay, remaining))
delay *= backoff_factor
result = await maybe_await(fn())
return result
async def collect_iterate_until_present(
iterator_factory: Callable[[], Iterator[_HasIdT] | AsyncIterator[_HasIdT]],
expected_ids: set[str],
*,
item_type: type[_HasIdT],
is_async: bool,
max_attempts: int = 5,
interval: float = 1.0,
) -> list[_HasIdT]:
"""Drain a collection `iterate()` until every expected ID is present.
Handles eventual consistency on listing endpoints: under parallel load a freshly
created resource may not appear in the listing for a short window. Each attempt
builds a fresh iterator via `iterator_factory`, drains it, and stops early once
`expected_ids` is a subset of the collected items' `.id` values. The most recent
collection is returned regardless of whether the condition was met, so the caller
can run its own assertion with a helpful failure message.
Args:
iterator_factory: No-arg callable returning a fresh iterator on each call.
expected_ids: IDs that must all appear in the collected items.
item_type: Asserted to match the runtime type of each yielded item.
is_async: Whether the iterator is async (and so are sleeps).
max_attempts: Maximum number of polling rounds, guaranteed regardless of how long each drain takes.
interval: Seconds to sleep between attempts.
Returns:
The most recently collected items.
"""
async def drain() -> list[_HasIdT]:
iterator = iterator_factory()
collected: list[_HasIdT] = []
if is_async:
assert isinstance(iterator, AsyncIterator)
async for item in iterator:
assert isinstance(item, item_type)
collected.append(item)
else:
assert isinstance(iterator, Iterator)
for item in iterator:
assert isinstance(item, item_type)
collected.append(item)
return collected
# Loop on attempt count rather than a wall-clock deadline: drains take HTTP time, and charging it
# against a deadline would mean fewer retries under load — exactly when they are needed most.
collected = await drain()
for _ in range(max_attempts - 1):
if expected_ids.issubset(item.id for item in collected):
break
await maybe_sleep(interval, is_async=is_async)
collected = await drain()
return collected
parametrized_api_urls = pytest.mark.parametrize(
('api_url', 'api_public_url'),
[
('https://api.apify.com', 'https://api.apify.com'),
('https://api.apify.com', None),
('https://api.apify.com', 'https://custom-public-url.com'),
('https://api.apify.com', 'https://custom-public-url.com/with/custom/path'),
('https://api.apify.com', 'https://custom-public-url.com/with/custom/path/'),
('http://10.0.88.214:8010', 'https://api.apify.com'),
('http://10.0.88.214:8010', None),
],
)
"""Parametrize decorator for testing various API URL and public URL combinations."""