Skip to content

Commit 307df23

Browse files
committed
WIP
Working tests and implementation. TODO: -Check KVS and RQ special cases -Figure out model mocking in some elegant way
1 parent 6b8001e commit 307df23

18 files changed

Lines changed: 1663 additions & 412 deletions
Lines changed: 269 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,269 @@
1+
from __future__ import annotations
2+
3+
from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Generator, Iterable, Iterator
4+
from typing import Any, Generic, TypeVar
5+
6+
from apify_client._docs import docs_group
7+
8+
T = TypeVar('T')
9+
10+
11+
def _min_for_limit_param(a: int | None, b: int | None) -> int | None:
12+
"""Return minimum of two limit parameters, treating `None` or `0` as infinity.
13+
14+
The Apify API treats `0` as no limit for the `limit` parameter, so `0` here means infinity.
15+
Returns `None` when both inputs represent infinity.
16+
"""
17+
if a == 0:
18+
a = None
19+
if b == 0:
20+
b = None
21+
if a is None:
22+
return b
23+
if b is None:
24+
return a
25+
return min(a, b)
26+
27+
28+
@docs_group('Other')
29+
class IterableListPage(Iterable[T], Generic[T]):
30+
"""A page of results that can also be iterated to yield items across subsequent pages.
31+
32+
Accessing fields such as `items`, `count`, or `total` returns the metadata of the first page,
33+
preserving the behavior of the previous non-iterable return value. Iterating with `for item in ...`
34+
yields individual items and performs additional API calls as needed to fetch further pages.
35+
"""
36+
37+
items: list[T]
38+
"""List of items on this page."""
39+
40+
count: int
41+
"""Number of items on this page."""
42+
43+
offset: int
44+
"""The starting offset of this page."""
45+
46+
limit: int
47+
"""The maximum number of items per page requested from the API."""
48+
49+
total: int
50+
"""Total number of items matching the query, as reported by the first page."""
51+
52+
desc: bool
53+
"""Whether the items are sorted in descending order."""
54+
55+
def __init__(self, first_page: Any, iterator: Iterator[T]) -> None:
56+
"""Initialize a page wrapper from a Pydantic paginated model and an iterator over all items."""
57+
self.items = first_page.items
58+
count = getattr(first_page, 'count', None)
59+
self.count = count if count is not None else len(first_page.items)
60+
self.offset = getattr(first_page, 'offset', 0) or 0
61+
self.limit = getattr(first_page, 'limit', 0) or 0
62+
self.total = getattr(first_page, 'total', None) or len(first_page.items)
63+
self.desc = getattr(first_page, 'desc', False) or False
64+
self._first_page = first_page
65+
self._iterator = iterator
66+
67+
def __iter__(self) -> Iterator[T]:
68+
"""Return an iterator over all items across pages, fetching additional pages as needed."""
69+
return self._iterator
70+
71+
72+
@docs_group('Other')
73+
class IterableListPageAsync(AsyncIterable[T], Generic[T]):
74+
"""An awaitable result that can also be asynchronously iterated to yield items across pages.
75+
76+
Awaiting the instance (`await client.list(...)`) performs a single API call and returns a
77+
populated `IterableListPage`. Iterating (`async for item in client.list(...)`) yields individual
78+
items and performs additional API calls as needed to fetch further pages.
79+
80+
A single instance supports either awaiting or iterating — not both.
81+
"""
82+
83+
def __init__(
84+
self,
85+
make_awaitable: Callable[[], Awaitable[IterableListPage[T]]],
86+
async_iterator: AsyncIterator[T],
87+
) -> None:
88+
"""Initialize with a factory that creates the awaitable on demand and the async iterator over items."""
89+
self._make_awaitable = make_awaitable
90+
self._async_iterator = async_iterator
91+
92+
def __aiter__(self) -> AsyncIterator[T]:
93+
"""Return an asynchronous iterator over all items across pages."""
94+
return self._async_iterator
95+
96+
def __await__(self) -> Generator[Any, Any, IterableListPage[T]]:
97+
"""Return an awaitable that resolves to an `IterableListPage` containing the first page."""
98+
return self._make_awaitable().__await__()
99+
100+
101+
def build_iterable_list_page(
102+
callback: Callable[..., Any],
103+
**kwargs: Any,
104+
) -> IterableListPage[Any]:
105+
"""Build an `IterableListPage` from a paginated sync callback.
106+
107+
The callback is invoked once immediately to fetch the first page, and again lazily during
108+
iteration to fetch further pages. The `total` field from the first page is not trusted for
109+
stopping iteration because it may change between calls; iteration stops when a page has
110+
no items or when the user-requested `limit` has been reached.
111+
112+
Recognized kwargs:
113+
chunk_size: Maximum number of items requested per API call during iteration. Pass `0`
114+
or `None` to let the API decide (effectively infinity).
115+
limit: User-requested total item limit. Stops iteration once this many items are yielded.
116+
offset: Starting offset for the first page.
117+
**other: Passed through to the callback unchanged.
118+
"""
119+
chunk_size = kwargs.pop('chunk_size', 0) or 0
120+
offset = kwargs.get('offset') or 0
121+
limit = kwargs.get('limit') or 0
122+
123+
first_page = callback(**{**kwargs, 'limit': _min_for_limit_param(kwargs.get('limit'), chunk_size)})
124+
125+
def iterator() -> Iterator[Any]:
126+
current_page = first_page
127+
yield from current_page.items
128+
129+
fetched_items = len(current_page.items)
130+
while current_page.items and (not limit or (limit > fetched_items)):
131+
new_kwargs = {
132+
**kwargs,
133+
'offset': offset + fetched_items,
134+
'limit': chunk_size if not limit else _min_for_limit_param(limit - fetched_items, chunk_size),
135+
}
136+
current_page = callback(**new_kwargs)
137+
yield from current_page.items
138+
fetched_items += len(current_page.items)
139+
140+
return IterableListPage(first_page, iterator())
141+
142+
143+
def build_iterable_list_page_async(
144+
callback: Callable[..., Awaitable[Any]],
145+
**kwargs: Any,
146+
) -> IterableListPageAsync[Any]:
147+
"""Build an `IterableListPageAsync` from a paginated async callback.
148+
149+
Mirrors `build_iterable_list_page` but for async callbacks. The returned object is both
150+
awaitable (resolves to the first page wrapped in `IterableListPage`) and asynchronously
151+
iterable (yields items across pages).
152+
"""
153+
chunk_size = kwargs.pop('chunk_size', 0) or 0
154+
offset = kwargs.get('offset') or 0
155+
limit = kwargs.get('limit') or 0
156+
157+
async def fetch_first_page() -> Any:
158+
return await callback(**{**kwargs, 'limit': _min_for_limit_param(kwargs.get('limit'), chunk_size)})
159+
160+
async def async_iterator() -> AsyncIterator[Any]:
161+
current_page = await fetch_first_page()
162+
for item in current_page.items:
163+
yield item
164+
165+
fetched_items = len(current_page.items)
166+
while current_page.items and (not limit or (limit > fetched_items)):
167+
new_kwargs = {
168+
**kwargs,
169+
'offset': offset + fetched_items,
170+
'limit': chunk_size if not limit else _min_for_limit_param(limit - fetched_items, chunk_size),
171+
}
172+
current_page = await callback(**new_kwargs)
173+
for item in current_page.items:
174+
yield item
175+
fetched_items += len(current_page.items)
176+
177+
async def wrap_first_page() -> IterableListPage[Any]:
178+
first_page = await fetch_first_page()
179+
return IterableListPage(first_page, iter(first_page.items))
180+
181+
return IterableListPageAsync(wrap_first_page, async_iterator())
182+
183+
184+
def build_cursor_iterable_list_page(
185+
callback: Callable[..., Any],
186+
*,
187+
cursor_param: str,
188+
next_cursor_fn: Callable[[Any], Any],
189+
initial_cursor: Any = None,
190+
limit: int | None = None,
191+
chunk_size: int | None = None,
192+
**kwargs: Any,
193+
) -> IterableListPage[Any]:
194+
"""Build an `IterableListPage` for endpoints that paginate with a cursor instead of an offset.
195+
196+
The callback is invoked with `{cursor_param: cursor, 'limit': effective_limit, **kwargs}` for each
197+
page, starting from `initial_cursor`. After each page, `next_cursor_fn(page)` is consulted to
198+
obtain the next cursor; returning `None` ends iteration. The iteration also stops when a page is
199+
empty or when the caller-requested `limit` has been reached.
200+
"""
201+
effective_chunk = chunk_size or 0
202+
user_limit = limit or 0
203+
204+
first_limit = _min_for_limit_param(limit, effective_chunk)
205+
first_page = callback(**{**kwargs, cursor_param: initial_cursor, 'limit': first_limit})
206+
207+
def iterator() -> Iterator[Any]:
208+
current_page = first_page
209+
yield from current_page.items
210+
211+
fetched = len(current_page.items)
212+
next_cursor = next_cursor_fn(current_page)
213+
214+
while current_page.items and next_cursor is not None and (not user_limit or user_limit > fetched):
215+
remaining = (user_limit - fetched) if user_limit else 0
216+
next_limit = effective_chunk if not user_limit else _min_for_limit_param(remaining, effective_chunk)
217+
current_page = callback(**{**kwargs, cursor_param: next_cursor, 'limit': next_limit})
218+
yield from current_page.items
219+
fetched += len(current_page.items)
220+
next_cursor = next_cursor_fn(current_page)
221+
222+
return IterableListPage(first_page, iterator())
223+
224+
225+
def build_cursor_iterable_list_page_async(
226+
callback: Callable[..., Awaitable[Any]],
227+
*,
228+
cursor_param: str,
229+
next_cursor_fn: Callable[[Any], Any],
230+
initial_cursor: Any = None,
231+
limit: int | None = None,
232+
chunk_size: int | None = None,
233+
**kwargs: Any,
234+
) -> IterableListPageAsync[Any]:
235+
"""Build an `IterableListPageAsync` for endpoints that paginate with a cursor instead of an offset.
236+
237+
Mirrors `build_cursor_iterable_list_page` but for async callbacks. The returned object is both
238+
awaitable (resolves to the first page wrapped in `IterableListPage`) and asynchronously iterable
239+
(yields items across pages using the supplied cursor strategy).
240+
"""
241+
effective_chunk = chunk_size or 0
242+
user_limit = limit or 0
243+
first_limit = _min_for_limit_param(limit, effective_chunk)
244+
245+
async def fetch_first_page() -> Any:
246+
return await callback(**{**kwargs, cursor_param: initial_cursor, 'limit': first_limit})
247+
248+
async def async_iterator() -> AsyncIterator[Any]:
249+
current_page = await fetch_first_page()
250+
for item in current_page.items:
251+
yield item
252+
253+
fetched = len(current_page.items)
254+
next_cursor = next_cursor_fn(current_page)
255+
256+
while current_page.items and next_cursor is not None and (not user_limit or user_limit > fetched):
257+
remaining = (user_limit - fetched) if user_limit else 0
258+
next_limit = effective_chunk if not user_limit else _min_for_limit_param(remaining, effective_chunk)
259+
current_page = await callback(**{**kwargs, cursor_param: next_cursor, 'limit': next_limit})
260+
for item in current_page.items:
261+
yield item
262+
fetched += len(current_page.items)
263+
next_cursor = next_cursor_fn(current_page)
264+
265+
async def wrap_first_page() -> IterableListPage[Any]:
266+
first_page = await fetch_first_page()
267+
return IterableListPage(first_page, iter(first_page.items))
268+
269+
return IterableListPageAsync(wrap_first_page, async_iterator())

‎src/apify_client/_resource_clients/actor_collection.py‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,12 @@
33
from typing import TYPE_CHECKING, Any, Literal
44

55
from apify_client._docs import docs_group
6+
from apify_client._iterable_list_page import (
7+
IterableListPage,
8+
IterableListPageAsync,
9+
build_iterable_list_page,
10+
build_iterable_list_page_async,
11+
)
612
from apify_client._models import (
713
Actor,
814
ActorResponse,
@@ -19,6 +25,7 @@
1925
if TYPE_CHECKING:
2026
from datetime import timedelta
2127

28+
from apify_client._models import ActorShort
2229
from apify_client._types import Timeout
2330

2431
_SORT_BY_TO_API: dict[str, str] = {
@@ -55,9 +62,12 @@ def list(
5562
desc: bool | None = None,
5663
sort_by: Literal['created_at', 'last_run_started_at'] | None = 'created_at',
5764
timeout: Timeout = 'medium',
58-
) -> ListOfActors:
65+
) -> IterableListPage[ActorShort]:
5966
"""List the Actors the user has created or used.
6067
68+
The returned page also supports iteration: `for item in client.list(...)` yields individual Actors
69+
and transparently fetches further pages from the API.
70+
6171
https://docs.apify.com/api/v2#/reference/actors/actor-collection/get-list-of-actors
6272
6373
Args:
@@ -72,8 +82,12 @@ def list(
7282
The list of available Actors matching the specified filters.
7383
"""
7484
api_sort_by = _SORT_BY_TO_API[sort_by] if sort_by is not None else None
75-
result = self._list(timeout=timeout, my=my, limit=limit, offset=offset, desc=desc, sortBy=api_sort_by)
76-
return ListOfActorsResponse.model_validate(result).data
85+
86+
def _callback(**kwargs: Any) -> ListOfActors:
87+
result = self._list(timeout=timeout, my=my, sortBy=api_sort_by, **kwargs)
88+
return ListOfActorsResponse.model_validate(result).data
89+
90+
return build_iterable_list_page(_callback, limit=limit, offset=offset, desc=desc)
7791

7892
def create(
7993
self,
@@ -192,7 +206,7 @@ def __init__(
192206
**kwargs,
193207
)
194208

195-
async def list(
209+
def list(
196210
self,
197211
*,
198212
my: bool | None = None,
@@ -201,9 +215,12 @@ async def list(
201215
desc: bool | None = None,
202216
sort_by: Literal['created_at', 'last_run_started_at'] | None = 'created_at',
203217
timeout: Timeout = 'medium',
204-
) -> ListOfActors:
218+
) -> IterableListPageAsync[ActorShort]:
205219
"""List the Actors the user has created or used.
206220
221+
The returned page also supports iteration: `for item in client.list(...)` yields individual Actors
222+
and transparently fetches further pages from the API.
223+
207224
https://docs.apify.com/api/v2#/reference/actors/actor-collection/get-list-of-actors
208225
209226
Args:
@@ -218,8 +235,12 @@ async def list(
218235
The list of available Actors matching the specified filters.
219236
"""
220237
api_sort_by = _SORT_BY_TO_API[sort_by] if sort_by is not None else None
221-
result = await self._list(timeout=timeout, my=my, limit=limit, offset=offset, desc=desc, sortBy=api_sort_by)
222-
return ListOfActorsResponse.model_validate(result).data
238+
239+
async def _callback(**kwargs: Any) -> ListOfActors:
240+
result = await self._list(timeout=timeout, my=my, sortBy=api_sort_by, **kwargs)
241+
return ListOfActorsResponse.model_validate(result).data
242+
243+
return build_iterable_list_page_async(_callback, limit=limit, offset=offset, desc=desc)
223244

224245
async def create(
225246
self,

0 commit comments

Comments
 (0)