Skip to content

Commit 3f395a2

Browse files
committed
Use new approach for all endpoints
1 parent 73e1531 commit 3f395a2

32 files changed

Lines changed: 991 additions & 398 deletions

‎src/apify_client/_iterable_list_page.py‎

Lines changed: 21 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
from __future__ import annotations
22

33
import asyncio
4-
from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Coroutine, Generator, Iterable, Iterator
5-
from typing import Any, Generic, Protocol, TypeVar
4+
from typing import TYPE_CHECKING, Any, Generic, Protocol, TypeVar
65

7-
from apify_client._docs import docs_group
6+
if TYPE_CHECKING:
7+
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Generator, Iterator
88

99
T = TypeVar('T')
1010

@@ -46,88 +46,6 @@ def __await__(self) -> Generator[Any, None, T]:
4646
return (yield from self._task.__await__())
4747

4848

49-
@docs_group('Other')
50-
class ListPage(Generic[T]):
51-
"""A page of API results.
52-
53-
Different endpoints may return different subsets of the available pagination metadata fields, the only field that
54-
is common for all endpoints is items.
55-
"""
56-
57-
items: list[T]
58-
"""List of items on this page."""
59-
60-
count: int | None
61-
"""Number of items on this page."""
62-
63-
offset: int | None
64-
"""The starting offset of this page."""
65-
66-
limit: int | None
67-
"""The maximum number of items per page requested from the API."""
68-
69-
total: int | None
70-
"""Total number of items matching the query, as reported by the first page."""
71-
72-
desc: bool | None
73-
"""Whether the items are sorted in descending order."""
74-
75-
def __init__(self, first_page: HasItems[T]) -> None:
76-
"""Initialize a page wrapper from a Pydantic paginated model."""
77-
self.items = first_page.items
78-
self.count = getattr(first_page, 'count', None)
79-
self.offset = getattr(first_page, 'offset', None)
80-
self.limit = getattr(first_page, 'limit', None)
81-
self.total = getattr(first_page, 'total', None)
82-
self.desc = getattr(first_page, 'desc', None)
83-
84-
85-
@docs_group('Other')
86-
class IterableListPage(ListPage[T], Iterable[T], Generic[T]):
87-
"""A page of results that can also be iterated to yield items across subsequent pages.
88-
89-
Accessing fields such as `items`, `count`, or `total` returns the metadata of the first page,
90-
preserving the behavior of the previous non-iterable return value. Iterating with `for item in ...`
91-
yields individual items and performs additional API calls as needed to fetch further pages.
92-
"""
93-
94-
def __init__(self, first_page: HasItems[T], get_iterator: Callable[[], Iterator[T]]) -> None:
95-
"""Initialize a page wrapper from a Pydantic paginated model and an iterator over all items."""
96-
super().__init__(first_page)
97-
self._get_iterator = get_iterator
98-
99-
def __iter__(self) -> Iterator[T]:
100-
"""Return an iterator over all items across pages, fetching additional pages as needed."""
101-
return self._get_iterator()
102-
103-
104-
@docs_group('Other')
105-
class IterableListPageAsync(AsyncIterable[T], Generic[T]):
106-
"""An awaitable result that can also be asynchronously iterated to yield items across pages.
107-
108-
Awaiting the instance (`await client.list(...)`) performs a single API call and returns a
109-
populated `ListPage`. Iterating (`async for item in client.list(...)`) yields individual
110-
items and performs additional API calls as needed to fetch further pages.
111-
"""
112-
113-
def __init__(
114-
self,
115-
awaitable_first_page: Callable[[], Awaitable[ListPage[T]]],
116-
get_async_iterator: Callable[[], AsyncIterator[T]],
117-
) -> None:
118-
"""Initialize with a factory that creates the awaitable on demand and the async iterator over items."""
119-
self._awaitable_first_page = awaitable_first_page
120-
self._get_async_iterator = get_async_iterator
121-
122-
def __aiter__(self) -> AsyncIterator[T]:
123-
"""Return an asynchronous iterator over all items across pages."""
124-
return self._get_async_iterator()
125-
126-
def __await__(self) -> Generator[Any, Any, ListPage[T]]:
127-
"""Return an awaitable that resolves to a `ListPage` containing the first page of results."""
128-
return self._awaitable_first_page().__await__()
129-
130-
13149
def build_iterable_list_page(
13250
callback: Callable[..., HasItems[T]],
13351
first_page: HasItems[T],
@@ -159,6 +77,7 @@ def build_iterable_list_page(
15977
chunk_size = kwargs.pop('chunk_size', 0) or 0
16078
offset = kwargs.get('offset') or 0
16179
limit = kwargs.get('limit') or 0
80+
16281
def get_iterator() -> Iterator[T]:
16382
current_page = first_page
16483
yield from current_page.items
@@ -179,7 +98,7 @@ def get_iterator() -> Iterator[T]:
17998

18099
def build_iterable_list_page_async(
181100
callback: Callable[..., Coroutine[Any, Any, HasItems[T]]],
182-
fetch_first_page,
101+
fetch_first_page: Awaitable[HasItems[T]],
183102
**kwargs: Any,
184103
) -> Callable[[], AsyncIterator[T]]:
185104
"""Build an `IterableListPageAsync` from a paginated async callback.
@@ -214,27 +133,24 @@ async def get_async_iterator() -> AsyncIterator[T]:
214133

215134
def build_cursor_iterable_list_page(
216135
callback: Callable[..., HasItems[T]],
136+
first_page: HasItems[T],
217137
*,
218138
cursor_param: str,
219-
initial_cursor: Any = None,
220139
limit: int | None = None,
221140
chunk_size: int | None = None,
222141
**kwargs: Any,
223-
) -> IterableListPage[T]:
224-
"""Build an `IterableListPage` for endpoints that paginate with a cursor instead of an offset.
142+
) -> Callable[[], Iterator[T]]:
143+
"""Build the iterator factory for endpoints that paginate with a cursor instead of an offset.
225144
226-
The callback is invoked with `{cursor_param: cursor, 'limit': effective_limit, **kwargs}` for each
227-
page, starting from `initial_cursor`. After each page, `next_cursor_fn(page)` is consulted to
228-
obtain the next cursor; returning `None` ends iteration. The iteration also stops when a page is
229-
empty or when the caller-requested `limit` has been reached.
145+
The caller is responsible for fetching the first page (typically by calling `callback` with
146+
the initial cursor). After each page, `getattr(page, f'next_{cursor_param}')` is consulted
147+
to obtain the next cursor; returning `None` ends iteration. The iteration also stops when a
148+
page is empty or when the caller-requested `limit` has been reached.
230149
"""
231150
effective_chunk = chunk_size or 0
232151
user_limit = limit or 0
233152

234-
first_limit = _min_for_limit_param(limit, effective_chunk)
235-
first_page = callback(**{**kwargs, cursor_param: initial_cursor, 'limit': first_limit})
236-
237-
def get_iterator() -> Iterator[Any]:
153+
def get_iterator() -> Iterator[T]:
238154
current_page = first_page
239155
yield from current_page.items
240156

@@ -249,32 +165,27 @@ def get_iterator() -> Iterator[Any]:
249165
fetched += len(current_page.items)
250166
next_cursor = getattr(current_page, f'next_{cursor_param}')
251167

252-
return IterableListPage(first_page, get_iterator)
168+
return get_iterator
253169

254170

255171
def build_cursor_iterable_list_page_async(
256172
callback: Callable[..., Coroutine[Any, Any, HasItems[T]]],
173+
fetch_first_page: Awaitable[HasItems[T]],
257174
*,
258175
cursor_param: str,
259-
initial_cursor: Any = None,
260176
limit: int | None = None,
261177
chunk_size: int | None = None,
262178
**kwargs: Any,
263-
) -> IterableListPageAsync[T]:
264-
"""Build an `IterableListPageAsync` for endpoints that paginate with a cursor instead of an offset.
179+
) -> Callable[[], AsyncIterator[T]]:
180+
"""Build the async iterator factory for endpoints that paginate with a cursor instead of an offset.
265181
266-
Mirrors `build_cursor_iterable_list_page` but for async callbacks. The returned object is both
267-
awaitable (resolves to the first page wrapped in `IterableListPage`) and asynchronously iterable
268-
(yields items across pages using the supplied cursor strategy).
182+
Mirrors `build_cursor_iterable_list_page` but for async callbacks. The caller is responsible
183+
for awaiting the first page (typically through a `_LazyTask` wrapping the initial callback).
269184
"""
270185
effective_chunk = chunk_size or 0
271186
user_limit = limit or 0
272-
first_limit = _min_for_limit_param(limit, effective_chunk)
273187

274-
# Can be awaited multiple times with same result, but not scheduled at this time yet, as it might be pre-emptive.
275-
fetch_first_page = _LazyTask(callback(**{**kwargs, cursor_param: initial_cursor, 'limit': first_limit}))
276-
277-
async def get_async_iterator() -> AsyncIterator[Any]:
188+
async def get_async_iterator() -> AsyncIterator[T]:
278189
current_page = await fetch_first_page
279190
for item in current_page.items:
280191
yield item
@@ -291,7 +202,4 @@ async def get_async_iterator() -> AsyncIterator[Any]:
291202
fetched += len(current_page.items)
292203
next_cursor = getattr(current_page, f'next_{cursor_param}')
293204

294-
async def wrap_first_page() -> ListPage[Any]:
295-
return ListPage(await fetch_first_page)
296-
297-
return IterableListPageAsync(wrap_first_page, get_async_iterator)
205+
return get_async_iterator

0 commit comments

Comments
 (0)