11from __future__ import annotations
22
33import asyncio
4- from collections .abc import AsyncIterable , AsyncIterator , Awaitable , Callable , Coroutine , Generator , Iterable , Iterator
5- from typing import Any , Generic , Protocol , TypeVar
4+ from typing import TYPE_CHECKING , Any , Generic , Protocol , TypeVar
65
7- from apify_client ._docs import docs_group
6+ if TYPE_CHECKING :
7+ from collections .abc import AsyncIterator , Awaitable , Callable , Coroutine , Generator , Iterator
88
99T = TypeVar ('T' )
1010
@@ -46,88 +46,6 @@ def __await__(self) -> Generator[Any, None, T]:
4646 return (yield from self ._task .__await__ ())
4747
4848
49- @docs_group ('Other' )
50- class ListPage (Generic [T ]):
51- """A page of API results.
52-
53- Different endpoints may return different subsets of the available pagination metadata fields, the only field that
54- is common for all endpoints is items.
55- """
56-
57- items : list [T ]
58- """List of items on this page."""
59-
60- count : int | None
61- """Number of items on this page."""
62-
63- offset : int | None
64- """The starting offset of this page."""
65-
66- limit : int | None
67- """The maximum number of items per page requested from the API."""
68-
69- total : int | None
70- """Total number of items matching the query, as reported by the first page."""
71-
72- desc : bool | None
73- """Whether the items are sorted in descending order."""
74-
75- def __init__ (self , first_page : HasItems [T ]) -> None :
76- """Initialize a page wrapper from a Pydantic paginated model."""
77- self .items = first_page .items
78- self .count = getattr (first_page , 'count' , None )
79- self .offset = getattr (first_page , 'offset' , None )
80- self .limit = getattr (first_page , 'limit' , None )
81- self .total = getattr (first_page , 'total' , None )
82- self .desc = getattr (first_page , 'desc' , None )
83-
84-
85- @docs_group ('Other' )
86- class IterableListPage (ListPage [T ], Iterable [T ], Generic [T ]):
87- """A page of results that can also be iterated to yield items across subsequent pages.
88-
89- Accessing fields such as `items`, `count`, or `total` returns the metadata of the first page,
90- preserving the behavior of the previous non-iterable return value. Iterating with `for item in ...`
91- yields individual items and performs additional API calls as needed to fetch further pages.
92- """
93-
94- def __init__ (self , first_page : HasItems [T ], get_iterator : Callable [[], Iterator [T ]]) -> None :
95- """Initialize a page wrapper from a Pydantic paginated model and an iterator over all items."""
96- super ().__init__ (first_page )
97- self ._get_iterator = get_iterator
98-
99- def __iter__ (self ) -> Iterator [T ]:
100- """Return an iterator over all items across pages, fetching additional pages as needed."""
101- return self ._get_iterator ()
102-
103-
104- @docs_group ('Other' )
105- class IterableListPageAsync (AsyncIterable [T ], Generic [T ]):
106- """An awaitable result that can also be asynchronously iterated to yield items across pages.
107-
108- Awaiting the instance (`await client.list(...)`) performs a single API call and returns a
109- populated `ListPage`. Iterating (`async for item in client.list(...)`) yields individual
110- items and performs additional API calls as needed to fetch further pages.
111- """
112-
113- def __init__ (
114- self ,
115- awaitable_first_page : Callable [[], Awaitable [ListPage [T ]]],
116- get_async_iterator : Callable [[], AsyncIterator [T ]],
117- ) -> None :
118- """Initialize with a factory that creates the awaitable on demand and the async iterator over items."""
119- self ._awaitable_first_page = awaitable_first_page
120- self ._get_async_iterator = get_async_iterator
121-
122- def __aiter__ (self ) -> AsyncIterator [T ]:
123- """Return an asynchronous iterator over all items across pages."""
124- return self ._get_async_iterator ()
125-
126- def __await__ (self ) -> Generator [Any , Any , ListPage [T ]]:
127- """Return an awaitable that resolves to a `ListPage` containing the first page of results."""
128- return self ._awaitable_first_page ().__await__ ()
129-
130-
13149def build_iterable_list_page (
13250 callback : Callable [..., HasItems [T ]],
13351 first_page : HasItems [T ],
@@ -159,6 +77,7 @@ def build_iterable_list_page(
15977 chunk_size = kwargs .pop ('chunk_size' , 0 ) or 0
16078 offset = kwargs .get ('offset' ) or 0
16179 limit = kwargs .get ('limit' ) or 0
80+
16281 def get_iterator () -> Iterator [T ]:
16382 current_page = first_page
16483 yield from current_page .items
@@ -179,7 +98,7 @@ def get_iterator() -> Iterator[T]:
17998
18099def build_iterable_list_page_async (
181100 callback : Callable [..., Coroutine [Any , Any , HasItems [T ]]],
182- fetch_first_page ,
101+ fetch_first_page : Awaitable [ HasItems [ T ]] ,
183102 ** kwargs : Any ,
184103) -> Callable [[], AsyncIterator [T ]]:
185104 """Build an `IterableListPageAsync` from a paginated async callback.
@@ -214,27 +133,24 @@ async def get_async_iterator() -> AsyncIterator[T]:
214133
215134def build_cursor_iterable_list_page (
216135 callback : Callable [..., HasItems [T ]],
136+ first_page : HasItems [T ],
217137 * ,
218138 cursor_param : str ,
219- initial_cursor : Any = None ,
220139 limit : int | None = None ,
221140 chunk_size : int | None = None ,
222141 ** kwargs : Any ,
223- ) -> IterableListPage [ T ]:
224- """Build an `IterableListPage` for endpoints that paginate with a cursor instead of an offset.
142+ ) -> Callable [[], Iterator [ T ] ]:
143+ """Build the iterator factory for endpoints that paginate with a cursor instead of an offset.
225144
226- The callback is invoked with `{cursor_param: cursor, 'limit': effective_limit, **kwargs}` for each
227- page, starting from `initial_cursor` . After each page, `next_cursor_fn (page)` is consulted to
228- obtain the next cursor; returning `None` ends iteration. The iteration also stops when a page is
229- empty or when the caller-requested `limit` has been reached.
145+ The caller is responsible for fetching the first page (typically by calling `callback` with
146+ the initial cursor) . After each page, `getattr (page, f'next_{cursor_param}' )` is consulted
147+ to obtain the next cursor; returning `None` ends iteration. The iteration also stops when a
148+ page is empty or when the caller-requested `limit` has been reached.
230149 """
231150 effective_chunk = chunk_size or 0
232151 user_limit = limit or 0
233152
234- first_limit = _min_for_limit_param (limit , effective_chunk )
235- first_page = callback (** {** kwargs , cursor_param : initial_cursor , 'limit' : first_limit })
236-
237- def get_iterator () -> Iterator [Any ]:
153+ def get_iterator () -> Iterator [T ]:
238154 current_page = first_page
239155 yield from current_page .items
240156
@@ -249,32 +165,27 @@ def get_iterator() -> Iterator[Any]:
249165 fetched += len (current_page .items )
250166 next_cursor = getattr (current_page , f'next_{ cursor_param } ' )
251167
252- return IterableListPage ( first_page , get_iterator )
168+ return get_iterator
253169
254170
255171def build_cursor_iterable_list_page_async (
256172 callback : Callable [..., Coroutine [Any , Any , HasItems [T ]]],
173+ fetch_first_page : Awaitable [HasItems [T ]],
257174 * ,
258175 cursor_param : str ,
259- initial_cursor : Any = None ,
260176 limit : int | None = None ,
261177 chunk_size : int | None = None ,
262178 ** kwargs : Any ,
263- ) -> IterableListPageAsync [ T ]:
264- """Build an `IterableListPageAsync` for endpoints that paginate with a cursor instead of an offset.
179+ ) -> Callable [[], AsyncIterator [ T ] ]:
180+ """Build the async iterator factory for endpoints that paginate with a cursor instead of an offset.
265181
266- Mirrors `build_cursor_iterable_list_page` but for async callbacks. The returned object is both
267- awaitable (resolves to the first page wrapped in `IterableListPage`) and asynchronously iterable
268- (yields items across pages using the supplied cursor strategy).
182+ Mirrors `build_cursor_iterable_list_page` but for async callbacks. The caller is responsible
183+ for awaiting the first page (typically through a `_LazyTask` wrapping the initial callback).
269184 """
270185 effective_chunk = chunk_size or 0
271186 user_limit = limit or 0
272- first_limit = _min_for_limit_param (limit , effective_chunk )
273187
274- # Can be awaited multiple times with same result, but not scheduled at this time yet, as it might be pre-emptive.
275- fetch_first_page = _LazyTask (callback (** {** kwargs , cursor_param : initial_cursor , 'limit' : first_limit }))
276-
277- async def get_async_iterator () -> AsyncIterator [Any ]:
188+ async def get_async_iterator () -> AsyncIterator [T ]:
278189 current_page = await fetch_first_page
279190 for item in current_page .items :
280191 yield item
@@ -291,7 +202,4 @@ async def get_async_iterator() -> AsyncIterator[Any]:
291202 fetched += len (current_page .items )
292203 next_cursor = getattr (current_page , f'next_{ cursor_param } ' )
293204
294- async def wrap_first_page () -> ListPage [Any ]:
295- return ListPage (await fetch_first_page )
296-
297- return IterableListPageAsync (wrap_first_page , get_async_iterator )
205+ return get_async_iterator
0 commit comments