11from __future__ import annotations
22
33import asyncio
4- from collections .abc import AsyncIterable , AsyncIterator , Awaitable , Callable , Generator , Iterable , Iterator , Coroutine
5- from typing import Any , Generic , TypeVar
4+ from collections .abc import AsyncIterable , AsyncIterator , Awaitable , Callable , Coroutine , Generator , Iterable , Iterator
5+ from typing import Any , Generic , Protocol , TypeVar
66
77from apify_client ._docs import docs_group
88
99T = TypeVar ('T' )
1010
1111
12+ class HasItems (Protocol [T ]):
13+ items : list [T ]
14+
15+
1216def _min_for_limit_param (a : int | None , b : int | None ) -> int | None :
1317 """Return minimum of two limit parameters, treating `None` or `0` as infinity.
1418
@@ -26,15 +30,13 @@ def _min_for_limit_param(a: int | None, b: int | None) -> int | None:
2630 return min (a , b )
2731
2832
29- T = TypeVar ('T' )
30-
31-
3233class _LazyTask (Generic [T ]):
3334 """Task that is created lazily upon awaiting.
3435
3536 This allows to reuse the same Task multiple times without the need to schedule the task when it is created.
3637 """
37- def __init__ (self , awaitable : Awaitable [T ]) -> None :
38+
39+ def __init__ (self , awaitable : Coroutine [Any , Any , T ]) -> None :
3840 self ._awaitable = awaitable
3941 self ._task : asyncio .Task [T ] | None = None
4042
@@ -44,7 +46,6 @@ def __await__(self) -> Generator[Any, None, T]:
4446 return (yield from self ._task .__await__ ())
4547
4648
47-
4849@docs_group ('Other' )
4950class IterableListPage (Iterable [T ], Generic [T ]):
5051 """A page of results that can also be iterated to yield items across subsequent pages.
@@ -72,7 +73,7 @@ class IterableListPage(Iterable[T], Generic[T]):
7273 desc : bool
7374 """Whether the items are sorted in descending order."""
7475
75- def __init__ (self , first_page : Any , iterator : Iterator [T ]) -> None :
76+ def __init__ (self , first_page : HasItems [ T ] , iterator : Iterator [T ]) -> None :
7677 """Initialize a page wrapper from a Pydantic paginated model and an iterator over all items."""
7778 self .items = first_page .items
7879 count = getattr (first_page , 'count' , None )
@@ -119,15 +120,24 @@ def __await__(self) -> Generator[Any, Any, IterableListPage[T]]:
119120
120121
121122def build_iterable_list_page (
122- callback : Callable [..., Any ],
123+ callback : Callable [..., HasItems [ T ] ],
123124 ** kwargs : Any ,
124- ) -> IterableListPage [Any ]:
125+ ) -> IterableListPage [T ]:
125126 """Build an `IterableListPage` from a paginated sync callback.
126127
127128 The callback is invoked once immediately to fetch the first page, and again lazily during
128- iteration to fetch further pages. The `total` field from the first page is not trusted for
129- stopping iteration because it may change between calls; iteration stops when a page has
130- no items or when the user-requested `limit` has been reached.
129+ iteration to fetch further pages.
130+
131+ There are several optional kwargs that control the pagination, but not all are accepted on each paginated endpoint.
132+ Some endpoints do not return all paginated metadata, so the implementation should be resilient to missing fields,
133+ but it can use them if available.
134+
135+ The `total` field from the first page is not trusted for stopping iteration because it may change between calls;
136+ iteration stops when a page has no items or when the user-requested `limit` has been reached.
137+
138+ The `count` field does not count objects returned, but object scanned by the API. For example when using filters,
139+ returned items can be smaller than `count`. Therefore, `count` should be used for correct offset calculation if
140+ available.
131141
132142 Iteration relevant kwargs:
133143 chunk_size: Maximum number of items requested per API call during iteration. Pass `0`
@@ -142,11 +152,11 @@ def build_iterable_list_page(
142152
143153 first_page = callback (** {** kwargs , 'limit' : _min_for_limit_param (kwargs .get ('limit' ), chunk_size )})
144154
145- def iterator () -> Iterator [Any ]:
155+ def iterator () -> Iterator [T ]:
146156 current_page = first_page
147157 yield from current_page .items
148158
149- fetched_items = current_page . count
159+ fetched_items = getattr ( current_page , ' count' , len ( current_page . items ))
150160 while current_page .items and (not limit or (limit > fetched_items )):
151161 new_kwargs = {
152162 ** kwargs ,
@@ -155,15 +165,15 @@ def iterator() -> Iterator[Any]:
155165 }
156166 current_page = callback (** new_kwargs )
157167 yield from current_page .items
158- fetched_items += current_page . count
168+ fetched_items += getattr ( current_page , ' count' , len ( current_page . items ))
159169
160170 return IterableListPage (first_page , iterator ())
161171
162172
163173def build_iterable_list_page_async (
164- callback : Callable [..., Awaitable [Any ]],
174+ callback : Callable [..., Coroutine [Any , Any , HasItems [ T ] ]],
165175 ** kwargs : Any ,
166- ) -> IterableListPageAsync [Any ]:
176+ ) -> IterableListPageAsync [T ]:
167177 """Build an `IterableListPageAsync` from a paginated async callback.
168178
169179 Mirrors `build_iterable_list_page` but for async callbacks. The returned object is both
@@ -182,7 +192,7 @@ async def async_iterator() -> AsyncIterator[Any]:
182192 for item in current_page .items :
183193 yield item
184194
185- fetched_items = current_page . count
195+ fetched_items = getattr ( current_page , ' count' , len ( current_page . items ))
186196 while current_page .items and (not limit or (limit > fetched_items )):
187197 new_kwargs = {
188198 ** kwargs ,
@@ -192,7 +202,7 @@ async def async_iterator() -> AsyncIterator[Any]:
192202 current_page = await callback (** new_kwargs )
193203 for item in current_page .items :
194204 yield item
195- fetched_items += current_page . count
205+ fetched_items += getattr ( current_page , ' count' , len ( current_page . items ))
196206
197207 async def wrap_first_page () -> IterableListPage [Any ]:
198208 first_page = await fetch_first_page
@@ -202,15 +212,14 @@ async def wrap_first_page() -> IterableListPage[Any]:
202212
203213
204214def build_cursor_iterable_list_page (
205- callback : Callable [..., Any ],
215+ callback : Callable [..., HasItems [ T ] ],
206216 * ,
207217 cursor_param : str ,
208- next_cursor_fn : Callable [[Any ], Any ],
209218 initial_cursor : Any = None ,
210219 limit : int | None = None ,
211220 chunk_size : int | None = None ,
212221 ** kwargs : Any ,
213- ) -> IterableListPage [Any ]:
222+ ) -> IterableListPage [T ]:
214223 """Build an `IterableListPage` for endpoints that paginate with a cursor instead of an offset.
215224
216225 The callback is invoked with `{cursor_param: cursor, 'limit': effective_limit, **kwargs}` for each
@@ -229,29 +238,28 @@ def iterator() -> Iterator[Any]:
229238 yield from current_page .items
230239
231240 fetched = len (current_page .items )
232- next_cursor = next_cursor_fn (current_page )
241+ next_cursor = getattr (current_page , f'next_ { cursor_param } ' )
233242
234243 while current_page .items and next_cursor is not None and (not user_limit or user_limit > fetched ):
235244 remaining = (user_limit - fetched ) if user_limit else 0
236245 next_limit = effective_chunk if not user_limit else _min_for_limit_param (remaining , effective_chunk )
237246 current_page = callback (** {** kwargs , cursor_param : next_cursor , 'limit' : next_limit })
238247 yield from current_page .items
239248 fetched += len (current_page .items )
240- next_cursor = next_cursor_fn (current_page )
249+ next_cursor = getattr (current_page , f'next_ { cursor_param } ' )
241250
242251 return IterableListPage (first_page , iterator ())
243252
244253
245254def build_cursor_iterable_list_page_async (
246- callback : Callable [..., Awaitable [Any ]],
255+ callback : Callable [..., Awaitable [HasItems [ T ] ]],
247256 * ,
248257 cursor_param : str ,
249- next_cursor_fn : Callable [[Any ], Any ],
250258 initial_cursor : Any = None ,
251259 limit : int | None = None ,
252260 chunk_size : int | None = None ,
253261 ** kwargs : Any ,
254- ) -> IterableListPageAsync [Any ]:
262+ ) -> IterableListPageAsync [T ]:
255263 """Build an `IterableListPageAsync` for endpoints that paginate with a cursor instead of an offset.
256264
257265 Mirrors `build_cursor_iterable_list_page` but for async callbacks. The returned object is both
@@ -271,7 +279,7 @@ async def async_iterator() -> AsyncIterator[Any]:
271279 yield item
272280
273281 fetched = len (current_page .items )
274- next_cursor = next_cursor_fn (current_page )
282+ next_cursor = getattr (current_page , f'next_ { cursor_param } ' )
275283
276284 while current_page .items and next_cursor is not None and (not user_limit or user_limit > fetched ):
277285 remaining = (user_limit - fetched ) if user_limit else 0
@@ -280,7 +288,7 @@ async def async_iterator() -> AsyncIterator[Any]:
280288 for item in current_page .items :
281289 yield item
282290 fetched += len (current_page .items )
283- next_cursor = next_cursor_fn (current_page )
291+ next_cursor = getattr (current_page , f'next_ { cursor_param } ' )
284292
285293 async def wrap_first_page () -> IterableListPage [Any ]:
286294 first_page = await fetch_first_page ()
0 commit comments