11from __future__ import annotations
22
3- from collections .abc import AsyncIterable , AsyncIterator , Awaitable , Callable , Generator , Iterable , Iterator
3+ import asyncio
4+ from collections .abc import AsyncIterable , AsyncIterator , Awaitable , Callable , Generator , Iterable , Iterator , Coroutine
45from typing import Any , Generic , TypeVar
56
67from apify_client ._docs import docs_group
@@ -25,6 +26,25 @@ def _min_for_limit_param(a: int | None, b: int | None) -> int | None:
2526 return min (a , b )
2627
2728
29+ T = TypeVar ('T' )
30+
31+
32+ class _LazyTask (Generic [T ]):
33+ """Task that is created lazily upon awaiting.
34+
35+ This allows to reuse the same Task multiple times without the need to schedule the task when it is created.
36+ """
37+ def __init__ (self , awaitable : Awaitable [T ]) -> None :
38+ self ._awaitable = awaitable
39+ self ._task : asyncio .Task [T ] | None = None
40+
41+ def __await__ (self ) -> Generator [Any , None , T ]:
42+ if self ._task is None :
43+ self ._task = asyncio .create_task (self ._awaitable )
44+ return (yield from self ._task .__await__ ())
45+
46+
47+
2848@docs_group ('Other' )
2949class IterableListPage (Iterable [T ], Generic [T ]):
3050 """A page of results that can also be iterated to yield items across subsequent pages.
@@ -109,7 +129,7 @@ def build_iterable_list_page(
109129 stopping iteration because it may change between calls; iteration stops when a page has
110130 no items or when the user-requested `limit` has been reached.
111131
112- Recognized kwargs:
132+ Iteration relevant kwargs:
113133 chunk_size: Maximum number of items requested per API call during iteration. Pass `0`
114134 or `None` to let the API decide (effectively infinity).
115135 limit: User-requested total item limit. Stops iteration once this many items are yielded.
@@ -126,7 +146,7 @@ def iterator() -> Iterator[Any]:
126146 current_page = first_page
127147 yield from current_page .items
128148
129- fetched_items = len ( current_page .items )
149+ fetched_items = current_page .count
130150 while current_page .items and (not limit or (limit > fetched_items )):
131151 new_kwargs = {
132152 ** kwargs ,
@@ -135,7 +155,7 @@ def iterator() -> Iterator[Any]:
135155 }
136156 current_page = callback (** new_kwargs )
137157 yield from current_page .items
138- fetched_items += len ( current_page .items )
158+ fetched_items += current_page .count
139159
140160 return IterableListPage (first_page , iterator ())
141161
@@ -154,15 +174,15 @@ def build_iterable_list_page_async(
154174 offset = kwargs .get ('offset' ) or 0
155175 limit = kwargs .get ('limit' ) or 0
156176
157- async def fetch_first_page () -> Any :
158- return await callback (** {** kwargs , 'limit' : _min_for_limit_param (kwargs .get ('limit' ), chunk_size )})
177+ # Can be awaited multiple times with same result, but not scheduled at this time yet, as it might be pre-emptive.
178+ fetch_first_page = _LazyTask ( callback (** {** kwargs , 'limit' : _min_for_limit_param (kwargs .get ('limit' ), chunk_size )}) )
159179
160180 async def async_iterator () -> AsyncIterator [Any ]:
161- current_page = await fetch_first_page ()
181+ current_page = await fetch_first_page
162182 for item in current_page .items :
163183 yield item
164184
165- fetched_items = len ( current_page .items )
185+ fetched_items = current_page .count
166186 while current_page .items and (not limit or (limit > fetched_items )):
167187 new_kwargs = {
168188 ** kwargs ,
@@ -172,10 +192,10 @@ async def async_iterator() -> AsyncIterator[Any]:
172192 current_page = await callback (** new_kwargs )
173193 for item in current_page .items :
174194 yield item
175- fetched_items += len ( current_page .items )
195+ fetched_items += current_page .count
176196
177197 async def wrap_first_page () -> IterableListPage [Any ]:
178- first_page = await fetch_first_page ()
198+ first_page = await fetch_first_page
179199 return IterableListPage (first_page , iter (first_page .items ))
180200
181201 return IterableListPageAsync (wrap_first_page , async_iterator ())
0 commit comments