11from __future__ import annotations
22
3- from typing import TYPE_CHECKING , Protocol , TypeVar , overload
4-
5- from apify_client ._models import KeyValueStoreKey , ListOfKeys , ListOfRequests , Request
3+ from typing import TYPE_CHECKING , Any , Protocol , TypeVar
64
75if TYPE_CHECKING :
86 from collections .abc import AsyncIterator , Awaitable , Callable , Iterator
@@ -14,23 +12,6 @@ class HasItems(Protocol[T]):
1412 items : list [T ]
1513
1614
17- def _min_for_limit_param (a : int | None , b : int | None ) -> int | None :
18- """Return minimum of two limit parameters, treating `None` or `0` as infinity.
19-
20- The Apify API treats `0` as no limit for the `limit` parameter, so `0` here means infinity.
21- Returns `None` when both inputs represent infinity.
22- """
23- if a == 0 :
24- a = None
25- if b == 0 :
26- b = None
27- if a is None :
28- return b
29- if b is None :
30- return a
31- return min (a , b )
32-
33-
3415def get_items_iterator (
3516 callback : Callable [..., HasItems [T ]],
3617 * ,
@@ -40,13 +21,13 @@ def get_items_iterator(
4021) -> Iterator [T ]:
4122 """Yield individual items from offset-based paginated API responses.
4223
43- The `callback` is invoked lazily to fetch each page from the API. It must accept `limit` and
44- `offset` keyword arguments and return an object whose `items` attribute is a list. If the
45- object also exposes a `count` attribute, it is used for offset bookkeeping (the Apify API's
46- `count` reflects items scanned, which can exceed items returned when filters are applied).
24+ The `callback` is invoked lazily to fetch each page from the API. It must accept `limit` and `offset` keyword
25+ arguments and return an object whose `items` attribute is a list. If the object also exposes a `count` attribute, it
26+ is used for offset bookkeeping (the Apify API's `count` reflects items scanned, which can exceed items returned when
27+ filters are applied).
4728
48- Iteration stops when a page returns no items or when the user-requested `limit` is reached.
49- The `total` field is intentionally not consulted, because it can change between calls.
29+ Iteration stops when a page returns no items or when the user-requested `limit` is reached. The `total` field is
30+ intentionally not consulted, because it can change between calls.
5031
5132 Args:
5233 callback: Function returning a single page of items.
@@ -61,14 +42,12 @@ def get_items_iterator(
6142
6243 while True :
6344 current_page = callback (
64- limit = effective_chunk
65- if not initial_limit
66- else _min_for_limit_param (initial_limit - fetched_items , effective_chunk ),
45+ limit = _next_page_limit (initial_limit , fetched_items , effective_chunk ),
6746 offset = initial_offset + fetched_items ,
6847 )
6948 yield from current_page .items
7049
71- fetched_items += getattr (current_page , 'count' , len (current_page .items ))
50+ fetched_items += max ( getattr (current_page , 'count' , 0 ) , len (current_page .items ))
7251
7352 if not current_page .items or (initial_limit and fetched_items >= initial_limit ):
7453 break
@@ -92,54 +71,35 @@ async def get_items_iterator_async(
9271
9372 while True :
9473 current_page = await callback (
95- limit = effective_chunk
96- if not initial_limit
97- else _min_for_limit_param (initial_limit - fetched_items , effective_chunk ),
74+ limit = _next_page_limit (initial_limit , fetched_items , effective_chunk ),
9875 offset = initial_offset + fetched_items ,
9976 )
10077 for item in current_page .items :
10178 yield item
10279
103- fetched_items += getattr (current_page , 'count' , len (current_page .items ))
80+ fetched_items += max ( getattr (current_page , 'count' , 0 ) , len (current_page .items ))
10481
10582 if not current_page .items or (initial_limit and fetched_items >= initial_limit ):
10683 break
10784
10885
109- @overload
110- def get_cursor_iterator (
111- callback : Callable [..., ListOfKeys ],
112- * ,
113- cursor : str | None = None ,
114- limit : int | None = None ,
115- chunk_size : int | None = None ,
116- ) -> Iterator [KeyValueStoreKey ]: ...
117- @overload
11886def get_cursor_iterator (
119- callback : Callable [..., ListOfRequests ],
120- * ,
121- cursor : str | None = None ,
122- limit : int | None = None ,
123- chunk_size : int | None = None ,
124- ) -> Iterator [Request ]: ...
125-
126-
127- def get_cursor_iterator (
128- callback : Callable [..., ListOfKeys ] | Callable [..., ListOfRequests ],
87+ callback : Callable [..., HasItems [T ]],
12988 * ,
89+ next_cursor : Callable [[Any ], str | None ],
13090 cursor : str | None = None ,
13191 limit : int | None = None ,
13292 chunk_size : int | None = None ,
133- ) -> Iterator [Request ] | Iterator [ KeyValueStoreKey ]:
93+ ) -> Iterator [T ]:
13494 """Yield individual items from cursor-paginated API responses.
13595
136- Each page is expected to expose `items` and `next_<cursor_param>` ; iteration ends when a
137- page returns no items, the next cursor is `None`, or the user-requested `limit` is reached.
96+ Each page is expected to expose `items`; iteration ends when a page returns no items, the cursor extracted by
97+ `next_cursor` is `None`, or the user-requested `limit` is reached.
13898
13999 Args:
140- callback: Function returning a single page of items. Receives the cursor as a kwarg
141- named after `cursor_param` and a `limit` kwarg.
142- cursor_param: Name of the cursor query-parameter (e.g. `cursor` or `exclusive_start_key`) .
100+ callback: Function returning a single page of items. Receives ` cursor` and `limit` kwargs.
101+ next_cursor: Callable that extracts the next-page cursor from the returned page (e.g. `lambda p: p.next_cursor`)
102+ and returns `None` when there are no more pages .
143103 cursor: Value of the cursor for the first request, or `None` to start from the beginning.
144104 limit: Maximum total number of items to yield across all pages.
145105 chunk_size: Maximum number of items requested per API call.
@@ -150,74 +110,55 @@ def get_cursor_iterator(
150110
151111 while True :
152112 current_page = callback (
153- limit = effective_chunk
154- if not initial_limit
155- else _min_for_limit_param (initial_limit - fetched_items , effective_chunk ),
113+ limit = _next_page_limit (initial_limit , fetched_items , effective_chunk ),
156114 cursor = cursor ,
157115 )
158116 yield from current_page .items
159117
160- fetched_items += getattr (current_page , 'count' , len (current_page .items ))
161-
162- if isinstance (current_page , ListOfKeys ):
163- cursor = current_page .next_exclusive_start_key
164- elif isinstance (current_page , ListOfRequests ):
165- cursor = current_page .next_cursor
166- else :
167- raise TypeError ('Unsupported page type returned by callback; expected ListOfKeys or ListOfRequests.' )
118+ fetched_items += max (getattr (current_page , 'count' , 0 ), len (current_page .items ))
119+ cursor = next_cursor (current_page )
168120
169121 if not current_page .items or cursor is None or (initial_limit and fetched_items >= initial_limit ):
170122 break
171123
172124
173- @overload
174- def get_cursor_iterator_async (
175- callback : Callable [..., Awaitable [ListOfKeys ]],
176- * ,
177- cursor : str | None = None ,
178- limit : int | None = None ,
179- chunk_size : int | None = None ,
180- ) -> AsyncIterator [KeyValueStoreKey ]: ...
181- @overload
182- def get_cursor_iterator_async (
183- callback : Callable [..., Awaitable [ListOfRequests ]],
184- * ,
185- cursor : str | None = None ,
186- limit : int | None = None ,
187- chunk_size : int | None = None ,
188- ) -> AsyncIterator [Request ]: ...
189-
190-
191125async def get_cursor_iterator_async (
192- callback : Callable [..., Awaitable [ListOfKeys ]] | Callable [..., Awaitable [ ListOfRequests ]],
126+ callback : Callable [..., Awaitable [HasItems [ T ] ]],
193127 * ,
128+ next_cursor : Callable [[Any ], str | None ],
194129 cursor : str | None = None ,
195130 limit : int | None = None ,
196131 chunk_size : int | None = None ,
197- ) -> AsyncIterator [KeyValueStoreKey ] | AsyncIterator [ Request ]:
132+ ) -> AsyncIterator [T ]:
198133 """Async variant of :func:`get_cursor_iterator`."""
199134 effective_chunk = chunk_size or 0
200135 initial_limit = limit or 0
201136 fetched_items = 0
202137
203138 while True :
204139 current_page = await callback (
205- limit = effective_chunk
206- if not initial_limit
207- else _min_for_limit_param (initial_limit - fetched_items , effective_chunk ),
140+ limit = _next_page_limit (initial_limit , fetched_items , effective_chunk ),
208141 cursor = cursor ,
209142 )
210143 for item in current_page .items :
211144 yield item
212145
213- fetched_items += getattr (current_page , 'count' , len (current_page .items ))
214-
215- if isinstance (current_page , ListOfKeys ):
216- cursor = current_page .next_exclusive_start_key
217- elif isinstance (current_page , ListOfRequests ):
218- cursor = current_page .next_cursor
219- else :
220- raise TypeError ('Unsupported page type returned by callback; expected ListOfKeys or ListOfRequests.' )
146+ fetched_items += max (getattr (current_page , 'count' , 0 ), len (current_page .items ))
147+ cursor = next_cursor (current_page )
221148
222149 if not current_page .items or cursor is None or (initial_limit and fetched_items >= initial_limit ):
223150 break
151+
152+
153+ def _next_page_limit (initial_limit : int , fetched_items : int , effective_chunk : int ) -> int :
154+ """Compute the `limit` value for the next API call.
155+
156+ `0` means no limit on the wire (matches the Apify API contract). When both an overall `initial_limit` and a per-page
157+ `effective_chunk` are set, the call is clamped to whichever is smaller; if either is unset (`0`), the other wins.
158+ """
159+ if not initial_limit :
160+ return effective_chunk
161+ remaining = initial_limit - fetched_items
162+ if not effective_chunk :
163+ return remaining
164+ return min (remaining , effective_chunk )
0 commit comments