Skip to content

Commit e8e4dfc

Browse files
committed
Fixes & minor polishment
1 parent 1259abb commit e8e4dfc

10 files changed

Lines changed: 129 additions & 151 deletions

File tree

‎docs/02_concepts/08_pagination.mdx‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ import PaginationAsyncExample from '!!raw-loader!./code/08_pagination_async.py';
1414
import PaginationSyncExample from '!!raw-loader!./code/08_pagination_sync.py';
1515
import IterateItemsAsyncExample from '!!raw-loader!./code/08_iterate_items_async.py';
1616
import IterateItemsSyncExample from '!!raw-loader!./code/08_iterate_items_sync.py';
17+
import IterateCollectionAsyncExample from '!!raw-loader!./code/08_iterate_collection_async.py';
18+
import IterateCollectionSyncExample from '!!raw-loader!./code/08_iterate_collection_sync.py';
1719

1820
Most methods named `list` or `list_something` in the Apify client return a <ApiLink to="class/ListPage">`ListPage`</ApiLink> object. This object provides a consistent interface for working with paginated data and includes the following properties:
1921

@@ -49,6 +51,24 @@ to retrieve every item matching the filters. For dataset, key-value store and re
4951
matching helpers are `iterate_items`, `iterate_keys` and `iterate_requests`. They handle pagination
5052
automatically, so you don't need to manage offsets, limits or cursors yourself.
5153

54+
The example below iterates over every Actor owned by the current user using a collection client's `iterate`
55+
method:
56+
57+
<Tabs>
58+
<TabItem value="AsyncExample" label="Async client" default>
59+
<CodeBlock className="language-python">
60+
{IterateCollectionAsyncExample}
61+
</CodeBlock>
62+
</TabItem>
63+
<TabItem value="SyncExample" label="Sync client">
64+
<CodeBlock className="language-python">
65+
{IterateCollectionSyncExample}
66+
</CodeBlock>
67+
</TabItem>
68+
</Tabs>
69+
70+
The next example uses `iterate_items` on a dataset client to stream items past a given offset:
71+
5272
<Tabs>
5373
<TabItem value="AsyncExample" label="Async client" default>
5474
<CodeBlock className="language-python">
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
from apify_client import ApifyClientAsync
2+
3+
TOKEN = 'MY-APIFY-TOKEN'
4+
5+
6+
async def main() -> None:
7+
apify_client = ApifyClientAsync(TOKEN)
8+
9+
# Iterate over all Actors owned by the current user, lazily fetching
10+
# as many pages as needed under the hood.
11+
async for actor in apify_client.actors().iterate(my=True):
12+
print(actor.id)
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
from apify_client import ApifyClient
2+
3+
TOKEN = 'MY-APIFY-TOKEN'
4+
5+
6+
def main() -> None:
7+
apify_client = ApifyClient(TOKEN)
8+
9+
# Iterate over all Actors owned by the current user, lazily fetching
10+
# as many pages as needed under the hood.
11+
for actor in apify_client.actors().iterate(my=True):
12+
print(actor.id)
13+
14+
15+
if __name__ == '__main__':
16+
main()

‎scripts/_utils.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
(re.compile(r'\bSynchronous\b'), 'Asynchronous'),
2828
(re.compile(r'Retry a function'), 'Retry an async function'),
2929
(re.compile(r'Function to retry'), 'Async function to retry'),
30-
(re.compile(r'returned page also supports iteration: `for'), 'returned page also supports iteration: `async for'),
3130
]
3231
"""Patterns for converting sync docstrings to async docstrings."""
3332

‎src/apify_client/_pagination.py‎

Lines changed: 42 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
from __future__ import annotations
22

3-
from typing import TYPE_CHECKING, Protocol, TypeVar, overload
4-
5-
from apify_client._models import KeyValueStoreKey, ListOfKeys, ListOfRequests, Request
3+
from typing import TYPE_CHECKING, Any, Protocol, TypeVar
64

75
if TYPE_CHECKING:
86
from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
@@ -14,23 +12,6 @@ class HasItems(Protocol[T]):
1412
items: list[T]
1513

1614

17-
def _min_for_limit_param(a: int | None, b: int | None) -> int | None:
18-
"""Return minimum of two limit parameters, treating `None` or `0` as infinity.
19-
20-
The Apify API treats `0` as no limit for the `limit` parameter, so `0` here means infinity.
21-
Returns `None` when both inputs represent infinity.
22-
"""
23-
if a == 0:
24-
a = None
25-
if b == 0:
26-
b = None
27-
if a is None:
28-
return b
29-
if b is None:
30-
return a
31-
return min(a, b)
32-
33-
3415
def get_items_iterator(
3516
callback: Callable[..., HasItems[T]],
3617
*,
@@ -40,13 +21,13 @@ def get_items_iterator(
4021
) -> Iterator[T]:
4122
"""Yield individual items from offset-based paginated API responses.
4223
43-
The `callback` is invoked lazily to fetch each page from the API. It must accept `limit` and
44-
`offset` keyword arguments and return an object whose `items` attribute is a list. If the
45-
object also exposes a `count` attribute, it is used for offset bookkeeping (the Apify API's
46-
`count` reflects items scanned, which can exceed items returned when filters are applied).
24+
The `callback` is invoked lazily to fetch each page from the API. It must accept `limit` and `offset` keyword
25+
arguments and return an object whose `items` attribute is a list. If the object also exposes a `count` attribute, it
26+
is used for offset bookkeeping (the Apify API's `count` reflects items scanned, which can exceed items returned when
27+
filters are applied).
4728
48-
Iteration stops when a page returns no items or when the user-requested `limit` is reached.
49-
The `total` field is intentionally not consulted, because it can change between calls.
29+
Iteration stops when a page returns no items or when the user-requested `limit` is reached. The `total` field is
30+
intentionally not consulted, because it can change between calls.
5031
5132
Args:
5233
callback: Function returning a single page of items.
@@ -61,14 +42,12 @@ def get_items_iterator(
6142

6243
while True:
6344
current_page = callback(
64-
limit=effective_chunk
65-
if not initial_limit
66-
else _min_for_limit_param(initial_limit - fetched_items, effective_chunk),
45+
limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
6746
offset=initial_offset + fetched_items,
6847
)
6948
yield from current_page.items
7049

71-
fetched_items += getattr(current_page, 'count', len(current_page.items))
50+
fetched_items += max(getattr(current_page, 'count', 0), len(current_page.items))
7251

7352
if not current_page.items or (initial_limit and fetched_items >= initial_limit):
7453
break
@@ -92,54 +71,35 @@ async def get_items_iterator_async(
9271

9372
while True:
9473
current_page = await callback(
95-
limit=effective_chunk
96-
if not initial_limit
97-
else _min_for_limit_param(initial_limit - fetched_items, effective_chunk),
74+
limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
9875
offset=initial_offset + fetched_items,
9976
)
10077
for item in current_page.items:
10178
yield item
10279

103-
fetched_items += getattr(current_page, 'count', len(current_page.items))
80+
fetched_items += max(getattr(current_page, 'count', 0), len(current_page.items))
10481

10582
if not current_page.items or (initial_limit and fetched_items >= initial_limit):
10683
break
10784

10885

109-
@overload
110-
def get_cursor_iterator(
111-
callback: Callable[..., ListOfKeys],
112-
*,
113-
cursor: str | None = None,
114-
limit: int | None = None,
115-
chunk_size: int | None = None,
116-
) -> Iterator[KeyValueStoreKey]: ...
117-
@overload
11886
def get_cursor_iterator(
119-
callback: Callable[..., ListOfRequests],
120-
*,
121-
cursor: str | None = None,
122-
limit: int | None = None,
123-
chunk_size: int | None = None,
124-
) -> Iterator[Request]: ...
125-
126-
127-
def get_cursor_iterator(
128-
callback: Callable[..., ListOfKeys] | Callable[..., ListOfRequests],
87+
callback: Callable[..., HasItems[T]],
12988
*,
89+
next_cursor: Callable[[Any], str | None],
13090
cursor: str | None = None,
13191
limit: int | None = None,
13292
chunk_size: int | None = None,
133-
) -> Iterator[Request] | Iterator[KeyValueStoreKey]:
93+
) -> Iterator[T]:
13494
"""Yield individual items from cursor-paginated API responses.
13595
136-
Each page is expected to expose `items` and `next_<cursor_param>`; iteration ends when a
137-
page returns no items, the next cursor is `None`, or the user-requested `limit` is reached.
96+
Each page is expected to expose `items`; iteration ends when a page returns no items, the cursor extracted by
97+
`next_cursor` is `None`, or the user-requested `limit` is reached.
13898
13999
Args:
140-
callback: Function returning a single page of items. Receives the cursor as a kwarg
141-
named after `cursor_param` and a `limit` kwarg.
142-
cursor_param: Name of the cursor query-parameter (e.g. `cursor` or `exclusive_start_key`).
100+
callback: Function returning a single page of items. Receives `cursor` and `limit` kwargs.
101+
next_cursor: Callable that extracts the next-page cursor from the returned page (e.g. `lambda p: p.next_cursor`)
102+
and returns `None` when there are no more pages.
143103
cursor: Value of the cursor for the first request, or `None` to start from the beginning.
144104
limit: Maximum total number of items to yield across all pages.
145105
chunk_size: Maximum number of items requested per API call.
@@ -150,74 +110,55 @@ def get_cursor_iterator(
150110

151111
while True:
152112
current_page = callback(
153-
limit=effective_chunk
154-
if not initial_limit
155-
else _min_for_limit_param(initial_limit - fetched_items, effective_chunk),
113+
limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
156114
cursor=cursor,
157115
)
158116
yield from current_page.items
159117

160-
fetched_items += getattr(current_page, 'count', len(current_page.items))
161-
162-
if isinstance(current_page, ListOfKeys):
163-
cursor = current_page.next_exclusive_start_key
164-
elif isinstance(current_page, ListOfRequests):
165-
cursor = current_page.next_cursor
166-
else:
167-
raise TypeError('Unsupported page type returned by callback; expected ListOfKeys or ListOfRequests.')
118+
fetched_items += max(getattr(current_page, 'count', 0), len(current_page.items))
119+
cursor = next_cursor(current_page)
168120

169121
if not current_page.items or cursor is None or (initial_limit and fetched_items >= initial_limit):
170122
break
171123

172124

173-
@overload
174-
def get_cursor_iterator_async(
175-
callback: Callable[..., Awaitable[ListOfKeys]],
176-
*,
177-
cursor: str | None = None,
178-
limit: int | None = None,
179-
chunk_size: int | None = None,
180-
) -> AsyncIterator[KeyValueStoreKey]: ...
181-
@overload
182-
def get_cursor_iterator_async(
183-
callback: Callable[..., Awaitable[ListOfRequests]],
184-
*,
185-
cursor: str | None = None,
186-
limit: int | None = None,
187-
chunk_size: int | None = None,
188-
) -> AsyncIterator[Request]: ...
189-
190-
191125
async def get_cursor_iterator_async(
192-
callback: Callable[..., Awaitable[ListOfKeys]] | Callable[..., Awaitable[ListOfRequests]],
126+
callback: Callable[..., Awaitable[HasItems[T]]],
193127
*,
128+
next_cursor: Callable[[Any], str | None],
194129
cursor: str | None = None,
195130
limit: int | None = None,
196131
chunk_size: int | None = None,
197-
) -> AsyncIterator[KeyValueStoreKey] | AsyncIterator[Request]:
132+
) -> AsyncIterator[T]:
198133
"""Async variant of :func:`get_cursor_iterator`."""
199134
effective_chunk = chunk_size or 0
200135
initial_limit = limit or 0
201136
fetched_items = 0
202137

203138
while True:
204139
current_page = await callback(
205-
limit=effective_chunk
206-
if not initial_limit
207-
else _min_for_limit_param(initial_limit - fetched_items, effective_chunk),
140+
limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
208141
cursor=cursor,
209142
)
210143
for item in current_page.items:
211144
yield item
212145

213-
fetched_items += getattr(current_page, 'count', len(current_page.items))
214-
215-
if isinstance(current_page, ListOfKeys):
216-
cursor = current_page.next_exclusive_start_key
217-
elif isinstance(current_page, ListOfRequests):
218-
cursor = current_page.next_cursor
219-
else:
220-
raise TypeError('Unsupported page type returned by callback; expected ListOfKeys or ListOfRequests.')
146+
fetched_items += max(getattr(current_page, 'count', 0), len(current_page.items))
147+
cursor = next_cursor(current_page)
221148

222149
if not current_page.items or cursor is None or (initial_limit and fetched_items >= initial_limit):
223150
break
151+
152+
153+
def _next_page_limit(initial_limit: int, fetched_items: int, effective_chunk: int) -> int:
154+
"""Compute the `limit` value for the next API call.
155+
156+
`0` means no limit on the wire (matches the Apify API contract). When both an overall `initial_limit` and a per-page
157+
`effective_chunk` are set, the call is clamped to whichever is smaller; if either is unset (`0`), the other wins.
158+
"""
159+
if not initial_limit:
160+
return effective_chunk
161+
remaining = initial_limit - fetched_items
162+
if not effective_chunk:
163+
return remaining
164+
return min(remaining, effective_chunk)

‎src/apify_client/_resource_clients/actor_version_collection.py‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,6 @@ def __init__(
4949
def list(self, *, timeout: Timeout = 'short') -> ListOfVersions:
5050
"""List the available Actor versions.
5151
52-
The returned page also supports iteration: `for item in client.list()` yields individual versions.
53-
5452
https://docs.apify.com/api/v2#/reference/actors/version-collection/get-list-of-versions
5553
5654
Args:
@@ -63,7 +61,7 @@ def list(self, *, timeout: Timeout = 'short') -> ListOfVersions:
6361
return ListOfVersionsResponse.model_validate(result).data
6462

6563
def iterate(self, *, timeout: Timeout = 'short') -> Iterator[Version]:
66-
"""Iterate over the available Actor environment variables.
64+
"""Iterate over the available Actor versions.
6765
6866
There is no possibility to control the pagination on this endpoint.
6967
"""
@@ -145,8 +143,6 @@ def __init__(
145143
async def list(self, *, timeout: Timeout = 'short') -> ListOfVersions:
146144
"""List the available Actor versions.
147145
148-
The returned page also supports iteration: `async for item in client.list()` yields individual versions.
149-
150146
https://docs.apify.com/api/v2#/reference/actors/version-collection/get-list-of-versions
151147
152148
Args:
@@ -159,7 +155,7 @@ async def list(self, *, timeout: Timeout = 'short') -> ListOfVersions:
159155
return ListOfVersionsResponse.model_validate(result).data
160156

161157
async def iterate(self, *, timeout: Timeout = 'short') -> AsyncIterator[Version]:
162-
"""Iterate over the available Actor environment variables.
158+
"""Iterate over the available Actor versions.
163159
164160
There is no possibility to control the pagination on this endpoint.
165161
"""

‎src/apify_client/_resource_clients/key_value_store.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,7 @@ def _callback(*, cursor: str | None = None, limit: int | None = None) -> ListOfK
223223

224224
return get_cursor_iterator(
225225
_callback,
226+
next_cursor=lambda page: page.next_exclusive_start_key,
226227
cursor=exclusive_start_key,
227228
limit=limit,
228229
chunk_size=chunk_size or 1000,
@@ -649,6 +650,7 @@ async def _callback(*, cursor: str | None = None, limit: int | None = None) -> L
649650

650651
return get_cursor_iterator_async(
651652
_callback,
653+
next_cursor=lambda page: page.next_exclusive_start_key,
652654
cursor=exclusive_start_key,
653655
limit=limit,
654656
chunk_size=chunk_size or 1000,

‎src/apify_client/_resource_clients/request_queue.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -561,7 +561,7 @@ def iterate_requests(
561561
562562
Simple `list_requests` does only one API call, possibly not listing all items matching the criteria.
563563
This method returns an iterator that is capable of making multiple API calls to retrieve all items
564-
matching the criteria using the opaque ``cursor`` returned by the API.
564+
matching the criteria using the opaque `cursor` returned by the API.
565565
566566
https://docs.apify.com/api/v2#/reference/request-queues/request-collection/list-requests
567567
@@ -581,9 +581,10 @@ def _callback(*, cursor: str | None = None, limit: int | None = None) -> ListOfR
581581

582582
return get_cursor_iterator(
583583
_callback,
584+
next_cursor=lambda page: page.next_cursor,
584585
cursor=cursor,
585586
limit=limit,
586-
chunk_size=chunk_size,
587+
chunk_size=chunk_size or 1000,
587588
)
588589

589590
def unlock_requests(self: RequestQueueClient, *, timeout: Timeout = 'long') -> UnlockRequestsResult:
@@ -1173,7 +1174,7 @@ def iterate_requests(
11731174
11741175
Simple `list_requests` does only one API call, possibly not listing all items matching the criteria.
11751176
This method returns an iterator that is capable of making multiple API calls to retrieve all items
1176-
matching the criteria using the opaque ``cursor`` returned by the API.
1177+
matching the criteria using the opaque `cursor` returned by the API.
11771178
11781179
https://docs.apify.com/api/v2#/reference/request-queues/request-collection/list-requests
11791180
@@ -1193,9 +1194,10 @@ async def _callback(*, cursor: str | None = None, limit: int | None = None) -> L
11931194

11941195
return get_cursor_iterator_async(
11951196
_callback,
1197+
next_cursor=lambda page: page.next_cursor,
11961198
cursor=cursor,
11971199
limit=limit,
1198-
chunk_size=chunk_size,
1200+
chunk_size=chunk_size or 1000,
11991201
)
12001202

12011203
async def unlock_requests(

0 commit comments

Comments
 (0)