Skip to content

Commit 82ee01f

Browse files
committed
Generic ListPage approach
1 parent 5659c0e commit 82ee01f

8 files changed

Lines changed: 76 additions & 79 deletions

File tree

‎docs/02_concepts/08_pagination.mdx‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@ import ApiLink from '@site/src/components/ApiLink';
1313
import PaginationAsyncExample from '!!raw-loader!./code/08_pagination_async.py';
1414
import PaginationSyncExample from '!!raw-loader!./code/08_pagination_sync.py';
1515

16-
import IterateItemsAsyncExample from '!!raw-loader!./code/08_iterate_items_async.py';
17-
import IterateItemsSyncExample from '!!raw-loader!./code/08_iterate_items_sync.py';
1816

1917
Most methods named `list` or `list_something` in the Apify client return a <ApiLink to="class/ListPage">`ListPage`</ApiLink> object. This object provides a consistent interface for working with paginated data and includes the following properties:
2018

@@ -45,7 +43,7 @@ The <ApiLink to="class/ListPage">`ListPage`</ApiLink> interface offers several k
4543

4644
## Generator-based iteration
4745

48-
For most use cases, `iterate_items()` is the recommended way to process all items in a dataset. It handles pagination automatically using a Python generator, fetching items in batches behind the scenes so you don't need to manage offsets or limits yourself.
46+
You can also use the `list` methods directly in iteration. It handles pagination automatically, fetching items in batches behind the scenes so you don't need to manage offsets or limits yourself.
4947

5048
<Tabs>
5149
<TabItem value="AsyncExample" label="Async client" default>
@@ -60,6 +58,4 @@ For most use cases, `iterate_items()` is the recommended way to process all item
6058
</TabItem>
6159
</Tabs>
6260

63-
`iterate_items()` accepts the same filtering parameters as `list_items()` (`clean`, `fields`, `omit`, `unwind`, `skip_empty`, `skip_hidden`), so you can combine automatic pagination with data filtering.
64-
6561
Similarly, `KeyValueStoreClient` provides an `iterate_keys()` method for iterating over all keys in a key-value store without manual pagination.

‎docs/02_concepts/code/08_iterate_items_async.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ async def main() -> None:
77
apify_client = ApifyClientAsync(TOKEN)
88
dataset_client = apify_client.dataset('dataset-id')
99

10-
# Iterate through all items automatically.
11-
async for item in dataset_client.iterate_items():
12-
print(item)
10+
# Define the pagination parameters
11+
limit = 1500 # Number of items in total
12+
offset = 100 # Starting offset
13+
14+
# Iterate through items automatically, lazily sending as many API calls
15+
# as needed and receiving items in chunks.
16+
async for item in dataset_client.list_items(limit=limit, offset=offset):
17+
print(item) # Process the item as needed

‎docs/02_concepts/code/08_iterate_items_sync.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,14 @@ def main() -> None:
77
apify_client = ApifyClient(TOKEN)
88
dataset_client = apify_client.dataset('dataset-id')
99

10-
# Iterate through all items automatically.
11-
for item in dataset_client.iterate_items():
12-
print(item)
10+
# Define the pagination parameters
11+
limit = 1500 # Number of items in total
12+
offset = 100 # Starting offset
13+
14+
# Iterate through items automatically, lazily sending as many API calls
15+
# as needed and receiving items in chunks.
16+
for item in dataset_client.list_items(limit=limit, offset=offset):
17+
print(item) # Process the item as needed
1318

1419

1520
if __name__ == '__main__':

‎docs/02_concepts/code/08_pagination_async.py‎

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -10,26 +10,15 @@ async def main() -> None:
1010
dataset_client = apify_client.dataset('dataset-id')
1111

1212
# Define the pagination parameters
13-
limit = 1000 # Number of items per page
13+
limit = 1000 # Number items to request from API
1414
offset = 0 # Starting offset
15-
all_items = [] # List to store all fetched items
1615

17-
while True:
18-
# Fetch a page of items
19-
response = await dataset_client.list_items(limit=limit, offset=offset)
20-
items = response.items
21-
total = response.total
16+
# Send single API call to fetch paginated items.
17+
# (number of items per single call can be limited by API)
18+
paginated_items = await dataset_client.list_items(limit=limit, offset=offset)
2219

23-
print(f'Fetched {len(items)} items')
20+
# Inspect pagination metadata returned by API
21+
print(paginated_items.total)
2422

25-
# Add the fetched items to the complete list
26-
all_items.extend(items)
27-
28-
# Exit the loop if there are no more items to fetch
29-
if offset + limit >= total:
30-
break
31-
32-
# Increment the offset for the next page
33-
offset += limit
34-
35-
print(f'Overall fetched {len(all_items)} items')
23+
for item in paginated_items.items:
24+
print(item) # Process the item as needed

‎docs/02_concepts/code/08_pagination_sync.py‎

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -10,26 +10,15 @@ def main() -> None:
1010
dataset_client = apify_client.dataset('dataset-id')
1111

1212
# Define the pagination parameters
13-
limit = 1000 # Number of items per page
13+
limit = 1000 # Number items to request from API
1414
offset = 0 # Starting offset
15-
all_items = [] # List to store all fetched items
1615

17-
while True:
18-
# Fetch a page of items
19-
response = dataset_client.list_items(limit=limit, offset=offset)
20-
items = response.items
21-
total = response.total
16+
# Send single API call to fetch paginated items.
17+
# (number of items per single call can be limited by API)
18+
paginated_items = dataset_client.list_items(limit=limit, offset=offset)
2219

23-
print(f'Fetched {len(items)} items')
20+
# Inspect pagination metadata returned by API
21+
print(paginated_items.total)
2422

25-
# Add the fetched items to the complete list
26-
all_items.extend(items)
27-
28-
# Exit the loop if there are no more items to fetch
29-
if offset + limit >= total:
30-
break
31-
32-
# Increment the offset for the next page
33-
offset += limit
34-
35-
print(f'Overall fetched {len(all_items)} items')
23+
for item in paginated_items.items:
24+
print(item) # Process the item as needed

‎src/apify_client/_iterable_list_page.py‎

Lines changed: 34 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -47,42 +47,53 @@ def __await__(self) -> Generator[Any, None, T]:
4747

4848

4949
@docs_group('Other')
50-
class IterableListPage(Iterable[T], Generic[T]):
51-
"""A page of results that can also be iterated to yield items across subsequent pages.
50+
class ListPage(Generic[T]):
51+
"""A page of API results.
5252
53-
Accessing fields such as `items`, `count`, or `total` returns the metadata of the first page,
54-
preserving the behavior of the previous non-iterable return value. Iterating with `for item in ...`
55-
yields individual items and performs additional API calls as needed to fetch further pages.
53+
Different endpoints may return different subsets of the available pagination metadata fields, the only field that
54+
is common for all endpoints is items.
5655
"""
5756

5857
items: list[T]
5958
"""List of items on this page."""
6059

61-
count: int
60+
count: int | None
6261
"""Number of items on this page."""
6362

64-
offset: int
63+
offset: int | None
6564
"""The starting offset of this page."""
6665

67-
limit: int
66+
limit: int | None
6867
"""The maximum number of items per page requested from the API."""
6968

70-
total: int
69+
total: int | None
7170
"""Total number of items matching the query, as reported by the first page."""
7271

73-
desc: bool
72+
desc: bool | None
7473
"""Whether the items are sorted in descending order."""
7574

75+
def __init__(self, first_page: HasItems[T]) -> None:
76+
"""Initialize a page wrapper from a Pydantic paginated model."""
77+
self.items = first_page.items
78+
self.count = getattr(first_page, 'count', None)
79+
self.offset = getattr(first_page, 'offset', None)
80+
self.limit = getattr(first_page, 'limit', None)
81+
self.total = getattr(first_page, 'total', None)
82+
self.desc = getattr(first_page, 'desc', None)
83+
84+
85+
@docs_group('Other')
86+
class IterableListPage(ListPage[T], Iterable[T], Generic[T]):
87+
"""A page of results that can also be iterated to yield items across subsequent pages.
88+
89+
Accessing fields such as `items`, `count`, or `total` returns the metadata of the first page,
90+
preserving the behavior of the previous non-iterable return value. Iterating with `for item in ...`
91+
yields individual items and performs additional API calls as needed to fetch further pages.
92+
"""
93+
7694
def __init__(self, first_page: HasItems[T], iterator: Iterator[T]) -> None:
7795
"""Initialize a page wrapper from a Pydantic paginated model and an iterator over all items."""
78-
self.items = first_page.items
79-
count = getattr(first_page, 'count', None)
80-
self.count = count if count is not None else len(first_page.items)
81-
self.offset = getattr(first_page, 'offset', 0) or 0
82-
self.limit = getattr(first_page, 'limit', 0) or 0
83-
self.total = getattr(first_page, 'total', None) or len(first_page.items)
84-
self.desc = getattr(first_page, 'desc', False) or False
85-
self._first_page = first_page
96+
super().__init__(first_page)
8697
self._iterator = iterator
8798

8899
def __iter__(self) -> Iterator[T]:
@@ -95,28 +106,26 @@ class IterableListPageAsync(AsyncIterable[T], Generic[T]):
95106
"""An awaitable result that can also be asynchronously iterated to yield items across pages.
96107
97108
Awaiting the instance (`await client.list(...)`) performs a single API call and returns a
98-
populated `IterableListPage`. Iterating (`async for item in client.list(...)`) yields individual
109+
populated `ListPage`. Iterating (`async for item in client.list(...)`) yields individual
99110
items and performs additional API calls as needed to fetch further pages.
100-
101-
A single instance supports either awaiting or iterating — not both.
102111
"""
103112

104113
def __init__(
105114
self,
106-
make_awaitable: Callable[[], Awaitable[IterableListPage[T]]],
115+
awaitable_first_page: Callable[[], Awaitable[ListPage[T]]],
107116
async_iterator: AsyncIterator[T],
108117
) -> None:
109118
"""Initialize with a factory that creates the awaitable on demand and the async iterator over items."""
110-
self._make_awaitable = make_awaitable
119+
self._awaitable_first_page = awaitable_first_page
111120
self._async_iterator = async_iterator
112121

113122
def __aiter__(self) -> AsyncIterator[T]:
114123
"""Return an asynchronous iterator over all items across pages."""
115124
return self._async_iterator
116125

117-
def __await__(self) -> Generator[Any, Any, IterableListPage[T]]:
126+
def __await__(self) -> Generator[Any, Any, ListPage[T]]:
118127
"""Return an awaitable that resolves to an `IterableListPage` containing the first page."""
119-
return self._make_awaitable().__await__()
128+
return self._awaitable_first_page().__await__()
120129

121130

122131
def build_iterable_list_page(

‎tests/integration/_utils.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,14 +89,14 @@ async def maybe_await(value: Coroutine[Any, Any, T]) -> T: ...
8989
async def maybe_await(value: T) -> T: ...
9090

9191

92-
async def maybe_await(value: T | Coroutine[Any, Any, T]) -> T:
92+
async def maybe_await(value: Any) -> Any:
9393
"""Await coroutines, pass through other values.
9494
9595
Enables unified test code for both sync and async clients:
9696
result = await maybe_await(client.datasets().list())
9797
"""
9898
if hasattr(value, '__await__'):
99-
return await value # ty: ignore[invalid-await]
99+
return await value
100100
return value
101101

102102

‎tests/integration/test_webhook.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
from typing import TYPE_CHECKING
66

7+
from apify_client._iterable_list_page import ListPage
8+
79
if TYPE_CHECKING:
810
from apify_client import ApifyClient, ApifyClientAsync
911

@@ -13,11 +15,11 @@
1315
ActorJobStatus,
1416
ListOfRuns,
1517
ListOfWebhookDispatches,
16-
ListOfWebhooks,
1718
Run,
1819
Webhook,
1920
WebhookDispatch,
2021
WebhookEventType,
22+
WebhookShort,
2123
)
2224

2325
HELLO_WORLD_ACTOR = 'apify/hello-world'
@@ -49,16 +51,18 @@ async def test_list_webhooks(client: ApifyClient | ApifyClientAsync) -> None:
4951
"""Test listing webhooks."""
5052
webhooks_page = await maybe_await(client.webhooks().list(limit=10))
5153

52-
assert isinstance(webhooks_page, ListOfWebhooks)
54+
assert isinstance(webhooks_page, ListPage)
5355
assert isinstance(webhooks_page.items, list)
56+
assert isinstance(webhooks_page.items[0], WebhookShort)
5457

5558

5659
async def test_list_webhooks_pagination(client: ApifyClient | ApifyClientAsync) -> None:
5760
"""Test listing webhooks with pagination."""
5861
webhooks_page = await maybe_await(client.webhooks().list(limit=5, offset=0))
5962

60-
assert isinstance(webhooks_page, ListOfWebhooks)
63+
assert isinstance(webhooks_page, ListPage)
6164
assert isinstance(webhooks_page.items, list)
65+
assert isinstance(webhooks_page.items[0], WebhookShort)
6266

6367

6468
async def test_webhook_create_and_get(client: ApifyClient | ApifyClientAsync) -> None:

0 commit comments

Comments
 (0)