Skip to content

Commit eeb6be1

Browse files
committed
Use HasItems and fix type errors
1 parent 9c4ee64 commit eeb6be1

4 files changed

Lines changed: 42 additions & 50 deletions

File tree

‎src/apify_client/_iterable_list_page.py‎

Lines changed: 38 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,18 @@
11
from __future__ import annotations
22

33
import asyncio
4-
from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Generator, Iterable, Iterator, Coroutine
5-
from typing import Any, Generic, TypeVar
4+
from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Coroutine, Generator, Iterable, Iterator
5+
from typing import Any, Generic, Protocol, TypeVar
66

77
from apify_client._docs import docs_group
88

99
T = TypeVar('T')
1010

1111

12+
class HasItems(Protocol[T]):
13+
items: list[T]
14+
15+
1216
def _min_for_limit_param(a: int | None, b: int | None) -> int | None:
1317
"""Return minimum of two limit parameters, treating `None` or `0` as infinity.
1418
@@ -26,15 +30,13 @@ def _min_for_limit_param(a: int | None, b: int | None) -> int | None:
2630
return min(a, b)
2731

2832

29-
T = TypeVar('T')
30-
31-
3233
class _LazyTask(Generic[T]):
3334
"""Task that is created lazily upon awaiting.
3435
3536
This allows to reuse the same Task multiple times without the need to schedule the task when it is created.
3637
"""
37-
def __init__(self, awaitable: Awaitable[T]) -> None:
38+
39+
def __init__(self, awaitable: Coroutine[Any, Any, T]) -> None:
3840
self._awaitable = awaitable
3941
self._task: asyncio.Task[T] | None = None
4042

@@ -44,7 +46,6 @@ def __await__(self) -> Generator[Any, None, T]:
4446
return (yield from self._task.__await__())
4547

4648

47-
4849
@docs_group('Other')
4950
class IterableListPage(Iterable[T], Generic[T]):
5051
"""A page of results that can also be iterated to yield items across subsequent pages.
@@ -72,7 +73,7 @@ class IterableListPage(Iterable[T], Generic[T]):
7273
desc: bool
7374
"""Whether the items are sorted in descending order."""
7475

75-
def __init__(self, first_page: Any, iterator: Iterator[T]) -> None:
76+
def __init__(self, first_page: HasItems[T], iterator: Iterator[T]) -> None:
7677
"""Initialize a page wrapper from a Pydantic paginated model and an iterator over all items."""
7778
self.items = first_page.items
7879
count = getattr(first_page, 'count', None)
@@ -119,15 +120,24 @@ def __await__(self) -> Generator[Any, Any, IterableListPage[T]]:
119120

120121

121122
def build_iterable_list_page(
122-
callback: Callable[..., Any],
123+
callback: Callable[..., HasItems[T]],
123124
**kwargs: Any,
124-
) -> IterableListPage[Any]:
125+
) -> IterableListPage[T]:
125126
"""Build an `IterableListPage` from a paginated sync callback.
126127
127128
The callback is invoked once immediately to fetch the first page, and again lazily during
128-
iteration to fetch further pages. The `total` field from the first page is not trusted for
129-
stopping iteration because it may change between calls; iteration stops when a page has
130-
no items or when the user-requested `limit` has been reached.
129+
iteration to fetch further pages.
130+
131+
There are several optional kwargs that control the pagination, but not all are accepted on each paginated endpoint.
132+
Some endpoints do not return all paginated metadata, so the implementation should be resilient to missing fields,
133+
but it can use them if available.
134+
135+
The `total` field from the first page is not trusted for stopping iteration because it may change between calls;
136+
iteration stops when a page has no items or when the user-requested `limit` has been reached.
137+
138+
The `count` field does not count objects returned, but object scanned by the API. For example when using filters,
139+
returned items can be smaller than `count`. Therefore, `count` should be used for correct offset calculation if
140+
available.
131141
132142
Iteration relevant kwargs:
133143
chunk_size: Maximum number of items requested per API call during iteration. Pass `0`
@@ -142,11 +152,11 @@ def build_iterable_list_page(
142152

143153
first_page = callback(**{**kwargs, 'limit': _min_for_limit_param(kwargs.get('limit'), chunk_size)})
144154

145-
def iterator() -> Iterator[Any]:
155+
def iterator() -> Iterator[T]:
146156
current_page = first_page
147157
yield from current_page.items
148158

149-
fetched_items = current_page.count
159+
fetched_items = getattr(current_page, 'count', len(current_page.items))
150160
while current_page.items and (not limit or (limit > fetched_items)):
151161
new_kwargs = {
152162
**kwargs,
@@ -155,15 +165,15 @@ def iterator() -> Iterator[Any]:
155165
}
156166
current_page = callback(**new_kwargs)
157167
yield from current_page.items
158-
fetched_items += current_page.count
168+
fetched_items += getattr(current_page, 'count', len(current_page.items))
159169

160170
return IterableListPage(first_page, iterator())
161171

162172

163173
def build_iterable_list_page_async(
164-
callback: Callable[..., Awaitable[Any]],
174+
callback: Callable[..., Coroutine[Any, Any, HasItems[T]]],
165175
**kwargs: Any,
166-
) -> IterableListPageAsync[Any]:
176+
) -> IterableListPageAsync[T]:
167177
"""Build an `IterableListPageAsync` from a paginated async callback.
168178
169179
Mirrors `build_iterable_list_page` but for async callbacks. The returned object is both
@@ -182,7 +192,7 @@ async def async_iterator() -> AsyncIterator[Any]:
182192
for item in current_page.items:
183193
yield item
184194

185-
fetched_items = current_page.count
195+
fetched_items = getattr(current_page, 'count', len(current_page.items))
186196
while current_page.items and (not limit or (limit > fetched_items)):
187197
new_kwargs = {
188198
**kwargs,
@@ -192,7 +202,7 @@ async def async_iterator() -> AsyncIterator[Any]:
192202
current_page = await callback(**new_kwargs)
193203
for item in current_page.items:
194204
yield item
195-
fetched_items += current_page.count
205+
fetched_items += getattr(current_page, 'count', len(current_page.items))
196206

197207
async def wrap_first_page() -> IterableListPage[Any]:
198208
first_page = await fetch_first_page
@@ -202,15 +212,14 @@ async def wrap_first_page() -> IterableListPage[Any]:
202212

203213

204214
def build_cursor_iterable_list_page(
205-
callback: Callable[..., Any],
215+
callback: Callable[..., HasItems[T]],
206216
*,
207217
cursor_param: str,
208-
next_cursor_fn: Callable[[Any], Any],
209218
initial_cursor: Any = None,
210219
limit: int | None = None,
211220
chunk_size: int | None = None,
212221
**kwargs: Any,
213-
) -> IterableListPage[Any]:
222+
) -> IterableListPage[T]:
214223
"""Build an `IterableListPage` for endpoints that paginate with a cursor instead of an offset.
215224
216225
The callback is invoked with `{cursor_param: cursor, 'limit': effective_limit, **kwargs}` for each
@@ -229,29 +238,28 @@ def iterator() -> Iterator[Any]:
229238
yield from current_page.items
230239

231240
fetched = len(current_page.items)
232-
next_cursor = next_cursor_fn(current_page)
241+
next_cursor = getattr(current_page, f'next_{cursor_param}')
233242

234243
while current_page.items and next_cursor is not None and (not user_limit or user_limit > fetched):
235244
remaining = (user_limit - fetched) if user_limit else 0
236245
next_limit = effective_chunk if not user_limit else _min_for_limit_param(remaining, effective_chunk)
237246
current_page = callback(**{**kwargs, cursor_param: next_cursor, 'limit': next_limit})
238247
yield from current_page.items
239248
fetched += len(current_page.items)
240-
next_cursor = next_cursor_fn(current_page)
249+
next_cursor = getattr(current_page, f'next_{cursor_param}')
241250

242251
return IterableListPage(first_page, iterator())
243252

244253

245254
def build_cursor_iterable_list_page_async(
246-
callback: Callable[..., Awaitable[Any]],
255+
callback: Callable[..., Awaitable[HasItems[T]]],
247256
*,
248257
cursor_param: str,
249-
next_cursor_fn: Callable[[Any], Any],
250258
initial_cursor: Any = None,
251259
limit: int | None = None,
252260
chunk_size: int | None = None,
253261
**kwargs: Any,
254-
) -> IterableListPageAsync[Any]:
262+
) -> IterableListPageAsync[T]:
255263
"""Build an `IterableListPageAsync` for endpoints that paginate with a cursor instead of an offset.
256264
257265
Mirrors `build_cursor_iterable_list_page` but for async callbacks. The returned object is both
@@ -271,7 +279,7 @@ async def async_iterator() -> AsyncIterator[Any]:
271279
yield item
272280

273281
fetched = len(current_page.items)
274-
next_cursor = next_cursor_fn(current_page)
282+
next_cursor = getattr(current_page, f'next_{cursor_param}')
275283

276284
while current_page.items and next_cursor is not None and (not user_limit or user_limit > fetched):
277285
remaining = (user_limit - fetched) if user_limit else 0
@@ -280,7 +288,7 @@ async def async_iterator() -> AsyncIterator[Any]:
280288
for item in current_page.items:
281289
yield item
282290
fetched += len(current_page.items)
283-
next_cursor = next_cursor_fn(current_page)
291+
next_cursor = getattr(current_page, f'next_{cursor_param}')
284292

285293
async def wrap_first_page() -> IterableListPage[Any]:
286294
first_page = await fetch_first_page()

‎src/apify_client/_resource_clients/dataset.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ def list_items(
151151
signature: str | None = None,
152152
chunk_size: int | None = None,
153153
timeout: Timeout = 'long',
154-
) -> IterableListPage[DatasetItemsPage]:
154+
) -> IterableListPage[dict[str, Any]]:
155155
"""List the items of the dataset.
156156
157157
The returned page also supports iteration: `for item in client.list_items(...)` yields individual
@@ -252,7 +252,7 @@ def iterate_items(
252252
skip_hidden: bool | None = None,
253253
signature: str | None = None,
254254
timeout: Timeout = 'long',
255-
) -> Iterator[DatasetItemsPage]:
255+
) -> Iterator[dict[str, Any]]:
256256
"""Iterate over the items in the dataset.
257257
258258
Deprecated: iterate the return value of `DatasetClient.list_items()` instead.
@@ -829,7 +829,7 @@ def list_items(
829829
signature: str | None = None,
830830
chunk_size: int | None = None,
831831
timeout: Timeout = 'long',
832-
) -> IterableListPageAsync[DatasetItemsPage]:
832+
) -> IterableListPageAsync[dict[str, Any]]:
833833
"""List the items of the dataset.
834834
835835
The returned page also supports iteration: `for item in client.list_items(...)` yields individual
@@ -930,7 +930,7 @@ async def iterate_items(
930930
skip_hidden: bool | None = None,
931931
signature: str | None = None,
932932
timeout: Timeout = 'long',
933-
) -> AsyncIterator[DatasetItemsPage]:
933+
) -> AsyncIterator[dict[str, Any]]:
934934
"""Iterate over the items in the dataset.
935935
936936
Deprecated: iterate the return value of `DatasetClientAsync.list_items()` instead.

‎src/apify_client/_resource_clients/key_value_store.py‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,6 @@
4040
from apify_client._types import Timeout
4141

4242

43-
def _kvs_next_cursor(page: ListOfKeys) -> str | None:
44-
"""Return the next cursor for KVS key pagination, or `None` when there are no more pages."""
45-
if not page.is_truncated:
46-
return None
47-
return page.next_exclusive_start_key
48-
49-
5043
def _parse_get_record_response(response: HttpResponse) -> Any:
5144
"""Parse an HTTP response based on its content type.
5245
@@ -202,7 +195,6 @@ def _callback(*, limit: int | None = None, exclusive_start_key: str | None = Non
202195
return build_cursor_iterable_list_page(
203196
_callback,
204197
cursor_param='exclusive_start_key',
205-
next_cursor_fn=_kvs_next_cursor,
206198
initial_cursor=exclusive_start_key,
207199
limit=limit,
208200
chunk_size=chunk_size,
@@ -641,7 +633,6 @@ async def _callback(*, limit: int | None = None, exclusive_start_key: str | None
641633
return build_cursor_iterable_list_page_async(
642634
_callback,
643635
cursor_param='exclusive_start_key',
644-
next_cursor_fn=_kvs_next_cursor,
645636
initial_cursor=exclusive_start_key,
646637
limit=limit,
647638
chunk_size=chunk_size,

‎src/apify_client/_resource_clients/request_queue.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,6 @@
5656
_SAFETY_BUFFER_PERCENT = 0.01 / 100 # 0.01%
5757

5858

59-
def _rq_next_cursor(page: ListOfRequests) -> str | None:
60-
"""Return the opaque `next_cursor` from the page, or `None` when there are no more pages."""
61-
return page.next_cursor
62-
63-
6459
@docs_group('Resource clients')
6560
class RequestQueueClient(ResourceClient):
6661
"""Sub-client for managing a specific request queue.
@@ -564,7 +559,6 @@ def _callback(*, limit: int | None = None, cursor: str | None = None) -> ListOfR
564559
return build_cursor_iterable_list_page(
565560
_callback,
566561
cursor_param='cursor',
567-
next_cursor_fn=_rq_next_cursor,
568562
initial_cursor=cursor,
569563
limit=limit,
570564
chunk_size=chunk_size,
@@ -1142,7 +1136,6 @@ async def _callback(*, limit: int | None = None, cursor: str | None = None) -> L
11421136
return build_cursor_iterable_list_page_async(
11431137
_callback,
11441138
cursor_param='cursor',
1145-
next_cursor_fn=_rq_next_cursor,
11461139
initial_cursor=cursor,
11471140
limit=limit,
11481141
chunk_size=chunk_size,

0 commit comments

Comments
 (0)