Skip to content

Commit 9c4ee64

Browse files
committed
Type unsafe
1 parent 307df23 commit 9c4ee64

3 files changed

Lines changed: 52 additions & 16 deletions

File tree

‎src/apify_client/_iterable_list_page.py‎

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from __future__ import annotations
22

3-
from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Generator, Iterable, Iterator
3+
import asyncio
4+
from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Generator, Iterable, Iterator, Coroutine
45
from typing import Any, Generic, TypeVar
56

67
from apify_client._docs import docs_group
@@ -25,6 +26,25 @@ def _min_for_limit_param(a: int | None, b: int | None) -> int | None:
2526
return min(a, b)
2627

2728

29+
T = TypeVar('T')
30+
31+
32+
class _LazyTask(Generic[T]):
33+
"""Task that is created lazily upon awaiting.
34+
35+
This allows to reuse the same Task multiple times without the need to schedule the task when it is created.
36+
"""
37+
def __init__(self, awaitable: Awaitable[T]) -> None:
38+
self._awaitable = awaitable
39+
self._task: asyncio.Task[T] | None = None
40+
41+
def __await__(self) -> Generator[Any, None, T]:
42+
if self._task is None:
43+
self._task = asyncio.create_task(self._awaitable)
44+
return (yield from self._task.__await__())
45+
46+
47+
2848
@docs_group('Other')
2949
class IterableListPage(Iterable[T], Generic[T]):
3050
"""A page of results that can also be iterated to yield items across subsequent pages.
@@ -109,7 +129,7 @@ def build_iterable_list_page(
109129
stopping iteration because it may change between calls; iteration stops when a page has
110130
no items or when the user-requested `limit` has been reached.
111131
112-
Recognized kwargs:
132+
Iteration relevant kwargs:
113133
chunk_size: Maximum number of items requested per API call during iteration. Pass `0`
114134
or `None` to let the API decide (effectively infinity).
115135
limit: User-requested total item limit. Stops iteration once this many items are yielded.
@@ -126,7 +146,7 @@ def iterator() -> Iterator[Any]:
126146
current_page = first_page
127147
yield from current_page.items
128148

129-
fetched_items = len(current_page.items)
149+
fetched_items = current_page.count
130150
while current_page.items and (not limit or (limit > fetched_items)):
131151
new_kwargs = {
132152
**kwargs,
@@ -135,7 +155,7 @@ def iterator() -> Iterator[Any]:
135155
}
136156
current_page = callback(**new_kwargs)
137157
yield from current_page.items
138-
fetched_items += len(current_page.items)
158+
fetched_items += current_page.count
139159

140160
return IterableListPage(first_page, iterator())
141161

@@ -154,15 +174,15 @@ def build_iterable_list_page_async(
154174
offset = kwargs.get('offset') or 0
155175
limit = kwargs.get('limit') or 0
156176

157-
async def fetch_first_page() -> Any:
158-
return await callback(**{**kwargs, 'limit': _min_for_limit_param(kwargs.get('limit'), chunk_size)})
177+
# Can be awaited multiple times with same result, but not scheduled at this time yet, as it might be pre-emptive.
178+
fetch_first_page = _LazyTask(callback(**{**kwargs, 'limit': _min_for_limit_param(kwargs.get('limit'), chunk_size)}))
159179

160180
async def async_iterator() -> AsyncIterator[Any]:
161-
current_page = await fetch_first_page()
181+
current_page = await fetch_first_page
162182
for item in current_page.items:
163183
yield item
164184

165-
fetched_items = len(current_page.items)
185+
fetched_items = current_page.count
166186
while current_page.items and (not limit or (limit > fetched_items)):
167187
new_kwargs = {
168188
**kwargs,
@@ -172,10 +192,10 @@ async def async_iterator() -> AsyncIterator[Any]:
172192
current_page = await callback(**new_kwargs)
173193
for item in current_page.items:
174194
yield item
175-
fetched_items += len(current_page.items)
195+
fetched_items += current_page.count
176196

177197
async def wrap_first_page() -> IterableListPage[Any]:
178-
first_page = await fetch_first_page()
198+
first_page = await fetch_first_page
179199
return IterableListPage(first_page, iter(first_page.items))
180200

181201
return IterableListPageAsync(wrap_first_page, async_iterator())

‎src/apify_client/_resource_clients/dataset.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -228,8 +228,9 @@ def _fetch_page(
228228
items=items,
229229
total=int(response.headers['x-apify-pagination-total']),
230230
offset=int(response.headers['x-apify-pagination-offset']),
231-
# x-apify-pagination-count returns invalid values when hidden/empty items are skipped
232-
count=len(items),
231+
# x-apify-pagination-count returns count of processed items, not count of returned items
232+
# This makes difference when items were filtered using hidden/empty
233+
count=int(response.headers['x-apify-pagination-count']),
233234
# API returns 999999999999 when no limit is used
234235
limit=int(response.headers['x-apify-pagination-limit']),
235236
desc=response.headers['x-apify-pagination-desc'].lower() == 'true',
@@ -905,8 +906,9 @@ async def _fetch_page(
905906
items=items,
906907
total=int(response.headers['x-apify-pagination-total']),
907908
offset=int(response.headers['x-apify-pagination-offset']),
908-
# x-apify-pagination-count returns invalid values when hidden/empty items are skipped
909-
count=len(items),
909+
# x-apify-pagination-count returns count of processed items, not count of returned items
910+
# This makes difference when items were filtered using hidden/empty
911+
count=int(response.headers['x-apify-pagination-count']),
910912
# API returns 999999999999 when no limit is used
911913
limit=int(response.headers['x-apify-pagination-limit']),
912914
desc=response.headers['x-apify-pagination-desc'].lower() == 'true',

‎tests/unit/test_client_pagination.py‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,9 +172,10 @@ def _build(_cls: type, obj: dict) -> _FakeResponseWrapper:
172172
p.stop()
173173

174174

175-
def create_items(start: int, end: int) -> list[dict[str, int]]:
175+
def create_items(start: int, end: int, step: int | None = None) -> list[dict[str, int]]:
176176
"""Create a list of test items for the given index range."""
177-
step = -1 if end < start else 1
177+
if not step:
178+
step = -1 if end < start else 1
178179
return [{'id': i, 'key': i} for i in range(start, end, step)]
179180

180181

@@ -211,6 +212,10 @@ def _mocked_api_pagination_logic(*, url: str, params: dict[str, Any] | None = No
211212

212213
selected_items = items[lower_index : min(upper_index, lower_index + max_items_per_page)]
213214

215+
# Every second item would be filtered out when using `skip_empty=True`, `skip_hidden=True`, or `clean=True`
216+
if params.get('skip_empty') or params.get('skip_hidden') or params.get('clean'):
217+
selected_items = selected_items[::2]
218+
214219
response = Mock()
215220

216221
# The dataset items endpoint returns items as a raw list
@@ -231,6 +236,7 @@ def _mocked_api_pagination_logic(*, url: str, params: dict[str, Any] | None = No
231236
}
232237

233238
response.headers = {
239+
'x-apify-pagination-count': count,
234240
'x-apify-pagination-total': str(total_items),
235241
'x-apify-pagination-offset': str(offset),
236242
'x-apify-pagination-limit': str(limit or count or 1),
@@ -324,6 +330,14 @@ def supports(self, client: ResourceClient | ResourceClientAsync) -> bool:
324330
create_items(2450, 1350),
325331
DATASET_CLIENTS,
326332
),
333+
_PaginationCase(
334+
'Offset, limit, descending, chunk_size, clean',
335+
{'limit': 1500, 'chunk_size': 100, 'clean': True},
336+
# API behavior with `clean=True` is to apply the cleaning after pagination, so we end up with missing items
337+
# being counted towards the limit and thus fewer total items returned.
338+
create_items(0, 1500, 2),
339+
DATASET_CLIENTS,
340+
),
327341
_PaginationCase(
328342
'Exclusive start key',
329343
{'exclusive_start_key': '1000'},

0 commit comments

Comments
 (0)