Skip to content

Commit 73e1531

Browse files
committed
Add for async as well
1 parent c5122e9 commit 73e1531

3 files changed

Lines changed: 69 additions & 44 deletions

File tree

‎src/apify_client/_iterable_list_page.py‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -179,8 +179,9 @@ def get_iterator() -> Iterator[T]:
179179

180180
def build_iterable_list_page_async(
181181
callback: Callable[..., Coroutine[Any, Any, HasItems[T]]],
182+
fetch_first_page,
182183
**kwargs: Any,
183-
) -> IterableListPageAsync[T]:
184+
) -> Callable[[], AsyncIterator[T]]:
184185
"""Build an `IterableListPageAsync` from a paginated async callback.
185186
186187
Mirrors `build_iterable_list_page` but for async callbacks. The returned object is both
@@ -191,10 +192,7 @@ def build_iterable_list_page_async(
191192
offset = kwargs.get('offset') or 0
192193
limit = kwargs.get('limit') or 0
193194

194-
# Can be awaited multiple times with same result, but not scheduled at this time yet, as it might be pre-emptive.
195-
fetch_first_page = _LazyTask(callback(**{**kwargs, 'limit': _min_for_limit_param(kwargs.get('limit'), chunk_size)}))
196-
197-
async def get_async_iterator() -> AsyncIterator[Any]:
195+
async def get_async_iterator() -> AsyncIterator[T]:
198196
current_page = await fetch_first_page
199197
for item in current_page.items:
200198
yield item
@@ -211,11 +209,7 @@ async def get_async_iterator() -> AsyncIterator[Any]:
211209
yield item
212210
fetched_items += getattr(current_page, 'count', len(current_page.items))
213211

214-
async def wrap_first_page() -> ListPage[Any]:
215-
first_page = await fetch_first_page
216-
return ListPage(first_page)
217-
218-
return IterableListPageAsync(wrap_first_page, get_async_iterator)
212+
return get_async_iterator
219213

220214

221215
def build_cursor_iterable_list_page(

‎src/apify_client/_pagination_classes.py‎

Lines changed: 52 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
from __future__ import annotations
22

3-
from collections.abc import Callable, Iterator
3+
from collections.abc import Callable, Iterator, AsyncIterator, Awaitable, Generator
44
from dataclasses import dataclass, InitVar, field
5-
from typing import TYPE_CHECKING, Generic, TypeVar
6-
5+
from typing import TYPE_CHECKING, Generic, TypeVar, Self, Any
76

7+
from apify_client._iterable_list_page import HasItems
88
from apify_client._models_generated import (
99
ActorShort,
1010
BuildShort,
@@ -28,66 +28,91 @@
2828

2929
@dataclass
3030
class IterablePage(Generic[T]):
31-
items: list[T]
32-
get_iterator: InitVar[Callable[[], Iterator[T]]]
33-
_get_iterator: Callable[[], Iterator[T]] = field(init=False, repr=False, compare=False)
34-
35-
def __post_init__(self, get_iterator: Callable[[], Iterator[T]]) -> None:
36-
self._get_iterator = get_iterator
31+
_get_iterator: Callable[[], Iterator[T]] = field(repr=False, compare=False)
3732

3833
def __iter__(self) -> Iterator[T]:
3934
return self._get_iterator()
4035

36+
@dataclass
37+
class IterablePageAsync(Generic[T]):
38+
_get_async_iterator: Callable[[], AsyncIterator[T]] = field(repr=False, compare=False)
39+
40+
def __aiter__(self) -> AsyncIterator[T]:
41+
return self._get_async_iterator()
42+
43+
@dataclass
44+
class AwaitablePage(Generic[T]):
45+
_awaitable_first_page: Awaitable[PaginatedPage[T]] = field(repr=False, compare=False)
46+
47+
def __await__(self) -> Generator[Any, Any, PaginatedPage[T]]:
48+
return self._awaitable_first_page.__await__()
49+
4150

4251
@dataclass
43-
class PaginatedPage:
52+
class PageWithItems(Generic[T]):
53+
items: list[T]
54+
55+
@dataclass
56+
class PageWithTotal(Generic[T]):
57+
total: int
58+
59+
@dataclass
60+
class PageWithLimit(Generic[T]):
61+
limit: int
62+
63+
@dataclass
64+
class PageWithCount(Generic[T]):
4465
count: int
66+
67+
@dataclass
68+
class PaginatedPage(PageWithItems[T], PageWithTotal, PageWithLimit, PageWithCount):
4569
offset: int
46-
limit: int
47-
total: int
4870
desc: bool
4971

5072
@dataclass
51-
class ListPageOfActors(PaginatedPage, IterablePage[ActorShort]): ...
73+
class PaginatedPageOnlyTotal(PageWithItems[T], PageWithTotal): ...
5274

5375
@dataclass
54-
class ListPageOfBuilds(PaginatedPage, IterablePage[BuildShort]): ...
76+
class ListPageOfActors(PaginatedPage[ActorShort], IterablePage[ActorShort]): ...
5577

5678
@dataclass
57-
class ListPageOfDatasets(PaginatedPage, IterablePage[DatasetListItem]): ...
79+
class ListPageOfBuilds(PaginatedPage[BuildShort], IterablePage[BuildShort]): ...
5880

5981
@dataclass
60-
class ListPageOfKeyValueStores(PaginatedPage, IterablePage[KeyValueStore]): ...
82+
class ListPageOfDatasets(PaginatedPage[DatasetListItem], IterablePage[DatasetListItem]): ...
6183

6284
@dataclass
63-
class ListPageOfRequestQueues(PaginatedPage, IterablePage[RequestQueueShort]): ...
85+
class ListPageOfKeyValueStores(PaginatedPage[KeyValueStore], IterablePage[KeyValueStore]): ...
6486

6587
@dataclass
66-
class ListPageOfRuns(PaginatedPage, IterablePage[RunShort]): ...
88+
class ListPageOfRequestQueues(PaginatedPage[RequestQueueShort], IterablePage[RequestQueueShort]): ...
6789

6890
@dataclass
69-
class ListPageOfSchedules(PaginatedPage, IterablePage[ScheduleShort]): ...
91+
class ListPageOfRuns(PaginatedPage[RunShort], IterablePage[RunShort]): ...
7092

7193
@dataclass
72-
class ListPageOfStoreActors(PaginatedPage, IterablePage[StoreListActor]): ...
94+
class ListPageOfSchedules(PaginatedPage[ScheduleShort], IterablePage[ScheduleShort]): ...
7395

7496
@dataclass
75-
class ListPageOfTasks(PaginatedPage, IterablePage[TaskShort]): ...
97+
class ListPageOfStoreActors(PaginatedPage[StoreListActor], IterablePage[StoreListActor]): ...
7698

7799
@dataclass
78-
class ListPageOfWebhookDispatches(PaginatedPage, IterablePage[WebhookDispatch]): ...
100+
class ListPageOfTasks(PaginatedPage[TaskShort], IterablePage[TaskShort]): ...
79101

80102
@dataclass
81-
class ListPageOfWebhooks(PaginatedPage, IterablePage[WebhookShort]): ...
103+
class ListPageOfWebhookDispatches(PaginatedPage[WebhookDispatch], IterablePage[WebhookDispatch]): ...
82104

83105
@dataclass
84-
class ListPageOfEnvVars(IterablePage[EnvVar]):
85-
total: int
106+
class ListPageOfWebhookDispatchesAsync(IterablePageAsync[WebhookDispatch], AwaitablePage[WebhookDispatch]): ...
107+
108+
@dataclass
109+
class ListPageOfWebhooks(PaginatedPage[WebhookShort], IterablePage[WebhookShort]): ...
86110

111+
@dataclass
112+
class ListPageOfEnvVars(IterablePage[EnvVar], PaginatedPageOnlyTotal): ...
87113

88114
@dataclass
89-
class ListPageOfVersions(IterablePage[Version]):
90-
total: int
115+
class ListPageOfVersions(IterablePage[Version], PaginatedPageOnlyTotal): ...
91116

92117
@dataclass
93118
class ListPageOfRequests(IterablePage[Request]):

‎src/apify_client/_resource_clients/webhook_dispatch_collection.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,11 @@
77
IterableListPage,
88
IterableListPageAsync,
99
build_iterable_list_page,
10-
build_iterable_list_page_async,
10+
build_iterable_list_page_async, _LazyTask
1111
)
1212
from apify_client._models_generated import ListOfWebhookDispatches, WebhookDispatchList
13-
from apify_client._pagination_classes import ListPageOfWebhookDispatches
13+
from apify_client._pagination_classes import ListPageOfWebhookDispatches, PaginatedPage, \
14+
ListPageOfWebhookDispatchesAsync
1415
from apify_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
1516

1617
if TYPE_CHECKING:
@@ -69,7 +70,7 @@ def _callback(**kwargs: Any) -> ListOfWebhookDispatches:
6970
first_page = _callback(limit=limit, offset=offset, desc=desc)
7071
get_iterator = build_iterable_list_page(_callback,first_page, limit=limit, offset=offset, desc=desc)
7172

72-
return ListPageOfWebhookDispatches(get_iterator=get_iterator,
73+
return ListPageOfWebhookDispatches(_get_iterator=get_iterator,
7374
items=first_page.items,
7475
count=first_page.count,
7576
offset=first_page.offset,
@@ -104,7 +105,7 @@ def list(
104105
offset: int | None = None,
105106
desc: bool | None = None,
106107
timeout: Timeout = 'medium',
107-
) -> IterableListPageAsync[WebhookDispatch]:
108+
) -> ListPageOfWebhookDispatchesAsync:
108109
"""List all webhook dispatches of a user.
109110
110111
The returned page also supports iteration: `async for item in client.list(...)` yields individual
@@ -122,8 +123,13 @@ def list(
122123
The retrieved webhook dispatches of a user.
123124
"""
124125

125-
async def _callback(**kwargs: Any) -> ListOfWebhookDispatches:
126+
async def _callback(**kwargs: Any) -> PaginatedPage[WebhookDispatch]:
126127
result = await self._list(timeout=timeout, **kwargs)
127-
return WebhookDispatchList.model_validate(result).data
128+
data = WebhookDispatchList.model_validate(result).data
129+
return PaginatedPage(items=data.items, count=data.count, offset=data.offset, desc=data.desc, total=data.total, limit=data.limit)
130+
131+
fetch_first_page = _LazyTask(_callback(limit=limit, offset=offset, desc=desc))
132+
get_async_iterator = build_iterable_list_page_async(_callback, fetch_first_page,
133+
limit=limit, offset=offset, desc=desc)
128134

129-
return build_iterable_list_page_async(_callback, limit=limit, offset=offset, desc=desc)
135+
return ListPageOfWebhookDispatchesAsync(_awaitable_first_page=fetch_first_page, _get_async_iterator=get_async_iterator)

0 commit comments

Comments
 (0)