Repository navigation
Expand file tree
/
Copy path05_custom_http_client_async.py
More file actions
132 lines (101 loc) · 3.46 KB
/
Copy path05_custom_http_client_async.py
File metadata and controls
132 lines (101 loc) · 3.46 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
from __future__ import annotations
import asyncio
import json as jsonlib
from typing import TYPE_CHECKING, Any
import aiohttp
from typing_extensions import override
from apify_client import ApifyClientAsync
from apify_client.http_clients import HttpClientAsync, HttpResponse
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterator, Mapping
TOKEN = 'MY-APIFY-TOKEN'
class AiohttpResponse:
"""Adapt an aiohttp response to the Apify client's HttpResponse protocol."""
def __init__(self, response: aiohttp.ClientResponse) -> None:
self._response = response
self._body: bytes | None = None
@property
def status_code(self) -> int:
return self._response.status
@property
def headers(self) -> Mapping[str, str]:
return self._response.headers
@property
def content(self) -> bytes:
if self._body is None:
raise RuntimeError(
'The streamed response has not been read yet; call aread() first'
)
return self._body
@property
def text(self) -> str:
encoding = self._response.charset or 'utf-8'
return self.content.decode(encoding, errors='replace')
def json(self) -> Any:
return jsonlib.loads(self.text)
def read(self) -> bytes:
return self.content
async def aread(self) -> bytes:
if self._body is None:
self._body = await self._response.read()
return self._body
def close(self) -> None:
self._response.close()
async def aclose(self) -> None:
await self._response.wait_for_close()
def iter_bytes(self) -> Iterator[bytes]:
body = self.content
if body:
yield body
async def aiter_bytes(self) -> AsyncIterator[bytes]:
if self._body is not None:
if self._body:
yield self._body
return
async for chunk in self._response.content.iter_chunked(64 * 1024):
yield chunk
class AiohttpHttpClient(HttpClientAsync):
"""Minimal custom asynchronous HTTP client backed by aiohttp."""
def __init__(self) -> None:
super().__init__()
self._session = aiohttp.ClientSession()
@override
def is_retryable_transport_error(self, exc: Exception) -> bool:
return isinstance(exc, (TimeoutError, aiohttp.ClientError))
@override
async def aclose(self) -> None:
await self._session.close()
@override
async def send_request(
self,
*,
method: str,
url: str,
headers: dict[str, str],
content: bytes | AsyncIterator[bytes] | None,
timeout: float | None,
stream: bool,
follow_redirects: bool = True,
) -> HttpResponse:
response = await self._session.request(
method=method,
url=url,
headers=headers,
data=content,
timeout=aiohttp.ClientTimeout(total=timeout),
allow_redirects=follow_redirects,
)
adapted_response = AiohttpResponse(response)
if not stream:
await adapted_response.aread()
return adapted_response
async def main() -> None:
async with AiohttpHttpClient() as http_client:
client = ApifyClientAsync.with_custom_http_client(
token=TOKEN,
http_client=http_client,
)
actor = await client.actor('apify/hello-world').get()
print(actor)
if __name__ == '__main__':
asyncio.run(main())