|
| 1 | +from __future__ import annotations |
| 2 | + |
| 3 | +from typing import TYPE_CHECKING |
| 4 | + |
| 5 | +import httpx2 as httpx |
| 6 | +from typing_extensions import override |
| 7 | + |
| 8 | +from apify_client._consts import ( |
| 9 | + DEFAULT_MAX_RETRIES, |
| 10 | + DEFAULT_MIN_DELAY_BETWEEN_RETRIES, |
| 11 | + DEFAULT_TIMEOUT_LONG, |
| 12 | + DEFAULT_TIMEOUT_MAX, |
| 13 | + DEFAULT_TIMEOUT_MEDIUM, |
| 14 | + DEFAULT_TIMEOUT_SHORT, |
| 15 | +) |
| 16 | +from apify_client._docs import docs_group |
| 17 | +from apify_client.http_clients._base import HttpClient, HttpClientAsync |
| 18 | + |
| 19 | +if TYPE_CHECKING: |
| 20 | + from datetime import timedelta |
| 21 | + |
| 22 | + from apify_client._statistics import ClientStatistics |
| 23 | + from apify_client.http_compressors._base import HttpCompressor |
| 24 | + |
| 25 | + |
| 26 | +_PERMANENT_ERRORS = ( |
| 27 | + # A request HTTPX rejects before sending it, e.g. one carrying an invalid header value. |
| 28 | + httpx.LocalProtocolError, |
| 29 | + # A URL scheme HTTPX refuses to speak, which repeating the request cannot change. |
| 30 | + httpx.UnsupportedProtocol, |
| 31 | + # An over-long redirect chain is a routing loop, which repeating the request cannot break. |
| 32 | + httpx.TooManyRedirects, |
| 33 | + # Only `Response.raise_for_status()` raises this, and the client never calls it - the shared pipeline decides on |
| 34 | + # status codes from the response itself. |
| 35 | + httpx.HTTPStatusError, |
| 36 | +) |
| 37 | +"""HTTPX errors that a retry cannot fix. Everything else in the `httpx.HTTPError` tree counts as transient.""" |
| 38 | + |
| 39 | + |
| 40 | +@docs_group('HTTP clients') |
| 41 | +class HttpxHttpClient(HttpClient): |
| 42 | + """Synchronous HTTP client for the Apify API built on top of [HTTPX](https://github.com/pydantic/httpx2). |
| 43 | +
|
| 44 | + This client wraps `httpx.Client` and adds automatic retries with exponential backoff for rate-limited |
| 45 | + (HTTP 429) and server error (HTTP 5xx) responses. |
| 46 | +
|
| 47 | + HTTPX applies a request timeout to each socket operation rather than to the request as a whole, so a response |
| 48 | + whose body arrives slowly keeps resetting it and can outlast both the requested timeout and `timeout_max`. The |
| 49 | + default Impit client enforces the same value as a deadline for the whole request, body included. |
| 50 | +
|
| 51 | + Requires the `httpx` extra: `pip install "apify-client[httpx]"`. That extra installs `httpx2`, Pydantic's |
| 52 | + maintained continuation of HTTPX, which this module imports under the `httpx` name. |
| 53 | + """ |
| 54 | + |
| 55 | + def __init__( |
| 56 | + self, |
| 57 | + *, |
| 58 | + token: str | None = None, |
| 59 | + timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT, |
| 60 | + timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM, |
| 61 | + timeout_long: timedelta = DEFAULT_TIMEOUT_LONG, |
| 62 | + timeout_max: timedelta = DEFAULT_TIMEOUT_MAX, |
| 63 | + max_retries: int = DEFAULT_MAX_RETRIES, |
| 64 | + min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, |
| 65 | + statistics: ClientStatistics | None = None, |
| 66 | + headers: dict[str, str] | None = None, |
| 67 | + http_compressor: HttpCompressor | None = None, |
| 68 | + ) -> None: |
| 69 | + """Initialize the HTTPX-based synchronous HTTP client. |
| 70 | +
|
| 71 | + Args: |
| 72 | + token: Apify API token for authentication. |
| 73 | + timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...). |
| 74 | + timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). |
| 75 | + timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). |
| 76 | + timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. |
| 77 | + max_retries: Maximum number of retry attempts for failed requests. |
| 78 | + min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). |
| 79 | + statistics: Statistics tracker for API calls. Created automatically if not provided. |
| 80 | + headers: Additional HTTP headers to include in all requests. |
| 81 | + http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. |
| 82 | + """ |
| 83 | + super().__init__( |
| 84 | + token=token, |
| 85 | + timeout_short=timeout_short, |
| 86 | + timeout_medium=timeout_medium, |
| 87 | + timeout_long=timeout_long, |
| 88 | + timeout_max=timeout_max, |
| 89 | + max_retries=max_retries, |
| 90 | + min_delay_between_retries=min_delay_between_retries, |
| 91 | + statistics=statistics, |
| 92 | + headers=headers, |
| 93 | + http_compressor=http_compressor, |
| 94 | + ) |
| 95 | + |
| 96 | + self._httpx_client = httpx.Client( |
| 97 | + follow_redirects=True, |
| 98 | + event_hooks={'response': [self._clear_response_cookies]}, |
| 99 | + ) |
| 100 | + |
| 101 | + @override |
| 102 | + def is_timeout_error(self, exc: Exception) -> bool: |
| 103 | + return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) |
| 104 | + |
| 105 | + @override |
| 106 | + def is_retryable_transport_error(self, exc: Exception) -> bool: |
| 107 | + # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in |
| 108 | + # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX adds later is retried rather than |
| 109 | + # silently treated as fatal. HTTP status code errors are handled by the shared pipeline based on the |
| 110 | + # response status code, not here. |
| 111 | + return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) |
| 112 | + |
| 113 | + @override |
| 114 | + def close(self) -> None: |
| 115 | + """Close the underlying HTTPX connection pool.""" |
| 116 | + self._httpx_client.close() |
| 117 | + |
| 118 | + @override |
| 119 | + def send_request( |
| 120 | + self, |
| 121 | + *, |
| 122 | + method: str, |
| 123 | + url: str, |
| 124 | + headers: dict[str, str], |
| 125 | + content: bytes | None, |
| 126 | + timeout: float | None, |
| 127 | + stream: bool, |
| 128 | + ) -> httpx.Response: |
| 129 | + request = self._httpx_client.build_request( |
| 130 | + method=method, |
| 131 | + url=url, |
| 132 | + headers=headers, |
| 133 | + content=content, |
| 134 | + timeout=timeout, |
| 135 | + ) |
| 136 | + _restore_explicit_cookie_header(request, headers) |
| 137 | + return self._httpx_client.send(request, stream=stream) |
| 138 | + |
| 139 | + def _clear_response_cookies(self, _response: httpx.Response) -> None: |
| 140 | + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" |
| 141 | + self._httpx_client.cookies.clear() |
| 142 | + |
| 143 | + |
| 144 | +@docs_group('HTTP clients') |
| 145 | +class HttpxHttpClientAsync(HttpClientAsync): |
| 146 | + """Asynchronous HTTP client for the Apify API built on top of [HTTPX](https://github.com/pydantic/httpx2). |
| 147 | +
|
| 148 | + This client wraps `httpx.AsyncClient` and adds automatic retries with exponential backoff for rate-limited |
| 149 | + (HTTP 429) and server error (HTTP 5xx) responses. |
| 150 | +
|
| 151 | + HTTPX applies a request timeout to each socket operation rather than to the request as a whole, so a response |
| 152 | + whose body arrives slowly keeps resetting it and can outlast both the requested timeout and `timeout_max`. The |
| 153 | + default Impit client enforces the same value as a deadline for the whole request, body included. |
| 154 | +
|
| 155 | + Requires the `httpx` extra: `pip install "apify-client[httpx]"`. That extra installs `httpx2`, Pydantic's |
| 156 | + maintained continuation of HTTPX, which this module imports under the `httpx` name. |
| 157 | + """ |
| 158 | + |
| 159 | + def __init__( |
| 160 | + self, |
| 161 | + *, |
| 162 | + token: str | None = None, |
| 163 | + timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT, |
| 164 | + timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM, |
| 165 | + timeout_long: timedelta = DEFAULT_TIMEOUT_LONG, |
| 166 | + timeout_max: timedelta = DEFAULT_TIMEOUT_MAX, |
| 167 | + max_retries: int = DEFAULT_MAX_RETRIES, |
| 168 | + min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, |
| 169 | + statistics: ClientStatistics | None = None, |
| 170 | + headers: dict[str, str] | None = None, |
| 171 | + http_compressor: HttpCompressor | None = None, |
| 172 | + ) -> None: |
| 173 | + """Initialize the HTTPX-based asynchronous HTTP client. |
| 174 | +
|
| 175 | + Args: |
| 176 | + token: Apify API token for authentication. |
| 177 | + timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...). |
| 178 | + timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). |
| 179 | + timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). |
| 180 | + timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. |
| 181 | + max_retries: Maximum number of retry attempts for failed requests. |
| 182 | + min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). |
| 183 | + statistics: Statistics tracker for API calls. Created automatically if not provided. |
| 184 | + headers: Additional HTTP headers to include in all requests. |
| 185 | + http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. |
| 186 | + """ |
| 187 | + super().__init__( |
| 188 | + token=token, |
| 189 | + timeout_short=timeout_short, |
| 190 | + timeout_medium=timeout_medium, |
| 191 | + timeout_long=timeout_long, |
| 192 | + timeout_max=timeout_max, |
| 193 | + max_retries=max_retries, |
| 194 | + min_delay_between_retries=min_delay_between_retries, |
| 195 | + statistics=statistics, |
| 196 | + headers=headers, |
| 197 | + http_compressor=http_compressor, |
| 198 | + ) |
| 199 | + |
| 200 | + self._httpx_async_client = httpx.AsyncClient( |
| 201 | + follow_redirects=True, |
| 202 | + event_hooks={'response': [self._clear_response_cookies]}, |
| 203 | + ) |
| 204 | + |
| 205 | + @override |
| 206 | + def is_timeout_error(self, exc: Exception) -> bool: |
| 207 | + return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) |
| 208 | + |
| 209 | + @override |
| 210 | + def is_retryable_transport_error(self, exc: Exception) -> bool: |
| 211 | + # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in |
| 212 | + # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX adds later is retried rather than |
| 213 | + # silently treated as fatal. HTTP status code errors are handled by the shared pipeline based on the |
| 214 | + # response status code, not here. |
| 215 | + return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) |
| 216 | + |
| 217 | + @override |
| 218 | + async def aclose(self) -> None: |
| 219 | + """Close the underlying asynchronous HTTPX connection pool.""" |
| 220 | + await self._httpx_async_client.aclose() |
| 221 | + |
| 222 | + @override |
| 223 | + async def send_request( |
| 224 | + self, |
| 225 | + *, |
| 226 | + method: str, |
| 227 | + url: str, |
| 228 | + headers: dict[str, str], |
| 229 | + content: bytes | None, |
| 230 | + timeout: float | None, |
| 231 | + stream: bool, |
| 232 | + ) -> httpx.Response: |
| 233 | + request = self._httpx_async_client.build_request( |
| 234 | + method=method, |
| 235 | + url=url, |
| 236 | + headers=headers, |
| 237 | + content=content, |
| 238 | + timeout=timeout, |
| 239 | + ) |
| 240 | + _restore_explicit_cookie_header(request, headers) |
| 241 | + return await self._httpx_async_client.send(request, stream=stream) |
| 242 | + |
| 243 | + async def _clear_response_cookies(self, _response: httpx.Response) -> None: |
| 244 | + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" |
| 245 | + self._httpx_async_client.cookies.clear() |
| 246 | + |
| 247 | + |
| 248 | +def _restore_explicit_cookie_header(request: httpx.Request, headers: dict[str, str]) -> None: |
| 249 | + """Keep only cookies explicitly supplied for this request, never cookies from HTTPX's shared jar. |
| 250 | +
|
| 251 | + HTTPX drops the `Cookie` header when it builds a redirect request and rebuilds it from the jar, so an explicit |
| 252 | + cookie only reaches the first hop of a redirected request. |
| 253 | + """ |
| 254 | + explicit_cookie = next((value for key, value in headers.items() if key.lower() == 'cookie'), None) |
| 255 | + if explicit_cookie is None: |
| 256 | + request.headers.pop('cookie', None) |
| 257 | + else: |
| 258 | + request.headers['cookie'] = explicit_cookie |
0 commit comments