Skip to content

Commit 175382d

Browse files
committed
fix: Keep proxy tunnel failures retryable
`ProxyError` was classified as permanently non-retryable, so a proxy answering CONNECT with a transient status (502, 503, 429) failed on the first attempt instead of being retried, as it was before. Other changes from the same review pass: - Rename `_send_request` to `send_request` and `_is_retryable_transport_error` to `is_retryable_transport_error`, so the whole documented transport contract is public. - Drop the no-op hook overrides from the custom-client docs examples and the test doubles, and drop the pass-through `LogClient._is_timeout_error`. - Derive the expected `Accept-Encoding` from HTTPX instead of hard-coding whichever codecs are installed. - Build `__all__` as one literal per branch, so RUF022 keeps checking it. - Assert the HTTPX no-timeout mapping through the real `build_request` rather than through a mock.
1 parent c4e20bc commit 175382d

17 files changed

Lines changed: 138 additions & 182 deletions

‎docs/02_concepts/10_custom_http_clients.mdx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,9 @@ All of these are available from the `apify_client.http_clients` module:
124124

125125
The public `call` method provides the shared request pipeline. A concrete transport implements these hooks:
126126

127-
- `_send_request(...)` sends one prepared request and returns an `HttpResponse`. The inherited `call` needs it, so
127+
- `send_request(...)` sends one prepared request and returns an `HttpResponse`. The inherited `call` needs it, so
128128
every transport adapter has to implement it.
129-
- `_is_retryable_transport_error(exc)` classifies transport failures for the shared retry loop. The default classifies
129+
- `is_retryable_transport_error(exc)` classifies transport failures for the shared retry loop. The default classifies
130130
nothing as retryable, so a transport that skips it gives up on the first connection failure.
131131
- `is_timeout_error(exc)` identifies transport-specific timeout exceptions for higher-level client features. The
132132
default recognizes Python's `TimeoutError`.

‎docs/02_concepts/code/10_plugging_in_async.py‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,7 @@ class MyHttpClientAsync(HttpClientAsync):
1010
"""Custom async HTTP client."""
1111

1212
@override
13-
def is_timeout_error(self, exc: Exception) -> bool:
14-
return super().is_timeout_error(exc)
15-
16-
@override
17-
async def aclose(self) -> None:
18-
"""Close the transport's connection pool."""
19-
20-
@override
21-
async def _send_request(
13+
async def send_request(
2214
self,
2315
*,
2416
method: str,
@@ -32,7 +24,7 @@ async def _send_request(
3224
raise NotImplementedError
3325

3426
@override
35-
def _is_retryable_transport_error(self, exc: Exception) -> bool:
27+
def is_retryable_transport_error(self, exc: Exception) -> bool:
3628
# List the transport's transient failures here, e.g. its timeout
3729
# and connection errors. Returning False for everything opts out
3830
# of transport retries entirely.

‎docs/02_concepts/code/10_plugging_in_sync.py‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,7 @@ class MyHttpClient(HttpClient):
1010
"""Custom sync HTTP client."""
1111

1212
@override
13-
def is_timeout_error(self, exc: Exception) -> bool:
14-
return super().is_timeout_error(exc)
15-
16-
@override
17-
def close(self) -> None:
18-
"""Close the transport's connection pool."""
19-
20-
@override
21-
def _send_request(
13+
def send_request(
2214
self,
2315
*,
2416
method: str,
@@ -32,7 +24,7 @@ def _send_request(
3224
raise NotImplementedError
3325

3426
@override
35-
def _is_retryable_transport_error(self, exc: Exception) -> bool:
27+
def is_retryable_transport_error(self, exc: Exception) -> bool:
3628
# List the transport's transient failures here, e.g. its timeout
3729
# and connection errors. Returning False for everything opts out
3830
# of transport retries entirely.

‎docs/03_guides/code/05_custom_http_client_async.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ async def aclose(self) -> None:
9696
await self._session.close()
9797

9898
@override
99-
async def _send_request(
99+
async def send_request(
100100
self,
101101
*,
102102
method: str,
@@ -121,7 +121,7 @@ async def _send_request(
121121
return adapted_response
122122

123123
@override
124-
def _is_retryable_transport_error(self, exc: Exception) -> bool:
124+
def is_retryable_transport_error(self, exc: Exception) -> bool:
125125
return isinstance(exc, (TimeoutError, aiohttp.ClientError))
126126

127127

‎docs/03_guides/code/05_custom_http_client_sync.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ def close(self) -> None:
8989
self._session.close()
9090

9191
@override
92-
def _send_request(
92+
def send_request(
9393
self,
9494
*,
9595
method: str,
@@ -115,7 +115,7 @@ def _send_request(
115115
return adapted_response
116116

117117
@override
118-
def _is_retryable_transport_error(self, exc: Exception) -> bool:
118+
def is_retryable_transport_error(self, exc: Exception) -> bool:
119119
return isinstance(
120120
exc,
121121
(

‎pyproject.toml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -208,11 +208,6 @@ include = ["docs/**/*.py", "website/**/*.py"]
208208
[tool.ty.overrides.rules]
209209
unresolved-import = "ignore"
210210

211-
[[tool.ty.overrides]]
212-
include = ["docs/**/10_plugging_in_async.py", "docs/**/10_plugging_in_sync.py"]
213-
[tool.ty.overrides.rules]
214-
empty-body = "ignore"
215-
216211
[[tool.ty.overrides]]
217212
include = ["docs/**/05_custom_http_client_async.py"]
218213
[tool.ty.overrides.rules]

‎src/apify_client/_resource_clients/log.py‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,6 @@ def __init__(
3434
**kwargs,
3535
)
3636

37-
def _is_timeout_error(self, exc: Exception) -> bool:
38-
"""Return whether an exception came from the configured HTTP client's timeout handling."""
39-
return self._http_client.is_timeout_error(exc)
40-
4137
def get(self, *, raw: bool = False, timeout: Timeout = 'long') -> str | None:
4238
"""Retrieve the log as text.
4339
@@ -146,10 +142,6 @@ def __init__(
146142
**kwargs,
147143
)
148144

149-
def _is_timeout_error(self, exc: Exception) -> bool:
150-
"""Return whether an exception came from the configured HTTP client's timeout handling."""
151-
return self._http_client.is_timeout_error(exc)
152-
153145
async def get(self, *, raw: bool = False, timeout: Timeout = 'long') -> str | None:
154146
"""Retrieve the log as text.
155147

‎src/apify_client/_streamed_log.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ def _stream_log(self) -> None:
160160
# Flush the last buffered part even if the read timed out or was stopped.
161161
self._log_buffer_content(include_last_part=True)
162162
except Exception as exc:
163-
if self._log_client._is_timeout_error(exc): # noqa: SLF001
163+
if self._log_client._http_client.is_timeout_error(exc): # noqa: SLF001
164164
# The stream cannot continue, so warn and let the thread end instead of leaking a traceback (#1040).
165165
self._to_logger.warning('Log streaming stopped: the log stream request timed out.')
166166
else:
@@ -239,7 +239,7 @@ async def _stream_log(self) -> None:
239239
# Flush the last buffered part even if the task is cancelled by `stop()`.
240240
self._log_buffer_content(include_last_part=True)
241241
except Exception as exc:
242-
if self._log_client._is_timeout_error(exc): # noqa: SLF001
242+
if self._log_client._http_client.is_timeout_error(exc): # noqa: SLF001
243243
# A timeout on the long-lived stream is an expected terminal condition, not an error.
244244
self._to_logger.warning('Log streaming stopped: the log stream request timed out.')
245245
else:

‎src/apify_client/http_clients/__init__.py‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,21 @@
1515
) as _httpx_import:
1616
from apify_client.http_clients._httpx import HttpxHttpClient, HttpxHttpClientAsync
1717

18-
__all__ = [
19-
'HttpClient',
20-
'HttpClientAsync',
21-
'HttpResponse',
22-
'ImpitHttpClient',
23-
'ImpitHttpClientAsync',
24-
]
25-
2618
if _httpx_import.available:
27-
__all__ += ['HttpxHttpClient', 'HttpxHttpClientAsync']
19+
__all__ = [
20+
'HttpClient',
21+
'HttpClientAsync',
22+
'HttpResponse',
23+
'HttpxHttpClient',
24+
'HttpxHttpClientAsync',
25+
'ImpitHttpClient',
26+
'ImpitHttpClientAsync',
27+
]
28+
else:
29+
__all__ = [
30+
'HttpClient',
31+
'HttpClientAsync',
32+
'HttpResponse',
33+
'ImpitHttpClient',
34+
'ImpitHttpClientAsync',
35+
]

‎src/apify_client/http_clients/_base.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -352,11 +352,11 @@ def _build_url_with_params(self, url: str, *, params: dict[str, Any] | None = No
352352
def _handle_request_exception(self, exc: Exception, *, stop_retrying: Callable[[], None]) -> None:
353353
"""Stop retrying when an exception is not a transient response or transport failure."""
354354
logger.debug('Request threw exception', exc_info=exc)
355-
if not isinstance(exc, InvalidResponseBodyError) and not self._is_retryable_transport_error(exc):
355+
if not isinstance(exc, InvalidResponseBodyError) and not self.is_retryable_transport_error(exc):
356356
logger.debug('Exception is not retryable', exc_info=exc)
357357
stop_retrying()
358358

359-
def _is_retryable_transport_error(self, exc: Exception) -> bool:
359+
def is_retryable_transport_error(self, exc: Exception) -> bool:
360360
"""Return whether an underlying HTTP-library exception is retryable.
361361
362362
The default classifies nothing as retryable, so a transport that doesn't override it gives up on the first
@@ -387,7 +387,7 @@ class HttpClient(HttpClientBase):
387387
"""Base class for synchronous HTTP clients used by `ApifyClient`.
388388
389389
Concrete clients inherit its request preparation, retry, and error handling, and implement the transport,
390-
error-classification, and lifecycle hooks. Only `_send_request` has to be implemented, the other hooks have
390+
error-classification, and lifecycle hooks. Only `send_request` has to be implemented, the other hooks have
391391
defaults. Replacing `call` itself also remains supported, and then none of the hooks are used. Helper methods
392392
from `HttpClientBase` remain available for request preparation, URL building, and parameter parsing.
393393
@@ -531,7 +531,7 @@ def _make_request(
531531
self._statistics.requests += 1
532532

533533
try:
534-
response = self._send_request(
534+
response = self.send_request(
535535
method=method,
536536
url=self._build_url_with_params(url, params=params),
537537
headers=headers,
@@ -568,7 +568,7 @@ def _make_request(
568568

569569
raise ApifyApiError(response, attempt, method=method)
570570

571-
def _send_request(
571+
def send_request(
572572
self,
573573
*,
574574
method: str,
@@ -742,7 +742,7 @@ async def _make_request(
742742
self._statistics.requests += 1
743743

744744
try:
745-
response = await self._send_request(
745+
response = await self.send_request(
746746
method=method,
747747
url=self._build_url_with_params(url, params=params),
748748
headers=headers,
@@ -779,7 +779,7 @@ async def _make_request(
779779

780780
raise ApifyApiError(response, attempt, method=method)
781781

782-
async def _send_request(
782+
async def send_request(
783783
self,
784784
*,
785785
method: str,

0 commit comments

Comments
 (0)