Skip to content

Commit 3b128d1

Browse files
authored
fix: allow max_retries=0 to disable retries (#1142)
`max_retries` counts retries, so `0` should mean a single attempt. Instead, every request with `max_retries=0` raised `ValueError` from the retry loop. Now `0` disables retries, and a negative value raises `ValueError` when `ApifyClient`, `ApifyClientAsync` or an HTTP client is constructed. This is a regression from 3.0.0. Up to 2.5.1, `max_retries=0` made exactly one attempt; the `ValueError` check came in with #604. The JS client also accepts `0`. The tests in `test_client_errors.py` that meant to avoid retries used `max_retries=1`, which still retries once. They now use `0`. Closes: #1139 *✍️ Drafted by Claude Code*
1 parent 9fb576c commit 3b128d1

7 files changed

Lines changed: 76 additions & 19 deletions

File tree

‎docs/02_concepts/05_retries.mdx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ By default, the client retries a failed request up to 4 times. The retry interva
2828

2929
You can customize this behavior using the following options in the <ApiLink to="class/ApifyClient">`ApifyClient`</ApiLink> constructor:
3030

31-
- `max_retries`: Defines the maximum number of retry attempts.
31+
- `max_retries`: Defines the maximum number of retry attempts. Set it to `0` to disable retries.
3232
- `min_delay_between_retries`: Sets the minimum delay between retries as a `timedelta`.
3333

3434
Retries with exponential backoff help reduce the load on the server and increase the chances of a successful request.

‎src/apify_client/_apify_client.py‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ def __init__(
140140
as well.
141141
api_public_url: The globally accessible URL of the Apify API server. Should be set only if `api_url`
142142
is an internal URL that is not globally accessible. Defaults to https://api.apify.com.
143-
max_retries: How many times to retry a failed request at most.
143+
max_retries: How many times to retry a failed request at most. Set to 0 to disable retries.
144144
min_delay_between_retries: How long will the client wait between retrying requests
145145
(increases exponentially from this value).
146146
timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
@@ -150,7 +150,13 @@ def __init__(
150150
headers: Additional HTTP headers to include in all API requests.
151151
compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm,
152152
or an `HttpCompressor` instance for finer-grained control.
153+
154+
Raises:
155+
ValueError: If `max_retries` is negative.
153156
"""
157+
if max_retries < 0:
158+
raise ValueError(f'max_retries must be a non-negative integer, got {max_retries}')
159+
154160
# We need to do this because of mocking in tests and default mutable arguments.
155161
api_url = DEFAULT_API_URL if api_url is None else api_url
156162
api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url
@@ -504,7 +510,7 @@ def __init__(
504510
as well.
505511
api_public_url: The globally accessible URL of the Apify API server. Should be set only if `api_url`
506512
is an internal URL that is not globally accessible. Defaults to https://api.apify.com.
507-
max_retries: How many times to retry a failed request at most.
513+
max_retries: How many times to retry a failed request at most. Set to 0 to disable retries.
508514
min_delay_between_retries: How long will the client wait between retrying requests
509515
(increases exponentially from this value).
510516
timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
@@ -514,7 +520,13 @@ def __init__(
514520
headers: Additional HTTP headers to include in all API requests.
515521
compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm,
516522
or an `HttpCompressor` instance for finer-grained control.
523+
524+
Raises:
525+
ValueError: If `max_retries` is negative.
517526
"""
527+
if max_retries < 0:
528+
raise ValueError(f'max_retries must be a non-negative integer, got {max_retries}')
529+
518530
# We need to do this because of mocking in tests and default mutable arguments.
519531
api_url = DEFAULT_API_URL if api_url is None else api_url
520532
api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url

‎src/apify_client/http_clients/_base.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -136,12 +136,18 @@ def __init__(
136136
timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
137137
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
138138
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
139-
max_retries: Maximum number of retries for failed requests.
139+
max_retries: Maximum number of retries for failed requests. Set to 0 to disable retries.
140140
min_delay_between_retries: Minimum delay between retries.
141141
statistics: Statistics tracker for API calls. Created automatically if not provided.
142142
headers: Additional HTTP headers to include in all requests.
143143
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
144+
145+
Raises:
146+
ValueError: If `max_retries` is negative.
144147
"""
148+
if max_retries < 0:
149+
raise ValueError(f'max_retries must be a non-negative integer, got {max_retries}')
150+
145151
self._http_compressor = http_compressor if http_compressor is not None else GzipHttpCompressor()
146152
self._timeout_short = timeout_short
147153
self._timeout_medium = timeout_medium
@@ -598,9 +604,6 @@ def _retry_with_exp_backoff(
598604
random_factor: float = 1,
599605
) -> T:
600606
"""Retry a function with exponential backoff and jitter."""
601-
if max_retries < 1:
602-
raise ValueError(f'max_retries must be at least 1, got {max_retries}')
603-
604607
random_factor = min(max(0, random_factor), 1)
605608
backoff_factor = min(max(1, backoff_factor), 10)
606609
swallow = True
@@ -825,9 +828,6 @@ async def _retry_with_exp_backoff(
825828
random_factor: float = 1,
826829
) -> T:
827830
"""Retry an async function with exponential backoff and jitter."""
828-
if max_retries < 1:
829-
raise ValueError(f'max_retries must be at least 1, got {max_retries}')
830-
831831
random_factor = min(max(0, random_factor), 1)
832832
backoff_factor = min(max(1, backoff_factor), 10)
833833
swallow = True

‎src/apify_client/http_clients/_httpx2.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ def __init__(
7575
timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
7676
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
7777
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
78-
max_retries: Maximum number of retry attempts for failed requests.
78+
max_retries: Maximum number of retry attempts for failed requests. Set to 0 to disable retries.
7979
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
8080
statistics: Statistics tracker for API calls. Created automatically if not provided.
8181
headers: Additional HTTP headers to include in all requests.
@@ -179,7 +179,7 @@ def __init__(
179179
timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
180180
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
181181
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
182-
max_retries: Maximum number of retry attempts for failed requests.
182+
max_retries: Maximum number of retry attempts for failed requests. Set to 0 to disable retries.
183183
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
184184
statistics: Statistics tracker for API calls. Created automatically if not provided.
185185
headers: Additional HTTP headers to include in all requests.

‎src/apify_client/http_clients/_impit.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ def __init__(
7171
timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
7272
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
7373
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
74-
max_retries: Maximum number of retry attempts for failed requests.
74+
max_retries: Maximum number of retry attempts for failed requests. Set to 0 to disable retries.
7575
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
7676
statistics: Statistics tracker for API calls. Created automatically if not provided.
7777
headers: Additional HTTP headers to include in all requests.
@@ -171,7 +171,7 @@ def __init__(
171171
timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
172172
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
173173
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
174-
max_retries: Maximum number of retry attempts for failed requests.
174+
max_retries: Maximum number of retry attempts for failed requests. Set to 0 to disable retries.
175175
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
176176
statistics: Statistics tracker for API calls. Created automatically if not provided.
177177
headers: Additional HTTP headers to include in all requests.

‎tests/unit/test_client_errors.py‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import pytest
88
from werkzeug import Response
99

10+
from apify_client import ApifyClient, ApifyClientAsync
1011
from apify_client.errors import (
1112
ApifyApiError,
1213
ConflictError,
@@ -25,7 +26,6 @@
2526
from pytest_httpserver import HTTPServer
2627
from werkzeug import Request
2728

28-
from apify_client import ApifyClient, ApifyClientAsync
2929
from apify_client.http_clients import HttpClient, HttpClientAsync
3030

3131
pytestmark = pytest.mark.usefixtures('http_client_classes')
@@ -195,7 +195,7 @@ def test_apify_api_error_dispatches_5xx_to_server_error(
195195
httpserver.expect_request('/server_error').respond_with_json(
196196
{'error': {'type': 'internal-error', 'message': 'boom'}}, status=503
197197
)
198-
client = http_client_class(max_retries=1)
198+
client = http_client_class(max_retries=0)
199199

200200
with pytest.raises(ServerError) as exc:
201201
client.call(method='GET', url=str(httpserver.url_for('/server_error')))
@@ -204,6 +204,32 @@ def test_apify_api_error_dispatches_5xx_to_server_error(
204204
assert exc.value.status_code == 503
205205

206206

207+
def test_max_retries_zero_makes_single_attempt(httpserver: HTTPServer) -> None:
208+
"""With `max_retries=0`, a retryable server error is requested once and raised as `ServerError`."""
209+
httpserver.expect_request('/v2/datasets/ds-1').respond_with_json(
210+
{'error': {'type': 'internal-error', 'message': 'boom'}}, status=500
211+
)
212+
client = ApifyClient(token='test', api_url=httpserver.url_for('/').removesuffix('/'), max_retries=0)
213+
214+
with pytest.raises(ServerError):
215+
client.dataset('ds-1').get()
216+
217+
assert len(httpserver.log) == 1
218+
219+
220+
async def test_max_retries_zero_makes_single_attempt_async(httpserver: HTTPServer) -> None:
221+
"""With `max_retries=0`, a retryable server error is requested once and raised as `ServerError`."""
222+
httpserver.expect_request('/v2/datasets/ds-1').respond_with_json(
223+
{'error': {'type': 'internal-error', 'message': 'boom'}}, status=500
224+
)
225+
client = ApifyClientAsync(token='test', api_url=httpserver.url_for('/').removesuffix('/'), max_retries=0)
226+
227+
with pytest.raises(ServerError):
228+
await client.dataset('ds-1').get()
229+
230+
assert len(httpserver.log) == 1
231+
232+
207233
def test_apify_api_error_falls_back_for_unmapped_status(
208234
httpserver: HTTPServer, http_client_class: type[HttpClient]
209235
) -> None:
@@ -242,8 +268,8 @@ def test_apify_api_error_dispatches_all_mapped_statuses(
242268
httpserver.expect_request('/dispatch_all').respond_with_json(
243269
{'error': {'type': 'some-type', 'message': 'msg'}}, status=status_code
244270
)
245-
# Use max_retries=1 so retryable statuses (429) don't loop during the test.
246-
client = http_client_class(max_retries=1)
271+
# `max_retries=0` keeps retryable statuses (429) from being retried.
272+
client = http_client_class(max_retries=0)
247273

248274
with pytest.raises(expected_cls) as exc:
249275
client.call(method='GET', url=str(httpserver.url_for('/dispatch_all')))
@@ -272,7 +298,7 @@ def test_apify_api_error_falls_back_for_unparsable_body(
272298
) -> None:
273299
"""When the body can't be parsed, status-based dispatch still applies and `.type` is None."""
274300
httpserver.expect_request('/unparsable').respond_with_data('<not json>', status=418, content_type='text/html')
275-
client = http_client_class(max_retries=1)
301+
client = http_client_class(max_retries=0)
276302

277303
with pytest.raises(ApifyApiError) as exc:
278304
client.call(method='GET', url=str(httpserver.url_for('/unparsable')))

‎tests/unit/test_http_clients.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import impit
1818
import pytest
1919

20+
from apify_client import ApifyClient, ApifyClientAsync
2021
from apify_client._consts import MIN_COMPRESSION_SIZE
2122
from apify_client._logging import logger_name
2223
from apify_client._statistics import ClientStatistics
@@ -210,6 +211,24 @@ def test_base_http_client_initialization() -> None:
210211
assert isinstance(client2._statistics, ClientStatistics)
211212

212213

214+
@pytest.mark.parametrize(
215+
'client_class',
216+
[
217+
pytest.param(ApifyClient, id='ApifyClient'),
218+
pytest.param(ApifyClientAsync, id='ApifyClientAsync'),
219+
pytest.param(ConcreteHttpClient, id='custom HttpClient'),
220+
pytest.param(ImpitHttpClient, id='ImpitHttpClient'),
221+
pytest.param(ImpitHttpClientAsync, id='ImpitHttpClientAsync'),
222+
pytest.param(Httpx2HttpClient, id='Httpx2HttpClient'),
223+
pytest.param(Httpx2HttpClientAsync, id='Httpx2HttpClientAsync'),
224+
],
225+
)
226+
def test_negative_max_retries_is_rejected_at_construction(client_class: Callable[..., object]) -> None:
227+
"""A negative `max_retries` raises `ValueError` from the constructor, before any request is made."""
228+
with pytest.raises(ValueError, match='max_retries must be a non-negative integer, got -1'):
229+
client_class(max_retries=-1)
230+
231+
213232
def test_http_client_init_headers_override_defaults_case_insensitively() -> None:
214233
"""Constructor headers replace same-named default headers even when their casings differ."""
215234
client = ConcreteHttpClient(token='default_token', headers={'authorization': 'Bearer custom'})

0 commit comments

Comments
 (0)