Skip to content

Commit c8f8ea2

Browse files
authored
fix: raise InvalidResponseBodyError for every unparsable response body (#1109)
Only `KeyValueStoreClient.get_record()` raised `InvalidResponseBodyError`. Every other method parses through `response_to_dict()` / `response_to_list()`, so a malformed JSON body came out as a bare `json.JSONDecodeError`. Both helpers now raise `InvalidResponseBodyError` and chain the decode error as its cause, which matches the JS client. `InvalidResponseBodyError` now subclasses `ValueError` as well as `ApifyClientError`, so code that catches `ValueError` today still catches it. A body that parses but has the wrong shape (a list where a dict is expected) still raises a plain `ValueError`, same as in JS. Closes: #1094 *✍️ Drafted by Claude Code*
1 parent dc30b12 commit c8f8ea2

6 files changed

Lines changed: 65 additions & 8 deletions

File tree

‎docs/02_concepts/10_custom_http_clients.mdx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ Decorate your implementations with `@override`, as the built-in Impit and HTTPX2
115115
| `text: str` | Response body as text |
116116
| `content: bytes` | Raw response body |
117117
| `headers: Mapping[str, str]` | Response headers |
118-
| `json() -> Any` | Parse body as JSON |
118+
| `json() -> Any` | Parse body as JSON, raising `ValueError` if it is not valid JSON |
119119
| `read() -> bytes` | Read entire response body |
120120
| `aread() -> bytes` | Read entire response body (async) |
121121
| `close() -> None` | Close the response |

‎src/apify_client/_utils/http.py‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from __future__ import annotations
22

33
import warnings
4-
from typing import TYPE_CHECKING
4+
from typing import TYPE_CHECKING, Any
55
from urllib.parse import quote
66

77
from apify_client._consts import (
@@ -11,6 +11,7 @@
1111
COMPRESSIBLE_MEDIA_TYPES,
1212
OVERRIDABLE_DEFAULT_HEADERS,
1313
)
14+
from apify_client.errors import InvalidResponseBodyError
1415

1516
if TYPE_CHECKING:
1617
from apify_client.http_clients import HttpResponse
@@ -80,6 +81,18 @@ def is_compressible_content_type(content_type: str | None) -> bool:
8081
)
8182

8283

84+
def _parse_json(response: HttpResponse) -> Any:
85+
"""Parse the response body as JSON.
86+
87+
Raises:
88+
InvalidResponseBodyError: If the response body is not valid JSON.
89+
"""
90+
try:
91+
return response.json()
92+
except ValueError as err:
93+
raise InvalidResponseBodyError(response) from err
94+
95+
8396
def response_to_dict(response: HttpResponse) -> dict:
8497
"""Parse the API response as a dictionary and validate its type.
8598
@@ -90,9 +103,10 @@ def response_to_dict(response: HttpResponse) -> dict:
90103
The parsed response as a dictionary.
91104
92105
Raises:
106+
InvalidResponseBodyError: If the response body is not valid JSON.
93107
ValueError: If the response is not a dictionary.
94108
"""
95-
data = response.json()
109+
data = _parse_json(response)
96110

97111
if isinstance(data, dict):
98112
return data
@@ -110,9 +124,10 @@ def response_to_list(response: HttpResponse) -> list:
110124
The parsed response as a list.
111125
112126
Raises:
127+
InvalidResponseBodyError: If the response body is not valid JSON.
113128
ValueError: If the response is not a list.
114129
"""
115-
data = response.json()
130+
data = _parse_json(response)
116131

117132
if isinstance(data, list):
118133
return data

‎src/apify_client/errors.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,12 +134,13 @@ class ServerError(ApifyApiError):
134134

135135

136136
@docs_group('Errors')
137-
class InvalidResponseBodyError(ApifyClientError):
137+
class InvalidResponseBodyError(ApifyClientError, ValueError):
138138
"""Error raised when a response body cannot be parsed.
139139
140140
This occurs when the API returns a body that does not match its content type, for example a malformed JSON
141141
document. It is raised on a response the client already accepted, so it is not retried - a transfer that breaks
142-
mid-body surfaces as a transport error inside the retry loop instead.
142+
mid-body surfaces as a transport error inside the retry loop instead. It subclasses `ValueError`, the error a
143+
failed JSON decode raises, so an `except ValueError` handler catches it too.
143144
"""
144145

145146
def __init__(self, response: HttpResponse) -> None:

‎src/apify_client/http_clients/_base.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,11 @@ def headers(self) -> Mapping[str, str]:
7878
"""Response headers as a mapping."""
7979

8080
def json(self) -> Any:
81-
"""Parse response body as JSON."""
81+
"""Parse response body as JSON.
82+
83+
Raises:
84+
ValueError: If the body is not valid JSON.
85+
"""
8286

8387
def read(self) -> bytes:
8488
"""Read the entire response body."""

‎tests/unit/test_client_errors.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
ConflictError,
1313
ForbiddenError,
1414
InvalidRequestError,
15+
InvalidResponseBodyError,
1516
NotFoundError,
1617
RateLimitError,
1718
ServerError,
@@ -280,6 +281,24 @@ def test_apify_api_error_falls_back_for_unparsable_body(
280281
assert exc.value.type is None
281282

282283

284+
def test_get_raises_invalid_response_body_error_on_malformed_json(
285+
httpserver: HTTPServer, sync_client: ApifyClient
286+
) -> None:
287+
"""A successful response whose body is not valid JSON raises `InvalidResponseBodyError`."""
288+
httpserver.expect_request('/v2/datasets/ds-1').respond_with_data(b'{"data": ', content_type='application/json')
289+
with pytest.raises(InvalidResponseBodyError):
290+
sync_client.dataset('ds-1').get()
291+
292+
293+
async def test_get_raises_invalid_response_body_error_on_malformed_json_async(
294+
httpserver: HTTPServer, async_client: ApifyClientAsync
295+
) -> None:
296+
"""A successful response whose body is not valid JSON raises `InvalidResponseBodyError`."""
297+
httpserver.expect_request('/v2/datasets/ds-1').respond_with_data(b'{"data": ', content_type='application/json')
298+
with pytest.raises(InvalidResponseBodyError):
299+
await async_client.dataset('ds-1').get()
300+
301+
283302
def test_direct_get_returns_none_on_404(httpserver: HTTPServer, sync_client: ApifyClient) -> None:
284303
httpserver.expect_request('/v2/datasets/missing').respond_with_json(_not_found_body(), status=404)
285304
assert sync_client.dataset('missing').get() is None

‎tests/unit/test_utils.py‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
to_safe_id,
2727
)
2828
from apify_client._utils.try_import import FailedImport, try_import
29-
from apify_client.errors import ApifyApiError
29+
from apify_client.errors import ApifyApiError, InvalidResponseBodyError
3030

3131
if TYPE_CHECKING:
3232
from collections.abc import AsyncIterator, Callable
@@ -485,6 +485,24 @@ def test_response_to_list_raises_for_non_list() -> None:
485485
response_to_list(mock_response)
486486

487487

488+
@pytest.mark.parametrize(
489+
'parse',
490+
[
491+
pytest.param(response_to_dict, id='response_to_dict'),
492+
pytest.param(response_to_list, id='response_to_list'),
493+
],
494+
)
495+
def test_response_parsing_raises_invalid_response_body_error(parse: Callable[[Any], object]) -> None:
496+
"""A body that is not valid JSON raises `InvalidResponseBodyError`, which is also a `ValueError`."""
497+
mock_response = Mock()
498+
mock_response.json.side_effect = json.JSONDecodeError('Expecting value', 'not json', 0)
499+
with pytest.raises(InvalidResponseBodyError) as exc_info:
500+
parse(mock_response)
501+
assert isinstance(exc_info.value, ValueError)
502+
assert exc_info.value.response is mock_response
503+
assert isinstance(exc_info.value.__cause__, json.JSONDecodeError)
504+
505+
488506
@pytest.mark.parametrize(
489507
('input_value', 'expected'),
490508
[

0 commit comments

Comments
 (0)