Skip to content

Commit 064b5cc

Browse files
authored
fix: Read file-like KVS values before upload and reject unencodable ones (#965)
Passing a file-like value (`open(...)`, `io.BytesIO(...)`) to `set_record` — or to `actor.start` / `actor.call` / `actor.validate_input` / `run.metamorph` via `run_input` — crashed with a raw `TypeError: 'BytesIO' object is not an instance of 'Sequence'`. `encode_key_value_store_record_value` set a content type for `io.IOBase` but passed the object through unread, and impit's `content=` only accepts bytes-like bodies. The encoder now reads file-like values into memory. Detection is duck-typed on a callable `read`, so non-`io.IOBase` wrappers work too. Binary maps to `application/octet-stream`, text to `text/plain`. `read()` is called with no arguments, so the value is consumed from its current position and the object is neither rewound nor closed. Values it cannot turn into a body now raise `TypeError` instead of reaching the transport: - async file-likes (`aiofiles`, Starlette `UploadFile`) — `read()` returns a coroutine - `read()` returning neither bytes nor `str`, e.g. `None` from a non-blocking raw stream - a non-bytes-like value with a non-JSON content type, e.g. `set_record('k', {'a': 1}, content_type='image/png')` Note: impit exposes no streaming `content=` API, so uploads are fully buffered rather than streamed. Tests: `set_record` over `BytesIO` / `StringIO` / a duck-typed reader, sync and async, over both compression algorithms, plus encoder coverage of the three rejection paths. *✍️ Drafted by Claude Code*
1 parent ac95e35 commit 064b5cc

3 files changed

Lines changed: 212 additions & 13 deletions

File tree

‎src/apify_client/_utils/encoding.py‎

Lines changed: 39 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
from __future__ import annotations
22

3-
import io
43
import json
54
from base64 import b64encode
65
from functools import cache
6+
from inspect import isawaitable, iscoroutine
77
from typing import TYPE_CHECKING, Any
88

99
from apify_client._models import WebhookCreate, WebhookRepresentation
@@ -12,29 +12,51 @@
1212
from apify_client.types import WebhooksList
1313

1414

15-
def encode_key_value_store_record_value(value: Any, *, content_type: str | None = None) -> tuple[Any, str]:
15+
def encode_key_value_store_record_value(
16+
value: Any, *, content_type: str | None = None
17+
) -> tuple[bytes | bytearray | str, str]:
1618
"""Encode a value for storage in a key-value store record.
1719
1820
Args:
19-
value: The value to encode (can be dict, str, bytes, or file-like object).
21+
value: The value to encode. Anything exposing a callable `read` is treated as a file-like object: `read`
22+
is called with no arguments, so the value is consumed from its current position and buffered in
23+
memory whole - the object is neither rewound nor closed, and async file-like objects are rejected.
24+
Any other value is JSON-serialized unless it is already bytes or a string.
2025
content_type: The content type; if None, it's inferred from the value type.
2126
2227
Returns:
2328
A tuple of (encoded_value, content_type).
29+
30+
Raises:
31+
TypeError: If the value cannot be encoded into a body the transport accepts.
2432
"""
33+
# Read file-like values into memory; the transport only accepts bytes-like bodies. Detect them by a
34+
# callable `read` (not `io.IOBase`) so duck-typed file-likes are read, not JSON-serialized. Impit exposes
35+
# no streaming `content=` API, so the value has to be buffered whole.
36+
read = getattr(value, 'read', None)
37+
if callable(read):
38+
value = read()
39+
40+
if isawaitable(value):
41+
if iscoroutine(value):
42+
value.close() # Prevent a "coroutine was never awaited" warning.
43+
raise TypeError(
44+
'Async file-like objects are not supported. Await the read yourself and pass the resulting '
45+
'bytes or string.'
46+
)
47+
48+
if not isinstance(value, (bytes, bytearray, str)):
49+
raise TypeError(f'Reading the file-like value returned {type(value).__name__}, expected bytes or str.')
50+
2551
if not content_type:
26-
if isinstance(value, (bytes, bytearray, io.IOBase)):
52+
if isinstance(value, (bytes, bytearray)):
2753
content_type = 'application/octet-stream'
2854
elif isinstance(value, str):
2955
content_type = 'text/plain; charset=utf-8'
3056
else:
3157
content_type = 'application/json; charset=utf-8'
3258

33-
if (
34-
'application/json' in content_type
35-
and not isinstance(value, (bytes, bytearray, io.IOBase))
36-
and not isinstance(value, str)
37-
):
59+
if 'application/json' in content_type and not isinstance(value, (bytes, bytearray, str)):
3860
# Don't use indentation to reduce size.
3961
value = json.dumps(
4062
value,
@@ -43,6 +65,14 @@ def encode_key_value_store_record_value(value: Any, *, content_type: str | None
4365
default=str,
4466
).encode('utf-8')
4567

68+
# A non-JSON content type skips the serialization above, so anything that is not bytes-like would reach the
69+
# transport unencoded and fail there with an opaque error.
70+
if not isinstance(value, (bytes, bytearray, str)):
71+
raise TypeError(
72+
f'Cannot encode a {type(value).__name__} value as {content_type!r}. Pass bytes, a string, or a '
73+
'file-like object, or use a JSON content type.'
74+
)
75+
4676
return (value, content_type)
4777

4878

‎tests/unit/test_key_value_store.py‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
from __future__ import annotations
2+
3+
import gzip
4+
import io
5+
from typing import TYPE_CHECKING, Any
6+
7+
import brotli
8+
import pytest
9+
from werkzeug import Request, Response
10+
11+
from apify_client import ApifyClient, ApifyClientAsync
12+
13+
if TYPE_CHECKING:
14+
from collections.abc import Callable
15+
16+
from pytest_httpserver import HTTPServer
17+
18+
from apify_client.types import HttpCompressionAlgorithm
19+
20+
_MOCKED_KVS_ID = 'test_kvs_id'
21+
_RECORD_PATH = f'/v2/key-value-stores/{_MOCKED_KVS_ID}/records/f'
22+
23+
24+
class DuckTypedReader:
25+
"""A file-like object that is not an `io.IOBase`, so only duck-typed detection picks it up."""
26+
27+
def read(self) -> bytes:
28+
return b'buffer data'
29+
30+
31+
# The values are built by a factory because reading consumes them, and each case runs once per compression
32+
# algorithm. Each case is (value factory, expected uploaded body, expected content type).
33+
_FILE_LIKE_VALUE_CASES = [
34+
pytest.param(lambda: io.BytesIO(b'buffer data'), b'buffer data', 'application/octet-stream', id='bytes io'),
35+
pytest.param(lambda: io.StringIO('buffer data'), b'buffer data', 'text/plain; charset=utf-8', id='string io'),
36+
pytest.param(DuckTypedReader, b'buffer data', 'application/octet-stream', id='duck-typed reader'),
37+
]
38+
39+
40+
@pytest.fixture(
41+
params=[
42+
pytest.param(('gzip', 'gzip'), id='gzip'),
43+
pytest.param(('brotli', 'br'), id='brotli'),
44+
]
45+
)
46+
def compression_case(request: pytest.FixtureRequest) -> tuple[HttpCompressionAlgorithm, str]:
47+
"""Run each test over both supported request-body compression algorithms, as (algorithm, content-encoding)."""
48+
return request.param
49+
50+
51+
@pytest.fixture
52+
def api_url(httpserver: HTTPServer) -> str:
53+
"""The base URL of the mock server, in the form the clients expect."""
54+
return httpserver.url_for('/').removesuffix('/')
55+
56+
57+
@pytest.fixture
58+
def captured_records(httpserver: HTTPServer) -> list[Request]:
59+
"""Answer record uploads with a 201 and collect the requests the client sent."""
60+
requests: list[Request] = []
61+
62+
def capture_request(request: Request) -> Response:
63+
requests.append(request)
64+
return Response(status=201)
65+
66+
httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(capture_request)
67+
return requests
68+
69+
70+
def decode_body(request: Request) -> bytes:
71+
"""Decompress a captured request body according to its `Content-Encoding`."""
72+
raw = request.get_data()
73+
encoding = request.headers.get('Content-Encoding')
74+
if encoding == 'gzip':
75+
return gzip.decompress(raw)
76+
if encoding == 'br':
77+
return brotli.decompress(raw)
78+
return raw
79+
80+
81+
@pytest.mark.parametrize(('make_value', 'expected_body', 'expected_content_type'), _FILE_LIKE_VALUE_CASES)
82+
def test_set_record_reads_file_like_value_sync(
83+
*,
84+
api_url: str,
85+
captured_records: list[Request],
86+
compression_case: tuple[HttpCompressionAlgorithm, str],
87+
make_value: Callable[[], Any],
88+
expected_body: bytes,
89+
expected_content_type: str,
90+
) -> None:
91+
"""A file-like value is read and its bytes are uploaded, not passed through unread."""
92+
algorithm, content_encoding = compression_case
93+
client = ApifyClient(token='test_token', api_url=api_url, compression=algorithm)
94+
95+
client.key_value_store(_MOCKED_KVS_ID).set_record('f', make_value())
96+
97+
assert len(captured_records) == 1
98+
assert captured_records[0].headers['content-encoding'] == content_encoding
99+
assert decode_body(captured_records[0]) == expected_body
100+
assert captured_records[0].headers['content-type'] == expected_content_type
101+
102+
103+
@pytest.mark.parametrize(('make_value', 'expected_body', 'expected_content_type'), _FILE_LIKE_VALUE_CASES)
104+
async def test_set_record_reads_file_like_value_async(
105+
*,
106+
api_url: str,
107+
captured_records: list[Request],
108+
compression_case: tuple[HttpCompressionAlgorithm, str],
109+
make_value: Callable[[], Any],
110+
expected_body: bytes,
111+
expected_content_type: str,
112+
) -> None:
113+
"""A file-like value is read and its bytes are uploaded, not passed through unread."""
114+
algorithm, content_encoding = compression_case
115+
client = ApifyClientAsync(token='test_token', api_url=api_url, compression=algorithm)
116+
117+
await client.key_value_store(_MOCKED_KVS_ID).set_record('f', make_value())
118+
119+
assert len(captured_records) == 1
120+
assert captured_records[0].headers['content-encoding'] == content_encoding
121+
assert decode_body(captured_records[0]) == expected_body
122+
assert captured_records[0].headers['content-type'] == expected_content_type

‎tests/unit/test_utils.py‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -223,8 +223,7 @@ def test__clean_json_payload(input_dict: dict, expected: dict) -> None:
223223
def test_encode_key_value_store_record_value_dict() -> None:
224224
"""Test that dictionaries are encoded as JSON."""
225225
value, content_type = encode_key_value_store_record_value({'key': 'value'})
226-
assert b'"key"' in value
227-
assert b'"value"' in value
226+
assert value == b'{"key": "value"}'
228227
assert content_type == 'application/json; charset=utf-8'
229228

230229

@@ -249,13 +248,61 @@ def test_encode_key_value_store_record_value(
249248

250249

251250
def test_encode_key_value_store_record_value_bytesio() -> None:
252-
"""Test that BytesIO is encoded as octet-stream."""
251+
"""Test that BytesIO is read into bytes and encoded as octet-stream."""
253252
buffer = io.BytesIO(b'buffer data')
254253
value, content_type = encode_key_value_store_record_value(buffer)
255-
assert value == buffer
254+
assert value == b'buffer data'
256255
assert content_type == 'application/octet-stream'
257256

258257

258+
def test_encode_key_value_store_record_value_stringio() -> None:
259+
"""Test that StringIO is read into text and encoded as text/plain."""
260+
buffer = io.StringIO('buffer data')
261+
value, content_type = encode_key_value_store_record_value(buffer)
262+
assert value == 'buffer data'
263+
assert content_type == 'text/plain; charset=utf-8'
264+
265+
266+
def test_encode_key_value_store_record_value_duck_typed_file_like() -> None:
267+
"""Test that a duck-typed file-like value (a callable `read`, not an `io.IOBase`) is read into bytes."""
268+
269+
class Reader:
270+
def read(self) -> bytes:
271+
return b'buffer data'
272+
273+
value, content_type = encode_key_value_store_record_value(Reader())
274+
assert value == b'buffer data'
275+
assert content_type == 'application/octet-stream'
276+
277+
278+
def test_encode_key_value_store_record_value_async_file_like_raises() -> None:
279+
"""Test that an async file-like value is rejected instead of storing the repr of an un-awaited coroutine."""
280+
281+
class AsyncReader:
282+
async def read(self) -> bytes:
283+
return b'buffer data'
284+
285+
with pytest.raises(TypeError, match='Async file-like objects are not supported'):
286+
encode_key_value_store_record_value(AsyncReader())
287+
288+
289+
def test_encode_key_value_store_record_value_non_bytes_read_raises() -> None:
290+
"""Test that a `read` returning neither bytes nor str is rejected instead of being JSON-serialized."""
291+
292+
class EmptyNonBlockingReader:
293+
def read(self) -> None:
294+
"""Mimic a non-blocking raw stream with no data available."""
295+
296+
with pytest.raises(TypeError, match='returned NoneType, expected bytes or str'):
297+
encode_key_value_store_record_value(EmptyNonBlockingReader())
298+
299+
300+
def test_encode_key_value_store_record_value_non_encodable_with_explicit_content_type_raises() -> None:
301+
"""Test that a non-bytes-like value with a non-JSON content type is rejected before it reaches the transport."""
302+
with pytest.raises(TypeError, match="Cannot encode a dict value as 'image/png'"):
303+
encode_key_value_store_record_value({'a': 1}, content_type='image/png')
304+
305+
259306
def test_response_to_dict() -> None:
260307
"""Test parsing response as dictionary."""
261308
mock_response = Mock()

0 commit comments

Comments
 (0)