Skip to content

Commit 2adf515

Browse files
authored
fix: Make batch_add_requests split batches by serialized payload size (#953)
The 9 MB payload guard in `batch_add_requests` was inert: `constrained_batches` was called without `get_len`, so the default `len()` measured each request dict's key count (~4) instead of its serialized size. Batches were therefore split only by the 25-request count limit, and large requests shipped as one oversized POST that the API rejects with 413, failing the whole call. The guard now measures each request as its UTF-8 JSON byte length, using the same serialization flags as the HTTP client's request body path. It also passes `strict=False`, which preserves the previous contract for an individually oversized request: it's sent in its own batch and left for the API to judge, instead of raising a client-side `ValueError`. Both the size-based splitting and the oversized-singleton path are covered by new sync/async regression tests. *✍️ Drafted by Claude Code*
1 parent 4324dcf commit 2adf515

2 files changed

Lines changed: 196 additions & 40 deletions

File tree

‎src/apify_client/_resource_clients/request_queue.py‎

Lines changed: 65 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from __future__ import annotations
22

33
import asyncio
4+
import json
45
import math
56
from collections.abc import Iterable
67
from queue import Queue
@@ -57,8 +58,37 @@
5758
from apify_client.types import Timeout
5859

5960
_RQ_MAX_REQUESTS_PER_BATCH = 25
60-
_MAX_PAYLOAD_SIZE_BYTES = 9 * 1024 * 1024 # 9 MB
61-
_SAFETY_BUFFER_PERCENT = 0.01 / 100 # 0.01%
61+
"""Maximum number of requests the API accepts in a single batch call."""
62+
63+
_MAX_PAYLOAD_SIZE_BYTES = 9 * 1024 * 1024
64+
"""Maximum payload size (9 MB) the API accepts for a single batch call."""
65+
66+
_SAFETY_BUFFER_PERCENT = 0.01 / 100
67+
"""Safety margin (0.01%) deducted from the maximum payload size when splitting requests into batches."""
68+
69+
70+
def _serialize_requests(
71+
requests: list[RequestDraft] | list[RequestDraftDict] | list[RequestDraftCamelDict],
72+
) -> list[bytes]:
73+
"""Validate requests and serialize each one into the JSON bytes it will occupy in the batch request body.
74+
75+
Each request is serialized exactly once: the same bytes are measured when splitting requests into batches and
76+
then assembled into the request body, so batch sizes are computed on exactly the bytes that get sent. Validation
77+
and serialization happen in a single pass, so each intermediate dict stays transient instead of a whole dict list
78+
being held in memory alongside the serialized requests. Uses the same `json.dumps` options as
79+
`HttpClientBase._prepare_request_call` to keep the wire format consistent with other endpoints.
80+
"""
81+
return [
82+
json.dumps(
83+
(request if isinstance(request, RequestDraft) else RequestDraft.model_validate(request)).model_dump(
84+
by_alias=True, exclude_none=True
85+
),
86+
ensure_ascii=False,
87+
allow_nan=False,
88+
default=str,
89+
).encode('utf-8')
90+
for request in requests
91+
]
6292

6393

6494
@docs_group('Resource clients')
@@ -400,44 +430,44 @@ def batch_add_requests(
400430
if max_parallel != 1:
401431
raise NotImplementedError('max_parallel is only supported in async client')
402432

403-
requests_as_dicts = [
404-
(r if isinstance(r, RequestDraft) else RequestDraft.model_validate(r)).model_dump(
405-
by_alias=True, exclude_none=True
406-
)
407-
for r in requests
408-
]
433+
# Validate the requests and serialize each of them into JSON bytes.
434+
serialized_requests = _serialize_requests(requests)
409435

436+
# Build the query parameters shared by all the batch API calls.
410437
request_params = self._build_params(clientKey=self.client_key, forefront=forefront)
411438

412439
# Compute the payload size limit to ensure it doesn't exceed the maximum allowed size.
413440
payload_size_limit_bytes = _MAX_PAYLOAD_SIZE_BYTES - math.ceil(_MAX_PAYLOAD_SIZE_BYTES * _SAFETY_BUFFER_PERCENT)
414441

415-
# Split the requests into batches, constrained by the max payload size and max requests per batch.
442+
# Split the requests into batches by payload size (counting commas and brackets) and max requests per batch.
416443
batches = constrained_batches(
417-
requests_as_dicts,
418-
max_size=payload_size_limit_bytes,
444+
serialized_requests,
445+
max_size=payload_size_limit_bytes - len(b'[]'),
419446
max_count=_RQ_MAX_REQUESTS_PER_BATCH,
447+
get_len=lambda serialized: len(serialized) + len(b','),
448+
strict=False,
420449
)
421450

422451
# Put the batches into the queue for processing.
423-
queue = Queue[Iterable[dict]]()
452+
batch_queue = Queue[Iterable[bytes]]()
424453

425454
for batch in batches:
426-
queue.put(batch)
455+
batch_queue.put(batch)
427456

428457
processed_requests = list[AddedRequest]()
429458
unprocessed_requests = list[RequestDraft]()
430459

431460
# Process all batches in the queue sequentially.
432-
while not queue.empty():
433-
request_batch = queue.get()
461+
while not batch_queue.empty():
462+
request_batch = batch_queue.get()
434463

435-
# Send the batch to the API.
464+
# Send the batch to the API, assembling the body from the already serialized requests.
436465
response = self._http_client.call(
437466
url=self._build_url('requests/batch'),
438467
method='POST',
468+
headers={'content-type': 'application/json'},
439469
params=request_params,
440-
json=list(request_batch),
470+
data=b'[' + b','.join(request_batch) + b']',
441471
timeout=timeout,
442472
)
443473

@@ -898,7 +928,7 @@ async def delete_request_lock(
898928
async def _batch_add_requests_worker(
899929
self,
900930
*,
901-
queue: asyncio.Queue[Iterable[dict]],
931+
queue: asyncio.Queue[Iterable[bytes]],
902932
request_params: dict,
903933
timeout: Timeout,
904934
) -> BatchAddResponse:
@@ -919,12 +949,13 @@ async def _batch_add_requests_worker(
919949
break
920950

921951
try:
922-
# Send the batch to the API.
952+
# Send the batch to the API, assembling the body from the already serialized requests.
923953
response = await self._http_client.call(
924954
url=self._build_url('requests/batch'),
925955
method='POST',
956+
headers={'content-type': 'application/json'},
926957
params=request_params,
927-
json=list(request_batch),
958+
data=b'[' + b','.join(request_batch) + b']',
928959
timeout=timeout,
929960
)
930961

@@ -969,51 +1000,45 @@ async def batch_add_requests(
9691000
Returns:
9701001
Result containing lists of processed and unprocessed requests.
9711002
"""
972-
requests_as_dicts = [
973-
(
974-
request
975-
if isinstance(request, RequestDraft)
976-
else RequestDraft.model_validate(
977-
request,
978-
)
979-
).model_dump(
980-
by_alias=True,
981-
exclude_none=True,
982-
)
983-
for request in requests
984-
]
1003+
# Validate and serialize the requests in a worker thread, as it is CPU-bound and would block the event loop.
1004+
serialized_requests = await asyncio.to_thread(_serialize_requests, requests)
9851005

986-
asyncio_queue: asyncio.Queue[Iterable[dict]] = asyncio.Queue()
1006+
# Build the query parameters shared by all the batch API calls.
9871007
request_params = self._build_params(clientKey=self.client_key, forefront=forefront)
9881008

9891009
# Compute the payload size limit to ensure it doesn't exceed the maximum allowed size.
9901010
payload_size_limit_bytes = _MAX_PAYLOAD_SIZE_BYTES - math.ceil(_MAX_PAYLOAD_SIZE_BYTES * _SAFETY_BUFFER_PERCENT)
9911011

992-
# Split the requests into batches, constrained by the max payload size and max requests per batch.
1012+
# Split the requests into batches by payload size (counting commas and brackets) and max requests per batch.
9931013
batches = constrained_batches(
994-
requests_as_dicts,
995-
max_size=payload_size_limit_bytes,
1014+
serialized_requests,
1015+
max_size=payload_size_limit_bytes - len(b'[]'),
9961016
max_count=_RQ_MAX_REQUESTS_PER_BATCH,
1017+
get_len=lambda serialized: len(serialized) + len(b','),
1018+
strict=False,
9971019
)
9981020

1021+
# Create a queue with all the batches, from which the worker tasks will consume them.
1022+
batch_queue: asyncio.Queue[Iterable[bytes]] = asyncio.Queue()
1023+
9991024
for batch in batches:
1000-
await asyncio_queue.put(batch)
1025+
await batch_queue.put(batch)
10011026

10021027
# Use TaskGroup for structured concurrency — automatic cleanup and error propagation.
10031028
try:
10041029
async with asyncio.TaskGroup() as tg:
10051030
workers = [
10061031
tg.create_task(
10071032
self._batch_add_requests_worker(
1008-
queue=asyncio_queue, request_params=request_params, timeout=timeout
1033+
queue=batch_queue, request_params=request_params, timeout=timeout
10091034
),
10101035
name=f'batch_add_requests_worker_{i}',
10111036
)
10121037
for i in range(max_parallel)
10131038
]
10141039

10151040
# Wait for all batches to be processed, then cancel idle workers.
1016-
await asyncio_queue.join()
1041+
await batch_queue.join()
10171042
for worker in workers:
10181043
worker.cancel()
10191044
except ExceptionGroup as eg:

‎tests/unit/test_client_request_queue.py‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,29 @@
11
from __future__ import annotations
22

3+
import gzip
4+
import json
35
import re
46
from typing import TYPE_CHECKING
57

68
import pytest
9+
from werkzeug.wrappers import Response
710

811
from apify_client import ApifyClient, ApifyClientAsync
912
from apify_client.errors import ApifyApiError
1013

1114
if TYPE_CHECKING:
15+
from collections.abc import Callable
16+
1217
from pytest_httpserver import HTTPServer
18+
from werkzeug.wrappers import Request
1319

1420
from apify_client._typeddicts import RequestDraftDict
1521

22+
# The Apify API limit on the payload size of a batch-add request, which the client's batching must respect.
23+
_API_MAX_PAYLOAD_SIZE_BYTES = 9 * 1024 * 1024
24+
25+
_EMPTY_BATCH_RESPONSE_CONTENT = '{"data": {"processedRequests": [], "unprocessedRequests": []}}'
26+
1627
_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT = """{
1728
"data": {
1829
"processedRequests": [
@@ -96,6 +107,126 @@ def test_batch_not_processed_raises_exception_sync(httpserver: HTTPServer) -> No
96107
rq_client.batch_add_requests(requests=requests)
97108

98109

110+
def _make_large_requests() -> list[RequestDraftDict]:
111+
"""Return 3 requests of ~4 MB each, so that all of them together exceed the 9 MB payload limit."""
112+
return [
113+
{
114+
'unique_key': f'http://example.com/{i}',
115+
'url': f'http://example.com/{i}?filler={"x" * (4 * 1024 * 1024)}',
116+
'method': 'GET',
117+
}
118+
for i in range(3)
119+
]
120+
121+
122+
def _payload_capturing_handler(payloads: list[bytes]) -> Callable[[Request], Response]:
123+
"""Return a handler that records each POST body (gzip-decompressed) and responds with an empty batch result."""
124+
125+
def handler(request: Request) -> Response:
126+
payloads.append(gzip.decompress(request.get_data()))
127+
return Response(_EMPTY_BATCH_RESPONSE_CONTENT, status=200, content_type='application/json')
128+
129+
return handler
130+
131+
132+
async def test_batch_add_requests_splits_batches_by_payload_size_async(httpserver: HTTPServer) -> None:
133+
"""Test that batches are split by serialized byte size so no POST payload exceeds the 9 MB API limit."""
134+
server_url = httpserver.url_for('/').removesuffix('/')
135+
client = ApifyClientAsync(
136+
token='placeholder_token',
137+
api_url=server_url,
138+
api_public_url=server_url,
139+
)
140+
141+
payloads = list[bytes]()
142+
httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler(
143+
_payload_capturing_handler(payloads)
144+
)
145+
rq_client = client.request_queue(request_queue_id='whatever')
146+
147+
await rq_client.batch_add_requests(requests=_make_large_requests())
148+
149+
assert len(payloads) > 1
150+
assert all(len(payload) <= _API_MAX_PAYLOAD_SIZE_BYTES for payload in payloads)
151+
assert sum(len(json.loads(payload)) for payload in payloads) == 3
152+
153+
154+
def test_batch_add_requests_splits_batches_by_payload_size_sync(httpserver: HTTPServer) -> None:
155+
"""Test that batches are split by serialized byte size so no POST payload exceeds the 9 MB API limit."""
156+
server_url = httpserver.url_for('/').removesuffix('/')
157+
client = ApifyClient(
158+
token='placeholder_token',
159+
api_url=server_url,
160+
api_public_url=server_url,
161+
)
162+
163+
payloads = list[bytes]()
164+
httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler(
165+
_payload_capturing_handler(payloads)
166+
)
167+
rq_client = client.request_queue(request_queue_id='whatever')
168+
169+
rq_client.batch_add_requests(requests=_make_large_requests())
170+
171+
assert len(payloads) > 1
172+
assert all(len(payload) <= _API_MAX_PAYLOAD_SIZE_BYTES for payload in payloads)
173+
assert sum(len(json.loads(payload)) for payload in payloads) == 3
174+
175+
176+
def _make_oversized_and_small_requests() -> list[RequestDraftDict]:
177+
"""Return a small request plus one whose serialized size alone exceeds the 9 MB payload limit."""
178+
return [
179+
{'unique_key': 'small', 'url': 'http://example.com/small', 'method': 'GET'},
180+
{
181+
'unique_key': 'oversized',
182+
'url': f'http://example.com/oversized?filler={"x" * (10 * 1024 * 1024)}',
183+
'method': 'GET',
184+
},
185+
]
186+
187+
188+
async def test_batch_add_requests_sends_oversized_request_alone_async(httpserver: HTTPServer) -> None:
189+
"""Test that a request exceeding the payload limit is sent in its own batch for the API to judge, not rejected."""
190+
server_url = httpserver.url_for('/').removesuffix('/')
191+
client = ApifyClientAsync(
192+
token='placeholder_token',
193+
api_url=server_url,
194+
api_public_url=server_url,
195+
)
196+
197+
payloads = list[bytes]()
198+
httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler(
199+
_payload_capturing_handler(payloads)
200+
)
201+
rq_client = client.request_queue(request_queue_id='whatever')
202+
203+
await rq_client.batch_add_requests(requests=_make_oversized_and_small_requests())
204+
205+
assert sum(len(json.loads(payload)) for payload in payloads) == 2
206+
assert any(len(payload) > _API_MAX_PAYLOAD_SIZE_BYTES for payload in payloads)
207+
208+
209+
def test_batch_add_requests_sends_oversized_request_alone_sync(httpserver: HTTPServer) -> None:
210+
"""Test that a request exceeding the payload limit is sent in its own batch for the API to judge, not rejected."""
211+
server_url = httpserver.url_for('/').removesuffix('/')
212+
client = ApifyClient(
213+
token='placeholder_token',
214+
api_url=server_url,
215+
api_public_url=server_url,
216+
)
217+
218+
payloads = list[bytes]()
219+
httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler(
220+
_payload_capturing_handler(payloads)
221+
)
222+
rq_client = client.request_queue(request_queue_id='whatever')
223+
224+
rq_client.batch_add_requests(requests=_make_oversized_and_small_requests())
225+
226+
assert sum(len(json.loads(payload)) for payload in payloads) == 2
227+
assert any(len(payload) > _API_MAX_PAYLOAD_SIZE_BYTES for payload in payloads)
228+
229+
99230
def test_batch_processed_partially_sync(httpserver: HTTPServer) -> None:
100231
server_url = httpserver.url_for('/').removesuffix('/')
101232
client = ApifyClient(

0 commit comments

Comments
 (0)