11from __future__ import annotations
22
33import asyncio
4+ import json
45import math
56from collections .abc import Iterable
67from queue import Queue
5758 from apify_client .types import Timeout
5859
5960_RQ_MAX_REQUESTS_PER_BATCH = 25
60- _MAX_PAYLOAD_SIZE_BYTES = 9 * 1024 * 1024 # 9 MB
61- _SAFETY_BUFFER_PERCENT = 0.01 / 100 # 0.01%
61+ """Maximum number of requests the API accepts in a single batch call."""
62+
63+ _MAX_PAYLOAD_SIZE_BYTES = 9 * 1024 * 1024
64+ """Maximum payload size (9 MB) the API accepts for a single batch call."""
65+
66+ _SAFETY_BUFFER_PERCENT = 0.01 / 100
67+ """Safety margin (0.01%) deducted from the maximum payload size when splitting requests into batches."""
68+
69+
70+ def _serialize_requests (
71+ requests : list [RequestDraft ] | list [RequestDraftDict ] | list [RequestDraftCamelDict ],
72+ ) -> list [bytes ]:
73+ """Validate requests and serialize each one into the JSON bytes it will occupy in the batch request body.
74+
75+ Each request is serialized exactly once: the same bytes are measured when splitting requests into batches and
76+ then assembled into the request body, so batch sizes are computed on exactly the bytes that get sent. Validation
77+ and serialization happen in a single pass, so each intermediate dict stays transient instead of a whole dict list
78+ being held in memory alongside the serialized requests. Uses the same `json.dumps` options as
79+ `HttpClientBase._prepare_request_call` to keep the wire format consistent with other endpoints.
80+ """
81+ return [
82+ json .dumps (
83+ (request if isinstance (request , RequestDraft ) else RequestDraft .model_validate (request )).model_dump (
84+ by_alias = True , exclude_none = True
85+ ),
86+ ensure_ascii = False ,
87+ allow_nan = False ,
88+ default = str ,
89+ ).encode ('utf-8' )
90+ for request in requests
91+ ]
6292
6393
6494@docs_group ('Resource clients' )
@@ -400,44 +430,44 @@ def batch_add_requests(
400430 if max_parallel != 1 :
401431 raise NotImplementedError ('max_parallel is only supported in async client' )
402432
403- requests_as_dicts = [
404- (r if isinstance (r , RequestDraft ) else RequestDraft .model_validate (r )).model_dump (
405- by_alias = True , exclude_none = True
406- )
407- for r in requests
408- ]
433+ # Validate the requests and serialize each of them into JSON bytes.
434+ serialized_requests = _serialize_requests (requests )
409435
436+ # Build the query parameters shared by all the batch API calls.
410437 request_params = self ._build_params (clientKey = self .client_key , forefront = forefront )
411438
412439 # Compute the payload size limit to ensure it doesn't exceed the maximum allowed size.
413440 payload_size_limit_bytes = _MAX_PAYLOAD_SIZE_BYTES - math .ceil (_MAX_PAYLOAD_SIZE_BYTES * _SAFETY_BUFFER_PERCENT )
414441
415- # Split the requests into batches, constrained by the max payload size and max requests per batch.
442+ # Split the requests into batches by payload size (counting commas and brackets) and max requests per batch.
416443 batches = constrained_batches (
417- requests_as_dicts ,
418- max_size = payload_size_limit_bytes ,
444+ serialized_requests ,
445+ max_size = payload_size_limit_bytes - len ( b'[]' ) ,
419446 max_count = _RQ_MAX_REQUESTS_PER_BATCH ,
447+ get_len = lambda serialized : len (serialized ) + len (b',' ),
448+ strict = False ,
420449 )
421450
422451 # Put the batches into the queue for processing.
423- queue = Queue [Iterable [dict ]]()
452+ batch_queue = Queue [Iterable [bytes ]]()
424453
425454 for batch in batches :
426- queue .put (batch )
455+ batch_queue .put (batch )
427456
428457 processed_requests = list [AddedRequest ]()
429458 unprocessed_requests = list [RequestDraft ]()
430459
431460 # Process all batches in the queue sequentially.
432- while not queue .empty ():
433- request_batch = queue .get ()
461+ while not batch_queue .empty ():
462+ request_batch = batch_queue .get ()
434463
435- # Send the batch to the API.
464+ # Send the batch to the API, assembling the body from the already serialized requests .
436465 response = self ._http_client .call (
437466 url = self ._build_url ('requests/batch' ),
438467 method = 'POST' ,
468+ headers = {'content-type' : 'application/json' },
439469 params = request_params ,
440- json = list (request_batch ),
470+ data = b'[' + b',' . join (request_batch ) + b']' ,
441471 timeout = timeout ,
442472 )
443473
@@ -898,7 +928,7 @@ async def delete_request_lock(
898928 async def _batch_add_requests_worker (
899929 self ,
900930 * ,
901- queue : asyncio .Queue [Iterable [dict ]],
931+ queue : asyncio .Queue [Iterable [bytes ]],
902932 request_params : dict ,
903933 timeout : Timeout ,
904934 ) -> BatchAddResponse :
@@ -919,12 +949,13 @@ async def _batch_add_requests_worker(
919949 break
920950
921951 try :
922- # Send the batch to the API.
952+ # Send the batch to the API, assembling the body from the already serialized requests .
923953 response = await self ._http_client .call (
924954 url = self ._build_url ('requests/batch' ),
925955 method = 'POST' ,
956+ headers = {'content-type' : 'application/json' },
926957 params = request_params ,
927- json = list (request_batch ),
958+ data = b'[' + b',' . join (request_batch ) + b']' ,
928959 timeout = timeout ,
929960 )
930961
@@ -969,51 +1000,45 @@ async def batch_add_requests(
9691000 Returns:
9701001 Result containing lists of processed and unprocessed requests.
9711002 """
972- requests_as_dicts = [
973- (
974- request
975- if isinstance (request , RequestDraft )
976- else RequestDraft .model_validate (
977- request ,
978- )
979- ).model_dump (
980- by_alias = True ,
981- exclude_none = True ,
982- )
983- for request in requests
984- ]
1003+ # Validate and serialize the requests in a worker thread, as it is CPU-bound and would block the event loop.
1004+ serialized_requests = await asyncio .to_thread (_serialize_requests , requests )
9851005
986- asyncio_queue : asyncio . Queue [ Iterable [ dict ]] = asyncio . Queue ()
1006+ # Build the query parameters shared by all the batch API calls.
9871007 request_params = self ._build_params (clientKey = self .client_key , forefront = forefront )
9881008
9891009 # Compute the payload size limit to ensure it doesn't exceed the maximum allowed size.
9901010 payload_size_limit_bytes = _MAX_PAYLOAD_SIZE_BYTES - math .ceil (_MAX_PAYLOAD_SIZE_BYTES * _SAFETY_BUFFER_PERCENT )
9911011
992- # Split the requests into batches, constrained by the max payload size and max requests per batch.
1012+ # Split the requests into batches by payload size (counting commas and brackets) and max requests per batch.
9931013 batches = constrained_batches (
994- requests_as_dicts ,
995- max_size = payload_size_limit_bytes ,
1014+ serialized_requests ,
1015+ max_size = payload_size_limit_bytes - len ( b'[]' ) ,
9961016 max_count = _RQ_MAX_REQUESTS_PER_BATCH ,
1017+ get_len = lambda serialized : len (serialized ) + len (b',' ),
1018+ strict = False ,
9971019 )
9981020
1021+ # Create a queue with all the batches, from which the worker tasks will consume them.
1022+ batch_queue : asyncio .Queue [Iterable [bytes ]] = asyncio .Queue ()
1023+
9991024 for batch in batches :
1000- await asyncio_queue .put (batch )
1025+ await batch_queue .put (batch )
10011026
10021027 # Use TaskGroup for structured concurrency — automatic cleanup and error propagation.
10031028 try :
10041029 async with asyncio .TaskGroup () as tg :
10051030 workers = [
10061031 tg .create_task (
10071032 self ._batch_add_requests_worker (
1008- queue = asyncio_queue , request_params = request_params , timeout = timeout
1033+ queue = batch_queue , request_params = request_params , timeout = timeout
10091034 ),
10101035 name = f'batch_add_requests_worker_{ i } ' ,
10111036 )
10121037 for i in range (max_parallel )
10131038 ]
10141039
10151040 # Wait for all batches to be processed, then cancel idle workers.
1016- await asyncio_queue .join ()
1041+ await batch_queue .join ()
10171042 for worker in workers :
10181043 worker .cancel ()
10191044 except ExceptionGroup as eg :
0 commit comments