From 867e5e0bdbe70c010ef9192c5dce4e55166fe295 Mon Sep 17 00:00:00 2001 From: Furkan Coban Date: Wed, 18 Nov 2026 21:40:28 +0300 Subject: [PATCH] feat: retry unprocessed requests in batch_add_requests --- src/apify_client/_consts.py | 6 + .../_resource_clients/request_queue.py | 203 +++++++++++++----- tests/unit/test_client_request_queue.py | 154 ++++++++++++- 3 files changed, 304 insertions(+), 59 deletions(-) diff --git a/src/apify_client/_consts.py b/src/apify_client/_consts.py index 5e6db6dd..1b06f4a5 100644 --- a/src/apify_client/_consts.py +++ b/src/apify_client/_consts.py @@ -26,6 +26,12 @@ DEFAULT_MIN_DELAY_BETWEEN_RETRIES = timedelta(milliseconds=500) """Default minimum delay between retries.""" +DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS = 3 +"""Default maximum number of retries for unprocessed requests in `batch_add_requests`.""" + +DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES = timedelta(milliseconds=500) +"""Default minimum delay between retries for unprocessed requests in `batch_add_requests`.""" + DEFAULT_WAIT_FOR_FINISH = timedelta(seconds=999999) """Default maximum wait time for job completion (effectively infinite).""" diff --git a/src/apify_client/_resource_clients/request_queue.py b/src/apify_client/_resource_clients/request_queue.py index baa1bf30..d1b537ef 100644 --- a/src/apify_client/_resource_clients/request_queue.py +++ b/src/apify_client/_resource_clients/request_queue.py @@ -3,12 +3,18 @@ import asyncio import json import math +import random +import time from collections.abc import Iterable from queue import Queue from typing import TYPE_CHECKING, Any, Literal from more_itertools import constrained_batches +from apify_client._consts import ( + DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES, + DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, +) from apify_client._docs import docs_group from apify_client._models import ( AddedRequest, @@ -48,6 +54,7 @@ from datetime import timedelta from apify_client._literals import GeneralAccess + from apify_client._statistics import ClientStatistics from apify_client._typeddicts import ( RequestResourceCamelDict, RequestResourceDict, @@ -70,26 +77,51 @@ def _serialize_requests( requests: list[RequestWithoutId] | list[RequestWithoutIdDict] | list[RequestWithoutIdCamelDict], -) -> list[bytes]: - """Validate requests and serialize each one into the JSON bytes it will occupy in the batch request body. +) -> list[tuple[str, bytes]]: + """Validate requests and serialize each one into `(unique_key, json_bytes)` for batch upload. Each request is serialized exactly once: the same bytes are measured when splitting requests into batches and - then assembled into the request body, so batch sizes are computed on exactly the bytes that get sent. Validation - and serialization happen in a single pass, so each intermediate dict stays transient instead of a whole dict list - being held in memory alongside the serialized requests. Uses the same `json.dumps` options as + then assembled into the request body, so batch sizes are computed on exactly the bytes that get sent. Pairing each + payload with its validated `unique_key` lets unprocessed-request retry rounds reassemble only the remaining items + without re-serializing or decoding payloads. Uses the same `json.dumps` options as `HttpClientBase._prepare_request_call` to keep the wire format consistent with other endpoints. """ - return [ - json.dumps( - (request if isinstance(request, RequestWithoutId) else RequestWithoutId.model_validate(request)).model_dump( - mode='json', by_alias=True, exclude_none=True, fallback=str - ), + serialized = list[tuple[str, bytes]]() + for request in requests: + validated = request if isinstance(request, RequestWithoutId) else RequestWithoutId.model_validate(request) + payload = json.dumps( + validated.model_dump(mode='json', by_alias=True, exclude_none=True, fallback=str), ensure_ascii=False, allow_nan=False, default=str, ).encode('utf-8') - for request in requests - ] + serialized.append((validated.unique_key, payload)) + return serialized + + +def _compute_retry_delay_seconds(attempt: int, min_delay: timedelta) -> float: + """Compute exponential backoff delay with jitter for an unprocessed-requests retry round.""" + return random.uniform(1.0, 2.0) * (2**attempt) * to_seconds(min_delay) + + +def _record_batch_attempt_result( + *, + batch_items: list[tuple[str, bytes]], + batch_response: BatchAddResponse, + processed_unique_keys: set[str], + processed_requests: list[AddedRequest], + attempt: int, + statistics: ClientStatistics, +) -> tuple[list[tuple[str, bytes]], list[UnprocessedRequest]]: + """Accumulate processed requests, record rate limits when needed, and return remaining items to retry.""" + processed_requests.extend(batch_response.data.processed_requests) + unprocessed = batch_response.data.unprocessed_requests + if not unprocessed: + return [], unprocessed + statistics.add_rate_limit_error(attempt + 1) + processed_unique_keys.update(req.unique_key for req in batch_response.data.processed_requests) + remaining = [item for item in batch_items if item[0] not in processed_unique_keys] + return remaining, unprocessed @docs_group('Resource clients') @@ -417,11 +449,16 @@ def batch_add_requests( *, forefront: bool = False, max_parallel: int = 1, + max_unprocessed_requests_retries: int = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, + min_delay_between_unprocessed_requests_retries: timedelta = ( + DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES + ), timeout: Timeout = 'medium', ) -> BatchAddResult: """Add requests to the request queue in batches. - Requests are split into batches based on size and processed in parallel. + Requests are split into batches based on size and processed in parallel. Any requests reported as unprocessed + by the API are retried with exponential backoff. https://docs.apify.com/api/v2#/reference/request-queues/batch-request-operations/add-requests @@ -431,6 +468,9 @@ def batch_add_requests( max_parallel: Specifies the maximum number of parallel tasks for API calls. This is only applicable to the async client. For the sync client, this value must be set to 1, as parallel execution is not supported. + max_unprocessed_requests_retries: Maximum number of retry attempts for requests returned as unprocessed + by the API. + min_delay_between_unprocessed_requests_retries: Minimum delay before retrying unprocessed requests. timeout: Timeout for the API HTTP request. Returns: @@ -438,8 +478,12 @@ def batch_add_requests( """ if max_parallel != 1: raise NotImplementedError('max_parallel is only supported in async client') + if max_unprocessed_requests_retries < 0: + raise ValueError( + f'max_unprocessed_requests_retries must be non-negative, got {max_unprocessed_requests_retries}' + ) - # Validate the requests and serialize each of them into JSON bytes. + # Validate the requests and serialize each of them into `(unique_key, json_bytes)`. serialized_requests = _serialize_requests(requests) # Build the query parameters shared by all the batch API calls. @@ -453,12 +497,12 @@ def batch_add_requests( serialized_requests, max_size=payload_size_limit_bytes - len(b'[]'), max_count=_RQ_MAX_REQUESTS_PER_BATCH, - get_len=lambda serialized: len(serialized) + len(b','), + get_len=lambda item: len(item[1]) + len(b','), strict=False, ) # Put the batches into the queue for processing. - batch_queue = Queue[Iterable[bytes]]() + batch_queue = Queue[Iterable[tuple[str, bytes]]]() for batch in batches: batch_queue.put(batch) @@ -468,22 +512,40 @@ def batch_add_requests( # Process all batches in the queue sequentially. while not batch_queue.empty(): - request_batch = batch_queue.get() + batch_items = list(batch_queue.get()) + remaining_items = batch_items + processed_unique_keys = set[str]() + batch_processed_requests = list[AddedRequest]() + batch_unprocessed_requests = list[UnprocessedRequest]() + + for attempt in range(1 + max_unprocessed_requests_retries): + response = self._http_client.call( + url=self._build_url('requests/batch'), + method='POST', + headers={'content-type': 'application/json'}, + params=request_params, + data=b'[' + b','.join(payload for _, payload in remaining_items) + b']', + timeout=timeout, + ) - # Send the batch to the API, assembling the body from the already serialized requests. - response = self._http_client.call( - url=self._build_url('requests/batch'), - method='POST', - headers={'content-type': 'application/json'}, - params=request_params, - data=b'[' + b','.join(request_batch) + b']', - timeout=timeout, - ) + result = response_to_dict(response) + batch_response = BatchAddResponse.model_validate(result) + remaining_items, batch_unprocessed_requests = _record_batch_attempt_result( + batch_items=batch_items, + batch_response=batch_response, + processed_unique_keys=processed_unique_keys, + processed_requests=batch_processed_requests, + attempt=attempt, + statistics=self._http_client.stats, + ) + if not remaining_items: + break - result = response_to_dict(response) - batch_response = BatchAddResponse.model_validate(result) - processed_requests.extend(batch_response.data.processed_requests) - unprocessed_requests.extend(batch_response.data.unprocessed_requests) + if attempt < max_unprocessed_requests_retries: + time.sleep(_compute_retry_delay_seconds(attempt, min_delay_between_unprocessed_requests_retries)) + + processed_requests.extend(batch_processed_requests) + unprocessed_requests.extend(batch_unprocessed_requests) return BatchAddResponse.model_construct( data=BatchAddResult.model_construct( @@ -945,13 +1007,15 @@ async def delete_request_lock( async def _batch_add_requests_worker( self, *, - queue: asyncio.Queue[Iterable[bytes]], + queue: asyncio.Queue[Iterable[tuple[str, bytes]]], request_params: QueryParams, + max_unprocessed_requests_retries: int, + min_delay_between_unprocessed_requests_retries: timedelta, timeout: Timeout, ) -> BatchAddResponse: """Worker function to process a batch of requests. - This worker will process batches from the queue. + This worker will process batches from the queue, retrying any unprocessed requests with exponential backoff. Return result containing lists of processed and unprocessed requests by the worker. """ @@ -966,20 +1030,42 @@ async def _batch_add_requests_worker( break try: - # Send the batch to the API, assembling the body from the already serialized requests. - response = await self._http_client.call( - url=self._build_url('requests/batch'), - method='POST', - headers={'content-type': 'application/json'}, - params=request_params, - data=b'[' + b','.join(request_batch) + b']', - timeout=timeout, - ) + batch_items = list(request_batch) + remaining_items = batch_items + processed_unique_keys = set[str]() + batch_processed_requests = list[AddedRequest]() + batch_unprocessed_requests = list[UnprocessedRequest]() + + for attempt in range(1 + max_unprocessed_requests_retries): + response = await self._http_client.call( + url=self._build_url('requests/batch'), + method='POST', + headers={'content-type': 'application/json'}, + params=request_params, + data=b'[' + b','.join(payload for _, payload in remaining_items) + b']', + timeout=timeout, + ) - result = response_to_dict(response) - batch_response = BatchAddResponse.model_validate(result) - processed_requests.extend(batch_response.data.processed_requests) - unprocessed_requests.extend(batch_response.data.unprocessed_requests) + result = response_to_dict(response) + batch_response = BatchAddResponse.model_validate(result) + remaining_items, batch_unprocessed_requests = _record_batch_attempt_result( + batch_items=batch_items, + batch_response=batch_response, + processed_unique_keys=processed_unique_keys, + processed_requests=batch_processed_requests, + attempt=attempt, + statistics=self._http_client.stats, + ) + if not remaining_items: + break + + if attempt < max_unprocessed_requests_retries: + await asyncio.sleep( + _compute_retry_delay_seconds(attempt, min_delay_between_unprocessed_requests_retries) + ) + + processed_requests.extend(batch_processed_requests) + unprocessed_requests.extend(batch_unprocessed_requests) finally: # Mark the batch as done whether it succeeded or failed. @@ -998,11 +1084,16 @@ async def batch_add_requests( *, forefront: bool = False, max_parallel: int = 5, + max_unprocessed_requests_retries: int = DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS, + min_delay_between_unprocessed_requests_retries: timedelta = ( + DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES + ), timeout: Timeout = 'medium', ) -> BatchAddResult: """Add requests to the request queue in batches. - Requests are split into batches based on size and processed in parallel. + Requests are split into batches based on size and processed in parallel. Any requests reported as unprocessed + by the API are retried with exponential backoff. https://docs.apify.com/api/v2#/reference/request-queues/batch-request-operations/add-requests @@ -1012,11 +1103,19 @@ async def batch_add_requests( max_parallel: Specifies the maximum number of parallel tasks for API calls. This is only applicable to the async client. For the sync client, this value must be set to 1, as parallel execution is not supported. + max_unprocessed_requests_retries: Maximum number of retry attempts for requests returned as unprocessed + by the API. + min_delay_between_unprocessed_requests_retries: Minimum delay before retrying unprocessed requests. timeout: Timeout for the API HTTP request. Returns: Result containing lists of processed and unprocessed requests. """ + if max_unprocessed_requests_retries < 0: + raise ValueError( + f'max_unprocessed_requests_retries must be non-negative, got {max_unprocessed_requests_retries}' + ) + # Validate and serialize the requests in a worker thread, as it is CPU-bound and would block the event loop. serialized_requests = await asyncio.to_thread(_serialize_requests, requests) @@ -1031,12 +1130,12 @@ async def batch_add_requests( serialized_requests, max_size=payload_size_limit_bytes - len(b'[]'), max_count=_RQ_MAX_REQUESTS_PER_BATCH, - get_len=lambda serialized: len(serialized) + len(b','), + get_len=lambda item: len(item[1]) + len(b','), strict=False, ) # Create a queue with all the batches, from which the worker tasks will consume them. - batch_queue: asyncio.Queue[Iterable[bytes]] = asyncio.Queue() + batch_queue: asyncio.Queue[Iterable[tuple[str, bytes]]] = asyncio.Queue() for batch in batches: await batch_queue.put(batch) @@ -1047,7 +1146,13 @@ async def batch_add_requests( workers = [ tg.create_task( self._batch_add_requests_worker( - queue=batch_queue, request_params=request_params, timeout=timeout + queue=batch_queue, + request_params=request_params, + max_unprocessed_requests_retries=max_unprocessed_requests_retries, + min_delay_between_unprocessed_requests_retries=( + min_delay_between_unprocessed_requests_retries + ), + timeout=timeout, ), name=f'batch_add_requests_worker_{i}', ) diff --git a/tests/unit/test_client_request_queue.py b/tests/unit/test_client_request_queue.py index 728fa459..91c44200 100644 --- a/tests/unit/test_client_request_queue.py +++ b/tests/unit/test_client_request_queue.py @@ -3,6 +3,7 @@ import gzip import json import re +from datetime import timedelta from typing import TYPE_CHECKING import pytest @@ -76,19 +77,35 @@ async def test_batch_processed_partially_async(httpserver: HTTPServer) -> None: api_public_url=server_url, ) - httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data( - status=200, response_data=_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT - ) + attempt = 0 + + def handler(_request: Request) -> Response: + nonlocal attempt + attempt += 1 + if attempt == 1: + return Response(_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT, status=200, content_type='application/json') + return Response( + '{"data": {"processedRequests": [], "unprocessedRequests": ' + '[{"uniqueKey": "http://example.com/2", "url": "http://example.com/2", "method": "GET"}]}}', + status=200, + content_type='application/json', + ) + + httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler(handler) requests: list[RequestWithoutIdDict] = [ {'unique_key': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'}, {'unique_key': 'http://example.com/2', 'url': 'http://example.com/2', 'method': 'GET'}, ] rq_client = client.request_queue(request_queue_id='whatever') - batch_response = await rq_client.batch_add_requests(requests=requests) - assert requests[0]['unique_key'] in {request.unique_key for request in batch_response.processed_requests} + batch_response = await rq_client.batch_add_requests( + requests=requests, + min_delay_between_unprocessed_requests_retries=timedelta(milliseconds=1), + ) + assert [request.unique_key for request in batch_response.processed_requests] == [requests[0]['unique_key']] assert len(batch_response.unprocessed_requests) == 1 assert batch_response.unprocessed_requests[0].unique_key == requests[1]['unique_key'] + assert client.stats.rate_limit_errors == {0: 1, 1: 1, 2: 1, 3: 1} def test_batch_not_processed_raises_exception_sync(httpserver: HTTPServer) -> None: @@ -247,19 +264,136 @@ def test_batch_processed_partially_sync(httpserver: HTTPServer) -> None: api_public_url=server_url, ) - httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data( - status=200, response_data=_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT - ) + attempt = 0 + + def handler(_request: Request) -> Response: + nonlocal attempt + attempt += 1 + if attempt == 1: + return Response(_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT, status=200, content_type='application/json') + return Response( + '{"data": {"processedRequests": [], "unprocessedRequests": ' + '[{"uniqueKey": "http://example.com/2", "url": "http://example.com/2", "method": "GET"}]}}', + status=200, + content_type='application/json', + ) + + httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler(handler) requests: list[RequestWithoutIdDict] = [ {'unique_key': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'}, {'unique_key': 'http://example.com/2', 'url': 'http://example.com/2', 'method': 'GET'}, ] rq_client = client.request_queue(request_queue_id='whatever') - batch_response = rq_client.batch_add_requests(requests=requests) - assert requests[0]['unique_key'] in {request.unique_key for request in batch_response.processed_requests} + batch_response = rq_client.batch_add_requests( + requests=requests, + min_delay_between_unprocessed_requests_retries=timedelta(milliseconds=1), + ) + assert [request.unique_key for request in batch_response.processed_requests] == [requests[0]['unique_key']] assert len(batch_response.unprocessed_requests) == 1 assert batch_response.unprocessed_requests[0].unique_key == requests[1]['unique_key'] + assert client.stats.rate_limit_errors == {0: 1, 1: 1, 2: 1, 3: 1} + + +def _progressive_retry_handler(sent_keys: list[list[str]]) -> Callable[[Request], Response]: + """Respond to 3 rounds of batch-add calls, processing one request per round.""" + round_num = 0 + + def handler(request: Request) -> Response: + nonlocal round_num + round_num += 1 + body = request.get_data() + raw = gzip.decompress(body) if request.headers.get('Content-Encoding') == 'gzip' else body + items = json.loads(raw) + sent_keys.append([item['uniqueKey'] for item in items]) + + processed = [ + { + 'requestId': f'id_{round_num}', + 'uniqueKey': f'http://example.com/{round_num}', + 'wasAlreadyPresent': False, + 'wasAlreadyHandled': False, + } + ] + unprocessed = [ + {'uniqueKey': f'http://example.com/{k}', 'url': f'http://example.com/{k}', 'method': 'GET'} + for k in range(round_num + 1, 4) + ] + return Response( + json.dumps({'data': {'processedRequests': processed, 'unprocessedRequests': unprocessed}}), + status=200, + content_type='application/json', + ) + + return handler + + +async def test_batch_add_requests_retries_unprocessed_requests_async(httpserver: HTTPServer) -> None: + """Unprocessed requests are retried by `unique_key` until processed and rate limits are tracked.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClientAsync(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + sent_keys = list[list[str]]() + httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler( + _progressive_retry_handler(sent_keys) + ) + requests: list[RequestWithoutIdDict] = [ + {'unique_key': f'http://example.com/{i}', 'url': f'http://example.com/{i}', 'method': 'GET'} + for i in range(1, 4) + ] + rq_client = client.request_queue(request_queue_id='whatever') + + result = await rq_client.batch_add_requests( + requests=requests, + min_delay_between_unprocessed_requests_retries=timedelta(milliseconds=1), + ) + + assert sent_keys == [ + ['http://example.com/1', 'http://example.com/2', 'http://example.com/3'], + ['http://example.com/2', 'http://example.com/3'], + ['http://example.com/3'], + ] + assert [req.unique_key for req in result.processed_requests] == [ + 'http://example.com/1', + 'http://example.com/2', + 'http://example.com/3', + ] + assert result.unprocessed_requests == [] + assert client.stats.rate_limit_errors == {0: 1, 1: 1} + + +def test_batch_add_requests_retries_unprocessed_requests_sync(httpserver: HTTPServer) -> None: + """Unprocessed requests are retried by `unique_key` until processed and rate limits are tracked.""" + server_url = httpserver.url_for('/').removesuffix('/') + client = ApifyClient(token='placeholder_token', api_url=server_url, api_public_url=server_url) + + sent_keys = list[list[str]]() + httpserver.expect_request(re.compile(r'.*'), method='POST').respond_with_handler( + _progressive_retry_handler(sent_keys) + ) + requests: list[RequestWithoutIdDict] = [ + {'unique_key': f'http://example.com/{i}', 'url': f'http://example.com/{i}', 'method': 'GET'} + for i in range(1, 4) + ] + rq_client = client.request_queue(request_queue_id='whatever') + + result = rq_client.batch_add_requests( + requests=requests, + min_delay_between_unprocessed_requests_retries=timedelta(milliseconds=1), + ) + + assert sent_keys == [ + ['http://example.com/1', 'http://example.com/2', 'http://example.com/3'], + ['http://example.com/2', 'http://example.com/3'], + ['http://example.com/3'], + ] + assert [req.unique_key for req in result.processed_requests] == [ + 'http://example.com/1', + 'http://example.com/2', + 'http://example.com/3', + ] + assert result.unprocessed_requests == [] + assert client.stats.rate_limit_errors == {0: 1, 1: 1} _REQUEST_REGISTRATION_RESPONSE_CONTENT = (