From 82f0ce6d6564b8ecd033abead548e51319aaacdf Mon Sep 17 00:00:00 2001 From: Niklas Semmler Date: Sat, 10 Oct 2026 11:57:19 +0200 Subject: [PATCH 1/4] fix(prompts): stop nesting get_prompt retries around the API client's retries get_prompt wrapped the fetch in a constant 1 s backoff on every exception, around the generated client's own 429/5xx retries. One 429 could become 9 requests, and 401/403/404 were retried too. HTTP retries are now left to the API client (exponential, Retry-After aware, bounded by max_retries); the outer loop only retries transport errors with exponential backoff. Refs LFE-17361 Co-Authored-By: Claude Opus 5.5 (1M context) --- langfuse/_client/client.py | 20 ++++--- tests/unit/test_prompt.py | 103 +++++++++++++++++++++++++++++++++++-- 2 files changed, 111 insertions(+), 12 deletions(-) diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index 5d128567c..467988590 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -3951,7 +3951,7 @@ def get_prompt( keyword argument. If not set, defaults to 60 seconds. Disables caching if set to 0. type: Literal["chat", "text"]: The type of the prompt to retrieve. Defaults to "text". fallback: Union[Optional[List[ChatMessageDict]], Optional[str]]: The prompt string to return if fetching the prompt fails. Important on the first call where no cached prompt is available. Follows Langfuse prompt formatting with double curly braces for variables. Defaults to None. - max_retries: Optional[int]: The maximum number of retries in case of API/network errors. Defaults to 2. The maximum value is 4. Retries have an exponential backoff with a maximum delay of 10 seconds. + max_retries: Optional[int]: The maximum number of retries in case of network errors or retryable API responses (408, 409, 429, 5xx). Other 4xx errors such as 401, 403, and 404 are not retried. Defaults to 2. The maximum value is 4. Retries use exponential backoff; rate-limited responses wait for the server's Retry-After header (up to 60 seconds). fetch_timeout_seconds: Optional[int]: The timeout in seconds for fetching the prompt. Defaults to the default timeout set on the SDK, which is 5 seconds per default. Returns: @@ -4081,20 +4081,26 @@ def _fetch_prompt_and_update_cache( langfuse_logger.debug("Fetching prompt '%s' from server...", cache_key) try: + request_options: RequestOptions = {"max_retries": max_retries} + if fetch_timeout_seconds is not None: + request_options["timeout_in_seconds"] = fetch_timeout_seconds + # The API client already retries 408/409/429/5xx responses with + # exponential backoff and Retry-After. Retry only transport errors + # here so retries are not multiplied and 4xx errors fail fast. @backoff.on_exception( - backoff.constant, Exception, max_tries=max_retries + 1, logger=None + backoff.expo, + httpx.TransportError, + max_tries=max_retries + 1, + max_value=10, + logger=None, ) def fetch_prompts() -> Any: return self.api.prompts.get( self._url_encode(name), version=version, label=label, - request_options={ - "timeout_in_seconds": fetch_timeout_seconds, - } - if fetch_timeout_seconds is not None - else None, + request_options=request_options, ) prompt_response = fetch_prompts() diff --git a/tests/unit/test_prompt.py b/tests/unit/test_prompt.py index eadfb8221..32a00fda4 100644 --- a/tests/unit/test_prompt.py +++ b/tests/unit/test_prompt.py @@ -1,5 +1,8 @@ +import random +import time from unittest.mock import Mock, patch +import httpx import pytest from langfuse._client.client import Langfuse @@ -9,7 +12,8 @@ PromptCacheItem, PromptCacheTaskManager, ) -from langfuse.api import NotFoundError, Prompt_Chat, Prompt_Text +from langfuse.api import LangfuseAPI, NotFoundError, Prompt_Chat, Prompt_Text +from langfuse.api.core.api_error import ApiError from langfuse.model import ChatPromptClient, TextPromptClient @@ -139,6 +143,14 @@ def langfuse(): return langfuse_instance +@pytest.fixture +def sleeps(monkeypatch: pytest.MonkeyPatch) -> list: + recorded: list = [] + monkeypatch.setattr(time, "sleep", recorded.append) + + return recorded + + def wait_for_prompt_refresh(langfuse: Langfuse) -> None: langfuse._resources.prompt_cache._task_manager.wait_for_idle() @@ -199,7 +211,7 @@ def test_get_fresh_prompt(langfuse): prompt_name, version=None, label=None, - request_options=None, + request_options={"max_retries": 2}, ) assert result == TextPromptClient(prompt) @@ -244,7 +256,7 @@ def test_using_custom_prompt_timeouts(langfuse): prompt_name, version=None, label=None, - request_options={"timeout_in_seconds": 1000}, + request_options={"max_retries": 2, "timeout_in_seconds": 1000}, ) assert result == TextPromptClient(prompt) @@ -620,7 +632,7 @@ def test_get_fresh_prompt_when_expired_cache_default_ttl(mock_time, langfuse: La @patch.object(PromptCacheItem, "get_epoch_seconds") -def test_get_expired_prompt_when_failing_fetch(mock_time, langfuse: Langfuse): +def test_get_expired_prompt_when_failing_fetch(mock_time, langfuse: Langfuse, sleeps): mock_time.return_value = 0 prompt_name = "test_get_expired_prompt_when_failing_fetch" @@ -643,7 +655,7 @@ def test_get_expired_prompt_when_failing_fetch(mock_time, langfuse: Langfuse): assert result_call_1 == prompt_client mock_time.return_value = DEFAULT_PROMPT_CACHE_TTL_SECONDS + 1 - mock_server_call.side_effect = Exception("Server error") + mock_server_call.side_effect = httpx.ConnectError("Server error") result_call_2 = langfuse.get_prompt(prompt_name, max_retries=1) wait_for_prompt_refresh(langfuse) @@ -748,3 +760,84 @@ def test_get_fresh_prompt_when_version_changes(langfuse: Langfuse): result_call_2 = langfuse.get_prompt(prompt_name, version=2) assert mock_server_call.call_count == 2 assert result_call_2 == version_changed_prompt_client + + +def _api_over_transport(handler) -> LangfuseAPI: + return LangfuseAPI( + base_url="http://test-host", + username="pk", + password="sk", + httpx_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) + + +def _prompt_json(name: str) -> dict: + return { + "name": name, + "version": 1, + "prompt": "Make me laugh", + "type": "text", + "labels": [], + "config": {}, + "tags": [], + } + + +def test_get_prompt_does_not_multiply_retries_on_rate_limit(langfuse, sleeps): + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(429, headers={"Retry-After": "3"}, json={}) + + langfuse.api = _api_over_transport(handler) + + with pytest.raises(ApiError) as exc_info: + langfuse.get_prompt("rate_limited_prompt", max_retries=2) + + assert exc_info.value.status_code == 429 + assert len(requests) == 3 + assert sleeps == [3, 3] + + +@pytest.mark.parametrize("status_code", [401, 403, 404]) +def test_get_prompt_does_not_retry_non_retryable_client_errors( + langfuse, sleeps, status_code +): + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(status_code, json={"message": "nope"}) + + langfuse.api = _api_over_transport(handler) + + with pytest.raises(ApiError): + langfuse.get_prompt("client_error_prompt", max_retries=2) + + assert len(requests) == 1 + assert sleeps == [] + + +def test_get_prompt_retries_network_errors_with_exponential_backoff( + langfuse, sleeps, monkeypatch: pytest.MonkeyPatch +): + # pin full jitter to its upper bound so the backoff sequence is observable + monkeypatch.setattr(random, "uniform", lambda _low, high: high) + prompt_name = "flaky_network_prompt" + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if len(requests) < 3: + raise httpx.ConnectError("connection refused", request=request) + + return httpx.Response(200, json=_prompt_json(prompt_name)) + + langfuse.api = _api_over_transport(handler) + + result = langfuse.get_prompt(prompt_name, max_retries=2) + + assert result.prompt == "Make me laugh" + assert len(requests) == 3 + assert sleeps == [1, 2] From 2b292cfa908a733d06dbcde92a1a498777f6409d Mon Sep 17 00:00:00 2001 From: Niklas Semmler Date: Sat, 10 Oct 2026 11:57:19 +0200 Subject: [PATCH 2/4] fix(media): stop retrying API errors the API client already retried media.get_upload_url and media.patch go through the generated client, which retries 429/5xx itself. The outer backoff retried them again, turning one 429 into 9 requests. Refs LFE-17361 Co-Authored-By: Claude Opus 5.5 (1M context) --- langfuse/_task_manager/media_manager.py | 8 +++---- tests/unit/test_media_manager.py | 32 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/langfuse/_task_manager/media_manager.py b/langfuse/_task_manager/media_manager.py index 3ad9ac62f..50cdec91d 100644 --- a/langfuse/_task_manager/media_manager.py +++ b/langfuse/_task_manager/media_manager.py @@ -513,12 +513,10 @@ def _request_with_backoff( self, func: Callable[P, T], *args: P.args, **kwargs: P.kwargs ) -> T: def _should_give_up(e: Exception) -> bool: + # The API client already retried 408/409/429/5xx responses before + # raising, so retrying here would multiply the requests. if isinstance(e, ApiError): - return ( - e.status_code is not None - and 400 <= e.status_code < 500 - and e.status_code != 429 - ) + return True if isinstance(e, httpx.HTTPStatusError): return ( e.response is not None diff --git a/tests/unit/test_media_manager.py b/tests/unit/test_media_manager.py index 3ab4e3226..eab624be1 100644 --- a/tests/unit/test_media_manager.py +++ b/tests/unit/test_media_manager.py @@ -1,3 +1,4 @@ +import time from queue import Queue from types import SimpleNamespace from unittest.mock import Mock @@ -6,6 +7,8 @@ import pytest from langfuse._task_manager.media_manager import MediaManager +from langfuse.api import LangfuseAPI +from langfuse.api.core.api_error import ApiError from langfuse.media import LangfuseMedia @@ -277,3 +280,32 @@ def test_find_and_process_media_gemini_inline_data_non_string_data_passes_throug assert result == data assert queue.empty() + + +@pytest.mark.parametrize("status_code", [429, 503]) +def test_media_api_calls_do_not_multiply_client_retries(monkeypatch, status_code): + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(status_code, json={}) + + api_client = LangfuseAPI( + base_url="http://test-host", + username="pk", + password="sk", + httpx_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) + manager = MediaManager( + api_client=api_client, + httpx_client=Mock(), + media_upload_queue=Queue(), + max_retries=3, + ) + + with pytest.raises(ApiError): + manager._process_upload_media_job(data=_upload_job()) + + # the API client's own retries (2) are the only ones: 3 requests in total + assert len(requests) == 3 From 62d34a6399071fe76ec72f0f7fa4d5aff0100a24 Mon Sep 17 00:00:00 2001 From: Niklas Semmler Date: Sat, 10 Oct 2026 12:10:15 +0200 Subject: [PATCH 3/4] fix(tracing): share one retry budget across network and HTTP failures A transport error raised during the API client's internal 429/5xx retries escaped with its retry count, and the outer TransportError retry then started a fresh inner budget, so mixed failures still sent up to (max_retries+1)^2 requests. get_prompt and MediaManager now call the API client with max_retries=0 and retry in one loop (call_with_retries) that covers network errors and 408/409/429/5xx and honors Retry-After / X-RateLimit-Reset. Refs LFE-17361 Co-Authored-By: Claude Opus 5.5 (1M context) --- langfuse/_client/client.py | 26 +++----- langfuse/_task_manager/media_manager.py | 43 ++++++------- langfuse/_utils/retry.py | 83 +++++++++++++++++++++++++ tests/unit/test_media_manager.py | 32 +++++++++- tests/unit/test_prompt.py | 43 ++++++++++++- tests/unit/test_retry.py | 67 ++++++++++++++++++++ 6 files changed, 251 insertions(+), 43 deletions(-) create mode 100644 langfuse/_utils/retry.py create mode 100644 tests/unit/test_retry.py diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index 467988590..fd4928316 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -30,7 +30,6 @@ overload, ) -import backoff import httpx from opentelemetry import context as otel_context_api from opentelemetry import trace as otel_trace_api @@ -98,6 +97,7 @@ from langfuse._utils.environment import get_common_release_envs from langfuse._utils.parse_error import handle_fern_exception from langfuse._utils.prompt_cache import PromptCache +from langfuse._utils.retry import call_with_retries from langfuse.api import ( AsyncLangfuseAPI, CreateChatPromptRequest, @@ -4081,29 +4081,21 @@ def _fetch_prompt_and_update_cache( langfuse_logger.debug("Fetching prompt '%s' from server...", cache_key) try: - request_options: RequestOptions = {"max_retries": max_retries} + # Retries happen only in call_with_retries so network errors and + # 408/409/429/5xx responses share one budget. + request_options: RequestOptions = {"max_retries": 0} if fetch_timeout_seconds is not None: request_options["timeout_in_seconds"] = fetch_timeout_seconds - # The API client already retries 408/409/429/5xx responses with - # exponential backoff and Retry-After. Retry only transport errors - # here so retries are not multiplied and 4xx errors fail fast. - @backoff.on_exception( - backoff.expo, - httpx.TransportError, - max_tries=max_retries + 1, - max_value=10, - logger=None, - ) - def fetch_prompts() -> Any: - return self.api.prompts.get( + prompt_response = call_with_retries( + lambda: self.api.prompts.get( self._url_encode(name), version=version, label=label, request_options=request_options, - ) - - prompt_response = fetch_prompts() + ), + max_retries=max_retries, + ) prompt: PromptClient if prompt_response.type == "chat": diff --git a/langfuse/_task_manager/media_manager.py b/langfuse/_task_manager/media_manager.py index 50cdec91d..bc72776a7 100644 --- a/langfuse/_task_manager/media_manager.py +++ b/langfuse/_task_manager/media_manager.py @@ -3,14 +3,14 @@ from queue import Empty, Full, Queue from typing import Any, Callable, Optional, TypeVar, cast -import backoff import httpx from typing_extensions import ParamSpec from langfuse._client.environment_variables import LANGFUSE_MEDIA_UPLOAD_ENABLED from langfuse._utils import _get_timestamp +from langfuse._utils.retry import call_with_retries, is_retryable_status from langfuse.api import LangfuseAPI, MediaContentType -from langfuse.api.core import ApiError +from langfuse.api.core import ApiError, RequestOptions from langfuse.logger import langfuse_logger as logger from langfuse.media import LangfuseMedia @@ -18,6 +18,9 @@ T = TypeVar("T") P = ParamSpec("P") + +# Retries happen in _request_with_backoff, so the API client must not retry too. +_NO_CLIENT_RETRIES: RequestOptions = {"max_retries": 0} _SHUTDOWN_SENTINEL = object() @@ -41,7 +44,7 @@ def __init__( self._api_client = api_client self._httpx_client = httpx_client self._queue = media_upload_queue - self._max_retries = max_retries + self._max_retries = max_retries if max_retries is not None else 3 self._enabled = os.environ.get( LANGFUSE_MEDIA_UPLOAD_ENABLED, "True" ).lower() not in ("false", "0") @@ -426,6 +429,7 @@ def _process_upload_media_job( dataset_id=data["dataset_id"], dataset_item_id=data["dataset_item_id"], field=data["field"], + request_options=_NO_CLIENT_RETRIES, ) upload_url = upload_url_response.upload_url @@ -484,6 +488,7 @@ def _upload_with_status_check() -> httpx.Response: upload_http_status=failed_response.status_code, upload_http_error=failed_response.text, upload_time_ms=upload_time_ms, + request_options=_NO_CLIENT_RETRIES, ) raise @@ -497,6 +502,7 @@ def _upload_with_status_check() -> httpx.Response: upload_http_status=upload_response.status_code, upload_http_error=upload_response.text, upload_time_ms=upload_time_ms, + request_options=_NO_CLIENT_RETRIES, ) logger.debug( @@ -512,27 +518,18 @@ def _upload_with_status_check() -> httpx.Response: def _request_with_backoff( self, func: Callable[P, T], *args: P.args, **kwargs: P.kwargs ) -> T: - def _should_give_up(e: Exception) -> bool: - # The API client already retried 408/409/429/5xx responses before - # raising, so retrying here would multiply the requests. + def _should_retry(e: Exception) -> bool: if isinstance(e, ApiError): - return True + return e.status_code is not None and is_retryable_status(e.status_code) if isinstance(e, httpx.HTTPStatusError): - return ( - e.response is not None - and e.response.status_code < 500 - and e.response.status_code != 429 + return e.response is not None and is_retryable_status( + e.response.status_code ) - return False - - @backoff.on_exception( - backoff.expo, - Exception, - max_tries=self._max_retries, - giveup=_should_give_up, - logger=None, - ) - def execute_task_with_backoff() -> T: - return func(*args, **kwargs) + return True - return execute_task_with_backoff() + # API client calls pass max_retries=0, so this is the only retry layer. + return call_with_retries( + lambda: func(*args, **kwargs), + max_retries=max(self._max_retries - 1, 0), + should_retry=_should_retry, + ) diff --git a/langfuse/_utils/retry.py b/langfuse/_utils/retry.py new file mode 100644 index 000000000..b4237fb29 --- /dev/null +++ b/langfuse/_utils/retry.py @@ -0,0 +1,83 @@ +"""Retries for calls made through the generated API client. + +Callers pass ``request_options={"max_retries": 0}`` to the API client and wrap +the call in :func:`call_with_retries`, so network errors and retryable HTTP +responses share a single retry budget instead of multiplying each other. +""" + +import random +import time +from typing import Callable, Optional, TypeVar + +import httpx + +from langfuse.api.core.api_error import ApiError +from langfuse.api.core.http_client import ( + _parse_retry_after, + _parse_x_ratelimit_reset, +) + +T = TypeVar("T") + +RETRYABLE_STATUS_CODES = frozenset({408, 409, 429}) + +# Matches the cap the generated API client applies to server-provided delays. +MAX_SERVER_RETRY_DELAY_SECONDS = 60.0 + + +def is_retryable_status(status_code: int) -> bool: + return status_code >= 500 or status_code in RETRYABLE_STATUS_CODES + + +def is_retryable_api_call_error(error: Exception) -> bool: + """Return True for network errors and 408/409/429/5xx API errors.""" + if isinstance(error, httpx.TransportError): + return True + if isinstance(error, ApiError) and error.status_code is not None: + return is_retryable_status(error.status_code) + + return False + + +def _response_headers(error: Exception) -> Optional[httpx.Headers]: + if isinstance(error, ApiError) and error.headers: + return httpx.Headers(error.headers) + if isinstance(error, httpx.HTTPStatusError) and error.response is not None: + return error.response.headers + + return None + + +def retry_delay_seconds( + error: Exception, retries: int, max_backoff_seconds: float +) -> float: + """Honor Retry-After / X-RateLimit-Reset, else exponential backoff with full jitter.""" + headers = _response_headers(error) + if headers is not None: + server_delay = _parse_retry_after(headers) + if server_delay is None or server_delay <= 0: + server_delay = _parse_x_ratelimit_reset(headers) + if server_delay is not None and server_delay > 0: + return min(server_delay, MAX_SERVER_RETRY_DELAY_SECONDS) + + return random.uniform(0, min(2.0**retries, max_backoff_seconds)) + + +def call_with_retries( + func: Callable[[], T], + *, + max_retries: int, + should_retry: Callable[[Exception], bool] = is_retryable_api_call_error, + max_backoff_seconds: float = 10.0, +) -> T: + """Call ``func``, retrying up to ``max_retries`` times on retryable errors.""" + retries = 0 + while True: + try: + return func() + except Exception as e: + if retries >= max_retries or not should_retry(e): + raise + + time.sleep(retry_delay_seconds(e, retries, max_backoff_seconds)) + retries += 1 diff --git a/tests/unit/test_media_manager.py b/tests/unit/test_media_manager.py index eab624be1..2ac4fd7b3 100644 --- a/tests/unit/test_media_manager.py +++ b/tests/unit/test_media_manager.py @@ -307,5 +307,35 @@ def handler(request: httpx.Request) -> httpx.Response: with pytest.raises(ApiError): manager._process_upload_media_job(data=_upload_job()) - # the API client's own retries (2) are the only ones: 3 requests in total + # max_retries=3 means 3 attempts in total, with no nested client retries + assert len(requests) == 3 + + +def test_media_api_calls_share_one_retry_budget_across_failure_kinds(monkeypatch): + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if len(requests) % 2 == 0: + raise httpx.ConnectError("connection reset", request=request) + + return httpx.Response(429, json={}) + + api_client = LangfuseAPI( + base_url="http://test-host", + username="pk", + password="sk", + httpx_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) + manager = MediaManager( + api_client=api_client, + httpx_client=Mock(), + media_upload_queue=Queue(), + max_retries=3, + ) + + with pytest.raises(ApiError): + manager._process_upload_media_job(data=_upload_job()) + assert len(requests) == 3 diff --git a/tests/unit/test_prompt.py b/tests/unit/test_prompt.py index 32a00fda4..9afec0507 100644 --- a/tests/unit/test_prompt.py +++ b/tests/unit/test_prompt.py @@ -211,7 +211,7 @@ def test_get_fresh_prompt(langfuse): prompt_name, version=None, label=None, - request_options={"max_retries": 2}, + request_options={"max_retries": 0}, ) assert result == TextPromptClient(prompt) @@ -256,7 +256,7 @@ def test_using_custom_prompt_timeouts(langfuse): prompt_name, version=None, label=None, - request_options={"max_retries": 2, "timeout_in_seconds": 1000}, + request_options={"max_retries": 0, "timeout_in_seconds": 1000}, ) assert result == TextPromptClient(prompt) @@ -841,3 +841,42 @@ def handler(request: httpx.Request) -> httpx.Response: assert result.prompt == "Make me laugh" assert len(requests) == 3 assert sleeps == [1, 2] + + +def test_get_prompt_shares_one_retry_budget_across_failure_kinds(langfuse, sleeps): + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if len(requests) % 2 == 0: + raise httpx.ConnectError("connection reset", request=request) + + return httpx.Response(503, json={}) + + langfuse.api = _api_over_transport(handler) + + with pytest.raises(httpx.ConnectError): + langfuse.get_prompt("degraded_server_prompt", max_retries=3) + + assert len(requests) == 4 + assert len(sleeps) == 3 + + +@pytest.mark.parametrize("failure", ["rate_limited", "network_error"]) +def test_get_prompt_with_zero_retries_sends_one_request(langfuse, sleeps, failure): + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if failure == "network_error": + raise httpx.ConnectError("connection refused", request=request) + + return httpx.Response(429, headers={"Retry-After": "1"}, json={}) + + langfuse.api = _api_over_transport(handler) + + with pytest.raises((ApiError, httpx.ConnectError)): + langfuse.get_prompt("no_retry_prompt", max_retries=0) + + assert len(requests) == 1 + assert sleeps == [] diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py new file mode 100644 index 000000000..04b77df24 --- /dev/null +++ b/tests/unit/test_retry.py @@ -0,0 +1,67 @@ +import random +import time + +import httpx +import pytest + +from langfuse._utils.retry import ( + MAX_SERVER_RETRY_DELAY_SECONDS, + call_with_retries, + retry_delay_seconds, +) +from langfuse.api.core.api_error import ApiError + + +def _status_error(status_code: int, headers: dict) -> httpx.HTTPStatusError: + request = httpx.Request("PUT", "https://example.com/upload") + response = httpx.Response(status_code, headers=headers, request=request) + return httpx.HTTPStatusError("failed", request=request, response=response) + + +def test_retry_delay_caps_retry_after(): + error = ApiError(status_code=429, headers={"retry-after": "3600"}) + + assert retry_delay_seconds(error, 0, 10) == MAX_SERVER_RETRY_DELAY_SECONDS + + +def test_retry_delay_uses_x_ratelimit_reset_without_retry_after(monkeypatch): + monkeypatch.setattr(time, "time", lambda: 1000.0) + error = ApiError(status_code=429, headers={"x-ratelimit-reset": "1005"}) + + assert retry_delay_seconds(error, 0, 10) == 5 + + +def test_retry_delay_reads_headers_from_http_status_errors(): + error = _status_error(503, {"Retry-After": "7"}) + + assert retry_delay_seconds(error, 0, 10) == 7 + + +def test_retry_delay_falls_back_to_capped_exponential_backoff(monkeypatch): + monkeypatch.setattr(random, "uniform", lambda _low, high: high) + error = ApiError(status_code=503, headers={}) + + assert [retry_delay_seconds(error, retries, 10) for retries in range(5)] == [ + 1, + 2, + 4, + 8, + 10, + ] + + +@pytest.mark.parametrize("status_code", [400, 401, 403, 404, 422]) +def test_call_with_retries_does_not_retry_non_retryable_api_errors( + monkeypatch, status_code +): + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + calls = [] + + def fail() -> None: + calls.append(1) + raise ApiError(status_code=status_code) + + with pytest.raises(ApiError): + call_with_retries(fail, max_retries=3) + + assert len(calls) == 1 From fb546c2f6200e523cc9ee2319b43aedba888837d Mon Sep 17 00:00:00 2001 From: Niklas Semmler Date: Tue, 17 Nov 2026 13:45:24 +0100 Subject: [PATCH 4/4] fix(retry): jitter X-RateLimit-Reset delays like the generated client Clients sharing a rate-limited key otherwise all retry at the reset instant. Retry-After stays exact. Co-Authored-By: Claude Opus 5.5 --- langfuse/_utils/retry.py | 17 ++++++++++++----- tests/unit/test_retry.py | 18 ++++++++++++++++-- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/langfuse/_utils/retry.py b/langfuse/_utils/retry.py index b4237fb29..d2f53a1ea 100644 --- a/langfuse/_utils/retry.py +++ b/langfuse/_utils/retry.py @@ -13,6 +13,7 @@ from langfuse.api.core.api_error import ApiError from langfuse.api.core.http_client import ( + _add_positive_jitter, _parse_retry_after, _parse_x_ratelimit_reset, ) @@ -54,11 +55,17 @@ def retry_delay_seconds( """Honor Retry-After / X-RateLimit-Reset, else exponential backoff with full jitter.""" headers = _response_headers(error) if headers is not None: - server_delay = _parse_retry_after(headers) - if server_delay is None or server_delay <= 0: - server_delay = _parse_x_ratelimit_reset(headers) - if server_delay is not None and server_delay > 0: - return min(server_delay, MAX_SERVER_RETRY_DELAY_SECONDS) + retry_after = _parse_retry_after(headers) + if retry_after is not None and retry_after > 0: + return min(retry_after, MAX_SERVER_RETRY_DELAY_SECONDS) + + # Like the generated client, jitter the reset delay so clients sharing a + # rate-limited key don't all retry at the same instant. + ratelimit_reset = _parse_x_ratelimit_reset(headers) + if ratelimit_reset is not None and ratelimit_reset > 0: + return _add_positive_jitter( + min(ratelimit_reset, MAX_SERVER_RETRY_DELAY_SECONDS) + ) return random.uniform(0, min(2.0**retries, max_backoff_seconds)) diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 04b77df24..efe98fe0f 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -9,6 +9,7 @@ call_with_retries, retry_delay_seconds, ) +from langfuse.api.core import http_client from langfuse.api.core.api_error import ApiError @@ -24,11 +25,24 @@ def test_retry_delay_caps_retry_after(): assert retry_delay_seconds(error, 0, 10) == MAX_SERVER_RETRY_DELAY_SECONDS -def test_retry_delay_uses_x_ratelimit_reset_without_retry_after(monkeypatch): +@pytest.mark.parametrize("draw, expected", [(0.0, 5.0), (1.0, 6.0)]) +def test_retry_delay_jitters_x_ratelimit_reset_without_retry_after( + monkeypatch, draw, expected +): + # Clients sharing a rate-limited key must not all retry at the reset instant, + # so the reset delay gets the generated client's 0-20% positive jitter. monkeypatch.setattr(time, "time", lambda: 1000.0) + monkeypatch.setattr(http_client, "random", lambda: draw) error = ApiError(status_code=429, headers={"x-ratelimit-reset": "1005"}) - assert retry_delay_seconds(error, 0, 10) == 5 + assert retry_delay_seconds(error, 0, 10) == pytest.approx(expected) + + +def test_retry_delay_does_not_jitter_retry_after(monkeypatch): + monkeypatch.setattr(http_client, "random", lambda: 1.0) + error = ApiError(status_code=429, headers={"retry-after": "3"}) + + assert retry_delay_seconds(error, 0, 10) == 3 def test_retry_delay_reads_headers_from_http_status_errors():