diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index 5d128567c..fd4928316 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -30,7 +30,6 @@ overload, ) -import backoff import httpx from opentelemetry import context as otel_context_api from opentelemetry import trace as otel_trace_api @@ -98,6 +97,7 @@ from langfuse._utils.environment import get_common_release_envs from langfuse._utils.parse_error import handle_fern_exception from langfuse._utils.prompt_cache import PromptCache +from langfuse._utils.retry import call_with_retries from langfuse.api import ( AsyncLangfuseAPI, CreateChatPromptRequest, @@ -3951,7 +3951,7 @@ def get_prompt( keyword argument. If not set, defaults to 60 seconds. Disables caching if set to 0. type: Literal["chat", "text"]: The type of the prompt to retrieve. Defaults to "text". fallback: Union[Optional[List[ChatMessageDict]], Optional[str]]: The prompt string to return if fetching the prompt fails. Important on the first call where no cached prompt is available. Follows Langfuse prompt formatting with double curly braces for variables. Defaults to None. - max_retries: Optional[int]: The maximum number of retries in case of API/network errors. Defaults to 2. The maximum value is 4. Retries have an exponential backoff with a maximum delay of 10 seconds. + max_retries: Optional[int]: The maximum number of retries in case of network errors or retryable API responses (408, 409, 429, 5xx). Other 4xx errors such as 401, 403, and 404 are not retried. Defaults to 2. The maximum value is 4. Retries use exponential backoff; rate-limited responses wait for the server's Retry-After header (up to 60 seconds). fetch_timeout_seconds: Optional[int]: The timeout in seconds for fetching the prompt. Defaults to the default timeout set on the SDK, which is 5 seconds per default. Returns: @@ -4081,23 +4081,21 @@ def _fetch_prompt_and_update_cache( langfuse_logger.debug("Fetching prompt '%s' from server...", cache_key) try: - - @backoff.on_exception( - backoff.constant, Exception, max_tries=max_retries + 1, logger=None - ) - def fetch_prompts() -> Any: - return self.api.prompts.get( + # Retries happen only in call_with_retries so network errors and + # 408/409/429/5xx responses share one budget. + request_options: RequestOptions = {"max_retries": 0} + if fetch_timeout_seconds is not None: + request_options["timeout_in_seconds"] = fetch_timeout_seconds + + prompt_response = call_with_retries( + lambda: self.api.prompts.get( self._url_encode(name), version=version, label=label, - request_options={ - "timeout_in_seconds": fetch_timeout_seconds, - } - if fetch_timeout_seconds is not None - else None, - ) - - prompt_response = fetch_prompts() + request_options=request_options, + ), + max_retries=max_retries, + ) prompt: PromptClient if prompt_response.type == "chat": diff --git a/langfuse/_task_manager/media_manager.py b/langfuse/_task_manager/media_manager.py index 3ad9ac62f..bc72776a7 100644 --- a/langfuse/_task_manager/media_manager.py +++ b/langfuse/_task_manager/media_manager.py @@ -3,14 +3,14 @@ from queue import Empty, Full, Queue from typing import Any, Callable, Optional, TypeVar, cast -import backoff import httpx from typing_extensions import ParamSpec from langfuse._client.environment_variables import LANGFUSE_MEDIA_UPLOAD_ENABLED from langfuse._utils import _get_timestamp +from langfuse._utils.retry import call_with_retries, is_retryable_status from langfuse.api import LangfuseAPI, MediaContentType -from langfuse.api.core import ApiError +from langfuse.api.core import ApiError, RequestOptions from langfuse.logger import langfuse_logger as logger from langfuse.media import LangfuseMedia @@ -18,6 +18,9 @@ T = TypeVar("T") P = ParamSpec("P") + +# Retries happen in _request_with_backoff, so the API client must not retry too. +_NO_CLIENT_RETRIES: RequestOptions = {"max_retries": 0} _SHUTDOWN_SENTINEL = object() @@ -41,7 +44,7 @@ def __init__( self._api_client = api_client self._httpx_client = httpx_client self._queue = media_upload_queue - self._max_retries = max_retries + self._max_retries = max_retries if max_retries is not None else 3 self._enabled = os.environ.get( LANGFUSE_MEDIA_UPLOAD_ENABLED, "True" ).lower() not in ("false", "0") @@ -426,6 +429,7 @@ def _process_upload_media_job( dataset_id=data["dataset_id"], dataset_item_id=data["dataset_item_id"], field=data["field"], + request_options=_NO_CLIENT_RETRIES, ) upload_url = upload_url_response.upload_url @@ -484,6 +488,7 @@ def _upload_with_status_check() -> httpx.Response: upload_http_status=failed_response.status_code, upload_http_error=failed_response.text, upload_time_ms=upload_time_ms, + request_options=_NO_CLIENT_RETRIES, ) raise @@ -497,6 +502,7 @@ def _upload_with_status_check() -> httpx.Response: upload_http_status=upload_response.status_code, upload_http_error=upload_response.text, upload_time_ms=upload_time_ms, + request_options=_NO_CLIENT_RETRIES, ) logger.debug( @@ -512,29 +518,18 @@ def _upload_with_status_check() -> httpx.Response: def _request_with_backoff( self, func: Callable[P, T], *args: P.args, **kwargs: P.kwargs ) -> T: - def _should_give_up(e: Exception) -> bool: + def _should_retry(e: Exception) -> bool: if isinstance(e, ApiError): - return ( - e.status_code is not None - and 400 <= e.status_code < 500 - and e.status_code != 429 - ) + return e.status_code is not None and is_retryable_status(e.status_code) if isinstance(e, httpx.HTTPStatusError): - return ( - e.response is not None - and e.response.status_code < 500 - and e.response.status_code != 429 + return e.response is not None and is_retryable_status( + e.response.status_code ) - return False - - @backoff.on_exception( - backoff.expo, - Exception, - max_tries=self._max_retries, - giveup=_should_give_up, - logger=None, - ) - def execute_task_with_backoff() -> T: - return func(*args, **kwargs) + return True - return execute_task_with_backoff() + # API client calls pass max_retries=0, so this is the only retry layer. + return call_with_retries( + lambda: func(*args, **kwargs), + max_retries=max(self._max_retries - 1, 0), + should_retry=_should_retry, + ) diff --git a/langfuse/_utils/retry.py b/langfuse/_utils/retry.py new file mode 100644 index 000000000..d2f53a1ea --- /dev/null +++ b/langfuse/_utils/retry.py @@ -0,0 +1,90 @@ +"""Retries for calls made through the generated API client. + +Callers pass ``request_options={"max_retries": 0}`` to the API client and wrap +the call in :func:`call_with_retries`, so network errors and retryable HTTP +responses share a single retry budget instead of multiplying each other. +""" + +import random +import time +from typing import Callable, Optional, TypeVar + +import httpx + +from langfuse.api.core.api_error import ApiError +from langfuse.api.core.http_client import ( + _add_positive_jitter, + _parse_retry_after, + _parse_x_ratelimit_reset, +) + +T = TypeVar("T") + +RETRYABLE_STATUS_CODES = frozenset({408, 409, 429}) + +# Matches the cap the generated API client applies to server-provided delays. +MAX_SERVER_RETRY_DELAY_SECONDS = 60.0 + + +def is_retryable_status(status_code: int) -> bool: + return status_code >= 500 or status_code in RETRYABLE_STATUS_CODES + + +def is_retryable_api_call_error(error: Exception) -> bool: + """Return True for network errors and 408/409/429/5xx API errors.""" + if isinstance(error, httpx.TransportError): + return True + if isinstance(error, ApiError) and error.status_code is not None: + return is_retryable_status(error.status_code) + + return False + + +def _response_headers(error: Exception) -> Optional[httpx.Headers]: + if isinstance(error, ApiError) and error.headers: + return httpx.Headers(error.headers) + if isinstance(error, httpx.HTTPStatusError) and error.response is not None: + return error.response.headers + + return None + + +def retry_delay_seconds( + error: Exception, retries: int, max_backoff_seconds: float +) -> float: + """Honor Retry-After / X-RateLimit-Reset, else exponential backoff with full jitter.""" + headers = _response_headers(error) + if headers is not None: + retry_after = _parse_retry_after(headers) + if retry_after is not None and retry_after > 0: + return min(retry_after, MAX_SERVER_RETRY_DELAY_SECONDS) + + # Like the generated client, jitter the reset delay so clients sharing a + # rate-limited key don't all retry at the same instant. + ratelimit_reset = _parse_x_ratelimit_reset(headers) + if ratelimit_reset is not None and ratelimit_reset > 0: + return _add_positive_jitter( + min(ratelimit_reset, MAX_SERVER_RETRY_DELAY_SECONDS) + ) + + return random.uniform(0, min(2.0**retries, max_backoff_seconds)) + + +def call_with_retries( + func: Callable[[], T], + *, + max_retries: int, + should_retry: Callable[[Exception], bool] = is_retryable_api_call_error, + max_backoff_seconds: float = 10.0, +) -> T: + """Call ``func``, retrying up to ``max_retries`` times on retryable errors.""" + retries = 0 + while True: + try: + return func() + except Exception as e: + if retries >= max_retries or not should_retry(e): + raise + + time.sleep(retry_delay_seconds(e, retries, max_backoff_seconds)) + retries += 1 diff --git a/tests/unit/test_media_manager.py b/tests/unit/test_media_manager.py index 3ab4e3226..2ac4fd7b3 100644 --- a/tests/unit/test_media_manager.py +++ b/tests/unit/test_media_manager.py @@ -1,3 +1,4 @@ +import time from queue import Queue from types import SimpleNamespace from unittest.mock import Mock @@ -6,6 +7,8 @@ import pytest from langfuse._task_manager.media_manager import MediaManager +from langfuse.api import LangfuseAPI +from langfuse.api.core.api_error import ApiError from langfuse.media import LangfuseMedia @@ -277,3 +280,62 @@ def test_find_and_process_media_gemini_inline_data_non_string_data_passes_throug assert result == data assert queue.empty() + + +@pytest.mark.parametrize("status_code", [429, 503]) +def test_media_api_calls_do_not_multiply_client_retries(monkeypatch, status_code): + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(status_code, json={}) + + api_client = LangfuseAPI( + base_url="http://test-host", + username="pk", + password="sk", + httpx_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) + manager = MediaManager( + api_client=api_client, + httpx_client=Mock(), + media_upload_queue=Queue(), + max_retries=3, + ) + + with pytest.raises(ApiError): + manager._process_upload_media_job(data=_upload_job()) + + # max_retries=3 means 3 attempts in total, with no nested client retries + assert len(requests) == 3 + + +def test_media_api_calls_share_one_retry_budget_across_failure_kinds(monkeypatch): + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if len(requests) % 2 == 0: + raise httpx.ConnectError("connection reset", request=request) + + return httpx.Response(429, json={}) + + api_client = LangfuseAPI( + base_url="http://test-host", + username="pk", + password="sk", + httpx_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) + manager = MediaManager( + api_client=api_client, + httpx_client=Mock(), + media_upload_queue=Queue(), + max_retries=3, + ) + + with pytest.raises(ApiError): + manager._process_upload_media_job(data=_upload_job()) + + assert len(requests) == 3 diff --git a/tests/unit/test_prompt.py b/tests/unit/test_prompt.py index eadfb8221..9afec0507 100644 --- a/tests/unit/test_prompt.py +++ b/tests/unit/test_prompt.py @@ -1,5 +1,8 @@ +import random +import time from unittest.mock import Mock, patch +import httpx import pytest from langfuse._client.client import Langfuse @@ -9,7 +12,8 @@ PromptCacheItem, PromptCacheTaskManager, ) -from langfuse.api import NotFoundError, Prompt_Chat, Prompt_Text +from langfuse.api import LangfuseAPI, NotFoundError, Prompt_Chat, Prompt_Text +from langfuse.api.core.api_error import ApiError from langfuse.model import ChatPromptClient, TextPromptClient @@ -139,6 +143,14 @@ def langfuse(): return langfuse_instance +@pytest.fixture +def sleeps(monkeypatch: pytest.MonkeyPatch) -> list: + recorded: list = [] + monkeypatch.setattr(time, "sleep", recorded.append) + + return recorded + + def wait_for_prompt_refresh(langfuse: Langfuse) -> None: langfuse._resources.prompt_cache._task_manager.wait_for_idle() @@ -199,7 +211,7 @@ def test_get_fresh_prompt(langfuse): prompt_name, version=None, label=None, - request_options=None, + request_options={"max_retries": 0}, ) assert result == TextPromptClient(prompt) @@ -244,7 +256,7 @@ def test_using_custom_prompt_timeouts(langfuse): prompt_name, version=None, label=None, - request_options={"timeout_in_seconds": 1000}, + request_options={"max_retries": 0, "timeout_in_seconds": 1000}, ) assert result == TextPromptClient(prompt) @@ -620,7 +632,7 @@ def test_get_fresh_prompt_when_expired_cache_default_ttl(mock_time, langfuse: La @patch.object(PromptCacheItem, "get_epoch_seconds") -def test_get_expired_prompt_when_failing_fetch(mock_time, langfuse: Langfuse): +def test_get_expired_prompt_when_failing_fetch(mock_time, langfuse: Langfuse, sleeps): mock_time.return_value = 0 prompt_name = "test_get_expired_prompt_when_failing_fetch" @@ -643,7 +655,7 @@ def test_get_expired_prompt_when_failing_fetch(mock_time, langfuse: Langfuse): assert result_call_1 == prompt_client mock_time.return_value = DEFAULT_PROMPT_CACHE_TTL_SECONDS + 1 - mock_server_call.side_effect = Exception("Server error") + mock_server_call.side_effect = httpx.ConnectError("Server error") result_call_2 = langfuse.get_prompt(prompt_name, max_retries=1) wait_for_prompt_refresh(langfuse) @@ -748,3 +760,123 @@ def test_get_fresh_prompt_when_version_changes(langfuse: Langfuse): result_call_2 = langfuse.get_prompt(prompt_name, version=2) assert mock_server_call.call_count == 2 assert result_call_2 == version_changed_prompt_client + + +def _api_over_transport(handler) -> LangfuseAPI: + return LangfuseAPI( + base_url="http://test-host", + username="pk", + password="sk", + httpx_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) + + +def _prompt_json(name: str) -> dict: + return { + "name": name, + "version": 1, + "prompt": "Make me laugh", + "type": "text", + "labels": [], + "config": {}, + "tags": [], + } + + +def test_get_prompt_does_not_multiply_retries_on_rate_limit(langfuse, sleeps): + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(429, headers={"Retry-After": "3"}, json={}) + + langfuse.api = _api_over_transport(handler) + + with pytest.raises(ApiError) as exc_info: + langfuse.get_prompt("rate_limited_prompt", max_retries=2) + + assert exc_info.value.status_code == 429 + assert len(requests) == 3 + assert sleeps == [3, 3] + + +@pytest.mark.parametrize("status_code", [401, 403, 404]) +def test_get_prompt_does_not_retry_non_retryable_client_errors( + langfuse, sleeps, status_code +): + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(status_code, json={"message": "nope"}) + + langfuse.api = _api_over_transport(handler) + + with pytest.raises(ApiError): + langfuse.get_prompt("client_error_prompt", max_retries=2) + + assert len(requests) == 1 + assert sleeps == [] + + +def test_get_prompt_retries_network_errors_with_exponential_backoff( + langfuse, sleeps, monkeypatch: pytest.MonkeyPatch +): + # pin full jitter to its upper bound so the backoff sequence is observable + monkeypatch.setattr(random, "uniform", lambda _low, high: high) + prompt_name = "flaky_network_prompt" + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if len(requests) < 3: + raise httpx.ConnectError("connection refused", request=request) + + return httpx.Response(200, json=_prompt_json(prompt_name)) + + langfuse.api = _api_over_transport(handler) + + result = langfuse.get_prompt(prompt_name, max_retries=2) + + assert result.prompt == "Make me laugh" + assert len(requests) == 3 + assert sleeps == [1, 2] + + +def test_get_prompt_shares_one_retry_budget_across_failure_kinds(langfuse, sleeps): + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if len(requests) % 2 == 0: + raise httpx.ConnectError("connection reset", request=request) + + return httpx.Response(503, json={}) + + langfuse.api = _api_over_transport(handler) + + with pytest.raises(httpx.ConnectError): + langfuse.get_prompt("degraded_server_prompt", max_retries=3) + + assert len(requests) == 4 + assert len(sleeps) == 3 + + +@pytest.mark.parametrize("failure", ["rate_limited", "network_error"]) +def test_get_prompt_with_zero_retries_sends_one_request(langfuse, sleeps, failure): + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if failure == "network_error": + raise httpx.ConnectError("connection refused", request=request) + + return httpx.Response(429, headers={"Retry-After": "1"}, json={}) + + langfuse.api = _api_over_transport(handler) + + with pytest.raises((ApiError, httpx.ConnectError)): + langfuse.get_prompt("no_retry_prompt", max_retries=0) + + assert len(requests) == 1 + assert sleeps == [] diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py new file mode 100644 index 000000000..efe98fe0f --- /dev/null +++ b/tests/unit/test_retry.py @@ -0,0 +1,81 @@ +import random +import time + +import httpx +import pytest + +from langfuse._utils.retry import ( + MAX_SERVER_RETRY_DELAY_SECONDS, + call_with_retries, + retry_delay_seconds, +) +from langfuse.api.core import http_client +from langfuse.api.core.api_error import ApiError + + +def _status_error(status_code: int, headers: dict) -> httpx.HTTPStatusError: + request = httpx.Request("PUT", "https://example.com/upload") + response = httpx.Response(status_code, headers=headers, request=request) + return httpx.HTTPStatusError("failed", request=request, response=response) + + +def test_retry_delay_caps_retry_after(): + error = ApiError(status_code=429, headers={"retry-after": "3600"}) + + assert retry_delay_seconds(error, 0, 10) == MAX_SERVER_RETRY_DELAY_SECONDS + + +@pytest.mark.parametrize("draw, expected", [(0.0, 5.0), (1.0, 6.0)]) +def test_retry_delay_jitters_x_ratelimit_reset_without_retry_after( + monkeypatch, draw, expected +): + # Clients sharing a rate-limited key must not all retry at the reset instant, + # so the reset delay gets the generated client's 0-20% positive jitter. + monkeypatch.setattr(time, "time", lambda: 1000.0) + monkeypatch.setattr(http_client, "random", lambda: draw) + error = ApiError(status_code=429, headers={"x-ratelimit-reset": "1005"}) + + assert retry_delay_seconds(error, 0, 10) == pytest.approx(expected) + + +def test_retry_delay_does_not_jitter_retry_after(monkeypatch): + monkeypatch.setattr(http_client, "random", lambda: 1.0) + error = ApiError(status_code=429, headers={"retry-after": "3"}) + + assert retry_delay_seconds(error, 0, 10) == 3 + + +def test_retry_delay_reads_headers_from_http_status_errors(): + error = _status_error(503, {"Retry-After": "7"}) + + assert retry_delay_seconds(error, 0, 10) == 7 + + +def test_retry_delay_falls_back_to_capped_exponential_backoff(monkeypatch): + monkeypatch.setattr(random, "uniform", lambda _low, high: high) + error = ApiError(status_code=503, headers={}) + + assert [retry_delay_seconds(error, retries, 10) for retries in range(5)] == [ + 1, + 2, + 4, + 8, + 10, + ] + + +@pytest.mark.parametrize("status_code", [400, 401, 403, 404, 422]) +def test_call_with_retries_does_not_retry_non_retryable_api_errors( + monkeypatch, status_code +): + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + calls = [] + + def fail() -> None: + calls.append(1) + raise ApiError(status_code=status_code) + + with pytest.raises(ApiError): + call_with_retries(fail, max_retries=3) + + assert len(calls) == 1