Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 14 additions & 16 deletions langfuse/_client/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
overload,
)

import backoff
import httpx
from opentelemetry import context as otel_context_api
from opentelemetry import trace as otel_trace_api
Expand Down Expand Up @@ -98,6 +97,7 @@
from langfuse._utils.environment import get_common_release_envs
from langfuse._utils.parse_error import handle_fern_exception
from langfuse._utils.prompt_cache import PromptCache
from langfuse._utils.retry import call_with_retries
from langfuse.api import (
AsyncLangfuseAPI,
CreateChatPromptRequest,
Expand Down Expand Up @@ -3951,7 +3951,7 @@ def get_prompt(
keyword argument. If not set, defaults to 60 seconds. Disables caching if set to 0.
type: Literal["chat", "text"]: The type of the prompt to retrieve. Defaults to "text".
fallback: Union[Optional[List[ChatMessageDict]], Optional[str]]: The prompt string to return if fetching the prompt fails. Important on the first call where no cached prompt is available. Follows Langfuse prompt formatting with double curly braces for variables. Defaults to None.
max_retries: Optional[int]: The maximum number of retries in case of API/network errors. Defaults to 2. The maximum value is 4. Retries have an exponential backoff with a maximum delay of 10 seconds.
max_retries: Optional[int]: The maximum number of retries in case of network errors or retryable API responses (408, 409, 429, 5xx). Other 4xx errors such as 401, 403, and 404 are not retried. Defaults to 2. The maximum value is 4. Retries use exponential backoff; rate-limited responses wait for the server's Retry-After header (up to 60 seconds).
fetch_timeout_seconds: Optional[int]: The timeout in seconds for fetching the prompt. Defaults to the default timeout set on the SDK, which is 5 seconds per default.

Returns:
Expand Down Expand Up @@ -4081,23 +4081,21 @@ def _fetch_prompt_and_update_cache(
langfuse_logger.debug("Fetching prompt '%s' from server...", cache_key)

try:

@backoff.on_exception(
backoff.constant, Exception, max_tries=max_retries + 1, logger=None
)
def fetch_prompts() -> Any:
return self.api.prompts.get(
# Retries happen only in call_with_retries so network errors and
# 408/409/429/5xx responses share one budget.
request_options: RequestOptions = {"max_retries": 0}
if fetch_timeout_seconds is not None:
request_options["timeout_in_seconds"] = fetch_timeout_seconds

prompt_response = call_with_retries(
lambda: self.api.prompts.get(
self._url_encode(name),
version=version,
label=label,
request_options={
"timeout_in_seconds": fetch_timeout_seconds,
}
if fetch_timeout_seconds is not None
else None,
)

prompt_response = fetch_prompts()
request_options=request_options,
),
max_retries=max_retries,
)

prompt: PromptClient
if prompt_response.type == "chat":
Expand Down
45 changes: 20 additions & 25 deletions langfuse/_task_manager/media_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,24 @@
from queue import Empty, Full, Queue
from typing import Any, Callable, Optional, TypeVar, cast

import backoff
import httpx
from typing_extensions import ParamSpec

from langfuse._client.environment_variables import LANGFUSE_MEDIA_UPLOAD_ENABLED
from langfuse._utils import _get_timestamp
from langfuse._utils.retry import call_with_retries, is_retryable_status
from langfuse.api import LangfuseAPI, MediaContentType
from langfuse.api.core import ApiError
from langfuse.api.core import ApiError, RequestOptions
from langfuse.logger import langfuse_logger as logger
from langfuse.media import LangfuseMedia

from .media_upload_queue import UploadMediaJob

T = TypeVar("T")
P = ParamSpec("P")

# Retries happen in _request_with_backoff, so the API client must not retry too.
_NO_CLIENT_RETRIES: RequestOptions = {"max_retries": 0}
_SHUTDOWN_SENTINEL = object()


Expand All @@ -41,7 +44,7 @@ def __init__(
self._api_client = api_client
self._httpx_client = httpx_client
self._queue = media_upload_queue
self._max_retries = max_retries
self._max_retries = max_retries if max_retries is not None else 3
self._enabled = os.environ.get(
LANGFUSE_MEDIA_UPLOAD_ENABLED, "True"
).lower() not in ("false", "0")
Expand Down Expand Up @@ -426,6 +429,7 @@ def _process_upload_media_job(
dataset_id=data["dataset_id"],
dataset_item_id=data["dataset_item_id"],
field=data["field"],
request_options=_NO_CLIENT_RETRIES,
)

upload_url = upload_url_response.upload_url
Expand Down Expand Up @@ -484,6 +488,7 @@ def _upload_with_status_check() -> httpx.Response:
upload_http_status=failed_response.status_code,
upload_http_error=failed_response.text,
upload_time_ms=upload_time_ms,
request_options=_NO_CLIENT_RETRIES,
)

raise
Expand All @@ -497,6 +502,7 @@ def _upload_with_status_check() -> httpx.Response:
upload_http_status=upload_response.status_code,
upload_http_error=upload_response.text,
upload_time_ms=upload_time_ms,
request_options=_NO_CLIENT_RETRIES,
)

logger.debug(
Expand All @@ -512,29 +518,18 @@ def _upload_with_status_check() -> httpx.Response:
def _request_with_backoff(
self, func: Callable[P, T], *args: P.args, **kwargs: P.kwargs
) -> T:
def _should_give_up(e: Exception) -> bool:
def _should_retry(e: Exception) -> bool:
if isinstance(e, ApiError):
return (
e.status_code is not None
and 400 <= e.status_code < 500
and e.status_code != 429
)
return e.status_code is not None and is_retryable_status(e.status_code)
if isinstance(e, httpx.HTTPStatusError):
return (
e.response is not None
and e.response.status_code < 500
and e.response.status_code != 429
return e.response is not None and is_retryable_status(
e.response.status_code
)
return False

@backoff.on_exception(
backoff.expo,
Exception,
max_tries=self._max_retries,
giveup=_should_give_up,
logger=None,
)
def execute_task_with_backoff() -> T:
return func(*args, **kwargs)
return True

return execute_task_with_backoff()
# API client calls pass max_retries=0, so this is the only retry layer.
return call_with_retries(
lambda: func(*args, **kwargs),
max_retries=max(self._max_retries - 1, 0),
should_retry=_should_retry,
)
90 changes: 90 additions & 0 deletions langfuse/_utils/retry.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
"""Retries for calls made through the generated API client.

Callers pass ``request_options={"max_retries": 0}`` to the API client and wrap
the call in :func:`call_with_retries`, so network errors and retryable HTTP
responses share a single retry budget instead of multiplying each other.
"""

import random
import time
from typing import Callable, Optional, TypeVar

import httpx

from langfuse.api.core.api_error import ApiError
from langfuse.api.core.http_client import (
_add_positive_jitter,
_parse_retry_after,
_parse_x_ratelimit_reset,
)

T = TypeVar("T")

RETRYABLE_STATUS_CODES = frozenset({408, 409, 429})

# Matches the cap the generated API client applies to server-provided delays.
MAX_SERVER_RETRY_DELAY_SECONDS = 60.0


def is_retryable_status(status_code: int) -> bool:
return status_code >= 500 or status_code in RETRYABLE_STATUS_CODES


def is_retryable_api_call_error(error: Exception) -> bool:
"""Return True for network errors and 408/409/429/5xx API errors."""
if isinstance(error, httpx.TransportError):
return True
if isinstance(error, ApiError) and error.status_code is not None:
return is_retryable_status(error.status_code)

return False


def _response_headers(error: Exception) -> Optional[httpx.Headers]:
if isinstance(error, ApiError) and error.headers:
return httpx.Headers(error.headers)
if isinstance(error, httpx.HTTPStatusError) and error.response is not None:
return error.response.headers

return None


def retry_delay_seconds(
error: Exception, retries: int, max_backoff_seconds: float
) -> float:
"""Honor Retry-After / X-RateLimit-Reset, else exponential backoff with full jitter."""
headers = _response_headers(error)
if headers is not None:
retry_after = _parse_retry_after(headers)
if retry_after is not None and retry_after > 0:
return min(retry_after, MAX_SERVER_RETRY_DELAY_SECONDS)

# Like the generated client, jitter the reset delay so clients sharing a
# rate-limited key don't all retry at the same instant.
ratelimit_reset = _parse_x_ratelimit_reset(headers)
if ratelimit_reset is not None and ratelimit_reset > 0:
return _add_positive_jitter(
min(ratelimit_reset, MAX_SERVER_RETRY_DELAY_SECONDS)
)

return random.uniform(0, min(2.0**retries, max_backoff_seconds))


def call_with_retries(
func: Callable[[], T],
*,
max_retries: int,
should_retry: Callable[[Exception], bool] = is_retryable_api_call_error,
max_backoff_seconds: float = 10.0,
) -> T:
"""Call ``func``, retrying up to ``max_retries`` times on retryable errors."""
retries = 0
while True:
try:
return func()
except Exception as e:
if retries >= max_retries or not should_retry(e):
raise

time.sleep(retry_delay_seconds(e, retries, max_backoff_seconds))
retries += 1
62 changes: 62 additions & 0 deletions tests/unit/test_media_manager.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import time
from queue import Queue
from types import SimpleNamespace
from unittest.mock import Mock
Expand All @@ -6,6 +7,8 @@
import pytest

from langfuse._task_manager.media_manager import MediaManager
from langfuse.api import LangfuseAPI
from langfuse.api.core.api_error import ApiError
from langfuse.media import LangfuseMedia


Expand Down Expand Up @@ -277,3 +280,62 @@ def test_find_and_process_media_gemini_inline_data_non_string_data_passes_throug

assert result == data
assert queue.empty()


@pytest.mark.parametrize("status_code", [429, 503])
def test_media_api_calls_do_not_multiply_client_retries(monkeypatch, status_code):
monkeypatch.setattr(time, "sleep", lambda _seconds: None)
requests = []

def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(status_code, json={})

api_client = LangfuseAPI(
base_url="http://test-host",
username="pk",
password="sk",
httpx_client=httpx.Client(transport=httpx.MockTransport(handler)),
)
manager = MediaManager(
api_client=api_client,
httpx_client=Mock(),
media_upload_queue=Queue(),
max_retries=3,
)

with pytest.raises(ApiError):
manager._process_upload_media_job(data=_upload_job())

# max_retries=3 means 3 attempts in total, with no nested client retries
assert len(requests) == 3


def test_media_api_calls_share_one_retry_budget_across_failure_kinds(monkeypatch):
monkeypatch.setattr(time, "sleep", lambda _seconds: None)
requests = []

def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
if len(requests) % 2 == 0:
raise httpx.ConnectError("connection reset", request=request)

return httpx.Response(429, json={})

api_client = LangfuseAPI(
base_url="http://test-host",
username="pk",
password="sk",
httpx_client=httpx.Client(transport=httpx.MockTransport(handler)),
)
manager = MediaManager(
api_client=api_client,
httpx_client=Mock(),
media_upload_queue=Queue(),
max_retries=3,
)

with pytest.raises(ApiError):
manager._process_upload_media_job(data=_upload_job())

assert len(requests) == 3
Loading
Loading