Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 15 additions & 16 deletions langfuse/_client/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
overload,
)

import backoff
import httpx
from opentelemetry import context as otel_context_api
from opentelemetry import trace as otel_trace_api
Expand Down Expand Up @@ -93,6 +92,7 @@
from langfuse._utils.environment import get_common_release_envs
from langfuse._utils.parse_error import handle_fern_exception
from langfuse._utils.prompt_cache import PromptCache
from langfuse._utils.retry import call_with_retries
from langfuse.api import (
AsyncLangfuseAPI,
CreateChatPromptRequest,
Expand All @@ -111,6 +111,7 @@
ScoreBody,
TraceBody,
)
from langfuse.api.core import RequestOptions
from langfuse.batch_evaluation import (
BatchEvaluationResult,
BatchEvaluationResumeToken,
Expand Down Expand Up @@ -3788,7 +3789,7 @@ def get_prompt(
keyword argument. If not set, defaults to 60 seconds. Disables caching if set to 0.
type: Literal["chat", "text"]: The type of the prompt to retrieve. Defaults to "text".
fallback: Union[Optional[List[ChatMessageDict]], Optional[str]]: The prompt string to return if fetching the prompt fails. Important on the first call where no cached prompt is available. Follows Langfuse prompt formatting with double curly braces for variables. Defaults to None.
max_retries: Optional[int]: The maximum number of retries in case of API/network errors. Defaults to 2. The maximum value is 4. Retries have an exponential backoff with a maximum delay of 10 seconds.
max_retries: Optional[int]: The maximum number of retries in case of network errors or retryable API responses (408, 409, 429, 5xx). Other 4xx errors such as 401, 403, and 404 are not retried. Defaults to 2. The maximum value is 4. Retries use exponential backoff; rate-limited responses wait for the server's Retry-After header (up to 60 seconds).
fetch_timeout_seconds: Optional[int]: The timeout in milliseconds for fetching the prompt. Defaults to the default timeout set on the SDK, which is 5 seconds per default.

Returns:
Expand Down Expand Up @@ -3918,23 +3919,21 @@ def _fetch_prompt_and_update_cache(
langfuse_logger.debug("Fetching prompt '%s' from server...", cache_key)

try:

@backoff.on_exception(
backoff.constant, Exception, max_tries=max_retries + 1, logger=None
)
def fetch_prompts() -> Any:
return self.api.prompts.get(
# Retries happen only in call_with_retries so network errors and
# 408/409/429/5xx responses share one budget.
request_options: RequestOptions = {"max_retries": 0}
if fetch_timeout_seconds is not None:
request_options["timeout_in_seconds"] = fetch_timeout_seconds

prompt_response = call_with_retries(
lambda: self.api.prompts.get(
self._url_encode(name),
version=version,
label=label,
request_options={
"timeout_in_seconds": fetch_timeout_seconds,
}
if fetch_timeout_seconds is not None
else None,
)

prompt_response = fetch_prompts()
request_options=request_options,
),
max_retries=max_retries,
)

prompt: PromptClient
if prompt_response.type == "chat":
Expand Down
45 changes: 20 additions & 25 deletions langfuse/_task_manager/media_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,24 @@
from queue import Empty, Full, Queue
from typing import Any, Callable, Optional, TypeVar, cast

import backoff
import httpx
from typing_extensions import ParamSpec

from langfuse._client.environment_variables import LANGFUSE_MEDIA_UPLOAD_ENABLED
from langfuse._utils import _get_timestamp
from langfuse._utils.retry import call_with_retries, is_retryable_status
from langfuse.api import LangfuseAPI, MediaContentType
from langfuse.api.core import ApiError
from langfuse.api.core import ApiError, RequestOptions
from langfuse.logger import langfuse_logger as logger
from langfuse.media import LangfuseMedia

from .media_upload_queue import UploadMediaJob

T = TypeVar("T")
P = ParamSpec("P")

# Retries happen in _request_with_backoff, so the API client must not retry too.
_NO_CLIENT_RETRIES: RequestOptions = {"max_retries": 0}
_SHUTDOWN_SENTINEL = object()


Expand All @@ -41,7 +44,7 @@ def __init__(
self._api_client = api_client
self._httpx_client = httpx_client
self._queue = media_upload_queue
self._max_retries = max_retries
self._max_retries = max_retries if max_retries is not None else 3
self._enabled = os.environ.get(
LANGFUSE_MEDIA_UPLOAD_ENABLED, "True"
).lower() not in ("false", "0")
Expand Down Expand Up @@ -422,6 +425,7 @@ def _process_upload_media_job(
dataset_id=data["dataset_id"],
dataset_item_id=data["dataset_item_id"],
field=data["field"],
request_options=_NO_CLIENT_RETRIES,
)

upload_url = upload_url_response.upload_url
Expand Down Expand Up @@ -480,6 +484,7 @@ def _upload_with_status_check() -> httpx.Response:
upload_http_status=failed_response.status_code,
upload_http_error=failed_response.text,
upload_time_ms=upload_time_ms,
request_options=_NO_CLIENT_RETRIES,
)

raise
Expand All @@ -493,6 +498,7 @@ def _upload_with_status_check() -> httpx.Response:
upload_http_status=upload_response.status_code,
upload_http_error=upload_response.text,
upload_time_ms=upload_time_ms,
request_options=_NO_CLIENT_RETRIES,
)

logger.debug(
Expand All @@ -508,29 +514,18 @@ def _upload_with_status_check() -> httpx.Response:
def _request_with_backoff(
self, func: Callable[P, T], *args: P.args, **kwargs: P.kwargs
) -> T:
def _should_give_up(e: Exception) -> bool:
def _should_retry(e: Exception) -> bool:
if isinstance(e, ApiError):
return (
e.status_code is not None
and 400 <= e.status_code < 500
and e.status_code != 429
)
return e.status_code is not None and is_retryable_status(e.status_code)
if isinstance(e, httpx.HTTPStatusError):
return (
e.response is not None
and e.response.status_code < 500
and e.response.status_code != 429
return e.response is not None and is_retryable_status(
e.response.status_code
)
return False

@backoff.on_exception(
backoff.expo,
Exception,
max_tries=self._max_retries,
giveup=_should_give_up,
logger=None,
)
def execute_task_with_backoff() -> T:
return func(*args, **kwargs)
return True

return execute_task_with_backoff()
# API client calls pass max_retries=0, so this is the only retry layer.
return call_with_retries(
lambda: func(*args, **kwargs),
max_retries=max(self._max_retries - 1, 0),
should_retry=_should_retry,
)
83 changes: 83 additions & 0 deletions langfuse/_utils/retry.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
"""Retries for calls made through the generated API client.

Callers pass ``request_options={"max_retries": 0}`` to the API client and wrap
the call in :func:`call_with_retries`, so network errors and retryable HTTP
responses share a single retry budget instead of multiplying each other.
"""

import random
import time
from typing import Callable, Optional, TypeVar

import httpx

from langfuse.api.core.api_error import ApiError
from langfuse.api.core.http_client import (
_parse_retry_after,
_parse_x_ratelimit_reset,
)

T = TypeVar("T")

RETRYABLE_STATUS_CODES = frozenset({408, 409, 429})

# Matches the cap the generated API client applies to server-provided delays.
MAX_SERVER_RETRY_DELAY_SECONDS = 60.0


def is_retryable_status(status_code: int) -> bool:
return status_code >= 500 or status_code in RETRYABLE_STATUS_CODES


def is_retryable_api_call_error(error: Exception) -> bool:
"""Return True for network errors and 408/409/429/5xx API errors."""
if isinstance(error, httpx.TransportError):
return True
if isinstance(error, ApiError) and error.status_code is not None:
return is_retryable_status(error.status_code)

return False


def _response_headers(error: Exception) -> Optional[httpx.Headers]:
if isinstance(error, ApiError) and error.headers:
return httpx.Headers(error.headers)
if isinstance(error, httpx.HTTPStatusError) and error.response is not None:
return error.response.headers

return None


def retry_delay_seconds(
error: Exception, retries: int, max_backoff_seconds: float
) -> float:
"""Honor Retry-After / X-RateLimit-Reset, else exponential backoff with full jitter."""
headers = _response_headers(error)
if headers is not None:
server_delay = _parse_retry_after(headers)
if server_delay is None or server_delay <= 0:
server_delay = _parse_x_ratelimit_reset(headers)
Comment on lines +56 to +59

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟣 pre-existing, not blocking: If the server sends a fractional Retry-After (e.g. "2.5") or a Retry-After-Ms header, get_prompt and media uploads ignore it and retry after uniform(0, 1s). That contradicts the updated get_prompt docstring. retry_delay_seconds hands the header to the generated _parse_retry_after. Its regex ^\s*[0-9]+\s*$ rejects decimals, and its retry_after_ms > 0 compares a str to an int, so it always falls through. Fix: parse Retry-After in retry.py as a float of seconds or an HTTP date, and Retry-After-Ms as a float of milliseconds, with a test for each.
A small fix can ride a push you are already making; otherwise a short reply is enough.

Why this was flagged

Trigger: any 429/503 whose Retry-After value is not a whole number, or that sends only retry-after-ms. The value enters through Langfuse.get_prompt (client.py:3928) or MediaManager._request_with_backoff, then call_with_retries and retry_delay_seconds (retry.py:56). In http_client.py:34, retry_after_ms > 0 compares a str to an int and raises TypeError. The bare except swallows it, so retry-after-ms is never used. In http_client.py:43, the regex rejects "2.5" and parsedate_tz returns None, so the function returns None. The code then falls back to random.uniform(0, 1) and then uniform(0, 2). All max_retries attempts land inside the rate-limit window and fail with 429. The new docstring at client.py:3792 says these responses "wait for the server's Retry-After header". The tests only use integer Retry-After values (test_retry.py:22,35; test_prompt.py:791), so the gap is not covered.

Verification: In langfuse/api/core/http_client.py:31-36, retry_after_ms > 0 compares a str to an int, which raises TypeError, and except Exception: pass drops the header. The regex ^\s*[0-9]+\s*$ (line 43) rejects "2.5". retry.py then falls back to random.uniform(0, min(2.0**retries, max_backoff_seconds)). On the base, _retry_timeout (http_client.py:107-120) used the same parser.

if server_delay is not None and server_delay > 0:
return min(server_delay, MAX_SERVER_RETRY_DELAY_SECONDS)
Comment on lines +59 to +61

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Clients retry together

retry_delay_seconds drops the small random delay previously added to X-RateLimit-Reset waits. Clients given the same reset time now retry together, which can trigger another rate limit when the server is busy. Add a small random delay to this path while keeping the overall wait capped.

Prompt To Fix With AI
This is a comment left during a code review.
Path: langfuse/_utils/retry.py
Line: 59-61

Comment:
**Clients retry together**

`retry_delay_seconds` drops the small random delay previously added to `X-RateLimit-Reset` waits. Clients given the same reset time now retry together, which can trigger another rate limit when the server is busy. Add a small random delay to this path while keeping the overall wait capped.

---

For each issue above, determine whether it is valid and should be fixed. If so, fix it directly.

Comment on lines +60 to +61

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 (optional) Concurrent SDK clients that get the same X-RateLimit-Reset now all retry at the same instant. On the base branch, the generated client spread these retries out with jitter. retry_delay_seconds returns min(server_delay, 60) exactly. The generated client's _retry_timeout instead wrapped the reset delay in _add_positive_jitter (http_client.py:113), which exists "to prevent thundering herd". Because the reset is an absolute timestamp, media consumer threads and processes all wake together and hit the limit again. Fix: add positive jitter (for example * (1 + random.random() * 0.2)) to server-derived delays, at least for X-RateLimit-Reset, and keep the 60s cap.

Why this was flagged

Trigger: several workers get a 429 carrying the same X-RateLimit-Reset epoch. Examples are the media upload consumer threads (MediaManager._request_with_backoff, media_manager.py:527) or many processes calling get_prompt (client.py:3928). In this case retry.py:60-61 returns min(server_delay, MAX_SERVER_RETRY_DELAY_SECONDS) with no jitter. Every caller then sleeps until the same wall-clock second and sends its retry at that moment. On the base branch these calls went through the generated client's own retry. Its _retry_timeout (http_client.py:111-113) applied _add_positive_jitter (0-20%), which http_client.py:67-68 documents as "to prevent thundering herd". The change sets max_retries=0 on the generated client, so that jittered path no longer runs. Nothing in call_with_retries adds jitter back for server-provided delays. The synchronized burst can trip the rate limit again for the whole group and burn the shared retry budget.

Verification: In langfuse/_utils/retry.py:57-61, if Retry-After is missing or <= 0, the helper returns min(server_delay, MAX_SERVER_RETRY_DELAY_SECONDS) with no jitter. So every caller that got the same reset value sleeps until the same wall-clock instant. On the base branch, _retry_timeout wrapped this in _add_positive_jitter, and the PR's {"max_retries": 0} skips that path.


return random.uniform(0, min(2.0**retries, max_backoff_seconds))


def call_with_retries(
func: Callable[[], T],
*,
max_retries: int,
should_retry: Callable[[Exception], bool] = is_retryable_api_call_error,
max_backoff_seconds: float = 10.0,
) -> T:
"""Call ``func``, retrying up to ``max_retries`` times on retryable errors."""
retries = 0
while True:
try:
return func()
except Exception as e:
if retries >= max_retries or not should_retry(e):
raise

time.sleep(retry_delay_seconds(e, retries, max_backoff_seconds))
retries += 1
62 changes: 62 additions & 0 deletions tests/unit/test_media_manager.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import time
from queue import Queue
from types import SimpleNamespace
from unittest.mock import Mock
Expand All @@ -6,6 +7,8 @@
import pytest

from langfuse._task_manager.media_manager import MediaManager
from langfuse.api import LangfuseAPI
from langfuse.api.core.api_error import ApiError
from langfuse.media import LangfuseMedia


Expand Down Expand Up @@ -277,3 +280,62 @@ def test_find_and_process_media_gemini_inline_data_non_string_data_passes_throug

assert result == data
assert queue.empty()


@pytest.mark.parametrize("status_code", [429, 503])
def test_media_api_calls_do_not_multiply_client_retries(monkeypatch, status_code):
monkeypatch.setattr(time, "sleep", lambda _seconds: None)
requests = []

def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(status_code, json={})
Comment on lines +286 to +292

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Upload reports lack retry tests

Both new media retry tests fail at get_upload_url, so neither reaches the changed media.patch calls. The existing upload tests mock patch as an immediate success. These tests would still pass if either upload report regained multiplied retries.

Add cases that reach both success and failure reports through the generated client, return retryable errors from patch, and check the request count.

Prompt To Fix With AI
This is a comment left during a code review.
Path: tests/unit/test_media_manager.py
Line: 286-292

Comment:
**Upload reports lack retry tests**

Both new media retry tests fail at `get_upload_url`, so neither reaches the changed `media.patch` calls. The existing upload tests mock `patch` as an immediate success. These tests would still pass if either upload report regained multiplied retries.

Add cases that reach both success and failure reports through the generated client, return retryable errors from `patch`, and check the request count.

---

For each issue above, determine whether it is valid and should be fixed. If so, fix it directly.


api_client = LangfuseAPI(
base_url="http://test-host",
username="pk",
password="sk",
httpx_client=httpx.Client(transport=httpx.MockTransport(handler)),
)
manager = MediaManager(
api_client=api_client,
httpx_client=Mock(),
media_upload_queue=Queue(),
max_retries=3,
)

with pytest.raises(ApiError):
manager._process_upload_media_job(data=_upload_job())

# max_retries=3 means 3 attempts in total, with no nested client retries
assert len(requests) == 3


def test_media_api_calls_share_one_retry_budget_across_failure_kinds(monkeypatch):
monkeypatch.setattr(time, "sleep", lambda _seconds: None)
requests = []

def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
if len(requests) % 2 == 0:
raise httpx.ConnectError("connection reset", request=request)

return httpx.Response(429, json={})

api_client = LangfuseAPI(
base_url="http://test-host",
username="pk",
password="sk",
httpx_client=httpx.Client(transport=httpx.MockTransport(handler)),
)
manager = MediaManager(
api_client=api_client,
httpx_client=Mock(),
media_upload_queue=Queue(),
max_retries=3,
)

with pytest.raises(ApiError):
manager._process_upload_media_job(data=_upload_job())

assert len(requests) == 3
Loading
Loading