Skip to content

Commit ccd2ef6

Browse files
niklassemmlerclaude
andcommitted
fix(tracing): share one retry budget across network and HTTP failures
A transport error raised during the API client's internal 429/5xx retries escaped with its retry count, and the outer TransportError retry then started a fresh inner budget, so mixed failures still sent up to (max_retries+1)^2 requests. get_prompt and MediaManager now call the API client with max_retries=0 and retry in one loop (call_with_retries) that covers network errors and 408/409/429/5xx and honors Retry-After / X-RateLimit-Reset. Refs LFE-17361 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent 1f7e1c9 commit ccd2ef6

6 files changed

Lines changed: 251 additions & 43 deletions

File tree

‎langfuse/_client/client.py‎

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
overload,
2828
)
2929

30-
import backoff
3130
import httpx
3231
from opentelemetry import context as otel_context_api
3332
from opentelemetry import trace as otel_trace_api
@@ -95,6 +94,7 @@
9594
from langfuse._utils.environment import get_common_release_envs
9695
from langfuse._utils.parse_error import handle_fern_exception
9796
from langfuse._utils.prompt_cache import PromptCache
97+
from langfuse._utils.retry import call_with_retries
9898
from langfuse.api import (
9999
AsyncLangfuseAPI,
100100
CreateChatPromptRequest,
@@ -4034,29 +4034,21 @@ def _fetch_prompt_and_update_cache(
40344034
langfuse_logger.debug("Fetching prompt '%s' from server...", cache_key)
40354035

40364036
try:
4037-
request_options: RequestOptions = {"max_retries": max_retries}
4037+
# Retries happen only in call_with_retries so network errors and
4038+
# 408/409/429/5xx responses share one budget.
4039+
request_options: RequestOptions = {"max_retries": 0}
40384040
if fetch_timeout_seconds is not None:
40394041
request_options["timeout_in_seconds"] = fetch_timeout_seconds
40404042

4041-
# The API client already retries 408/409/429/5xx responses with
4042-
# exponential backoff and Retry-After. Retry only transport errors
4043-
# here so retries are not multiplied and 4xx errors fail fast.
4044-
@backoff.on_exception(
4045-
backoff.expo,
4046-
httpx.TransportError,
4047-
max_tries=max_retries + 1,
4048-
max_value=10,
4049-
logger=None,
4050-
)
4051-
def fetch_prompts() -> Any:
4052-
return self.api.prompts.get(
4043+
prompt_response = call_with_retries(
4044+
lambda: self.api.prompts.get(
40534045
self._url_encode(name),
40544046
version=version,
40554047
label=label,
40564048
request_options=request_options,
4057-
)
4058-
4059-
prompt_response = fetch_prompts()
4049+
),
4050+
max_retries=max_retries,
4051+
)
40604052

40614053
prompt: PromptClient
40624054
if prompt_response.type == "chat":

‎langfuse/_task_manager/media_manager.py‎

Lines changed: 20 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,24 @@
33
from queue import Empty, Full, Queue
44
from typing import Any, Callable, Optional, TypeVar, cast
55

6-
import backoff
76
import httpx
87
from typing_extensions import ParamSpec
98

109
from langfuse._client.environment_variables import LANGFUSE_MEDIA_UPLOAD_ENABLED
1110
from langfuse._utils import _get_timestamp
11+
from langfuse._utils.retry import call_with_retries, is_retryable_status
1212
from langfuse.api import LangfuseAPI, MediaContentType
13-
from langfuse.api.core import ApiError
13+
from langfuse.api.core import ApiError, RequestOptions
1414
from langfuse.logger import langfuse_logger as logger
1515
from langfuse.media import LangfuseMedia
1616

1717
from .media_upload_queue import UploadMediaJob
1818

1919
T = TypeVar("T")
2020
P = ParamSpec("P")
21+
22+
# Retries happen in _request_with_backoff, so the API client must not retry too.
23+
_NO_CLIENT_RETRIES: RequestOptions = {"max_retries": 0}
2124
_SHUTDOWN_SENTINEL = object()
2225

2326

@@ -41,7 +44,7 @@ def __init__(
4144
self._api_client = api_client
4245
self._httpx_client = httpx_client
4346
self._queue = media_upload_queue
44-
self._max_retries = max_retries
47+
self._max_retries = max_retries if max_retries is not None else 3
4548
self._enabled = os.environ.get(
4649
LANGFUSE_MEDIA_UPLOAD_ENABLED, "True"
4750
).lower() not in ("false", "0")
@@ -422,6 +425,7 @@ def _process_upload_media_job(
422425
dataset_id=data["dataset_id"],
423426
dataset_item_id=data["dataset_item_id"],
424427
field=data["field"],
428+
request_options=_NO_CLIENT_RETRIES,
425429
)
426430

427431
upload_url = upload_url_response.upload_url
@@ -480,6 +484,7 @@ def _upload_with_status_check() -> httpx.Response:
480484
upload_http_status=failed_response.status_code,
481485
upload_http_error=failed_response.text,
482486
upload_time_ms=upload_time_ms,
487+
request_options=_NO_CLIENT_RETRIES,
483488
)
484489

485490
raise
@@ -493,6 +498,7 @@ def _upload_with_status_check() -> httpx.Response:
493498
upload_http_status=upload_response.status_code,
494499
upload_http_error=upload_response.text,
495500
upload_time_ms=upload_time_ms,
501+
request_options=_NO_CLIENT_RETRIES,
496502
)
497503

498504
logger.debug(
@@ -508,27 +514,18 @@ def _upload_with_status_check() -> httpx.Response:
508514
def _request_with_backoff(
509515
self, func: Callable[P, T], *args: P.args, **kwargs: P.kwargs
510516
) -> T:
511-
def _should_give_up(e: Exception) -> bool:
512-
# The API client already retried 408/409/429/5xx responses before
513-
# raising, so retrying here would multiply the requests.
517+
def _should_retry(e: Exception) -> bool:
514518
if isinstance(e, ApiError):
515-
return True
519+
return e.status_code is not None and is_retryable_status(e.status_code)
516520
if isinstance(e, httpx.HTTPStatusError):
517-
return (
518-
e.response is not None
519-
and e.response.status_code < 500
520-
and e.response.status_code != 429
521+
return e.response is not None and is_retryable_status(
522+
e.response.status_code
521523
)
522-
return False
523-
524-
@backoff.on_exception(
525-
backoff.expo,
526-
Exception,
527-
max_tries=self._max_retries,
528-
giveup=_should_give_up,
529-
logger=None,
530-
)
531-
def execute_task_with_backoff() -> T:
532-
return func(*args, **kwargs)
524+
return True
533525

534-
return execute_task_with_backoff()
526+
# API client calls pass max_retries=0, so this is the only retry layer.
527+
return call_with_retries(
528+
lambda: func(*args, **kwargs),
529+
max_retries=max(self._max_retries - 1, 0),
530+
should_retry=_should_retry,
531+
)

‎langfuse/_utils/retry.py‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
"""Retries for calls made through the generated API client.
2+
3+
Callers pass ``request_options={"max_retries": 0}`` to the API client and wrap
4+
the call in :func:`call_with_retries`, so network errors and retryable HTTP
5+
responses share a single retry budget instead of multiplying each other.
6+
"""
7+
8+
import random
9+
import time
10+
from typing import Callable, Optional, TypeVar
11+
12+
import httpx
13+
14+
from langfuse.api.core.api_error import ApiError
15+
from langfuse.api.core.http_client import (
16+
_parse_retry_after,
17+
_parse_x_ratelimit_reset,
18+
)
19+
20+
T = TypeVar("T")
21+
22+
RETRYABLE_STATUS_CODES = frozenset({408, 409, 429})
23+
24+
# Matches the cap the generated API client applies to server-provided delays.
25+
MAX_SERVER_RETRY_DELAY_SECONDS = 60.0
26+
27+
28+
def is_retryable_status(status_code: int) -> bool:
29+
return status_code >= 500 or status_code in RETRYABLE_STATUS_CODES
30+
31+
32+
def is_retryable_api_call_error(error: Exception) -> bool:
33+
"""Return True for network errors and 408/409/429/5xx API errors."""
34+
if isinstance(error, httpx.TransportError):
35+
return True
36+
if isinstance(error, ApiError) and error.status_code is not None:
37+
return is_retryable_status(error.status_code)
38+
39+
return False
40+
41+
42+
def _response_headers(error: Exception) -> Optional[httpx.Headers]:
43+
if isinstance(error, ApiError) and error.headers:
44+
return httpx.Headers(error.headers)
45+
if isinstance(error, httpx.HTTPStatusError) and error.response is not None:
46+
return error.response.headers
47+
48+
return None
49+
50+
51+
def retry_delay_seconds(
52+
error: Exception, retries: int, max_backoff_seconds: float
53+
) -> float:
54+
"""Honor Retry-After / X-RateLimit-Reset, else exponential backoff with full jitter."""
55+
headers = _response_headers(error)
56+
if headers is not None:
57+
server_delay = _parse_retry_after(headers)
58+
if server_delay is None or server_delay <= 0:
59+
server_delay = _parse_x_ratelimit_reset(headers)
60+
if server_delay is not None and server_delay > 0:
61+
return min(server_delay, MAX_SERVER_RETRY_DELAY_SECONDS)
62+
63+
return random.uniform(0, min(2.0**retries, max_backoff_seconds))
64+
65+
66+
def call_with_retries(
67+
func: Callable[[], T],
68+
*,
69+
max_retries: int,
70+
should_retry: Callable[[Exception], bool] = is_retryable_api_call_error,
71+
max_backoff_seconds: float = 10.0,
72+
) -> T:
73+
"""Call ``func``, retrying up to ``max_retries`` times on retryable errors."""
74+
retries = 0
75+
while True:
76+
try:
77+
return func()
78+
except Exception as e:
79+
if retries >= max_retries or not should_retry(e):
80+
raise
81+
82+
time.sleep(retry_delay_seconds(e, retries, max_backoff_seconds))
83+
retries += 1

‎tests/unit/test_media_manager.py‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -307,5 +307,35 @@ def handler(request: httpx.Request) -> httpx.Response:
307307
with pytest.raises(ApiError):
308308
manager._process_upload_media_job(data=_upload_job())
309309

310-
# the API client's own retries (2) are the only ones: 3 requests in total
310+
# max_retries=3 means 3 attempts in total, with no nested client retries
311+
assert len(requests) == 3
312+
313+
314+
def test_media_api_calls_share_one_retry_budget_across_failure_kinds(monkeypatch):
315+
monkeypatch.setattr(time, "sleep", lambda _seconds: None)
316+
requests = []
317+
318+
def handler(request: httpx.Request) -> httpx.Response:
319+
requests.append(request)
320+
if len(requests) % 2 == 0:
321+
raise httpx.ConnectError("connection reset", request=request)
322+
323+
return httpx.Response(429, json={})
324+
325+
api_client = LangfuseAPI(
326+
base_url="http://test-host",
327+
username="pk",
328+
password="sk",
329+
httpx_client=httpx.Client(transport=httpx.MockTransport(handler)),
330+
)
331+
manager = MediaManager(
332+
api_client=api_client,
333+
httpx_client=Mock(),
334+
media_upload_queue=Queue(),
335+
max_retries=3,
336+
)
337+
338+
with pytest.raises(ApiError):
339+
manager._process_upload_media_job(data=_upload_job())
340+
311341
assert len(requests) == 3

‎tests/unit/test_prompt.py‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ def test_get_fresh_prompt(langfuse):
211211
prompt_name,
212212
version=None,
213213
label=None,
214-
request_options={"max_retries": 2},
214+
request_options={"max_retries": 0},
215215
)
216216

217217
assert result == TextPromptClient(prompt)
@@ -256,7 +256,7 @@ def test_using_custom_prompt_timeouts(langfuse):
256256
prompt_name,
257257
version=None,
258258
label=None,
259-
request_options={"max_retries": 2, "timeout_in_seconds": 1000},
259+
request_options={"max_retries": 0, "timeout_in_seconds": 1000},
260260
)
261261

262262
assert result == TextPromptClient(prompt)
@@ -841,3 +841,42 @@ def handler(request: httpx.Request) -> httpx.Response:
841841
assert result.prompt == "Make me laugh"
842842
assert len(requests) == 3
843843
assert sleeps == [1, 2]
844+
845+
846+
def test_get_prompt_shares_one_retry_budget_across_failure_kinds(langfuse, sleeps):
847+
requests = []
848+
849+
def handler(request: httpx.Request) -> httpx.Response:
850+
requests.append(request)
851+
if len(requests) % 2 == 0:
852+
raise httpx.ConnectError("connection reset", request=request)
853+
854+
return httpx.Response(503, json={})
855+
856+
langfuse.api = _api_over_transport(handler)
857+
858+
with pytest.raises(httpx.ConnectError):
859+
langfuse.get_prompt("degraded_server_prompt", max_retries=3)
860+
861+
assert len(requests) == 4
862+
assert len(sleeps) == 3
863+
864+
865+
@pytest.mark.parametrize("failure", ["rate_limited", "network_error"])
866+
def test_get_prompt_with_zero_retries_sends_one_request(langfuse, sleeps, failure):
867+
requests = []
868+
869+
def handler(request: httpx.Request) -> httpx.Response:
870+
requests.append(request)
871+
if failure == "network_error":
872+
raise httpx.ConnectError("connection refused", request=request)
873+
874+
return httpx.Response(429, headers={"Retry-After": "1"}, json={})
875+
876+
langfuse.api = _api_over_transport(handler)
877+
878+
with pytest.raises((ApiError, httpx.ConnectError)):
879+
langfuse.get_prompt("no_retry_prompt", max_retries=0)
880+
881+
assert len(requests) == 1
882+
assert sleeps == []

0 commit comments

Comments
 (0)