Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions langfuse/_task_manager/score_ingestion_consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from pydantic import BaseModel

from langfuse._utils.parse_error import handle_exception
from langfuse._utils.request import APIError, LangfuseClient
from langfuse._utils.request import APIError, APIErrors, LangfuseClient
from langfuse._utils.serializer import EventSerializer
from langfuse.logger import langfuse_logger as logger

Expand Down Expand Up @@ -181,6 +181,23 @@ def _upload_batch(self, batch: List[Any]) -> None:
def execute_task_with_backoff(batch: List[Any]) -> None:
try:
self._client.batch_post(batch=batch, metadata=metadata)
logger.debug(
"API: Successfully sent %s score events to Langfuse API in batch mode",
len(batch),
)
except APIErrors as e:
# 207 partial failure: the other items were already accepted, so
# re-posting the whole batch cannot fix the rejected ones.
# Known limitation: per-item 429/5xx are dropped too; retrying only those
# needs the item id, which APIErrors does not carry yet.
logger.warning(
"API: %s of %s score events were rejected by Langfuse "
"(showing up to 5): %s",
len(e.errors),
len(batch),
"; ".join(repr(str(err))[:200] for err in e.errors[:5]),
)
return
except Exception as e:
if (
isinstance(e, APIError)
Expand All @@ -192,7 +209,3 @@ def execute_task_with_backoff(batch: List[Any]) -> None:
raise e

execute_task_with_backoff(batch)
logger.debug(
"API: Successfully sent %s score events to Langfuse API in batch mode",
len(batch),
)
66 changes: 66 additions & 0 deletions tests/unit/test_resource_manager.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Test the LangfuseResourceManager and get_client() function."""

import logging
from queue import Queue
from types import SimpleNamespace
from typing import Sequence
Expand All @@ -14,6 +15,7 @@
from langfuse._task_manager.media_manager import MediaManager
from langfuse._task_manager.media_upload_consumer import MediaUploadConsumer
from langfuse._task_manager.score_ingestion_consumer import ScoreIngestionConsumer
from langfuse._utils.request import APIError, APIErrors
from langfuse.types import MaskOtelSpansResult


Expand Down Expand Up @@ -156,6 +158,70 @@ def test_score_ingestion_consumer_pause_wakes_blocked_thread():
assert not consumer.is_alive()


def test_score_ingestion_consumer_does_not_retry_207_partial_failure(caplog):
client = Mock()
client.batch_post.side_effect = APIErrors(
[APIError(400, "invalid score", "bad value")]
)
queue: Queue = Queue()
consumer = ScoreIngestionConsumer(
ingestion_queue=queue,
identifier=0,
client=client,
public_key="pk-test",
flush_at=2,
flush_interval=0.1,
)
for i in range(2):
queue.put({"id": str(i), "type": "score-create", "body": {"name": "s"}})

with caplog.at_level(logging.DEBUG, logger="langfuse"):
consumer.upload()

assert client.batch_post.call_count == 1
assert "1 of 2 score events were rejected" in caplog.text
assert "invalid score" in caplog.text
# A rejected batch must not also be reported as successfully sent.
assert "Successfully sent" not in caplog.text


def test_score_ingestion_consumer_207_warning_is_bounded_and_escaped(caplog):
long_message = "x" * 300
errors = [APIError(400, f"error-{i}", "detail") for i in range(7)]
errors[0] = APIError(400, "multi\nline\r\nmessage", long_message)
client = Mock()
client.batch_post.side_effect = APIErrors(errors)
queue: Queue = Queue()
consumer = ScoreIngestionConsumer(
ingestion_queue=queue,
identifier=0,
client=client,
public_key="pk-test",
flush_at=2,
flush_interval=0.1,
)
for i in range(2):
queue.put({"id": str(i), "type": "score-create", "body": {"name": "s"}})

with caplog.at_level(logging.DEBUG, logger="langfuse"):
consumer.upload()

assert client.batch_post.call_count == 1
warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
assert len(warnings) == 1
message = warnings[0]
assert "7 of 2 score events were rejected" in message
# Only the first 5 errors are shown.
assert "error-4" in message
assert "error-5" not in message
assert "error-6" not in message
# Control characters are escaped, so the log stays on one line.
assert "\n" not in message and "\r" not in message
assert "multi\\nline\\r\\nmessage" in message
# Each error is truncated to 200 characters.
assert "x" * 200 not in message


def test_media_upload_consumer_signal_shutdown_wakes_blocked_thread():
media_manager = MediaManager(
api_client=Mock(),
Expand Down