diff --git a/langfuse/_task_manager/score_ingestion_consumer.py b/langfuse/_task_manager/score_ingestion_consumer.py index 1dce00f80..e118b3326 100644 --- a/langfuse/_task_manager/score_ingestion_consumer.py +++ b/langfuse/_task_manager/score_ingestion_consumer.py @@ -9,7 +9,7 @@ from pydantic import BaseModel from langfuse._utils.parse_error import handle_exception -from langfuse._utils.request import APIError, LangfuseClient +from langfuse._utils.request import APIError, APIErrors, LangfuseClient from langfuse._utils.serializer import EventSerializer from langfuse.logger import langfuse_logger as logger @@ -181,6 +181,23 @@ def _upload_batch(self, batch: List[Any]) -> None: def execute_task_with_backoff(batch: List[Any]) -> None: try: self._client.batch_post(batch=batch, metadata=metadata) + logger.debug( + "API: Successfully sent %s score events to Langfuse API in batch mode", + len(batch), + ) + except APIErrors as e: + # 207 partial failure: the other items were already accepted, so + # re-posting the whole batch cannot fix the rejected ones. + # Known limitation: per-item 429/5xx are dropped too; retrying only those + # needs the item id, which APIErrors does not carry yet. + logger.warning( + "API: %s of %s score events were rejected by Langfuse " + "(showing up to 5): %s", + len(e.errors), + len(batch), + "; ".join(repr(str(err))[:200] for err in e.errors[:5]), + ) + return except Exception as e: if ( isinstance(e, APIError) @@ -192,7 +209,3 @@ def execute_task_with_backoff(batch: List[Any]) -> None: raise e execute_task_with_backoff(batch) - logger.debug( - "API: Successfully sent %s score events to Langfuse API in batch mode", - len(batch), - ) diff --git a/tests/unit/test_resource_manager.py b/tests/unit/test_resource_manager.py index f66a1e052..929a85122 100644 --- a/tests/unit/test_resource_manager.py +++ b/tests/unit/test_resource_manager.py @@ -1,5 +1,6 @@ """Test the LangfuseResourceManager and get_client() function.""" +import logging from queue import Queue from types import SimpleNamespace from typing import Sequence @@ -14,6 +15,7 @@ from langfuse._task_manager.media_manager import MediaManager from langfuse._task_manager.media_upload_consumer import MediaUploadConsumer from langfuse._task_manager.score_ingestion_consumer import ScoreIngestionConsumer +from langfuse._utils.request import APIError, APIErrors from langfuse.types import MaskOtelSpansResult @@ -156,6 +158,70 @@ def test_score_ingestion_consumer_pause_wakes_blocked_thread(): assert not consumer.is_alive() +def test_score_ingestion_consumer_does_not_retry_207_partial_failure(caplog): + client = Mock() + client.batch_post.side_effect = APIErrors( + [APIError(400, "invalid score", "bad value")] + ) + queue: Queue = Queue() + consumer = ScoreIngestionConsumer( + ingestion_queue=queue, + identifier=0, + client=client, + public_key="pk-test", + flush_at=2, + flush_interval=0.1, + ) + for i in range(2): + queue.put({"id": str(i), "type": "score-create", "body": {"name": "s"}}) + + with caplog.at_level(logging.DEBUG, logger="langfuse"): + consumer.upload() + + assert client.batch_post.call_count == 1 + assert "1 of 2 score events were rejected" in caplog.text + assert "invalid score" in caplog.text + # A rejected batch must not also be reported as successfully sent. + assert "Successfully sent" not in caplog.text + + +def test_score_ingestion_consumer_207_warning_is_bounded_and_escaped(caplog): + long_message = "x" * 300 + errors = [APIError(400, f"error-{i}", "detail") for i in range(7)] + errors[0] = APIError(400, "multi\nline\r\nmessage", long_message) + client = Mock() + client.batch_post.side_effect = APIErrors(errors) + queue: Queue = Queue() + consumer = ScoreIngestionConsumer( + ingestion_queue=queue, + identifier=0, + client=client, + public_key="pk-test", + flush_at=2, + flush_interval=0.1, + ) + for i in range(2): + queue.put({"id": str(i), "type": "score-create", "body": {"name": "s"}}) + + with caplog.at_level(logging.DEBUG, logger="langfuse"): + consumer.upload() + + assert client.batch_post.call_count == 1 + warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert len(warnings) == 1 + message = warnings[0] + assert "7 of 2 score events were rejected" in message + # Only the first 5 errors are shown. + assert "error-4" in message + assert "error-5" not in message + assert "error-6" not in message + # Control characters are escaped, so the log stays on one line. + assert "\n" not in message and "\r" not in message + assert "multi\\nline\\r\\nmessage" in message + # Each error is truncated to 200 characters. + assert "x" * 200 not in message + + def test_media_upload_consumer_signal_shutdown_wakes_blocked_thread(): media_manager = MediaManager( api_client=Mock(),