Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion langfuse/_task_manager/score_ingestion_consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,13 @@ def upload(self) -> None:
try:
self._upload_batch(batch)
except Exception as e:
handle_exception(e)
try:
handle_exception(e)
except Exception:
# error reporting must never kill the consumer thread
logger.exception(
"Error while handling score upload failure (original error: %s)", e
)
finally:
# mark items as acknowledged from queue
for _ in batch:
Expand Down
21 changes: 10 additions & 11 deletions langfuse/_utils/parse_error.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,21 +72,20 @@ def handle_fern_exception(exception: Error) -> None:
logger.error(error_message)


def _error_response_for_status(status: Union[int, str, None]) -> str:
"""Look up the message for a status; non-numeric/missing statuses fall back to the default."""
try:
return errorResponseByCode.get(int(status), defaultErrorResponse) # type: ignore[arg-type]
except (TypeError, ValueError, OverflowError):
return defaultErrorResponse


def generate_error_message(exception: Union[APIError, APIErrors, Exception]) -> str:
if isinstance(exception, APIError):
status_code = (
int(exception.status)
if isinstance(exception.status, str)
else exception.status
)
return f"API error occurred: {errorResponseByCode.get(status_code, defaultErrorResponse)}"
return f"API error occurred: {_error_response_for_status(exception.status)}"
elif isinstance(exception, APIErrors):
error_messages = [
errorResponseByCode.get(
int(error.status) if isinstance(error.status, str) else error.status,
defaultErrorResponse,
)
for error in exception.errors
_error_response_for_status(error.status) for error in exception.errors
]
return "API errors occurred: " + "\n".join(error_messages)
else:
Expand Down
22 changes: 22 additions & 0 deletions tests/unit/test_error_parsing.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
"""@private"""

import pytest

from langfuse._utils.parse_error import (
defaultErrorResponse,
errorResponseByCode,
generate_error_message,
generate_error_message_fern,
)
Expand Down Expand Up @@ -75,3 +79,21 @@ def test_generate_error_message_generic():
exception = ApiError(status_code=503)
expected_message = "Service unavailable. This is an unusual occurrence and we are monitoring it closely. For help, please contact support: https://langfuse.com/support."
assert generate_error_message_fern(exception) == expected_message


@pytest.mark.parametrize("status", ["abc", "", None, [400], float("inf")])
def test_generate_error_message_non_numeric_status(status):
expected_default = f"API error occurred: {defaultErrorResponse}"
assert generate_error_message(APIError(status=status, message="Bad")) == (
expected_default
)
assert generate_error_message(
APIErrors([APIError(status=status, message="Bad")])
) == ("API errors occurred: " + defaultErrorResponse)


@pytest.mark.parametrize("status", [404, "404"])
def test_generate_error_message_numeric_status_unchanged(status):
assert errorResponseByCode[404] in generate_error_message(
APIError(status=status, message="Not found")
)
64 changes: 64 additions & 0 deletions tests/unit/test_resource_manager.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Test the LangfuseResourceManager and get_client() function."""

import threading
from queue import Queue
from types import SimpleNamespace
from typing import Sequence
Expand All @@ -14,6 +15,7 @@
from langfuse._task_manager.media_manager import MediaManager
from langfuse._task_manager.media_upload_consumer import MediaUploadConsumer
from langfuse._task_manager.score_ingestion_consumer import ScoreIngestionConsumer
from langfuse._utils.request import APIError, APIErrors
from langfuse.types import MaskOtelSpansResult


Expand All @@ -27,6 +29,32 @@ def shutdown(self) -> None:
pass


def _join_with_timeout(queue: Queue, timeout: float = 5) -> bool:
"""Return True if the queue drained; False (instead of hanging) if it did not."""
joiner = threading.Thread(target=queue.join, daemon=True)
joiner.start()
joiner.join(timeout)
return not joiner.is_alive()


def _assert_consumer_survives(consumer, queue):
queue.put({"id": "1", "type": "score-create", "body": {}})
consumer.start()
try:
assert _join_with_timeout(queue), "queue.join() hung: item was not acknowledged"
assert consumer.is_alive()
# task_done() runs in a finally block before an exception can leave run(),
# so the first drain alone does not prove the worker survived. A second
# item must also be taken and acknowledged.
queue.put({"id": "2", "type": "score-create", "body": {}})
assert _join_with_timeout(queue), "worker stopped taking new work"
assert consumer.is_alive()
finally:
consumer.pause()
consumer.join(timeout=5)
assert not consumer.is_alive()


def test_get_client_preserves_all_settings(monkeypatch):
"""Test that get_client() preserves environment and all client settings."""
with LangfuseResourceManager._lock:
Expand Down Expand Up @@ -156,6 +184,42 @@ def test_score_ingestion_consumer_pause_wakes_blocked_thread():
assert not consumer.is_alive()


def test_score_ingestion_consumer_survives_error_handler_failure(monkeypatch):
queue = Queue()
consumer = ScoreIngestionConsumer(
ingestion_queue=queue,
identifier=0,
client=Mock(),
public_key="pk-test",
flush_interval=0.1,
)
monkeypatch.setattr(consumer, "_upload_batch", Mock(side_effect=RuntimeError("x")))
monkeypatch.setattr(
"langfuse._task_manager.score_ingestion_consumer.handle_exception",
Mock(side_effect=ValueError("boom")),
)

_assert_consumer_survives(consumer, queue)


def test_score_ingestion_consumer_survives_non_numeric_207_status():
queue = Queue()
client = Mock()
client.batch_post.side_effect = APIErrors(
[APIError(status="abc", message="m", details="d")]
)
consumer = ScoreIngestionConsumer(
ingestion_queue=queue,
identifier=0,
client=client,
public_key="pk-test",
flush_interval=0.1,
max_retries=1,
)

_assert_consumer_survives(consumer, queue)


def test_media_upload_consumer_signal_shutdown_wakes_blocked_thread():
media_manager = MediaManager(
api_client=Mock(),
Expand Down