diff --git a/langfuse/_task_manager/score_ingestion_consumer.py b/langfuse/_task_manager/score_ingestion_consumer.py index 1dce00f80..9a998915c 100644 --- a/langfuse/_task_manager/score_ingestion_consumer.py +++ b/langfuse/_task_manager/score_ingestion_consumer.py @@ -147,7 +147,13 @@ def upload(self) -> None: try: self._upload_batch(batch) except Exception as e: - handle_exception(e) + try: + handle_exception(e) + except Exception: + # error reporting must never kill the consumer thread + logger.exception( + "Error while handling score upload failure (original error: %s)", e + ) finally: # mark items as acknowledged from queue for _ in batch: diff --git a/langfuse/_utils/parse_error.py b/langfuse/_utils/parse_error.py index 2b9a7bd6f..82e88d635 100644 --- a/langfuse/_utils/parse_error.py +++ b/langfuse/_utils/parse_error.py @@ -72,21 +72,20 @@ def handle_fern_exception(exception: Error) -> None: logger.error(error_message) +def _error_response_for_status(status: Union[int, str, None]) -> str: + """Look up the message for a status; non-numeric/missing statuses fall back to the default.""" + try: + return errorResponseByCode.get(int(status), defaultErrorResponse) # type: ignore[arg-type] + except (TypeError, ValueError, OverflowError): + return defaultErrorResponse + + def generate_error_message(exception: Union[APIError, APIErrors, Exception]) -> str: if isinstance(exception, APIError): - status_code = ( - int(exception.status) - if isinstance(exception.status, str) - else exception.status - ) - return f"API error occurred: {errorResponseByCode.get(status_code, defaultErrorResponse)}" + return f"API error occurred: {_error_response_for_status(exception.status)}" elif isinstance(exception, APIErrors): error_messages = [ - errorResponseByCode.get( - int(error.status) if isinstance(error.status, str) else error.status, - defaultErrorResponse, - ) - for error in exception.errors + _error_response_for_status(error.status) for error in exception.errors ] return "API errors occurred: " + "\n".join(error_messages) else: diff --git a/tests/unit/test_error_parsing.py b/tests/unit/test_error_parsing.py index 36a7decbf..a5967ba56 100644 --- a/tests/unit/test_error_parsing.py +++ b/tests/unit/test_error_parsing.py @@ -1,6 +1,10 @@ """@private""" +import pytest + from langfuse._utils.parse_error import ( + defaultErrorResponse, + errorResponseByCode, generate_error_message, generate_error_message_fern, ) @@ -75,3 +79,21 @@ def test_generate_error_message_generic(): exception = ApiError(status_code=503) expected_message = "Service unavailable. This is an unusual occurrence and we are monitoring it closely. For help, please contact support: https://langfuse.com/support." assert generate_error_message_fern(exception) == expected_message + + +@pytest.mark.parametrize("status", ["abc", "", None, [400], float("inf")]) +def test_generate_error_message_non_numeric_status(status): + expected_default = f"API error occurred: {defaultErrorResponse}" + assert generate_error_message(APIError(status=status, message="Bad")) == ( + expected_default + ) + assert generate_error_message( + APIErrors([APIError(status=status, message="Bad")]) + ) == ("API errors occurred: " + defaultErrorResponse) + + +@pytest.mark.parametrize("status", [404, "404"]) +def test_generate_error_message_numeric_status_unchanged(status): + assert errorResponseByCode[404] in generate_error_message( + APIError(status=status, message="Not found") + ) diff --git a/tests/unit/test_resource_manager.py b/tests/unit/test_resource_manager.py index f66a1e052..e4b0f8bdc 100644 --- a/tests/unit/test_resource_manager.py +++ b/tests/unit/test_resource_manager.py @@ -1,5 +1,6 @@ """Test the LangfuseResourceManager and get_client() function.""" +import threading from queue import Queue from types import SimpleNamespace from typing import Sequence @@ -14,6 +15,7 @@ from langfuse._task_manager.media_manager import MediaManager from langfuse._task_manager.media_upload_consumer import MediaUploadConsumer from langfuse._task_manager.score_ingestion_consumer import ScoreIngestionConsumer +from langfuse._utils.request import APIError, APIErrors from langfuse.types import MaskOtelSpansResult @@ -27,6 +29,32 @@ def shutdown(self) -> None: pass +def _join_with_timeout(queue: Queue, timeout: float = 5) -> bool: + """Return True if the queue drained; False (instead of hanging) if it did not.""" + joiner = threading.Thread(target=queue.join, daemon=True) + joiner.start() + joiner.join(timeout) + return not joiner.is_alive() + + +def _assert_consumer_survives(consumer, queue): + queue.put({"id": "1", "type": "score-create", "body": {}}) + consumer.start() + try: + assert _join_with_timeout(queue), "queue.join() hung: item was not acknowledged" + assert consumer.is_alive() + # task_done() runs in a finally block before an exception can leave run(), + # so the first drain alone does not prove the worker survived. A second + # item must also be taken and acknowledged. + queue.put({"id": "2", "type": "score-create", "body": {}}) + assert _join_with_timeout(queue), "worker stopped taking new work" + assert consumer.is_alive() + finally: + consumer.pause() + consumer.join(timeout=5) + assert not consumer.is_alive() + + def test_get_client_preserves_all_settings(monkeypatch): """Test that get_client() preserves environment and all client settings.""" with LangfuseResourceManager._lock: @@ -156,6 +184,42 @@ def test_score_ingestion_consumer_pause_wakes_blocked_thread(): assert not consumer.is_alive() +def test_score_ingestion_consumer_survives_error_handler_failure(monkeypatch): + queue = Queue() + consumer = ScoreIngestionConsumer( + ingestion_queue=queue, + identifier=0, + client=Mock(), + public_key="pk-test", + flush_interval=0.1, + ) + monkeypatch.setattr(consumer, "_upload_batch", Mock(side_effect=RuntimeError("x"))) + monkeypatch.setattr( + "langfuse._task_manager.score_ingestion_consumer.handle_exception", + Mock(side_effect=ValueError("boom")), + ) + + _assert_consumer_survives(consumer, queue) + + +def test_score_ingestion_consumer_survives_non_numeric_207_status(): + queue = Queue() + client = Mock() + client.batch_post.side_effect = APIErrors( + [APIError(status="abc", message="m", details="d")] + ) + consumer = ScoreIngestionConsumer( + ingestion_queue=queue, + identifier=0, + client=client, + public_key="pk-test", + flush_interval=0.1, + max_retries=1, + ) + + _assert_consumer_survives(consumer, queue) + + def test_media_upload_consumer_signal_shutdown_wakes_blocked_thread(): media_manager = MediaManager( api_client=Mock(),