Skip to content

Commit f93d505

Browse files
authored
fix(langchain): mark handled tool errors as errors (#1718)
1 parent 818b155 commit f93d505

2 files changed

Lines changed: 53 additions & 6 deletions

File tree

‎langfuse/langchain/CallbackHandler.py‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1091,7 +1091,7 @@ def on_retriever_end(
10911091

10921092
def on_tool_end(
10931093
self,
1094-
output: str,
1094+
output: Any,
10951095
*,
10961096
run_id: UUID,
10971097
parent_run_id: Optional[UUID] = None,
@@ -1105,10 +1105,24 @@ def on_tool_end(
11051105
if observation is not None:
11061106
if parent_run_id is None:
11071107
self._clear_root_run_resume_key(run_id)
1108-
observation.update(
1109-
output=output,
1110-
input=kwargs.get("inputs"),
1111-
).end()
1108+
1109+
update_kwargs: Dict[str, Any] = {
1110+
"output": output,
1111+
"input": kwargs.get("inputs"),
1112+
}
1113+
1114+
if (
1115+
isinstance(output, ToolMessage)
1116+
and getattr(output, "status", None) == "error"
1117+
):
1118+
update_kwargs["level"] = "ERROR"
1119+
update_kwargs["status_message"] = (
1120+
output.content
1121+
if isinstance(output.content, str)
1122+
else str(output.content)
1123+
)
1124+
1125+
observation.update(**update_kwargs).end()
11121126

11131127
except Exception as e:
11141128
langfuse_logger.exception(e)

‎tests/unit/test_langchain.py‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
import pytest
77
from langchain.messages import HumanMessage
8-
from langchain_core.messages import AIMessage
8+
from langchain_core.messages import AIMessage, ToolMessage
99
from langchain_core.output_parsers import StrOutputParser
1010
from langchain_core.outputs import ChatGeneration, ChatResult, Generation, LLMResult
1111
from langchain_core.prompts import ChatPromptTemplate
@@ -791,6 +791,39 @@ class DummyControlFlowError(RuntimeError):
791791
assert not _has_run_state(handler, retriever_run_id)
792792

793793

794+
def test_handled_tool_error_marks_observation_error(
795+
langfuse_memory_client, get_span, json_attr
796+
):
797+
handler = CallbackHandler()
798+
run_id = uuid4()
799+
800+
handler.on_tool_start(
801+
{"name": "failing_tool"},
802+
'{"query": "x"}',
803+
run_id=run_id,
804+
)
805+
handler.on_tool_end(
806+
ToolMessage(
807+
content="handled failure",
808+
tool_call_id="call_1",
809+
status="error",
810+
),
811+
run_id=run_id,
812+
)
813+
814+
langfuse_memory_client.flush()
815+
span = get_span("failing_tool")
816+
817+
assert span.attributes[LangfuseOtelSpanAttributes.OBSERVATION_LEVEL] == "ERROR"
818+
assert (
819+
span.attributes[LangfuseOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE]
820+
== "handled failure"
821+
)
822+
assert json_attr(span, LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT)["status"] == (
823+
"error"
824+
)
825+
826+
794827
def test_pending_resume_contexts_are_capped(langfuse_memory_client, monkeypatch):
795828
class DummyControlFlowError(RuntimeError):
796829
pass

0 commit comments

Comments
 (0)