Skip to content

Commit 50fc1fa

Browse files
committed
fix(langchain): normalize tool definitions to OpenAI function format
Tool definitions from invocation_params["tools"] were forwarded into the generation input without format normalization. Setups that bypass bind_tools() (model.bind(tools=...), tools passed as invoke kwargs, or custom model integrations) deliver raw BaseTool instances, which were serialized as their attribute dict (args_schema, return_direct, ...) and not recognized as tool definitions by the Langfuse UI. The CallbackHandler now normalizes each tool definition before appending it: OpenAI-format dicts pass through unchanged, everything else is converted via langchain_core's convert_to_openai_tool with a fallback to the original value. Fixes #11850
1 parent 8520890 commit 50fc1fa

3 files changed

Lines changed: 150 additions & 2 deletions

File tree

‎langfuse/langchain/CallbackHandler.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
LangfuseTool,
3434
)
3535
from langfuse._utils import _get_timestamp
36-
from langfuse.langchain.utils import _extract_model_name
36+
from langfuse.langchain.utils import _extract_model_name, _normalize_tool_definition
3737
from langfuse.logger import langfuse_logger
3838
from langfuse.types import TraceContext
3939

@@ -1189,7 +1189,12 @@ def __on_llm_action(
11891189
try:
11901190
tools = kwargs.get("invocation_params", {}).get("tools", None)
11911191
if tools and isinstance(tools, list):
1192-
prompts.extend([{"role": "tool", "content": tool} for tool in tools])
1192+
prompts.extend(
1193+
[
1194+
{"role": "tool", "content": _normalize_tool_definition(tool)}
1195+
for tool in tools
1196+
]
1197+
)
11931198

11941199
model_name = self._parse_model_and_log_errors(
11951200
serialized=serialized, metadata=metadata, kwargs=kwargs

‎langfuse/langchain/utils.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,32 @@
99
# - langchain_community is loaded as a dependency of langchain, but extremely unreliable. Decided to not depend on it.
1010

1111

12+
def _normalize_tool_definition(tool: Any) -> Any:
13+
"""Best-effort normalization of a tool definition into the OpenAI function-tool format.
14+
15+
LangChain integrations are expected to pass tool definitions through
16+
`invocation_params["tools"]` in the OpenAI format
17+
`{"type": "function", "function": {...}}`. Some code paths (custom chat
18+
models, `model.bind(tools=...)` instead of `bind_tools`) forward raw
19+
`BaseTool` instances instead, which would otherwise be serialized as their
20+
attribute dict (`args_schema`, `return_direct`, ...) and not be recognized
21+
as tools by the Langfuse UI.
22+
"""
23+
if (
24+
isinstance(tool, dict)
25+
and tool.get("type") == "function"
26+
and isinstance(tool.get("function"), dict)
27+
):
28+
return tool
29+
30+
try:
31+
from langchain_core.utils.function_calling import convert_to_openai_tool
32+
33+
return convert_to_openai_tool(tool)
34+
except Exception:
35+
return tool
36+
37+
1238
def _extract_model_name(
1339
serialized: Optional[Dict[str, Any]],
1440
**kwargs: Any,

‎tests/unit/test_langchain.py‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,14 @@
99
from langchain_core.output_parsers import StrOutputParser
1010
from langchain_core.outputs import ChatGeneration, ChatResult, Generation, LLMResult
1111
from langchain_core.prompts import ChatPromptTemplate
12+
from langchain_core.tools import StructuredTool
1213
from langchain_openai import ChatOpenAI, OpenAI
1314
from opentelemetry import context as otel_context
15+
from pydantic import BaseModel, Field
1416

1517
from langfuse._client.attributes import LangfuseOtelSpanAttributes
1618
from langfuse.langchain import CallbackHandler
19+
from langfuse.langchain.utils import _normalize_tool_definition
1720

1821
callback_handler_module = importlib.import_module("langfuse.langchain.CallbackHandler")
1922

@@ -940,3 +943,117 @@ def import_without_langgraph_command(
940943
)
941944

942945
importlib.reload(callback_handler_module)
946+
947+
948+
class _ExampleToolArgs(BaseModel):
949+
query: str = Field(description="What to search for")
950+
951+
952+
def _make_example_tool() -> StructuredTool:
953+
return StructuredTool.from_function(
954+
func=lambda query: query,
955+
name="example_tool",
956+
description="Example tool description",
957+
args_schema=_ExampleToolArgs,
958+
)
959+
960+
961+
def _invoke_chat_model_with_tools(langfuse_memory_client, bind):
962+
response = ChatResult(
963+
generations=[ChatGeneration(message=AIMessage(content="ok"), text="ok")],
964+
llm_output={"token_usage": {}, "model_name": "gpt-4o-mini"},
965+
)
966+
967+
with patch.object(ChatOpenAI, "_generate", return_value=response):
968+
handler = CallbackHandler()
969+
970+
with langfuse_memory_client.start_as_current_observation(name="parent"):
971+
bind(ChatOpenAI(api_key="test", temperature=0)).invoke(
972+
[HumanMessage(content="hello")],
973+
config={"callbacks": [handler]},
974+
)
975+
976+
langfuse_memory_client.flush()
977+
978+
979+
def _tool_definition_messages(generation_span, json_attr):
980+
input_messages = json_attr(
981+
generation_span, LangfuseOtelSpanAttributes.OBSERVATION_INPUT
982+
)
983+
return [message for message in input_messages if message.get("role") == "tool"]
984+
985+
986+
def test_normalize_tool_definition_passes_openai_format_through():
987+
openai_tool = {
988+
"type": "function",
989+
"function": {"name": "example_tool", "parameters": {"type": "object"}},
990+
}
991+
992+
assert _normalize_tool_definition(openai_tool) is openai_tool
993+
994+
995+
def test_normalize_tool_definition_converts_base_tool_instances():
996+
normalized = _normalize_tool_definition(_make_example_tool())
997+
998+
assert normalized["type"] == "function"
999+
assert normalized["function"]["name"] == "example_tool"
1000+
assert normalized["function"]["description"] == "Example tool description"
1001+
assert "query" in normalized["function"]["parameters"]["properties"]
1002+
1003+
1004+
def test_normalize_tool_definition_wraps_bare_function_schemas():
1005+
bare_function = {
1006+
"name": "example_tool",
1007+
"description": "Example tool description",
1008+
"parameters": {"type": "object", "properties": {}},
1009+
}
1010+
1011+
normalized = _normalize_tool_definition(bare_function)
1012+
1013+
assert normalized["type"] == "function"
1014+
assert normalized["function"]["name"] == "example_tool"
1015+
1016+
1017+
def test_normalize_tool_definition_returns_unconvertible_values_unchanged():
1018+
unconvertible = object()
1019+
1020+
assert _normalize_tool_definition(unconvertible) is unconvertible
1021+
1022+
1023+
def test_raw_bound_tools_are_normalized_in_generation_input(
1024+
langfuse_memory_client, get_span, json_attr
1025+
):
1026+
# .bind() (unlike .bind_tools()) forwards raw tool objects into
1027+
# invocation_params -- the path that produced unrenderable tool
1028+
# definitions in the Langfuse UI.
1029+
_invoke_chat_model_with_tools(
1030+
langfuse_memory_client, lambda model: model.bind(tools=[_make_example_tool()])
1031+
)
1032+
1033+
tool_messages = _tool_definition_messages(get_span("ChatOpenAI"), json_attr)
1034+
1035+
assert len(tool_messages) == 1
1036+
tool_definition = tool_messages[0]["content"]
1037+
assert tool_definition["type"] == "function"
1038+
assert tool_definition["function"]["name"] == "example_tool"
1039+
assert "query" in tool_definition["function"]["parameters"]["properties"], (
1040+
"args schema should be exposed as OpenAI-style parameters"
1041+
)
1042+
1043+
1044+
def test_bind_tools_tool_definitions_stay_in_openai_format(
1045+
langfuse_memory_client, get_span, json_attr
1046+
):
1047+
# Regression guard for the standard path: bind_tools() already provides
1048+
# OpenAI-format definitions, which must pass through unchanged.
1049+
_invoke_chat_model_with_tools(
1050+
langfuse_memory_client, lambda model: model.bind_tools([_make_example_tool()])
1051+
)
1052+
1053+
tool_messages = _tool_definition_messages(get_span("ChatOpenAI"), json_attr)
1054+
1055+
assert len(tool_messages) == 1
1056+
tool_definition = tool_messages[0]["content"]
1057+
assert tool_definition["type"] == "function"
1058+
assert tool_definition["function"]["name"] == "example_tool"
1059+
assert "query" in tool_definition["function"]["parameters"]["properties"]

0 commit comments

Comments
 (0)