Skip to content

Commit 55b1ed0

Browse files
milanagmhassiebp
andauthored
fix(langchain): normalize tool definitions to OpenAI function format (#1758)
* fix(langchain): normalize tool definitions to OpenAI function format Tool definitions from invocation_params["tools"] were forwarded into the generation input without format normalization. Setups that bypass bind_tools() (model.bind(tools=...), tools passed as invoke kwargs, or custom model integrations) deliver raw BaseTool instances, which were serialized as their attribute dict (args_schema, return_direct, ...) and not recognized as tool definitions by the Langfuse UI. The CallbackHandler now normalizes each tool definition before appending it: OpenAI-format dicts pass through unchanged, everything else is converted via langchain_core's convert_to_openai_tool with a fallback to the original value. Fixes #11850 * fix(langchain): log tool normalization fallback at debug level --------- Co-authored-by: Hassieb Pakzad <68423100+hassiebp@users.noreply.github.com>
1 parent 5a590f0 commit 55b1ed0

3 files changed

Lines changed: 156 additions & 2 deletions

File tree

‎langfuse/langchain/CallbackHandler.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
LangfuseTool,
3434
)
3535
from langfuse._utils import _get_timestamp
36-
from langfuse.langchain.utils import _extract_model_name
36+
from langfuse.langchain.utils import _extract_model_name, _normalize_tool_definition
3737
from langfuse.logger import langfuse_logger
3838
from langfuse.types import TraceContext
3939

@@ -1189,7 +1189,12 @@ def __on_llm_action(
11891189
try:
11901190
tools = kwargs.get("invocation_params", {}).get("tools", None)
11911191
if tools and isinstance(tools, list):
1192-
prompts.extend([{"role": "tool", "content": tool} for tool in tools])
1192+
prompts.extend(
1193+
[
1194+
{"role": "tool", "content": _normalize_tool_definition(tool)}
1195+
for tool in tools
1196+
]
1197+
)
11931198

11941199
model_name = self._parse_model_and_log_errors(
11951200
serialized=serialized, metadata=metadata, kwargs=kwargs

‎langfuse/langchain/utils.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,44 @@
33
import re
44
from typing import Any, Dict, List, Literal, Optional, cast
55

6+
from langfuse.logger import langfuse_logger
7+
68
# NOTE ON DEPENDENCIES:
79
# - since Jan 2024, there is https://pypi.org/project/langchain-openai/ which is a separate package and imports openai models.
810
# Decided to not make this a dependency of langfuse as few people will have this. Need to match these models manually
911
# - langchain_community is loaded as a dependency of langchain, but extremely unreliable. Decided to not depend on it.
1012

1113

14+
def _normalize_tool_definition(tool: Any) -> Any:
15+
"""Best-effort normalization of a tool definition into the OpenAI function-tool format.
16+
17+
LangChain integrations are expected to pass tool definitions through
18+
`invocation_params["tools"]` in the OpenAI format
19+
`{"type": "function", "function": {...}}`. Some code paths (custom chat
20+
models, `model.bind(tools=...)` instead of `bind_tools`) forward raw
21+
`BaseTool` instances instead, which would otherwise be serialized as their
22+
attribute dict (`args_schema`, `return_direct`, ...) and not be recognized
23+
as tools by the Langfuse UI.
24+
"""
25+
if (
26+
isinstance(tool, dict)
27+
and tool.get("type") == "function"
28+
and isinstance(tool.get("function"), dict)
29+
):
30+
return tool
31+
32+
try:
33+
from langchain_core.utils.function_calling import convert_to_openai_tool
34+
35+
return convert_to_openai_tool(tool)
36+
except Exception:
37+
langfuse_logger.debug(
38+
"Could not normalize tool definition to OpenAI format; keeping original.",
39+
exc_info=True,
40+
)
41+
return tool
42+
43+
1244
def _extract_model_name(
1345
serialized: Optional[Dict[str, Any]],
1446
**kwargs: Any,

‎tests/unit/test_langchain.py‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,14 @@
99
from langchain_core.output_parsers import StrOutputParser
1010
from langchain_core.outputs import ChatGeneration, ChatResult, Generation, LLMResult
1111
from langchain_core.prompts import ChatPromptTemplate
12+
from langchain_core.tools import StructuredTool
1213
from langchain_openai import ChatOpenAI, OpenAI
1314
from opentelemetry import context as otel_context
15+
from pydantic import BaseModel, Field
1416

1517
from langfuse._client.attributes import LangfuseOtelSpanAttributes
1618
from langfuse.langchain import CallbackHandler
19+
from langfuse.langchain.utils import _normalize_tool_definition
1720

1821
callback_handler_module = importlib.import_module("langfuse.langchain.CallbackHandler")
1922

@@ -940,3 +943,117 @@ def import_without_langgraph_command(
940943
)
941944

942945
importlib.reload(callback_handler_module)
946+
947+
948+
class _ExampleToolArgs(BaseModel):
949+
query: str = Field(description="What to search for")
950+
951+
952+
def _make_example_tool() -> StructuredTool:
953+
return StructuredTool.from_function(
954+
func=lambda query: query,
955+
name="example_tool",
956+
description="Example tool description",
957+
args_schema=_ExampleToolArgs,
958+
)
959+
960+
961+
def _invoke_chat_model_with_tools(langfuse_memory_client, bind):
962+
response = ChatResult(
963+
generations=[ChatGeneration(message=AIMessage(content="ok"), text="ok")],
964+
llm_output={"token_usage": {}, "model_name": "gpt-4o-mini"},
965+
)
966+
967+
with patch.object(ChatOpenAI, "_generate", return_value=response):
968+
handler = CallbackHandler()
969+
970+
with langfuse_memory_client.start_as_current_observation(name="parent"):
971+
bind(ChatOpenAI(api_key="test", temperature=0)).invoke(
972+
[HumanMessage(content="hello")],
973+
config={"callbacks": [handler]},
974+
)
975+
976+
langfuse_memory_client.flush()
977+
978+
979+
def _tool_definition_messages(generation_span, json_attr):
980+
input_messages = json_attr(
981+
generation_span, LangfuseOtelSpanAttributes.OBSERVATION_INPUT
982+
)
983+
return [message for message in input_messages if message.get("role") == "tool"]
984+
985+
986+
def test_normalize_tool_definition_passes_openai_format_through():
987+
openai_tool = {
988+
"type": "function",
989+
"function": {"name": "example_tool", "parameters": {"type": "object"}},
990+
}
991+
992+
assert _normalize_tool_definition(openai_tool) is openai_tool
993+
994+
995+
def test_normalize_tool_definition_converts_base_tool_instances():
996+
normalized = _normalize_tool_definition(_make_example_tool())
997+
998+
assert normalized["type"] == "function"
999+
assert normalized["function"]["name"] == "example_tool"
1000+
assert normalized["function"]["description"] == "Example tool description"
1001+
assert "query" in normalized["function"]["parameters"]["properties"]
1002+
1003+
1004+
def test_normalize_tool_definition_wraps_bare_function_schemas():
1005+
bare_function = {
1006+
"name": "example_tool",
1007+
"description": "Example tool description",
1008+
"parameters": {"type": "object", "properties": {}},
1009+
}
1010+
1011+
normalized = _normalize_tool_definition(bare_function)
1012+
1013+
assert normalized["type"] == "function"
1014+
assert normalized["function"]["name"] == "example_tool"
1015+
1016+
1017+
def test_normalize_tool_definition_returns_unconvertible_values_unchanged():
1018+
unconvertible = object()
1019+
1020+
assert _normalize_tool_definition(unconvertible) is unconvertible
1021+
1022+
1023+
def test_raw_bound_tools_are_normalized_in_generation_input(
1024+
langfuse_memory_client, get_span, json_attr
1025+
):
1026+
# .bind() (unlike .bind_tools()) forwards raw tool objects into
1027+
# invocation_params -- the path that produced unrenderable tool
1028+
# definitions in the Langfuse UI.
1029+
_invoke_chat_model_with_tools(
1030+
langfuse_memory_client, lambda model: model.bind(tools=[_make_example_tool()])
1031+
)
1032+
1033+
tool_messages = _tool_definition_messages(get_span("ChatOpenAI"), json_attr)
1034+
1035+
assert len(tool_messages) == 1
1036+
tool_definition = tool_messages[0]["content"]
1037+
assert tool_definition["type"] == "function"
1038+
assert tool_definition["function"]["name"] == "example_tool"
1039+
assert "query" in tool_definition["function"]["parameters"]["properties"], (
1040+
"args schema should be exposed as OpenAI-style parameters"
1041+
)
1042+
1043+
1044+
def test_bind_tools_tool_definitions_stay_in_openai_format(
1045+
langfuse_memory_client, get_span, json_attr
1046+
):
1047+
# Regression guard for the standard path: bind_tools() already provides
1048+
# OpenAI-format definitions, which must pass through unchanged.
1049+
_invoke_chat_model_with_tools(
1050+
langfuse_memory_client, lambda model: model.bind_tools([_make_example_tool()])
1051+
)
1052+
1053+
tool_messages = _tool_definition_messages(get_span("ChatOpenAI"), json_attr)
1054+
1055+
assert len(tool_messages) == 1
1056+
tool_definition = tool_messages[0]["content"]
1057+
assert tool_definition["type"] == "function"
1058+
assert tool_definition["function"]["name"] == "example_tool"
1059+
assert "query" in tool_definition["function"]["parameters"]["properties"]

0 commit comments

Comments
 (0)