|
9 | 9 | from langchain_core.output_parsers import StrOutputParser |
10 | 10 | from langchain_core.outputs import ChatGeneration, ChatResult, Generation, LLMResult |
11 | 11 | from langchain_core.prompts import ChatPromptTemplate |
| 12 | +from langchain_core.tools import StructuredTool |
12 | 13 | from langchain_openai import ChatOpenAI, OpenAI |
13 | 14 | from opentelemetry import context as otel_context |
| 15 | +from pydantic import BaseModel, Field |
14 | 16 |
|
15 | 17 | from langfuse._client.attributes import LangfuseOtelSpanAttributes |
16 | 18 | from langfuse.langchain import CallbackHandler |
| 19 | +from langfuse.langchain.utils import _normalize_tool_definition |
17 | 20 |
|
18 | 21 | callback_handler_module = importlib.import_module("langfuse.langchain.CallbackHandler") |
19 | 22 |
|
@@ -940,3 +943,117 @@ def import_without_langgraph_command( |
940 | 943 | ) |
941 | 944 |
|
942 | 945 | importlib.reload(callback_handler_module) |
| 946 | + |
| 947 | + |
| 948 | +class _ExampleToolArgs(BaseModel): |
| 949 | + query: str = Field(description="What to search for") |
| 950 | + |
| 951 | + |
| 952 | +def _make_example_tool() -> StructuredTool: |
| 953 | + return StructuredTool.from_function( |
| 954 | + func=lambda query: query, |
| 955 | + name="example_tool", |
| 956 | + description="Example tool description", |
| 957 | + args_schema=_ExampleToolArgs, |
| 958 | + ) |
| 959 | + |
| 960 | + |
| 961 | +def _invoke_chat_model_with_tools(langfuse_memory_client, bind): |
| 962 | + response = ChatResult( |
| 963 | + generations=[ChatGeneration(message=AIMessage(content="ok"), text="ok")], |
| 964 | + llm_output={"token_usage": {}, "model_name": "gpt-4o-mini"}, |
| 965 | + ) |
| 966 | + |
| 967 | + with patch.object(ChatOpenAI, "_generate", return_value=response): |
| 968 | + handler = CallbackHandler() |
| 969 | + |
| 970 | + with langfuse_memory_client.start_as_current_observation(name="parent"): |
| 971 | + bind(ChatOpenAI(api_key="test", temperature=0)).invoke( |
| 972 | + [HumanMessage(content="hello")], |
| 973 | + config={"callbacks": [handler]}, |
| 974 | + ) |
| 975 | + |
| 976 | + langfuse_memory_client.flush() |
| 977 | + |
| 978 | + |
| 979 | +def _tool_definition_messages(generation_span, json_attr): |
| 980 | + input_messages = json_attr( |
| 981 | + generation_span, LangfuseOtelSpanAttributes.OBSERVATION_INPUT |
| 982 | + ) |
| 983 | + return [message for message in input_messages if message.get("role") == "tool"] |
| 984 | + |
| 985 | + |
| 986 | +def test_normalize_tool_definition_passes_openai_format_through(): |
| 987 | + openai_tool = { |
| 988 | + "type": "function", |
| 989 | + "function": {"name": "example_tool", "parameters": {"type": "object"}}, |
| 990 | + } |
| 991 | + |
| 992 | + assert _normalize_tool_definition(openai_tool) is openai_tool |
| 993 | + |
| 994 | + |
| 995 | +def test_normalize_tool_definition_converts_base_tool_instances(): |
| 996 | + normalized = _normalize_tool_definition(_make_example_tool()) |
| 997 | + |
| 998 | + assert normalized["type"] == "function" |
| 999 | + assert normalized["function"]["name"] == "example_tool" |
| 1000 | + assert normalized["function"]["description"] == "Example tool description" |
| 1001 | + assert "query" in normalized["function"]["parameters"]["properties"] |
| 1002 | + |
| 1003 | + |
| 1004 | +def test_normalize_tool_definition_wraps_bare_function_schemas(): |
| 1005 | + bare_function = { |
| 1006 | + "name": "example_tool", |
| 1007 | + "description": "Example tool description", |
| 1008 | + "parameters": {"type": "object", "properties": {}}, |
| 1009 | + } |
| 1010 | + |
| 1011 | + normalized = _normalize_tool_definition(bare_function) |
| 1012 | + |
| 1013 | + assert normalized["type"] == "function" |
| 1014 | + assert normalized["function"]["name"] == "example_tool" |
| 1015 | + |
| 1016 | + |
| 1017 | +def test_normalize_tool_definition_returns_unconvertible_values_unchanged(): |
| 1018 | + unconvertible = object() |
| 1019 | + |
| 1020 | + assert _normalize_tool_definition(unconvertible) is unconvertible |
| 1021 | + |
| 1022 | + |
| 1023 | +def test_raw_bound_tools_are_normalized_in_generation_input( |
| 1024 | + langfuse_memory_client, get_span, json_attr |
| 1025 | +): |
| 1026 | + # .bind() (unlike .bind_tools()) forwards raw tool objects into |
| 1027 | + # invocation_params -- the path that produced unrenderable tool |
| 1028 | + # definitions in the Langfuse UI. |
| 1029 | + _invoke_chat_model_with_tools( |
| 1030 | + langfuse_memory_client, lambda model: model.bind(tools=[_make_example_tool()]) |
| 1031 | + ) |
| 1032 | + |
| 1033 | + tool_messages = _tool_definition_messages(get_span("ChatOpenAI"), json_attr) |
| 1034 | + |
| 1035 | + assert len(tool_messages) == 1 |
| 1036 | + tool_definition = tool_messages[0]["content"] |
| 1037 | + assert tool_definition["type"] == "function" |
| 1038 | + assert tool_definition["function"]["name"] == "example_tool" |
| 1039 | + assert "query" in tool_definition["function"]["parameters"]["properties"], ( |
| 1040 | + "args schema should be exposed as OpenAI-style parameters" |
| 1041 | + ) |
| 1042 | + |
| 1043 | + |
| 1044 | +def test_bind_tools_tool_definitions_stay_in_openai_format( |
| 1045 | + langfuse_memory_client, get_span, json_attr |
| 1046 | +): |
| 1047 | + # Regression guard for the standard path: bind_tools() already provides |
| 1048 | + # OpenAI-format definitions, which must pass through unchanged. |
| 1049 | + _invoke_chat_model_with_tools( |
| 1050 | + langfuse_memory_client, lambda model: model.bind_tools([_make_example_tool()]) |
| 1051 | + ) |
| 1052 | + |
| 1053 | + tool_messages = _tool_definition_messages(get_span("ChatOpenAI"), json_attr) |
| 1054 | + |
| 1055 | + assert len(tool_messages) == 1 |
| 1056 | + tool_definition = tool_messages[0]["content"] |
| 1057 | + assert tool_definition["type"] == "function" |
| 1058 | + assert tool_definition["function"]["name"] == "example_tool" |
| 1059 | + assert "query" in tool_definition["function"]["parameters"]["properties"] |
0 commit comments