From c73ca3cc903f7c3aec9b5958cb85f94d282acf00 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Sat, 12 Sep 2026 12:53:49 -0700 Subject: [PATCH 1/3] fix(openai): capture server tool output items Assisted-by: Codex --- .../genai/openai/response_extractors.py | 76 ++++++++++ ...onses_server_tool_calling_conformance.yaml | 77 ++++++++++ .../responses_server_tool_calling.py | 77 ++++++++++ .../tests/test_conformance.py | 4 + .../tests/test_response_extractors.py | 140 ++++++++++++++++++ 5 files changed, 374 insertions(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/responses_server_tool_calling_conformance.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_server_tool_calling.py diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py index a1e40b9ed..8334734fc 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py @@ -71,6 +71,8 @@ OutputMessage, ReasoningPart, Role, + ServerToolCallPart, + ServerToolCallResponsePart, TextPart, ) from opentelemetry.util.genai.types import ( @@ -84,6 +86,8 @@ OutputMessage = None ReasoningPart = None Role = None + ServerToolCallPart = None + ServerToolCallResponsePart = None TextPart = None ToolCall = None @@ -298,6 +302,68 @@ def _extract_reasoning_parts( return parts +_SERVER_TOOL_NAMES = { + "code_interpreter_call": "code_interpreter", + "file_search_call": "file_search", + "image_generation_call": "image_generation", + "mcp_call": "mcp", + "mcp_list_tools": "mcp_list_tools", + "tool_search_call": "tool_search", + "web_search_call": "web_search", +} + +_SERVER_TOOL_RESPONSE_NAMES = { + "tool_search_output": "tool_search", +} + + +def _extract_server_tool_part( + item: object, +) -> ServerToolCallPart | ServerToolCallResponsePart | None: + if ServerToolCallPart is None or ServerToolCallResponsePart is None: + return None + + item_type = _get_field(item, "type") + if not isinstance(item_type, str): + return None + tool_name = _SERVER_TOOL_NAMES.get(item_type) + response_name = _SERVER_TOOL_RESPONSE_NAMES.get(item_type) + if tool_name is None and response_name is None: + return None + if ( + item_type.startswith("tool_search_") + and _get_field(item, "execution") != "server" + ): + return None + + if isinstance(item, Mapping): + payload = dict(item) + else: + model_dump = getattr(item, "model_dump", None) + if not callable(model_dump): + return None + payload = model_dump(exclude_none=True, mode="json") + + item_id = payload.pop("id", None) + payload.pop("type", None) + name = payload.pop("name", None) + canonical_name = tool_name or response_name + if canonical_name is None: + return None + payload["type"] = canonical_name + if response_name is not None: + call_id = payload.pop("call_id", None) + return ServerToolCallResponsePart( + server_tool_call_response=payload, + id=call_id if isinstance(call_id, str) else None, + ) + return ServerToolCallPart( + name=name if isinstance(name, str) else canonical_name, + server_tool_call=payload, + id=item_id if isinstance(item_id, str) else None, + ) + + # `incomplete_details.reason` values that map onto a cross-provider finish # reason; an unrecognized reason is reported as-is. _INCOMPLETE_REASON_TO_FINISH_REASON = { @@ -448,6 +514,16 @@ def get_output_messages_from_response( finish_reason=finish_reason, ) ) + continue + + if server_tool_part := _extract_server_tool_part(item): + messages.append( + OutputMessage( + role=Role.ASSISTANT.value, + parts=[server_tool_part], + finish_reason="stop", + ) + ) return messages diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/responses_server_tool_calling_conformance.yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/responses_server_tool_calling_conformance.yaml new file mode 100644 index 000000000..ac411c5a9 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/responses_server_tool_calling_conformance.yaml @@ -0,0 +1,77 @@ +# TODO: this is generated by AI, re-record +interactions: +- request: + body: |- + {"input":"Search for OpenTelemetry.","model":"gpt-4o-mini","tools":[{"type":"web_search"}]} + headers: + Accept: + - application/json + Content-Type: + - application/json + Host: + - api.openai.com + authorization: + - Bearer test_openai_api_key + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: |- + { + "id": "resp_server_tool", + "object": "response", + "created_at": 1776481259, + "status": "completed", + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "ws_1", + "type": "web_search_call", + "status": "completed", + "action": { + "type": "search", + "query": "OpenTelemetry" + } + }, + { + "id": "msg_1", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "text": "OpenTelemetry is an observability framework." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "usage": { + "input_tokens": 20, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 15, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 35 + }, + "metadata": {} + } + headers: + Content-Type: + - application/json + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_server_tool_calling.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_server_tool_calling.py new file mode 100644 index 000000000..79862b2d3 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_server_tool_calling.py @@ -0,0 +1,77 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: OpenAI Responses API hosted tool calling.""" + +from __future__ import annotations + +import json +from typing import Any + +from openai import OpenAI + +from opentelemetry.instrumentation.genai.openai import OpenAIInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + + +class ResponsesServerToolCallingScenario(Scenario): + expected_spans = {"chat": 1} + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + output_messages = [ + json.loads(attribute["value"]) + for entry in report["samples"] + if "span" in entry + for attribute in entry["span"]["attributes"] + if attribute["name"] == "gen_ai.output.messages" + ] + assert len(output_messages) == 1 + assert output_messages[0][0]["parts"] == [ + { + "name": "web_search", + "server_tool_call": { + "action": { + "query": "OpenTelemetry", + "type": "search", + }, + "status": "completed", + "type": "web_search", + }, + "id": "ws_1", + "type": "server_tool_call", + } + ] + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + OpenAIInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette( + "responses_server_tool_calling_conformance.yaml" + ): + OpenAI().responses.create( + model="gpt-4o-mini", + input="Search for OpenTelemetry.", + tools=[{"type": "web_search"}], + ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py index 2d9158c04..3603af463 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py @@ -32,6 +32,9 @@ ) from .conformance.responses_conversation import ResponsesConversationScenario from .conformance.responses_fetch import ResponsesFetchScenario +from .conformance.responses_server_tool_calling import ( + ResponsesServerToolCallingScenario, +) from .conformance.responses_stream import ResponsesStreamScenario from .conformance.responses_streaming import ResponsesStreamingScenario from .conformance.responses_tool_calling import ( @@ -54,6 +57,7 @@ ResponsesStreamScenario(), ResponsesStreamingScenario(), ResponsesToolCallingScenario(), + ResponsesServerToolCallingScenario(), ], ids=lambda s: type(s).__name__, ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py index b85efc935..d79763f59 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py @@ -19,6 +19,8 @@ FunctionToolDefinition, GenericToolDefinition, LLMInvocation, + ServerToolCallPart, + ServerToolCallResponsePart, TextPart, UriPart, ) @@ -335,6 +337,144 @@ def test_extract_output_messages_maps_parts_and_finish_reasons(loaded_module): assert messages[3].parts[0].content == "Thought step" +@pytest.mark.parametrize( + ("item", "expected_name", "expected_payload"), + [ + ( + { + "id": "fs_1", + "type": "file_search_call", + "status": "completed", + "queries": ["OpenTelemetry"], + "results": [], + }, + "file_search", + { + "type": "file_search", + "status": "completed", + "queries": ["OpenTelemetry"], + "results": [], + }, + ), + ( + { + "id": "ws_1", + "type": "web_search_call", + "status": "completed", + "action": {"type": "search", "query": "OpenTelemetry"}, + }, + "web_search", + { + "type": "web_search", + "status": "completed", + "action": {"type": "search", "query": "OpenTelemetry"}, + }, + ), + ( + { + "id": "ci_1", + "type": "code_interpreter_call", + "status": "completed", + "code": "print(1)", + "container_id": "container_1", + "outputs": [{"type": "logs", "logs": "1"}], + }, + "code_interpreter", + { + "type": "code_interpreter", + "status": "completed", + "code": "print(1)", + "container_id": "container_1", + "outputs": [{"type": "logs", "logs": "1"}], + }, + ), + ( + { + "id": "mcp_1", + "type": "mcp_call", + "status": "completed", + "name": "get_weather", + "server_label": "weather", + "arguments": '{"city":"Seattle"}', + "output": "rain", + }, + "get_weather", + { + "type": "mcp", + "status": "completed", + "server_label": "weather", + "arguments": '{"city":"Seattle"}', + "output": "rain", + }, + ), + ], +) +def test_extract_output_messages_maps_server_tools( + loaded_module, item, expected_name, expected_payload +): + try: + response = _make_response(output=[item]) + except ValueError: + pytest.skip("server tool output type is unavailable in this SDK") + + messages = loaded_module.get_output_messages_from_response(response) + + assert len(messages) == 1 + assert messages[0].finish_reason == "stop" + assert len(messages[0].parts) == 1 + part = messages[0].parts[0] + assert isinstance(part, ServerToolCallPart) + assert part.id == item["id"] + assert part.name == expected_name + assert part.server_tool_call == expected_payload + + +def test_extract_output_messages_maps_server_tool_search_result(loaded_module): + item = { + "id": "ts_2", + "type": "tool_search_output", + "call_id": "ts_1", + "execution": "server", + "status": "completed", + "tools": [], + } + try: + response = _make_response(output=[item]) + except ValueError: + pytest.skip("server tool search output is unavailable in this SDK") + + messages = loaded_module.get_output_messages_from_response(response) + + part = messages[0].parts[0] + assert isinstance(part, ServerToolCallResponsePart) + assert part.id == "ts_1" + assert part.server_tool_call_response == { + "execution": "server", + "status": "completed", + "tools": [], + "type": "tool_search", + } + + +def test_extract_output_messages_does_not_classify_client_tool_search( + loaded_module, +): + item = { + "id": "ts_1", + "type": "tool_search_call", + "call_id": "call_1", + "execution": "client", + "status": "completed", + "arguments": {}, + } + try: + response = _make_response(output=[item]) + except ValueError: + pytest.skip("tool search output is unavailable in this SDK") + + assert loaded_module.get_output_messages_from_response(response) == [] + + def test_extract_finish_reasons_maps_terminal_message_and_tool_items( loaded_module, ): From 2ad11d529dc9b9dc84f1d3527aa077ad739f210f Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Sat, 12 Sep 2026 12:54:13 -0700 Subject: [PATCH 2/3] chore(openai): add changelog fragment Assisted-by: Codex --- .../.changelog/700.fixed | 1 + 1 file changed, 1 insertion(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/700.fixed diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/700.fixed b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/700.fixed new file mode 100644 index 000000000..1bc574f14 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/700.fixed @@ -0,0 +1 @@ +Capture provider-executed Responses API tools in output messages. From f9c29f99a951d65cadc7d65e7d50fd5f8ad460b7 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Sat, 12 Sep 2026 13:10:29 -0700 Subject: [PATCH 3/3] fix(openai): address server tool review feedback Assisted-by: Codex --- .../genai/openai/response_extractors.py | 79 +++++++++---- .../tests/test_response_extractors.py | 111 ++++++++++++++++-- 2 files changed, 153 insertions(+), 37 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py index 8334734fc..3878b09e7 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py @@ -26,6 +26,7 @@ if TYPE_CHECKING: from openai.types.responses.response import Response + from openai.types.responses.response_output_item import ResponseOutputItem from openai.types.responses.response_usage import ResponseUsage from openai.types.responses.tool_param import ToolParam @@ -318,33 +319,35 @@ def _extract_reasoning_parts( def _extract_server_tool_part( - item: object, -) -> ServerToolCallPart | ServerToolCallResponsePart | None: + item: ResponseOutputItem, +) -> tuple[ServerToolCallPart | ServerToolCallResponsePart, str] | None: if ServerToolCallPart is None or ServerToolCallResponsePart is None: return None - item_type = _get_field(item, "type") - if not isinstance(item_type, str): - return None + item_type = item.type tool_name = _SERVER_TOOL_NAMES.get(item_type) response_name = _SERVER_TOOL_RESPONSE_NAMES.get(item_type) if tool_name is None and response_name is None: return None if ( - item_type.startswith("tool_search_") - and _get_field(item, "execution") != "server" + item_type + in ( + "tool_search_call", + "tool_search_output", + ) + and item.execution != "server" ): return None - if isinstance(item, Mapping): - payload = dict(item) - else: - model_dump = getattr(item, "model_dump", None) - if not callable(model_dump): - return None - payload = model_dump(exclude_none=True, mode="json") + finish_reason = _server_tool_finish_reason(item) + if finish_reason is None: + return None + + payload = item.model_dump(exclude_none=True, mode="json") item_id = payload.pop("id", None) + call_id = payload.pop("call_id", None) + part_id = call_id if isinstance(call_id, str) else item_id payload.pop("type", None) name = payload.pop("name", None) canonical_name = tool_name or response_name @@ -352,18 +355,45 @@ def _extract_server_tool_part( return None payload["type"] = canonical_name if response_name is not None: - call_id = payload.pop("call_id", None) - return ServerToolCallResponsePart( - server_tool_call_response=payload, - id=call_id if isinstance(call_id, str) else None, + return ( + ServerToolCallResponsePart( + server_tool_call_response=payload, + id=call_id if isinstance(call_id, str) else None, + ), + finish_reason, ) - return ServerToolCallPart( - name=name if isinstance(name, str) else canonical_name, - server_tool_call=payload, - id=item_id if isinstance(item_id, str) else None, + return ( + ServerToolCallPart( + name=name if isinstance(name, str) else canonical_name, + server_tool_call=payload, + id=part_id if isinstance(part_id, str) else None, + ), + finish_reason, ) +def _server_tool_finish_reason(item: ResponseOutputItem) -> str | None: + match item.type: + case "mcp_list_tools": + return "error" if item.error else "stop" + case ( + "code_interpreter_call" + | "file_search_call" + | "image_generation_call" + | "mcp_call" + | "tool_search_call" + | "tool_search_output" + | "web_search_call" + ): + return ( + _finish_reason_from_status(item.status) + if item.status is not None + else None + ) + case _: + return None + + # `incomplete_details.reason` values that map onto a cross-provider finish # reason; an unrecognized reason is reported as-is. _INCOMPLETE_REASON_TO_FINISH_REASON = { @@ -516,12 +546,13 @@ def get_output_messages_from_response( ) continue - if server_tool_part := _extract_server_tool_part(item): + if server_tool := _extract_server_tool_part(item): + server_tool_part, finish_reason = server_tool messages.append( OutputMessage( role=Role.ASSISTANT.value, parts=[server_tool_part], - finish_reason="stop", + finish_reason=finish_reason, ) ) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py index d79763f59..5c498f6d9 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_response_extractors.py @@ -407,15 +407,57 @@ def test_extract_output_messages_maps_parts_and_finish_reasons(loaded_module): "output": "rain", }, ), + ( + { + "id": "ig_1", + "type": "image_generation_call", + "status": "completed", + "result": "image-data", + }, + "image_generation", + { + "type": "image_generation", + "status": "completed", + "result": "image-data", + }, + ), + ( + { + "id": "mcp_list_1", + "type": "mcp_list_tools", + "server_label": "weather", + "tools": [], + }, + "mcp_list_tools", + { + "type": "mcp_list_tools", + "server_label": "weather", + "tools": [], + }, + ), + ( + { + "id": "ts_item_1", + "type": "tool_search_call", + "call_id": "ts_call_1", + "execution": "server", + "status": "completed", + "arguments": {"query": "weather"}, + }, + "tool_search", + { + "type": "tool_search", + "execution": "server", + "status": "completed", + "arguments": {"query": "weather"}, + }, + ), ], ) def test_extract_output_messages_maps_server_tools( loaded_module, item, expected_name, expected_payload ): - try: - response = _make_response(output=[item]) - except ValueError: - pytest.skip("server tool output type is unavailable in this SDK") + response = _make_response(output=[item]) messages = loaded_module.get_output_messages_from_response(response) @@ -424,7 +466,7 @@ def test_extract_output_messages_maps_server_tools( assert len(messages[0].parts) == 1 part = messages[0].parts[0] assert isinstance(part, ServerToolCallPart) - assert part.id == item["id"] + assert part.id == item.get("call_id", item["id"]) assert part.name == expected_name assert part.server_tool_call == expected_payload @@ -438,10 +480,7 @@ def test_extract_output_messages_maps_server_tool_search_result(loaded_module): "status": "completed", "tools": [], } - try: - response = _make_response(output=[item]) - except ValueError: - pytest.skip("server tool search output is unavailable in this SDK") + response = _make_response(output=[item]) messages = loaded_module.get_output_messages_from_response(response) @@ -467,14 +506,60 @@ def test_extract_output_messages_does_not_classify_client_tool_search( "status": "completed", "arguments": {}, } - try: - response = _make_response(output=[item]) - except ValueError: - pytest.skip("tool search output is unavailable in this SDK") + response = _make_response(output=[item]) assert loaded_module.get_output_messages_from_response(response) == [] +@pytest.mark.parametrize( + ("status", "expected_finish_reason"), + [ + ("in_progress", None), + ("failed", "error"), + ("incomplete", "incomplete"), + ], +) +def test_extract_output_messages_uses_server_tool_status( + loaded_module, status, expected_finish_reason +): + response = _make_response( + output=[ + { + "id": "fs_1", + "type": "file_search_call", + "status": status, + "queries": ["OpenTelemetry"], + "results": [], + } + ] + ) + + messages = loaded_module.get_output_messages_from_response(response) + + if expected_finish_reason is None: + assert messages == [] + else: + assert messages[0].finish_reason == expected_finish_reason + + +def test_extract_output_messages_maps_failed_mcp_list_tools(loaded_module): + response = _make_response( + output=[ + { + "id": "mcp_list_1", + "type": "mcp_list_tools", + "server_label": "weather", + "tools": [], + "error": "unavailable", + } + ] + ) + + messages = loaded_module.get_output_messages_from_response(response) + + assert messages[0].finish_reason == "error" + + def test_extract_finish_reasons_maps_terminal_message_and_tool_items( loaded_module, ):