fix(openai): capture server tool output items (#700)

* fix(openai): capture server tool output items

Assisted-by: Codex

* chore(openai): add changelog fragment

Assisted-by: Codex

* fix(openai): address server tool review feedback

Assisted-by: Codex

Author: lmolkova

instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/700.fixed

@@ -0,0 +1 @@
+Capture provider-executed Responses API tools in output messages.

instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py

@@ -26,6 +26,7 @@
 
 if TYPE_CHECKING:
     from openai.types.responses.response import Response
+    from openai.types.responses.response_output_item import ResponseOutputItem
     from openai.types.responses.response_usage import ResponseUsage
     from openai.types.responses.tool_param import ToolParam
 
@@ -71,6 +72,8 @@
         OutputMessage,
         ReasoningPart,
         Role,
+        ServerToolCallPart,
+        ServerToolCallResponsePart,
         TextPart,
     )
     from opentelemetry.util.genai.types import (
@@ -84,6 +87,8 @@
     OutputMessage = None
     ReasoningPart = None
     Role = None
+    ServerToolCallPart = None
+    ServerToolCallResponsePart = None
     TextPart = None
     ToolCall = None
 
@@ -298,6 +303,97 @@ def _extract_reasoning_parts(
     return parts
 
 
+_SERVER_TOOL_NAMES = {
+    "code_interpreter_call": "code_interpreter",
+    "file_search_call": "file_search",
+    "image_generation_call": "image_generation",
+    "mcp_call": "mcp",
+    "mcp_list_tools": "mcp_list_tools",
+    "tool_search_call": "tool_search",
+    "web_search_call": "web_search",
+}
+
+_SERVER_TOOL_RESPONSE_NAMES = {
+    "tool_search_output": "tool_search",
+}
+
+
+def _extract_server_tool_part(
+    item: ResponseOutputItem,
+) -> tuple[ServerToolCallPart | ServerToolCallResponsePart, str] | None:
+    if ServerToolCallPart is None or ServerToolCallResponsePart is None:
+        return None
+
+    item_type = item.type
+    tool_name = _SERVER_TOOL_NAMES.get(item_type)
+    response_name = _SERVER_TOOL_RESPONSE_NAMES.get(item_type)
+    if tool_name is None and response_name is None:
+        return None
+    if (
+        item_type
+        in (
+            "tool_search_call",
+            "tool_search_output",
+        )
+        and item.execution != "server"
+    ):
+        return None
+
+    finish_reason = _server_tool_finish_reason(item)
+    if finish_reason is None:
+        return None
+
+    payload = item.model_dump(exclude_none=True, mode="json")
+
+    item_id = payload.pop("id", None)
+    call_id = payload.pop("call_id", None)
+    part_id = call_id if isinstance(call_id, str) else item_id
+    payload.pop("type", None)
+    name = payload.pop("name", None)
+    canonical_name = tool_name or response_name
+    if canonical_name is None:
+        return None
+    payload["type"] = canonical_name
+    if response_name is not None:
+        return (
+            ServerToolCallResponsePart(
+                server_tool_call_response=payload,
+                id=call_id if isinstance(call_id, str) else None,
+            ),
+            finish_reason,
+        )
+    return (
+        ServerToolCallPart(
+            name=name if isinstance(name, str) else canonical_name,
+            server_tool_call=payload,
+            id=part_id if isinstance(part_id, str) else None,
+        ),
+        finish_reason,
+    )
+
+
+def _server_tool_finish_reason(item: ResponseOutputItem) -> str | None:
+    match item.type:
+        case "mcp_list_tools":
+            return "error" if item.error else "stop"
+        case (
+            "code_interpreter_call"
+            | "file_search_call"
+            | "image_generation_call"
+            | "mcp_call"
+            | "tool_search_call"
+            | "tool_search_output"
+            | "web_search_call"
+        ):
+            return (
+                _finish_reason_from_status(item.status)
+                if item.status is not None
+                else None
+            )
+        case _:
+            return None
+
+
 # `incomplete_details.reason` values that map onto a cross-provider finish
 # reason; an unrecognized reason is reported as-is.
 _INCOMPLETE_REASON_TO_FINISH_REASON = {
@@ -448,6 +544,17 @@ def get_output_messages_from_response(
                         finish_reason=finish_reason,
                     )
                 )
+            continue
+
+        if server_tool := _extract_server_tool_part(item):
+            server_tool_part, finish_reason = server_tool
+            messages.append(
+                OutputMessage(
+                    role=Role.ASSISTANT.value,
+                    parts=[server_tool_part],
+                    finish_reason=finish_reason,
+                )
+            )
 
     return messages
 

instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/responses_server_tool_calling_conformance.yaml

@@ -0,0 +1,77 @@
+# TODO: this is generated by AI, re-record
+interactions:
+- request:
+    body: |-
+      {"input":"Search for OpenTelemetry.","model":"gpt-4o-mini","tools":[{"type":"web_search"}]}
+    headers:
+      Accept:
+      - application/json
+      Content-Type:
+      - application/json
+      Host:
+      - api.openai.com
+      authorization:
+      - Bearer test_openai_api_key
+    method: POST
+    uri: https://api.openai.com/v1/responses
+  response:
+    body:
+      string: |-
+        {
+          "id": "resp_server_tool",
+          "object": "response",
+          "created_at": 1776481259,
+          "status": "completed",
+          "error": null,
+          "incomplete_details": null,
+          "instructions": null,
+          "max_output_tokens": null,
+          "model": "gpt-4o-mini-2024-07-18",
+          "output": [
+            {
+              "id": "ws_1",
+              "type": "web_search_call",
+              "status": "completed",
+              "action": {
+                "type": "search",
+                "query": "OpenTelemetry"
+              }
+            },
+            {
+              "id": "msg_1",
+              "type": "message",
+              "status": "completed",
+              "content": [
+                {
+                  "type": "output_text",
+                  "annotations": [],
+                  "text": "OpenTelemetry is an observability framework."
+                }
+              ],
+              "role": "assistant"
+            }
+          ],
+          "parallel_tool_calls": true,
+          "previous_response_id": null,
+          "service_tier": "default",
+          "store": true,
+          "temperature": 1.0,
+          "tool_choice": "auto",
+          "tools": [],
+          "top_p": 1.0,
+          "usage": {
+            "input_tokens": 20,
+            "input_tokens_details": {"cached_tokens": 0},
+            "output_tokens": 15,
+            "output_tokens_details": {"reasoning_tokens": 0},
+            "total_tokens": 35
+          },
+          "metadata": {}
+        }
+    headers:
+      Content-Type:
+      - application/json
+    status:
+      code: 200
+      message: OK
+version: 1

instrumentation/opentelemetry-instrumentation-genai-openai/tests/conformance/responses_server_tool_calling.py

@@ -0,0 +1,77 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Conformance scenario: OpenAI Responses API hosted tool calling."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from openai import OpenAI
+
+from opentelemetry.instrumentation.genai.openai import OpenAIInstrumentor
+from opentelemetry.sdk._logs import LoggerProvider
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.test.weaver_live_check import LiveCheckReport
+from opentelemetry.test_util_genai.conformance import Scenario
+from opentelemetry.test_util_genai.instrumentor import instrument
+
+
+class ResponsesServerToolCallingScenario(Scenario):
+    expected_spans = {"chat": 1}
+    expected_metrics = (
+        "gen_ai.client.operation.duration",
+        "gen_ai.client.token.usage",
+    )
+
+    def validate(self, report: LiveCheckReport) -> None:
+        super().validate(report)
+        output_messages = [
+            json.loads(attribute["value"])
+            for entry in report["samples"]
+            if "span" in entry
+            for attribute in entry["span"]["attributes"]
+            if attribute["name"] == "gen_ai.output.messages"
+        ]
+        assert len(output_messages) == 1
+        assert output_messages[0][0]["parts"] == [
+            {
+                "name": "web_search",
+                "server_tool_call": {
+                    "action": {
+                        "query": "OpenTelemetry",
+                        "type": "search",
+                    },
+                    "status": "completed",
+                    "type": "web_search",
+                },
+                "id": "ws_1",
+                "type": "server_tool_call",
+            }
+        ]
+
+    def run(
+        self,
+        *,
+        tracer_provider: TracerProvider,
+        meter_provider: MeterProvider,
+        logger_provider: LoggerProvider,
+        vcr: Any,
+    ) -> None:
+        with instrument(
+            OpenAIInstrumentor(),
+            tracer_provider=tracer_provider,
+            logger_provider=logger_provider,
+            meter_provider=meter_provider,
+            content_capture="SPAN_ONLY",
+        ):
+            with vcr.use_cassette(
+                "responses_server_tool_calling_conformance.yaml"
+            ):
+                OpenAI().responses.create(
+                    model="gpt-4o-mini",
+                    input="Search for OpenTelemetry.",
+                    tools=[{"type": "web_search"}],
+                )

instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_conformance.py

@@ -32,6 +32,9 @@
 )
 from .conformance.responses_conversation import ResponsesConversationScenario
 from .conformance.responses_fetch import ResponsesFetchScenario
+from .conformance.responses_server_tool_calling import (
+    ResponsesServerToolCallingScenario,
+)
 from .conformance.responses_stream import ResponsesStreamScenario
 from .conformance.responses_streaming import ResponsesStreamingScenario
 from .conformance.responses_tool_calling import (
@@ -54,6 +57,7 @@
         ResponsesStreamScenario(),
         ResponsesStreamingScenario(),
         ResponsesToolCallingScenario(),
+        ResponsesServerToolCallingScenario(),
     ],
     ids=lambda s: type(s).__name__,
 )