diff --git a/py/src/braintrust/integrations/litellm/tracing.py b/py/src/braintrust/integrations/litellm/tracing.py index 412fdd8a..48ba8a1f 100644 --- a/py/src/braintrust/integrations/litellm/tracing.py +++ b/py/src/braintrust/integrations/litellm/tracing.py @@ -11,10 +11,10 @@ from braintrust.integrations.openai.tracing import _decisions_input, _decisions_output, _process_attachments_in_input from braintrust.integrations.utils import ( _extract_audio_output, + _image_attachment_from_base64, _materialize_attachment, _parse_openai_usage_metrics, _prettify_response_params, - _ResolvedAttachment, _timing_metrics, _try_to_dict, ) @@ -440,24 +440,6 @@ async def _adecisions_wrapper_async(wrapped, instance, args, kwargs): return response -def _image_attachment_from_base64( - data: Any, *, output_format: Any, index: int -) -> tuple[_ResolvedAttachment | None, int | None]: - if not isinstance(data, str): - return None, None - - extension = output_format if isinstance(output_format, str) and output_format else "png" - mime_type = extension if "/" in extension else f"image/{extension}" - resolved_attachment = _materialize_attachment( - data, - mime_type=mime_type, - prefix=f"generated_image_{index}", - ) - if resolved_attachment is None: - return None, None - return resolved_attachment, len(resolved_attachment.attachment.data) - - def _extract_image_generation_output(response: Any) -> Any: images: list[dict[str, Any]] = [] output_format = _get_field(response, "output_format") diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_images_generate_svg.yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_images_generate_svg.yaml new file mode 100644 index 00000000..eb9f0d78 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_images_generate_svg.yaml @@ -0,0 +1,104 @@ +interactions: +- request: + body: '{"model":"gpt-image-2.6-flare","prompt":"A simple flat icon of a red circle","complexity":"low","n":1,"output_format":"svg","size":"1024x1024","svg":{"model":"gpt-5.6","reasoning":{"effort":"low"}}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '198' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.46.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.46.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/images/generations + response: + body: + string: "{\n \"created\": 1794252924,\n \"background\": \"transparent\",\n + \ \"data\": [\n {\n \"b64_json\": \"PHN2ZyB2aWV3Qm94PSIwIDAgMTAyNCAxMDI0IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIxMDI0IiBoZWlnaHQ9IjEwMjQiPgogIDx0aXRsZT5SZWQgY2lyY2xlIGljb248L3RpdGxlPgogIDxjaXJjbGUgaWQ9InJlZC1jaXJjbGUiIGN4PSI1MTIiIGN5PSI1MTIiIHI9IjQ1OCIgZmlsbD0iI2YwMDAxOCIgLz4KPC9zdmc+\"\n + \ }\n ],\n \"output_format\": \"svg\",\n \"quality\": \"high\",\n \"safety_identifier\": + null,\n \"size\": \"1024x1024\",\n \"svg_usage\": {\n \"input_tokens\": + 1590,\n \"model\": \"gpt-5.6-sol\",\n \"output_tokens\": 71,\n \"reasoning_tokens\": + 0,\n \"total_tokens\": 1661\n },\n \"usage\": {\n \"input_tokens\": + 81,\n \"input_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": + 81\n },\n \"output_tokens\": 1758,\n \"output_tokens_details\": {\n + \ \"image_tokens\": 1758,\n \"text_tokens\": 0\n },\n \"total_tokens\": + 1839\n }\n}" + headers: + access-control-expose-headers: + - X-Request-ID, Server-Timing-Total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a57f4284ac417aa8-YYZ + connection: + - keep-alive + content-length: + - '897' + content-type: + - application/json + date: + - Mon, 09 Nov 2026 19:35:26 GMT + openai-processing-ms: + - '7854' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-input-images: + - '250000' + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-input-images: + - '249999' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '39999867' + x-ratelimit-reset-input-images: + - 0s + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_d9b157b3e93f4c8d8fa4626406bc7132 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_responses_image_generation_svg.yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_responses_image_generation_svg.yaml new file mode 100644 index 00000000..85a693ce --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_responses_image_generation_svg.yaml @@ -0,0 +1,145 @@ +interactions: +- request: + body: '{"model":"gpt-5.6","image":{"context_compression":"auto"},"input":"Draw + a simple flat icon of a red circle.","prompt_cache_options":{"ttl":"12h"},"tool_choice":{"type":"image_generation"},"tools":[{"type":"image_generation","model":"gpt-image-2.6-flare","output_format":"svg","size":"1024x1024","svg":{"model":"gpt-5.6","reasoning":{"effort":"low"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '352' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.46.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.46.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"id\": \"resp_0307e38d3728b13b016af2238664a887d2a8f3eae80d925b27\",\n + \ \"object\": \"response\",\n \"created_at\": 1794253702,\n \"status\": + \"completed\",\n \"access_programs\": null,\n \"background\": false,\n \"billing\": + {\n \"payer\": \"developer\"\n },\n \"completed_at\": 1794253713,\n \"error\": + null,\n \"frequency_penalty\": 0.0,\n \"image\": {\n \"context_compression\": + \"disabled\"\n },\n \"incomplete_details\": null,\n \"instructions\": null,\n + \ \"max_output_tokens\": null,\n \"max_tool_calls\": null,\n \"model\": + \"gpt-5.6-sol\",\n \"moderation\": null,\n \"output\": [\n {\n \"id\": + \"ig_0307e38d3728b13b016af22387433487d28ef6d4262a72d9f8\",\n \"type\": + \"image_generation_call\",\n \"status\": \"completed\",\n \"action\": + \"generate\",\n \"background\": \"transparent\",\n \"output_format\": + \"svg\",\n \"quality\": \"high\",\n \"result\": \"PHN2ZyB2aWV3Qm94PSIwIDAgMTAyNCAxMDI0IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIxMDI0IiBoZWlnaHQ9IjEwMjQiPgogIDxjaXJjbGUgaWQ9InJlZC1jaXJjbGUiIGN4PSI1MTIiIGN5PSI1MTIiIHI9IjM5NSIgZmlsbD0iI2ZmMDAwMCIgLz4KPC9zdmc+\",\n + \ \"revised_prompt\": \"A simple flat vector-style icon: one perfectly + centered solid red circle on a transparent background, crisp clean edges, + no outline, no shadow, no texture, no text, square canvas.\",\n \"size\": + \"1024x1024\",\n \"svg_usage\": {\n \"input_tokens\": 1618,\n + \ \"model\": \"gpt-5.6-sol\",\n \"output_tokens\": 61,\n \"reasoning_tokens\": + 0,\n \"total_tokens\": 1679\n }\n },\n {\n \"id\": + \"msg_0307e38d3728b13b016af22391eee087d2930f39cbd2cf2794\",\n \"type\": + \"message\",\n \"status\": \"completed\",\n \"content\": [\n {\n + \ \"type\": \"output_text\",\n \"annotations\": [],\n \"logprobs\": + [],\n \"text\": \"\"\n }\n ],\n \"phase\": \"final_answer\",\n + \ \"role\": \"assistant\"\n }\n ],\n \"parallel_tool_calls\": true,\n + \ \"presence_penalty\": 0.0,\n \"previous_response_id\": null,\n \"prompt_cache_key\": + null,\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": + \"12h\"\n },\n \"prompt_cache_retention\": \"24h\",\n \"reasoning\": {\n + \ \"context\": \"all_turns\",\n \"effort\": \"medium\",\n \"mode\": + \"standard\",\n \"summary\": null\n },\n \"safety_identifier\": null,\n + \ \"service_tier\": \"default\",\n \"store\": false,\n \"temperature\": + 1.0,\n \"text\": {\n \"format\": {\n \"type\": \"text\"\n },\n + \ \"verbosity\": \"medium\"\n },\n \"tool_choice\": {\n \"type\": \"image_generation\"\n + \ },\n \"tool_usage\": {\n \"image_gen\": {\n \"input_tokens\": 109,\n + \ \"input_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": + 109\n },\n \"output_tokens\": 1758,\n \"output_tokens_details\": + {\n \"image_tokens\": 1758,\n \"text_tokens\": 0\n },\n + \ \"total_tokens\": 1867\n },\n \"web_search\": {\n \"num_requests\": + 0\n }\n },\n \"tools\": [\n {\n \"type\": \"image_generation\",\n + \ \"background\": \"auto\",\n \"model\": \"gpt-image-2.6-flare\",\n + \ \"moderation\": \"auto\",\n \"n\": 1,\n \"output_compression\": + 100,\n \"output_format\": \"svg\",\n \"quality\": \"auto\",\n \"size\": + \"1024x1024\",\n \"svg\": {\n \"model\": \"gpt-5.6\",\n \"reasoning\": + {\n \"effort\": \"low\"\n }\n }\n }\n ],\n \"top_logprobs\": + 0,\n \"top_p\": 0.98,\n \"truncation\": \"disabled\",\n \"usage\": {\n + \ \"input_tokens\": 1650,\n \"input_tokens_details\": {\n \"cache_write_tokens\": + 0,\n \"cache_write_tokens_details\": {\n \"ttl_12h_tokens\": 0,\n + \ \"ttl_30m_tokens\": 0\n },\n \"cached_tokens\": 0\n },\n + \ \"output_tokens\": 56,\n \"output_tokens_details\": {\n \"reasoning_tokens\": + 0\n },\n \"total_tokens\": 1706\n },\n \"user\": null,\n \"metadata\": + {}\n}" + headers: + access-control-expose-headers: + - X-Request-ID, Server-Timing-Total + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a57f55a6fff03896-YYZ + connection: + - keep-alive + content-length: + - '3637' + content-type: + - application/json + date: + - Mon, 09 Nov 2026 19:48:34 GMT + openai-processing-ms: + - '11660' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-input-images: + - '250000' + x-ratelimit-limit-requests: + - '15000' + x-ratelimit-limit-tokens: + - '40000000' + x-ratelimit-remaining-input-images: + - '249999' + x-ratelimit-remaining-requests: + - '14999' + x-ratelimit-remaining-tokens: + - '39998928' + x-ratelimit-reset-input-images: + - 0s + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 1ms + x-request-id: + - req_1de3473cc1ac423a85917e5ccd7a31d5 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index 4344bba4..2cd08850 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -43,6 +43,9 @@ PROJECT_NAME = "test-project-openai-py-tracing" TEST_MODEL = "gpt-4o-mini" # cheapest model for tests RESPONSES_TOOL_MODEL = "gpt-4.1-mini" +_requires_svg_output = pytest.mark.skipif( + not version_satisfies(openai.__version__, ">=3.46.0"), reason="SVG image output requires OpenAI 3.46.0+" +) TEST_PROMPT = "What's 12 + 12?" @@ -1944,6 +1947,85 @@ def test_openai_images_generate(memory_logger): assert span["metrics"]["duration"] >= 0 +@pytest.mark.vcr +@_requires_svg_output +def test_openai_images_generate_svg(memory_logger): + assert not memory_logger.pop() + + client = wrap_openai(openai.OpenAI()) + response = client.images.generate( + model="gpt-image-2.6-flare", + prompt="A simple flat icon of a red circle", + output_format="svg", + complexity="low", + n=1, + size="1024x1024", + svg={"model": "gpt-5.6", "reasoning": {"effort": "low"}}, + ) + + assert response.output_format == "svg" + assert response.data and response.data[0].b64_json + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert span["metadata"]["output_format"] == "svg" + assert span["metadata"]["complexity"] == "low" + assert span["metadata"]["svg"] == {"model": "gpt-5.6", "reasoning": {"effort": "low"}} + + image = span["output"]["images"][0] + assert image["mime_type"] == "image/svg+xml" + attachment = image["image_url"]["url"] + assert isinstance(attachment, Attachment) + assert attachment.reference["content_type"] == "image/svg+xml" + assert attachment.reference["filename"].endswith(".svg") + assert b" Any: "call_id", "server_label", "error", + "svg_usage", *(_RESPONSE_TOOL_ITEM_INPUT_KEYS.get(item.get("type"), ())), } output = clean_nones( @@ -1202,6 +1209,7 @@ def _response_tool_span_metadata(item: dict[str, Any]) -> dict[str, Any] | None: "call_id": item.get("call_id"), "status": item.get("status"), "server_label": item.get("server_label"), + **_svg_usage_metadata(item), } ) or None @@ -1856,27 +1864,6 @@ def _postprocess_streaming_results(cls, all_results: list[Any]) -> dict[str, Any } -def _image_attachment_from_base64( - data: Any, - *, - output_format: Any, - index: int, -) -> tuple[_ResolvedAttachment | None, int | None]: - if not isinstance(data, str): - return None, None - - extension = output_format if isinstance(output_format, str) and output_format else "png" - mime_type = extension if "/" in extension else f"image/{extension}" - resolved_attachment = _materialize_attachment( - data, - mime_type=mime_type, - prefix=f"generated_image_{index}", - ) - if resolved_attachment is None: - return None, None - return resolved_attachment, len(resolved_attachment.attachment.data) - - def _extract_images_output(response: dict[str, Any]) -> dict[str, Any]: images = [] output_format = response.get("output_format") @@ -1989,7 +1976,7 @@ def _log_result(self, log_response: Any, start: float, span: Span) -> None: if isinstance(log_response, dict): metrics.update(_parse_metrics_from_usage(log_response.get("usage"))) output = _extract_images_output(log_response) if isinstance(log_response, dict) else log_response - span.log(metrics=metrics, output=output) + span.log(metrics=metrics, output=output, metadata=_svg_usage_metadata(log_response)) def create(self, *args: Any, **kwargs: Any) -> Any: params = self._parse_params(kwargs) @@ -2561,6 +2548,17 @@ def _parse_metrics_from_usage(usage: Any) -> dict[str, Any]: return metrics +def _svg_usage_metadata(response: Any) -> dict[str, Any]: + """Return `svg_usage` (usage of the model that authors SVG image output) as metadata. + + It is reported separately from the image model's `usage` and may come from a + different model. It has no standard metric keys and must not be folded into + the image model's token totals, so it is kept verbatim as metadata. + """ + svg_usage = _try_to_dict(response.get("svg_usage")) if isinstance(response, Mapping) else None + return {"svg_usage": svg_usage} if svg_usage else {} + + def prettify_params(params: dict[str, Any]) -> dict[str, Any]: # Filter out NOT_GIVEN parameters # https://linear.app/braintrustdata/issue/BRA-2467 diff --git a/py/src/braintrust/integrations/utils.py b/py/src/braintrust/integrations/utils.py index ffa25d5b..482285b2 100644 --- a/py/src/braintrust/integrations/utils.py +++ b/py/src/braintrust/integrations/utils.py @@ -428,6 +428,34 @@ def _normalize_chat_messages(messages: Any) -> Any: return normalized +# Image `output_format` values whose MIME type is not `image/`. +_IMAGE_FORMAT_TO_MIME_TYPE = { + "svg": "image/svg+xml", +} + + +def _image_attachment_from_base64( + data: Any, + *, + output_format: Any, + index: int, +) -> tuple[_ResolvedAttachment | None, int | None]: + """Materialize a generated base64 image as an attachment, returning it and its size in bytes.""" + if not isinstance(data, str): + return None, None + + extension = output_format if isinstance(output_format, str) and output_format else "png" + mime_type = extension if "/" in extension else _IMAGE_FORMAT_TO_MIME_TYPE.get(extension, f"image/{extension}") + resolved_attachment = _materialize_attachment( + data, + mime_type=mime_type, + prefix=f"generated_image_{index}", + ) + if resolved_attachment is None: + return None, None + return resolved_attachment, len(resolved_attachment.attachment.data) + + _AUDIO_FORMAT_TO_MIME_TYPE = { "mp3": "audio/mpeg", "wav": "audio/wav",