feat(adk): support multimodal input types (image/audio/video/document)

integrations/adk-middleware/python/CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- **NEW**: Support for multimodal input types (`ImageInputContent`, `AudioInputContent`, `VideoInputContent`, `DocumentInputContent`) (#1405)
+  - Replaces reliance on the deprecated `BinaryInputContent` with the newer modality-specific types defined in the AG-UI protocol
+  - `InputContentDataSource` (inline base64) converts to `types.Part(inline_data=types.Blob(...))`, same as before
+  - `InputContentUrlSource` (HTTPS/GCS URLs) converts to `types.Part(file_data=types.FileData(file_uri=...))`, leveraging ADK's native URI support
+  - Legacy `BinaryInputContent` continues to work for backward compatibility
+  - Adds E2E tests gated on `GOOGLE_API_KEY` covering inline images, document URLs (RFC 2549 via IETF), multi-image messages, and mixed text+image content
+
+### Fixed
+
+- **FIX**: Disable `save_input_blobs_as_artifacts` so inline images reach the model (#1405)
+  - ADK's runner was converting `inline_data` parts to artifact references before the model could see them, replacing images with text like `"Uploaded file: artifact_xxx. It is saved into artifacts"`
+  - Setting `save_input_blobs_as_artifacts=False` in `RunConfig` preserves inline binary data so the model receives the actual image/audio/video/document content
+
 ## [0.5.2] - 2026-03-26
 
 ### Changed

integrations/adk-middleware/python/examples/test_multimodal_live.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+"""Interactive script to test multimodal messaging against a running ADK server.
+
+Usage:
+    python test_multimodal_live.py                          # text-only chat
+    python test_multimodal_live.py --image photo.png        # send an image
+    python test_multimodal_live.py --image photo.jpg --text "What's in this image?"
+    python test_multimodal_live.py --url https://example.com/doc.pdf --text "Summarize this"
+
+Requires the ADK server to be running on http://localhost:8000 (or set --server).
+"""
+
+import argparse
+import base64
+import json
+import mimetypes
+import sys
+import uuid
+from pathlib import Path
+
+import httpx
+
+
+def build_message(text: str, image_path: str | None, url: str | None) -> dict:
+    """Build an AG-UI UserMessage with optional multimodal content."""
+    content_parts = []
+
+    if text:
+        content_parts.append({"type": "text", "text": text})
+
+    if image_path:
+        path = Path(image_path)
+        if not path.exists():
+            print(f"Error: file not found: {image_path}", file=sys.stderr)
+            sys.exit(1)
+
+        mime_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream"
+        data = base64.b64encode(path.read_bytes()).decode("ascii")
+        content_parts.append({
+            "type": "image",
+            "source": {
+                "type": "data",
+                "value": data,
+                "mimeType": mime_type,
+            },
+        })
+        print(f"  Attached image: {path.name} ({mime_type}, {len(data)} bytes base64)")
+
+    if url:
+        # Guess mime type from URL extension
+        mime_type = mimetypes.guess_type(url)[0]
+        content_parts.append({
+            "type": "document",
+            "source": {
+                "type": "url",
+                "value": url,
+                **({"mimeType": mime_type} if mime_type else {}),
+            },
+        })
+        print(f"  Attached URL: {url} ({mime_type or 'auto-detect'})")
+
+    # If only text, send as plain string; otherwise send content array
+    if len(content_parts) == 1 and content_parts[0]["type"] == "text":
+        msg_content = text
+    elif not content_parts:
+        msg_content = text or "Hello"
+    else:
+        msg_content = content_parts
+
+    return {
+        "id": f"msg-{uuid.uuid4().hex[:8]}",
+        "role": "user",
+        "content": msg_content,
+    }
+
+
+def send_message(server_url: str, message: dict, thread_id: str):
+    """Send a message to the ADK server and stream the response."""
+    payload = {
+        "threadId": thread_id,
+        "runId": f"run-{uuid.uuid4().hex[:8]}",
+        "messages": [message],
+        "context": [],
+        "state": {},
+        "tools": [],
+        "forwardedProps": {},
+    }
+
+    print(f"\n--- Sending to {server_url} (thread: {thread_id}) ---\n")
+
+    with httpx.stream(
+        "POST",
+        server_url,
+        json=payload,
+        headers={"Accept": "text/event-stream"},
+        timeout=60.0,
+    ) as response:
+        if response.status_code != 200:
+            print(f"Error: HTTP {response.status_code}")
+            print(response.read().decode())
+            return
+
+        full_text = []
+        for line in response.iter_lines():
+            if not line.strip():
+                continue
+
+            # Parse SSE format
+            if line.startswith("data: "):
+                data_str = line[6:]
+                try:
+                    event = json.loads(data_str)
+                except json.JSONDecodeError:
+                    continue
+
+                event_type = event.get("type")
+
+                if event_type == "TEXT_MESSAGE_CONTENT":
+                    delta = event.get("delta", "")
+                    print(delta, end="", flush=True)
+                    full_text.append(delta)
+                elif event_type == "RUN_STARTED":
+                    print("[Run started]")
+                elif event_type == "RUN_FINISHED":
+                    print("\n[Run finished]")
+                elif event_type == "RUN_ERROR":
+                    print(f"\n[ERROR] {event.get('message', 'Unknown error')}")
+                elif event_type == "TEXT_MESSAGE_START":
+                    pass  # beginning of message
+                elif event_type == "TEXT_MESSAGE_END":
+                    pass  # end of message
+
+        if full_text:
+            print(f"\n\n--- Full response ({len(''.join(full_text))} chars) ---")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Test multimodal messaging against ADK server")
+    parser.add_argument("--server", default="http://localhost:8000/chat/", help="Server endpoint URL")
+    parser.add_argument("--text", "-t", default=None, help="Text message to send")
+    parser.add_argument("--image", "-i", default=None, help="Path to an image file to attach")
+    parser.add_argument("--url", "-u", default=None, help="URL of a document to attach")
+    parser.add_argument("--thread", default=None, help="Thread ID (default: random)")
+    parser.add_argument("--interactive", action="store_true", help="Interactive chat mode")
+    args = parser.parse_args()
+
+    thread_id = args.thread or f"thread-{uuid.uuid4().hex[:8]}"
+
+    if args.interactive:
+        print("Interactive multimodal chat (type 'quit' to exit)")
+        print("  Prefix with /image <path> to attach an image")
+        print("  Prefix with /url <url> to attach a document URL")
+        print()
+
+        while True:
+            try:
+                user_input = input("You: ").strip()
+            except (EOFError, KeyboardInterrupt):
+                print("\nBye!")
+                break
+
+            if user_input.lower() in ("quit", "exit", "/quit"):
+                break
+
+            image_path = None
+            url = None
+            text = user_input
+
+            if user_input.startswith("/image "):
+                parts = user_input[7:].split(" ", 1)
+                image_path = parts[0]
+                text = parts[1] if len(parts) > 1 else "Describe this image."
+
+            elif user_input.startswith("/url "):
+                parts = user_input[5:].split(" ", 1)
+                url = parts[0]
+                text = parts[1] if len(parts) > 1 else "What is this document about?"
+
+            message = build_message(text, image_path, url)
+            send_message(args.server, message, thread_id)
+            print()
+    else:
+        if not args.text and not args.image and not args.url:
+            args.text = "Hello! What can you help me with?"
+
+        message = build_message(args.text or "", args.image, args.url)
+        send_message(args.server, message, thread_id)
+
+
+if __name__ == "__main__":
+    main()

integrations/adk-middleware/python/pyproject.toml

@@ -8,7 +8,7 @@ authors = [
 ]
 requires-python = ">=3.10, <3.15"
 dependencies = [
-    "ag-ui-protocol>=0.1.11",
+    "ag-ui-protocol>=0.1.15",
     "aiohttp>=3.12.0",
     "asyncio>=3.4.3",
     "fastapi>=0.115.2",

integrations/adk-middleware/python/src/ag_ui_adk/adk_agent.py

@@ -725,7 +725,7 @@ def _default_run_config(self, input: RunAgentInput) -> ADKRunConfig:
         """
         config_kwargs = {
             'streaming_mode': StreamingMode.SSE,
-            'save_input_blobs_as_artifacts': True,
+            'save_input_blobs_as_artifacts': False,
         }
 
         # For ADK 1.22.0+, also include context in custom_metadata

integrations/adk-middleware/python/src/ag_ui_adk/utils/converters.py

@@ -10,7 +10,9 @@
 
 from ag_ui.core import (
     Message, UserMessage, AssistantMessage, SystemMessage, ToolMessage,
-    ToolCall, FunctionCall, TextInputContent, BinaryInputContent, InputContent
+    ToolCall, FunctionCall, TextInputContent, BinaryInputContent, InputContent,
+    ImageInputContent, AudioInputContent, VideoInputContent, DocumentInputContent,
+    InputContentDataSource, InputContentUrlSource,
 )
 from google.adk.events import Event as ADKEvent
 from google.genai import types
@@ -90,13 +92,85 @@ def _is_binary_content(item: Union[dict, InputContent]) -> bool:
     is_binary_input_content = isinstance(item, BinaryInputContent)
     return is_binary_dict or is_binary_input_content
 
+_MEDIA_CONTENT_TYPES = (ImageInputContent, AudioInputContent, VideoInputContent, DocumentInputContent)
+_MEDIA_TYPE_STRINGS = {"image", "audio", "video", "document"}
+
+def _is_media_content(item: Union[dict, InputContent]) -> bool:
+    if isinstance(item, _MEDIA_CONTENT_TYPES):
+        return True
+    return isinstance(item, dict) and item.get("type") in _MEDIA_TYPE_STRINGS
+
+def _media_content_to_part(item: Union[dict, InputContent]) -> Optional[types.Part]:
+    """Convert a media content item (image/audio/video/document) to a types.Part."""
+    if isinstance(item, _MEDIA_CONTENT_TYPES):
+        source = item.source
+    elif isinstance(item, dict):
+        source = item.get("source")
+    else:
+        return None
+
+    if source is None:
+        logger.warning("Media content item has no source; ignoring.")
+        return None
+
+    # Handle InputContentDataSource (inline base64)
+    if isinstance(source, InputContentDataSource):
+        mime_type = source.mime_type
+        data_value = source.value
+    elif isinstance(source, dict) and source.get("type") == "data":
+        mime_type = source.get("mimeType") or source.get("mime_type")
+        data_value = source.get("value")
+    else:
+        mime_type = None
+        data_value = None
+
+    if data_value is not None:
+        if not mime_type:
+            logger.warning("Media content data source missing mime_type; ignoring.")
+            return None
+        try:
+            decoded = base64.b64decode(data_value, validate=True)
+            return types.Part(
+                inline_data=types.Blob(
+                    mime_type=mime_type,
+                    data=decoded,
+                )
+            )
+        except (binascii.Error, ValueError) as e:
+            logger.warning("Failed to base64 decode media content data: %s", e)
+            return None
+
+    # Handle InputContentUrlSource (URI reference)
+    if isinstance(source, InputContentUrlSource):
+        url_value = source.value
+        url_mime = source.mime_type
+    elif isinstance(source, dict) and source.get("type") == "url":
+        url_value = source.get("value")
+        url_mime = source.get("mimeType") or source.get("mime_type")
+    else:
+        logger.warning("Media content has unrecognized source type; ignoring.")
+        return None
+
+    if not url_value:
+        logger.warning("Media content URL source missing value; ignoring.")
+        return None
+
+    return types.Part(
+        file_data=types.FileData(
+            file_uri=url_value,
+            mime_type=url_mime,
+        )
+    )
+
 def convert_message_content_to_parts(content: Optional[Union[str, List[Any]]]) -> List[types.Part]:
     """Convert AG-UI message content into google.genai types.Part list.
 
     Supports:
     - str -> [Part(text=...)]
-    - List[InputContent] -> text parts + binary parts (inline_data only; data/base64 only)
-    - List[dict] -> dict-shaped text/binary items (data/base64 only)
+    - List[InputContent] -> text parts + media parts (image/audio/video/document) + binary parts
+    - Media data sources (base64) -> Part(inline_data=Blob(...))
+    - Media URL sources -> Part(file_data=FileData(file_uri=...))
+    - Legacy BinaryInputContent -> Part(inline_data=Blob(...)) (deprecated)
     """
     if content is None:
         return []
@@ -111,6 +185,10 @@ def convert_message_content_to_parts(content: Optional[Union[str, List[Any]]]) -
             part = _to_text_part(text_value)
             if part:
                 parts.append(part)
+        elif _is_media_content(item):
+            part = _media_content_to_part(item)
+            if part:
+                parts.append(part)
         elif _is_binary_content(item):
             data, mime_type, url, binary_id = _get_binary_attributes(item)
             part = _to_binary_part(data, mime_type, url, binary_id)