Merge branch 'main' into fix/langgraph-concurrent-requests-and-tool-name-none

Author: ranst91

docs/concepts/messages.mdx

@@ -55,25 +55,59 @@ interface UserMessage {
   name?: string // Optional user identifier
 }
 
-type InputContent = TextInputContent | BinaryInputContent
+type InputContent =
+  | TextInputContent
+  | ImageInputContent
+  | AudioInputContent
+  | VideoInputContent
+  | DocumentInputContent
+
+interface InputContentDataSource {
+  type: "data"
+  value: string
+  mimeType: string
+}
+
+interface InputContentUrlSource {
+  type: "url"
+  value: string
+  mimeType?: string
+}
+
+type InputContentSource = InputContentDataSource | InputContentUrlSource
 
 interface TextInputContent {
   type: "text"
   text: string
 }
 
-interface BinaryInputContent {
-  type: "binary"
-  mimeType: string
-  id?: string
-  url?: string
-  data?: string
-  filename?: string
+interface ImageInputContent {
+  type: "image"
+  source: InputContentSource
+  metadata?: Record<string, unknown>
+}
+
+interface AudioInputContent {
+  type: "audio"
+  source: InputContentSource
+  metadata?: Record<string, unknown>
+}
+
+interface VideoInputContent {
+  type: "video"
+  source: InputContentSource
+  metadata?: Record<string, unknown>
+}
+
+interface DocumentInputContent {
+  type: "document"
+  source: InputContentSource
+  metadata?: Record<string, unknown>
 }
 ```
 
-> For `BinaryInputContent`, provide at least one of `id`, `url`, or `data` to
-> reference the payload.
+> In Python, the previous `BinaryInputContent` model is deprecated and remains
+> temporarily available as a compatibility path.
 
 This structure keeps traditional plain-text inputs working while enabling richer
 payloads such as images, audio clips, or uploaded files in the same message.

docs/docs.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://mintlify.com/docs.json",
-  "theme": "willow",
+  "theme": "aspen",
   "name": "Agent User Interaction Protocol",
   "colors": {
     "primary": "#09090b",
@@ -90,6 +90,7 @@
                 "pages": [
                   "sdk/js/core/overview",
                   "sdk/js/core/types",
+                  "sdk/js/core/multimodal-inputs",
                   "sdk/js/core/events"
                 ]
               },
@@ -116,6 +117,7 @@
                 "pages": [
                   "sdk/python/core/overview",
                   "sdk/python/core/types",
+                  "sdk/python/core/multimodal-inputs",
                   "sdk/python/core/events"
                 ]
               },
@@ -193,4 +195,4 @@
       "destination": "/quickstart/server"
     }
   ]
-}
+}

docs/drafts/multimodal-messages.mdx

@@ -33,9 +33,9 @@ straightforward to map to any LLM provider's API.
 ### Overview
 
 Extend the `UserMessage` `content` property to be either a string or an array of
-`InputContentPart` objects. Each modality (image, audio, video, document) has
-its own dedicated part type with a typed `source` that is either inline `data`
-or a `url` reference. This makes it trivial to map content parts to any LLM
+`InputContent` objects. Each modality (image, audio, video, document) has its
+own dedicated part type with a typed `source` that is either inline `data` or a
+`url` reference. This makes it trivial to map content parts to any LLM
 provider's API.
 
 ```typescript
@@ -106,7 +106,7 @@ interface DocumentInputPart<TMetadata = unknown> {
   metadata?: TMetadata
 }
 
-type InputContentPart =
+type InputContent =
   | TextInputPart
   | ImageInputPart
   | AudioInputPart
@@ -118,7 +118,7 @@ type InputContentPart =
 type UserMessage = {
   id: string
   role: "user"
-  content: string | InputContentPart[]
+  content: string | InputContent[]
   name?: string
 }
 ```
@@ -422,9 +422,9 @@ Examples:
 
 TypeScript SDK:
 
-- New `Modality` type and all `InputContentPart` types in `@ag-ui/core`
+- New `Modality` type and all `InputContent` types in `@ag-ui/core`
 - `InputContentSource`, `InputContentDataSource`, `InputContentUrlSource` types
-- Updated `UserMessage` with `content: string | InputContentPart[]`
+- Updated `UserMessage` with `content: string | InputContent[]`
 - Helper methods for constructing typed content parts
 - Provider-specific metadata generics on each content part type
 
@@ -440,7 +440,7 @@ Python SDK:
 
 Frameworks need to:
 
-- Parse typed `InputContentPart` parts and dispatch on `part.type`
+- Parse typed `InputContent` parts and dispatch on `part.type`
 - Map content parts to provider-specific formats (the typed structure makes this
   straightforward)
 - Use `source.type` to determine whether to send inline data or a URL to the
@@ -479,7 +479,7 @@ Share screenshots (`ImageInputPart`) for UI/UX feedback or debugging assistance.
 
 ## Testing Strategy
 
-- Unit tests for each `InputContentPart` type and `InputContentSource` variant
+- Unit tests for each `InputContent` type and `InputContentSource` variant
 - Validate `source.type` discriminator correctly narrows the union
 - Integration tests with multimodal LLMs (OpenAI, Anthropic, Google)
 - Backward compatibility tests with plain `string` content