From 7590ed0c15d1c9f2b73dd3bdfa6793f793b52a50 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Tue, 20 Oct 2026 11:10:29 +0800 Subject: [PATCH 01/20] [AI Agents] Set up stacked GPT Live WebSocket work Prepare a draft stacked PR for handwritten GPT Live WebSocket support, examples, and tests on top of #50875. No implementation changes yet. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> From 5ba462d43cebdaeca5d4af10c846c4a7d552c511 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Tue, 20 Oct 2026 16:04:58 +0800 Subject: [PATCH 02/20] [AI Agents] Add native GPT Live WebSocket clients and voice samples Add synchronous and asynchronous native Live clients, connection options, interactive text/audio samples, and regression coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-agents/CHANGELOG.md | 26 +- sdk/ai/azure-ai-agents/README.md | 141 ++ .../azure/ai/agents/AgentsClientBuilder.java | 30 + ...etaVoiceAgentLiveWebSocketAsyncClient.java | 54 + .../BetaVoiceAgentLiveWebSocketClient.java | 54 + ...eAgentLiveWebSocketSessionAsyncClient.java | 104 ++ ...aVoiceAgentLiveWebSocketSessionClient.java | 121 ++ ...VoiceAgentWebSocketSessionAsyncClient.java | 14 +- .../BetaVoiceAgentWebSocketSessionClient.java | 10 +- ...oiceAgentWebSocketClientConfiguration.java | 24 + .../realtime/VoiceAgentWebSocketUtils.java | 41 +- ...ceAgentLiveWebSocketConnectionOptions.java | 202 +++ .../GptLiveAudioConversationAsyncSample.java | 138 ++ .../voice/GptLiveFunctionToolSample.java | 145 +++ .../ai/agents/voice/GptLiveSampleUtils.java | 306 +++++ .../GptLiveTextConversationAsyncSample.java | 347 +++++ .../voice/GptLiveTextConversationSample.java | 150 +++ ...AgentLiveAudioConversationAsyncSample.java | 15 +- .../VoiceAgentLiveTextConversationSample.java | 50 +- .../voice/VoiceAgentRealtimeSampleUtils.java | 6 + .../ai/agents/voice/GptLiveSamplesTests.java | 1154 +++++++++++++++++ ...eAgentLiveTextConversationSampleTests.java | 185 +++ .../voice/VoiceAgentLiveWebSocketTests.java | 407 ++++++ 23 files changed, 3697 insertions(+), 27 deletions(-) create mode 100644 sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketAsyncClient.java create mode 100644 sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketClient.java create mode 100644 sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java create mode 100644 sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java create mode 100644 sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java create mode 100644 sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java create mode 100644 sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSampleTests.java create mode 100644 sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java diff --git a/sdk/ai/azure-ai-agents/CHANGELOG.md b/sdk/ai/azure-ai-agents/CHANGELOG.md index a82892ffe3da5..fc2edd5d217d0 100644 --- a/sdk/ai/azure-ai-agents/CHANGELOG.md +++ b/sdk/ai/azure-ai-agents/CHANGELOG.md @@ -7,7 +7,7 @@ - Added preview voice-agent post-call delivery through `VoiceAgentPostCallConfiguration` for Event Grid and routine destinations. - Added Live voice-agent definitions and preview Live conversation REST clients for persisted conversations, items, - and audio. Live WebSocket transport is not included. + and audio. - Added SIP telephony bindings, inline voice subagents and handoff configuration, and system tools for language detection, digit recognition, transfer, SMS, and DTMF, with related server events. - Added polymorphic voice subagent configuration through `VoiceAgentSubagentConfiguration`, with @@ -18,6 +18,28 @@ `VoiceAgentLiveClientDelegation`, and `VoiceAgentLiveResponsesDelegation`, with managed-agent delegation offsets represented as `Duration` and outputs as `List`. - Added preview telephony campaign clients and models for recipients, scheduling, progress, and validation errors. +- Added preview synchronous and asynchronous GPT Live WebSocket clients through + `AgentsClientBuilder.beta().buildBetaVoiceAgentLiveWebSocketClient()` and + `buildBetaVoiceAgentLiveWebSocketAsyncClient()`. Native Live events are sent and received as lossless JSON + using `BinaryData`, separately from the Realtime protocol clients. +- Added runnable native GPT Live samples for synchronous/asynchronous multi-turn text, asynchronous microphone + and speaker audio, and Responses backend function tools, including sample-agent creation and cleanup. + Model deployment names default to `gpt-live-1` and `gpt-5.6-terra` when their environment variables are unset. + The audio sample requests an opening greeting, labels spoken transcripts without duplicating backend text, + reports moderation failures explicitly, and shuts down shared Reactor schedulers when its standalone main exits. + Both text samples support interactive terminal input with blank-line, `exit`, and `quit` termination, + preserve scripted command-line prompts, and shut down shared Reactor schedulers when their standalone mains exit. + Both interactive text modes request a spoken greeting, play frontend audio and print spoken transcripts + instead of backend working notes. It preserves server closure diagnostics and handles remote hangup normally. + Interactive voice mode streams paced silent PCM input while typing (consuming voice-session usage), uses + blocking console line reads instead of readiness polling, and uses the native console reader when attached. + Assistant transcript fragments are assembled into utterances independently of backend and typed turns. + Completed text is buffered while typing on the `You:` line; audio and input remain available, and incomplete + text is not split or relabeled at Enter. Speech-inactivity boundaries remain heuristic. + The first input prompt waits for greeting audio and text, a one-second quiet period, and queued speaker playback. + Continuous silent output does not extend that quiet period, and paced silent input tolerates send backpressure + without overflowing a periodic timer. + Removed redundant blank lines between console input, assistant transcripts, and session closure. ### Breaking Changes @@ -31,6 +53,8 @@ ### Bugs Fixed +- Updated the synchronous live text sample to replace an existing sample agent on a creation conflict and + delete the replacement on exit, including when the conversation fails. - Fixed voice-agent definition deserialization through `AgentDefinition` and `VoiceAgentBaseDefinition` to select Realtime for an omitted or `realtime` sub-protocol and Live for `live`. - Fixed response input and handoff error JSON serialization to preserve objects rather than encode them as strings. diff --git a/sdk/ai/azure-ai-agents/README.md b/sdk/ai/azure-ai-agents/README.md index 003d4a3770b34..63438caca04db 100644 --- a/sdk/ai/azure-ai-agents/README.md +++ b/sdk/ai/azure-ai-agents/README.md @@ -71,6 +71,7 @@ The Agents client library has the following sub-clients which group the differen - `BetaMemoryStoresClient` / `BetaMemoryStoresAsyncClient` **(preview)**: Manage memory stores and individual memory items for agents. - `ToolboxesClient` / `ToolboxesAsyncClient`: Manage toolboxes and toolbox versions. - `BetaVoiceAgentWebSocketClient` / `BetaVoiceAgentWebSocketAsyncClient` **(preview)**: Open typed realtime WebSocket sessions with voice agents. +- `BetaVoiceAgentLiveWebSocketClient` / `BetaVoiceAgentLiveWebSocketAsyncClient` **(preview)**: Open native GPT Live WebSocket sessions with persisted Live voice agents. - `BetaVoiceAgentsTelephonyClient` / `BetaVoiceAgentsTelephonyAsyncClient` **(preview)**: Manage telephony bindings, calls, transfer targets, and outbound call jobs. - `BetaVoiceAgentsConversationsClient` / `BetaVoiceAgentsConversationsAsyncClient` **(preview)**: Read and delete persisted voice-agent conversations and retrieve their responses, items, and audio. @@ -198,6 +199,7 @@ Build clients whose names start with `Beta` from `AgentsClientBuilder.beta()`. T | `BetaAgentsClient` | `WorkflowAgents=V1Preview,ExternalAgents=V1Preview,DraftAgents=V1Preview,VoiceAgents=V1Preview,DigitalWorker=V1Preview,GitHubCopilot=V1Preview,Skills=V1Preview,MicrosoftCopilot=V1Preview` | | `BetaMemoryStoresClient` | `MemoryStores=V1Preview` | | `BetaVoiceAgentWebSocketClient` | `VoiceAgents=V1Preview` | +| `BetaVoiceAgentLiveWebSocketClient` / `BetaVoiceAgentLiveWebSocketAsyncClient` | `VoiceAgents=V1Preview` | | `BetaVoiceAgentsTelephonyClient` | `VoiceAgents=V1Preview` | | `BetaVoiceAgentsConversationsClient` | `VoiceAgents=V1Preview` | @@ -1101,8 +1103,147 @@ When a `RealtimeResponseFunctionCallArgumentsDoneEvent` requests a client-side t All realtime examples require `FOUNDRY_PROJECT_ENDPOINT` and optionally use `FOUNDRY_VOICE_AGENT_NAME`. The function-tool example also optionally uses `FOUNDRY_VOICE_MODEL` and `FOUNDRY_VOICE_MODEL_TYPE`. The asynchronous text and audio examples delete their generated agents by default; set `FOUNDRY_KEEP_VOICE_AGENT=true` to retain them. +The synchronous live text sample deletes and replaces the selected agent if creation returns a conflict, +then deletes the replacement on exit, including on failure. Use a dedicated sample-agent name: this also +applies when `FOUNDRY_VOICE_AGENT_NAME` is set. This synchronous sample does not support retaining its agent. + The live audio example requires a Java Sound-compatible microphone and speaker. It streams signed, little-endian, mono PCM16 audio at 24 kHz. These examples use WebSocket transport. Although the generated protocol models include WebRTC signaling events, the Java client does not provide a WebRTC peer connection or media implementation. +### GPT Live WebSocket examples (preview) + +GPT Live is a different wire protocol from Realtime. Create a persisted agent with `VoiceAgentLiveDefinition`, +then use `BetaVoiceAgentLiveWebSocketClient` or `BetaVoiceAgentLiveWebSocketAsyncClient` to connect. +These clients use +`wss://{project-endpoint}/agents/{agent_name}/endpoint/protocols/voice/live/sessions?api-version=v1`, +send the `Foundry-Features: VoiceAgents=V1Preview` header, and **do not** negotiate the `realtime` WebSocket +subprotocol. `allowPreview(true)` is not required. + +Foundry sends the initial native `session.start`. Wait for `session.started` before sending application events; +do not send `session.start` yourself. Live events are exposed as complete `BinaryData` JSON objects, not +`RealtimeClientEvent` or `RealtimeServerEvent` models. This preserves native events, unknown properties, +nested `response.event` backend events, and the Foundry `conversation_id` on `session.started`. +The existing samples named `VoiceAgentLive*ConversationSample` above demonstrate live connections using the +**Realtime** protocol; use the separate GPT Live clients for `VoiceAgentLiveDefinition` agents. + +Use `sendEvent(BinaryData)` with native Live JSON commands valid for the configured delegation mode. +In particular, `response.item.create` and `response.create` are Responses-delegation-only commands. +Service `error` events arrive as JSON events for the application to inspect; transport and malformed JSON +errors fail the receive stream unless a malformed-event callback was explicitly configured. + +Configure timeouts, bounded receive capacity, overflow handling, message size, persistence, agent version, +and structured inputs with `VoiceAgentLiveWebSocketConnectionOptions`. Structured inputs are passed as +unencoded JSON to `setStructuredInputs` and URL-encoded by the SDK into the `structured_inputs` query parameter. +Do not log endpoint URLs containing sensitive template inputs. Realtime-only transport and agent-session-ID +options are not exposed. Options are copied when opening a session. + +Both transports retain the realtime clients' credential, proxy, custom-header and builder validation behavior. +Custom HTTP clients, pipelines, policies, and retry settings are unsupported and fail at build time rather +than being ignored. Each session permits one receiver. The default receive buffer holds 256 events; overflow +terminates the session unless a drop strategy explicitly permits event loss. + +#### Runnable GPT Live scenarios + +These examples create a uniquely named `VoiceAgentLiveDefinition` agent with **Responses delegation**, connect +using native GPT Live WebSockets, and delete the agent on exit, including on failure. They do not modify or delete +an existing agent. Set these environment variables before running a sample's `main` method: + +| Variable | Value | +|---|---| +| `FOUNDRY_PROJECT_ENDPOINT` | Your Foundry project endpoint. Authentication uses `DefaultAzureCredential`. | +| `FOUNDRY_GPT_LIVE_MODEL` | Optional. Your GPT Live **deployment name**, not a model catalog ID. Defaults to `gpt-live-1` when unset. | +| `FOUNDRY_GPT_LIVE_BACKEND_MODEL` | Optional. Your Responses backend **deployment name in the Live resource**. Defaults to `gpt-5.6-terra` when unset. | +| `FOUNDRY_KEEP_GPT_LIVE_AGENT` | Optional. Set to `true` to retain the sample-created agent; defaults to `false`. | + +Each sample reads, defaults, and validates these settings and configures its client builder directly in its own `main` method. +Shared helpers receive explicit values rather than reading environment variables. +Override the model variables if your deployment names differ from these defaults. Explicitly blank values are invalid. +Generated agent names use `gpt-live--` and are at most 57 characters for these scenarios, +leaving headroom below the service's 63-character limit. + +| Scenario | Runnable sample | +|---|---| +| Synchronous multi-turn text | [GptLiveTextConversationSample.java](src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java) | +| Asynchronous multi-turn text | [GptLiveTextConversationAsyncSample.java](src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java) | +| Asynchronous microphone and speaker | [GptLiveAudioConversationAsyncSample.java](src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java) | +| Responses backend function tool | [GptLiveFunctionToolSample.java](src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java) | + +Both text examples read terminal input interactively when no arguments are supplied. After each reply, +they prompt for another message; submit a blank line or enter `exit` or `quit` to close the session. +With arguments, they instead process each argument as a prompt and exit after the last reply. +They wait for `session.started`, send `response.item.create` followed by `response.create`, and wait for the +nested `response.completed` before advancing to the next turn. Scripted mode prints backend text without +speaker playback. Both interactive examples instead request a spoken opening greeting with +`session.commentary.append`, play the voice frontend's 24-kHz PCM16 audio, and print its spoken transcript +rather than backend working notes. The synchronous example uses synchronous session send/receive calls on +workers and shares the asynchronous example's voice-loop coordination. +Before displaying the first `You:` prompt, it waits for both greeting audio and transcript, one second without +speech activity, and completion of queued speaker playback. Continuous silent audio does not restart the timer. +For this heuristic, PCM16 frames with peak sample magnitude at most 256 (about -42 dBFS) are treated as silence. +The quiet period is a heuristic because native GPT Live +has no speech-completed event: a long pause within a greeting can be mistaken for its end. Missing greeting output +fails with a timeout instead of silently starting input before the introduction. +It sends 20 milliseconds of silent PCM (960 bytes) every 20 milliseconds after startup, providing continuous +audio input without opening a microphone. **This consumes voice-session usage while you type or wait.** +The silent stream is stopped and pending audio sends complete before the sample requests `session.close`. +If sending falls behind, silent-frame production waits for demand instead of accumulating timer ticks or +failing with an interval overflow; it resumes without a burst of stale frames. +Speaker playback runs off the WebSocket receive thread. If a speaker is unavailable, the sample reports that +and still prints transcripts, like the Realtime text example. +Agent creation and deletion occur on the main thread. + +Console input is line-oriented: press Enter to submit a message. The sample flushes the `You:` prompt immediately; +character echo while typing is controlled by Windows CMD/PowerShell and its terminal host, not by the service. +Interactive input uses a normal blocking line read on a worker rather than polling `Reader.ready()`, +matching the Realtime sample's console interaction. The WebSocket receiver continues independently. +When a native Java console is attached, input comes from `System.console().reader()` rather than `System.in`. +Without a console, the sample reads `System.in`, supporting redirected input. End-of-input (EOF), a blank line, +or an explicit exit command closes the session. +The terminal echoes characters on the `You:` line as you type. Assistant transcript fragments are assembled into +utterances independently of typed turns and backend completion. A one-second speech-inactivity period marks an +utterance boundary using the same heuristic as the greeting; a long pause within speech can still divide the text. +Completed utterances are printed with one `Assistant:` label each. While input is active, completed text waits +until Enter so it cannot overwrite typed characters; incomplete text stays buffered across input submissions. +Audio playback and input remain available. This dependency-free approach can delay captions until the next Enter. +Session closure flushes any remaining transcript. Completed and incomplete text share a 65,536-character buffer +limit; exceeding it fails the sample explicitly instead of dropping transcript text or growing memory indefinitely. +The native console's echoed Enter already starts a new line; the sample avoids adding another blank line. +After a remote disconnect, a console read that the operating system cannot interrupt may require Enter to release +the worker. The sample does not close shared standard input to force a pending read to return. +Backend `response.completed` permits the next typed prompt but does not mean that the frontend finished speaking: +native GPT Live has no output-audio-done event. The sample keeps receiving audio and transcripts while you type. +See [GPT-Live audio and transcript events](https://learn.microsoft.com/azure/foundry/openai/gpt-live-reference#sessionoutput_audiodelta). +Server `remote_hangup` closes the conversation normally. Other unexpected closures remain failures and include +the complete `session.closed` event, including its reason, rather than hiding the service diagnostic. + +The audio example requires a Java Sound-compatible microphone and speaker supporting raw, mono PCM16LE at +24 kHz. After `session.started`, it sends a greeting request followed by `response.create` so the agent can +open the conversation. It labels spoken transcript fragments as `You:` or `Assistant:` and omits backend +text deltas, which are not necessarily the words spoken by the voice frontend. +Press Enter to end the call; asking the model to end it does not itself close the session. +Service moderation (`content_filter`) ends the call with an explicit error and the original service event; +it is not treated as a successful completion. Audio devices and the WebSocket are released on failure. +The standalone audio and text samples' `main` methods also shut down shared Reactor schedulers on exit +to avoid lingering timer threads under `exec:java`. Do not invoke those `main` methods inside an application that +shares Reactor schedulers. + +The audio sample sends `session.input_audio.append`, plays base64-decoded `session.output_audio.delta` chunks, and +prints native input/output transcript deltas. It does not send Realtime audio-buffer commit or response-cancel +commands. Capture and playback use dedicated threads; the playback backlog is bounded to 60 seconds. +Device errors and overflow end the call explicitly. Press Enter to stop capture and finalize the session. +This example must be the only reader of standard input. + +The function-tool example configures `get_weather` on the **Responses backend**, consumes completed +`function_call` items from nested `response.output_item.done` events, then sends `function_call_output` items +with the original `call_id` before requesting continuation. Its weather results are **simulated demonstration +data**, not live weather. It does not assume lifecycle snapshots contain output items, and does not add +Foundry-managed-agent tool-result routing. + +All four scenarios request `session.close` and wait for `session.closed` before normal transport cleanup. +The asynchronous text example waits for terminal input without a timeout, while applying a 60-second timeout to +startup, each response, and finalization. The synchronous text and tool examples use a 60-second event-wait timeout; +the audio example uses a 60-second startup/finalization timeout. +An unexpected disconnect or native service error fails the sample rather than being reported as success. + ### Service API versions The client library targets the latest service API version by default. diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java index add94d052cdb2..0062221ae3060 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java @@ -631,11 +631,13 @@ public BetaAgentsClientBuilder beta() { BetaAgentsClient.class, BetaMemoryStoresClient.class, BetaVoiceAgentWebSocketClient.class, + BetaVoiceAgentLiveWebSocketClient.class, BetaVoiceAgentsTelephonyClient.class, BetaVoiceAgentsConversationsClient.class, BetaAgentsAsyncClient.class, BetaMemoryStoresAsyncClient.class, BetaVoiceAgentWebSocketAsyncClient.class, + BetaVoiceAgentLiveWebSocketAsyncClient.class, BetaVoiceAgentsTelephonyAsyncClient.class, BetaVoiceAgentsConversationsAsyncClient.class }) public final class BetaAgentsClientBuilder { @@ -771,6 +773,34 @@ public BetaVoiceAgentWebSocketClient buildBetaVoiceAgentWebSocketClient() { return new BetaVoiceAgentWebSocketClient(createVoiceAgentWebSocketConfiguration()); } + /** + * Builds an asynchronous WebSocket client for persisted GPT Live voice agents. + *

+ * Applies the same endpoint, credential, proxy and client options as the realtime WebSocket client. + * Custom HTTP clients, pipelines, policies and retry settings are not supported. + * + * @return an asynchronous Live WebSocket client. + * @throws IllegalStateException if unsupported HTTP pipeline configuration is present. + */ + @Beta + public BetaVoiceAgentLiveWebSocketAsyncClient buildBetaVoiceAgentLiveWebSocketAsyncClient() { + return new BetaVoiceAgentLiveWebSocketAsyncClient(createVoiceAgentWebSocketConfiguration().forLive()); + } + + /** + * Builds a synchronous WebSocket client for persisted GPT Live voice agents. + *

+ * Applies the same endpoint, credential, proxy and client options as the realtime WebSocket client. + * Custom HTTP clients, pipelines, policies and retry settings are not supported. + * + * @return a synchronous Live WebSocket client. + * @throws IllegalStateException if unsupported HTTP pipeline configuration is present. + */ + @Beta + public BetaVoiceAgentLiveWebSocketClient buildBetaVoiceAgentLiveWebSocketClient() { + return new BetaVoiceAgentLiveWebSocketClient(createVoiceAgentWebSocketConfiguration().forLive()); + } + /** * Builds a synchronous beta client for preview voice-agent telephony operations. *

diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketAsyncClient.java new file mode 100644 index 0000000000000..192aef2a4f5bc --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketAsyncClient.java @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents; + +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.utils.Beta; +import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; +import com.azure.core.annotation.ServiceClient; +import java.util.Objects; +import reactor.core.publisher.Mono; + +/** + * An asynchronous client for opening native GPT Live sessions with persisted Foundry Live voice agents. + * Create instances with {@link AgentsClientBuilder#beta()}. + */ +@ServiceClient(builder = AgentsClientBuilder.class, isAsync = true) +@Beta(warningText = "This class is in preview and may change in future releases.") +public final class BetaVoiceAgentLiveWebSocketAsyncClient { + private final VoiceAgentWebSocketClientConfiguration configuration; + + BetaVoiceAgentLiveWebSocketAsyncClient(VoiceAgentWebSocketClientConfiguration configuration) { + this.configuration = Objects.requireNonNull(configuration, "'configuration' cannot be null."); + } + + /** + * Opens a WebSocket session using the persisted Live agent configuration. + * @param agentName the Live voice agent name. + * @return the connected session. Each subscription opens a new connection. + */ + public Mono openWebSocketSession(String agentName) { + return openWebSocketSession(agentName, new VoiceAgentLiveWebSocketConnectionOptions()); + } + + /** + * Opens a WebSocket session. Foundry sends the initial {@code session.start}; consume {@code session.started} + * from the event stream before sending application events. No Realtime subprotocol is negotiated. + * @param agentName the Live voice agent name. + * @param options the connection options, copied when this method is called. + * @return the connected session. Each subscription opens a new connection. + */ + public Mono openWebSocketSession(String agentName, + VoiceAgentLiveWebSocketConnectionOptions options) { + Objects.requireNonNull(agentName, "'agentName' cannot be null."); + VoiceAgentWebSocketConnectionOptions snapshot = VoiceAgentWebSocketUtils.liveOptions(options); + return Mono.defer(() -> { + BetaVoiceAgentWebSocketSessionAsyncClient transport + = new BetaVoiceAgentWebSocketSessionAsyncClient(configuration, agentName, snapshot); + return transport.connect().thenReturn(new BetaVoiceAgentLiveWebSocketSessionAsyncClient(transport)); + }); + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketClient.java new file mode 100644 index 0000000000000..85afd6cc86b5c --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketClient.java @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents; + +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.utils.Beta; +import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; +import com.azure.core.annotation.ServiceClient; +import com.azure.core.util.logging.ClientLogger; +import java.util.Objects; + +/** + * A synchronous client for opening native GPT Live sessions with persisted Foundry Live voice agents. + * Create instances with {@link AgentsClientBuilder#beta()}. + */ +@ServiceClient(builder = AgentsClientBuilder.class) +@Beta(warningText = "This class is in preview and may change in future releases.") +public final class BetaVoiceAgentLiveWebSocketClient { + private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentLiveWebSocketClient.class); + private final VoiceAgentWebSocketClientConfiguration configuration; + + BetaVoiceAgentLiveWebSocketClient(VoiceAgentWebSocketClientConfiguration configuration) { + this.configuration = Objects.requireNonNull(configuration, "'configuration' cannot be null."); + } + + /** + * Opens a WebSocket session using the persisted Live agent configuration. + * @param agentName the Live voice agent name. + * @return the connected session. + */ + public BetaVoiceAgentLiveWebSocketSessionClient openWebSocketSession(String agentName) { + return openWebSocketSession(agentName, new VoiceAgentLiveWebSocketConnectionOptions()); + } + + /** + * Opens a WebSocket session. Foundry sends the initial {@code session.start}; consume {@code session.started} + * from the event stream before sending application events. No Realtime subprotocol is negotiated. + * @param agentName the Live voice agent name. + * @param options the connection options, copied before connecting. + * @return the connected session. + * @throws IllegalArgumentException if the agent name is empty. + */ + public BetaVoiceAgentLiveWebSocketSessionClient openWebSocketSession(String agentName, + VoiceAgentLiveWebSocketConnectionOptions options) { + Objects.requireNonNull(agentName, "'agentName' cannot be null."); + if (agentName.isEmpty()) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException("'agentName' cannot be empty.")); + } + return new BetaVoiceAgentLiveWebSocketSessionClient(BetaVoiceAgentWebSocketSessionClient.connect(configuration, + agentName, VoiceAgentWebSocketUtils.liveOptions(options))); + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java new file mode 100644 index 0000000000000..de0d598015bef --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents; + +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.utils.Beta; +import com.azure.core.util.AsyncCloseable; +import com.azure.core.util.BinaryData; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * An asynchronous bidirectional native GPT Live session. + *

+ * Live events are complete JSON objects, not {@code RealtimeServerEvent} or {@code RealtimeClientEvent} models. + * Unknown fields, nested {@code response.event} payloads and Foundry {@code conversation_id} values are preserved. + * Sessions support one event subscriber and must be closed when no longer needed. + */ +@Beta(warningText = "This class is in preview and may change in future releases.") +public final class BetaVoiceAgentLiveWebSocketSessionAsyncClient implements AsyncCloseable, AutoCloseable { + private final BetaVoiceAgentWebSocketSessionAsyncClient transport; + + BetaVoiceAgentLiveWebSocketSessionAsyncClient(BetaVoiceAgentWebSocketSessionAsyncClient transport) { + this.transport = transport; + } + + /** + * Gets the WebSocket endpoint. It may contain structured inputs; do not log sensitive query values. + * @return the endpoint. + */ + public String getEndpoint() { + return transport.getEndpoint(); + } + + /** + * Determines whether the transport is open, not whether {@code session.started} has been received. + * @return whether the WebSocket is open. + */ + public boolean isOpen() { + return transport.isOpen(); + } + + /** + * Gets the peer close code. + * @return the code, or null if no close frame has been received. + */ + public Integer getCloseCode() { + return transport.getCloseCode(); + } + + /** + * Gets the peer close reason. + * @return the reason, or null if no close frame has been received. + */ + public String getCloseReason() { + return transport.getCloseReason(); + } + + /** + * Receives native Live server events in wire order, including unknown events. + * Service {@code error} events are delivered as JSON; transport and malformed JSON errors terminate the stream. + * @return the single-subscriber stream of complete JSON events. + */ + public Flux receiveEvents() { + return transport.receiveEvents().map(VoiceAgentWebSocketUtils::liveEventPayload); + } + + /** + * Sends a native Live JSON event. Foundry supplies {@code session.start}; clients must not send it. + * The service validates native command semantics, including delegation restrictions. + * @param event the complete JSON object. + * @return completion after the frame is written, or an error for invalid JSON or a closed session. + */ + public Mono sendEvent(BinaryData event) { + return transport.sendEvent(event); + } + + /** + * Closes the WebSocket normally. + * @return completion when the session is closed. + */ + @Override + public Mono closeAsync() { + return transport.closeAsync(); + } + + /** + * Closes the WebSocket with the specified close frame. The first asynchronous close request wins. + * @param code a valid WebSocket close code. + * @param reason a non-null reason of at most 123 UTF-8 bytes. + * @return completion when the session is closed. + * @throws IllegalArgumentException if the code or reason is invalid. + */ + public Mono closeAsync(int code, String reason) { + return transport.closeAsync(code, reason); + } + + /** Closes the session, blocking until closure or the configured close timeout. */ + @Override + public void close() { + transport.close(); + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java new file mode 100644 index 0000000000000..10d07cca45371 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents; + +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.utils.Beta; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.core.util.BinaryData; +import com.azure.core.util.IterableStream; +import java.time.Duration; +import java.util.Iterator; + +/** + * A synchronous bidirectional native GPT Live session. + *

+ * Events are complete JSON objects, not Realtime protocol models. Unknown fields, nested {@code response.event} + * payloads and Foundry {@code conversation_id} values are preserved. Only one event iterator is supported. + * Close sessions when they are no longer needed. + */ +@Beta(warningText = "This class is in preview and may change in future releases.") +public final class BetaVoiceAgentLiveWebSocketSessionClient implements AutoCloseable { + private final BetaVoiceAgentWebSocketSessionClient transport; + + BetaVoiceAgentLiveWebSocketSessionClient(BetaVoiceAgentWebSocketSessionClient transport) { + this.transport = transport; + } + + /** + * Gets the WebSocket endpoint. It may contain structured inputs; do not log sensitive query values. + * @return the endpoint. + */ + public String getEndpoint() { + return transport.getEndpoint(); + } + + /** + * Determines whether the transport is open, not whether {@code session.started} has been received. + * @return whether the WebSocket is open. + */ + public boolean isOpen() { + return transport.isOpen(); + } + + /** + * Gets the peer close code. + * @return the code, or null if no close frame has been received. + */ + public Integer getCloseCode() { + return transport.getCloseCode(); + } + + /** + * Gets the peer close reason. + * @return the reason, or null if no close frame has been received. + */ + public String getCloseReason() { + return transport.getCloseReason(); + } + + /** + * Receives native Live server events in wire order, including unknown events. + * Service {@code error} events are delivered as JSON; transport and malformed JSON errors terminate iteration. + * @return the single-consumer stream of complete JSON events. + * @throws IllegalStateException if the receive stream was already claimed. + */ + public IterableStream receiveEvents() { + return receiveEvents(null); + } + + /** + * Receives native Live events with a timeout per wait. Timeouts leave the session open so iteration can resume. + * @param timeout a positive timeout, or null to wait indefinitely. + * @return the single-consumer event stream. + * @throws IllegalArgumentException if the timeout is zero or negative. + * @throws IllegalStateException if the receive stream was already claimed, or a wait times out. + */ + public IterableStream receiveEvents(Duration timeout) { + IterableStream events = transport.receiveEvents(timeout); + return IterableStream.of(() -> new Iterator() { + private final Iterator iterator = events.iterator(); + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public BinaryData next() { + return VoiceAgentWebSocketUtils.liveEventPayload(iterator.next()); + } + }); + } + + /** + * Sends a native Live JSON event. Foundry supplies {@code session.start}; clients must not send it. + * The service validates native command semantics, including delegation restrictions. + * @param event the complete JSON object. + * @throws IllegalArgumentException if the event is not a JSON object. + * @throws IllegalStateException if the session cannot accept the event. + */ + public void sendEvent(BinaryData event) { + transport.sendEvent(event); + } + + /** Closes the WebSocket normally. */ + @Override + public void close() { + transport.close(); + } + + /** + * Closes the WebSocket with the specified close frame. + * @param code a valid WebSocket close code. + * @param reason a non-null reason of at most 123 UTF-8 bytes. + * @throws IllegalArgumentException if the code or reason is invalid. + */ + public void close(int code, String reason) { + transport.close(code, reason); + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java index 06cba1221d988..f8418480f2d05 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java @@ -425,10 +425,12 @@ private Mono openWebSocket(String token) { headers.set(header.getName(), header.getValue()); } }); - WebsocketClientSpec spec = WebsocketClientSpec.builder() - .protocols(VoiceAgentWebSocketUtils.SUBPROTOCOL) - .maxFramePayloadLength(options.getMaxMessageSize()) - .build(); + WebsocketClientSpec.Builder specBuilder + = WebsocketClientSpec.builder().maxFramePayloadLength(options.getMaxMessageSize()); + if (!configuration.isLive()) { + specBuilder.protocols(VoiceAgentWebSocketUtils.SUBPROTOCOL); + } + WebsocketClientSpec spec = specBuilder.build(); return client.websocket(spec).uri(websocketUri.toString()).connect().flatMap(connection -> { if (!(connection instanceof WebsocketInbound) || !(connection instanceof WebsocketOutbound)) { @@ -468,8 +470,8 @@ private void handleFrame(WebSocketFrame frame) { try { byte[] bytes = new byte[frame.content().readableBytes()]; frame.content().getBytes(frame.content().readerIndex(), bytes); - RealtimeServerEvent event - = VoiceAgentWebSocketUtils.deserializeEvent(VoiceAgentWebSocketUtils.decodeEvent(bytes)); + RealtimeServerEvent event = VoiceAgentWebSocketUtils + .deserializeEvent(VoiceAgentWebSocketUtils.decodeEvent(bytes), configuration.isLive()); Sinks.EmitResult result = events.tryEmitNext(event); if (result == Sinks.EmitResult.FAIL_OVERFLOW || result == Sinks.EmitResult.FAIL_ZERO_SUBSCRIBER) { switch (options.getOverflowStrategy()) { diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java index 7227280cd8326..b4e1cadc67c36 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java @@ -68,6 +68,7 @@ public final class BetaVoiceAgentWebSocketSessionClient implements AutoCloseable private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentWebSocketSessionClient.class); private final URI websocketUri; private final VoiceAgentWebSocketConnectionOptions options; + private final boolean live; private final OkHttpClient httpClient; private final BlockingQueue events; private final int receiveBufferCapacity; @@ -87,6 +88,7 @@ public final class BetaVoiceAgentWebSocketSessionClient implements AutoCloseable private BetaVoiceAgentWebSocketSessionClient(VoiceAgentWebSocketClientConfiguration configuration, String agentName, VoiceAgentWebSocketConnectionOptions options) { this.options = options; + this.live = configuration.isLive(); this.receiveBufferCapacity = options.getReceiveBufferCapacity(); this.events = new ArrayBlockingQueue<>(receiveBufferCapacity + 1); this.websocketUri = VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, options); @@ -94,8 +96,10 @@ private BetaVoiceAgentWebSocketSessionClient(VoiceAgentWebSocketClientConfigurat .getTokenSync(VoiceAgentWebSocketUtils.createTokenRequestContext()) .getToken(); this.httpClient = createHttpClient(configuration, options); - Request.Builder request = new Request.Builder().url(websocketUri.toString()) - .header("Sec-WebSocket-Protocol", VoiceAgentWebSocketUtils.SUBPROTOCOL); + Request.Builder request = new Request.Builder().url(websocketUri.toString()); + if (!live) { + request.header("Sec-WebSocket-Protocol", VoiceAgentWebSocketUtils.SUBPROTOCOL); + } for (HttpHeader header : VoiceAgentWebSocketUtils.buildHeaders(configuration, options, token)) { request.header(header.getName(), header.getValue()); } @@ -516,7 +520,7 @@ public void onMessage(WebSocket webSocket, String text) { return; } try { - signal(EventSignal.event(VoiceAgentWebSocketUtils.deserializeEvent(text))); + signal(EventSignal.event(VoiceAgentWebSocketUtils.deserializeEvent(text, live))); } catch (IOException | RuntimeException error) { malformedEvent(webSocket, error); } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java index d123c4d966303..1fc87e1ad297c 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java @@ -20,6 +20,7 @@ public final class VoiceAgentWebSocketClientConfiguration { private final String userAgent; private final HttpHeaders headers; private final ProxyOptions proxyOptions; + private final boolean live; /** * Creates the connection configuration. @@ -33,12 +34,35 @@ public final class VoiceAgentWebSocketClientConfiguration { */ public VoiceAgentWebSocketClientConfiguration(URI endpoint, TokenCredential credential, String apiVersion, String userAgent, HttpHeaders headers, ProxyOptions proxyOptions) { + this(endpoint, credential, apiVersion, userAgent, headers, proxyOptions, false); + } + + private VoiceAgentWebSocketClientConfiguration(URI endpoint, TokenCredential credential, String apiVersion, + String userAgent, HttpHeaders headers, ProxyOptions proxyOptions, boolean live) { this.endpoint = Objects.requireNonNull(endpoint, "'endpoint' cannot be null."); this.credential = Objects.requireNonNull(credential, "'credential' cannot be null."); this.apiVersion = Objects.requireNonNull(apiVersion, "'apiVersion' cannot be null."); this.userAgent = Objects.requireNonNull(userAgent, "'userAgent' cannot be null."); this.headers = headers; this.proxyOptions = proxyOptions; + this.live = live; + } + + /** + * Creates a configuration for the native GPT Live protocol. + * @return the Live configuration. + */ + public VoiceAgentWebSocketClientConfiguration forLive() { + return new VoiceAgentWebSocketClientConfiguration(endpoint, credential, apiVersion, userAgent, headers, + proxyOptions, true); + } + + /** + * Indicates whether this connection uses native GPT Live events and routing. + * @return whether the Live protocol is selected. + */ + public boolean isLive() { + return live; } /** diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java index 97aade8190dd7..864d5119b96df 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java @@ -7,6 +7,7 @@ import com.azure.ai.agents.models.RawRealtimeServerEvent; import com.azure.ai.agents.models.RealtimeServerEvent; import com.azure.ai.agents.models.VoiceAgentTransport; +import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.credential.TokenRequestContext; import com.azure.core.http.HttpHeader; @@ -54,9 +55,13 @@ public static String validateEvent(BinaryData event) { } public static RealtimeServerEvent deserializeEvent(String json) throws IOException { + return deserializeEvent(json, false); + } + + public static RealtimeServerEvent deserializeEvent(String json, boolean live) throws IOException { validateEvent(BinaryData.fromString(json)); RawRealtimeServerEvent raw = new RawRealtimeServerEvent(BinaryData.fromString(json)); - if (raw.getType() == null) { + if (live || raw.getType() == null) { return raw; } try (JsonReader reader = JsonProviders.createReader(json)) { @@ -71,6 +76,26 @@ public static RealtimeServerEvent deserializeEvent(String json) throws IOExcepti private VoiceAgentWebSocketUtils() { } + public static VoiceAgentWebSocketConnectionOptions liveOptions(VoiceAgentLiveWebSocketConnectionOptions options) { + Objects.requireNonNull(options, "'options' cannot be null."); + return new VoiceAgentWebSocketConnectionOptions().setReceiveBufferCapacity(options.getReceiveBufferCapacity()) + .setMaxMessageSize(options.getMaxMessageSize()) + .setOverflowStrategy(options.getOverflowStrategy()) + .setMalformedEventHandler(options.getMalformedEventHandler()) + .setHandshakeTimeout(options.getHandshakeTimeout()) + .setCloseTimeout(options.getCloseTimeout()) + .setStoreEnabled(options.isStoreEnabled()) + .setAgentVersionOverride(options.getAgentVersionOverride()) + .setStructuredInputs(options.getStructuredInputs()); + } + + public static BinaryData liveEventPayload(RealtimeServerEvent event) { + if (event instanceof RawRealtimeServerEvent) { + return ((RawRealtimeServerEvent) event).getRawEvent(); + } + throw new IllegalStateException("Expected a raw native Live event."); + } + public static void validateClose(int code, String reason) { if (code < 1000 || code >= 5000 @@ -112,13 +137,16 @@ public static URI buildWebSocketUri(VoiceAgentWebSocketClientConfiguration confi } String basePath = endpoint.getRawPath() == null ? "" : endpoint.getRawPath().replaceAll("/$", ""); - String path = basePath + "/agents/" + encode(agentName) + "/endpoint/protocols/voice"; + String path = basePath + "/agents/" + encode(agentName) + "/endpoint/protocols/voice" + + (configuration.isLive() ? "/live/sessions" : ""); URI baseUri = URI.create(scheme + "://" + endpoint.getRawAuthority() + path); UrlBuilder url = UrlBuilder.parse(baseUri.toString()); url.setQueryParameter("api-version", encode(configuration.getApiVersion())); url.setQueryParameter("x-ms-client-sdk", encode(configuration.getUserAgent())); - VoiceAgentTransport transport = options.getTransport(); - url.setQueryParameter("transport", transport == null ? "websocket" : encode(transport.toString())); + if (!configuration.isLive()) { + VoiceAgentTransport transport = options.getTransport(); + url.setQueryParameter("transport", transport == null ? "websocket" : encode(transport.toString())); + } if (options.isStoreEnabled() != null) { url.setQueryParameter("store", options.isStoreEnabled().toString()); } @@ -128,6 +156,9 @@ public static URI buildWebSocketUri(VoiceAgentWebSocketClientConfiguration confi if (options.getAgentSessionId() != null) { url.setQueryParameter("agent_session_id", encode(options.getAgentSessionId())); } + if (configuration.isLive() && options.getStructuredInputs() != null) { + url.setQueryParameter("structured_inputs", encode(options.getStructuredInputs())); + } return URI.create(url.toString()); } @@ -147,7 +178,7 @@ public static HttpHeaders buildHeaders(VoiceAgentWebSocketClientConfiguration co } headers.set(HttpHeaderName.fromString("Foundry-Features"), AgentDefinitionOptInKeys.VOICE_AGENTS_V1_PREVIEW.toString()); - if (options.getStructuredInputs() != null) { + if (!configuration.isLive() && options.getStructuredInputs() != null) { headers.set(HttpHeaderName.fromString("x-ms-voice-structured-inputs"), options.getStructuredInputs()); } return headers.set(HttpHeaderName.AUTHORIZATION, "Bearer " + token); diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java new file mode 100644 index 0000000000000..79fd1507df458 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.models; + +import com.azure.ai.agents.implementation.utils.Beta; +import com.azure.core.annotation.Fluent; +import java.time.Duration; +import java.util.Objects; +import java.util.function.Consumer; + +/** + * Connection and buffering options for a persisted GPT Live voice agent. + * Configure these options before opening a session; clients snapshot them when opening. + */ +@Beta(warningText = "Preview API. VoiceAgents=V1Preview") +@Fluent +public final class VoiceAgentLiveWebSocketConnectionOptions { + private final VoiceAgentWebSocketConnectionOptions options; + + /** Creates options with a 30-second handshake timeout and a 256-event receive buffer. */ + public VoiceAgentLiveWebSocketConnectionOptions() { + options = new VoiceAgentWebSocketConnectionOptions(); + } + + /** + * Copies connection options. + * @param source the options to copy. + */ + public VoiceAgentLiveWebSocketConnectionOptions(VoiceAgentLiveWebSocketConnectionOptions source) { + options = new VoiceAgentWebSocketConnectionOptions( + Objects.requireNonNull(source, "'source' cannot be null.").options); + } + + /** + * Gets the receive queue capacity. + * @return the maximum queued events, default 256. + */ + public int getReceiveBufferCapacity() { + return options.getReceiveBufferCapacity(); + } + + /** + * Sets the receive queue capacity. + * @param capacity the capacity, between 1 and 65536. + * @return this options instance. + * @throws IllegalArgumentException if the capacity is outside the supported range. + */ + public VoiceAgentLiveWebSocketConnectionOptions setReceiveBufferCapacity(int capacity) { + options.setReceiveBufferCapacity(capacity); + return this; + } + + /** + * Gets the maximum accepted message size. + * @return the size in bytes, default 32 MiB. + */ + public int getMaxMessageSize() { + return options.getMaxMessageSize(); + } + + /** + * Sets the maximum accepted message size. + * @param bytes the positive maximum message size in bytes. + * @return this options instance. + * @throws IllegalArgumentException if the size is not positive. + */ + public VoiceAgentLiveWebSocketConnectionOptions setMaxMessageSize(int bytes) { + options.setMaxMessageSize(bytes); + return this; + } + + /** + * Gets the receive queue overflow strategy. + * @return the strategy, default ERROR. + */ + public VoiceAgentWebSocketOverflowStrategy getOverflowStrategy() { + return options.getOverflowStrategy(); + } + + /** + * Sets the overflow strategy. Drop strategies explicitly permit event loss. + * @param strategy the non-null strategy. + * @return this options instance. + */ + public VoiceAgentLiveWebSocketConnectionOptions setOverflowStrategy(VoiceAgentWebSocketOverflowStrategy strategy) { + options.setOverflowStrategy(strategy); + return this; + } + + /** + * Gets the callback for malformed events. + * @return the callback, or null to terminate the session. + */ + public Consumer getMalformedEventHandler() { + return options.getMalformedEventHandler(); + } + + /** + * Sets a callback to report and skip malformed events. It runs on the receive thread and must not block. + * If the callback throws, the session terminates. Transport errors and oversized messages are not skipped. + * @param handler the callback, or null to terminate on malformed events. + * @return this options instance. + */ + public VoiceAgentLiveWebSocketConnectionOptions setMalformedEventHandler(Consumer handler) { + options.setMalformedEventHandler(handler); + return this; + } + + /** + * Gets the handshake timeout. + * @return the timeout, default 30 seconds. + */ + public Duration getHandshakeTimeout() { + return options.getHandshakeTimeout(); + } + + /** + * Sets the handshake timeout. + * @param timeout a non-null positive timeout. + * @return this options instance. + * @throws IllegalArgumentException if the timeout is not positive. + */ + public VoiceAgentLiveWebSocketConnectionOptions setHandshakeTimeout(Duration timeout) { + options.setHandshakeTimeout(timeout); + return this; + } + + /** + * Gets the close timeout. + * @return the timeout, default 10 seconds. + */ + public Duration getCloseTimeout() { + return options.getCloseTimeout(); + } + + /** + * Sets the close timeout. + * @param timeout a non-null positive timeout. + * @return this options instance. + * @throws IllegalArgumentException if the timeout is not positive. + */ + public VoiceAgentLiveWebSocketConnectionOptions setCloseTimeout(Duration timeout) { + options.setCloseTimeout(timeout); + return this; + } + + /** + * Gets the persistence override. + * @return the override, or null to use the persisted agent configuration. + */ + public Boolean isStoreEnabled() { + return options.isStoreEnabled(); + } + + /** + * Overrides Foundry conversation and audio persistence for this session. + * @param store the override, or null to use the persisted agent configuration. + * @return this options instance. + */ + public VoiceAgentLiveWebSocketConnectionOptions setStoreEnabled(Boolean store) { + options.setStoreEnabled(store); + return this; + } + + /** + * Gets the agent version override. + * @return the version override, or null. + */ + public String getAgentVersionOverride() { + return options.getAgentVersionOverride(); + } + + /** + * Sets the agent version to connect to. + * @param version the version override, or null to use the default version. + * @return this options instance. + */ + public VoiceAgentLiveWebSocketConnectionOptions setAgentVersionOverride(String version) { + options.setAgentVersionOverride(version); + return this; + } + + /** + * Gets structured inputs for instruction and greeting templates. + * @return the JSON inputs, or null. + */ + public String getStructuredInputs() { + return options.getStructuredInputs(); + } + + /** + * Sets the JSON inputs for instruction and greeting templates. + * The SDK URL-encodes this value into the {@code structured_inputs} query parameter. + * @param inputs the unencoded JSON inputs, or null. + * @return this options instance. + */ + public VoiceAgentLiveWebSocketConnectionOptions setStructuredInputs(String inputs) { + options.setStructuredInputs(inputs); + return this; + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java new file mode 100644 index 0000000000000..de82b687cae37 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import java.io.InputStream; +import java.util.Base64; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import reactor.core.publisher.Mono; +import reactor.core.publisher.Sinks; +import reactor.core.scheduler.Schedulers; + +/** + * Streams microphone and speaker audio over a native GPT Live WebSocket. + * Requires Java Sound devices supporting 24-kHz mono PCM16LE. Press Enter to finish. + * Audio capture and playback run on dedicated threads, not the WebSocket receive thread. + * Playback is bounded to 60 seconds; overflow and device failures terminate the call explicitly. + * The uniquely named agent is deleted unless FOUNDRY_KEEP_GPT_LIVE_AGENT=true. + * Requests an opening greeting and labels spoken transcripts by speaker. + * The standalone main method shuts down shared Reactor schedulers on exit; do not invoke it inside a shared application. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_GPT_LIVE_MODEL} - Optional. The GPT Live deployment name. Defaults to {@code gpt-live-1}.
    • + *
  • {@code FOUNDRY_GPT_LIVE_BACKEND_MODEL} - Optional. The Responses backend deployment name in the Live + * resource. Defaults to {@code gpt-5.6-terra}.
    • + *
  • {@code FOUNDRY_KEEP_GPT_LIVE_AGENT} - Optional. Set to {@code true} to keep the agent after the sample. + * Defaults to {@code false}.
    • + *
+ */ +public class GptLiveAudioConversationAsyncSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_GPT_LIVE_MODEL"); + String backendModel = configuration.get("FOUNDRY_GPT_LIVE_BACKEND_MODEL"); + boolean keepAgent = Boolean.parseBoolean(configuration.get("FOUNDRY_KEEP_GPT_LIVE_AGENT", "false")); + + if (endpoint == null || endpoint.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable FOUNDRY_PROJECT_ENDPOINT before running this sample."); + } + if (model == null) { + model = "gpt-live-1"; + } else if (model.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable FOUNDRY_GPT_LIVE_MODEL before running this sample."); + } + if (backendModel == null) { + backendModel = "gpt-5.6-terra"; + } else if (backendModel.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable FOUNDRY_GPT_LIVE_BACKEND_MODEL before running this sample."); + } + + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .allowPreview(true); + try (GptLiveSampleUtils.AgentResource agent + = GptLiveSampleUtils.createAgent(builder, model, backendModel, "audio-async", false, keepAgent)) { + Mono.usingWhen(agent.builder.beta().buildBetaVoiceAgentLiveWebSocketAsyncClient() + .openWebSocketSession(agent.name), GptLiveAudioConversationAsyncSample::runConversation, + BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync, + (session, error) -> session.closeAsync(), + BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(); + } finally { + Schedulers.shutdownNow(); + } + } + + static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session) { + return Mono.defer(() -> runConversation(session, + new VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor( + null, null, pcm -> session.sendEvent(GptLiveSampleUtils.audio(pcm.toBytes()))), System.in)); + } + + static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, + VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor processor, InputStream input) { + return Mono.usingWhen(Mono.just(processor), + audio -> { + AtomicBoolean started = new AtomicBoolean(); + AtomicBoolean finalized = new AtomicBoolean(); + GptLiveSampleUtils.AudioTranscriptPrinter transcripts = new GptLiveSampleUtils.AudioTranscriptPrinter(); + Sinks.Empty ready = Sinks.empty(); + Mono receive = session.receiveEvents().concatMap(data -> { + Map event = GptLiveSampleUtils.object(data); + transcripts.print(event); + if ("session.started".equals(GptLiveSampleUtils.type(event))) { + if (!started.compareAndSet(false, true)) { + return Mono.error(new IllegalStateException("Duplicate session.started event.")); + } + return Mono.fromRunnable(() -> { + audio.start(); + }).subscribeOn(Schedulers.boundedElastic()) + .then(session.sendEvent(GptLiveSampleUtils.userMessage( + "Greet me briefly with: Hi, how can I help you today?"))) + .then(session.sendEvent(GptLiveSampleUtils.event("response.create"))) + .then(Mono.fromRunnable(() -> { + ready.tryEmitEmpty(); + System.out.println("Opening greeting requested. Speak into the microphone. " + + "Press Enter to end the call."); + })).thenReturn(data); + } else if ("session.output_audio.delta".equals(GptLiveSampleUtils.type(event))) { + audio.queueAudio(decodeAudio(event)); + } else if ("session.closed".equals(GptLiveSampleUtils.type(event))) { + finalized.set(true); + } + return Mono.just(data); + }).takeUntil(data -> + "session.closed".equals(GptLiveSampleUtils.type(GptLiveSampleUtils.object(data)))) + .then(Mono.defer(() -> finalized.get() ? Mono.empty() + : Mono.error(new IllegalStateException("GPT Live disconnected without session.closed.")))); + Mono stop = ready.asMono().timeout(GptLiveSampleUtils.TIMEOUT) + .then(VoiceAgentLiveAudioConversationAsyncSample.waitForEnter(input)) + .then(Mono.fromRunnable(audio::close).subscribeOn(Schedulers.boundedElastic())) + .then(session.sendEvent(GptLiveSampleUtils.event("session.close"))) + .then(Mono.delay(GptLiveSampleUtils.TIMEOUT)) + .then(Mono.error(new IllegalStateException("Timed out waiting for session.closed."))); + return Mono.firstWithSignal(receive, audio.failureSignal(), stop); + }, GptLiveAudioConversationAsyncSample::closeAudio, + (audio, error) -> closeAudio(audio), GptLiveAudioConversationAsyncSample::closeAudio); + } + + static byte[] decodeAudio(Map event) { + byte[] pcm = Base64.getDecoder().decode(GptLiveSampleUtils.string(event, "delta")); + if (pcm.length % 2 != 0) { + throw new IllegalArgumentException("GPT Live PCM16 audio must contain complete two-byte samples."); + } + return pcm; + } + + private static Mono closeAudio(VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor audio) { + return Mono.fromRunnable(audio::close).subscribeOn(Schedulers.boundedElastic()); + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java new file mode 100644 index 0000000000000..08b60a99f75a9 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Executes a sample weather function requested by the GPT Live Responses backend. + * Weather results are explicitly simulated, not a call to a weather service. + * Uses response.event envelopes and native response.item.create function_call_output items. + * This is Responses delegation, not Foundry-managed-agent tool-result routing. + * The uniquely named agent is deleted unless FOUNDRY_KEEP_GPT_LIVE_AGENT=true. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_GPT_LIVE_MODEL} - Optional. The GPT Live deployment name. Defaults to {@code gpt-live-1}.
    • + *
  • {@code FOUNDRY_GPT_LIVE_BACKEND_MODEL} - Optional. The Responses backend deployment name in the Live + * resource. Defaults to {@code gpt-5.6-terra}.
    • + *
  • {@code FOUNDRY_KEEP_GPT_LIVE_AGENT} - Optional. Set to {@code true} to keep the agent after the sample. + * Defaults to {@code false}.
    • + *
+ */ +public class GptLiveFunctionToolSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_GPT_LIVE_MODEL"); + String backendModel = configuration.get("FOUNDRY_GPT_LIVE_BACKEND_MODEL"); + boolean keepAgent = Boolean.parseBoolean(configuration.get("FOUNDRY_KEEP_GPT_LIVE_AGENT", "false")); + + if (endpoint == null || endpoint.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable FOUNDRY_PROJECT_ENDPOINT before running this sample."); + } + if (model == null) { + model = "gpt-live-1"; + } else if (model.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable FOUNDRY_GPT_LIVE_MODEL before running this sample."); + } + if (backendModel == null) { + backendModel = "gpt-5.6-terra"; + } else if (backendModel.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable FOUNDRY_GPT_LIVE_BACKEND_MODEL before running this sample."); + } + + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .allowPreview(true); + try (GptLiveSampleUtils.AgentResource agent + = GptLiveSampleUtils.createAgent(builder, model, backendModel, "tools", true, keepAgent); + BetaVoiceAgentLiveWebSocketSessionClient session = agent.builder.beta() + .buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession(agent.name)) { + runConversation(session, args.length == 0 ? "What is the weather in Seattle?" : args[0]); + } + } + + static BinaryData weatherTool() { + Map schema = new LinkedHashMap<>(); + schema.put("type", "object"); + schema.put("properties", Collections.singletonMap("city", Collections.singletonMap("type", "string"))); + schema.put("required", Collections.singletonList("city")); + schema.put("additionalProperties", false); + Map tool = new LinkedHashMap<>(); + tool.put("type", "function"); + tool.put("name", "get_weather"); + tool.put("description", "Get simulated weather for a city. This sample returns demonstration data only."); + tool.put("parameters", schema); + tool.put("strict", true); + return BinaryData.fromObject(tool); + } + + static BinaryData toolResult(Map call) { + if (!"get_weather".equals(GptLiveSampleUtils.string(call, "name"))) { + throw new IllegalArgumentException("Unsupported sample tool: " + call.get("name")); + } + Map arguments = GptLiveSampleUtils.object( + BinaryData.fromString(GptLiveSampleUtils.string(call, "arguments"))); + Map result = new LinkedHashMap<>(); + result.put("city", GptLiveSampleUtils.string(arguments, "city")); + result.put("condition", "sunny"); + result.put("temperature_f", 72); + result.put("simulated", true); + Map item = new LinkedHashMap<>(); + item.put("type", "function_call_output"); + item.put("call_id", GptLiveSampleUtils.string(call, "call_id")); + item.put("output", BinaryData.fromObject(result).toString()); + return GptLiveSampleUtils.event("response.item.create", "item", item); + } + + static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, String prompt) { + Iterator events = session.receiveEvents(GptLiveSampleUtils.TIMEOUT).iterator(); + GptLiveTextConversationSample.awaitType(events, "session.started"); + session.sendEvent(GptLiveSampleUtils.userMessage(prompt)); + session.sendEvent(GptLiveSampleUtils.event("response.create")); + List results = new ArrayList<>(); + int continuations = 0; + boolean complete = false; + while (events.hasNext()) { + Map event = GptLiveSampleUtils.object(events.next()); + GptLiveSampleUtils.printEvent(event); + if ("session.closed".equals(GptLiveSampleUtils.type(event))) { + throw new IllegalStateException("GPT Live closed before the tool conversation completed."); + } + if (!"response.event".equals(GptLiveSampleUtils.type(event))) { + continue; + } + Map backend = GptLiveSampleUtils.backendEvent(event); + if ("response.output_item.done".equals(GptLiveSampleUtils.type(backend))) { + Map item = GptLiveSampleUtils.object(backend.get("item")); + if ("function_call".equals(GptLiveSampleUtils.type(item))) { + results.add(toolResult(item)); + } + } else if ("response.completed".equals(GptLiveSampleUtils.type(backend))) { + if (results.isEmpty()) { + complete = true; + break; + } + if (++continuations > 8) { + throw new IllegalStateException("Sample tool continuation limit exceeded."); + } + for (BinaryData result : results) { + session.sendEvent(result); + } + results.clear(); + session.sendEvent(GptLiveSampleUtils.event("response.create")); + } + } + if (!complete) { + throw new IllegalStateException("GPT Live disconnected before the tool conversation completed."); + } + session.sendEvent(GptLiveSampleUtils.event("session.close")); + GptLiveTextConversationSample.awaitType(events, "session.closed"); + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java new file mode 100644 index 0000000000000..bb1fe0420583c --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java @@ -0,0 +1,306 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.LiveInitialSessionAudioParam; +import com.azure.ai.agents.models.LiveResponsesDelegationParam; +import com.azure.ai.agents.models.LiveResponsesDelegationSettingsInputParam; +import com.azure.ai.agents.models.LiveSessionAudioFormatPCMParam; +import com.azure.ai.agents.models.LiveSessionAudioFormatPCMParamRate; +import com.azure.ai.agents.models.VoiceAgentLiveDefinition; +import com.azure.ai.agents.models.VoiceAgentSubProtocol; +import com.azure.core.util.BinaryData; +import com.azure.json.JsonProviders; +import com.azure.json.JsonWriter; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.time.Duration; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; + +final class GptLiveSampleUtils { + static final Duration TIMEOUT = Duration.ofSeconds(60); + + private GptLiveSampleUtils() { + } + + static Map object(BinaryData data) { + return data.toObject(Map.class); + } + + static Map object(Object value) { + if (!(value instanceof Map)) { + throw new IllegalArgumentException("Expected a native Live JSON object: " + value); + } + return (Map) value; + } + + static String type(Map event) { + return string(event, "type"); + } + + static String string(Map event, String field) { + Object value = event.get(field); + if (!(value instanceof String)) { + throw new IllegalArgumentException("Missing or invalid Live event field: " + field); + } + return (String) value; + } + + static BinaryData event(String type) { + return event(type, null, null); + } + + static BinaryData event(String type, String field, Object value) { + Map event = new LinkedHashMap<>(); + event.put("type", type); + if (field != null) { + event.put(field, value); + } + return BinaryData.fromObject(event); + } + + static BinaryData userMessage(String text) { + Map part = new LinkedHashMap<>(); + part.put("type", "input_text"); + part.put("text", text); + Map item = new LinkedHashMap<>(); + item.put("type", "message"); + item.put("role", "user"); + item.put("content", java.util.Collections.singletonList(part)); + return event("response.item.create", "item", item); + } + + static BinaryData audio(byte[] pcm) { + return event("session.input_audio.append", "audio", Base64.getEncoder().encodeToString(pcm)); + } + + static BinaryData commentary(String content) { + try (ByteArrayOutputStream output = new ByteArrayOutputStream(); + JsonWriter writer = JsonProviders.createWriter(output)) { + writer.writeStartObject() + .writeStringField("type", "session.commentary.append") + .writeStringField("content", content) + .writeNullField("delegation_id") + .writeEndObject() + .flush(); + return BinaryData.fromBytes(output.toByteArray()); + } catch (IOException error) { + throw new IllegalStateException("Unable to serialize GPT Live commentary.", error); + } + } + + static Map backendEvent(Map event) { + return "response.event".equals(type(event)) ? object(event.get("event")) : event; + } + + static void checkError(Map event) { + if ("error".equals(type(event)) || "response.failed".equals(type(event)) + || "response.incomplete".equals(type(event))) { + Object error = event.get("error"); + if (error instanceof Map && ("content_filter".equals(((Map) error).get("code")) + || "content_filter".equals(((Map) error).get("type")))) { + throw new IllegalStateException("GPT Live service error: generation stopped by service moderation " + + "(content_filter). The call has ended; review the conversation and service diagnostics. Event: " + + BinaryData.fromObject(event)); + } + throw new IllegalStateException("GPT Live service error: " + BinaryData.fromObject(event)); + } + } + + static void printEvent(Map event) { + checkError(event); + if ("session.started".equals(type(event))) { + Object conversationId = event.get("conversation_id"); + System.out.println(conversationId == null ? "Live session started." + : "Live session started; conversation_id=" + conversationId); + } else if ("session.input_transcript.delta".equals(type(event))) { + System.out.print(string(event, "delta")); + } else if ("session.output_transcript.delta".equals(type(event))) { + System.out.print(string(event, "delta")); + } else if ("response.event".equals(type(event))) { + Map backend = backendEvent(event); + checkError(backend); + if ("response.output_text.delta".equals(type(backend))) { + System.out.print(string(backend, "delta")); + } + } + } + + static final class AudioTranscriptPrinter { + static final int MAX_BUFFERED_TRANSCRIPT_CHARACTERS = 65536; + private String speaker; + private boolean readingInput; + private boolean lineOpen; + private final boolean inputEchoed; + private final boolean completeUtterances; + private final StringBuilder pending = new StringBuilder(); + private final StringBuilder utterance = new StringBuilder(); + + AudioTranscriptPrinter() { + this(false); + } + + AudioTranscriptPrinter(boolean inputEchoed) { + this(inputEchoed, false); + } + + AudioTranscriptPrinter(boolean inputEchoed, boolean completeUtterances) { + this.inputEchoed = inputEchoed; + this.completeUtterances = completeUtterances; + } + + synchronized void beginInput() { + if (readingInput) { + throw new IllegalStateException("A console read is already active."); + } + if (lineOpen) { + System.out.println(); + } + speaker = null; + lineOpen = false; + readingInput = true; + System.out.print("You: "); + System.out.flush(); + } + + synchronized void endInput() { + if (!readingInput) { + return; + } + readingInput = false; + if (!inputEchoed) { + System.out.println(); + } + if (pending.length() > 0) { + System.out.print(pending); + pending.setLength(0); + } + speaker = null; + } + + synchronized void beginTurn() { + speaker = null; + } + + synchronized void endUtterance() { + if (utterance.length() == 0) { + return; + } + String spokenText = utterance.toString().trim(); + if (spokenText.isEmpty()) { + utterance.setLength(0); + return; + } + String text = (lineOpen ? System.lineSeparator() : "") + "Assistant: " + spokenText + + System.lineSeparator(); + if (readingInput && text.length() > MAX_BUFFERED_TRANSCRIPT_CHARACTERS - pending.length()) { + throw new IllegalStateException("Buffered transcript exceeded 65536 characters while reading input."); + } + utterance.setLength(0); + speaker = null; + write(text); + } + + synchronized void print(Map event) { + checkError(event); + String eventType = type(event); + if ("response.event".equals(eventType)) { + checkError(backendEvent(event)); + } else if ("session.input_transcript.delta".equals(eventType) + || "session.output_transcript.delta".equals(eventType)) { + if (completeUtterances && "session.output_transcript.delta".equals(eventType)) { + String delta = string(event, "delta"); + if (delta.length() > MAX_BUFFERED_TRANSCRIPT_CHARACTERS - pending.length() - utterance.length()) { + throw new IllegalStateException("Buffered transcript exceeded 65536 characters."); + } + utterance.append(delta); + return; + } + String nextSpeaker = "session.input_transcript.delta".equals(eventType) ? "You" : "Assistant"; + if (!nextSpeaker.equals(speaker)) { + write((lineOpen ? System.lineSeparator() : "") + nextSpeaker + ": "); + speaker = nextSpeaker; + } + write(string(event, "delta")); + } else if ("session.started".equals(eventType)) { + printEvent(event); + } else if ("session.closed".equals(eventType)) { + endUtterance(); + if (lineOpen) { + write(System.lineSeparator()); + } + speaker = null; + } + } + + private void write(String text) { + if (readingInput) { + if (text.length() > MAX_BUFFERED_TRANSCRIPT_CHARACTERS - pending.length() - utterance.length()) { + throw new IllegalStateException("Buffered transcript exceeded 65536 characters while reading input."); + } + pending.append(text); + } else { + System.out.print(text); + } + if (!text.isEmpty()) { + lineOpen = !text.endsWith("\n"); + } + } + } + + static AgentResource createAgent(AgentsClientBuilder builder, String model, String backendModel, + String scenario, boolean tools, boolean keepAgent) { + LiveResponsesDelegationSettingsInputParam responses = new LiveResponsesDelegationSettingsInputParam(backendModel) + .setInstructions(tools + ? "Use get_weather for weather requests. After receiving its result, answer briefly." + : "Answer the user briefly and helpfully."); + if (tools) { + responses.setTools(java.util.Collections.singletonList(GptLiveFunctionToolSample.weatherTool())); + } + VoiceAgentLiveDefinition definition = new VoiceAgentLiveDefinition(VoiceAgentSubProtocol.LIVE, model) + .setInstructions("You are a helpful voice assistant. Delegate requests to your Responses backend.") + .setAudio(new LiveInitialSessionAudioParam().setFormat(new LiveSessionAudioFormatPCMParam( + LiveSessionAudioFormatPCMParamRate.TWO_FOUR_ZERO_ZERO_ZERO))) + .setDelegation(BinaryData.fromObject(new LiveResponsesDelegationParam(responses))); + String agentName = createAgentName(scenario); + AgentsClient agents = builder.buildAgentsClient(); + agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition)); + System.out.println("Created GPT Live agent: " + agentName); + return new AgentResource(builder, agents, agentName, keepAgent); + } + + static String createAgentName(String scenario) { + return "gpt-live-" + scenario + "-" + UUID.randomUUID(); + } + + static final class AgentResource implements AutoCloseable { + final AgentsClientBuilder builder; + final String name; + private final AgentsClient agents; + private final boolean keep; + + AgentResource(AgentsClientBuilder builder, AgentsClient agents, String name, boolean keep) { + this.builder = builder; + this.agents = agents; + this.name = name; + this.keep = keep; + } + + @Override + public void close() { + if (keep) { + System.out.println("Kept GPT Live agent: " + name); + } else { + agents.deleteAgent(name); + System.out.println("Deleted GPT Live agent: " + name); + } + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java new file mode 100644 index 0000000000000..cbffe655726c0 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java @@ -0,0 +1,347 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import java.io.BufferedReader; +import java.io.Console; +import java.io.InputStreamReader; +import java.io.Reader; +import java.time.Duration; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.function.Consumer; +import java.util.function.Supplier; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.publisher.Sinks; +import reactor.core.scheduler.Schedulers; + +/** + * Runs a nonblocking, multi-turn GPT Live text conversation using Responses delegation. + * Type messages interactively; a blank line, exit or quit ends the session. + * Alternatively, pass prompts as command-line arguments for a scripted conversation. + * Console lines are read on a worker, off the WebSocket thread. + * After a remote disconnect, a pending console read may require Enter to release the input worker. + * Interactive mode requests a spoken greeting, plays 24-kHz PCM16 replies, and prints the voice frontend's transcripts. + * Before the first prompt, it waits for greeting audio and text, one second of speech inactivity, and speaker drain. + * The quiet-period boundary is a heuristic, not a service speech-completion event. + * Assistant transcript fragments are assembled into utterances using the speech-inactivity heuristic. + * Completed utterances wait until Enter if input is active; incomplete text is not flushed by input submission. + * Audio continues playing while typing on the You: line. + * It streams real-time silent PCM input instead of a microphone; voice usage continues while you type. + * Backend completion allows another typed prompt; it does not mean that spoken output has finished. + * Scripted mode prints backend text without requiring a speaker. + * Agent creation/deletion run outside the asynchronous session on the main thread. + * The uniquely named agent is deleted unless FOUNDRY_KEEP_GPT_LIVE_AGENT=true. + * The standalone main method shuts down shared Reactor schedulers on exit; do not invoke it inside a shared application. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_GPT_LIVE_MODEL} - Optional. The GPT Live deployment name. Defaults to {@code gpt-live-1}.
    • + *
  • {@code FOUNDRY_GPT_LIVE_BACKEND_MODEL} - Optional. The Responses backend deployment name in the Live + * resource. Defaults to {@code gpt-5.6-terra}.
    • + *
  • {@code FOUNDRY_KEEP_GPT_LIVE_AGENT} - Optional. Set to {@code true} to keep the agent after the sample. + * Defaults to {@code false}.
    • + *
+ */ +public class GptLiveTextConversationAsyncSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_GPT_LIVE_MODEL"); + String backendModel = configuration.get("FOUNDRY_GPT_LIVE_BACKEND_MODEL"); + boolean keepAgent = Boolean.parseBoolean(configuration.get("FOUNDRY_KEEP_GPT_LIVE_AGENT", "false")); + + if (endpoint == null || endpoint.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable FOUNDRY_PROJECT_ENDPOINT before running this sample."); + } + if (model == null) { + model = "gpt-live-1"; + } else if (model.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable FOUNDRY_GPT_LIVE_MODEL before running this sample."); + } + if (backendModel == null) { + backendModel = "gpt-5.6-terra"; + } else if (backendModel.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable FOUNDRY_GPT_LIVE_BACKEND_MODEL before running this sample."); + } + + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .allowPreview(true); + Console console = System.console(); + try (GptLiveSampleUtils.AgentResource agent + = GptLiveSampleUtils.createAgent(builder, model, backendModel, "text-async", false, keepAgent)) { + Mono.usingWhen(agent.builder.beta().buildBetaVoiceAgentLiveWebSocketAsyncClient() + .openWebSocketSession(agent.name), + session -> args.length == 0 + ? runConversation(session, console == null ? new InputStreamReader(System.in) : console.reader()) + : runConversation(session, Arrays.asList(args)), + BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync, + (session, error) -> session.closeAsync(), + BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(); + } finally { + Schedulers.shutdownNow(); + } + } + + static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, List prompts) { + return Mono.defer(() -> { + Iterator pending = prompts.iterator(); + return runConversation(session, transcripts -> Mono.defer(() -> { + if (!pending.hasNext()) { + return Mono.empty(); + } + String prompt = pending.next(); + System.out.println("You: " + prompt); + return Mono.just(prompt); + }), pcm -> { }, () -> { }, false); + }); + } + + static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input) { + return Mono.usingWhen( + Mono.fromCallable(VoiceAgentRealtimeSampleUtils.SpeakerPlayer::new) + .subscribeOn(Schedulers.boundedElastic()), + player -> runConversation(session, input, player::play, player::drain), + player -> Mono.fromRunnable(() -> { + boolean speakerEnabled = player.isEnabled(); + try { + player.close(); + } finally { + System.out.printf("(received %.2fs of reply audio%s)%n", + player.getSecondsReceived(), speakerEnabled ? " and played it" : ""); + } + }).subscribeOn(Schedulers.boundedElastic())); + } + + static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input, + Consumer playAudio) { + return runConversation(session, input, playAudio, () -> { }); + } + + static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input, + Consumer playAudio, Runnable drainAudio) { + return Mono.defer(() -> runInteractiveConversation(session.receiveEvents(), session::sendEvent, + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), + input, playAudio, drainAudio)); + } + + static Mono runInteractiveConversation(Flux events, + Function> sendEvent, Supplier closeDetails, Reader input, + Consumer playAudio, Runnable drainAudio) { + return Mono.defer(() -> { + System.out.println("Type a message and press Enter. Blank line, 'exit' or 'quit' ends the session."); + System.out.println("Interactive voice streams silent audio while you type; voice usage is billable."); + BufferedReader console = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); + return runConversation(events, sendEvent, closeDetails, transcripts -> readPrompt(console, transcripts), + playAudio, drainAudio, true); + }); + } + + private static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, + Function> nextPrompt, + Consumer playAudio, Runnable drainAudio, boolean spoken) { + return Mono.defer(() -> runConversation(session.receiveEvents(), session::sendEvent, + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), + nextPrompt, playAudio, drainAudio, spoken)); + } + + private static Mono runConversation(Flux events, + Function> sendEvent, Supplier closeDetails, + Function> nextPrompt, + Consumer playAudio, Runnable drainAudio, boolean spoken) { + return Mono.defer(() -> { + AtomicBoolean finalized = new AtomicBoolean(); + AtomicBoolean started = new AtomicBoolean(); + AtomicBoolean closeRequested = new AtomicBoolean(); + AtomicBoolean remoteHangup = new AtomicBoolean(); + GptLiveSampleUtils.AudioTranscriptPrinter transcripts + = new GptLiveSampleUtils.AudioTranscriptPrinter(System.console() != null, spoken); + Greeting greeting = new Greeting(); + Sinks.Empty ready = Sinks.empty(); + Sinks.Empty closed = Sinks.empty(); + Sinks.Empty audioStarted = Sinks.empty(); + Sinks.Empty stopAudio = Sinks.empty(); + Sinks.Empty audioStopped = Sinks.empty(); + AtomicReference> responseCompleted = new AtomicReference<>(); + Mono receive = events + .publishOn(Schedulers.boundedElastic(), 1) + .doOnNext(data -> { + Map event = GptLiveSampleUtils.object(data); + String eventType = GptLiveSampleUtils.type(event); + if (spoken) { + transcripts.print(event); + if ("session.output_audio.delta".equals(eventType)) { + byte[] pcm = GptLiveAudioConversationAsyncSample.decodeAudio(event); + playAudio.accept(pcm); + greeting.onAudio(pcm); + } else if ("session.output_transcript.delta".equals(eventType) + && !GptLiveSampleUtils.string(event, "delta").trim().isEmpty()) { + greeting.onTranscript(); + } + } else { + GptLiveSampleUtils.printEvent(event); + } + String type = GptLiveSampleUtils.type(GptLiveSampleUtils.backendEvent(event)); + if ("session.started".equals(type)) { + if (!started.compareAndSet(false, true)) { + throw new IllegalStateException("Duplicate session.started event."); + } + ready.tryEmitEmpty(); + } else if ("response.completed".equals(type)) { + Sinks.Empty completion = responseCompleted.getAndSet(null); + if (spoken && started.get() && completion == null) { + return; + } + if (!started.get() || closeRequested.get() || completion == null) { + throw new IllegalStateException("Unexpected GPT Live response lifecycle event."); + } + if (!spoken) { + System.out.println(); + } + completion.tryEmitEmpty(); + } else if ("session.closed".equals(type)) { + boolean remote = started.get() && "remote_hangup".equals(event.get("reason")); + boolean requested = closeRequested.get() && "close_requested".equals(event.get("reason")); + if (!requested && !remote) { + throw new IllegalStateException("GPT Live closed before all turns completed. Event: " + + data + "; " + closeDetails.get()); + } + remoteHangup.set(remote); + System.out.println("Live session closed (" + event.get("reason") + ")."); + finalized.set(true); + closed.tryEmitEmpty(); + } + }) + .takeUntil(data -> "session.closed".equals(GptLiveSampleUtils.type(GptLiveSampleUtils.object(data)))) + .then(Mono.defer(() -> remoteHangup.get() ? Mono.empty() : finalized.get() ? Mono.never() + : Mono.error(new IllegalStateException("GPT Live disconnected without session.closed.")))); + Mono audio = ready.asMono() + .thenMany(silenceFrames().takeUntilOther(stopAudio.asMono()) + .concatMap(frame -> sendEvent.apply(frame).doOnSuccess(ignored -> audioStarted.tryEmitEmpty()), 1)) + .then() + .doOnSuccess(ignored -> audioStopped.tryEmitEmpty()) + .then(Mono.never()); + Mono renderTranscripts = greeting.quietPeriods() + .doOnNext(ignored -> transcripts.endUtterance()).then(); + Mono conversation = ready.asMono().timeout(GptLiveSampleUtils.TIMEOUT) + .then(Mono.defer(() -> spoken ? audioStarted.asMono().timeout(GptLiveSampleUtils.TIMEOUT) : Mono.empty())) + .then(Mono.defer(() -> spoken + ? sendEvent.apply(GptLiveSampleUtils.commentary("Hi, how can I help you today?")) + .timeout(GptLiveSampleUtils.TIMEOUT) + .then(greeting.awaitPlayback(drainAudio)) + .then(Mono.fromRunnable(transcripts::endUtterance)) + : Mono.empty())) + .then(Mono.defer(() -> prompt(sendEvent, nextPrompt, responseCompleted, spoken, transcripts))) + .then(Mono.defer(() -> { + closeRequested.set(true); + stopAudio.tryEmitEmpty(); + return (spoken ? audioStopped.asMono() : Mono.empty()) + .then(sendEvent.apply(GptLiveSampleUtils.event("session.close"))) + .then(closed.asMono()).timeout(GptLiveSampleUtils.TIMEOUT); + })); + return spoken ? Mono.firstWithSignal(receive, audio, renderTranscripts, conversation) + : Mono.firstWithSignal(receive, conversation); + }); + } + + private static Mono prompt(Function> sendEvent, + Function> nextPrompt, + AtomicReference> responseCompleted, boolean spoken, + GptLiveSampleUtils.AudioTranscriptPrinter transcripts) { + return nextPrompt.apply(transcripts).flatMap(text -> { + Sinks.Empty completion = Sinks.empty(); + responseCompleted.set(completion); + if (!spoken) { + System.out.print("Assistant: "); + System.out.flush(); + } + return sendEvent.apply(GptLiveSampleUtils.userMessage(text)) + .then(sendEvent.apply(GptLiveSampleUtils.event("response.create"))) + .then(completion.asMono()).timeout(GptLiveSampleUtils.TIMEOUT) + .thenReturn(true); + }).defaultIfEmpty(false) + .flatMap(keepGoing -> keepGoing + ? Mono.defer(() -> prompt(sendEvent, nextPrompt, responseCompleted, spoken, transcripts)) : Mono.empty()); + } + + static Flux silenceFrames() { + // Twenty milliseconds of 24-kHz mono PCM16LE. + BinaryData silence = GptLiveSampleUtils.audio(new byte[960]); + return Mono.just(silence) + .concatWith(Mono.just(silence).delayElement(Duration.ofMillis(20)).repeat()); + } + + static final class Greeting { + // PCM16 peak levels up to 256 (about -42 dBFS) are treated as silence for this heuristic. + static final int SILENCE_PEAK_THRESHOLD = 256; + private final Sinks.Many progress = Sinks.many().replay().latest(); + private int received; + + void onAudio(byte[] pcm) { + for (int offset = 0; offset < pcm.length; offset += 2) { + int sample = (short) ((pcm[offset] & 0xff) | (pcm[offset + 1] << 8)); + if (sample < -SILENCE_PEAK_THRESHOLD || sample > SILENCE_PEAK_THRESHOLD) { + received |= 1; + progress.tryEmitNext(received); + return; + } + } + } + + void onTranscript() { + received |= 2; + progress.tryEmitNext(received); + } + + Mono awaitPlayback(Runnable drainAudio) { + // Output can continue with silent PCM after speaking; only speech activity restarts this timer. + return quietPeriods().next() + .timeout(GptLiveSampleUtils.TIMEOUT) + .then(Mono.fromRunnable(drainAudio).subscribeOn(Schedulers.boundedElastic())); + } + + Flux quietPeriods() { + return progress.asFlux().filter(parts -> parts == 3) + .switchMap(parts -> Mono.delay(Duration.ofSeconds(1))); + } + } + + static Mono readPrompt(Reader input) { + return readPrompt(input, new GptLiveSampleUtils.AudioTranscriptPrinter()); + } + + private static Mono readPrompt(Reader input, GptLiveSampleUtils.AudioTranscriptPrinter transcripts) { + return Mono.defer(() -> { + BufferedReader reader = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); + return Mono.fromCallable(() -> { + transcripts.beginInput(); + String line; + try { + line = reader.readLine(); + } finally { + transcripts.endInput(); + } + if (line == null) { + return null; + } + String text = line.trim(); + return text.isEmpty() || "exit".equalsIgnoreCase(text) || "quit".equalsIgnoreCase(text) ? null : text; + }).subscribeOn(Schedulers.boundedElastic()) + .doOnCancel(transcripts::endInput); + }); + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java new file mode 100644 index 0000000000000..226d2ab3211cd --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import java.io.Console; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import java.util.function.Consumer; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +/** + * Runs a synchronous, multi-turn GPT Live text conversation using Responses delegation. + * Type messages interactively; a blank line, exit or quit ends the session. + * Alternatively, pass prompts as command-line arguments for a scripted conversation. + * Interactive mode plays a spoken greeting and replies, and prints frontend transcripts instead of backend notes. + * It streams silent PCM while typing; voice usage is billable. Greeting completion uses the same speech-inactivity + * heuristic and speaker drain as the asynchronous sample. Assistant transcript fragments are assembled into + * utterances independently of typed turns; completed text waits until Enter when a You: line is active. + * Synchronous session I/O runs on workers using the asynchronous sample's shared voice-loop coordination. + * After a remote disconnect, a pending console read may require Enter to release the input worker. + * Scripted command-line prompts print backend text without speaker playback. + * A uniquely named agent is deleted on exit unless FOUNDRY_KEEP_GPT_LIVE_AGENT=true. + * The standalone main method shuts down shared Reactor schedulers on exit; do not invoke it inside a shared application. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_GPT_LIVE_MODEL} - Optional. The GPT Live deployment name. Defaults to {@code gpt-live-1}.
    • + *
  • {@code FOUNDRY_GPT_LIVE_BACKEND_MODEL} - Optional. The Responses backend deployment name in the Live + * resource. Defaults to {@code gpt-5.6-terra}.
    • + *
  • {@code FOUNDRY_KEEP_GPT_LIVE_AGENT} - Optional. Set to {@code true} to keep the agent after the sample. + * Defaults to {@code false}.
    • + *
+ */ +public class GptLiveTextConversationSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_GPT_LIVE_MODEL"); + String backendModel = configuration.get("FOUNDRY_GPT_LIVE_BACKEND_MODEL"); + boolean keepAgent = Boolean.parseBoolean(configuration.get("FOUNDRY_KEEP_GPT_LIVE_AGENT", "false")); + + if (endpoint == null || endpoint.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable FOUNDRY_PROJECT_ENDPOINT before running this sample."); + } + if (model == null) { + model = "gpt-live-1"; + } else if (model.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable FOUNDRY_GPT_LIVE_MODEL before running this sample."); + } + if (backendModel == null) { + backendModel = "gpt-5.6-terra"; + } else if (backendModel.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable FOUNDRY_GPT_LIVE_BACKEND_MODEL before running this sample."); + } + + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .allowPreview(true); + try (GptLiveSampleUtils.AgentResource agent + = GptLiveSampleUtils.createAgent(builder, model, backendModel, "text", false, keepAgent); + BetaVoiceAgentLiveWebSocketSessionClient session = agent.builder.beta() + .buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession(agent.name)) { + if (args.length == 0) { + Console console = System.console(); + runConversation(session, console == null ? new InputStreamReader(System.in) : console.reader()); + } else { + runConversation(session, Arrays.asList(args)); + } + } finally { + Schedulers.shutdownNow(); + } + } + + static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, List prompts) { + Iterator pending = prompts.iterator(); + runConversation(session, () -> { + if (!pending.hasNext()) { + return null; + } + String prompt = pending.next(); + System.out.println("You: " + prompt); + return prompt; + }); + } + + static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Reader input) { + try (VoiceAgentRealtimeSampleUtils.SpeakerPlayer player = new VoiceAgentRealtimeSampleUtils.SpeakerPlayer()) { + runConversation(session, input, player::play, player::drain); + player.drain(); + System.out.printf("(received %.2fs of reply audio%s)%n", player.getSecondsReceived(), + player.isEnabled() ? " and played it" : ""); + } + } + + static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Reader input, + Consumer playAudio, Runnable drainAudio) { + Flux events = Flux.defer(() -> Flux.fromIterable(session.receiveEvents(GptLiveSampleUtils.TIMEOUT))) + .subscribeOn(Schedulers.boundedElastic()); + GptLiveTextConversationAsyncSample.runInteractiveConversation(events, + event -> Mono.fromRunnable(() -> session.sendEvent(event)).subscribeOn(Schedulers.boundedElastic()), + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), + input, playAudio, drainAudio) + .onErrorMap(IOException.class, error -> new IllegalStateException("Unable to read console input.", error)) + .block(); + } + + private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Supplier nextPrompt) { + Iterator events = session.receiveEvents(GptLiveSampleUtils.TIMEOUT).iterator(); + awaitType(events, "session.started"); + String prompt; + while ((prompt = nextPrompt.get()) != null) { + System.out.print("Assistant: "); + session.sendEvent(GptLiveSampleUtils.userMessage(prompt)); + session.sendEvent(GptLiveSampleUtils.event("response.create")); + awaitType(events, "response.completed"); + System.out.println(); + } + session.sendEvent(GptLiveSampleUtils.event("session.close")); + awaitType(events, "session.closed"); + } + + static void awaitType(Iterator events, String expected) { + while (events.hasNext()) { + Map event = GptLiveSampleUtils.object(events.next()); + GptLiveSampleUtils.printEvent(event); + if (expected.equals(GptLiveSampleUtils.type(GptLiveSampleUtils.backendEvent(event)))) { + return; + } + if ("session.closed".equals(GptLiveSampleUtils.type(event))) { + throw new IllegalStateException("GPT Live session closed before " + expected); + } + } + throw new IllegalStateException("GPT Live connection ended before " + expected); + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java index 2a80cc060fbae..baaf6ff0f9481 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java @@ -42,6 +42,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; /** * Demonstrates an asynchronous hands-free voice conversation using Java Sound and server-side VAD. @@ -195,7 +196,7 @@ static final class AudioProcessor implements AutoCloseable { private static final int CHUNK_BYTES = 2400; static final int MAX_PLAYBACK_BYTES = VoiceAgentRealtimeSampleUtils.SAMPLE_RATE * 2 * 60; private static final byte[] STOP = new byte[0]; - private final BetaVoiceAgentWebSocketSessionAsyncClient session; + private final Function> sendAudio; private final AudioFormat format = new AudioFormat(VoiceAgentRealtimeSampleUtils.SAMPLE_RATE, 16, 1, true, false); private final BlockingQueue playback = new LinkedBlockingQueue<>(MAX_PLAYBACK_BYTES / 2); private int queuedPlaybackBytes; @@ -212,11 +213,19 @@ static final class AudioProcessor implements AutoCloseable { } AudioProcessor(BetaVoiceAgentWebSocketSessionAsyncClient session, TargetDataLine microphone, SourceDataLine speaker) { - this.session = session; + this(microphone, speaker, pcm -> session.appendInputAudio(pcm)); + } + + AudioProcessor(TargetDataLine microphone, SourceDataLine speaker, Function> sendAudio) { + this.sendAudio = sendAudio; this.microphone = microphone; this.speaker = speaker; } + Mono failureSignal() { + return failure.asMono(); + } + synchronized void start() { if (closed) { throw new IllegalStateException("Audio processor is already closed."); @@ -252,7 +261,7 @@ private void capture() { while (running.get()) { int read = microphone.read(buffer, 0, buffer.length); if (read > 0 && running.get()) { - session.appendInputAudio(BinaryData.fromBytes(Arrays.copyOf(buffer, read))).block(SEND_TIMEOUT); + sendAudio.apply(BinaryData.fromBytes(Arrays.copyOf(buffer, read))).block(SEND_TIMEOUT); } } } catch (RuntimeException error) { diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java index 6b9ce05b1c00c..9903d76639c14 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java @@ -16,6 +16,7 @@ import com.azure.ai.agents.models.VoiceAgentDefinition; import com.azure.core.util.BinaryData; import com.azure.core.util.Configuration; +import com.azure.core.exception.ResourceModifiedException; import com.azure.identity.DefaultAzureCredentialBuilder; import java.time.Duration; @@ -40,6 +41,8 @@ *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - Optional. The voice agent name. Defaults to * {@code sample-live-text-conversation-agent-java}.
    • * + *

    This sample deletes and replaces an existing agent with the selected name on a creation conflict. + * Use a name reserved for this sample. The replacement is deleted on exit, including on failure.

    */ public class VoiceAgentLiveTextConversationSample { private static final Duration RESPONSE_TIMEOUT = Duration.ofSeconds(45); @@ -59,13 +62,8 @@ public static void main(String[] args) { BetaVoiceAgentWebSocketClient realtime = builder.beta().buildBetaVoiceAgentWebSocketClient(); BetaVoiceAgentsConversationsClient conversations = builder.beta().buildBetaVoiceAgentsConversationsClient(); - Map request = new LinkedHashMap<>(); - request.put("kind", "voice"); - request.put("name", agentName); - AgentDetails generated = betaAgents.createAgentFromPrompt(BinaryData.fromObject(request)); - - try { - AgentVersionDetails latest = generated.getVersions().getLatest(); + try (SampleAgent agent = createAgent(agents, betaAgents, agentName)) { + AgentVersionDetails latest = agent.details.getVersions().getLatest(); VoiceAgentDefinition definition = (VoiceAgentDefinition) latest.getDefinition(); agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition.setStore(true))); @@ -124,9 +122,43 @@ public static void main(String[] args) { } else { System.out.println("No persisted conversation ID was returned."); } - } finally { + } + } + + static SampleAgent createAgent(AgentsClient agents, BetaAgentsClient betaAgents, String agentName) { + Map request = new LinkedHashMap<>(); + request.put("kind", "voice"); + request.put("name", agentName); + BinaryData body = BinaryData.fromObject(request); + AgentDetails details; + try { + details = betaAgents.createAgentFromPrompt(body); + } catch (ResourceModifiedException error) { + if (error.getResponse() == null || error.getResponse().getStatusCode() != 409) { + throw error; + } + System.err.println("Deleting existing voice agent before replacement: " + agentName); agents.deleteAgent(agentName); - System.out.println("Deleted voice agent: " + agentName); + details = betaAgents.createAgentFromPrompt(body); + } + return new SampleAgent(agents, agentName, details); + } + + static final class SampleAgent implements AutoCloseable { + final AgentDetails details; + private final AgentsClient agents; + private final String name; + + private SampleAgent(AgentsClient agents, String name, AgentDetails details) { + this.agents = agents; + this.name = name; + this.details = details; + } + + @Override + public void close() { + agents.deleteAgent(name); + System.out.println("Deleted voice agent: " + name); } } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java index 6ce3f33a22d51..0e905c78c1142 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java @@ -139,6 +139,12 @@ boolean isEnabled() { return line != null; } + synchronized void drain() { + if (line != null) { + line.drain(); + } + } + @Override public synchronized void close() { if (line != null) { diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java new file mode 100644 index 0000000000000..355d4be9fcc53 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java @@ -0,0 +1,1154 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; +import com.azure.core.credential.AccessToken; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import java.io.ByteArrayOutputStream; +import java.io.FilterReader; +import java.io.InputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.io.Reader; +import java.io.PipedReader; +import java.io.PipedWriter; +import java.io.StringReader; +import java.lang.reflect.Proxy; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.stream.Stream; +import javax.sound.sampled.SourceDataLine; +import javax.sound.sampled.TargetDataLine; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.publisher.Sinks; +import reactor.core.scheduler.Schedulers; +import reactor.netty.DisposableServer; +import reactor.test.StepVerifier; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ResourceLock("voice-agent-websocket-tls") +@Timeout(20) +public class GptLiveSamplesTests { + private static final Duration TIMEOUT = Duration.ofSeconds(5); + private static final String STARTED = "{\"type\":\"session.started\",\"session\":{\"id\":\"live-1\"}}"; + private static final String CLOSED = "{\"type\":\"session.closed\",\"reason\":\"close_requested\"}"; + private static final String COMPLETED = "{\"type\":\"response.event\",\"delegation_id\":\"del-1\",\"event\":" + + "{\"type\":\"response.completed\",\"response\":{\"id\":\"resp-1\",\"output\":[]}}}"; + private static final String MODERATED = "{\"type\":\"error\",\"event_id\":\"moderated-1\"," + + "\"error\":{\"type\":\"content_filter\",\"code\":\"content_filter\"," + + "\"message\":\"The generation was stopped due to moderation.\"}}"; + private final List> commands = new CopyOnWriteArrayList<>(); + private DisposableServer server; + + @ParameterizedTest + @ValueSource(strings = { "text", "text-async", "audio-async", "tools" }) + public void agentNamesStayBelowServiceLimit(String scenario) { + String prefix = "gpt-live-" + scenario + "-"; + String name = GptLiveSampleUtils.createAgentName(scenario); + assertTrue(name.startsWith(prefix)); + assertTrue(name.length() <= 57, "Sample agent names must leave headroom below the 63-character limit"); + assertTrue(name.matches("[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9]")); + assertEquals(36, name.substring(prefix.length()).length()); + assertEquals(4, UUID.fromString(name.substring(prefix.length())).version()); + assertFalse(name.equals(GptLiveSampleUtils.createAgentName(scenario))); + } + + @Test + @ResourceLock("java.lang.System.out") + public void audioTranscriptsLabelSpeakersWithoutPrintingBackendText() { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream original = System.out; + try (PrintStream capture = new PrintStream(output)) { + System.setOut(capture); + GptLiveSampleUtils.AudioTranscriptPrinter printer = new GptLiveSampleUtils.AudioTranscriptPrinter(); + printer.print(GptLiveSampleUtils.object(BinaryData.fromString(STARTED))); + printer.print(GptLiveSampleUtils + .object(GptLiveSampleUtils.event("session.input_transcript.delta", "delta", "Hello"))); + printer.print(GptLiveSampleUtils + .object(GptLiveSampleUtils.event("session.input_transcript.delta", "delta", " there"))); + printer.print(GptLiveSampleUtils.object(BinaryData.fromString("{\"type\":\"response.event\"," + + "\"event\":{\"type\":\"response.output_text.delta\",\"delta\":\"backend-only text\"}}"))); + printer.print( + GptLiveSampleUtils.object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Hi!"))); + printer.beginTurn(); + printer.print(GptLiveSampleUtils + .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "How can I help?"))); + } finally { + System.setOut(original); + } + String transcript = new String(output.toByteArray(), StandardCharsets.UTF_8); + assertEquals("Live session started." + System.lineSeparator() + "You: Hello there" + System.lineSeparator() + + "Assistant: Hi!" + System.lineSeparator() + "Assistant: How can I help?", transcript); + } + + @Test + @ResourceLock("java.lang.System.out") + public void transcriptBufferIsBoundedWhileReadingInput() { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream original = System.out; + GptLiveSampleUtils.AudioTranscriptPrinter printer = new GptLiveSampleUtils.AudioTranscriptPrinter(); + try (PrintStream capture = new PrintStream(output)) { + System.setOut(capture); + printer.print(GptLiveSampleUtils + .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Previous response."))); + printer.beginInput(); + assertTrue(new String(output.toByteArray(), StandardCharsets.UTF_8) + .endsWith("Previous response." + System.lineSeparator() + "You: ")); + int promptLength = output.size(); + char[] characters = new char[GptLiveSampleUtils.AudioTranscriptPrinter.MAX_BUFFERED_TRANSCRIPT_CHARACTERS + - "Assistant: ".length()]; + Arrays.fill(characters, 'a'); + printer.print(GptLiveSampleUtils + .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", new String(characters)))); + assertEquals(promptLength, output.size(), "Transcript output must not overwrite the input line"); + IllegalStateException error = assertThrows(IllegalStateException.class, () -> printer.print( + GptLiveSampleUtils.object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "x")))); + assertTrue(error.getMessage().contains("65536")); + printer.endInput(); + assertTrue(output.size() > promptLength, "Buffered text must be flushed after input ends"); + } finally { + System.setOut(original); + } + } + + @ParameterizedTest + @ValueSource(booleans = { true, false }) + @ResourceLock("java.lang.System.out") + public void consoleTranscriptUsesSingleLineBreaks(boolean inputEchoed) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream original = System.out; + try (PrintStream capture = new PrintStream(output)) { + System.setOut(capture); + GptLiveSampleUtils.AudioTranscriptPrinter printer + = new GptLiveSampleUtils.AudioTranscriptPrinter(inputEchoed); + printer.print(GptLiveSampleUtils + .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Welcome."))); + printer.beginInput(); + if (inputEchoed) { + System.out.println("testing"); + } + printer.print(GptLiveSampleUtils + .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Reply."))); + printer.endInput(); + printer.beginInput(); + if (inputEchoed) { + System.out.println("exit"); + } + printer.endInput(); + printer.print(GptLiveSampleUtils.object(BinaryData.fromString(CLOSED))); + System.out.println("Live session closed."); + } finally { + System.setOut(original); + } + String newline = System.lineSeparator(); + assertEquals( + "Assistant: Welcome." + newline + "You: " + (inputEchoed ? "testing" : "") + newline + "Assistant: Reply." + + newline + "You: " + (inputEchoed ? "exit" : "") + newline + "Live session closed." + newline, + new String(output.toByteArray(), StandardCharsets.UTF_8)); + } + + @Test + @ResourceLock("java.lang.System.out") + public void partialUtteranceSurvivesInputSubmissionWithoutBeingPrinted() { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream original = System.out; + try (PrintStream capture = new PrintStream(output)) { + System.setOut(capture); + GptLiveSampleUtils.AudioTranscriptPrinter printer + = new GptLiveSampleUtils.AudioTranscriptPrinter(false, true); + printer.beginInput(); + printer.print(GptLiveSampleUtils + .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "This sentence "))); + printer.endInput(); + assertFalse(new String(output.toByteArray(), StandardCharsets.UTF_8).contains("This sentence")); + printer.beginTurn(); + printer.beginInput(); + printer.print(GptLiveSampleUtils.object( + GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "continues after Enter."))); + printer.endUtterance(); + assertTrue(new String(output.toByteArray(), StandardCharsets.UTF_8).endsWith("You: ")); + printer.endInput(); + } finally { + System.setOut(original); + } + String newline = System.lineSeparator(); + assertEquals( + "You: " + newline + "You: " + newline + "Assistant: This sentence continues after Enter." + newline, + new String(output.toByteArray(), StandardCharsets.UTF_8)); + } + + @Test + @ResourceLock("java.lang.System.out") + public void completedAndPartialUtterancesShareTheTranscriptBufferLimit() { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream original = System.out; + try (PrintStream capture = new PrintStream(output)) { + System.setOut(capture); + GptLiveSampleUtils.AudioTranscriptPrinter printer + = new GptLiveSampleUtils.AudioTranscriptPrinter(false, true); + printer.beginInput(); + printer.print(GptLiveSampleUtils + .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Completed."))); + printer.endUtterance(); + int queuedLength = ("Assistant: Completed." + System.lineSeparator()).length(); + char[] characters + = new char[GptLiveSampleUtils.AudioTranscriptPrinter.MAX_BUFFERED_TRANSCRIPT_CHARACTERS - queuedLength]; + Arrays.fill(characters, 'a'); + printer.print(GptLiveSampleUtils + .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", new String(characters)))); + assertThrows(IllegalStateException.class, () -> printer.print( + GptLiveSampleUtils.object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "x")))); + printer.endInput(); + assertTrue(new String(output.toByteArray(), StandardCharsets.UTF_8).contains("Assistant: Completed.")); + assertFalse(new String(output.toByteArray(), StandardCharsets.UTF_8).contains("aaaa")); + } finally { + System.setOut(original); + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + @ResourceLock("java.lang.System.out") + public void spokenSentenceIsNotSplitByBackendCompletionOrAnotherTypedTurn(boolean async) { + AtomicInteger responses = new AtomicInteger(); + server = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, response) -> response.sendWebsocket((inbound, outbound) -> outbound + .sendString(Flux.concat(Mono.just(STARTED), inbound.receive().asString().concatMap(json -> { + Map command = GptLiveSampleUtils.object(BinaryData.fromString(json)); + commands.add(command); + if ("session.commentary.append".equals(command.get("type"))) { + return Flux.just("{\"type\":\"session.output_transcript.delta\",\"delta\":\"Welcome.\"}", + "{\"type\":\"session.output_audio.delta\",\"delta\":\"AQI=\"}"); + } + if ("response.create".equals(command.get("type"))) { + String fragment + = responses.incrementAndGet() == 1 ? "This sentence " : "continues across typed turns."; + return Flux.just( + GptLiveSampleUtils.event("session.output_transcript.delta", "delta", fragment).toString(), + "{\"type\":\"session.output_audio.delta\",\"delta\":\"AQI=\"}", COMPLETED); + } + return "session.close".equals(command.get("type")) ? Flux.just(CLOSED) : Flux.empty(); + }))) + .then())) + .bindNow(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream original = System.out; + try (PrintStream capture = new PrintStream(output)) { + System.setOut(capture); + runInteractiveConversation(async, new StringReader("one\ntwo\nexit\n")); + } finally { + System.setOut(original); + } + assertEquals(2, responses.get()); + String transcript = new String(output.toByteArray(), StandardCharsets.UTF_8); + assertTrue(transcript.contains("Assistant: This sentence continues across typed turns.")); + assertFalse(transcript.contains("Assistant: continues")); + assertEquals(2, transcript.split("Assistant: ", -1).length - 1); + assertEquals(3, transcript.split("You: ", -1).length - 1, "Input must remain available between backend turns"); + } + + @Test + public void moderationIsReportedExplicitlyAndPreservesServiceDiagnostics() { + Map event = GptLiveSampleUtils.object(BinaryData.fromString(MODERATED)); + IllegalStateException error = assertThrows(IllegalStateException.class, + () -> new GptLiveSampleUtils.AudioTranscriptPrinter().print(event)); + assertTrue(error.getMessage().contains("generation stopped by service moderation")); + assertTrue(error.getMessage().contains("content_filter")); + assertTrue(error.getMessage().contains("moderated-1")); + Map wrapper = new java.util.LinkedHashMap<>(); + wrapper.put("type", "response.event"); + wrapper.put("event", event); + assertThrows(IllegalStateException.class, () -> new GptLiveSampleUtils.AudioTranscriptPrinter().print(wrapper)); + } + + @ParameterizedTest + @MethodSource("invalidSampleSettingCases") + @ResourceLock("com.azure.core.util.Configuration") + public void examplesValidateSettingsBeforeCreatingClients(String sampleName, Consumer main, + String setting, String value) { + Configuration configuration = Configuration.getGlobalConfiguration(); + Map original = new LinkedHashMap<>(); + for (String name : Arrays.asList("FOUNDRY_PROJECT_ENDPOINT", "FOUNDRY_GPT_LIVE_MODEL", + "FOUNDRY_GPT_LIVE_BACKEND_MODEL")) { + original.put(name, configuration.get(name)); + } + try { + configuration.put("FOUNDRY_PROJECT_ENDPOINT", "not-an-endpoint"); + configuration.put("FOUNDRY_GPT_LIVE_MODEL", "gpt-live-1"); + configuration.put("FOUNDRY_GPT_LIVE_BACKEND_MODEL", "gpt-5.6-terra"); + configuration.put(setting, value); + IllegalArgumentException error = assertThrows(IllegalArgumentException.class, + () -> main.accept(new String[0]), sampleName); + assertTrue(error.getMessage().contains(setting)); + } finally { + original.forEach((name, previous) -> { + if (previous == null) { + configuration.remove(name); + } else { + configuration.put(name, previous); + } + }); + } + } + + private static Stream invalidSampleSettingCases() { + Map> samples = new LinkedHashMap<>(); + samples.put("audio-async", GptLiveAudioConversationAsyncSample::main); + samples.put("text", GptLiveTextConversationSample::main); + samples.put("text-async", GptLiveTextConversationAsyncSample::main); + samples.put("tools", GptLiveFunctionToolSample::main); + return samples.entrySet().stream() + .flatMap(sample -> Stream.of("FOUNDRY_PROJECT_ENDPOINT", "FOUNDRY_GPT_LIVE_MODEL", + "FOUNDRY_GPT_LIVE_BACKEND_MODEL") + .flatMap(setting -> Stream.of("", " ") + .map(value -> Arguments.of(sample.getKey(), sample.getValue(), setting, value)))); + } + + @BeforeAll + static void installTlsCertificate() { + VoiceAgentWebSocketSessionTests.installTlsCertificate(); + } + + @AfterAll + static void restoreTlsConfiguration() { + VoiceAgentWebSocketSessionTests.restoreTlsConfiguration(); + } + + @AfterEach + void closeServer() { + if (server != null) { + server.disposeNow(); + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void textSamplesWaitForStartupAndCompleteMultipleTurns(boolean async) { + startTextServer(false, false); + if (async) { + Mono.usingWhen(builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), + session -> GptLiveTextConversationAsyncSample.runConversation(session, Arrays.asList("one", "two")), + BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(TIMEOUT); + } else { + try (BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + GptLiveTextConversationSample.runConversation(session, Arrays.asList("one", "two")); + } + } + List expected = Arrays.asList("response.item.create", "response.create", "response.item.create", + "response.create", "session.close"); + assertEquals(expected, commandTypes()); + Map item = GptLiveSampleUtils.object(commands.get(0).get("item")); + assertEquals("user", item.get("role")); + assertEquals("message", item.get("type")); + assertEquals("one", GptLiveSampleUtils.object(((List) item.get("content")).get(0)).get("text")); + } + + @ParameterizedTest + @MethodSource("interactiveExitCases") + public void interactiveTextSamplesReadMultipleTurnsAndCloseOnExit(boolean async, String exit) { + startTextServer(false, false); + AtomicBoolean inputClosed = new AtomicBoolean(); + Reader input = new StringReader("one\r\ntwo\n" + exit) { + @Override + public void close() { + inputClosed.set(true); + super.close(); + } + }; + runInteractiveConversation(async, input); + assertEquals(Arrays.asList("session.commentary.append", "response.item.create", "response.create", + "response.item.create", "response.create", "session.close"), commandsWithoutInputAudio()); + List sentPrompts = commands.stream() + .filter(command -> "response.item.create".equals(command.get("type"))) + .map(command -> GptLiveSampleUtils.object(command.get("item"))) + .map(item -> GptLiveSampleUtils.object(((List) item.get("content")).get(0))) + .map(part -> GptLiveSampleUtils.string(part, "text")) + .collect(java.util.stream.Collectors.toList()); + assertEquals(Arrays.asList("one", "two"), sentPrompts); + assertFalse(inputClosed.get(), "Conversation cleanup must not close standard input"); + } + + @ParameterizedTest + @MethodSource("interactiveExitCases") + public void interactiveTextSamplesCanExitWithoutSendingAPrompt(boolean async, String text) { + startTextServer(false, false); + runInteractiveConversation(async, new StringReader(text)); + assertEquals(Arrays.asList("session.commentary.append", "session.close"), commandsWithoutInputAudio()); + } + + private static Stream interactiveExitCases() { + return Stream.of(false, true) + .flatMap(async -> Stream.of("", "\n", "exit\n", " QuIt \r\n").map(exit -> Arguments.of(async, exit))); + } + + private List commandsWithoutInputAudio() { + return commandTypes().stream() + .filter(type -> !"session.input_audio.append".equals(type)) + .collect(java.util.stream.Collectors.toList()); + } + + private void runInteractiveConversation(boolean async, Reader input) { + runInteractiveConversation(async, input, pcm -> { + }, () -> { + }); + } + + private void runInteractiveConversation(boolean async, Reader input, Consumer playAudio, + Runnable drainAudio) { + if (async) { + Mono.usingWhen(builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), + session -> GptLiveTextConversationAsyncSample.runConversation(session, input, playAudio, drainAudio), + BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(TIMEOUT); + } else { + try (BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + GptLiveTextConversationSample.runConversation(session, input, playAudio, drainAudio); + } + } + } + + @Test + public void interactiveTextSampleReceivesServiceErrorsWhileWaitingForInput() { + server = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, + response) -> response.sendWebsocket((inbound, outbound) -> outbound + .sendString(Flux.just(STARTED, MODERATED).delayElements(Duration.ofMillis(100))) + .then())) + .bindNow(); + CountDownLatch releaseRead = new CountDownLatch(1); + Reader idleInput = new Reader() { + @Override + public boolean ready() { + throw new AssertionError("Console input must use a blocking line read, not readiness polling"); + } + + @Override + public int read(char[] buffer, int offset, int length) throws IOException { + try { + releaseRead.await(); + return -1; + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IOException("Console read interrupted", error); + } + } + + @Override + public void close() { + throw new AssertionError("Standard input must not be closed"); + } + }; + try { + StepVerifier.create(Mono.usingWhen( + builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), + session -> GptLiveTextConversationAsyncSample.runConversation(session, idleInput, pcm -> { + }), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync)) + .expectErrorMatches( + error -> error instanceof IllegalStateException && error.getMessage().contains("content_filter")) + .verify(TIMEOUT); + } finally { + releaseRead.countDown(); + } + } + + @Test + public void consoleReadFailuresPropagate() { + Reader input = new StringReader("") { + @Override + public int read(char[] buffer, int offset, int length) throws IOException { + throw new IOException("Console unavailable"); + } + }; + StepVerifier.create(GptLiveTextConversationAsyncSample.readPrompt(input)) + .expectErrorMatches( + error -> error instanceof IOException && "Console unavailable".equals(error.getMessage())) + .verify(TIMEOUT); + } + + @Test + public void consoleInputIsSubmittedOnlyAfterEnterWithoutLosingCharacters() throws Exception { + try (PipedReader input = new PipedReader(); PipedWriter keyboard = new PipedWriter(input)) { + CompletableFuture prompt = GptLiveTextConversationAsyncSample.readPrompt(input).toFuture(); + try { + keyboard.write("testtesttes testing"); + keyboard.flush(); + assertThrows(TimeoutException.class, () -> prompt.get(200, TimeUnit.MILLISECONDS)); + keyboard.write("\r\n"); + keyboard.flush(); + assertEquals("testtesttes testing", prompt.get(5, TimeUnit.SECONDS)); + } finally { + prompt.cancel(true); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + @ResourceLock("java.lang.System.out") + public void greetingIsPrintedAndPlayedBeforeReadingInput(boolean async) throws Exception { + Sinks.Empty allowGreeting = Sinks.empty(); + CountDownLatch greetingRequested = new CountDownLatch(1); + CountDownLatch inputStarted = new CountDownLatch(1); + startTextServer(false, false, Mono.fromRunnable(greetingRequested::countDown).then(allowGreeting.asMono())); + CountDownLatch greetingPlayed = new CountDownLatch(1); + CountDownLatch replyPlayed = new CountDownLatch(1); + CountDownLatch drainStarted = new CountDownLatch(1); + CountDownLatch allowDrain = new CountDownLatch(1); + AtomicBoolean drained = new AtomicBoolean(); + AtomicInteger played = new AtomicInteger(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream original = System.out; + try (PipedReader pipe = new PipedReader(); + PipedWriter keyboard = new PipedWriter(pipe); + PrintStream capture = new PrintStream(output)) { + Reader input = new FilterReader(pipe) { + @Override + public int read(char[] buffer, int offset, int length) throws IOException { + assertTrue(drained.get(), "Queued greeting audio must finish before input begins"); + inputStarted.countDown(); + return super.read(buffer, offset, length); + } + }; + System.setOut(capture); + CompletableFuture conversation + = Mono.fromRunnable(() -> runInteractiveConversation(async, input, pcm -> { + if (played.incrementAndGet() == 1) { + greetingPlayed.countDown(); + } else { + replyPlayed.countDown(); + } + }, () -> { + drainStarted.countDown(); + try { + assertTrue(allowDrain.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new AssertionError(error); + } + drained.set(true); + })).subscribeOn(Schedulers.boundedElastic()).toFuture(); + try { + assertTrue(greetingRequested.await(5, TimeUnit.SECONDS)); + assertFalse(inputStarted.await(200, TimeUnit.MILLISECONDS)); + allowGreeting.tryEmitEmpty(); + assertTrue(greetingPlayed.await(5, TimeUnit.SECONDS)); + assertFalse(inputStarted.await(200, TimeUnit.MILLISECONDS), + "The quiet period must elapse before prompting"); + assertTrue(drainStarted.await(5, TimeUnit.SECONDS)); + assertFalse(inputStarted.await(200, TimeUnit.MILLISECONDS), + "The first prompt must wait for queued speaker playback"); + allowDrain.countDown(); + assertTrue(inputStarted.await(5, TimeUnit.SECONDS)); + String whileTyping = new String(output.toByteArray(), StandardCharsets.UTF_8); + assertTrue(whileTyping.endsWith("You: ")); + assertTrue(whileTyping.contains("Assistant: Welcome." + System.lineSeparator() + "You: ")); + keyboard.write("testing\n"); + keyboard.flush(); + assertTrue(replyPlayed.await(5, TimeUnit.SECONDS)); + keyboard.write("exit\n"); + keyboard.flush(); + conversation.get(5, TimeUnit.SECONDS); + String transcript = new String(output.toByteArray(), StandardCharsets.UTF_8); + assertTrue(transcript.indexOf("Assistant: Welcome.") < transcript.indexOf("You: ")); + assertTrue(transcript.contains("Spoken reply.")); + assertFalse(transcript.contains(System.lineSeparator() + System.lineSeparator())); + } finally { + allowDrain.countDown(); + conversation.cancel(true); + } + } finally { + System.setOut(original); + } + } + + @Test + public void greetingWaitsForBothOutputsAndRestartsQuietPeriodForEachChunk() { + AtomicReference greeting = new AtomicReference<>(); + AtomicBoolean drained = new AtomicBoolean(); + StepVerifier.withVirtualTime(() -> { + GptLiveTextConversationAsyncSample.Greeting pending = new GptLiveTextConversationAsyncSample.Greeting(); + greeting.set(pending); + return pending.awaitPlayback(() -> drained.set(true)); + }) + .expectSubscription() + .then(() -> greeting.get().onTranscript()) + .expectNoEvent(Duration.ofSeconds(2)) + .then(() -> greeting.get().onAudio(new byte[] { 1, 2 })) + .expectNoEvent(Duration.ofMillis(750)) + .then(() -> greeting.get().onAudio(new byte[] { 1, 2 })) + .expectNoEvent(Duration.ofMillis(750)) + .then(() -> greeting.get().onTranscript()) + .expectNoEvent(Duration.ofMillis(999)) + .then(() -> assertFalse(drained.get())) + .thenAwait(Duration.ofMillis(1)) + .verifyComplete(); + assertTrue(drained.get()); + } + + @ParameterizedTest + @ValueSource(strings = { "session.output_transcript.delta", "session.output_audio.delta" }) + public void missingGreetingOutputTimesOutInsteadOfStartingInput(String type) { + AtomicBoolean drained = new AtomicBoolean(); + StepVerifier.withVirtualTime(() -> { + GptLiveTextConversationAsyncSample.Greeting greeting = new GptLiveTextConversationAsyncSample.Greeting(); + if ("session.output_audio.delta".equals(type)) { + greeting.onAudio(new byte[] { 1, 2 }); + } else { + greeting.onTranscript(); + } + return greeting.awaitPlayback(() -> drained.set(true)); + }).expectSubscription().thenAwait(GptLiveSampleUtils.TIMEOUT).expectError(TimeoutException.class).verify(); + assertFalse(drained.get()); + } + + @Test + public void greetingDrainFailureIsReported() { + StepVerifier.withVirtualTime(() -> { + GptLiveTextConversationAsyncSample.Greeting greeting = new GptLiveTextConversationAsyncSample.Greeting(); + greeting.onAudio(new byte[] { 1, 2 }); + greeting.onTranscript(); + return greeting.awaitPlayback(() -> { + throw new IllegalStateException("Speaker drain failed"); + }); + }).expectSubscription().thenAwait(Duration.ofSeconds(1)).expectErrorMessage("Speaker drain failed").verify(); + } + + @Test + public void continuousSilentOutputDoesNotKeepTheGreetingOpen() { + AtomicBoolean drained = new AtomicBoolean(); + StepVerifier.withVirtualTime(() -> { + GptLiveTextConversationAsyncSample.Greeting greeting = new GptLiveTextConversationAsyncSample.Greeting(); + greeting.onAudio(new byte[] { 1, 2 }); + greeting.onTranscript(); + Mono silence = Flux.interval(Duration.ofMillis(20)) + .take(50) + .doOnNext(tick -> greeting.onAudio(new byte[960])) + .then(); + return Mono.when(silence, greeting.awaitPlayback(() -> drained.set(true))); + }).expectSubscription().thenAwait(Duration.ofSeconds(1)).verifyComplete(); + assertTrue(drained.get()); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + @ResourceLock("java.lang.System.out") + public void continuousSilentAudioStillAllowsInputAfterTheGreeting(boolean async) { + startTextServer(false, false, Mono.empty(), true); + AtomicInteger silentFrames = new AtomicInteger(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream original = System.out; + try (PrintStream capture = new PrintStream(output)) { + System.setOut(capture); + runInteractiveConversation(async, new StringReader("exit\n"), pcm -> { + if (pcm.length == 960) { + assertArrayEquals(new byte[960], pcm); + silentFrames.incrementAndGet(); + } + }, () -> { + }); + } finally { + System.setOut(original); + } + assertTrue(silentFrames.get() >= 20, "The server must keep streaming silence throughout the quiet period"); + String transcript = new String(output.toByteArray(), StandardCharsets.UTF_8); + assertTrue(transcript.contains("Assistant: Welcome." + System.lineSeparator() + "You: ")); + assertEquals(Arrays.asList("session.commentary.append", "session.close"), commandsWithoutInputAudio()); + } + + @ParameterizedTest + @ValueSource(ints = { 0, 256, -256, 257, -257, 32767, -32768 }) + public void greetingSpeechDetectionHonorsSignedPcm16PeakThreshold(int peak) { + boolean audible = Math.abs(peak) > GptLiveTextConversationAsyncSample.Greeting.SILENCE_PEAK_THRESHOLD; + AtomicBoolean drained = new AtomicBoolean(); + StepVerifier.Step verification = StepVerifier.withVirtualTime(() -> { + GptLiveTextConversationAsyncSample.Greeting greeting = new GptLiveTextConversationAsyncSample.Greeting(); + greeting.onTranscript(); + greeting.onAudio(new byte[] { (byte) peak, (byte) (peak >> 8) }); + return greeting.awaitPlayback(() -> drained.set(true)); + }).expectSubscription().thenAwait(audible ? Duration.ofSeconds(1) : GptLiveSampleUtils.TIMEOUT); + if (audible) { + verification.verifyComplete(); + } else { + verification.expectError(TimeoutException.class).verify(); + } + assertEquals(audible, drained.get()); + } + + @ParameterizedTest + @ValueSource(strings = { "", "\r\n", "exit\n", "quit\n" }) + public void consoleEofBlankAndExitProduceNoPrompt(String input) { + StepVerifier.create(GptLiveTextConversationAsyncSample.readPrompt(new StringReader(input))).verifyComplete(); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void interactiveTextPlaybackFailureTerminatesTheConversation(boolean async) { + startTextServer(false, false); + StepVerifier.create( + Mono.fromRunnable(() -> runInteractiveConversation(async, new StringReader("testing\n"), pcm -> { + throw new IllegalStateException("Speaker failed"); + }, () -> { + })).subscribeOn(Schedulers.boundedElastic())) + .expectErrorMatches( + error -> error instanceof IllegalStateException && "Speaker failed".equals(error.getMessage())) + .verify(TIMEOUT); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + @ResourceLock("java.lang.System.out") + public void interactiveTextPlaysGreetingAndRepliesWithoutPrintingBackendNotes(boolean async) { + startTextServer(false, false); + List played = new CopyOnWriteArrayList<>(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream original = System.out; + try (PrintStream capture = new PrintStream(output)) { + System.setOut(capture); + runInteractiveConversation(async, new StringReader("testing\nexit\n"), played::add, () -> { + }); + } finally { + System.setOut(original); + } + assertEquals(2, played.size()); + assertArrayEquals(new byte[] { 1, 2 }, played.get(0)); + assertArrayEquals(new byte[] { 1, 2 }, played.get(1)); + String transcript = new String(output.toByteArray(), StandardCharsets.UTF_8); + assertTrue(transcript.contains("Assistant: Welcome.")); + assertTrue(transcript.contains("Spoken reply.")); + assertFalse(transcript.contains("Hello"), "Backend text must not be printed as the spoken answer"); + assertFalse(transcript.contains("[GPT Live diagnostics]")); + assertFalse(transcript.contains("\"type\":\"session.closed\"")); + Map greeting = commands.stream() + .filter(command -> "session.commentary.append".equals(command.get("type"))) + .findFirst() + .get(); + assertEquals("session.commentary.append", greeting.get("type")); + assertEquals("Hi, how can I help you today?", greeting.get("content")); + assertTrue(greeting.containsKey("delegation_id")); + assertEquals(null, greeting.get("delegation_id")); + assertEquals("session.input_audio.append", commands.get(0).get("type")); + assertEquals("session.close", commands.get(commands.size() - 1).get("type")); + } + + @Test + public void silentAudioFramesArePacedAtRealTimePcmRate() { + StepVerifier.withVirtualTime(() -> GptLiveTextConversationAsyncSample.silenceFrames().take(3)) + .expectSubscription() + .thenAwait(Duration.ZERO) + .assertNext(this::assertSilentAudioFrame) + .expectNoEvent(Duration.ofMillis(19)) + .thenAwait(Duration.ofMillis(1)) + .assertNext(this::assertSilentAudioFrame) + .expectNoEvent(Duration.ofMillis(19)) + .thenAwait(Duration.ofMillis(1)) + .assertNext(this::assertSilentAudioFrame) + .verifyComplete(); + } + + @Test + public void silentAudioFramesWaitForDemandWithoutTimerOverflow() { + StepVerifier.withVirtualTime(() -> GptLiveTextConversationAsyncSample.silenceFrames().take(3), 0) + .expectSubscription() + .expectNoEvent(Duration.ofSeconds(40)) + .thenRequest(1) + .assertNext(this::assertSilentAudioFrame) + .expectNoEvent(Duration.ofSeconds(40)) + .thenRequest(1) + .thenAwait(Duration.ofMillis(20)) + .assertNext(this::assertSilentAudioFrame) + .expectNoEvent(Duration.ofSeconds(40)) + .thenRequest(1) + .thenAwait(Duration.ofMillis(20)) + .assertNext(this::assertSilentAudioFrame) + .verifyComplete(); + } + + @Test + public void silentAudioToleratesSendsSlowerThanItsFrameRate() { + StepVerifier + .withVirtualTime(() -> GptLiveTextConversationAsyncSample.silenceFrames() + .concatMap(frame -> Mono.delay(Duration.ofMillis(75)).thenReturn(frame), 1) + .take(500)) + .expectSubscription() + .thenAwait(Duration.ofSeconds(40)) + .expectNextCount(500) + .verifyComplete(); + } + + private void assertSilentAudioFrame(BinaryData frame) { + Map event = GptLiveSampleUtils.object(frame); + assertEquals("session.input_audio.append", event.get("type")); + assertArrayEquals(new byte[960], Base64.getDecoder().decode(GptLiveSampleUtils.string(event, "audio"))); + } + + @ParameterizedTest + @MethodSource("interactiveClosureCases") + public void interactiveTextPreservesServerClosureReason(boolean async, String reason) { + String closed = "{\"type\":\"session.closed\",\"reason\":\"" + reason + "\",\"event_id\":\"closed-1\"}"; + server = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, response) -> response.sendWebsocket((inbound, outbound) -> outbound + .sendString(Flux.concat(Mono.just(STARTED), inbound.receive().asString().concatMap(json -> { + Map command = GptLiveSampleUtils.object(BinaryData.fromString(json)); + commands.add(command); + if ("session.commentary.append".equals(command.get("type"))) { + return Flux.just("{\"type\":\"session.output_transcript.delta\",\"delta\":\"Welcome.\"}", + "{\"type\":\"session.output_audio.delta\",\"delta\":\"AQI=\"}"); + } + return "response.create".equals(command.get("type")) ? Mono.just(closed) : Mono.empty(); + }))) + .then())) + .bindNow(); + if ("remote_hangup".equals(reason)) { + runInteractiveConversation(async, new StringReader("testing\n")); + } else { + IllegalStateException error = assertThrows(IllegalStateException.class, + () -> runInteractiveConversation(async, new StringReader("testing\n"))); + assertTrue(error.getMessage().contains(reason)); + assertTrue(error.getMessage().contains("closed-1")); + } + assertFalse(commandTypes().contains("session.close")); + } + + private static Stream interactiveClosureCases() { + return Stream.of(false, true) + .flatMap(async -> Stream.of("remote_hangup", "expired", "content", "connection_lost") + .map(reason -> Arguments.of(async, reason))); + } + + @Test + public void synchronousConsoleReadFailuresPropagateAndReleaseTheSession() { + startTextServer(false, false); + IOException failure = new IOException("Console unavailable"); + Reader input = new Reader() { + @Override + public int read(char[] buffer, int offset, int length) throws IOException { + throw failure; + } + + @Override + public void close() { + throw new AssertionError("Standard input must not be closed"); + } + }; + try (BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + IllegalStateException error = assertThrows(IllegalStateException.class, + () -> GptLiveTextConversationSample.runConversation(session, input, pcm -> { + }, () -> { + })); + assertEquals("Unable to read console input.", error.getMessage()); + assertEquals(failure, error.getCause()); + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void serviceErrorsFailBothTextSamples(boolean async) { + startTextServer(false, true); + if (async) { + StepVerifier + .create(Mono.usingWhen( + builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), + session -> GptLiveTextConversationAsyncSample.runConversation(session, + Collections.singletonList("one")), + BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync)) + .expectErrorMatches(error -> error instanceof IllegalStateException + && error.getMessage().contains("GPT Live service error")) + .verify(TIMEOUT); + } else { + try (BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + assertThrows(IllegalStateException.class, + () -> GptLiveTextConversationSample.runConversation(session, Collections.singletonList("one"))); + } + } + assertFalse(commandTypes().contains("session.close")); + } + + @Test + public void toolsUseGranularEventsAndReturnResultsBeforeContinuation() { + startTextServer(true, false); + try (BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + GptLiveFunctionToolSample.runConversation(session, "Weather in Seattle?"); + } + assertEquals(Arrays.asList("response.item.create", "response.create", "response.item.create", "response.create", + "session.close"), commandTypes()); + Map item = GptLiveSampleUtils.object(commands.get(2).get("item")); + assertEquals("function_call_output", item.get("type")); + assertEquals("call-1", item.get("call_id")); + Map result = GptLiveSampleUtils.object(BinaryData.fromString((String) item.get("output"))); + assertEquals("Seattle", result.get("city")); + assertEquals(true, result.get("simulated")); + assertFalse(item.containsKey("delegation_id")); + } + + @Test + public void toolSchemaAndArgumentsAreValidated() { + Map tool = GptLiveSampleUtils.object(GptLiveFunctionToolSample.weatherTool()); + assertEquals("function", tool.get("type")); + assertEquals("get_weather", tool.get("name")); + assertEquals(true, tool.get("strict")); + assertEquals(false, GptLiveSampleUtils.object(tool.get("parameters")).get("additionalProperties")); + Map unknown = GptLiveSampleUtils + .object(BinaryData.fromString("{\"name\":\"unknown\",\"call_id\":\"c\",\"arguments\":\"{}\"}")); + assertThrows(IllegalArgumentException.class, () -> GptLiveFunctionToolSample.toolResult(unknown)); + Map invalid = GptLiveSampleUtils + .object(BinaryData.fromString("{\"name\":\"get_weather\",\"call_id\":\"c\",\"arguments\":\"{}\"}")); + assertThrows(IllegalArgumentException.class, () -> GptLiveFunctionToolSample.toolResult(invalid)); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void prematureClosureDoesNotReportSuccessfulConversation(boolean async) { + server = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, response) -> response.sendWebsocket( + (inbound, outbound) -> outbound.sendString(Flux.just(STARTED, CLOSED)).then(inbound.receive().then()))) + .bindNow(); + if (async) { + StepVerifier + .create(Mono.usingWhen( + builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), + session -> GptLiveTextConversationAsyncSample.runConversation(session, + Collections.singletonList("one")), + BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync)) + .expectError(IllegalStateException.class) + .verify(TIMEOUT); + } else { + try (BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + assertThrows(IllegalStateException.class, + () -> GptLiveTextConversationSample.runConversation(session, Collections.singletonList("one"))); + } + } + } + + @Test + public void nativeAudioEncodingAndDecodingPreservePcmBytes() { + byte[] pcm = new byte[] { 0, 1, -1, 127 }; + Map command = GptLiveSampleUtils.object(GptLiveSampleUtils.audio(pcm)); + assertEquals("session.input_audio.append", command.get("type")); + assertArrayEquals(pcm, Base64.getDecoder().decode((String) command.get("audio"))); + Map event = Collections.singletonMap("delta", Base64.getEncoder().encodeToString(pcm)); + assertArrayEquals(pcm, GptLiveAudioConversationAsyncSample.decodeAudio(event)); + assertThrows(IllegalArgumentException.class, + () -> GptLiveAudioConversationAsyncSample.decodeAudio(Collections.singletonMap("delta", "%%%"))); + assertThrows(IllegalArgumentException.class, + () -> GptLiveAudioConversationAsyncSample.decodeAudio(Collections.singletonMap("delta", "AA=="))); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void audioSampleGreetsCapturesPlaysAndReleasesDevices(boolean moderated) { + AtomicBoolean captured = new AtomicBoolean(); + AtomicInteger closed = new AtomicInteger(); + CountDownLatch unblockRead = new CountDownLatch(1); + AtomicReference played = new AtomicReference<>(); + AtomicBoolean firstRead = new AtomicBoolean(true); + TargetDataLine microphone = (TargetDataLine) Proxy.newProxyInstance(getClass().getClassLoader(), + new Class[] { TargetDataLine.class }, (proxy, method, args) -> { + if ("read".equals(method.getName())) { + if (firstRead.compareAndSet(true, false)) { + byte[] bytes = (byte[]) args[0]; + bytes[0] = 1; + bytes[1] = 2; + return 2; + } + unblockRead.await(5, TimeUnit.SECONDS); + return 0; + } else if ("close".equals(method.getName())) { + closed.incrementAndGet(); + unblockRead.countDown(); + } + return null; + }); + SourceDataLine speaker = (SourceDataLine) Proxy.newProxyInstance(getClass().getClassLoader(), + new Class[] { SourceDataLine.class }, (proxy, method, args) -> { + if ("write".equals(method.getName())) { + played.set(((byte[]) args[0]).clone()); + return args[2]; + } else if ("close".equals(method.getName())) { + closed.incrementAndGet(); + } + return null; + }); + server + = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, + response) -> response.sendWebsocket((inbound, + outbound) -> outbound.sendString(Flux.concat( + Flux.just(STARTED, "{\"type\":\"session.output_audio.delta\",\"delta\":\"AQI=\"}"), + inbound.receive().asString().concatMap(json -> { + Map command = GptLiveSampleUtils.object(BinaryData.fromString(json)); + commands.add(command); + if ("session.input_audio.append".equals(command.get("type"))) { + captured.set(true); + } + if (moderated && "response.create".equals(command.get("type"))) { + return Mono.just(MODERATED); + } + return "session.close".equals(command.get("type")) ? Mono.just(CLOSED) : Mono.empty(); + }))).then())) + .bindNow(); + BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentLiveWebSocketAsyncClient() + .openWebSocketSession("agent") + .block(TIMEOUT); + assertNotNull(session); + InputStream enterAfterAudio = new InputStream() { + @Override + public int available() { + return captured.get() && played.get() != null && commandTypes().contains("response.create") ? 1 : 0; + } + + @Override + public int read() { + return '\n'; + } + }; + try { + VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor processor + = new VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor(microphone, speaker, + pcm -> session.sendEvent(GptLiveSampleUtils.audio(pcm.toBytes()))); + Mono conversation + = GptLiveAudioConversationAsyncSample.runConversation(session, processor, enterAfterAudio); + if (moderated) { + IllegalStateException error + = assertThrows(IllegalStateException.class, () -> conversation.block(TIMEOUT)); + assertTrue(error.getMessage().contains("content_filter")); + assertTrue(error.getMessage().contains("moderated-1")); + } else { + conversation.block(TIMEOUT); + assertTrue(captured.get()); + assertArrayEquals(new byte[] { 1, 2 }, played.get()); + assertTrue(commandTypes().contains("session.close")); + } + assertEquals(2, closed.get()); + List greetingCommands = commandTypes().stream() + .filter(type -> "response.item.create".equals(type) || "response.create".equals(type)) + .collect(java.util.stream.Collectors.toList()); + assertEquals(Arrays.asList("response.item.create", "response.create"), greetingCommands); + Map greeting = commands.stream() + .filter(command -> "response.item.create".equals(command.get("type"))) + .findFirst() + .get(); + Map item = GptLiveSampleUtils.object(greeting.get("item")); + assertEquals("Greet me briefly with: Hi, how can I help you today?", + GptLiveSampleUtils.object(((List) item.get("content")).get(0)).get("text")); + } finally { + session.closeAsync().block(TIMEOUT); + } + } + + private List commandTypes() { + return commands.stream().map(GptLiveSampleUtils::type).collect(java.util.stream.Collectors.toList()); + } + + private AgentsClientBuilder builder() { + return new AgentsClientBuilder().endpoint("https://localhost:" + server.port()) + .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))) + .configuration(Configuration.NONE); + } + + private void startTextServer(boolean tools, boolean fail) { + startTextServer(tools, fail, Mono.empty()); + } + + private void startTextServer(boolean tools, boolean fail, Mono beforeGreeting) { + startTextServer(tools, fail, beforeGreeting, false); + } + + private void startTextServer(boolean tools, boolean fail, Mono beforeGreeting, boolean continuousSilence) { + AtomicInteger responses = new AtomicInteger(); + String silentOutput = "{\"type\":\"session.output_audio.delta\",\"delta\":\"" + + Base64.getEncoder().encodeToString(new byte[960]) + "\"}"; + server = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, + response) -> response.sendWebsocket((inbound, outbound) -> outbound + .sendString(Flux.concat(Mono.just(STARTED).delayElement(Duration.ofMillis(50)), + inbound.receive().asString().concatMap(json -> { + Map command = GptLiveSampleUtils.object(BinaryData.fromString(json)); + commands.add(command); + if ("session.commentary.append".equals(command.get("type"))) { + return beforeGreeting.thenMany( + Flux.just("{\"type\":\"session.output_transcript.delta\",\"delta\":\"Welcome.\"}", + "{\"type\":\"session.output_audio.delta\",\"delta\":\"AQI=\"}")); + } + if ("response.create".equals(command.get("type"))) { + int responseIndex = responses.incrementAndGet(); + if (fail) { + return Flux.just("{\"type\":\"error\",\"error\":{\"message\":\"failed\"}}"); + } + if (tools && responseIndex == 1) { + return Flux.just("{\"type\":\"response.event\",\"delegation_id\":\"del-1\"," + + "\"event\":{\"type\":\"response.output_item.done\",\"item\":" + + "{\"type\":\"function_call\",\"call_id\":\"call-1\",\"name\":\"get_weather\"," + + "\"arguments\":\"{\\\"city\\\":\\\"Seattle\\\"}\"}}}", COMPLETED); + } + return Flux.just( + "{\"type\":\"response.event\",\"event\":" + + "{\"type\":\"response.output_text.delta\",\"delta\":\"Hello\"}}", + "{\"type\":\"session.output_transcript.delta\",\"delta\":\"Spoken reply.\"}", + "{\"type\":\"session.output_audio.delta\",\"delta\":\"AQI=\"}", COMPLETED); + } + if (continuousSilence && "session.input_audio.append".equals(command.get("type"))) { + return Flux.just(silentOutput); + } + return "session.close".equals(command.get("type")) ? Flux.just(CLOSED) : Flux.empty(); + }))) + .then())) + .bindNow(); + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSampleTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSampleTests.java new file mode 100644 index 0000000000000..65068289eb980 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSampleTests.java @@ -0,0 +1,185 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.exception.HttpResponseException; +import com.azure.core.http.HttpClient; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import com.azure.core.http.policy.ExponentialBackoffOptions; +import com.azure.core.http.policy.RetryOptions; +import com.azure.core.test.http.MockHttpResponse; +import com.azure.core.test.utils.MockTokenCredential; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.core.util.Context; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.function.Function; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Mono; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class VoiceAgentLiveTextConversationSampleTests { + private static final String NAME = "sample-live-text-conversation-agent-java"; + private static final String GENERATE = "/agents:generate"; + private static final String AGENT = "/agents/" + NAME; + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void createdAgentIsDeletedOnNormalExitAndFailure(boolean failConversation) { + ScriptedTransport transport = new ScriptedTransport(); + transport.expect(HttpMethod.POST, GENERATE, 200); + transport.expect(HttpMethod.DELETE, AGENT, 200); + AgentsClientBuilder builder = builder(transport); + IllegalStateException failure = new IllegalStateException("Conversation failed"); + Runnable conversation = () -> { + try (VoiceAgentLiveTextConversationSample.SampleAgent agent = VoiceAgentLiveTextConversationSample + .createAgent(builder.buildAgentsClient(), builder.beta().buildBetaAgentsClient(), NAME)) { + assertNotNull(agent.details); + if (failConversation) { + throw failure; + } + } + }; + if (failConversation) { + assertSame(failure, assertThrows(IllegalStateException.class, conversation::run)); + } else { + conversation.run(); + } + transport.assertConsumed(); + } + + @Test + public void conflictDeletesExistingAgentRetriesOnceAndDeletesReplacement() { + ScriptedTransport transport = new ScriptedTransport(); + transport.expect(HttpMethod.POST, GENERATE, 409); + transport.expect(HttpMethod.DELETE, AGENT, 200); + transport.expect(HttpMethod.POST, GENERATE, 200); + transport.expect(HttpMethod.DELETE, AGENT, 200); + AgentsClientBuilder builder = builder(transport); + try (VoiceAgentLiveTextConversationSample.SampleAgent agent = VoiceAgentLiveTextConversationSample + .createAgent(builder.buildAgentsClient(), builder.beta().buildBetaAgentsClient(), NAME)) { + assertNotNull(agent.details); + } + transport.assertConsumed(); + } + + @ParameterizedTest + @ValueSource(ints = { 400, 403, 412 }) + public void nonConflictCreationFailureDoesNotDeleteAnAgent(int status) { + ScriptedTransport transport = new ScriptedTransport(); + transport.expect(HttpMethod.POST, GENERATE, status); + AgentsClientBuilder builder = builder(transport); + HttpResponseException error + = assertThrows(HttpResponseException.class, () -> VoiceAgentLiveTextConversationSample + .createAgent(builder.buildAgentsClient(), builder.beta().buildBetaAgentsClient(), NAME)); + assertEquals(status, error.getResponse().getStatusCode()); + transport.assertConsumed(); + } + + @Test + public void replacementDeletionFailureStopsWithoutRetryingCreation() { + ScriptedTransport transport = new ScriptedTransport(); + transport.expect(HttpMethod.POST, GENERATE, 409); + transport.expect(HttpMethod.DELETE, AGENT, 403); + AgentsClientBuilder builder = builder(transport); + HttpResponseException error + = assertThrows(HttpResponseException.class, () -> VoiceAgentLiveTextConversationSample + .createAgent(builder.buildAgentsClient(), builder.beta().buildBetaAgentsClient(), NAME)); + assertEquals(403, error.getResponse().getStatusCode()); + transport.assertConsumed(); + } + + @Test + public void repeatedConflictIsNotRetriedOrDeletedAgain() { + ScriptedTransport transport = new ScriptedTransport(); + transport.expect(HttpMethod.POST, GENERATE, 409); + transport.expect(HttpMethod.DELETE, AGENT, 200); + transport.expect(HttpMethod.POST, GENERATE, 409); + AgentsClientBuilder builder = builder(transport); + HttpResponseException error + = assertThrows(HttpResponseException.class, () -> VoiceAgentLiveTextConversationSample + .createAgent(builder.buildAgentsClient(), builder.beta().buildBetaAgentsClient(), NAME)); + assertEquals(409, error.getResponse().getStatusCode()); + transport.assertConsumed(); + } + + @Test + public void cleanupFailureIsPreservedAlongsideConversationFailure() { + ScriptedTransport transport = new ScriptedTransport(); + transport.expect(HttpMethod.POST, GENERATE, 200); + transport.expect(HttpMethod.DELETE, AGENT, 403); + AgentsClientBuilder builder = builder(transport); + IllegalStateException failure = new IllegalStateException("Conversation failed"); + IllegalStateException error = assertThrows(IllegalStateException.class, () -> { + try (VoiceAgentLiveTextConversationSample.SampleAgent agent = VoiceAgentLiveTextConversationSample + .createAgent(builder.buildAgentsClient(), builder.beta().buildBetaAgentsClient(), NAME)) { + assertNotNull(agent.details); + throw failure; + } + }); + assertSame(failure, error); + assertEquals(1, error.getSuppressed().length); + assertTrue(error.getSuppressed()[0] instanceof HttpResponseException); + transport.assertConsumed(); + } + + private static AgentsClientBuilder builder(HttpClient transport) { + return new AgentsClientBuilder().endpoint("https://localhost") + .credential(new MockTokenCredential()) + .configuration(Configuration.NONE) + .httpClient(transport) + .retryOptions(new RetryOptions(new ExponentialBackoffOptions().setMaxRetries(0))) + .allowPreview(true); + } + + private static final class ScriptedTransport implements HttpClient { + private final Deque> requests = new ArrayDeque<>(); + + void expect(HttpMethod method, String path, int status) { + requests.add(request -> { + assertEquals(method, request.getHttpMethod()); + assertEquals(path, request.getUrl().getPath()); + if (method == HttpMethod.POST) { + assertEquals(NAME, request.getBodyAsBinaryData().toObject(java.util.Map.class).get("name")); + } + String body = status >= 400 + ? "{\"error\":{\"code\":\"" + (status == 409 ? "conflict" : "failed") + + "\",\"message\":\"failed\"}}" + : "{}"; + return new MockHttpResponse(request, status, + new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, "application/json"), + BinaryData.fromString(body).toBytes()); + }); + } + + @Override + public Mono send(HttpRequest request) { + throw new AssertionError("The sync sample must not use asynchronous HTTP."); + } + + @Override + public HttpResponse sendSync(HttpRequest request, Context context) { + assertFalse(requests.isEmpty(), "Unexpected request: " + request.getUrl()); + return requests.removeFirst().apply(request); + } + + void assertConsumed() { + assertTrue(requests.isEmpty(), "Expected HTTP requests were not sent"); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java new file mode 100644 index 0000000000000..cd7b225d6c60a --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java @@ -0,0 +1,407 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; +import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; +import com.azure.core.credential.AccessToken; +import com.azure.core.exception.ResourceNotFoundException; +import com.azure.core.util.BinaryData; +import com.azure.core.util.ClientOptions; +import com.azure.core.util.Configuration; +import com.azure.core.util.Header; +import io.netty.handler.codec.http.HttpHeaderNames; +import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.netty.DisposableServer; +import reactor.test.StepVerifier; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ResourceLock("voice-agent-websocket-tls") +@Timeout(20) +public class VoiceAgentLiveWebSocketTests { + private static final Duration TIMEOUT = Duration.ofSeconds(5); + private static final String STARTED + = "{\"type\":\"session.started\",\"conversation_id\":\"conversation-1\",\"session\":{\"id\":\"live-1\"}}"; + private static final String RESPONSE = "{\"type\":\"response.done\",\"future_field\":{\"native\":true}}"; + private static final String FUTURE = "{\"type\":\"future.live.event\",\"unknown\":[1,2,3]}"; + private static final String ERROR + = "{\"type\":\"error\",\"error\":{\"type\":\"server_error\",\"message\":\"native Live error\"}}"; + private static final String COMMAND = "{\"type\":\"session.update\",\"session\":{\"future_setting\":true}}"; + private DisposableServer server; + + @BeforeAll + static void installTlsCertificate() { + VoiceAgentWebSocketSessionTests.installTlsCertificate(); + } + + @AfterAll + static void restoreTlsConfiguration() { + VoiceAgentWebSocketSessionTests.restoreTlsConfiguration(); + } + + @AfterEach + void stopServer() { + if (server != null) { + server.disposeNow(); + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void handshakeUsesLiveRouteWithoutRealtimeSubprotocol(boolean async) throws Exception { + AtomicReference requestUri = new AtomicReference<>(); + AtomicReference headers = new AtomicReference<>(); + server = VoiceAgentWebSocketSessionTests.tlsServer().host("localhost").port(0).handle((request, response) -> { + requestUri.set(request.uri()); + headers.set(request.requestHeaders().copy()); + return response.sendWebsocket((inbound, outbound) -> inbound.receive().then()); + }).bindNow(); + AgentsClientBuilder builder + = builder().endpoint("https://localhost:" + server.port() + "/api/projects/project/") + .clientOptions(new ClientOptions().setApplicationId("live-app") + .setHeaders(Arrays.asList(new Header("X-Custom", "value"), new Header("Authorization", "override"), + new Header("Foundry-Features", "override"), new Header("Sec-WebSocket-Protocol", "realtime")))); + String inputs = "{\"name\":\"a & b/\u4f60\u597d\"}"; + VoiceAgentLiveWebSocketConnectionOptions options + = new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(true) + .setAgentVersionOverride("version 1") + .setStructuredInputs(inputs); + if (async) { + Mono connection = builder.beta() + .buildBetaVoiceAgentLiveWebSocketAsyncClient() + .openWebSocketSession("agent /name", options); + options.setStoreEnabled(false).setAgentVersionOverride("changed").setStructuredInputs("{}"); + BetaVoiceAgentLiveWebSocketSessionAsyncClient session = connection.block(TIMEOUT); + assertNotNull(session); + assertTrue(session.isOpen()); + session.closeAsync().block(TIMEOUT); + assertFalse(session.isOpen()); + } else { + try (BetaVoiceAgentLiveWebSocketSessionClient session = builder.beta() + .buildBetaVoiceAgentLiveWebSocketClient() + .openWebSocketSession("agent /name", options)) { + assertTrue(session.isOpen()); + } + } + + URI uri = URI.create(requestUri.get()); + assertEquals("/api/projects/project/agents/agent%20%2Fname/endpoint/protocols/voice/live/sessions", + uri.getRawPath()); + String query = URLDecoder.decode(uri.getRawQuery(), StandardCharsets.UTF_8.name()); + assertTrue(query.contains("api-version=v1"), query); + assertTrue(query.contains("store=true"), query); + assertTrue(query.contains("x-agent-version-override=version 1"), query); + assertTrue(query.contains("structured_inputs=" + inputs), query); + assertFalse(query.contains("transport="), query); + assertFalse(query.contains("agent_session_id="), query); + assertNull(headers.get().get(HttpHeaderNames.SEC_WEBSOCKET_PROTOCOL)); + assertNull(headers.get().get("x-ms-voice-structured-inputs")); + assertEquals("VoiceAgents=V1Preview", headers.get().get("Foundry-Features")); + assertEquals("value", headers.get().get("X-Custom")); + assertTrue(headers.get().get(HttpHeaderNames.USER_AGENT).startsWith("live-app azsdk-java-")); + assertFalse("override".equals(headers.get().get(HttpHeaderNames.AUTHORIZATION))); + } + + @Test + public void syncNativeEventsAreLosslessAndSessionIsBidirectional() { + List sent = new CopyOnWriteArrayList<>(); + startEchoServer(sent); + BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent"); + try { + Iterator events = session.receiveEvents(TIMEOUT).iterator(); + assertEquals(STARTED, events.next().toString()); + assertEquals(RESPONSE, events.next().toString()); + assertEquals(FUTURE, events.next().toString()); + assertEquals(ERROR, events.next().toString()); + assertThrows(IllegalStateException.class, session::receiveEvents); + assertThrows(IllegalArgumentException.class, () -> session.sendEvent(BinaryData.fromString("[]"))); + session.sendEvent(BinaryData.fromString(COMMAND)); + assertEquals(COMMAND, events.next().toString()); + assertEquals(Arrays.asList(COMMAND), sent); + session.close(1000, "done"); + assertFalse(session.isOpen()); + assertThrows(IllegalStateException.class, () -> session.sendEvent(BinaryData.fromString(COMMAND))); + } finally { + session.close(); + } + } + + @Test + public void asyncNativeEventsAreLosslessAndSessionIsBidirectional() { + List sent = new CopyOnWriteArrayList<>(); + startEchoServer(sent); + BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentLiveWebSocketAsyncClient() + .openWebSocketSession("agent") + .block(TIMEOUT); + assertNotNull(session); + try { + StepVerifier.create(session.sendEvent(BinaryData.fromString("[]"))) + .expectError(IllegalArgumentException.class) + .verify(TIMEOUT); + StepVerifier + .create(session.sendEvent(BinaryData.fromString(COMMAND)) + .thenMany(session.receiveEvents()) + .map(BinaryData::toString)) + .expectNext(STARTED, RESPONSE, FUTURE, ERROR) + .expectNext(COMMAND) + .thenCancel() + .verify(TIMEOUT); + assertEquals(Arrays.asList(COMMAND), sent); + StepVerifier.create(session.receiveEvents()).expectError(IllegalStateException.class).verify(TIMEOUT); + session.closeAsync(1000, "done").block(TIMEOUT); + assertFalse(session.isOpen()); + StepVerifier.create(session.sendEvent(BinaryData.fromString(COMMAND))) + .expectError(IllegalStateException.class) + .verify(TIMEOUT); + } finally { + session.closeAsync().block(TIMEOUT); + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void handshakeFailuresSurfaceAsAzureExceptions(boolean async) { + server = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, response) -> response.status(404).send()) + .bindNow(); + if (async) { + StepVerifier + .create(builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("missing")) + .expectError(ResourceNotFoundException.class) + .verify(TIMEOUT); + } else { + assertThrows(ResourceNotFoundException.class, + () -> builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("missing")); + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void malformedJsonTerminatesReception(boolean async) { + server = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, response) -> response.sendWebsocket( + (inbound, outbound) -> outbound.sendString(Mono.just("[]")).then(inbound.receive().then()))) + .bindNow(); + if (async) { + BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentLiveWebSocketAsyncClient() + .openWebSocketSession("agent") + .block(TIMEOUT); + assertNotNull(session); + try { + StepVerifier.create(session.receiveEvents()) + .expectError(IllegalArgumentException.class) + .verify(TIMEOUT); + } finally { + session.closeAsync().block(TIMEOUT); + } + } else { + try (BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + assertThrows(IllegalArgumentException.class, () -> session.receiveEvents(TIMEOUT).iterator().hasNext()); + } + } + } + + @Test + public void connectionOptionsValidateBoundsAndCopyAllSettings() { + VoiceAgentLiveWebSocketConnectionOptions options = new VoiceAgentLiveWebSocketConnectionOptions(); + assertEquals(256, options.getReceiveBufferCapacity()); + assertEquals(32 * 1024 * 1024, options.getMaxMessageSize()); + assertThrows(IllegalArgumentException.class, () -> options.setReceiveBufferCapacity(0)); + assertThrows(IllegalArgumentException.class, () -> options.setReceiveBufferCapacity(65537)); + assertThrows(IllegalArgumentException.class, () -> options.setMaxMessageSize(0)); + assertThrows(IllegalArgumentException.class, () -> options.setHandshakeTimeout(Duration.ZERO)); + assertThrows(IllegalArgumentException.class, () -> options.setCloseTimeout(Duration.ofSeconds(-1))); + options.setReceiveBufferCapacity(1) + .setMaxMessageSize(1024) + .setStoreEnabled(true) + .setHandshakeTimeout(TIMEOUT) + .setCloseTimeout(TIMEOUT) + .setAgentVersionOverride("1") + .setStructuredInputs("{\"name\":\"value\"}"); + VoiceAgentLiveWebSocketConnectionOptions copy = new VoiceAgentLiveWebSocketConnectionOptions(options); + options.setReceiveBufferCapacity(65536).setStoreEnabled(false).setStructuredInputs("{}"); + assertEquals(1, copy.getReceiveBufferCapacity()); + assertEquals(1024, copy.getMaxMessageSize()); + assertEquals(true, copy.isStoreEnabled()); + assertEquals(TIMEOUT, copy.getHandshakeTimeout()); + assertEquals(TIMEOUT, copy.getCloseTimeout()); + assertEquals("1", copy.getAgentVersionOverride()); + assertEquals("{\"name\":\"value\"}", copy.getStructuredInputs()); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void peerClosurePreservesEventsAndCloseMetadata(boolean async) { + server = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, + response) -> response.sendWebsocket((inbound, outbound) -> outbound.sendString(Mono.just(STARTED)) + .then(outbound.sendClose(1000, "finished")))) + .bindNow(); + if (async) { + BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentLiveWebSocketAsyncClient() + .openWebSocketSession("agent") + .block(TIMEOUT); + assertNotNull(session); + try { + StepVerifier.create(session.receiveEvents().map(BinaryData::toString)) + .expectNext(STARTED) + .expectComplete() + .verify(TIMEOUT); + assertEquals(1000, session.getCloseCode()); + assertEquals("finished", session.getCloseReason()); + assertFalse(session.isOpen()); + } finally { + session.closeAsync().block(TIMEOUT); + } + } else { + try (BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + Iterator events = session.receiveEvents(TIMEOUT).iterator(); + assertEquals(STARTED, events.next().toString()); + assertFalse(events.hasNext()); + assertEquals(1000, session.getCloseCode()); + assertEquals("finished", session.getCloseReason()); + assertFalse(session.isOpen()); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void malformedEventCallbackCanSkipInvalidJson(boolean async) { + AtomicInteger malformed = new AtomicInteger(); + VoiceAgentLiveWebSocketConnectionOptions options = new VoiceAgentLiveWebSocketConnectionOptions() + .setMalformedEventHandler(error -> malformed.incrementAndGet()); + server = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, response) -> response.sendWebsocket( + (inbound, outbound) -> outbound.sendString(Flux.just("[]", STARTED)).then(inbound.receive().then()))) + .bindNow(); + if (async) { + BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentLiveWebSocketAsyncClient() + .openWebSocketSession("agent", options) + .block(TIMEOUT); + assertNotNull(session); + try { + StepVerifier.create(session.receiveEvents().map(BinaryData::toString)) + .expectNext(STARTED) + .thenCancel() + .verify(TIMEOUT); + } finally { + session.closeAsync().block(TIMEOUT); + } + } else { + try (BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent", options)) { + assertEquals(STARTED, session.receiveEvents(TIMEOUT).iterator().next().toString()); + } + } + assertEquals(1, malformed.get()); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void oversizedEventsFailEvenWhenMalformedCallbackIsConfigured(boolean async) { + AtomicInteger malformed = new AtomicInteger(); + VoiceAgentLiveWebSocketConnectionOptions options + = new VoiceAgentLiveWebSocketConnectionOptions().setMaxMessageSize(32) + .setMalformedEventHandler(error -> malformed.incrementAndGet()); + server = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, response) -> response.sendWebsocket( + (inbound, outbound) -> outbound.sendString(Mono.just(STARTED)).then(inbound.receive().then()))) + .bindNow(); + if (async) { + BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentLiveWebSocketAsyncClient() + .openWebSocketSession("agent", options) + .block(TIMEOUT); + assertNotNull(session); + try { + StepVerifier.create(session.receiveEvents()).expectError().verify(TIMEOUT); + } finally { + session.closeAsync().block(TIMEOUT); + } + } else { + try (BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent", options)) { + assertThrows(IllegalArgumentException.class, () -> session.receiveEvents(TIMEOUT).iterator().hasNext()); + } + } + assertEquals(0, malformed.get()); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void unsupportedHttpSettingsAreRejected(boolean async) { + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint("https://example.com") + .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))) + .httpClient(com.azure.core.http.HttpClient.createDefault()); + if (async) { + assertThrows(IllegalStateException.class, + () -> builder.beta().buildBetaVoiceAgentLiveWebSocketAsyncClient()); + } else { + assertThrows(IllegalStateException.class, () -> builder.beta().buildBetaVoiceAgentLiveWebSocketClient()); + } + } + + private AgentsClientBuilder builder() { + return new AgentsClientBuilder().endpoint("https://localhost:" + server.port()) + .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))) + .configuration(Configuration.NONE); + } + + private void startEchoServer(List sent) { + server = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, + response) -> response.sendWebsocket((inbound, + outbound) -> outbound.sendString(Flux.concat(Flux.just(STARTED, RESPONSE, FUTURE, ERROR), + inbound.receive().asString().doOnNext(sent::add))).then())) + .bindNow(); + } +} From 5c21c30d30fb4d9ce101665536745552b65ac272 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Wed, 21 Oct 2026 10:26:07 +0800 Subject: [PATCH 03/20] Add GPT Live persisted conversation and audio samples --- sdk/ai/azure-ai-agents/CHANGELOG.md | 3 + sdk/ai/azure-ai-agents/README.md | 47 +++ .../GptLiveAudioConversationAsyncSample.java | 7 +- .../GptLiveReadConversationAsyncSample.java | 84 +++++ ...tLiveReadConversationAudioAsyncSample.java | 91 +++++ .../GptLiveReadConversationAudioSample.java | 122 +++++++ .../voice/GptLiveReadConversationSample.java | 119 +++++++ .../ai/agents/voice/GptLiveSampleUtils.java | 3 +- .../GptLiveTextConversationAsyncSample.java | 6 +- .../voice/GptLiveTextConversationSample.java | 7 +- .../GptLiveConversationSamplesTests.java | 324 ++++++++++++++++++ .../ai/agents/voice/GptLiveSamplesTests.java | 93 ++++- 12 files changed, 883 insertions(+), 23 deletions(-) create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAsyncSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioAsyncSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationSample.java create mode 100644 sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveConversationSamplesTests.java diff --git a/sdk/ai/azure-ai-agents/CHANGELOG.md b/sdk/ai/azure-ai-agents/CHANGELOG.md index fc2edd5d217d0..ff8b1d364b558 100644 --- a/sdk/ai/azure-ai-agents/CHANGELOG.md +++ b/sdk/ai/azure-ai-agents/CHANGELOG.md @@ -22,6 +22,9 @@ `AgentsClientBuilder.beta().buildBetaVoiceAgentLiveWebSocketClient()` and `buildBetaVoiceAgentLiveWebSocketAsyncClient()`. Native Live events are sent and received as lossless JSON using `BinaryData`, separately from the Realtime protocol clients. +- Added synchronous/asynchronous persisted GPT Live conversation history and audio-download examples with offline tests. +- Added opt-in persistence to the GPT Live text and microphone/speaker samples using `FOUNDRY_GPT_LIVE_STORE` (default `false`), + with explicit startup output when the service does not provide a persisted conversation ID. - Added runnable native GPT Live samples for synchronous/asynchronous multi-turn text, asynchronous microphone and speaker audio, and Responses backend function tools, including sample-agent creation and cleanup. Model deployment names default to `gpt-live-1` and `gpt-5.6-terra` when their environment variables are unset. diff --git a/sdk/ai/azure-ai-agents/README.md b/sdk/ai/azure-ai-agents/README.md index 63438caca04db..86afd0fe9a2d8 100644 --- a/sdk/ai/azure-ai-agents/README.md +++ b/sdk/ai/azure-ai-agents/README.md @@ -1152,6 +1152,7 @@ an existing agent. Set these environment variables before running a sample's `ma | `FOUNDRY_PROJECT_ENDPOINT` | Your Foundry project endpoint. Authentication uses `DefaultAzureCredential`. | | `FOUNDRY_GPT_LIVE_MODEL` | Optional. Your GPT Live **deployment name**, not a model catalog ID. Defaults to `gpt-live-1` when unset. | | `FOUNDRY_GPT_LIVE_BACKEND_MODEL` | Optional. Your Responses backend **deployment name in the Live resource**. Defaults to `gpt-5.6-terra` when unset. | +| `FOUNDRY_GPT_LIVE_STORE` | Optional, text and microphone/speaker examples. Set to `true` to persist conversation content and audio. Defaults to `false`. | | `FOUNDRY_KEEP_GPT_LIVE_AGENT` | Optional. Set to `true` to retain the sample-created agent; defaults to `false`. | Each sample reads, defaults, and validates these settings and configures its client builder directly in its own `main` method. @@ -1167,6 +1168,52 @@ leaving headroom below the service's 63-character limit. | Asynchronous microphone and speaker | [GptLiveAudioConversationAsyncSample.java](src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java) | | Responses backend function tool | [GptLiveFunctionToolSample.java](src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java) | +#### Read persisted GPT Live conversations and audio + +These REST examples read existing persisted data; they do not start a WebSocket session, create an agent, +or delete resources. + +| Scenario | Runnable sample | +|---|---| +| Synchronous history and transcripts | [GptLiveReadConversationSample.java](src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationSample.java) | +| Asynchronous history and transcripts | [GptLiveReadConversationAsyncSample.java](src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAsyncSample.java) | +| Synchronous stored audio download | [GptLiveReadConversationAudioSample.java](src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioSample.java) | +| Asynchronous stored audio download | [GptLiveReadConversationAudioAsyncSample.java](src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioAsyncSample.java) | + +Use an agent/session configured with `store=true` and retain the agent. With either text WebSocket example or the +microphone/speaker example, set +both `FOUNDRY_GPT_LIVE_STORE=true` and `FOUNDRY_KEEP_GPT_LIVE_AGENT=true`. The first sets the session persistence +override; the second prevents sample cleanup from deleting the agent. Keeping the agent alone does **not** enable +persistence. Both interactive and scripted modes print the Foundry `conversation_id` from `session.started` +when provided. If absent, startup output explicitly reports that no persisted conversation ID was provided; +it never substitutes the native session ID. +Use `session.started.conversation_id`, not the native session ID. Set: + +| Variable | Value | +|---|---| +| `FOUNDRY_PROJECT_ENDPOINT` | Required. Your Foundry project endpoint; authentication uses `DefaultAzureCredential`. No default. | +| `FOUNDRY_GPT_LIVE_AGENT_NAME` | Required. The existing Live agent's name. No default. | +| `FOUNDRY_GPT_LIVE_CONVERSATION_ID` | Required. The persisted Foundry conversation ID. No default. | +| `FOUNDRY_GPT_LIVE_ITEM_ID` | Optional, audio examples only: a speech item's ID, not a delegation item. Defaults to unset (merged audio only); explicitly blank values are invalid. | + +Model deployment settings such as `FOUNDRY_GPT_LIVE_MODEL` and `FOUNDRY_GPT_LIVE_BACKEND_MODEL` are not needed: +these samples read existing conversations rather than creating agents. +Run a history sample's `main` method with no arguments. For audio, pass one existing local output directory; +there is no default output directory. +Audio examples create uniquely named WAV files without overwriting existing files. History examples explicitly +advance the `after` cursor until an empty page because the generated list clients do not expose automatic +continuation; missing or repeated cursors fail explicitly. They show conversation status and usage seconds, +input/output transcripts, and delegation targets/response IDs. Stored speech segments are not native turns or +proof that audio was played. Treat the printed transcripts and downloaded audio as sensitive data. + +Wait for session completion and recording finalization before downloading. HTTP errors propagate: `404` can mean +missing audio; `409` can mean `recording_not_ready` or `recording_unavailable`. Inspect `Retry-After` before retrying. +BYOS metadata is reported without calling Foundry content endpoints; use the metadata's blob URI and your own +Azure Storage credentials to download BYOS audio. The current generated Live REST builder methods require explicit +preview opt-in, so these samples add `Foundry-Features: VoiceAgents=V1Preview` using an HTTP pipeline header policy. + +#### Using the interactive WebSocket samples + Both text examples read terminal input interactively when no arguments are supplied. After each reply, they prompt for another message; submit a blank line or enter `exit` or `quit` to close the session. With arguments, they instead process each argument as a prompt and exit after the last reply. diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java index de82b687cae37..683ffdb6f2613 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java @@ -5,6 +5,7 @@ import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.core.util.Configuration; import com.azure.identity.DefaultAzureCredentialBuilder; import java.io.InputStream; @@ -30,6 +31,8 @@ *
  • {@code FOUNDRY_GPT_LIVE_MODEL} - Optional. The GPT Live deployment name. Defaults to {@code gpt-live-1}.
    • *
  • {@code FOUNDRY_GPT_LIVE_BACKEND_MODEL} - Optional. The Responses backend deployment name in the Live * resource. Defaults to {@code gpt-5.6-terra}.
    • + *
  • {@code FOUNDRY_GPT_LIVE_STORE} - Optional. Set to {@code true} to persist conversation content and audio + * and request a Foundry conversation ID. Defaults to {@code false}. Also keep the agent for later REST reads.
    • *
  • {@code FOUNDRY_KEEP_GPT_LIVE_AGENT} - Optional. Set to {@code true} to keep the agent after the sample. * Defaults to {@code false}.
    • * @@ -41,6 +44,7 @@ public static void main(String[] args) { String model = configuration.get("FOUNDRY_GPT_LIVE_MODEL"); String backendModel = configuration.get("FOUNDRY_GPT_LIVE_BACKEND_MODEL"); boolean keepAgent = Boolean.parseBoolean(configuration.get("FOUNDRY_KEEP_GPT_LIVE_AGENT", "false")); + boolean store = Boolean.parseBoolean(configuration.get("FOUNDRY_GPT_LIVE_STORE", "false")); if (endpoint == null || endpoint.trim().isEmpty()) { throw new IllegalArgumentException("Set environment variable FOUNDRY_PROJECT_ENDPOINT before running this sample."); @@ -62,7 +66,8 @@ public static void main(String[] args) { try (GptLiveSampleUtils.AgentResource agent = GptLiveSampleUtils.createAgent(builder, model, backendModel, "audio-async", false, keepAgent)) { Mono.usingWhen(agent.builder.beta().buildBetaVoiceAgentLiveWebSocketAsyncClient() - .openWebSocketSession(agent.name), GptLiveAudioConversationAsyncSample::runConversation, + .openWebSocketSession(agent.name, new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store)), + GptLiveAudioConversationAsyncSample::runConversation, BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(); diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAsyncSample.java new file mode 100644 index 0000000000000..61e4005660565 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAsyncSample.java @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentsLiveConversationsAsyncClient; +import com.azure.ai.agents.models.PageOrder; +import com.azure.ai.agents.models.VoiceAgentLiveConversation; +import com.azure.ai.agents.models.VoiceAgentLiveConversationItem; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * Asynchronously reads persisted GPT Live conversations and transcript items. + * The session must have used {@code store=true}. The agent and conversation must still exist. + * Like {@link GptLiveReadConversationSample}, this sample prints sensitive stored content without modifying data. + * Blocking occurs only at the standalone main method boundary. + * + *

    Before running the sample, set these environment variables:

    + *
      + *
    • {@code FOUNDRY_PROJECT_ENDPOINT} - Required. The Azure AI Project endpoint. No default.
      • + *
    • {@code FOUNDRY_GPT_LIVE_AGENT_NAME} - Required. The existing GPT Live agent's name. No default.
      • + *
    • {@code FOUNDRY_GPT_LIVE_CONVERSATION_ID} - Required. The persisted Foundry conversation ID from + * {@code session.started.conversation_id}, not the native session ID. No default.
      • + *
    + * + *

    Authenticates with {@code DefaultAzureCredential}. Run without command-line arguments. + * Model deployment settings are not needed because this sample reads an existing conversation.

    + */ +public class GptLiveReadConversationAsyncSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_PROJECT_ENDPOINT"); + String agentName = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_GPT_LIVE_AGENT_NAME"); + String conversationId + = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_GPT_LIVE_CONVERSATION_ID"); + BetaVoiceAgentsLiveConversationsAsyncClient client = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .addPolicy(GptLiveReadConversationSample.previewHeaders()) + .buildBetaVoiceAgentsLiveConversationsAsyncClient(); + readConversation(client, agentName, conversationId).block(); + } + + static Mono readConversation(BetaVoiceAgentsLiveConversationsAsyncClient client, String agentName, + String conversationId) { + return listConversations(client, agentName, null) + .doOnNext(conversation -> System.out.printf("Available conversation: %s (%s)%n", + conversation.getId(), conversation.getStatus())) + .then(Mono.defer(() -> client.getAgentConversation(agentName, conversationId) + .doOnNext(GptLiveReadConversationSample::printConversation))) + .thenMany(listItems(client, agentName, conversationId, null)) + .concatMap(item -> client.getAgentConversationItem(agentName, conversationId, item.getId()), 0) + .doOnNext(GptLiveReadConversationSample::printItem) + .then(); + } + + private static Flux listConversations(BetaVoiceAgentsLiveConversationsAsyncClient client, + String agentName, String after) { + return client.listAgentConversations(agentName, 100, PageOrder.ASC, after, null).collectList() + .flatMapMany(page -> { + if (page.isEmpty()) { + return Flux.empty(); + } + String next = GptLiveReadConversationSample.nextCursor(page.get(page.size() - 1).getId(), after); + return Flux.fromIterable(page).concatWith(Flux.defer(() -> listConversations(client, agentName, next))); + }); + } + + private static Flux listItems(BetaVoiceAgentsLiveConversationsAsyncClient client, + String agentName, String conversationId, String after) { + return client.listAgentConversationItems(agentName, conversationId, 100, PageOrder.ASC, after, null).collectList() + .flatMapMany(page -> { + if (page.isEmpty()) { + return Flux.empty(); + } + String next = GptLiveReadConversationSample.nextCursor(page.get(page.size() - 1).getId(), after); + return Flux.fromIterable(page) + .concatWith(Flux.defer(() -> listItems(client, agentName, conversationId, next))); + }); + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioAsyncSample.java new file mode 100644 index 0000000000000..11afc71d5d84f --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioAsyncSample.java @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentsLiveConversationsAsyncClient; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import java.nio.file.Path; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +/** + * Asynchronously downloads persisted GPT Live merged audio and optionally a speech item's WAV to new files. + * The session must have used {@code store=true}. The agent and conversation must still exist. + * Wait for recording finalization after the session ends. Like {@link GptLiveReadConversationAudioSample}, + * HTTP errors propagate and BYOS audio requires separate Azure Storage credentials. + * File writes run on boundedElastic, not an HTTP I/O thread. + * Blocking occurs only at the standalone main method boundary. Global Reactor schedulers are not shut down. + * No agent or persisted data is modified or deleted. + * + *

    Before running the sample, set these environment variables:

    + *
      + *
    • {@code FOUNDRY_PROJECT_ENDPOINT} - Required. The Azure AI Project endpoint. No default.
      • + *
    • {@code FOUNDRY_GPT_LIVE_AGENT_NAME} - Required. The existing GPT Live agent's name. No default.
      • + *
    • {@code FOUNDRY_GPT_LIVE_CONVERSATION_ID} - Required. The persisted Foundry conversation ID from + * {@code session.started.conversation_id}, not the native session ID. No default.
      • + *
    • {@code FOUNDRY_GPT_LIVE_ITEM_ID} - Optional. A speech item's ID, not a delegation item. + * Defaults to unset, downloading only the merged conversation audio. Explicitly blank values are invalid.
      • + *
    + * + *

    To download an individual speech item's audio:

    + *
      + *
    1. Run {@link GptLiveReadConversationSample} with the same project endpoint, agent name, and conversation ID.
      2. + *
    2. Find the desired transcript in its output and copy the ID from the preceding + * {@code Item : type=input} or {@code Item : type=output} line. Do not select a delegation item.
      3. + *
    3. Set {@code FOUNDRY_GPT_LIVE_ITEM_ID} to that ID, then run this sample with an existing output directory + * as its sole command-line argument. It downloads both the merged call audio and the selected item's WAV.
      4. + *
    + *

    For merged call audio only, skip the item lookup and leave {@code FOUNDRY_GPT_LIVE_ITEM_ID} unset.

    + * + *

    Authenticates with {@code DefaultAzureCredential}. Pass one existing output directory as the sole + * command-line argument; there is no default directory. Treat downloaded audio as sensitive data. + * Model deployment settings are not needed because this sample reads an existing conversation.

    + */ +public class GptLiveReadConversationAudioAsyncSample { + public static void main(String[] args) { + Path directory = GptLiveReadConversationAudioSample.outputDirectory(args); + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_PROJECT_ENDPOINT"); + String agentName = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_GPT_LIVE_AGENT_NAME"); + String conversationId + = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_GPT_LIVE_CONVERSATION_ID"); + String itemId = configuration.get("FOUNDRY_GPT_LIVE_ITEM_ID"); + BetaVoiceAgentsLiveConversationsAsyncClient client = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .addPolicy(GptLiveReadConversationSample.previewHeaders()) + .buildBetaVoiceAgentsLiveConversationsAsyncClient(); + downloadAudio(client, agentName, conversationId, itemId, directory).block(); + } + + static Mono downloadAudio(BetaVoiceAgentsLiveConversationsAsyncClient client, String agentName, + String conversationId, String itemId, Path directory) { + if (itemId != null && itemId.trim().isEmpty()) { + return Mono.error(new IllegalArgumentException("FOUNDRY_GPT_LIVE_ITEM_ID must not be blank when set.")); + } + Mono merged = client.getAgentConversationAudio(agentName, conversationId).flatMap(recording -> { + GptLiveReadConversationAudioSample.printRecording(recording); + return GptLiveReadConversationAudioSample.isFoundryManaged(recording.getBlobUri()) + ? client.downloadAgentConversationAudio(agentName, conversationId) + .flatMap(audio -> saveAudio(audio, directory, "gpt-live-call-")) + : Mono.empty(); + }); + return merged.then(Mono.defer(() -> itemId == null ? Mono.empty() + : client.getAgentConversationAudioItem(agentName, conversationId, itemId).flatMap(item -> { + System.out.printf("Item %s: format=%s, rate=%d%n", itemId, item.getFormat(), item.getSampleRate()); + return GptLiveReadConversationAudioSample.isFoundryManaged(item.getBlobUri()) + ? client.downloadAgentConversationAudioItem(agentName, conversationId, itemId) + .flatMap(audio -> saveAudio(audio, directory, "gpt-live-item-")) + : Mono.empty(); + }))); + } + + private static Mono saveAudio(BinaryData audio, Path directory, String prefix) { + return Mono.fromCallable(() -> GptLiveReadConversationAudioSample.saveAudio(audio, directory, prefix)) + .subscribeOn(Schedulers.boundedElastic()) + .then(); + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioSample.java new file mode 100644 index 0000000000000..79c3c2db37198 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioSample.java @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentsLiveConversationsClient; +import com.azure.ai.agents.models.VoiceAudioItem; +import com.azure.ai.agents.models.VoiceRecording; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Downloads persisted GPT Live merged audio and optionally a speech item's WAV to new files. + * The session must have used {@code store=true}. The agent and conversation must still exist. + * Wait for recording finalization after the session ends. HTTP errors propagate: inspect Retry-After on + * recording_not_ready (409), and distinguish missing audio (404) from recording_unavailable (409). + * BYOS audio requires separate Azure Storage credentials; this sample does not download it. + * No agent or persisted data is modified or deleted. + * + *

    Before running the sample, set these environment variables:

    + *
      + *
    • {@code FOUNDRY_PROJECT_ENDPOINT} - Required. The Azure AI Project endpoint. No default.
      • + *
    • {@code FOUNDRY_GPT_LIVE_AGENT_NAME} - Required. The existing GPT Live agent's name. No default.
      • + *
    • {@code FOUNDRY_GPT_LIVE_CONVERSATION_ID} - Required. The persisted Foundry conversation ID from + * {@code session.started.conversation_id}, not the native session ID. No default.
      • + *
    • {@code FOUNDRY_GPT_LIVE_ITEM_ID} - Optional. A speech item's ID, not a delegation item. + * Defaults to unset, downloading only the merged conversation audio. Explicitly blank values are invalid.
      • + *
    + * + *

    To download an individual speech item's audio:

    + *
      + *
    1. Run {@link GptLiveReadConversationSample} with the same project endpoint, agent name, and conversation ID.
      2. + *
    2. Find the desired transcript in its output and copy the ID from the preceding + * {@code Item : type=input} or {@code Item : type=output} line. Do not select a delegation item.
      3. + *
    3. Set {@code FOUNDRY_GPT_LIVE_ITEM_ID} to that ID, then run this sample with an existing output directory + * as its sole command-line argument. It downloads both the merged call audio and the selected item's WAV.
      4. + *
    + *

    For merged call audio only, skip the item lookup and leave {@code FOUNDRY_GPT_LIVE_ITEM_ID} unset.

    + * + *

    Authenticates with {@code DefaultAzureCredential}. Pass one existing output directory as the sole + * command-line argument; there is no default directory. Treat downloaded audio as sensitive data. + * Model deployment settings are not needed because this sample reads an existing conversation.

    + */ +public class GptLiveReadConversationAudioSample { + public static void main(String[] args) throws IOException { + Path directory = outputDirectory(args); + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_PROJECT_ENDPOINT"); + String agentName = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_GPT_LIVE_AGENT_NAME"); + String conversationId + = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_GPT_LIVE_CONVERSATION_ID"); + String itemId = configuration.get("FOUNDRY_GPT_LIVE_ITEM_ID"); + BetaVoiceAgentsLiveConversationsClient client = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .addPolicy(GptLiveReadConversationSample.previewHeaders()) + .buildBetaVoiceAgentsLiveConversationsClient(); + downloadAudio(client, agentName, conversationId, itemId, directory); + } + + static void downloadAudio(BetaVoiceAgentsLiveConversationsClient client, String agentName, + String conversationId, String itemId, Path directory) throws IOException { + if (itemId != null && itemId.trim().isEmpty()) { + throw new IllegalArgumentException("FOUNDRY_GPT_LIVE_ITEM_ID must not be blank when set."); + } + VoiceRecording recording = client.getAgentConversationAudio(agentName, conversationId); + printRecording(recording); + if (isFoundryManaged(recording.getBlobUri())) { + saveAudio(client.downloadAgentConversationAudio(agentName, conversationId), directory, "gpt-live-call-"); + } + if (itemId != null) { + VoiceAudioItem item = client.getAgentConversationAudioItem(agentName, conversationId, itemId); + System.out.printf("Item %s: format=%s, rate=%d%n", itemId, item.getFormat(), item.getSampleRate()); + if (isFoundryManaged(item.getBlobUri())) { + saveAudio(client.downloadAgentConversationAudioItem(agentName, conversationId, itemId), + directory, "gpt-live-item-"); + } + } + } + + static void printRecording(VoiceRecording recording) { + System.out.printf("Recording: format=%s, rate=%d, channels=%d, durationMs=%s%n", + recording.getFormat(), recording.getSampleRate(), recording.getChannels(), recording.getDurationMs()); + } + + static boolean isFoundryManaged(String blobUri) { + if (blobUri == null || blobUri.isEmpty()) { + return true; + } + System.out.println("BYOS audio: use metadata.getBlobUri() with your own Azure Storage credentials."); + return false; + } + + static Path saveAudio(BinaryData audio, Path directory, String prefix) throws IOException { + Path output = Files.createTempFile(directory, prefix, ".wav"); + try (OutputStream stream = Files.newOutputStream(output)) { + audio.writeTo(stream); + } catch (IOException | RuntimeException error) { + try { + Files.deleteIfExists(output); + } catch (IOException cleanupError) { + error.addSuppressed(cleanupError); + } + throw error; + } + System.out.println("Saved WAV: " + output.toAbsolutePath()); + return output; + } + + static Path outputDirectory(String[] args) { + if (args.length != 1 || !Files.isDirectory(Paths.get(args[0]))) { + throw new IllegalArgumentException("Pass one existing output directory as the command-line argument."); + } + return Paths.get(args[0]); + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationSample.java new file mode 100644 index 0000000000000..b427f9e646fd3 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationSample.java @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentsLiveConversationsClient; +import com.azure.ai.agents.models.PageOrder; +import com.azure.ai.agents.models.VoiceAgentLiveConversation; +import com.azure.ai.agents.models.VoiceAgentLiveConversationItem; +import com.azure.ai.agents.models.VoiceAgentLiveDelegationConversationItem; +import com.azure.ai.agents.models.VoiceAgentLiveInputConversationItem; +import com.azure.ai.agents.models.VoiceAgentLiveOutputConversationItem; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.policy.AddHeadersPolicy; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Lists persisted GPT Live conversations and reads one conversation's status, usage, and transcript items. + * The session must have used {@code store=true}. Use {@code session.started.conversation_id}, not the native session ID. + * Stored speech segments are not native turns or proof of playback; delegation results are not spoken transcripts. + * This sample prints sensitive stored content. It does not create sessions, modify agents, or delete data. + * + *

    Before running the sample, set these environment variables:

    + *
      + *
    • {@code FOUNDRY_PROJECT_ENDPOINT} - Required. The Azure AI Project endpoint. No default.
      • + *
    • {@code FOUNDRY_GPT_LIVE_AGENT_NAME} - Required. The existing GPT Live agent's name. No default.
      • + *
    • {@code FOUNDRY_GPT_LIVE_CONVERSATION_ID} - Required. The persisted Foundry conversation ID from + * {@code session.started.conversation_id}. No default.
      • + *
    + * + *

    Authenticates with {@code DefaultAzureCredential}. Run without command-line arguments. + * Model deployment settings are not needed because this sample reads an existing conversation.

    + */ +public class GptLiveReadConversationSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = requireSetting(configuration, "FOUNDRY_PROJECT_ENDPOINT"); + String agentName = requireSetting(configuration, "FOUNDRY_GPT_LIVE_AGENT_NAME"); + String conversationId = requireSetting(configuration, "FOUNDRY_GPT_LIVE_CONVERSATION_ID"); + BetaVoiceAgentsLiveConversationsClient client = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .addPolicy(previewHeaders()) + .buildBetaVoiceAgentsLiveConversationsClient(); + readConversation(client, agentName, conversationId); + } + + static AddHeadersPolicy previewHeaders() { + // The generated Live REST builder methods do not automatically opt in to the voice preview. + return new AddHeadersPolicy(new HttpHeaders().set("Foundry-Features", "VoiceAgents=V1Preview")); + } + + static void readConversation(BetaVoiceAgentsLiveConversationsClient client, String agentName, + String conversationId) { + // Generated paging exposes cursor parameters but no automatic continuation token. + String after = null; + while (true) { + List page = client + .listAgentConversations(agentName, 100, PageOrder.ASC, after, null).stream().collect(Collectors.toList()); + if (page.isEmpty()) { + break; + } + for (VoiceAgentLiveConversation conversation : page) { + System.out.printf("Available conversation: %s (%s)%n", conversation.getId(), conversation.getStatus()); + } + after = nextCursor(page.get(page.size() - 1).getId(), after); + } + printConversation(client.getAgentConversation(agentName, conversationId)); + after = null; + while (true) { + List page = client + .listAgentConversationItems(agentName, conversationId, 100, PageOrder.ASC, after, null) + .stream().collect(Collectors.toList()); + if (page.isEmpty()) { + break; + } + for (VoiceAgentLiveConversationItem item : page) { + printItem(client.getAgentConversationItem(agentName, conversationId, item.getId())); + } + after = nextCursor(page.get(page.size() - 1).getId(), after); + } + } + + static String nextCursor(String id, String previous) { + if (id == null || id.isEmpty() || id.equals(previous)) { + throw new IllegalStateException("The service returned a missing or non-advancing pagination cursor."); + } + return id; + } + + static void printConversation(VoiceAgentLiveConversation conversation) { + System.out.printf("Conversation %s: status=%s, created=%s, completed=%s, usageSeconds=%s%n", + conversation.getId(), conversation.getStatus(), conversation.getCreatedAt(), conversation.getCompletedAt(), + conversation.getUsage() == null ? null : conversation.getUsage().getSeconds()); + } + + static void printItem(VoiceAgentLiveConversationItem item) { + System.out.printf("Item %s: type=%s%n", item.getId(), item.getType()); + if (item instanceof VoiceAgentLiveInputConversationItem) { + System.out.println("User transcript: " + ((VoiceAgentLiveInputConversationItem) item).getTranscription()); + } else if (item instanceof VoiceAgentLiveOutputConversationItem) { + System.out.println("Assistant transcript: " + ((VoiceAgentLiveOutputConversationItem) item).getTranscription()); + } else if (item instanceof VoiceAgentLiveDelegationConversationItem) { + VoiceAgentLiveDelegationConversationItem delegation = (VoiceAgentLiveDelegationConversationItem) item; + System.out.printf("Delegation target=%s, response=%s%n", delegation.getTarget(), delegation.getResponseId()); + } + } + + static String requireSetting(Configuration configuration, String name) { + String value = configuration.get(name); + if (value == null || value.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable " + name + " before running this sample."); + } + return value; + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java index bb1fe0420583c..f107c6dbc841d 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java @@ -118,7 +118,8 @@ static void printEvent(Map event) { checkError(event); if ("session.started".equals(type(event))) { Object conversationId = event.get("conversation_id"); - System.out.println(conversationId == null ? "Live session started." + System.out.println(conversationId == null + ? "Live session started; conversation_id not provided (persistence may be disabled)." : "Live session started; conversation_id=" + conversationId); } else if ("session.input_transcript.delta".equals(type(event))) { System.out.print(string(event, "delta")); diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java index cbffe655726c0..13a71a4dd9540 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java @@ -7,6 +7,7 @@ import com.azure.ai.agents.AgentsClientBuilder; import com.azure.core.util.BinaryData; import com.azure.core.util.Configuration; +import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.identity.DefaultAzureCredentialBuilder; import java.io.BufferedReader; import java.io.Console; @@ -52,6 +53,8 @@ *
  • {@code FOUNDRY_GPT_LIVE_MODEL} - Optional. The GPT Live deployment name. Defaults to {@code gpt-live-1}.
    • *
  • {@code FOUNDRY_GPT_LIVE_BACKEND_MODEL} - Optional. The Responses backend deployment name in the Live * resource. Defaults to {@code gpt-5.6-terra}.
    • + *
  • {@code FOUNDRY_GPT_LIVE_STORE} - Optional. Set to {@code true} to persist conversation content and audio + * and request a Foundry conversation ID. Defaults to {@code false}. Also keep the agent for later REST reads.
    • *
  • {@code FOUNDRY_KEEP_GPT_LIVE_AGENT} - Optional. Set to {@code true} to keep the agent after the sample. * Defaults to {@code false}.
    • * @@ -63,6 +66,7 @@ public static void main(String[] args) { String model = configuration.get("FOUNDRY_GPT_LIVE_MODEL"); String backendModel = configuration.get("FOUNDRY_GPT_LIVE_BACKEND_MODEL"); boolean keepAgent = Boolean.parseBoolean(configuration.get("FOUNDRY_KEEP_GPT_LIVE_AGENT", "false")); + boolean store = Boolean.parseBoolean(configuration.get("FOUNDRY_GPT_LIVE_STORE", "false")); if (endpoint == null || endpoint.trim().isEmpty()) { throw new IllegalArgumentException("Set environment variable FOUNDRY_PROJECT_ENDPOINT before running this sample."); @@ -85,7 +89,7 @@ public static void main(String[] args) { try (GptLiveSampleUtils.AgentResource agent = GptLiveSampleUtils.createAgent(builder, model, backendModel, "text-async", false, keepAgent)) { Mono.usingWhen(agent.builder.beta().buildBetaVoiceAgentLiveWebSocketAsyncClient() - .openWebSocketSession(agent.name), + .openWebSocketSession(agent.name, new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store)), session -> args.length == 0 ? runConversation(session, console == null ? new InputStreamReader(System.in) : console.reader()) : runConversation(session, Arrays.asList(args)), diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java index 226d2ab3211cd..46893afac9c13 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java @@ -7,6 +7,7 @@ import com.azure.ai.agents.AgentsClientBuilder; import com.azure.core.util.BinaryData; import com.azure.core.util.Configuration; +import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.identity.DefaultAzureCredentialBuilder; import java.io.Console; import java.io.IOException; @@ -42,6 +43,8 @@ *
  • {@code FOUNDRY_GPT_LIVE_MODEL} - Optional. The GPT Live deployment name. Defaults to {@code gpt-live-1}.
    • *
  • {@code FOUNDRY_GPT_LIVE_BACKEND_MODEL} - Optional. The Responses backend deployment name in the Live * resource. Defaults to {@code gpt-5.6-terra}.
    • + *
  • {@code FOUNDRY_GPT_LIVE_STORE} - Optional. Set to {@code true} to persist conversation content and audio + * and request a Foundry conversation ID. Defaults to {@code false}. Also keep the agent for later REST reads.
    • *
  • {@code FOUNDRY_KEEP_GPT_LIVE_AGENT} - Optional. Set to {@code true} to keep the agent after the sample. * Defaults to {@code false}.
    • * @@ -53,6 +56,7 @@ public static void main(String[] args) { String model = configuration.get("FOUNDRY_GPT_LIVE_MODEL"); String backendModel = configuration.get("FOUNDRY_GPT_LIVE_BACKEND_MODEL"); boolean keepAgent = Boolean.parseBoolean(configuration.get("FOUNDRY_KEEP_GPT_LIVE_AGENT", "false")); + boolean store = Boolean.parseBoolean(configuration.get("FOUNDRY_GPT_LIVE_STORE", "false")); if (endpoint == null || endpoint.trim().isEmpty()) { throw new IllegalArgumentException("Set environment variable FOUNDRY_PROJECT_ENDPOINT before running this sample."); @@ -74,7 +78,8 @@ public static void main(String[] args) { try (GptLiveSampleUtils.AgentResource agent = GptLiveSampleUtils.createAgent(builder, model, backendModel, "text", false, keepAgent); BetaVoiceAgentLiveWebSocketSessionClient session = agent.builder.beta() - .buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession(agent.name)) { + .buildBetaVoiceAgentLiveWebSocketClient() + .openWebSocketSession(agent.name, new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store))) { if (args.length == 0) { Console console = System.console(); runConversation(session, console == null ? new InputStreamReader(System.in) : console.reader()); diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveConversationSamplesTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveConversationSamplesTests.java new file mode 100644 index 0000000000000..b229838f0efe8 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveConversationSamplesTests.java @@ -0,0 +1,324 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.exception.HttpResponseException; +import com.azure.core.http.HttpClient; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import com.azure.core.http.policy.ExponentialBackoffOptions; +import com.azure.core.http.policy.RetryOptions; +import com.azure.core.test.http.MockHttpResponse; +import com.azure.core.test.utils.MockTokenCredential; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.core.util.Context; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Mono; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class GptLiveConversationSamplesTests { + private static final String ROOT = "/agents/live-agent/endpoint/protocols/voice/live/conversations"; + private static final String PATH = ROOT + "/conversation-1"; + private static final String CONVERSATION + = "{\"id\":\"conversation-1\",\"status\":\"completed\",\"created_at\":1700000000,\"usage\":{\"seconds\":2.5}}"; + private static final String INPUT = "{\"id\":\"input-1\",\"type\":\"input\",\"transcription\":\"Hello\"}"; + private static final String OUTPUT = "{\"id\":\"output-1\",\"type\":\"output\",\"transcription\":\"Hi\"}"; + private static final String DELEGATION + = "{\"id\":\"delegation-1\",\"type\":\"delegation\",\"target\":\"responses\",\"response_id\":\"response-1\"}"; + private static final byte[] WAV = { 82, 73, 70, 70, 0, 0, 0, 0, 87, 65, 86, 69 }; + private static final Duration TIMEOUT = Duration.ofSeconds(10); + + @TempDir + Path directory; + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void readsCursorPagesAndFetchesSpeechAndDelegationDetails(boolean async) { + ScriptedTransport transport = new ScriptedTransport(async); + transport.json(ROOT, null, 200, page(CONVERSATION)); + transport.json(ROOT, "conversation-1", 200, page()); + transport.json(PATH, null, 200, CONVERSATION); + transport.json(PATH + "/items", null, 200, page(INPUT)); + transport.json(PATH + "/items/input-1", null, 200, INPUT); + transport.json(PATH + "/items", "input-1", 200, page(OUTPUT, DELEGATION)); + transport.json(PATH + "/items/output-1", null, 200, OUTPUT); + transport.json(PATH + "/items/delegation-1", null, 200, DELEGATION); + transport.json(PATH + "/items", "delegation-1", 200, page()); + readConversation(builder(transport), async); + transport.assertConsumed(); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void missingConversationPropagates(boolean async) { + ScriptedTransport transport = new ScriptedTransport(async); + transport.json(ROOT, null, 200, page()); + transport.json(PATH, null, 404, error("not_found")); + HttpResponseException failure + = assertThrows(HttpResponseException.class, () -> readConversation(builder(transport), async)); + assertEquals(404, failure.getResponse().getStatusCode()); + transport.assertConsumed(); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void repeatedCursorFailsRatherThanLooping(boolean async) { + ScriptedTransport transport = new ScriptedTransport(async); + transport.json(ROOT, null, 200, page(CONVERSATION)); + transport.json(ROOT, "conversation-1", 200, page(CONVERSATION)); + assertThrows(IllegalStateException.class, () -> readConversation(builder(transport), async)); + transport.assertConsumed(); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void downloadsMergedAndItemBytesWithoutOverwritingFiles(boolean async) throws IOException { + Path existing = directory.resolve("existing.wav"); + byte[] original = { 1, 2, 3 }; + Files.write(existing, original); + ScriptedTransport transport = audioTransport(async, false); + downloadAudio(builder(transport), async, "input-1"); + transport.assertConsumed(); + assertEquals(3, files().size()); + assertArrayEquals(original, Files.readAllBytes(existing)); + for (Path file : files()) { + if (!file.equals(existing)) { + assertArrayEquals(WAV, Files.readAllBytes(file)); + assertTrue(file.getFileName().toString().endsWith(".wav")); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void byosMetadataDoesNotCallFoundryContentEndpoints(boolean async) throws IOException { + ScriptedTransport transport = audioTransport(async, true); + downloadAudio(builder(transport), async, "input-1"); + transport.assertConsumed(); + assertTrue(files().isEmpty()); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void optionalItemCanBeOmitted(boolean async) throws IOException { + ScriptedTransport transport = new ScriptedTransport(async); + transport.json(PATH + "/audio", null, 200, recording(false)); + transport.audio(PATH + "/audio/content"); + downloadAudio(builder(transport), async, null); + transport.assertConsumed(); + assertEquals(1, files().size()); + } + + @ParameterizedTest + @MethodSource("audioFailures") + public void audioErrorsPreserveStatusAndRetryAfter(boolean async, int status, boolean content) throws IOException { + ScriptedTransport transport = new ScriptedTransport(async); + if (content) { + transport.json(PATH + "/audio", null, 200, recording(false)); + } + transport.json(PATH + (content ? "/audio/content" : "/audio"), null, status, error("recording_not_ready")); + HttpResponseException failure + = assertThrows(HttpResponseException.class, () -> downloadAudio(builder(transport), async, null)); + assertEquals(status, failure.getResponse().getStatusCode()); + assertEquals("3", failure.getResponse().getHeaderValue("Retry-After")); + assertTrue(files().isEmpty()); + transport.assertConsumed(); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void blankItemFailsBeforeNetworkAccess(boolean async) { + ScriptedTransport transport = new ScriptedTransport(async); + assertThrows(IllegalArgumentException.class, () -> downloadAudio(builder(transport), async, " ")); + transport.assertConsumed(); + } + + @Test + public void failedFileWriteRemovesPartialOutput() throws IOException { + InputStream failingStream = new InputStream() { + @Override + public int read() throws IOException { + throw new IOException("Simulated read failure"); + } + }; + assertThrows(IOException.class, () -> GptLiveReadConversationAudioSample + .saveAudio(BinaryData.fromStream(failingStream), directory, "failure-")); + assertTrue(files().isEmpty()); + } + + @Test + public void configurationDirectoryAndCursorValidation() { + assertThrows(IllegalArgumentException.class, + () -> GptLiveReadConversationSample.requireSetting(Configuration.NONE, "FOUNDRY_PROJECT_ENDPOINT")); + assertThrows(IllegalArgumentException.class, + () -> GptLiveReadConversationAudioSample.outputDirectory(new String[0])); + assertThrows(IllegalArgumentException.class, () -> GptLiveReadConversationAudioSample + .outputDirectory(new String[] { directory.resolve("missing").toString() })); + assertEquals(directory, + GptLiveReadConversationAudioSample.outputDirectory(new String[] { directory.toString() })); + assertThrows(IllegalStateException.class, () -> GptLiveReadConversationSample.nextCursor(null, null)); + assertThrows(IllegalStateException.class, () -> GptLiveReadConversationSample.nextCursor("", null)); + } + + private static Stream audioFailures() { + return Stream.of(false, true) + .flatMap(async -> Stream.of(403, 404, 409) + .flatMap(status -> Stream.of(false, true).map(content -> Arguments.of(async, status, content)))); + } + + private static void readConversation(AgentsClientBuilder builder, boolean async) { + if (async) { + GptLiveReadConversationAsyncSample + .readConversation(builder.buildBetaVoiceAgentsLiveConversationsAsyncClient(), "live-agent", + "conversation-1") + .block(TIMEOUT); + } else { + GptLiveReadConversationSample.readConversation(builder.buildBetaVoiceAgentsLiveConversationsClient(), + "live-agent", "conversation-1"); + } + } + + private void downloadAudio(AgentsClientBuilder builder, boolean async, String itemId) throws IOException { + if (async) { + GptLiveReadConversationAudioAsyncSample + .downloadAudio(builder.buildBetaVoiceAgentsLiveConversationsAsyncClient(), "live-agent", + "conversation-1", itemId, directory) + .block(TIMEOUT); + } else { + GptLiveReadConversationAudioSample.downloadAudio(builder.buildBetaVoiceAgentsLiveConversationsClient(), + "live-agent", "conversation-1", itemId, directory); + } + } + + private static ScriptedTransport audioTransport(boolean async, boolean byos) { + ScriptedTransport transport = new ScriptedTransport(async); + transport.json(PATH + "/audio", null, 200, recording(byos)); + if (!byos) { + transport.audio(PATH + "/audio/content"); + } + transport.json(PATH + "/items/input-1/audio", null, 200, + "{\"conversation_id\":\"conversation-1\",\"item_id\":\"input-1\",\"format\":\"wav\",\"sample_rate\":24000" + + (byos ? ",\"blob_uri\":\"https://storage.example/item.wav\"" : "") + "}"); + if (!byos) { + transport.audio(PATH + "/items/input-1/audio/content"); + } + return transport; + } + + private List files() throws IOException { + try (Stream paths = Files.list(directory)) { + return paths.collect(Collectors.toList()); + } + } + + private static String recording(boolean byos) { + return "{\"conversation_id\":\"conversation-1\",\"format\":\"wav\",\"sample_rate\":24000,\"channels\":2," + + "\"channel_layout\":{\"left\":\"user\",\"right\":\"assistant\"},\"duration_ms\":2000" + + (byos ? ",\"blob_uri\":\"https://storage.example/call.wav\"" : "") + "}"; + } + + private static String page(String... entries) { + return "{\"data\":[" + String.join(",", entries) + "]}"; + } + + private static String error(String code) { + return "{\"error\":{\"code\":\"" + code + "\",\"message\":\"Sample service error\"}}"; + } + + private static AgentsClientBuilder builder(HttpClient transport) { + return new AgentsClientBuilder().endpoint("https://localhost") + .credential(new MockTokenCredential()) + .configuration(Configuration.NONE) + .addPolicy(GptLiveReadConversationSample.previewHeaders()) + .httpClient(transport) + .retryOptions(new RetryOptions(new ExponentialBackoffOptions().setMaxRetries(0))); + } + + private static final class ScriptedTransport implements HttpClient { + private final Deque> requests = new ArrayDeque<>(); + private final boolean async; + + ScriptedTransport(boolean async) { + this.async = async; + } + + void json(String path, String after, int status, String body) { + expect(path, after, status, "application/json", BinaryData.fromString(body).toBytes()); + } + + void audio(String path) { + expect(path, null, 200, "audio/wav", WAV); + } + + private void expect(String path, String after, int status, String contentType, byte[] body) { + requests.add(request -> { + assertEquals(HttpMethod.GET, request.getHttpMethod(), "Samples must not modify or delete data"); + assertEquals(path, request.getUrl().getPath()); + assertEquals("VoiceAgents=V1Preview", request.getHeaders().getValue("Foundry-Features")); + String query = request.getUrl().getQuery(); + assertTrue(query.contains("api-version=")); + if (path.equals(ROOT) || path.endsWith("/items")) { + assertTrue(query.contains("limit=100")); + assertTrue(query.contains("order=asc")); + } + if (after != null) { + assertTrue(query.contains("after=" + after), "Missing cursor: " + query); + } else { + assertFalse(query.contains("after="), "Unexpected cursor: " + query); + } + return new MockHttpResponse(request, status, + new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, contentType) + .set(HttpHeaderName.fromString("Retry-After"), "3"), + body); + }); + } + + @Override + public Mono send(HttpRequest request) { + assertTrue(async, "Synchronous examples must use synchronous HTTP"); + return Mono.fromSupplier(() -> respond(request)); + } + + @Override + public HttpResponse sendSync(HttpRequest request, Context context) { + assertFalse(async, "Asynchronous examples must use asynchronous HTTP"); + return respond(request); + } + + private HttpResponse respond(HttpRequest request) { + assertFalse(requests.isEmpty(), "Unexpected request: " + request.getUrl()); + return requests.removeFirst().apply(request); + } + + void assertConsumed() { + assertTrue(requests.isEmpty(), "Not all expected requests were made"); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java index 355d4be9fcc53..1b8c6db6a7e05 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java @@ -6,6 +6,7 @@ import com.azure.ai.agents.AgentsClientBuilder; import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; +import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.core.credential.AccessToken; import com.azure.core.util.BinaryData; import com.azure.core.util.Configuration; @@ -116,8 +117,52 @@ public void audioTranscriptsLabelSpeakersWithoutPrintingBackendText() { System.setOut(original); } String transcript = new String(output.toByteArray(), StandardCharsets.UTF_8); - assertEquals("Live session started." + System.lineSeparator() + "You: Hello there" + System.lineSeparator() - + "Assistant: Hi!" + System.lineSeparator() + "Assistant: How can I help?", transcript); + assertEquals("Live session started; conversation_id not provided (persistence may be disabled)." + + System.lineSeparator() + "You: Hello there" + System.lineSeparator() + "Assistant: Hi!" + + System.lineSeparator() + "Assistant: How can I help?", transcript); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + @ResourceLock("java.lang.System.out") + public void startupPrintsFoundryConversationIdInScriptedAndInteractiveModes(boolean interactive) { + Map event = GptLiveSampleUtils.object(BinaryData.fromString( + "{\"type\":\"session.started\",\"conversation_id\":\"conversation-1\",\"session\":{\"id\":\"native-1\"}}")); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream original = System.out; + try (PrintStream capture = new PrintStream(output)) { + System.setOut(capture); + if (interactive) { + new GptLiveSampleUtils.AudioTranscriptPrinter(false, true).print(event); + } else { + GptLiveSampleUtils.printEvent(event); + } + } finally { + System.setOut(original); + } + assertEquals("Live session started; conversation_id=conversation-1" + System.lineSeparator(), + new String(output.toByteArray(), StandardCharsets.UTF_8)); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + @ResourceLock("java.lang.System.out") + public void startupWithoutPersistenceDoesNotSubstituteNativeSessionId(boolean interactive) { + Map event = GptLiveSampleUtils.object(BinaryData.fromString(STARTED)); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream original = System.out; + try (PrintStream capture = new PrintStream(output)) { + System.setOut(capture); + if (interactive) { + new GptLiveSampleUtils.AudioTranscriptPrinter(false, true).print(event); + } else { + GptLiveSampleUtils.printEvent(event); + } + } finally { + System.setOut(original); + } + assertEquals("Live session started; conversation_id not provided (persistence may be disabled)." + + System.lineSeparator(), new String(output.toByteArray(), StandardCharsets.UTF_8)); } @Test @@ -318,8 +363,8 @@ public void examplesValidateSettingsBeforeCreatingClients(String sampleName, Con configuration.put("FOUNDRY_GPT_LIVE_MODEL", "gpt-live-1"); configuration.put("FOUNDRY_GPT_LIVE_BACKEND_MODEL", "gpt-5.6-terra"); configuration.put(setting, value); - IllegalArgumentException error = assertThrows(IllegalArgumentException.class, - () -> main.accept(new String[0]), sampleName); + IllegalArgumentException error + = assertThrows(IllegalArgumentException.class, () -> main.accept(new String[0]), sampleName); assertTrue(error.getMessage().contains(setting)); } finally { original.forEach((name, previous) -> { @@ -338,9 +383,10 @@ private static Stream invalidSampleSettingCases() { samples.put("text", GptLiveTextConversationSample::main); samples.put("text-async", GptLiveTextConversationAsyncSample::main); samples.put("tools", GptLiveFunctionToolSample::main); - return samples.entrySet().stream() - .flatMap(sample -> Stream.of("FOUNDRY_PROJECT_ENDPOINT", "FOUNDRY_GPT_LIVE_MODEL", - "FOUNDRY_GPT_LIVE_BACKEND_MODEL") + return samples.entrySet() + .stream() + .flatMap(sample -> Stream + .of("FOUNDRY_PROJECT_ENDPOINT", "FOUNDRY_GPT_LIVE_MODEL", "FOUNDRY_GPT_LIVE_BACKEND_MODEL") .flatMap(setting -> Stream.of("", " ") .map(value -> Arguments.of(sample.getKey(), sample.getValue(), setting, value)))); } @@ -985,9 +1031,14 @@ public void nativeAudioEncodingAndDecodingPreservePcmBytes() { () -> GptLiveAudioConversationAsyncSample.decodeAudio(Collections.singletonMap("delta", "AA=="))); } + private static Stream audioSessionCases() { + return Stream.of(false, true) + .flatMap(moderated -> Stream.of(false, true).map(store -> Arguments.of(moderated, store))); + } + @ParameterizedTest - @ValueSource(booleans = { false, true }) - public void audioSampleGreetsCapturesPlaysAndReleasesDevices(boolean moderated) { + @MethodSource("audioSessionCases") + public void audioSampleGreetsCapturesPlaysAndReleasesDevices(boolean moderated, boolean store) { AtomicBoolean captured = new AtomicBoolean(); AtomicInteger closed = new AtomicInteger(); CountDownLatch unblockRead = new CountDownLatch(1); @@ -1020,14 +1071,17 @@ public void audioSampleGreetsCapturesPlaysAndReleasesDevices(boolean moderated) } return null; }); - server - = VoiceAgentWebSocketSessionTests.tlsServer() - .host("localhost") - .port(0) - .handle((request, - response) -> response.sendWebsocket((inbound, + AtomicReference requestUri = new AtomicReference<>(); + String started = store + ? "{\"type\":\"session.started\",\"conversation_id\":\"conversation-1\",\"session\":{\"id\":\"live-1\"}}" + : STARTED; + server = VoiceAgentWebSocketSessionTests.tlsServer().host("localhost").port(0).handle((request, response) -> { + requestUri.set(request.uri()); + return response + .sendWebsocket( + (inbound, outbound) -> outbound.sendString(Flux.concat( - Flux.just(STARTED, "{\"type\":\"session.output_audio.delta\",\"delta\":\"AQI=\"}"), + Flux.just(started, "{\"type\":\"session.output_audio.delta\",\"delta\":\"AQI=\"}"), inbound.receive().asString().concatMap(json -> { Map command = GptLiveSampleUtils.object(BinaryData.fromString(json)); commands.add(command); @@ -1038,13 +1092,14 @@ public void audioSampleGreetsCapturesPlaysAndReleasesDevices(boolean moderated) return Mono.just(MODERATED); } return "session.close".equals(command.get("type")) ? Mono.just(CLOSED) : Mono.empty(); - }))).then())) - .bindNow(); + }))).then()); + }).bindNow(); BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() .buildBetaVoiceAgentLiveWebSocketAsyncClient() - .openWebSocketSession("agent") + .openWebSocketSession("agent", new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store)) .block(TIMEOUT); assertNotNull(session); + assertTrue(requestUri.get().contains("store=" + store), requestUri.get()); InputStream enterAfterAudio = new InputStream() { @Override public int available() { From 03bbc5899bfaba11c158755de7a0f6d0b425ab5f Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Sun, 25 Oct 2026 11:54:40 +0800 Subject: [PATCH 04/20] Refactor GPT Live samples to share conversation utilities --- .../GptLiveAudioConversationAsyncSample.java | 71 +- .../voice/GptLiveFunctionToolSample.java | 51 +- .../GptLiveReadConversationAsyncSample.java | 6 +- ...tLiveReadConversationAudioAsyncSample.java | 6 +- .../GptLiveReadConversationAudioSample.java | 6 +- .../voice/GptLiveReadConversationSample.java | 13 +- .../ai/agents/voice/GptLiveSampleUtils.java | 770 ++++++++++++++++++ .../GptLiveTextConversationAsyncSample.java | 266 +----- .../voice/GptLiveTextConversationSample.java | 78 +- .../GptLiveConversationSamplesTests.java | 2 +- .../ai/agents/voice/GptLiveSamplesTests.java | 105 ++- 11 files changed, 856 insertions(+), 518 deletions(-) diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java index 683ffdb6f2613..10f71f7fb2241 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java @@ -8,12 +8,7 @@ import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.core.util.Configuration; import com.azure.identity.DefaultAzureCredentialBuilder; -import java.io.InputStream; -import java.util.Base64; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; import reactor.core.publisher.Mono; -import reactor.core.publisher.Sinks; import reactor.core.scheduler.Schedulers; /** @@ -67,7 +62,7 @@ public static void main(String[] args) { = GptLiveSampleUtils.createAgent(builder, model, backendModel, "audio-async", false, keepAgent)) { Mono.usingWhen(agent.builder.beta().buildBetaVoiceAgentLiveWebSocketAsyncClient() .openWebSocketSession(agent.name, new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store)), - GptLiveAudioConversationAsyncSample::runConversation, + GptLiveSampleUtils::runConversation, BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(); @@ -76,68 +71,4 @@ public static void main(String[] args) { } } - static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session) { - return Mono.defer(() -> runConversation(session, - new VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor( - null, null, pcm -> session.sendEvent(GptLiveSampleUtils.audio(pcm.toBytes()))), System.in)); - } - - static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, - VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor processor, InputStream input) { - return Mono.usingWhen(Mono.just(processor), - audio -> { - AtomicBoolean started = new AtomicBoolean(); - AtomicBoolean finalized = new AtomicBoolean(); - GptLiveSampleUtils.AudioTranscriptPrinter transcripts = new GptLiveSampleUtils.AudioTranscriptPrinter(); - Sinks.Empty ready = Sinks.empty(); - Mono receive = session.receiveEvents().concatMap(data -> { - Map event = GptLiveSampleUtils.object(data); - transcripts.print(event); - if ("session.started".equals(GptLiveSampleUtils.type(event))) { - if (!started.compareAndSet(false, true)) { - return Mono.error(new IllegalStateException("Duplicate session.started event.")); - } - return Mono.fromRunnable(() -> { - audio.start(); - }).subscribeOn(Schedulers.boundedElastic()) - .then(session.sendEvent(GptLiveSampleUtils.userMessage( - "Greet me briefly with: Hi, how can I help you today?"))) - .then(session.sendEvent(GptLiveSampleUtils.event("response.create"))) - .then(Mono.fromRunnable(() -> { - ready.tryEmitEmpty(); - System.out.println("Opening greeting requested. Speak into the microphone. " - + "Press Enter to end the call."); - })).thenReturn(data); - } else if ("session.output_audio.delta".equals(GptLiveSampleUtils.type(event))) { - audio.queueAudio(decodeAudio(event)); - } else if ("session.closed".equals(GptLiveSampleUtils.type(event))) { - finalized.set(true); - } - return Mono.just(data); - }).takeUntil(data -> - "session.closed".equals(GptLiveSampleUtils.type(GptLiveSampleUtils.object(data)))) - .then(Mono.defer(() -> finalized.get() ? Mono.empty() - : Mono.error(new IllegalStateException("GPT Live disconnected without session.closed.")))); - Mono stop = ready.asMono().timeout(GptLiveSampleUtils.TIMEOUT) - .then(VoiceAgentLiveAudioConversationAsyncSample.waitForEnter(input)) - .then(Mono.fromRunnable(audio::close).subscribeOn(Schedulers.boundedElastic())) - .then(session.sendEvent(GptLiveSampleUtils.event("session.close"))) - .then(Mono.delay(GptLiveSampleUtils.TIMEOUT)) - .then(Mono.error(new IllegalStateException("Timed out waiting for session.closed."))); - return Mono.firstWithSignal(receive, audio.failureSignal(), stop); - }, GptLiveAudioConversationAsyncSample::closeAudio, - (audio, error) -> closeAudio(audio), GptLiveAudioConversationAsyncSample::closeAudio); - } - - static byte[] decodeAudio(Map event) { - byte[] pcm = Base64.getDecoder().decode(GptLiveSampleUtils.string(event, "delta")); - if (pcm.length % 2 != 0) { - throw new IllegalArgumentException("GPT Live PCM16 audio must contain complete two-byte samples."); - } - return pcm; - } - - private static Mono closeAudio(VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor audio) { - return Mono.fromRunnable(audio::close).subscribeOn(Schedulers.boundedElastic()); - } } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java index 08b60a99f75a9..df9e0beeac618 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java @@ -8,11 +8,8 @@ import com.azure.core.util.BinaryData; import com.azure.core.util.Configuration; import com.azure.identity.DefaultAzureCredentialBuilder; -import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; /** @@ -61,7 +58,9 @@ public static void main(String[] args) { = GptLiveSampleUtils.createAgent(builder, model, backendModel, "tools", true, keepAgent); BetaVoiceAgentLiveWebSocketSessionClient session = agent.builder.beta() .buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession(agent.name)) { - runConversation(session, args.length == 0 ? "What is the weather in Seattle?" : args[0]); + GptLiveSampleUtils.runConversation(session, + Collections.singletonList(args.length == 0 ? "What is the weather in Seattle?" : args[0]), + GptLiveFunctionToolSample::toolResult); } } @@ -98,48 +97,4 @@ static BinaryData toolResult(Map call) { return GptLiveSampleUtils.event("response.item.create", "item", item); } - static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, String prompt) { - Iterator events = session.receiveEvents(GptLiveSampleUtils.TIMEOUT).iterator(); - GptLiveTextConversationSample.awaitType(events, "session.started"); - session.sendEvent(GptLiveSampleUtils.userMessage(prompt)); - session.sendEvent(GptLiveSampleUtils.event("response.create")); - List results = new ArrayList<>(); - int continuations = 0; - boolean complete = false; - while (events.hasNext()) { - Map event = GptLiveSampleUtils.object(events.next()); - GptLiveSampleUtils.printEvent(event); - if ("session.closed".equals(GptLiveSampleUtils.type(event))) { - throw new IllegalStateException("GPT Live closed before the tool conversation completed."); - } - if (!"response.event".equals(GptLiveSampleUtils.type(event))) { - continue; - } - Map backend = GptLiveSampleUtils.backendEvent(event); - if ("response.output_item.done".equals(GptLiveSampleUtils.type(backend))) { - Map item = GptLiveSampleUtils.object(backend.get("item")); - if ("function_call".equals(GptLiveSampleUtils.type(item))) { - results.add(toolResult(item)); - } - } else if ("response.completed".equals(GptLiveSampleUtils.type(backend))) { - if (results.isEmpty()) { - complete = true; - break; - } - if (++continuations > 8) { - throw new IllegalStateException("Sample tool continuation limit exceeded."); - } - for (BinaryData result : results) { - session.sendEvent(result); - } - results.clear(); - session.sendEvent(GptLiveSampleUtils.event("response.create")); - } - } - if (!complete) { - throw new IllegalStateException("GPT Live disconnected before the tool conversation completed."); - } - session.sendEvent(GptLiveSampleUtils.event("session.close")); - GptLiveTextConversationSample.awaitType(events, "session.closed"); - } } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAsyncSample.java index 61e4005660565..f233d0c572ca6 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAsyncSample.java @@ -33,10 +33,10 @@ public class GptLiveReadConversationAsyncSample { public static void main(String[] args) { Configuration configuration = Configuration.getGlobalConfiguration(); - String endpoint = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_PROJECT_ENDPOINT"); - String agentName = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_GPT_LIVE_AGENT_NAME"); + String endpoint = GptLiveSampleUtils.requireSetting(configuration, "FOUNDRY_PROJECT_ENDPOINT"); + String agentName = GptLiveSampleUtils.requireSetting(configuration, "FOUNDRY_GPT_LIVE_AGENT_NAME"); String conversationId - = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_GPT_LIVE_CONVERSATION_ID"); + = GptLiveSampleUtils.requireSetting(configuration, "FOUNDRY_GPT_LIVE_CONVERSATION_ID"); BetaVoiceAgentsLiveConversationsAsyncClient client = new AgentsClientBuilder().endpoint(endpoint) .credential(new DefaultAzureCredentialBuilder().build()) .addPolicy(GptLiveReadConversationSample.previewHeaders()) diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioAsyncSample.java index 11afc71d5d84f..39af834b887c4 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioAsyncSample.java @@ -49,10 +49,10 @@ public class GptLiveReadConversationAudioAsyncSample { public static void main(String[] args) { Path directory = GptLiveReadConversationAudioSample.outputDirectory(args); Configuration configuration = Configuration.getGlobalConfiguration(); - String endpoint = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_PROJECT_ENDPOINT"); - String agentName = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_GPT_LIVE_AGENT_NAME"); + String endpoint = GptLiveSampleUtils.requireSetting(configuration, "FOUNDRY_PROJECT_ENDPOINT"); + String agentName = GptLiveSampleUtils.requireSetting(configuration, "FOUNDRY_GPT_LIVE_AGENT_NAME"); String conversationId - = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_GPT_LIVE_CONVERSATION_ID"); + = GptLiveSampleUtils.requireSetting(configuration, "FOUNDRY_GPT_LIVE_CONVERSATION_ID"); String itemId = configuration.get("FOUNDRY_GPT_LIVE_ITEM_ID"); BetaVoiceAgentsLiveConversationsAsyncClient client = new AgentsClientBuilder().endpoint(endpoint) .credential(new DefaultAzureCredentialBuilder().build()) diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioSample.java index 79c3c2db37198..ee83c0d92f853 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationAudioSample.java @@ -52,10 +52,10 @@ public class GptLiveReadConversationAudioSample { public static void main(String[] args) throws IOException { Path directory = outputDirectory(args); Configuration configuration = Configuration.getGlobalConfiguration(); - String endpoint = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_PROJECT_ENDPOINT"); - String agentName = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_GPT_LIVE_AGENT_NAME"); + String endpoint = GptLiveSampleUtils.requireSetting(configuration, "FOUNDRY_PROJECT_ENDPOINT"); + String agentName = GptLiveSampleUtils.requireSetting(configuration, "FOUNDRY_GPT_LIVE_AGENT_NAME"); String conversationId - = GptLiveReadConversationSample.requireSetting(configuration, "FOUNDRY_GPT_LIVE_CONVERSATION_ID"); + = GptLiveSampleUtils.requireSetting(configuration, "FOUNDRY_GPT_LIVE_CONVERSATION_ID"); String itemId = configuration.get("FOUNDRY_GPT_LIVE_ITEM_ID"); BetaVoiceAgentsLiveConversationsClient client = new AgentsClientBuilder().endpoint(endpoint) .credential(new DefaultAzureCredentialBuilder().build()) diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationSample.java index b427f9e646fd3..cfb51e9180d9f 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveReadConversationSample.java @@ -38,9 +38,9 @@ public class GptLiveReadConversationSample { public static void main(String[] args) { Configuration configuration = Configuration.getGlobalConfiguration(); - String endpoint = requireSetting(configuration, "FOUNDRY_PROJECT_ENDPOINT"); - String agentName = requireSetting(configuration, "FOUNDRY_GPT_LIVE_AGENT_NAME"); - String conversationId = requireSetting(configuration, "FOUNDRY_GPT_LIVE_CONVERSATION_ID"); + String endpoint = GptLiveSampleUtils.requireSetting(configuration, "FOUNDRY_PROJECT_ENDPOINT"); + String agentName = GptLiveSampleUtils.requireSetting(configuration, "FOUNDRY_GPT_LIVE_AGENT_NAME"); + String conversationId = GptLiveSampleUtils.requireSetting(configuration, "FOUNDRY_GPT_LIVE_CONVERSATION_ID"); BetaVoiceAgentsLiveConversationsClient client = new AgentsClientBuilder().endpoint(endpoint) .credential(new DefaultAzureCredentialBuilder().build()) .addPolicy(previewHeaders()) @@ -109,11 +109,4 @@ static void printItem(VoiceAgentLiveConversationItem item) { } } - static String requireSetting(Configuration configuration, String name) { - String value = configuration.get(name); - if (value == null || value.trim().isEmpty()) { - throw new IllegalArgumentException("Set environment variable " + name + " before running this sample."); - } - return value; - } } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java index f107c6dbc841d..19fd0955a4806 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java @@ -5,6 +5,9 @@ import com.azure.ai.agents.AgentsClient; import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; import com.azure.ai.agents.models.CreateAgentVersionInput; import com.azure.ai.agents.models.LiveInitialSessionAudioParam; import com.azure.ai.agents.models.LiveResponsesDelegationParam; @@ -14,15 +17,44 @@ import com.azure.ai.agents.models.VoiceAgentLiveDefinition; import com.azure.ai.agents.models.VoiceAgentSubProtocol; import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; import com.azure.json.JsonProviders; import com.azure.json.JsonWriter; +import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Base64; +import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.LineUnavailableException; +import javax.sound.sampled.SourceDataLine; +import javax.sound.sampled.TargetDataLine; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.publisher.Sinks; +import reactor.core.scheduler.Schedulers; final class GptLiveSampleUtils { static final Duration TIMEOUT = Duration.ofSeconds(60); @@ -30,6 +62,553 @@ final class GptLiveSampleUtils { private GptLiveSampleUtils() { } + static String requireSetting(Configuration configuration, String name) { + String value = configuration.get(name); + if (value == null || value.trim().isEmpty()) { + throw new IllegalArgumentException("Set environment variable " + name + " before running this sample."); + } + return value; + } + + /** + * Microphone entry point called by {@link GptLiveAudioConversationAsyncSample}. + */ + static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session) { + return Mono.defer(() -> runConversation(session, + new AudioProcessor( + null, null, pcm -> session.sendEvent(audio(pcm.toBytes()))), System.in)); + } + + static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, + AudioProcessor processor, InputStream input) { + return Mono.usingWhen(Mono.just(processor), + audio -> Mono.firstWithSignal(runConversation(session.receiveEvents(), session::sendEvent, + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), + transcripts -> VoiceAgentLiveAudioConversationAsyncSample.waitForEnter(input) + .then(closeAudio(audio)).then(Mono.empty()), + audio::queueAudio, () -> { }, true, audio), audio.failureSignal()), GptLiveSampleUtils::closeAudio, + (audio, error) -> closeAudio(audio), GptLiveSampleUtils::closeAudio); + } + + static byte[] decodeAudio(Map event) { + byte[] pcm = Base64.getDecoder().decode(string(event, "delta")); + if (pcm.length % 2 != 0) { + throw new IllegalArgumentException("GPT Live PCM16 audio must contain complete two-byte samples."); + } + return pcm; + } + + private static Mono closeAudio(AudioProcessor audio) { + return Mono.fromRunnable(audio::close).subscribeOn(Schedulers.boundedElastic()); + } + + /** + * Scripted-text entry point called by {@link GptLiveTextConversationSample} when command-line prompts are supplied. + */ + static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, List prompts) { + runConversation(session, prompts, null); + } + + /** + * Called by {@link GptLiveFunctionToolSample}; also serves scripted {@link GptLiveTextConversationSample} calls + * without a tool handler. + */ + static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, List prompts, + Function, BinaryData> toolHandler) { + Iterator pending = prompts.iterator(); + runConversation(session, () -> { + if (!pending.hasNext()) { + return null; + } + String prompt = pending.next(); + System.out.println("You: " + prompt); + return prompt; + }, toolHandler); + } + + /** + * Interactive entry point called by {@link GptLiveTextConversationSample} when no command-line prompts are supplied. + */ + static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Reader input) { + try (VoiceAgentRealtimeSampleUtils.SpeakerPlayer player = new VoiceAgentRealtimeSampleUtils.SpeakerPlayer()) { + runConversation(session, input, player::play, player::drain); + player.drain(); + System.out.printf("(received %.2fs of reply audio%s)%n", player.getSecondsReceived(), + player.isEnabled() ? " and played it" : ""); + } + } + + static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Reader input, + Consumer playAudio, Runnable drainAudio) { + System.out.println("Type a message and press Enter. Blank line, 'exit' or 'quit' ends the session."); + System.out.println("Interactive voice streams silent audio while you type; voice usage is billable."); + BufferedReader console = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); + runConversation(session, transcripts -> { + try { + return readPromptBlocking(console, transcripts); + } catch (IOException error) { + throw new IllegalStateException("Unable to read console input.", error); + } + }, playAudio, drainAudio, true, null); + } + + /** + * Scripted-text entry point called by {@link GptLiveTextConversationAsyncSample} with command-line prompts. + */ + static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, List prompts) { + return Mono.defer(() -> { + Iterator pending = prompts.iterator(); + return runConversation(session, transcripts -> Mono.defer(() -> { + if (!pending.hasNext()) { + return Mono.empty(); + } + String prompt = pending.next(); + System.out.println("You: " + prompt); + return Mono.just(prompt); + }), pcm -> { }, () -> { }, false); + }); + } + + private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Supplier nextPrompt, + Function, BinaryData> toolHandler) { + runConversation(session, transcripts -> nextPrompt.get(), pcm -> { }, () -> { }, false, toolHandler); + } + + private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, + Function nextPrompt, Consumer playAudio, Runnable drainAudio, + boolean spoken, + Function, BinaryData> toolHandler) { + ExecutorService workers = Executors.newFixedThreadPool(2, task -> { + Thread thread = new Thread(task, "gpt-live-sync-conversation"); + thread.setDaemon(true); + return thread; + }); + BlockingQueue pending = new LinkedBlockingQueue<>(256); + AtomicReference receiveFailure = new AtomicReference<>(); + AtomicBoolean receiveEnded = new AtomicBoolean(); + AudioTranscriptPrinter transcripts = new AudioTranscriptPrinter(System.console() != null, spoken); + Future receiver = workers.submit(() -> { + try { + for (BinaryData data : session.receiveEvents(TIMEOUT)) { + pending.put(data); + if ("session.closed".equals(type(object(data)))) { + break; + } + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } catch (RuntimeException error) { + receiveFailure.set(error); + } finally { + receiveEnded.set(true); + } + }); + List results = new ArrayList<>(); + Future prompt = null; + boolean started = false; + boolean greetingComplete = !spoken; + boolean greetingAudio = false; + boolean greetingTranscript = false; + boolean awaitingResponse = false; + boolean closing = false; + int continuations = 0; + long deadline = System.nanoTime() + TIMEOUT.toNanos(); + long nextAudio = 0; + long quietSince = 0; + BinaryData silence = audio(new byte[960]); + try { + while (true) { + BinaryData data = pending.poll(20, TimeUnit.MILLISECONDS); + long now = System.nanoTime(); + if (data != null) { + Map event = object(data); + if (spoken) { + transcripts.print(event); + if ("session.output_audio.delta".equals(type(event))) { + byte[] pcm = decodeAudio(event); + playAudio.accept(pcm); + if (hasSpeech(pcm)) { + greetingAudio = true; + quietSince = now; + } + } else if ("session.output_transcript.delta".equals(type(event)) + && !string(event, "delta").trim().isEmpty()) { + greetingTranscript = true; + quietSince = now; + } + } else { + printEvent(event); + } + Map backend = backendEvent(event); + String type = type(backend); + if ("session.started".equals(type)) { + if (started) { + throw new IllegalStateException("Duplicate session.started event."); + } + started = true; + if (spoken) { + session.sendEvent(silence); + nextAudio = now + TimeUnit.MILLISECONDS.toNanos(20); + session.sendEvent(commentary("Hi, how can I help you today?")); + } + } else if ("response.completed".equals(type)) { + if (!awaitingResponse && spoken && started) { + continue; + } + if (!awaitingResponse || closing) { + throw new IllegalStateException("Unexpected GPT Live response lifecycle event."); + } + if (results.isEmpty()) { + awaitingResponse = false; + continuations = 0; + if (!spoken) { + System.out.println(); + } + } else { + if (++continuations > 8) { + throw new IllegalStateException("Sample tool continuation limit exceeded."); + } + for (BinaryData result : results) { + session.sendEvent(result); + } + results.clear(); + session.sendEvent(event("response.create")); + } + } else if (toolHandler != null && "response.output_item.done".equals(type)) { + Map item = object(backend.get("item")); + if ("function_call".equals(type(item))) { + results.add(toolHandler.apply(item)); + } + } else if ("session.closed".equals(type)) { + if (started && ("remote_hangup".equals(event.get("reason")) + || closing && "close_requested".equals(event.get("reason")))) { + System.out.println("Live session closed (" + event.get("reason") + ")."); + return; + } + throw new IllegalStateException("GPT Live closed before all turns completed. Event: " + data + + "; transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason()); + } + } else if (receiveEnded.get()) { + if (receiveFailure.get() != null) { + throw receiveFailure.get(); + } + throw new IllegalStateException("GPT Live disconnected without session.closed."); + } + + if (started && spoken && !closing && now >= nextAudio) { + session.sendEvent(silence); + nextAudio = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(20); + } + if (spoken && greetingAudio && greetingTranscript && quietSince != 0 + && now - quietSince >= TimeUnit.SECONDS.toNanos(1)) { + transcripts.endUtterance(); + quietSince = 0; + if (!greetingComplete) { + drainAudio.run(); + greetingComplete = true; + } + } + if (started && greetingComplete && !awaitingResponse && !closing && prompt == null) { + prompt = workers.submit(() -> nextPrompt.apply(transcripts)); + } + if (prompt != null && prompt.isDone()) { + String text = prompt.get(); + prompt = null; + deadline = System.nanoTime() + TIMEOUT.toNanos(); + if (text == null) { + closing = true; + session.sendEvent(event("session.close")); + } else { + awaitingResponse = true; + if (!spoken) { + System.out.print("Assistant: "); + } + session.sendEvent(userMessage(text)); + session.sendEvent(event("response.create")); + } + } + if ((!started || !greetingComplete || awaitingResponse || closing) && now >= deadline) { + throw new IllegalStateException("Timed out waiting for the GPT Live conversation."); + } + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("GPT Live conversation interrupted.", error); + } catch (ExecutionException error) { + if (error.getCause() instanceof RuntimeException) { + throw (RuntimeException) error.getCause(); + } + throw new IllegalStateException("Unable to read console input.", error.getCause()); + } finally { + receiver.cancel(true); + if (prompt != null) { + prompt.cancel(true); + } + transcripts.endInput(); + workers.shutdownNow(); + } + } + + /** + * Interactive entry point called by {@link GptLiveTextConversationAsyncSample} without command-line prompts. + */ + static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input) { + return Mono.usingWhen( + Mono.fromCallable(VoiceAgentRealtimeSampleUtils.SpeakerPlayer::new) + .subscribeOn(Schedulers.boundedElastic()), + player -> runConversation(session, input, player::play, player::drain), + player -> Mono.fromRunnable(() -> { + boolean speakerEnabled = player.isEnabled(); + try { + player.close(); + } finally { + System.out.printf("(received %.2fs of reply audio%s)%n", + player.getSecondsReceived(), speakerEnabled ? " and played it" : ""); + } + }).subscribeOn(Schedulers.boundedElastic())); + } + + static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input, + Consumer playAudio) { + return runConversation(session, input, playAudio, () -> { }); + } + + static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input, + Consumer playAudio, Runnable drainAudio) { + return Mono.defer(() -> runInteractiveConversation(session.receiveEvents(), session::sendEvent, + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), + input, playAudio, drainAudio)); + } + + static Mono runInteractiveConversation(Flux events, + Function> sendEvent, Supplier closeDetails, Reader input, + Consumer playAudio, Runnable drainAudio) { + return Mono.defer(() -> { + System.out.println("Type a message and press Enter. Blank line, 'exit' or 'quit' ends the session."); + System.out.println("Interactive voice streams silent audio while you type; voice usage is billable."); + BufferedReader console = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); + return runConversation(events, sendEvent, closeDetails, transcripts -> readPrompt(console, transcripts), + playAudio, drainAudio, true); + }); + } + + private static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, + Function> nextPrompt, + Consumer playAudio, Runnable drainAudio, boolean spoken) { + return Mono.defer(() -> runConversation(session.receiveEvents(), session::sendEvent, + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), + nextPrompt, playAudio, drainAudio, spoken)); + } + + private static Mono runConversation(Flux events, + Function> sendEvent, Supplier closeDetails, + Function> nextPrompt, + Consumer playAudio, Runnable drainAudio, boolean spoken) { + return runConversation(events, sendEvent, closeDetails, nextPrompt, playAudio, drainAudio, spoken, null); + } + + private static Mono runConversation(Flux events, + Function> sendEvent, Supplier closeDetails, + Function> nextPrompt, + Consumer playAudio, Runnable drainAudio, boolean spoken, AudioProcessor microphone) { + return Mono.defer(() -> { + boolean interactive = spoken && microphone == null; + AtomicBoolean finalized = new AtomicBoolean(); + AtomicBoolean started = new AtomicBoolean(); + AtomicBoolean closeRequested = new AtomicBoolean(); + AtomicBoolean remoteHangup = new AtomicBoolean(); + AudioTranscriptPrinter transcripts = new AudioTranscriptPrinter( + microphone == null && System.console() != null, interactive); + Greeting greeting = new Greeting(); + Sinks.Empty ready = Sinks.empty(); + Sinks.Empty closed = Sinks.empty(); + Sinks.Empty audioStarted = Sinks.empty(); + Sinks.Empty stopAudio = Sinks.empty(); + Sinks.Empty audioStopped = Sinks.empty(); + AtomicReference> responseCompleted = new AtomicReference<>(); + Mono receive = events + .publishOn(Schedulers.boundedElastic(), 1) + .doOnNext(data -> { + Map event = object(data); + String eventType = type(event); + if (spoken) { + transcripts.print(event); + if ("session.output_audio.delta".equals(eventType)) { + byte[] pcm = decodeAudio(event); + playAudio.accept(pcm); + greeting.onAudio(pcm); + } else if ("session.output_transcript.delta".equals(eventType) + && !string(event, "delta").trim().isEmpty()) { + greeting.onTranscript(); + } + } else { + printEvent(event); + } + String type = type(backendEvent(event)); + if ("session.started".equals(type)) { + if (!started.compareAndSet(false, true)) { + throw new IllegalStateException("Duplicate session.started event."); + } + if (microphone != null) { + microphone.start(); + } + ready.tryEmitEmpty(); + } else if ("response.completed".equals(type)) { + Sinks.Empty completion = responseCompleted.getAndSet(null); + if (spoken && started.get() && completion == null) { + return; + } + if (!started.get() || closeRequested.get() || completion == null) { + throw new IllegalStateException("Unexpected GPT Live response lifecycle event."); + } + if (!spoken) { + System.out.println(); + } + completion.tryEmitEmpty(); + } else if ("session.closed".equals(type)) { + boolean remote = started.get() && "remote_hangup".equals(event.get("reason")); + boolean requested = microphone != null + || closeRequested.get() && "close_requested".equals(event.get("reason")); + if (!requested && !remote) { + throw new IllegalStateException("GPT Live closed before all turns completed. Event: " + + data + "; " + closeDetails.get()); + } + remoteHangup.set(remote || microphone != null); + System.out.println("Live session closed (" + event.get("reason") + ")."); + finalized.set(true); + closed.tryEmitEmpty(); + } + }) + .takeUntil(data -> "session.closed".equals(type(object(data)))) + .then(Mono.defer(() -> remoteHangup.get() ? Mono.empty() : finalized.get() ? Mono.never() + : Mono.error(new IllegalStateException("GPT Live disconnected without session.closed.")))); + Mono audio = ready.asMono() + .thenMany(silenceFrames().takeUntilOther(stopAudio.asMono()) + .concatMap(frame -> sendEvent.apply(frame).doOnSuccess(ignored -> audioStarted.tryEmitEmpty()), 1)) + .then() + .doOnSuccess(ignored -> audioStopped.tryEmitEmpty()) + .then(Mono.never()); + Mono renderTranscripts = greeting.quietPeriods() + .doOnNext(ignored -> transcripts.endUtterance()).then(); + Mono conversation = ready.asMono().timeout(TIMEOUT) + .then(Mono.defer(() -> interactive ? audioStarted.asMono().timeout(TIMEOUT) : Mono.empty())) + .then(Mono.defer(() -> interactive + ? sendEvent.apply(commentary("Hi, how can I help you today?")) + .timeout(TIMEOUT) + .then(greeting.awaitPlayback(drainAudio)) + .then(Mono.fromRunnable(transcripts::endUtterance)) + : microphone == null ? Mono.empty() + : sendEvent.apply(userMessage("Greet me briefly with: Hi, how can I help you today?")) + .then(sendEvent.apply(event("response.create"))) + .then(Mono.fromRunnable(() -> System.out.println( + "Opening greeting requested. Speak into the microphone. Press Enter to end the call."))))) + .then(Mono.defer(() -> prompt(sendEvent, nextPrompt, responseCompleted, spoken, transcripts))) + .then(Mono.defer(() -> { + closeRequested.set(true); + stopAudio.tryEmitEmpty(); + return (interactive ? audioStopped.asMono() : Mono.empty()) + .then(sendEvent.apply(event("session.close"))) + .then(closed.asMono()).timeout(TIMEOUT); + })); + return interactive ? Mono.firstWithSignal(receive, audio, renderTranscripts, conversation) + : Mono.firstWithSignal(receive, conversation); + }); + } + + private static Mono prompt(Function> sendEvent, + Function> nextPrompt, + AtomicReference> responseCompleted, boolean spoken, + AudioTranscriptPrinter transcripts) { + return nextPrompt.apply(transcripts).flatMap(text -> { + Sinks.Empty completion = Sinks.empty(); + responseCompleted.set(completion); + if (!spoken) { + System.out.print("Assistant: "); + System.out.flush(); + } + return sendEvent.apply(userMessage(text)) + .then(sendEvent.apply(event("response.create"))) + .then(completion.asMono()).timeout(TIMEOUT) + .thenReturn(true); + }).defaultIfEmpty(false) + .flatMap(keepGoing -> keepGoing + ? Mono.defer(() -> prompt(sendEvent, nextPrompt, responseCompleted, spoken, transcripts)) : Mono.empty()); + } + + static Flux silenceFrames() { + // Twenty milliseconds of 24-kHz mono PCM16LE. + BinaryData silence = audio(new byte[960]); + return Mono.just(silence) + .concatWith(Mono.just(silence).delayElement(Duration.ofMillis(20)).repeat()); + } + + static final class Greeting { + // PCM16 peak levels up to 256 (about -42 dBFS) are treated as silence for this heuristic. + static final int SILENCE_PEAK_THRESHOLD = 256; + private final Sinks.Many progress = Sinks.many().replay().latest(); + private int received; + + void onAudio(byte[] pcm) { + if (hasSpeech(pcm)) { + received |= 1; + progress.tryEmitNext(received); + } + } + + void onTranscript() { + received |= 2; + progress.tryEmitNext(received); + } + + Mono awaitPlayback(Runnable drainAudio) { + // Output can continue with silent PCM after speaking; only speech activity restarts this timer. + return quietPeriods().next() + .timeout(TIMEOUT) + .then(Mono.fromRunnable(drainAudio).subscribeOn(Schedulers.boundedElastic())); + } + + Flux quietPeriods() { + return progress.asFlux().filter(parts -> parts == 3) + .switchMap(parts -> Mono.delay(Duration.ofSeconds(1))); + } + } + + static Mono readPrompt(Reader input) { + return readPrompt(input, new AudioTranscriptPrinter()); + } + + private static Mono readPrompt(Reader input, AudioTranscriptPrinter transcripts) { + return Mono.defer(() -> { + BufferedReader reader = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); + return Mono.fromCallable(() -> readPromptBlocking(reader, transcripts)).subscribeOn(Schedulers.boundedElastic()) + .doOnCancel(transcripts::endInput); + }); + } + + private static String readPromptBlocking(BufferedReader reader, AudioTranscriptPrinter transcripts) throws IOException { + transcripts.beginInput(); + try { + String line = reader.readLine(); + if (line == null) { + return null; + } + String text = line.trim(); + return text.isEmpty() || "exit".equalsIgnoreCase(text) || "quit".equalsIgnoreCase(text) ? null : text; + } finally { + transcripts.endInput(); + } + } + + private static boolean hasSpeech(byte[] pcm) { + for (int offset = 0; offset < pcm.length; offset += 2) { + int sample = (short) ((pcm[offset] & 0xff) | (pcm[offset + 1] << 8)); + if (sample < -Greeting.SILENCE_PEAK_THRESHOLD || sample > Greeting.SILENCE_PEAK_THRESHOLD) { + return true; + } + } + return false; + } + static Map object(BinaryData data) { return data.toObject(Map.class); } @@ -134,6 +713,197 @@ static void printEvent(Map event) { } } + static final class AudioProcessor implements AutoCloseable { + private static final Duration SEND_TIMEOUT = Duration.ofSeconds(10); + private static final int CHUNK_BYTES = 2400; + static final int MAX_PLAYBACK_BYTES = VoiceAgentRealtimeSampleUtils.SAMPLE_RATE * 2 * 60; + private static final byte[] STOP = new byte[0]; + private final Function> sendAudio; + private final AudioFormat format = new AudioFormat(VoiceAgentRealtimeSampleUtils.SAMPLE_RATE, 16, 1, true, false); + private final BlockingQueue playback = new LinkedBlockingQueue<>(MAX_PLAYBACK_BYTES / 2); + private int queuedPlaybackBytes; + private final AtomicBoolean running = new AtomicBoolean(); + private final Sinks.Empty failure = Sinks.empty(); + private boolean closed; + private TargetDataLine microphone; + private SourceDataLine speaker; + private Thread captureThread; + private Thread playbackThread; + + AudioProcessor(BetaVoiceAgentWebSocketSessionAsyncClient session) { + this(session, null, null); + } + + AudioProcessor(BetaVoiceAgentWebSocketSessionAsyncClient session, TargetDataLine microphone, SourceDataLine speaker) { + this(microphone, speaker, pcm -> session.appendInputAudio(pcm)); + } + + AudioProcessor(TargetDataLine microphone, SourceDataLine speaker, Function> sendAudio) { + this.sendAudio = sendAudio; + this.microphone = microphone; + this.speaker = speaker; + } + + Mono failureSignal() { + return failure.asMono(); + } + + synchronized void start() { + if (closed) { + throw new IllegalStateException("Audio processor is already closed."); + } + try { + if (microphone == null) { + microphone = AudioSystem.getTargetDataLine(format); + } + microphone.open(format, CHUNK_BYTES * 4); + if (speaker == null) { + speaker = AudioSystem.getSourceDataLine(format); + } + speaker.open(format); + microphone.start(); + speaker.start(); + } catch (LineUnavailableException | IllegalArgumentException error) { + close(); + throw new IllegalStateException("A 24-kHz mono PCM16 microphone and speaker are required.", error); + } + + running.set(true); + captureThread = new Thread(this::capture, "voice-agent-microphone"); + playbackThread = new Thread(this::playback, "voice-agent-speaker"); + captureThread.setDaemon(true); + playbackThread.setDaemon(true); + captureThread.start(); + playbackThread.start(); + } + + private void capture() { + byte[] buffer = new byte[CHUNK_BYTES]; + try { + while (running.get()) { + int read = microphone.read(buffer, 0, buffer.length); + if (read > 0 && running.get()) { + sendAudio.apply(BinaryData.fromBytes(Arrays.copyOf(buffer, read))).block(SEND_TIMEOUT); + } + } + } catch (RuntimeException error) { + fail(error); + } + } + + private void playback() { + try { + while (running.get()) { + byte[] pcm = playback.take(); + if (pcm == STOP) { + break; + } + synchronized (playback) { + queuedPlaybackBytes -= pcm.length; + } + speaker.write(pcm, 0, pcm.length); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + fail(error); + } catch (RuntimeException error) { + fail(error); + } + } + + void queueAudio(byte[] pcm) { + if (pcm == null || pcm.length == 0) { + return; + } + if (pcm.length % 2 != 0) { + fail(new IllegalArgumentException("PCM16 audio must contain complete two-byte samples.")); + return; + } + synchronized (playback) { + if (!running.get()) { + return; + } + if (pcm.length <= MAX_PLAYBACK_BYTES - queuedPlaybackBytes && playback.offer(pcm)) { + queuedPlaybackBytes += pcm.length; + return; + } + } + fail(new IllegalStateException("Audio playback backlog exceeded 60 seconds.")); + } + + private void fail(Throwable error) { + if (running.compareAndSet(true, false)) { + failure.tryEmitError(error); + } + } + + synchronized void skipPendingAudio() { + clearPlayback(); + if (speaker != null) { + speaker.flush(); + } + } + + @Override + public synchronized void close() { + if (closed) { + return; + } + closed = true; + running.set(false); + clearPlayback(); + playback.offer(STOP); + try { + closeLine(microphone); + } finally { + try { + closeLine(speaker); + } finally { + if (captureThread != null) { + captureThread.interrupt(); + } + if (playbackThread != null) { + playbackThread.interrupt(); + } + join(captureThread); + join(playbackThread); + } + } + } + + private void clearPlayback() { + synchronized (playback) { + byte[] discarded; + while ((discarded = playback.poll()) != null) { + queuedPlaybackBytes -= discarded.length; + } + } + } + + private static void closeLine(javax.sound.sampled.DataLine line) { + if (line != null) { + try { + line.stop(); + } finally { + line.close(); + } + } + } + + private static void join(Thread thread) { + if (thread != null && thread != Thread.currentThread()) { + try { + thread.join(SEND_TIMEOUT.toMillis()); + if (thread.isAlive()) { + System.err.println("Audio thread did not stop: " + thread.getName()); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } + } + } + } + static final class AudioTranscriptPrinter { static final int MAX_BUFFERED_TRANSCRIPT_CHARACTERS = 65536; private String speaker; diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java index 13a71a4dd9540..0dafec9b90006 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java @@ -5,27 +5,13 @@ import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; import com.azure.ai.agents.AgentsClientBuilder; -import com.azure.core.util.BinaryData; import com.azure.core.util.Configuration; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.identity.DefaultAzureCredentialBuilder; -import java.io.BufferedReader; import java.io.Console; import java.io.InputStreamReader; -import java.io.Reader; -import java.time.Duration; import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; -import java.util.function.Consumer; -import java.util.function.Supplier; -import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import reactor.core.publisher.Sinks; import reactor.core.scheduler.Schedulers; /** @@ -91,8 +77,9 @@ public static void main(String[] args) { Mono.usingWhen(agent.builder.beta().buildBetaVoiceAgentLiveWebSocketAsyncClient() .openWebSocketSession(agent.name, new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store)), session -> args.length == 0 - ? runConversation(session, console == null ? new InputStreamReader(System.in) : console.reader()) - : runConversation(session, Arrays.asList(args)), + ? GptLiveSampleUtils.runConversation(session, + console == null ? new InputStreamReader(System.in) : console.reader()) + : GptLiveSampleUtils.runConversation(session, Arrays.asList(args)), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(); @@ -101,251 +88,4 @@ public static void main(String[] args) { } } - static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, List prompts) { - return Mono.defer(() -> { - Iterator pending = prompts.iterator(); - return runConversation(session, transcripts -> Mono.defer(() -> { - if (!pending.hasNext()) { - return Mono.empty(); - } - String prompt = pending.next(); - System.out.println("You: " + prompt); - return Mono.just(prompt); - }), pcm -> { }, () -> { }, false); - }); - } - - static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input) { - return Mono.usingWhen( - Mono.fromCallable(VoiceAgentRealtimeSampleUtils.SpeakerPlayer::new) - .subscribeOn(Schedulers.boundedElastic()), - player -> runConversation(session, input, player::play, player::drain), - player -> Mono.fromRunnable(() -> { - boolean speakerEnabled = player.isEnabled(); - try { - player.close(); - } finally { - System.out.printf("(received %.2fs of reply audio%s)%n", - player.getSecondsReceived(), speakerEnabled ? " and played it" : ""); - } - }).subscribeOn(Schedulers.boundedElastic())); - } - - static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input, - Consumer playAudio) { - return runConversation(session, input, playAudio, () -> { }); - } - - static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input, - Consumer playAudio, Runnable drainAudio) { - return Mono.defer(() -> runInteractiveConversation(session.receiveEvents(), session::sendEvent, - () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), - input, playAudio, drainAudio)); - } - - static Mono runInteractiveConversation(Flux events, - Function> sendEvent, Supplier closeDetails, Reader input, - Consumer playAudio, Runnable drainAudio) { - return Mono.defer(() -> { - System.out.println("Type a message and press Enter. Blank line, 'exit' or 'quit' ends the session."); - System.out.println("Interactive voice streams silent audio while you type; voice usage is billable."); - BufferedReader console = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); - return runConversation(events, sendEvent, closeDetails, transcripts -> readPrompt(console, transcripts), - playAudio, drainAudio, true); - }); - } - - private static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, - Function> nextPrompt, - Consumer playAudio, Runnable drainAudio, boolean spoken) { - return Mono.defer(() -> runConversation(session.receiveEvents(), session::sendEvent, - () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), - nextPrompt, playAudio, drainAudio, spoken)); - } - - private static Mono runConversation(Flux events, - Function> sendEvent, Supplier closeDetails, - Function> nextPrompt, - Consumer playAudio, Runnable drainAudio, boolean spoken) { - return Mono.defer(() -> { - AtomicBoolean finalized = new AtomicBoolean(); - AtomicBoolean started = new AtomicBoolean(); - AtomicBoolean closeRequested = new AtomicBoolean(); - AtomicBoolean remoteHangup = new AtomicBoolean(); - GptLiveSampleUtils.AudioTranscriptPrinter transcripts - = new GptLiveSampleUtils.AudioTranscriptPrinter(System.console() != null, spoken); - Greeting greeting = new Greeting(); - Sinks.Empty ready = Sinks.empty(); - Sinks.Empty closed = Sinks.empty(); - Sinks.Empty audioStarted = Sinks.empty(); - Sinks.Empty stopAudio = Sinks.empty(); - Sinks.Empty audioStopped = Sinks.empty(); - AtomicReference> responseCompleted = new AtomicReference<>(); - Mono receive = events - .publishOn(Schedulers.boundedElastic(), 1) - .doOnNext(data -> { - Map event = GptLiveSampleUtils.object(data); - String eventType = GptLiveSampleUtils.type(event); - if (spoken) { - transcripts.print(event); - if ("session.output_audio.delta".equals(eventType)) { - byte[] pcm = GptLiveAudioConversationAsyncSample.decodeAudio(event); - playAudio.accept(pcm); - greeting.onAudio(pcm); - } else if ("session.output_transcript.delta".equals(eventType) - && !GptLiveSampleUtils.string(event, "delta").trim().isEmpty()) { - greeting.onTranscript(); - } - } else { - GptLiveSampleUtils.printEvent(event); - } - String type = GptLiveSampleUtils.type(GptLiveSampleUtils.backendEvent(event)); - if ("session.started".equals(type)) { - if (!started.compareAndSet(false, true)) { - throw new IllegalStateException("Duplicate session.started event."); - } - ready.tryEmitEmpty(); - } else if ("response.completed".equals(type)) { - Sinks.Empty completion = responseCompleted.getAndSet(null); - if (spoken && started.get() && completion == null) { - return; - } - if (!started.get() || closeRequested.get() || completion == null) { - throw new IllegalStateException("Unexpected GPT Live response lifecycle event."); - } - if (!spoken) { - System.out.println(); - } - completion.tryEmitEmpty(); - } else if ("session.closed".equals(type)) { - boolean remote = started.get() && "remote_hangup".equals(event.get("reason")); - boolean requested = closeRequested.get() && "close_requested".equals(event.get("reason")); - if (!requested && !remote) { - throw new IllegalStateException("GPT Live closed before all turns completed. Event: " - + data + "; " + closeDetails.get()); - } - remoteHangup.set(remote); - System.out.println("Live session closed (" + event.get("reason") + ")."); - finalized.set(true); - closed.tryEmitEmpty(); - } - }) - .takeUntil(data -> "session.closed".equals(GptLiveSampleUtils.type(GptLiveSampleUtils.object(data)))) - .then(Mono.defer(() -> remoteHangup.get() ? Mono.empty() : finalized.get() ? Mono.never() - : Mono.error(new IllegalStateException("GPT Live disconnected without session.closed.")))); - Mono audio = ready.asMono() - .thenMany(silenceFrames().takeUntilOther(stopAudio.asMono()) - .concatMap(frame -> sendEvent.apply(frame).doOnSuccess(ignored -> audioStarted.tryEmitEmpty()), 1)) - .then() - .doOnSuccess(ignored -> audioStopped.tryEmitEmpty()) - .then(Mono.never()); - Mono renderTranscripts = greeting.quietPeriods() - .doOnNext(ignored -> transcripts.endUtterance()).then(); - Mono conversation = ready.asMono().timeout(GptLiveSampleUtils.TIMEOUT) - .then(Mono.defer(() -> spoken ? audioStarted.asMono().timeout(GptLiveSampleUtils.TIMEOUT) : Mono.empty())) - .then(Mono.defer(() -> spoken - ? sendEvent.apply(GptLiveSampleUtils.commentary("Hi, how can I help you today?")) - .timeout(GptLiveSampleUtils.TIMEOUT) - .then(greeting.awaitPlayback(drainAudio)) - .then(Mono.fromRunnable(transcripts::endUtterance)) - : Mono.empty())) - .then(Mono.defer(() -> prompt(sendEvent, nextPrompt, responseCompleted, spoken, transcripts))) - .then(Mono.defer(() -> { - closeRequested.set(true); - stopAudio.tryEmitEmpty(); - return (spoken ? audioStopped.asMono() : Mono.empty()) - .then(sendEvent.apply(GptLiveSampleUtils.event("session.close"))) - .then(closed.asMono()).timeout(GptLiveSampleUtils.TIMEOUT); - })); - return spoken ? Mono.firstWithSignal(receive, audio, renderTranscripts, conversation) - : Mono.firstWithSignal(receive, conversation); - }); - } - - private static Mono prompt(Function> sendEvent, - Function> nextPrompt, - AtomicReference> responseCompleted, boolean spoken, - GptLiveSampleUtils.AudioTranscriptPrinter transcripts) { - return nextPrompt.apply(transcripts).flatMap(text -> { - Sinks.Empty completion = Sinks.empty(); - responseCompleted.set(completion); - if (!spoken) { - System.out.print("Assistant: "); - System.out.flush(); - } - return sendEvent.apply(GptLiveSampleUtils.userMessage(text)) - .then(sendEvent.apply(GptLiveSampleUtils.event("response.create"))) - .then(completion.asMono()).timeout(GptLiveSampleUtils.TIMEOUT) - .thenReturn(true); - }).defaultIfEmpty(false) - .flatMap(keepGoing -> keepGoing - ? Mono.defer(() -> prompt(sendEvent, nextPrompt, responseCompleted, spoken, transcripts)) : Mono.empty()); - } - - static Flux silenceFrames() { - // Twenty milliseconds of 24-kHz mono PCM16LE. - BinaryData silence = GptLiveSampleUtils.audio(new byte[960]); - return Mono.just(silence) - .concatWith(Mono.just(silence).delayElement(Duration.ofMillis(20)).repeat()); - } - - static final class Greeting { - // PCM16 peak levels up to 256 (about -42 dBFS) are treated as silence for this heuristic. - static final int SILENCE_PEAK_THRESHOLD = 256; - private final Sinks.Many progress = Sinks.many().replay().latest(); - private int received; - - void onAudio(byte[] pcm) { - for (int offset = 0; offset < pcm.length; offset += 2) { - int sample = (short) ((pcm[offset] & 0xff) | (pcm[offset + 1] << 8)); - if (sample < -SILENCE_PEAK_THRESHOLD || sample > SILENCE_PEAK_THRESHOLD) { - received |= 1; - progress.tryEmitNext(received); - return; - } - } - } - - void onTranscript() { - received |= 2; - progress.tryEmitNext(received); - } - - Mono awaitPlayback(Runnable drainAudio) { - // Output can continue with silent PCM after speaking; only speech activity restarts this timer. - return quietPeriods().next() - .timeout(GptLiveSampleUtils.TIMEOUT) - .then(Mono.fromRunnable(drainAudio).subscribeOn(Schedulers.boundedElastic())); - } - - Flux quietPeriods() { - return progress.asFlux().filter(parts -> parts == 3) - .switchMap(parts -> Mono.delay(Duration.ofSeconds(1))); - } - } - - static Mono readPrompt(Reader input) { - return readPrompt(input, new GptLiveSampleUtils.AudioTranscriptPrinter()); - } - - private static Mono readPrompt(Reader input, GptLiveSampleUtils.AudioTranscriptPrinter transcripts) { - return Mono.defer(() -> { - BufferedReader reader = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); - return Mono.fromCallable(() -> { - transcripts.beginInput(); - String line; - try { - line = reader.readLine(); - } finally { - transcripts.endInput(); - } - if (line == null) { - return null; - } - String text = line.trim(); - return text.isEmpty() || "exit".equalsIgnoreCase(text) || "quit".equalsIgnoreCase(text) ? null : text; - }).subscribeOn(Schedulers.boundedElastic()) - .doOnCancel(transcripts::endInput); - }); - } } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java index 46893afac9c13..f02f6bab3d30e 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java @@ -5,22 +5,12 @@ import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; import com.azure.ai.agents.AgentsClientBuilder; -import com.azure.core.util.BinaryData; import com.azure.core.util.Configuration; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.identity.DefaultAzureCredentialBuilder; import java.io.Console; -import java.io.IOException; import java.io.InputStreamReader; -import java.io.Reader; import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; -import java.util.function.Consumer; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; /** @@ -31,7 +21,7 @@ * It streams silent PCM while typing; voice usage is billable. Greeting completion uses the same speech-inactivity * heuristic and speaker drain as the asynchronous sample. Assistant transcript fragments are assembled into * utterances independently of typed turns; completed text waits until Enter when a You: line is active. - * Synchronous session I/O runs on workers using the asynchronous sample's shared voice-loop coordination. + * Synchronous session I/O runs on workers using the shared voice-loop coordination in GptLiveSampleUtils. * After a remote disconnect, a pending console read may require Enter to release the input worker. * Scripted command-line prompts print backend text without speaker playback. * A uniquely named agent is deleted on exit unless FOUNDRY_KEEP_GPT_LIVE_AGENT=true. @@ -82,74 +72,14 @@ public static void main(String[] args) { .openWebSocketSession(agent.name, new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store))) { if (args.length == 0) { Console console = System.console(); - runConversation(session, console == null ? new InputStreamReader(System.in) : console.reader()); + GptLiveSampleUtils.runConversation(session, + console == null ? new InputStreamReader(System.in) : console.reader()); } else { - runConversation(session, Arrays.asList(args)); + GptLiveSampleUtils.runConversation(session, Arrays.asList(args)); } } finally { Schedulers.shutdownNow(); } } - static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, List prompts) { - Iterator pending = prompts.iterator(); - runConversation(session, () -> { - if (!pending.hasNext()) { - return null; - } - String prompt = pending.next(); - System.out.println("You: " + prompt); - return prompt; - }); - } - - static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Reader input) { - try (VoiceAgentRealtimeSampleUtils.SpeakerPlayer player = new VoiceAgentRealtimeSampleUtils.SpeakerPlayer()) { - runConversation(session, input, player::play, player::drain); - player.drain(); - System.out.printf("(received %.2fs of reply audio%s)%n", player.getSecondsReceived(), - player.isEnabled() ? " and played it" : ""); - } - } - - static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Reader input, - Consumer playAudio, Runnable drainAudio) { - Flux events = Flux.defer(() -> Flux.fromIterable(session.receiveEvents(GptLiveSampleUtils.TIMEOUT))) - .subscribeOn(Schedulers.boundedElastic()); - GptLiveTextConversationAsyncSample.runInteractiveConversation(events, - event -> Mono.fromRunnable(() -> session.sendEvent(event)).subscribeOn(Schedulers.boundedElastic()), - () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), - input, playAudio, drainAudio) - .onErrorMap(IOException.class, error -> new IllegalStateException("Unable to read console input.", error)) - .block(); - } - - private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Supplier nextPrompt) { - Iterator events = session.receiveEvents(GptLiveSampleUtils.TIMEOUT).iterator(); - awaitType(events, "session.started"); - String prompt; - while ((prompt = nextPrompt.get()) != null) { - System.out.print("Assistant: "); - session.sendEvent(GptLiveSampleUtils.userMessage(prompt)); - session.sendEvent(GptLiveSampleUtils.event("response.create")); - awaitType(events, "response.completed"); - System.out.println(); - } - session.sendEvent(GptLiveSampleUtils.event("session.close")); - awaitType(events, "session.closed"); - } - - static void awaitType(Iterator events, String expected) { - while (events.hasNext()) { - Map event = GptLiveSampleUtils.object(events.next()); - GptLiveSampleUtils.printEvent(event); - if (expected.equals(GptLiveSampleUtils.type(GptLiveSampleUtils.backendEvent(event)))) { - return; - } - if ("session.closed".equals(GptLiveSampleUtils.type(event))) { - throw new IllegalStateException("GPT Live session closed before " + expected); - } - } - throw new IllegalStateException("GPT Live connection ended before " + expected); - } } diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveConversationSamplesTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveConversationSamplesTests.java index b229838f0efe8..1eb8fda65990f 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveConversationSamplesTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveConversationSamplesTests.java @@ -176,7 +176,7 @@ public int read() throws IOException { @Test public void configurationDirectoryAndCursorValidation() { assertThrows(IllegalArgumentException.class, - () -> GptLiveReadConversationSample.requireSetting(Configuration.NONE, "FOUNDRY_PROJECT_ENDPOINT")); + () -> GptLiveSampleUtils.requireSetting(Configuration.NONE, "FOUNDRY_PROJECT_ENDPOINT")); assertThrows(IllegalArgumentException.class, () -> GptLiveReadConversationAudioSample.outputDirectory(new String[0])); assertThrows(IllegalArgumentException.class, () -> GptLiveReadConversationAudioSample diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java index 1b8c6db6a7e05..664b051b3390a 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java @@ -414,12 +414,12 @@ public void textSamplesWaitForStartupAndCompleteMultipleTurns(boolean async) { startTextServer(false, false); if (async) { Mono.usingWhen(builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), - session -> GptLiveTextConversationAsyncSample.runConversation(session, Arrays.asList("one", "two")), + session -> GptLiveSampleUtils.runConversation(session, Arrays.asList("one", "two")), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(TIMEOUT); } else { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - GptLiveTextConversationSample.runConversation(session, Arrays.asList("one", "two")); + GptLiveSampleUtils.runConversation(session, Arrays.asList("one", "two")); } } List expected = Arrays.asList("response.item.create", "response.create", "response.item.create", @@ -485,25 +485,32 @@ private void runInteractiveConversation(boolean async, Reader input, Consumer GptLiveTextConversationAsyncSample.runConversation(session, input, playAudio, drainAudio), + session -> GptLiveSampleUtils.runConversation(session, input, playAudio, drainAudio), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(TIMEOUT); } else { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - GptLiveTextConversationSample.runConversation(session, input, playAudio, drainAudio); + GptLiveSampleUtils.runConversation(session, input, playAudio, drainAudio); } } } - @Test - public void interactiveTextSampleReceivesServiceErrorsWhileWaitingForInput() { + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void interactiveTextSampleReceivesServiceErrorsWhileWaitingForInput(boolean async) { + Sinks.Empty inputStarted = Sinks.empty(); server = VoiceAgentWebSocketSessionTests.tlsServer() .host("localhost") .port(0) - .handle((request, - response) -> response.sendWebsocket((inbound, outbound) -> outbound - .sendString(Flux.just(STARTED, MODERATED).delayElements(Duration.ofMillis(100))) - .then())) + .handle((request, response) -> response.sendWebsocket((inbound, outbound) -> outbound + .sendString(Flux.concat(Mono.just(STARTED), inbound.receive().asString().concatMap(json -> { + Map command = GptLiveSampleUtils.object(BinaryData.fromString(json)); + return "session.commentary.append".equals(command.get("type")) + ? Flux.just("{\"type\":\"session.output_transcript.delta\",\"delta\":\"Welcome.\"}", + "{\"type\":\"session.output_audio.delta\",\"delta\":\"AQI=\"}") + : Flux.empty(); + })).mergeWith(inputStarted.asMono().thenReturn(MODERATED))) + .then())) .bindNow(); CountDownLatch releaseRead = new CountDownLatch(1); Reader idleInput = new Reader() { @@ -514,6 +521,7 @@ public boolean ready() { @Override public int read(char[] buffer, int offset, int length) throws IOException { + inputStarted.tryEmitEmpty(); try { releaseRead.await(); return -1; @@ -529,10 +537,9 @@ public void close() { } }; try { - StepVerifier.create(Mono.usingWhen( - builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), - session -> GptLiveTextConversationAsyncSample.runConversation(session, idleInput, pcm -> { - }), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync)) + StepVerifier + .create(Mono.fromRunnable(() -> runInteractiveConversation(async, idleInput)) + .subscribeOn(Schedulers.boundedElastic())) .expectErrorMatches( error -> error instanceof IllegalStateException && error.getMessage().contains("content_filter")) .verify(TIMEOUT); @@ -549,7 +556,7 @@ public int read(char[] buffer, int offset, int length) throws IOException { throw new IOException("Console unavailable"); } }; - StepVerifier.create(GptLiveTextConversationAsyncSample.readPrompt(input)) + StepVerifier.create(GptLiveSampleUtils.readPrompt(input)) .expectErrorMatches( error -> error instanceof IOException && "Console unavailable".equals(error.getMessage())) .verify(TIMEOUT); @@ -558,7 +565,7 @@ public int read(char[] buffer, int offset, int length) throws IOException { @Test public void consoleInputIsSubmittedOnlyAfterEnterWithoutLosingCharacters() throws Exception { try (PipedReader input = new PipedReader(); PipedWriter keyboard = new PipedWriter(input)) { - CompletableFuture prompt = GptLiveTextConversationAsyncSample.readPrompt(input).toFuture(); + CompletableFuture prompt = GptLiveSampleUtils.readPrompt(input).toFuture(); try { keyboard.write("testtesttes testing"); keyboard.flush(); @@ -653,10 +660,10 @@ public int read(char[] buffer, int offset, int length) throws IOException { @Test public void greetingWaitsForBothOutputsAndRestartsQuietPeriodForEachChunk() { - AtomicReference greeting = new AtomicReference<>(); + AtomicReference greeting = new AtomicReference<>(); AtomicBoolean drained = new AtomicBoolean(); StepVerifier.withVirtualTime(() -> { - GptLiveTextConversationAsyncSample.Greeting pending = new GptLiveTextConversationAsyncSample.Greeting(); + GptLiveSampleUtils.Greeting pending = new GptLiveSampleUtils.Greeting(); greeting.set(pending); return pending.awaitPlayback(() -> drained.set(true)); }) @@ -680,7 +687,7 @@ public void greetingWaitsForBothOutputsAndRestartsQuietPeriodForEachChunk() { public void missingGreetingOutputTimesOutInsteadOfStartingInput(String type) { AtomicBoolean drained = new AtomicBoolean(); StepVerifier.withVirtualTime(() -> { - GptLiveTextConversationAsyncSample.Greeting greeting = new GptLiveTextConversationAsyncSample.Greeting(); + GptLiveSampleUtils.Greeting greeting = new GptLiveSampleUtils.Greeting(); if ("session.output_audio.delta".equals(type)) { greeting.onAudio(new byte[] { 1, 2 }); } else { @@ -694,7 +701,7 @@ public void missingGreetingOutputTimesOutInsteadOfStartingInput(String type) { @Test public void greetingDrainFailureIsReported() { StepVerifier.withVirtualTime(() -> { - GptLiveTextConversationAsyncSample.Greeting greeting = new GptLiveTextConversationAsyncSample.Greeting(); + GptLiveSampleUtils.Greeting greeting = new GptLiveSampleUtils.Greeting(); greeting.onAudio(new byte[] { 1, 2 }); greeting.onTranscript(); return greeting.awaitPlayback(() -> { @@ -707,7 +714,7 @@ public void greetingDrainFailureIsReported() { public void continuousSilentOutputDoesNotKeepTheGreetingOpen() { AtomicBoolean drained = new AtomicBoolean(); StepVerifier.withVirtualTime(() -> { - GptLiveTextConversationAsyncSample.Greeting greeting = new GptLiveTextConversationAsyncSample.Greeting(); + GptLiveSampleUtils.Greeting greeting = new GptLiveSampleUtils.Greeting(); greeting.onAudio(new byte[] { 1, 2 }); greeting.onTranscript(); Mono silence = Flux.interval(Duration.ofMillis(20)) @@ -748,10 +755,10 @@ public void continuousSilentAudioStillAllowsInputAfterTheGreeting(boolean async) @ParameterizedTest @ValueSource(ints = { 0, 256, -256, 257, -257, 32767, -32768 }) public void greetingSpeechDetectionHonorsSignedPcm16PeakThreshold(int peak) { - boolean audible = Math.abs(peak) > GptLiveTextConversationAsyncSample.Greeting.SILENCE_PEAK_THRESHOLD; + boolean audible = Math.abs(peak) > GptLiveSampleUtils.Greeting.SILENCE_PEAK_THRESHOLD; AtomicBoolean drained = new AtomicBoolean(); StepVerifier.Step verification = StepVerifier.withVirtualTime(() -> { - GptLiveTextConversationAsyncSample.Greeting greeting = new GptLiveTextConversationAsyncSample.Greeting(); + GptLiveSampleUtils.Greeting greeting = new GptLiveSampleUtils.Greeting(); greeting.onTranscript(); greeting.onAudio(new byte[] { (byte) peak, (byte) (peak >> 8) }); return greeting.awaitPlayback(() -> drained.set(true)); @@ -767,7 +774,7 @@ public void greetingSpeechDetectionHonorsSignedPcm16PeakThreshold(int peak) { @ParameterizedTest @ValueSource(strings = { "", "\r\n", "exit\n", "quit\n" }) public void consoleEofBlankAndExitProduceNoPrompt(String input) { - StepVerifier.create(GptLiveTextConversationAsyncSample.readPrompt(new StringReader(input))).verifyComplete(); + StepVerifier.create(GptLiveSampleUtils.readPrompt(new StringReader(input))).verifyComplete(); } @ParameterizedTest @@ -822,7 +829,7 @@ public void interactiveTextPlaysGreetingAndRepliesWithoutPrintingBackendNotes(bo @Test public void silentAudioFramesArePacedAtRealTimePcmRate() { - StepVerifier.withVirtualTime(() -> GptLiveTextConversationAsyncSample.silenceFrames().take(3)) + StepVerifier.withVirtualTime(() -> GptLiveSampleUtils.silenceFrames().take(3)) .expectSubscription() .thenAwait(Duration.ZERO) .assertNext(this::assertSilentAudioFrame) @@ -837,7 +844,7 @@ public void silentAudioFramesArePacedAtRealTimePcmRate() { @Test public void silentAudioFramesWaitForDemandWithoutTimerOverflow() { - StepVerifier.withVirtualTime(() -> GptLiveTextConversationAsyncSample.silenceFrames().take(3), 0) + StepVerifier.withVirtualTime(() -> GptLiveSampleUtils.silenceFrames().take(3), 0) .expectSubscription() .expectNoEvent(Duration.ofSeconds(40)) .thenRequest(1) @@ -856,7 +863,7 @@ public void silentAudioFramesWaitForDemandWithoutTimerOverflow() { @Test public void silentAudioToleratesSendsSlowerThanItsFrameRate() { StepVerifier - .withVirtualTime(() -> GptLiveTextConversationAsyncSample.silenceFrames() + .withVirtualTime(() -> GptLiveSampleUtils.silenceFrames() .concatMap(frame -> Mono.delay(Duration.ofMillis(75)).thenReturn(frame), 1) .take(500)) .expectSubscription() @@ -925,7 +932,7 @@ public void close() { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { IllegalStateException error = assertThrows(IllegalStateException.class, - () -> GptLiveTextConversationSample.runConversation(session, input, pcm -> { + () -> GptLiveSampleUtils.runConversation(session, input, pcm -> { }, () -> { })); assertEquals("Unable to read console input.", error.getMessage()); @@ -941,8 +948,7 @@ public void serviceErrorsFailBothTextSamples(boolean async) { StepVerifier .create(Mono.usingWhen( builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), - session -> GptLiveTextConversationAsyncSample.runConversation(session, - Collections.singletonList("one")), + session -> GptLiveSampleUtils.runConversation(session, Collections.singletonList("one")), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync)) .expectErrorMatches(error -> error instanceof IllegalStateException && error.getMessage().contains("GPT Live service error")) @@ -951,7 +957,7 @@ public void serviceErrorsFailBothTextSamples(boolean async) { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { assertThrows(IllegalStateException.class, - () -> GptLiveTextConversationSample.runConversation(session, Collections.singletonList("one"))); + () -> GptLiveSampleUtils.runConversation(session, Collections.singletonList("one"))); } } assertFalse(commandTypes().contains("session.close")); @@ -962,7 +968,8 @@ public void toolsUseGranularEventsAndReturnResultsBeforeContinuation() { startTextServer(true, false); try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - GptLiveFunctionToolSample.runConversation(session, "Weather in Seattle?"); + GptLiveSampleUtils.runConversation(session, Collections.singletonList("Weather in Seattle?"), + GptLiveFunctionToolSample::toolResult); } assertEquals(Arrays.asList("response.item.create", "response.create", "response.item.create", "response.create", "session.close"), commandTypes()); @@ -975,6 +982,21 @@ public void toolsUseGranularEventsAndReturnResultsBeforeContinuation() { assertFalse(item.containsKey("delegation_id")); } + @Test + public void toolConversationUsesSharedMultiTurnLoop() { + startTextServer(true, false); + try (BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + GptLiveSampleUtils.runConversation(session, Arrays.asList("Weather in Seattle?", "Thanks!"), + GptLiveFunctionToolSample::toolResult); + } + assertEquals(Arrays.asList("response.item.create", "response.create", "response.item.create", "response.create", + "response.item.create", "response.create", "session.close"), commandTypes()); + Map followUp = GptLiveSampleUtils.object(commands.get(4).get("item")); + assertEquals("user", followUp.get("role")); + assertEquals("Thanks!", GptLiveSampleUtils.object(((List) followUp.get("content")).get(0)).get("text")); + } + @Test public void toolSchemaAndArgumentsAreValidated() { Map tool = GptLiveSampleUtils.object(GptLiveFunctionToolSample.weatherTool()); @@ -1003,8 +1025,7 @@ public void prematureClosureDoesNotReportSuccessfulConversation(boolean async) { StepVerifier .create(Mono.usingWhen( builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), - session -> GptLiveTextConversationAsyncSample.runConversation(session, - Collections.singletonList("one")), + session -> GptLiveSampleUtils.runConversation(session, Collections.singletonList("one")), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync)) .expectError(IllegalStateException.class) .verify(TIMEOUT); @@ -1012,7 +1033,7 @@ public void prematureClosureDoesNotReportSuccessfulConversation(boolean async) { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { assertThrows(IllegalStateException.class, - () -> GptLiveTextConversationSample.runConversation(session, Collections.singletonList("one"))); + () -> GptLiveSampleUtils.runConversation(session, Collections.singletonList("one"))); } } } @@ -1024,11 +1045,11 @@ public void nativeAudioEncodingAndDecodingPreservePcmBytes() { assertEquals("session.input_audio.append", command.get("type")); assertArrayEquals(pcm, Base64.getDecoder().decode((String) command.get("audio"))); Map event = Collections.singletonMap("delta", Base64.getEncoder().encodeToString(pcm)); - assertArrayEquals(pcm, GptLiveAudioConversationAsyncSample.decodeAudio(event)); + assertArrayEquals(pcm, GptLiveSampleUtils.decodeAudio(event)); assertThrows(IllegalArgumentException.class, - () -> GptLiveAudioConversationAsyncSample.decodeAudio(Collections.singletonMap("delta", "%%%"))); + () -> GptLiveSampleUtils.decodeAudio(Collections.singletonMap("delta", "%%%"))); assertThrows(IllegalArgumentException.class, - () -> GptLiveAudioConversationAsyncSample.decodeAudio(Collections.singletonMap("delta", "AA=="))); + () -> GptLiveSampleUtils.decodeAudio(Collections.singletonMap("delta", "AA=="))); } private static Stream audioSessionCases() { @@ -1112,11 +1133,9 @@ public int read() { } }; try { - VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor processor - = new VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor(microphone, speaker, - pcm -> session.sendEvent(GptLiveSampleUtils.audio(pcm.toBytes()))); - Mono conversation - = GptLiveAudioConversationAsyncSample.runConversation(session, processor, enterAfterAudio); + GptLiveSampleUtils.AudioProcessor processor = new GptLiveSampleUtils.AudioProcessor(microphone, speaker, + pcm -> session.sendEvent(GptLiveSampleUtils.audio(pcm.toBytes()))); + Mono conversation = GptLiveSampleUtils.runConversation(session, processor, enterAfterAudio); if (moderated) { IllegalStateException error = assertThrows(IllegalStateException.class, () -> conversation.block(TIMEOUT)); From 1d4960bfb248bed5a6fe8ec17aaa7fb3321daf50 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Sun, 25 Oct 2026 13:06:18 +0800 Subject: [PATCH 05/20] Add typed GPT Live events and extract shared WebSocket transports --- sdk/ai/azure-ai-agents/CHANGELOG.md | 6 + sdk/ai/azure-ai-agents/README.md | 35 +- .../azure/ai/agents/AgentsClientBuilder.java | 4 +- ...etaVoiceAgentLiveWebSocketAsyncClient.java | 11 +- .../BetaVoiceAgentLiveWebSocketClient.java | 11 +- ...eAgentLiveWebSocketSessionAsyncClient.java | 37 +- ...aVoiceAgentLiveWebSocketSessionClient.java | 63 ++- ...VoiceAgentWebSocketSessionAsyncClient.java | 389 +------------- .../BetaVoiceAgentWebSocketSessionClient.java | 441 +-------------- ...oiceAgentWebSocketClientConfiguration.java | 24 - .../realtime/VoiceAgentWebSocketUtils.java | 65 ++- .../WebSocketSessionAsyncTransport.java | 403 ++++++++++++++ .../realtime/WebSocketSessionTransport.java | 500 ++++++++++++++++++ ...VoiceAgentWebSocketSessionClientTests.java | 5 +- .../voice/VoiceAgentLiveWebSocketTests.java | 155 ++++++ 15 files changed, 1264 insertions(+), 885 deletions(-) create mode 100644 sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java create mode 100644 sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java diff --git a/sdk/ai/azure-ai-agents/CHANGELOG.md b/sdk/ai/azure-ai-agents/CHANGELOG.md index ff8b1d364b558..65afc1092917b 100644 --- a/sdk/ai/azure-ai-agents/CHANGELOG.md +++ b/sdk/ai/azure-ai-agents/CHANGELOG.md @@ -22,6 +22,10 @@ `AgentsClientBuilder.beta().buildBetaVoiceAgentLiveWebSocketClient()` and `buildBetaVoiceAgentLiveWebSocketAsyncClient()`. Native Live events are sent and received as lossless JSON using `BinaryData`, separately from the Realtime protocol clients. +- Added typed GPT Live sends through `sendEvent(com.openai.models.live.ClientEvent)` and typed receives through + `receiveTypedEvents()`, retaining the raw JSON APIs. The synchronous client also supports per-event receive timeouts. + Live transports use native OpenAI `ServerEvent` models while retaining original JSON for raw receivers. + Live and Realtime clients select their own routes and headers without a shared protocol-mode flag. - Added synchronous/asynchronous persisted GPT Live conversation history and audio-download examples with offline tests. - Added opt-in persistence to the GPT Live text and microphone/speaker samples using `FOUNDRY_GPT_LIVE_STORE` (default `false`), with explicit startup output when the service does not provide a persisted conversation ID. @@ -155,6 +159,8 @@ ### Other Changes +- Extracted shared protocol-neutral WebSocket transports for Live and Realtime, keeping independent blocking + and asynchronous implementations and protocol-specific event decoding in the respective clients. - Regenerated from the updated Foundry TypeSpec specification. - Updated `SessionConfiguration` documentation to reflect the service-supported maximum idle timeout of 14,400 seconds (four hours); the default remains 900 seconds. diff --git a/sdk/ai/azure-ai-agents/README.md b/sdk/ai/azure-ai-agents/README.md index 86afd0fe9a2d8..5f0ab82b32af8 100644 --- a/sdk/ai/azure-ai-agents/README.md +++ b/sdk/ai/azure-ai-agents/README.md @@ -1119,15 +1119,37 @@ send the `Foundry-Features: VoiceAgents=V1Preview` header, and **do not** negoti subprotocol. `allowPreview(true)` is not required. Foundry sends the initial native `session.start`. Wait for `session.started` before sending application events; -do not send `session.start` yourself. Live events are exposed as complete `BinaryData` JSON objects, not -`RealtimeClientEvent` or `RealtimeServerEvent` models. This preserves native events, unknown properties, -nested `response.event` backend events, and the Foundry `conversation_id` on `session.started`. +do not send `session.start` yourself. Use `sendEvent(ClientEvent)` and `receiveTypedEvents()` with the +`com.openai.models.live` event models from the SDK's existing `openai-java` dependency. These are distinct +from Realtime event models. The synchronous client also offers `receiveTypedEvents(Duration)` for bounded waits. +The existing `sendEvent(BinaryData)` and `receiveEvents()` APIs expose complete JSON objects when the original +wire representation is needed, including unknown properties, nested `response.event` backend events, and the +Foundry `conversation_id` on `session.started`. OpenAI typed models retain extra properties and unknown event +variants, but typed serialization need not preserve the original JSON formatting. The existing samples named `VoiceAgentLive*ConversationSample` above demonstrate live connections using the **Realtime** protocol; use the separate GPT Live clients for `VoiceAgentLiveDefinition` agents. -Use `sendEvent(BinaryData)` with native Live JSON commands valid for the configured delegation mode. +For example, with a connected synchronous Live session configured for Responses delegation: + +```java +import com.openai.models.live.ClientEvent; +import com.openai.models.live.ResponseCreateEvent; +import com.openai.models.live.ServerEvent; + +for (ServerEvent event : session.receiveTypedEvents()) { + if (event.isSessionStarted()) { + session.sendEvent(ClientEvent.ofResponseCreate(ResponseCreateEvent.builder().build())); + } else if (event.isSessionOutputTranscriptDelta()) { + System.out.print(event.asSessionOutputTranscriptDelta().delta()); + } else if (event.isSessionClosed() || event.isErrorEvent()) { + break; + } +} +``` + +Send only native Live commands valid for the configured delegation mode, whether typed or raw. In particular, `response.item.create` and `response.create` are Responses-delegation-only commands. -Service `error` events arrive as JSON events for the application to inspect; transport and malformed JSON +Service `error` events arrive as events for the application to inspect; transport and malformed JSON errors fail the receive stream unless a malformed-event callback was explicitly configured. Configure timeouts, bounded receive capacity, overflow handling, message size, persistence, agent version, @@ -1138,7 +1160,8 @@ options are not exposed. Options are copied when opening a session. Both transports retain the realtime clients' credential, proxy, custom-header and builder validation behavior. Custom HTTP clients, pipelines, policies, and retry settings are unsupported and fail at build time rather -than being ignored. Each session permits one receiver. The default receive buffer holds 256 events; overflow +than being ignored. Each session permits one receiver, shared by typed and raw receive APIs. +The default receive buffer holds 256 events; overflow terminates the session unless a drop strategy explicitly permits event loss. #### Runnable GPT Live scenarios diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java index 0062221ae3060..4a932609eabc8 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java @@ -784,7 +784,7 @@ public BetaVoiceAgentWebSocketClient buildBetaVoiceAgentWebSocketClient() { */ @Beta public BetaVoiceAgentLiveWebSocketAsyncClient buildBetaVoiceAgentLiveWebSocketAsyncClient() { - return new BetaVoiceAgentLiveWebSocketAsyncClient(createVoiceAgentWebSocketConfiguration().forLive()); + return new BetaVoiceAgentLiveWebSocketAsyncClient(createVoiceAgentWebSocketConfiguration()); } /** @@ -798,7 +798,7 @@ public BetaVoiceAgentLiveWebSocketAsyncClient buildBetaVoiceAgentLiveWebSocketAs */ @Beta public BetaVoiceAgentLiveWebSocketClient buildBetaVoiceAgentLiveWebSocketClient() { - return new BetaVoiceAgentLiveWebSocketClient(createVoiceAgentWebSocketConfiguration().forLive()); + return new BetaVoiceAgentLiveWebSocketClient(createVoiceAgentWebSocketConfiguration()); } /** diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketAsyncClient.java index 192aef2a4f5bc..386e2fcb0b94a 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketAsyncClient.java @@ -5,12 +5,16 @@ import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.realtime.WebSocketSessionAsyncTransport; +import com.azure.ai.agents.implementation.OpenAIJsonHelper; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.annotation.ServiceClient; +import com.openai.models.live.ServerEvent; import java.util.Objects; import reactor.core.publisher.Mono; +import reactor.netty.http.client.HttpClient; /** * An asynchronous client for opening native GPT Live sessions with persisted Foundry Live voice agents. @@ -46,8 +50,11 @@ public Mono openWebSocketSession( Objects.requireNonNull(agentName, "'agentName' cannot be null."); VoiceAgentWebSocketConnectionOptions snapshot = VoiceAgentWebSocketUtils.liveOptions(options); return Mono.defer(() -> { - BetaVoiceAgentWebSocketSessionAsyncClient transport - = new BetaVoiceAgentWebSocketSessionAsyncClient(configuration, agentName, snapshot); + WebSocketSessionAsyncTransport transport = new WebSocketSessionAsyncTransport<>(configuration, + VoiceAgentWebSocketUtils.buildLiveWebSocketUri(configuration, agentName, snapshot), snapshot, + HttpClient.create(), null, token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, token), + json -> OpenAIJsonHelper.fromBinaryData(VoiceAgentWebSocketUtils.deserializeRawEvent(json), + ServerEvent.class)); return transport.connect().thenReturn(new BetaVoiceAgentLiveWebSocketSessionAsyncClient(transport)); }); } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketClient.java index 85afd6cc86b5c..4e497d4517914 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketClient.java @@ -5,8 +5,12 @@ import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport; +import com.azure.ai.agents.implementation.OpenAIJsonHelper; +import com.openai.models.live.ServerEvent; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.annotation.ServiceClient; import com.azure.core.util.logging.ClientLogger; import java.util.Objects; @@ -48,7 +52,10 @@ public BetaVoiceAgentLiveWebSocketSessionClient openWebSocketSession(String agen if (agentName.isEmpty()) { throw LOGGER.logExceptionAsError(new IllegalArgumentException("'agentName' cannot be empty.")); } - return new BetaVoiceAgentLiveWebSocketSessionClient(BetaVoiceAgentWebSocketSessionClient.connect(configuration, - agentName, VoiceAgentWebSocketUtils.liveOptions(options))); + VoiceAgentWebSocketConnectionOptions snapshot = VoiceAgentWebSocketUtils.liveOptions(options); + return new BetaVoiceAgentLiveWebSocketSessionClient(WebSocketSessionTransport.connect(configuration, + VoiceAgentWebSocketUtils.buildLiveWebSocketUri(configuration, agentName, snapshot), snapshot, null, + token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, token), json -> OpenAIJsonHelper + .fromBinaryData(VoiceAgentWebSocketUtils.deserializeRawEvent(json), ServerEvent.class))); } } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java index de0d598015bef..51bd3895dcba6 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java @@ -3,25 +3,29 @@ package com.azure.ai.agents; -import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.OpenAIJsonHelper; +import com.azure.ai.agents.implementation.realtime.WebSocketSessionAsyncTransport; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.core.util.AsyncCloseable; import com.azure.core.util.BinaryData; +import com.openai.models.live.ClientEvent; +import com.openai.models.live.ServerEvent; +import java.util.Objects; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; /** * An asynchronous bidirectional native GPT Live session. *

    - * Live events are complete JSON objects, not {@code RealtimeServerEvent} or {@code RealtimeClientEvent} models. - * Unknown fields, nested {@code response.event} payloads and Foundry {@code conversation_id} values are preserved. - * Sessions support one event subscriber and must be closed when no longer needed. + * Live events can be sent and received using {@link ClientEvent} and {@link ServerEvent}, or as complete JSON objects. + * Raw JSON preserves unknown fields, nested {@code response.event} payloads and Foundry {@code conversation_id} values. + * Typed and raw receivers share one event subscriber. Close sessions when they are no longer needed. */ @Beta(warningText = "This class is in preview and may change in future releases.") public final class BetaVoiceAgentLiveWebSocketSessionAsyncClient implements AsyncCloseable, AutoCloseable { - private final BetaVoiceAgentWebSocketSessionAsyncClient transport; + private final WebSocketSessionAsyncTransport transport; - BetaVoiceAgentLiveWebSocketSessionAsyncClient(BetaVoiceAgentWebSocketSessionAsyncClient transport) { + BetaVoiceAgentLiveWebSocketSessionAsyncClient(WebSocketSessionAsyncTransport transport) { this.transport = transport; } @@ -63,7 +67,26 @@ public String getCloseReason() { * @return the single-subscriber stream of complete JSON events. */ public Flux receiveEvents() { - return transport.receiveEvents().map(VoiceAgentWebSocketUtils::liveEventPayload); + return transport.receiveRawEvents(); + } + + /** + * Receives typed native Live server events. Typed and raw receivers share the single-subscriber limit. + * Use {@link #receiveEvents()} when the original JSON representation is needed. + * @return the typed server event stream. + */ + public Flux receiveTypedEvents() { + return transport.receiveEvents(); + } + + /** + * Sends a typed native Live command. Foundry supplies {@code session.start}; clients must not send it. + * @param event the command to send. + * @return completion after the frame is written. + */ + public Mono sendEvent(ClientEvent event) { + Objects.requireNonNull(event, "'event' cannot be null."); + return Mono.defer(() -> sendEvent(OpenAIJsonHelper.toBinaryData(event))); } /** diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java index 10d07cca45371..ba8e905f7d3f8 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java @@ -3,26 +3,28 @@ package com.azure.ai.agents; -import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.OpenAIJsonHelper; +import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport; import com.azure.ai.agents.implementation.utils.Beta; -import com.azure.ai.agents.models.RealtimeServerEvent; import com.azure.core.util.BinaryData; import com.azure.core.util.IterableStream; +import com.openai.models.live.ClientEvent; +import com.openai.models.live.ServerEvent; import java.time.Duration; -import java.util.Iterator; +import java.util.Objects; /** * A synchronous bidirectional native GPT Live session. *

    - * Events are complete JSON objects, not Realtime protocol models. Unknown fields, nested {@code response.event} - * payloads and Foundry {@code conversation_id} values are preserved. Only one event iterator is supported. - * Close sessions when they are no longer needed. + * Events can be sent and received using {@link ClientEvent} and {@link ServerEvent}, or as complete JSON objects. + * Raw JSON preserves unknown fields, nested {@code response.event} payloads and Foundry {@code conversation_id} values. + * Typed and raw receivers share one event iterator. Close sessions when they are no longer needed. */ @Beta(warningText = "This class is in preview and may change in future releases.") public final class BetaVoiceAgentLiveWebSocketSessionClient implements AutoCloseable { - private final BetaVoiceAgentWebSocketSessionClient transport; + private final WebSocketSessionTransport transport; - BetaVoiceAgentLiveWebSocketSessionClient(BetaVoiceAgentWebSocketSessionClient transport) { + BetaVoiceAgentLiveWebSocketSessionClient(WebSocketSessionTransport transport) { this.transport = transport; } @@ -68,6 +70,36 @@ public IterableStream receiveEvents() { return receiveEvents(null); } + /** + * Receives typed native Live server events. Typed and raw receivers share the single-consumer limit. + * @return the typed server event stream. + */ + public IterableStream receiveTypedEvents() { + return receiveTypedEvents(null); + } + + /** + * Receives typed native Live events with a timeout per wait. Timeouts leave the session open. + * Use {@link #receiveEvents(Duration)} when the original JSON representation is needed. + * @param timeout a positive timeout, or null to wait indefinitely. + * @return the single-consumer typed event stream. + * @throws IllegalArgumentException if the timeout is zero or negative. + * @throws IllegalStateException if the receive stream was already claimed, or a wait times out. + */ + public IterableStream receiveTypedEvents(Duration timeout) { + return transport.receiveEvents(timeout); + } + + /** + * Sends a typed native Live command. Foundry supplies {@code session.start}; clients must not send it. + * @param event the command to send. + * @throws IllegalStateException if the session cannot accept the event. + */ + public void sendEvent(ClientEvent event) { + Objects.requireNonNull(event, "'event' cannot be null."); + sendEvent(OpenAIJsonHelper.toBinaryData(event)); + } + /** * Receives native Live events with a timeout per wait. Timeouts leave the session open so iteration can resume. * @param timeout a positive timeout, or null to wait indefinitely. @@ -76,20 +108,7 @@ public IterableStream receiveEvents() { * @throws IllegalStateException if the receive stream was already claimed, or a wait times out. */ public IterableStream receiveEvents(Duration timeout) { - IterableStream events = transport.receiveEvents(timeout); - return IterableStream.of(() -> new Iterator() { - private final Iterator iterator = events.iterator(); - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public BinaryData next() { - return VoiceAgentWebSocketUtils.liveEventPayload(iterator.next()); - } - }); + return transport.receiveRawEvents(timeout); } /** diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java index f8418480f2d05..fc3e7d04ce012 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java @@ -4,9 +4,8 @@ package com.azure.ai.agents; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; -import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketHandshakeHandler; -import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketHttpResponse; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.realtime.WebSocketSessionAsyncTransport; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.RealtimeClientEvent; import com.azure.ai.agents.models.RealtimeConversationItemCreateEvent; @@ -21,46 +20,16 @@ import com.azure.ai.agents.models.RealtimeServerEvent; import com.azure.ai.agents.models.VoiceAgentResponseCreateOptions; import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; -import com.azure.core.credential.AccessToken; -import com.azure.core.credential.TokenRequestContext; -import com.azure.core.exception.ClientAuthenticationException; -import com.azure.core.exception.HttpResponseException; -import com.azure.core.exception.ResourceModifiedException; -import com.azure.core.exception.ResourceNotFoundException; -import com.azure.core.http.HttpHeader; -import com.azure.core.http.ProxyOptions; import com.azure.core.util.AsyncCloseable; import com.azure.core.util.BinaryData; import com.openai.models.realtime.RealtimeConversationItemUserMessage.Content; -import com.azure.core.util.logging.ClientLogger; -import io.netty.channel.Channel; -import io.netty.channel.ChannelOption; -import io.netty.handler.codec.http.websocketx.BinaryWebSocketFrame; -import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; -import io.netty.handler.codec.http.websocketx.WebSocketClientHandshakeException; -import io.netty.handler.codec.http.websocketx.WebSocketFrame; import java.io.IOException; -import java.net.URI; -import java.time.Duration; import java.util.Base64; import java.util.Collections; import java.util.Objects; -import java.util.Queue; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.CancellationException; -import java.util.concurrent.Semaphore; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import reactor.core.Disposable; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import reactor.core.publisher.Sinks; -import reactor.netty.Connection; import reactor.netty.http.client.HttpClient; -import reactor.netty.http.client.WebsocketClientSpec; -import reactor.netty.http.websocket.WebsocketInbound; -import reactor.netty.http.websocket.WebsocketOutbound; -import reactor.netty.transport.ProxyProvider; /** * An asynchronous bidirectional realtime session connected to a Foundry voice agent. @@ -70,28 +39,7 @@ */ @Beta(warningText = "This class is in preview and may change in future releases.") public final class BetaVoiceAgentWebSocketSessionAsyncClient implements AsyncCloseable, AutoCloseable { - private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentWebSocketSessionAsyncClient.class); - private static final int MAX_OUTSTANDING_SENDS = 256; - - private final VoiceAgentWebSocketClientConfiguration configuration; - private final VoiceAgentWebSocketConnectionOptions options; - private final HttpClient httpClient; - private final URI websocketUri; - private final AtomicReference state = new AtomicReference<>(State.NEW); - private final AtomicReference channel = new AtomicReference<>(); - private final AtomicReference outbound = new AtomicReference<>(); - private final AtomicReference connectionOperation = new AtomicReference<>(); - private final AtomicReference lifecycle = new AtomicReference<>(); - private final AtomicBoolean receiveClaimed = new AtomicBoolean(); - private final Semaphore sendPermits = new Semaphore(MAX_OUTSTANDING_SENDS); - private final Queue eventQueue; - private final Sinks.Many events; - private final Sinks.One ready = Sinks.one(); - private final Sinks.One closeSignal = Sinks.one(); - private final AtomicReference> closeOperation = new AtomicReference<>(); - - private volatile Integer closeCode; - private volatile String closeReason; + private final WebSocketSessionAsyncTransport transport; BetaVoiceAgentWebSocketSessionAsyncClient(VoiceAgentWebSocketClientConfiguration configuration, String agentName, VoiceAgentWebSocketConnectionOptions options) { @@ -100,34 +48,17 @@ public final class BetaVoiceAgentWebSocketSessionAsyncClient implements AsyncClo BetaVoiceAgentWebSocketSessionAsyncClient(VoiceAgentWebSocketClientConfiguration configuration, String agentName, VoiceAgentWebSocketConnectionOptions options, HttpClient httpClient) { - this.configuration = Objects.requireNonNull(configuration, "'configuration' cannot be null."); - Objects.requireNonNull(agentName, "'agentName' cannot be null."); - if (agentName.isEmpty()) { - throw new IllegalArgumentException("'agentName' cannot be empty."); - } - this.options = options == null ? new VoiceAgentWebSocketConnectionOptions() : options; - this.eventQueue = new ArrayBlockingQueue<>(this.options.getReceiveBufferCapacity()); - this.events = Sinks.many().unicast().onBackpressureBuffer(eventQueue); - this.httpClient = Objects.requireNonNull(httpClient, "'httpClient' cannot be null."); - this.websocketUri = VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, this.options); + VoiceAgentWebSocketConnectionOptions sessionOptions + = options == null ? new VoiceAgentWebSocketConnectionOptions() : options; + transport = new WebSocketSessionAsyncTransport<>(configuration, + VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, sessionOptions), sessionOptions, + httpClient, VoiceAgentWebSocketUtils.SUBPROTOCOL, + token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, sessionOptions, token), + VoiceAgentWebSocketUtils::deserializeEvent); } Mono connect() { - return Mono.defer(() -> { - if (!state.compareAndSet(State.NEW, State.CONNECTING)) { - return Mono.error(new IllegalStateException("The voice-agent session has already been started.")); - } - - TokenRequestContext tokenContext = VoiceAgentWebSocketUtils.createTokenRequestContext(); - return configuration.getCredential().getToken(tokenContext).map(AccessToken::getToken).flatMap(token -> { - Disposable connection = openWebSocket(token).subscribe(unused -> { - }, this::terminateWithError, this::terminateNormally); - connectionOperation.set(connection); - return ready.asMono().timeout(options.getHandshakeTimeout()); - }) - .doOnError(this::terminateWithError) - .doOnCancel(() -> terminateWithError(new CancellationException("WebSocket connection cancelled."))); - }); + return transport.connect(); } /** @@ -136,7 +67,7 @@ Mono connect() { * @return the WebSocket endpoint. */ public String getEndpoint() { - return websocketUri.toString(); + return transport.getEndpoint(); } /** @@ -145,8 +76,7 @@ public String getEndpoint() { * @return {@code true} when the session is open. */ public boolean isOpen() { - Channel current = channel.get(); - return state.get() == State.OPEN && current != null && current.isActive(); + return transport.isOpen(); } /** @@ -155,7 +85,7 @@ public boolean isOpen() { * @return the close code, or {@code null} when no close frame has been received. */ public Integer getCloseCode() { - return closeCode; + return transport.getCloseCode(); } /** @@ -164,7 +94,7 @@ public Integer getCloseCode() { * @return the close reason, or {@code null} when no close frame has been received. */ public String getCloseReason() { - return closeReason; + return transport.getCloseReason(); } /** @@ -173,9 +103,7 @@ public String getCloseReason() { * @return the server event stream. */ public Flux receiveEvents() { - return Flux.defer(() -> receiveClaimed.compareAndSet(false, true) - ? events.asFlux() - : Flux.error(new IllegalStateException("Only one receiveEvents subscriber is supported per session."))); + return transport.receiveEvents(); } /** @@ -187,28 +115,12 @@ public Flux receiveEvents() { public Mono sendEvent(RealtimeClientEvent event) { Objects.requireNonNull(event, "'event' cannot be null."); return Mono.defer(() -> { - Channel current = requireOpenChannel(); - if (!sendPermits.tryAcquire()) { - return Mono.error(new IllegalStateException("Too many voice-agent WebSocket sends are outstanding.")); - } - - final String json; try { - json = serialize(event); + return transport.sendEvent(BinaryData.fromString(event.toJsonString())); } catch (IOException error) { - sendPermits.release(); return Mono .error(new IllegalArgumentException("Failed to serialize the realtime client event.", error)); } - - return Mono.create(sink -> current.writeAndFlush(new TextWebSocketFrame(json)).addListener(result -> { - sendPermits.release(); - if (result.isSuccess()) { - sink.success(); - } else { - sink.error(result.cause()); - } - })); }); } @@ -218,22 +130,7 @@ public Mono sendEvent(RealtimeClientEvent event) { * @return completion after the frame is written, or an error for invalid JSON or a closed session. */ public Mono sendEvent(BinaryData event) { - Objects.requireNonNull(event, "'event' cannot be null."); - return Mono.defer(() -> { - String json = VoiceAgentWebSocketUtils.validateEvent(event); - Channel current = requireOpenChannel(); - if (!sendPermits.tryAcquire()) { - return Mono.error(new IllegalStateException("Too many voice-agent WebSocket sends are outstanding.")); - } - return Mono.create(sink -> current.writeAndFlush(new TextWebSocketFrame(json)).addListener(result -> { - sendPermits.release(); - if (result.isSuccess()) { - sink.success(); - } else { - sink.error(result.cause()); - } - })); - }); + return transport.sendEvent(event); } /** @@ -360,7 +257,7 @@ public Mono sendFunctionCallOutput(String callId, String output) { */ @Override public Mono closeAsync() { - return closeAsync(1000, ""); + return transport.closeAsync(); } /** @@ -371,40 +268,7 @@ public Mono closeAsync() { * @return a completion signal, or an error if the code or reason is invalid. */ public Mono closeAsync(int code, String reason) { - try { - VoiceAgentWebSocketUtils.validateClose(code, reason); - } catch (IllegalArgumentException exception) { - return Mono.error(exception); - } - Mono existing = closeOperation.get(); - if (existing != null) { - return existing; - } - - Mono created = Mono.defer(() -> { - State current = state.get(); - if (current == State.CLOSED || current == State.NEW) { - state.set(State.CLOSED); - events.tryEmitComplete(); - return Mono.empty(); - } - state.set(State.CLOSING); - WebsocketOutbound currentOutbound = outbound.get(); - Channel currentChannel = channel.get(); - Mono graceful = currentOutbound == null ? Mono.empty() : currentOutbound.sendClose(code, reason); - Mono disposed = currentChannel == null ? Mono.empty() : Connection.from(currentChannel).onDispose(); - return graceful.then(disposed).timeout(options.getCloseTimeout()).onErrorResume(error -> { - if (currentChannel != null) { - currentChannel.close(); - } - return Mono.empty(); - }).doFinally(signal -> terminateNormally()); - }).cache(); - - if (closeOperation.compareAndSet(null, created)) { - return created; - } - return closeOperation.get(); + return transport.closeAsync(code, reason); } /** @@ -412,219 +276,6 @@ public Mono closeAsync(int code, String reason) { */ @Override public void close() { - closeAsync().block(options.getCloseTimeout().plusSeconds(1)); - } - - private Mono openWebSocket(String token) { - HttpClient client = configureProxy(httpClient).followRedirect(false) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, toConnectTimeoutMillis(options.getHandshakeTimeout())) - .doOnConnected(connection -> connection.addHandlerLast("voiceAgentHandshakeResponseObserver", - new VoiceAgentWebSocketHandshakeHandler(this::terminateWithError))) - .headers(headers -> { - for (HttpHeader header : VoiceAgentWebSocketUtils.buildHeaders(configuration, options, token)) { - headers.set(header.getName(), header.getValue()); - } - }); - WebsocketClientSpec.Builder specBuilder - = WebsocketClientSpec.builder().maxFramePayloadLength(options.getMaxMessageSize()); - if (!configuration.isLive()) { - specBuilder.protocols(VoiceAgentWebSocketUtils.SUBPROTOCOL); - } - WebsocketClientSpec spec = specBuilder.build(); - - return client.websocket(spec).uri(websocketUri.toString()).connect().flatMap(connection -> { - if (!(connection instanceof WebsocketInbound) || !(connection instanceof WebsocketOutbound)) { - return Mono.error(new IllegalStateException("The WebSocket transport returned an invalid connection.")); - } - return handleConnection((WebsocketInbound) connection, (WebsocketOutbound) connection); - }); - } - - private static int toConnectTimeoutMillis(Duration timeout) { - if (timeout.compareTo(Duration.ofMillis(Integer.MAX_VALUE)) >= 0) { - return Integer.MAX_VALUE; - } - return Math.toIntExact(Math.max(1L, timeout.toMillis())); - } - - private Mono handleConnection(WebsocketInbound inbound, WebsocketOutbound outbound) { - this.outbound.set(outbound); - inbound.withConnection(connection -> channel.set(connection.channel())); - state.set(State.OPEN); - ready.tryEmitEmpty(); - - inbound.receiveCloseStatus().subscribe(status -> { - closeCode = status.code(); - closeReason = status.reasonText(); - }, error -> LOGGER.atVerbose().addKeyValue("error", error.getMessage()).log("Close status unavailable.")); - - Disposable receive = inbound.aggregateFrames(options.getMaxMessageSize()) - .receiveFrames() - .subscribe(this::handleFrame, this::terminateWithError, this::terminateNormally); - lifecycle.set(receive); - return closeSignal.asMono(); - } - - private void handleFrame(WebSocketFrame frame) { - if (frame instanceof TextWebSocketFrame || frame instanceof BinaryWebSocketFrame) { - try { - byte[] bytes = new byte[frame.content().readableBytes()]; - frame.content().getBytes(frame.content().readerIndex(), bytes); - RealtimeServerEvent event = VoiceAgentWebSocketUtils - .deserializeEvent(VoiceAgentWebSocketUtils.decodeEvent(bytes), configuration.isLive()); - Sinks.EmitResult result = events.tryEmitNext(event); - if (result == Sinks.EmitResult.FAIL_OVERFLOW || result == Sinks.EmitResult.FAIL_ZERO_SUBSCRIBER) { - switch (options.getOverflowStrategy()) { - case DROP_LATEST: - return; - - case DROP_OLDEST: - eventQueue.poll(); - result = events.tryEmitNext(event); - break; - - default: - break; - } - } - if (result.isFailure()) { - terminateWithError(new IllegalStateException("Voice-agent event emission failed: " + result)); - } - } catch (IOException | RuntimeException error) { - Throwable failure = error; - if (options.getMalformedEventHandler() != null) { - try { - options.getMalformedEventHandler().accept(error); - return; - } catch (RuntimeException callbackError) { - failure = callbackError; - } - } - closeWithProtocolError(1007, "Invalid JSON event", failure); - } - } - } - - private void closeWithProtocolError(int code, String reason, Throwable error) { - WebsocketOutbound current = outbound.get(); - terminateWithError(error); - if (current != null) { - current.sendClose(code, reason).subscribe(unused -> { - }, ignored -> { - }); - } - } - - private void terminateWithError(Throwable error) { - Throwable mappedError = mapHandshakeError(error); - State previous = state.getAndSet(State.CLOSED); - if (previous == State.CLOSED) { - return; - } - ready.tryEmitError(mappedError); - events.tryEmitError(mappedError); - closeSignal.tryEmitError(mappedError); - disposeReceive(); - } - - private Throwable mapHandshakeError(Throwable error) { - Throwable current = error; - while (current != null && !(current instanceof WebSocketClientHandshakeException)) { - current = current.getCause(); - } - if (current == null) { - return error; - } - WebSocketClientHandshakeException handshakeError = (WebSocketClientHandshakeException) current; - if (handshakeError.response() == null) { - return error; - } - VoiceAgentWebSocketHttpResponse response - = new VoiceAgentWebSocketHttpResponse(websocketUri, handshakeError.response()); - String message = "Voice-agent WebSocket handshake failed with status " + response.getStatusCode() + "."; - switch (response.getStatusCode()) { - case 401: - return new ClientAuthenticationException(message, response, error); - - case 404: - return new ResourceNotFoundException(message, response, error); - - case 409: - return new ResourceModifiedException(message, response, error); - - default: - return new HttpResponseException(message, response, error); - } - } - - private void terminateNormally() { - State previous = state.getAndSet(State.CLOSED); - if (previous == State.CLOSED) { - return; - } - if (previous == State.CONNECTING) { - ready.tryEmitError(new IllegalStateException("The WebSocket closed before the handshake completed.")); - } - events.tryEmitComplete(); - closeSignal.tryEmitEmpty(); - disposeReceive(); - } - - private void disposeReceive() { - Disposable connection = connectionOperation.getAndSet(null); - if (connection != null && !connection.isDisposed()) { - connection.dispose(); - } - Disposable receive = lifecycle.getAndSet(null); - if (receive != null && !receive.isDisposed()) { - receive.dispose(); - } - } - - private Channel requireOpenChannel() { - Channel current = channel.get(); - if (state.get() != State.OPEN || current == null || !current.isActive()) { - throw LOGGER - .logExceptionAsError(new IllegalStateException("The voice-agent WebSocket session is not open.")); - } - return current; - } - - private HttpClient configureProxy(HttpClient client) { - ProxyOptions proxy = configuration.getProxyOptions(); - if (proxy == null) { - return client; - } - return client.proxy(typeSpec -> { - ProxyProvider.Proxy proxyType; - switch (proxy.getType()) { - case SOCKS4: - proxyType = ProxyProvider.Proxy.SOCKS4; - break; - - case SOCKS5: - proxyType = ProxyProvider.Proxy.SOCKS5; - break; - - default: - proxyType = ProxyProvider.Proxy.HTTP; - break; - } - ProxyProvider.Builder builder = typeSpec.type(proxyType).socketAddress(proxy.getAddress()); - if (proxy.getUsername() != null) { - builder.username(proxy.getUsername()).password(ignored -> proxy.getPassword()); - } - if (proxy.getNonProxyHosts() != null) { - builder.nonProxyHosts(proxy.getNonProxyHosts()); - } - }); - } - - private static String serialize(RealtimeClientEvent event) throws IOException { - return event.toJsonString(); - } - - private enum State { - NEW, CONNECTING, OPEN, CLOSING, CLOSED + transport.close(); } } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java index b4e1cadc67c36..118cf90909cbe 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java @@ -4,8 +4,8 @@ package com.azure.ai.agents; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; -import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketHttpResponse; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.RealtimeClientEvent; import com.azure.ai.agents.models.RealtimeConversationItemCreateEvent; @@ -20,45 +20,15 @@ import com.azure.ai.agents.models.RealtimeServerEvent; import com.azure.ai.agents.models.VoiceAgentResponseCreateOptions; import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; -import com.azure.core.exception.ClientAuthenticationException; -import com.azure.core.exception.HttpResponseException; -import com.azure.core.exception.ResourceModifiedException; -import com.azure.core.exception.ResourceNotFoundException; -import com.azure.core.http.HttpHeader; -import com.azure.core.http.ProxyOptions; import com.azure.core.util.BinaryData; import com.openai.models.realtime.RealtimeConversationItemUserMessage.Content; import com.azure.core.util.IterableStream; import com.azure.core.util.logging.ClientLogger; import java.io.IOException; -import java.net.Proxy; -import java.net.ProxySelector; -import java.net.SocketAddress; -import java.net.URI; -import java.nio.charset.CharacterCodingException; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Base64; import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; import java.util.Objects; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Pattern; -import okhttp3.Credentials; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.WebSocket; -import okhttp3.WebSocketListener; -import okio.ByteString; /** * A synchronous bidirectional realtime session connected to a Foundry voice agent. @@ -66,58 +36,19 @@ @Beta(warningText = "This class is in preview and may change in future releases.") public final class BetaVoiceAgentWebSocketSessionClient implements AutoCloseable { private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentWebSocketSessionClient.class); - private final URI websocketUri; - private final VoiceAgentWebSocketConnectionOptions options; - private final boolean live; - private final OkHttpClient httpClient; - private final BlockingQueue events; - private final int receiveBufferCapacity; - private final CountDownLatch handshakeCompleted = new CountDownLatch(1); - private final CountDownLatch closeCompleted = new CountDownLatch(1); - private final AtomicReference connectionError = new AtomicReference<>(); - private final AtomicBoolean handshakeSucceeded = new AtomicBoolean(); - private final AtomicBoolean receiveClaimed = new AtomicBoolean(); - private final AtomicBoolean open = new AtomicBoolean(); - private final AtomicBoolean closed = new AtomicBoolean(); - private final AtomicBoolean clientShutdown = new AtomicBoolean(); + private final WebSocketSessionTransport transport; - private volatile WebSocket webSocket; - private volatile Integer closeCode; - private volatile String closeReason; - - private BetaVoiceAgentWebSocketSessionClient(VoiceAgentWebSocketClientConfiguration configuration, String agentName, - VoiceAgentWebSocketConnectionOptions options) { - this.options = options; - this.live = configuration.isLive(); - this.receiveBufferCapacity = options.getReceiveBufferCapacity(); - this.events = new ArrayBlockingQueue<>(receiveBufferCapacity + 1); - this.websocketUri = VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, options); - String token = configuration.getCredential() - .getTokenSync(VoiceAgentWebSocketUtils.createTokenRequestContext()) - .getToken(); - this.httpClient = createHttpClient(configuration, options); - Request.Builder request = new Request.Builder().url(websocketUri.toString()); - if (!live) { - request.header("Sec-WebSocket-Protocol", VoiceAgentWebSocketUtils.SUBPROTOCOL); - } - for (HttpHeader header : VoiceAgentWebSocketUtils.buildHeaders(configuration, options, token)) { - request.header(header.getName(), header.getValue()); - } - this.webSocket = httpClient.newWebSocket(request.build(), new Listener()); + private BetaVoiceAgentWebSocketSessionClient(WebSocketSessionTransport transport) { + this.transport = transport; } static BetaVoiceAgentWebSocketSessionClient connect(VoiceAgentWebSocketClientConfiguration configuration, String agentName, VoiceAgentWebSocketConnectionOptions options) { - BetaVoiceAgentWebSocketSessionClient session - = new BetaVoiceAgentWebSocketSessionClient(configuration, agentName, options); - try { - session.awaitHandshake(); - return session; - } catch (RuntimeException error) { - session.webSocket.cancel(); - session.shutdownHttpClient(); - throw error; - } + return new BetaVoiceAgentWebSocketSessionClient(WebSocketSessionTransport.connect(configuration, + VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, options), options, + VoiceAgentWebSocketUtils.SUBPROTOCOL, + token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, options, token), + VoiceAgentWebSocketUtils::deserializeEvent)); } /** @@ -126,7 +57,7 @@ static BetaVoiceAgentWebSocketSessionClient connect(VoiceAgentWebSocketClientCon * @return the WebSocket endpoint. */ public String getEndpoint() { - return websocketUri.toString(); + return transport.getEndpoint(); } /** @@ -135,7 +66,7 @@ public String getEndpoint() { * @return {@code true} when the session is open. */ public boolean isOpen() { - return open.get() && !closed.get(); + return transport.isOpen(); } /** @@ -144,7 +75,7 @@ public boolean isOpen() { * @return the close code, or {@code null}. */ public Integer getCloseCode() { - return closeCode; + return transport.getCloseCode(); } /** @@ -153,7 +84,7 @@ public Integer getCloseCode() { * @return the close reason, or {@code null}. */ public String getCloseReason() { - return closeReason; + return transport.getCloseReason(); } /** @@ -174,21 +105,7 @@ public IterableStream receiveEvents() { * @throws IllegalStateException if another receiver exists or a wait times out (with a TimeoutException cause). */ public IterableStream receiveEvents(Duration timeout) { - if (timeout != null && (timeout.isZero() || timeout.isNegative())) { - throw LOGGER.logExceptionAsError(new IllegalArgumentException("Receive timeout must be positive.")); - } - if (!receiveClaimed.compareAndSet(false, true)) { - throw LOGGER.logExceptionAsError( - new IllegalStateException("Only one receiveEvents iterator is supported per session.")); - } - AtomicBoolean iteratorCreated = new AtomicBoolean(); - return IterableStream.of(() -> { - if (!iteratorCreated.compareAndSet(false, true)) { - throw LOGGER.logExceptionAsError( - new IllegalStateException("The receiveEvents stream may only be iterated once.")); - } - return new EventIterator(events, timeout); - }); + return transport.receiveEvents(timeout); } /** @@ -200,18 +117,12 @@ public IterableStream receiveEvents(Duration timeout) { */ public void sendEvent(RealtimeClientEvent event) { Objects.requireNonNull(event, "'event' cannot be null."); - ensureOpen(); - final String json; try { - json = event.toJsonString(); + transport.sendEvent(BinaryData.fromString(event.toJsonString())); } catch (IOException error) { throw LOGGER.logExceptionAsError( new IllegalArgumentException("Failed to serialize the realtime client event.", error)); } - if (!webSocket.send(json)) { - throw LOGGER.logExceptionAsError( - new IllegalStateException("The voice-agent WebSocket send queue is full or closed.")); - } } /** @@ -221,12 +132,7 @@ public void sendEvent(RealtimeClientEvent event) { * @throws IllegalStateException if the session cannot accept the event. */ public void sendEvent(BinaryData event) { - String json = VoiceAgentWebSocketUtils.validateEvent(event); - ensureOpen(); - if (!webSocket.send(json)) { - throw LOGGER.logExceptionAsError( - new IllegalStateException("The voice-agent WebSocket send queue is full or closed.")); - } + transport.sendEvent(event); } /** @@ -324,7 +230,7 @@ public void sendFunctionCallOutput(String callId, String output) { /** Closes the session. */ @Override public void close() { - close(1000, ""); + transport.close(); } /** @@ -334,317 +240,6 @@ public void close() { * @throws IllegalArgumentException if the code or reason is invalid. */ public void close(int code, String reason) { - VoiceAgentWebSocketUtils.validateClose(code, reason); - if (!closed.compareAndSet(false, true)) { - shutdownHttpClient(); - return; - } - open.set(false); - if (!webSocket.close(code, reason)) { - webSocket.cancel(); - closeCompleted.countDown(); - } - try { - if (!closeCompleted.await(options.getCloseTimeout().toMillis(), TimeUnit.MILLISECONDS)) { - webSocket.cancel(); - } - } catch (InterruptedException error) { - Thread.currentThread().interrupt(); - webSocket.cancel(); - } finally { - signal(EventSignal.complete()); - shutdownHttpClient(); - } - } - - private void awaitHandshake() { - try { - if (!handshakeCompleted.await(options.getHandshakeTimeout().toMillis(), TimeUnit.MILLISECONDS)) { - webSocket.cancel(); - throw LOGGER - .logExceptionAsError(new IllegalStateException("Voice-agent WebSocket handshake timed out.")); - } - } catch (InterruptedException error) { - Thread.currentThread().interrupt(); - webSocket.cancel(); - throw LOGGER.logExceptionAsError( - new IllegalStateException("Interrupted while opening the voice-agent WebSocket session.", error)); - } - if (handshakeSucceeded.get()) { - return; - } - Throwable error = connectionError.get(); - if (error instanceof RuntimeException) { - throw LOGGER.logExceptionAsError((RuntimeException) error); - } - if (error != null) { - throw LOGGER.logExceptionAsError( - new IllegalStateException("Failed to open the voice-agent WebSocket session.", error)); - } - } - - private void ensureOpen() { - if (!isOpen()) { - throw LOGGER - .logExceptionAsError(new IllegalStateException("The voice-agent WebSocket session is not open.")); - } - } - - private void fail(Throwable error, Response response) { - Throwable mapped; - try { - mapped = response == null ? error : mapHandshakeError(error, response); - } finally { - if (response != null) { - response.close(); - } - } - connectionError.compareAndSet(null, mapped); - open.set(false); - closed.set(true); - handshakeCompleted.countDown(); - signal(EventSignal.error(mapped)); - closeCompleted.countDown(); - shutdownHttpClient(); - } - - private Throwable mapHandshakeError(Throwable error, Response response) { - VoiceAgentWebSocketHttpResponse azureResponse = new VoiceAgentWebSocketHttpResponse(websocketUri, response); - String message = "Voice-agent WebSocket handshake failed with status " + response.code() + "."; - switch (response.code()) { - case 401: - return new ClientAuthenticationException(message, azureResponse, error); - - case 404: - return new ResourceNotFoundException(message, azureResponse, error); - - case 409: - return new ResourceModifiedException(message, azureResponse, error); - - default: - return new HttpResponseException(message, azureResponse, error); - } - } - - private synchronized void signal(EventSignal signal) { - if (signal.event != null && events.size() >= receiveBufferCapacity) { - switch (options.getOverflowStrategy()) { - case DROP_LATEST: - return; - - case DROP_OLDEST: - events.poll(); - break; - - default: - break; - } - } - if ((signal.event != null && events.size() >= receiveBufferCapacity) || !events.offer(signal)) { - events.clear(); - events.add(EventSignal.error(new IllegalStateException("Voice-agent receive buffer overflow."))); - WebSocket current = webSocket; - if (current != null) { - current.cancel(); - } - open.set(false); - closed.set(true); - shutdownHttpClient(); - } - } - - private void shutdownHttpClient() { - if (clientShutdown.compareAndSet(false, true)) { - httpClient.dispatcher().executorService().shutdown(); - httpClient.connectionPool().evictAll(); - } - } - - static OkHttpClient createHttpClient(VoiceAgentWebSocketClientConfiguration configuration, - VoiceAgentWebSocketConnectionOptions options) { - OkHttpClient.Builder builder = new OkHttpClient.Builder(); - builder.connectTimeout(options.getHandshakeTimeout().toMillis(), TimeUnit.MILLISECONDS) - .readTimeout(0, TimeUnit.MILLISECONDS) - .followRedirects(false); - ProxyOptions proxyOptions = configuration.getProxyOptions(); - if (proxyOptions != null) { - Proxy.Type proxyType = proxyOptions.getType() == ProxyOptions.Type.SOCKS4 - || proxyOptions.getType() == ProxyOptions.Type.SOCKS5 ? Proxy.Type.SOCKS : Proxy.Type.HTTP; - Proxy proxy = new Proxy(proxyType, proxyOptions.getAddress()); - if (proxyOptions.getNonProxyHosts() == null) { - builder.proxy(proxy); - } else { - Pattern nonProxyHosts = Pattern.compile(proxyOptions.getNonProxyHosts(), Pattern.CASE_INSENSITIVE); - builder.proxySelector(new ProxySelector() { - @Override - public List select(URI uri) { - return Collections - .singletonList(uri.getHost() != null && nonProxyHosts.matcher(uri.getHost()).matches() - ? Proxy.NO_PROXY - : proxy); - } - - @Override - public void connectFailed(URI uri, SocketAddress address, IOException error) { - LOGGER.atVerbose() - .addKeyValue("uri", uri) - .addKeyValue("proxyAddress", address) - .log("Failed to connect through the configured proxy."); - } - }); - } - if (proxyOptions.getUsername() != null) { - builder.proxyAuthenticator((route, response) -> response.request() - .newBuilder() - .header("Proxy-Authorization", - Credentials.basic(proxyOptions.getUsername(), proxyOptions.getPassword())) - .build()); - } - } - return builder.build(); - } - - private final class Listener extends WebSocketListener { - @Override - public void onOpen(WebSocket webSocket, Response response) { - handshakeSucceeded.set(true); - open.set(true); - handshakeCompleted.countDown(); - } - - @Override - public void onMessage(WebSocket webSocket, String text) { - if (text.getBytes(StandardCharsets.UTF_8).length > options.getMaxMessageSize()) { - fail(new IllegalArgumentException("Voice-agent message exceeds the configured size limit."), null); - webSocket.close(1009, "Message too large"); - return; - } - try { - signal(EventSignal.event(VoiceAgentWebSocketUtils.deserializeEvent(text, live))); - } catch (IOException | RuntimeException error) { - malformedEvent(webSocket, error); - } - } - - @Override - public void onMessage(WebSocket webSocket, ByteString bytes) { - if (bytes.size() > options.getMaxMessageSize()) { - fail(new IllegalArgumentException("Voice-agent message exceeds the configured size limit."), null); - webSocket.close(1009, "Message too large"); - return; - } - try { - onMessage(webSocket, VoiceAgentWebSocketUtils.decodeEvent(bytes.toByteArray())); - } catch (CharacterCodingException error) { - malformedEvent(webSocket, error); - } - } - - private void malformedEvent(WebSocket webSocket, Throwable error) { - if (options.getMalformedEventHandler() != null) { - try { - options.getMalformedEventHandler().accept(error); - return; - } catch (RuntimeException callbackError) { - error = callbackError; - } - } - fail(new IllegalArgumentException("Invalid JSON event.", error), null); - webSocket.close(1007, "Invalid JSON event"); - } - - @Override - public void onClosing(WebSocket webSocket, int code, String reason) { - closeCode = code; - closeReason = reason; - webSocket.close(code == 1005 ? 1000 : code, reason); - } - - @Override - public void onClosed(WebSocket webSocket, int code, String reason) { - closeCode = code; - closeReason = reason; - open.set(false); - closed.set(true); - signal(EventSignal.complete()); - handshakeCompleted.countDown(); - closeCompleted.countDown(); - shutdownHttpClient(); - } - - @Override - public void onFailure(WebSocket webSocket, Throwable error, Response response) { - fail(error, response); - } - } - - private static final class EventSignal { - private final RealtimeServerEvent event; - private final Throwable error; - private final boolean complete; - - private EventSignal(RealtimeServerEvent event, Throwable error, boolean complete) { - this.event = event; - this.error = error; - this.complete = complete; - } - - private static EventSignal event(RealtimeServerEvent event) { - return new EventSignal(event, null, false); - } - - private static EventSignal error(Throwable error) { - return new EventSignal(null, error, false); - } - - private static EventSignal complete() { - return new EventSignal(null, null, true); - } - } - - private static final class EventIterator implements Iterator { - private final BlockingQueue events; - private final Duration timeout; - private EventSignal next; - - private EventIterator(BlockingQueue events, Duration timeout) { - this.events = events; - this.timeout = timeout; - } - - @Override - public boolean hasNext() { - if (next == null) { - try { - next = timeout == null ? events.take() : events.poll(timeout.toNanos(), TimeUnit.NANOSECONDS); - if (next == null) { - throw new IllegalStateException("Timed out waiting for a voice-agent event.", - new TimeoutException()); - } - } catch (InterruptedException error) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("Interrupted while waiting for a voice-agent event.", error); - } - } - if (next.error != null) { - Throwable error = next.error; - next = null; - if (error instanceof RuntimeException) { - throw (RuntimeException) error; - } - throw new IllegalStateException("Voice-agent event stream failed.", error); - } - return !next.complete; - } - - @Override - public RealtimeServerEvent next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - RealtimeServerEvent event = next.event; - next = null; - return event; - } + transport.close(code, reason); } } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java index 1fc87e1ad297c..d123c4d966303 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java @@ -20,7 +20,6 @@ public final class VoiceAgentWebSocketClientConfiguration { private final String userAgent; private final HttpHeaders headers; private final ProxyOptions proxyOptions; - private final boolean live; /** * Creates the connection configuration. @@ -34,35 +33,12 @@ public final class VoiceAgentWebSocketClientConfiguration { */ public VoiceAgentWebSocketClientConfiguration(URI endpoint, TokenCredential credential, String apiVersion, String userAgent, HttpHeaders headers, ProxyOptions proxyOptions) { - this(endpoint, credential, apiVersion, userAgent, headers, proxyOptions, false); - } - - private VoiceAgentWebSocketClientConfiguration(URI endpoint, TokenCredential credential, String apiVersion, - String userAgent, HttpHeaders headers, ProxyOptions proxyOptions, boolean live) { this.endpoint = Objects.requireNonNull(endpoint, "'endpoint' cannot be null."); this.credential = Objects.requireNonNull(credential, "'credential' cannot be null."); this.apiVersion = Objects.requireNonNull(apiVersion, "'apiVersion' cannot be null."); this.userAgent = Objects.requireNonNull(userAgent, "'userAgent' cannot be null."); this.headers = headers; this.proxyOptions = proxyOptions; - this.live = live; - } - - /** - * Creates a configuration for the native GPT Live protocol. - * @return the Live configuration. - */ - public VoiceAgentWebSocketClientConfiguration forLive() { - return new VoiceAgentWebSocketClientConfiguration(endpoint, credential, apiVersion, userAgent, headers, - proxyOptions, true); - } - - /** - * Indicates whether this connection uses native GPT Live events and routing. - * @return whether the Live protocol is selected. - */ - public boolean isLive() { - return live; } /** diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java index 864d5119b96df..34d1e80261448 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java @@ -55,13 +55,9 @@ public static String validateEvent(BinaryData event) { } public static RealtimeServerEvent deserializeEvent(String json) throws IOException { - return deserializeEvent(json, false); - } - - public static RealtimeServerEvent deserializeEvent(String json, boolean live) throws IOException { validateEvent(BinaryData.fromString(json)); RawRealtimeServerEvent raw = new RawRealtimeServerEvent(BinaryData.fromString(json)); - if (live || raw.getType() == null) { + if (raw.getType() == null) { return raw; } try (JsonReader reader = JsonProviders.createReader(json)) { @@ -89,11 +85,10 @@ public static VoiceAgentWebSocketConnectionOptions liveOptions(VoiceAgentLiveWeb .setStructuredInputs(options.getStructuredInputs()); } - public static BinaryData liveEventPayload(RealtimeServerEvent event) { - if (event instanceof RawRealtimeServerEvent) { - return ((RawRealtimeServerEvent) event).getRawEvent(); - } - throw new IllegalStateException("Expected a raw native Live event."); + public static BinaryData deserializeRawEvent(String json) { + BinaryData event = BinaryData.fromString(json); + validateEvent(event); + return event; } public static void validateClose(int code, String reason) { @@ -123,6 +118,30 @@ private static boolean isProtectedHeader(String name) { public static URI buildWebSocketUri(VoiceAgentWebSocketClientConfiguration configuration, String agentName, VoiceAgentWebSocketConnectionOptions options) { + UrlBuilder url = buildSessionUrl(configuration, agentName, "", options); + VoiceAgentTransport transport = options.getTransport(); + url.setQueryParameter("transport", transport == null ? "websocket" : encode(transport.toString())); + if (options.getAgentSessionId() != null) { + url.setQueryParameter("agent_session_id", encode(options.getAgentSessionId())); + } + return URI.create(url.toString()); + } + + public static URI buildLiveWebSocketUri(VoiceAgentWebSocketClientConfiguration configuration, String agentName, + VoiceAgentWebSocketConnectionOptions options) { + UrlBuilder url = buildSessionUrl(configuration, agentName, "/live/sessions", options); + if (options.getStructuredInputs() != null) { + url.setQueryParameter("structured_inputs", encode(options.getStructuredInputs())); + } + return URI.create(url.toString()); + } + + private static UrlBuilder buildSessionUrl(VoiceAgentWebSocketClientConfiguration configuration, String agentName, + String suffix, VoiceAgentWebSocketConnectionOptions options) { + Objects.requireNonNull(agentName, "'agentName' cannot be null."); + if (agentName.isEmpty()) { + throw new IllegalArgumentException("'agentName' cannot be empty."); + } URI endpoint = configuration.getEndpoint(); String scheme; if ("https".equalsIgnoreCase(endpoint.getScheme()) || "wss".equalsIgnoreCase(endpoint.getScheme())) { @@ -137,29 +156,18 @@ public static URI buildWebSocketUri(VoiceAgentWebSocketClientConfiguration confi } String basePath = endpoint.getRawPath() == null ? "" : endpoint.getRawPath().replaceAll("/$", ""); - String path = basePath + "/agents/" + encode(agentName) + "/endpoint/protocols/voice" - + (configuration.isLive() ? "/live/sessions" : ""); + String path = basePath + "/agents/" + encode(agentName) + "/endpoint/protocols/voice" + suffix; URI baseUri = URI.create(scheme + "://" + endpoint.getRawAuthority() + path); UrlBuilder url = UrlBuilder.parse(baseUri.toString()); url.setQueryParameter("api-version", encode(configuration.getApiVersion())); url.setQueryParameter("x-ms-client-sdk", encode(configuration.getUserAgent())); - if (!configuration.isLive()) { - VoiceAgentTransport transport = options.getTransport(); - url.setQueryParameter("transport", transport == null ? "websocket" : encode(transport.toString())); - } if (options.isStoreEnabled() != null) { url.setQueryParameter("store", options.isStoreEnabled().toString()); } if (options.getAgentVersionOverride() != null) { url.setQueryParameter("x-agent-version-override", encode(options.getAgentVersionOverride())); } - if (options.getAgentSessionId() != null) { - url.setQueryParameter("agent_session_id", encode(options.getAgentSessionId())); - } - if (configuration.isLive() && options.getStructuredInputs() != null) { - url.setQueryParameter("structured_inputs", encode(options.getStructuredInputs())); - } - return URI.create(url.toString()); + return url; } public static TokenRequestContext createTokenRequestContext() { @@ -168,6 +176,14 @@ public static TokenRequestContext createTokenRequestContext() { public static HttpHeaders buildHeaders(VoiceAgentWebSocketClientConfiguration configuration, VoiceAgentWebSocketConnectionOptions options, String token) { + HttpHeaders headers = buildHeaders(configuration, token); + if (options.getStructuredInputs() != null) { + headers.set(HttpHeaderName.fromString("x-ms-voice-structured-inputs"), options.getStructuredInputs()); + } + return headers; + } + + public static HttpHeaders buildHeaders(VoiceAgentWebSocketClientConfiguration configuration, String token) { HttpHeaders headers = new HttpHeaders().set(HttpHeaderName.USER_AGENT, configuration.getUserAgent()); if (configuration.getHeaders() != null) { for (HttpHeader header : configuration.getHeaders()) { @@ -178,9 +194,6 @@ public static HttpHeaders buildHeaders(VoiceAgentWebSocketClientConfiguration co } headers.set(HttpHeaderName.fromString("Foundry-Features"), AgentDefinitionOptInKeys.VOICE_AGENTS_V1_PREVIEW.toString()); - if (!configuration.isLive() && options.getStructuredInputs() != null) { - headers.set(HttpHeaderName.fromString("x-ms-voice-structured-inputs"), options.getStructuredInputs()); - } return headers.set(HttpHeaderName.AUTHORIZATION, "Bearer " + token); } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java new file mode 100644 index 0000000000000..6d13ca944d239 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java @@ -0,0 +1,403 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.implementation.realtime; + +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; +import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport.ReceivedEvent; +import com.azure.core.credential.AccessToken; +import com.azure.core.exception.ClientAuthenticationException; +import com.azure.core.exception.HttpResponseException; +import com.azure.core.exception.ResourceModifiedException; +import com.azure.core.exception.ResourceNotFoundException; +import com.azure.core.http.HttpHeader; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.ProxyOptions; +import com.azure.core.util.BinaryData; +import com.azure.core.util.logging.ClientLogger; +import io.netty.channel.Channel; +import io.netty.channel.ChannelOption; +import io.netty.handler.codec.http.websocketx.BinaryWebSocketFrame; +import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; +import io.netty.handler.codec.http.websocketx.WebSocketClientHandshakeException; +import io.netty.handler.codec.http.websocketx.WebSocketFrame; +import java.io.IOException; +import java.net.URI; +import java.time.Duration; +import java.util.Objects; +import java.util.Queue; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CancellationException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import reactor.core.Disposable; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.publisher.Sinks; +import reactor.netty.Connection; +import reactor.netty.http.client.HttpClient; +import reactor.netty.http.client.WebsocketClientSpec; +import reactor.netty.http.websocket.WebsocketInbound; +import reactor.netty.http.websocket.WebsocketOutbound; +import reactor.netty.transport.ProxyProvider; + +/** + * Shared asynchronous WebSocket connection and I/O. Protocol clients supply the incoming message decoder. + * @param the decoded message type. + */ +public final class WebSocketSessionAsyncTransport implements AutoCloseable { + private static final ClientLogger LOGGER = new ClientLogger(WebSocketSessionAsyncTransport.class); + private static final int MAX_OUTSTANDING_SENDS = 256; + private final VoiceAgentWebSocketClientConfiguration configuration; + private final VoiceAgentWebSocketConnectionOptions options; + private final HttpClient httpClient; + private final URI websocketUri; + private final String subprotocol; + private final Function headersFactory; + private final WebSocketSessionTransport.EventDecoder decoder; + private final AtomicReference state = new AtomicReference<>(State.NEW); + private final AtomicReference channel = new AtomicReference<>(); + private final AtomicReference outbound = new AtomicReference<>(); + private final AtomicReference connectionOperation = new AtomicReference<>(); + private final AtomicReference lifecycle = new AtomicReference<>(); + private final AtomicBoolean receiveClaimed = new AtomicBoolean(); + private final Semaphore sendPermits = new Semaphore(MAX_OUTSTANDING_SENDS); + private final Queue> eventQueue; + private final Sinks.Many> events; + private final Sinks.One ready = Sinks.one(); + private final Sinks.One closeSignal = Sinks.one(); + private final AtomicReference> closeOperation = new AtomicReference<>(); + private volatile Integer closeCode; + private volatile String closeReason; + + public WebSocketSessionAsyncTransport(VoiceAgentWebSocketClientConfiguration configuration, URI websocketUri, + VoiceAgentWebSocketConnectionOptions options, HttpClient httpClient, String subprotocol, + Function headersFactory, WebSocketSessionTransport.EventDecoder decoder) { + this.configuration = Objects.requireNonNull(configuration, "'configuration' cannot be null."); + this.options = options == null ? new VoiceAgentWebSocketConnectionOptions() : options; + this.eventQueue = new ArrayBlockingQueue<>(this.options.getReceiveBufferCapacity()); + this.events = Sinks.many().unicast().onBackpressureBuffer(eventQueue); + this.httpClient = Objects.requireNonNull(httpClient, "'httpClient' cannot be null."); + this.websocketUri = Objects.requireNonNull(websocketUri, "'websocketUri' cannot be null."); + this.subprotocol = subprotocol; + this.headersFactory = Objects.requireNonNull(headersFactory, "'headersFactory' cannot be null."); + this.decoder = Objects.requireNonNull(decoder, "'decoder' cannot be null."); + } + + public Mono connect() { + return Mono.defer(() -> { + if (!state.compareAndSet(State.NEW, State.CONNECTING)) { + return Mono.error(new IllegalStateException("The voice-agent session has already been started.")); + } + return configuration.getCredential() + .getToken(VoiceAgentWebSocketUtils.createTokenRequestContext()) + .map(AccessToken::getToken) + .flatMap(token -> { + Disposable connection = openWebSocket(token).subscribe(unused -> { + }, this::terminateWithError, this::terminateNormally); + connectionOperation.set(connection); + return ready.asMono().timeout(options.getHandshakeTimeout()); + }) + .doOnError(this::terminateWithError) + .doOnCancel(() -> terminateWithError(new CancellationException("WebSocket connection cancelled."))); + }); + } + + public String getEndpoint() { + return websocketUri.toString(); + } + + public boolean isOpen() { + Channel current = channel.get(); + return state.get() == State.OPEN && current != null && current.isActive(); + } + + public Integer getCloseCode() { + return closeCode; + } + + public String getCloseReason() { + return closeReason; + } + + public Flux receiveEvents() { + return receiveMessages().map(ReceivedEvent::getEvent); + } + + public Flux receiveRawEvents() { + return receiveMessages().map(ReceivedEvent::getRawEvent); + } + + private Flux> receiveMessages() { + return Flux.defer(() -> receiveClaimed.compareAndSet(false, true) + ? events.asFlux() + : Flux.error(new IllegalStateException("Only one receiveEvents subscriber is supported per session."))); + } + + public Mono sendEvent(BinaryData event) { + Objects.requireNonNull(event, "'event' cannot be null."); + return Mono.defer(() -> { + String json = VoiceAgentWebSocketUtils.validateEvent(event); + Channel current = requireOpenChannel(); + if (!sendPermits.tryAcquire()) { + return Mono.error(new IllegalStateException("Too many voice-agent WebSocket sends are outstanding.")); + } + return Mono.create(sink -> current.writeAndFlush(new TextWebSocketFrame(json)).addListener(result -> { + sendPermits.release(); + if (result.isSuccess()) { + sink.success(); + } else { + sink.error(result.cause()); + } + })); + }); + } + + public Mono closeAsync() { + return closeAsync(1000, ""); + } + + public Mono closeAsync(int code, String reason) { + try { + VoiceAgentWebSocketUtils.validateClose(code, reason); + } catch (IllegalArgumentException exception) { + return Mono.error(exception); + } + Mono existing = closeOperation.get(); + if (existing != null) { + return existing; + } + Mono created = Mono.defer(() -> { + State current = state.get(); + if (current == State.CLOSED || current == State.NEW) { + state.set(State.CLOSED); + events.tryEmitComplete(); + return Mono.empty(); + } + state.set(State.CLOSING); + WebsocketOutbound currentOutbound = outbound.get(); + Channel currentChannel = channel.get(); + Mono graceful = currentOutbound == null ? Mono.empty() : currentOutbound.sendClose(code, reason); + Mono disposed = currentChannel == null ? Mono.empty() : Connection.from(currentChannel).onDispose(); + return graceful.then(disposed).timeout(options.getCloseTimeout()).onErrorResume(error -> { + if (currentChannel != null) { + currentChannel.close(); + } + return Mono.empty(); + }).doFinally(signal -> terminateNormally()); + }).cache(); + return closeOperation.compareAndSet(null, created) ? created : closeOperation.get(); + } + + @Override + public void close() { + closeAsync().block(options.getCloseTimeout().plusSeconds(1)); + } + + private Mono openWebSocket(String token) { + HttpClient client = configureProxy(httpClient).followRedirect(false) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, toConnectTimeoutMillis(options.getHandshakeTimeout())) + .doOnConnected(connection -> connection.addHandlerLast("voiceAgentHandshakeResponseObserver", + new VoiceAgentWebSocketHandshakeHandler(this::terminateWithError))) + .headers(headers -> { + for (HttpHeader header : headersFactory.apply(token)) { + headers.set(header.getName(), header.getValue()); + } + }); + WebsocketClientSpec.Builder specBuilder + = WebsocketClientSpec.builder().maxFramePayloadLength(options.getMaxMessageSize()); + if (subprotocol != null) { + specBuilder.protocols(subprotocol); + } + return client.websocket(specBuilder.build()).uri(websocketUri.toString()).connect().flatMap(connection -> { + if (!(connection instanceof WebsocketInbound) || !(connection instanceof WebsocketOutbound)) { + return Mono.error(new IllegalStateException("The WebSocket transport returned an invalid connection.")); + } + return handleConnection((WebsocketInbound) connection, (WebsocketOutbound) connection); + }); + } + + private static int toConnectTimeoutMillis(Duration timeout) { + if (timeout.compareTo(Duration.ofMillis(Integer.MAX_VALUE)) >= 0) { + return Integer.MAX_VALUE; + } + return Math.toIntExact(Math.max(1L, timeout.toMillis())); + } + + private Mono handleConnection(WebsocketInbound inbound, WebsocketOutbound outbound) { + this.outbound.set(outbound); + inbound.withConnection(connection -> channel.set(connection.channel())); + state.set(State.OPEN); + ready.tryEmitEmpty(); + inbound.receiveCloseStatus().subscribe(status -> { + closeCode = status.code(); + closeReason = status.reasonText(); + }, error -> LOGGER.atVerbose().addKeyValue("error", error.getMessage()).log("Close status unavailable.")); + Disposable receive = inbound.aggregateFrames(options.getMaxMessageSize()) + .receiveFrames() + .subscribe(this::handleFrame, this::terminateWithError, this::terminateNormally); + lifecycle.set(receive); + return closeSignal.asMono(); + } + + private void handleFrame(WebSocketFrame frame) { + if (frame instanceof TextWebSocketFrame || frame instanceof BinaryWebSocketFrame) { + try { + byte[] bytes = new byte[frame.content().readableBytes()]; + frame.content().getBytes(frame.content().readerIndex(), bytes); + String json = VoiceAgentWebSocketUtils.decodeEvent(bytes); + ReceivedEvent event = new ReceivedEvent<>(decoder.decode(json), BinaryData.fromString(json)); + Sinks.EmitResult result = events.tryEmitNext(event); + if (result == Sinks.EmitResult.FAIL_OVERFLOW || result == Sinks.EmitResult.FAIL_ZERO_SUBSCRIBER) { + switch (options.getOverflowStrategy()) { + case DROP_LATEST: + return; + + case DROP_OLDEST: + eventQueue.poll(); + result = events.tryEmitNext(event); + break; + + default: + break; + } + } + if (result.isFailure()) { + terminateWithError(new IllegalStateException("Voice-agent event emission failed: " + result)); + } + } catch (IOException | RuntimeException error) { + Throwable failure = error; + if (options.getMalformedEventHandler() != null) { + try { + options.getMalformedEventHandler().accept(error); + return; + } catch (RuntimeException callbackError) { + failure = callbackError; + } + } + closeWithProtocolError(1007, "Invalid JSON event", failure); + } + } + } + + private void closeWithProtocolError(int code, String reason, Throwable error) { + WebsocketOutbound current = outbound.get(); + terminateWithError(error); + if (current != null) { + current.sendClose(code, reason).subscribe(unused -> { + }, ignored -> { + }); + } + } + + private void terminateWithError(Throwable error) { + Throwable mappedError = mapHandshakeError(error); + State previous = state.getAndSet(State.CLOSED); + if (previous == State.CLOSED) { + return; + } + ready.tryEmitError(mappedError); + events.tryEmitError(mappedError); + closeSignal.tryEmitError(mappedError); + disposeReceive(); + } + + private Throwable mapHandshakeError(Throwable error) { + Throwable current = error; + while (current != null && !(current instanceof WebSocketClientHandshakeException)) { + current = current.getCause(); + } + if (current == null) { + return error; + } + WebSocketClientHandshakeException handshakeError = (WebSocketClientHandshakeException) current; + if (handshakeError.response() == null) { + return error; + } + VoiceAgentWebSocketHttpResponse response + = new VoiceAgentWebSocketHttpResponse(websocketUri, handshakeError.response()); + String message = "Voice-agent WebSocket handshake failed with status " + response.getStatusCode() + "."; + switch (response.getStatusCode()) { + case 401: + return new ClientAuthenticationException(message, response, error); + + case 404: + return new ResourceNotFoundException(message, response, error); + + case 409: + return new ResourceModifiedException(message, response, error); + + default: + return new HttpResponseException(message, response, error); + } + } + + private void terminateNormally() { + State previous = state.getAndSet(State.CLOSED); + if (previous == State.CLOSED) { + return; + } + if (previous == State.CONNECTING) { + ready.tryEmitError(new IllegalStateException("The WebSocket closed before the handshake completed.")); + } + events.tryEmitComplete(); + closeSignal.tryEmitEmpty(); + disposeReceive(); + } + + private void disposeReceive() { + Disposable connection = connectionOperation.getAndSet(null); + if (connection != null && !connection.isDisposed()) { + connection.dispose(); + } + Disposable receive = lifecycle.getAndSet(null); + if (receive != null && !receive.isDisposed()) { + receive.dispose(); + } + } + + private Channel requireOpenChannel() { + Channel current = channel.get(); + if (state.get() != State.OPEN || current == null || !current.isActive()) { + throw LOGGER + .logExceptionAsError(new IllegalStateException("The voice-agent WebSocket session is not open.")); + } + return current; + } + + private HttpClient configureProxy(HttpClient client) { + ProxyOptions proxy = configuration.getProxyOptions(); + if (proxy == null) { + return client; + } + return client.proxy(typeSpec -> { + ProxyProvider.Proxy proxyType; + switch (proxy.getType()) { + case SOCKS4: + proxyType = ProxyProvider.Proxy.SOCKS4; + break; + + case SOCKS5: + proxyType = ProxyProvider.Proxy.SOCKS5; + break; + + default: + proxyType = ProxyProvider.Proxy.HTTP; + break; + } + ProxyProvider.Builder builder = typeSpec.type(proxyType).socketAddress(proxy.getAddress()); + if (proxy.getUsername() != null) { + builder.username(proxy.getUsername()).password(ignored -> proxy.getPassword()); + } + if (proxy.getNonProxyHosts() != null) { + builder.nonProxyHosts(proxy.getNonProxyHosts()); + } + }); + } + + private enum State { + NEW, CONNECTING, OPEN, CLOSING, CLOSED + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java new file mode 100644 index 0000000000000..59a2904a85a62 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java @@ -0,0 +1,500 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.implementation.realtime; + +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; +import com.azure.core.exception.ClientAuthenticationException; +import com.azure.core.exception.HttpResponseException; +import com.azure.core.exception.ResourceModifiedException; +import com.azure.core.exception.ResourceNotFoundException; +import com.azure.core.http.HttpHeader; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.ProxyOptions; +import com.azure.core.util.BinaryData; +import com.azure.core.util.IterableStream; +import com.azure.core.util.logging.ClientLogger; +import java.io.IOException; +import java.net.Proxy; +import java.net.ProxySelector; +import java.net.SocketAddress; +import java.net.URI; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; +import java.util.function.Function; +import okhttp3.Credentials; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.WebSocket; +import okhttp3.WebSocketListener; +import okio.ByteString; + +/** + * Shared blocking WebSocket connection and I/O, independent of the asynchronous transport. + * @param the decoded message type. + */ +public final class WebSocketSessionTransport implements AutoCloseable { + private static final ClientLogger LOGGER = new ClientLogger(WebSocketSessionTransport.class); + private final URI websocketUri; + private final VoiceAgentWebSocketConnectionOptions options; + private final EventDecoder decoder; + private final OkHttpClient httpClient; + private final BlockingQueue> events; + private final int receiveBufferCapacity; + private final CountDownLatch handshakeCompleted = new CountDownLatch(1); + private final CountDownLatch closeCompleted = new CountDownLatch(1); + private final AtomicReference connectionError = new AtomicReference<>(); + private final AtomicBoolean handshakeSucceeded = new AtomicBoolean(); + private final AtomicBoolean receiveClaimed = new AtomicBoolean(); + private final AtomicBoolean open = new AtomicBoolean(); + private final AtomicBoolean closed = new AtomicBoolean(); + private final AtomicBoolean clientShutdown = new AtomicBoolean(); + private volatile WebSocket webSocket; + private volatile Integer closeCode; + private volatile String closeReason; + + @FunctionalInterface + public interface EventDecoder { + T decode(String json) throws IOException; + } + + private WebSocketSessionTransport(VoiceAgentWebSocketClientConfiguration configuration, URI websocketUri, + VoiceAgentWebSocketConnectionOptions options, String subprotocol, Function headersFactory, + EventDecoder decoder) { + this.options = options; + this.decoder = Objects.requireNonNull(decoder, "'decoder' cannot be null."); + this.receiveBufferCapacity = options.getReceiveBufferCapacity(); + this.events = new ArrayBlockingQueue<>(receiveBufferCapacity + 1); + this.websocketUri = Objects.requireNonNull(websocketUri, "'websocketUri' cannot be null."); + String token = configuration.getCredential() + .getTokenSync(VoiceAgentWebSocketUtils.createTokenRequestContext()) + .getToken(); + this.httpClient = createHttpClient(configuration, options); + Request.Builder request = new Request.Builder().url(websocketUri.toString()); + if (subprotocol != null) { + request.header("Sec-WebSocket-Protocol", subprotocol); + } + for (HttpHeader header : headersFactory.apply(token)) { + request.header(header.getName(), header.getValue()); + } + this.webSocket = httpClient.newWebSocket(request.build(), new Listener()); + } + + public static WebSocketSessionTransport connect(VoiceAgentWebSocketClientConfiguration configuration, + URI websocketUri, VoiceAgentWebSocketConnectionOptions options, String subprotocol, + Function headersFactory, EventDecoder decoder) { + WebSocketSessionTransport session = new WebSocketSessionTransport<>(configuration, websocketUri, options, + subprotocol, headersFactory, decoder); + try { + session.awaitHandshake(); + return session; + } catch (RuntimeException error) { + session.webSocket.cancel(); + session.shutdownHttpClient(); + throw error; + } + } + + public String getEndpoint() { + return websocketUri.toString(); + } + + public boolean isOpen() { + return open.get() && !closed.get(); + } + + public Integer getCloseCode() { + return closeCode; + } + + public String getCloseReason() { + return closeReason; + } + + public IterableStream receiveEvents(Duration timeout) { + return receiveMessages(timeout, ReceivedEvent::getEvent); + } + + public IterableStream receiveRawEvents(Duration timeout) { + return receiveMessages(timeout, ReceivedEvent::getRawEvent); + } + + private IterableStream receiveMessages(Duration timeout, Function, R> mapper) { + if (timeout != null && (timeout.isZero() || timeout.isNegative())) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException("Receive timeout must be positive.")); + } + if (!receiveClaimed.compareAndSet(false, true)) { + throw LOGGER.logExceptionAsError( + new IllegalStateException("Only one receiveEvents iterator is supported per session.")); + } + AtomicBoolean iteratorCreated = new AtomicBoolean(); + return IterableStream.of(() -> { + if (!iteratorCreated.compareAndSet(false, true)) { + throw LOGGER.logExceptionAsError( + new IllegalStateException("The receiveEvents stream may only be iterated once.")); + } + return new EventIterator<>(events, timeout, mapper); + }); + } + + public void sendEvent(BinaryData event) { + String json = VoiceAgentWebSocketUtils.validateEvent(event); + if (!isOpen()) { + throw LOGGER + .logExceptionAsError(new IllegalStateException("The voice-agent WebSocket session is not open.")); + } + if (!webSocket.send(json)) { + throw LOGGER.logExceptionAsError( + new IllegalStateException("The voice-agent WebSocket send queue is full or closed.")); + } + } + + @Override + public void close() { + close(1000, ""); + } + + public void close(int code, String reason) { + VoiceAgentWebSocketUtils.validateClose(code, reason); + if (!closed.compareAndSet(false, true)) { + shutdownHttpClient(); + return; + } + open.set(false); + if (!webSocket.close(code, reason)) { + webSocket.cancel(); + closeCompleted.countDown(); + } + try { + if (!closeCompleted.await(options.getCloseTimeout().toMillis(), TimeUnit.MILLISECONDS)) { + webSocket.cancel(); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + webSocket.cancel(); + } finally { + signal(EventSignal.complete()); + shutdownHttpClient(); + } + } + + private void awaitHandshake() { + try { + if (!handshakeCompleted.await(options.getHandshakeTimeout().toMillis(), TimeUnit.MILLISECONDS)) { + webSocket.cancel(); + throw LOGGER + .logExceptionAsError(new IllegalStateException("Voice-agent WebSocket handshake timed out.")); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + webSocket.cancel(); + throw LOGGER.logExceptionAsError( + new IllegalStateException("Interrupted while opening the voice-agent WebSocket session.", error)); + } + if (handshakeSucceeded.get()) { + return; + } + Throwable error = connectionError.get(); + if (error instanceof RuntimeException) { + throw LOGGER.logExceptionAsError((RuntimeException) error); + } + if (error != null) { + throw LOGGER.logExceptionAsError( + new IllegalStateException("Failed to open the voice-agent WebSocket session.", error)); + } + } + + private void fail(Throwable error, Response response) { + Throwable mapped; + try { + mapped = response == null ? error : mapHandshakeError(error, response); + } finally { + if (response != null) { + response.close(); + } + } + connectionError.compareAndSet(null, mapped); + open.set(false); + closed.set(true); + handshakeCompleted.countDown(); + signal(EventSignal.error(mapped)); + closeCompleted.countDown(); + shutdownHttpClient(); + } + + private Throwable mapHandshakeError(Throwable error, Response response) { + VoiceAgentWebSocketHttpResponse azureResponse = new VoiceAgentWebSocketHttpResponse(websocketUri, response); + String message = "Voice-agent WebSocket handshake failed with status " + response.code() + "."; + switch (response.code()) { + case 401: + return new ClientAuthenticationException(message, azureResponse, error); + + case 404: + return new ResourceNotFoundException(message, azureResponse, error); + + case 409: + return new ResourceModifiedException(message, azureResponse, error); + + default: + return new HttpResponseException(message, azureResponse, error); + } + } + + private synchronized void signal(EventSignal signal) { + if (signal.event != null && events.size() >= receiveBufferCapacity) { + switch (options.getOverflowStrategy()) { + case DROP_LATEST: + return; + + case DROP_OLDEST: + events.poll(); + break; + + default: + break; + } + } + if ((signal.event != null && events.size() >= receiveBufferCapacity) || !events.offer(signal)) { + events.clear(); + events.add(EventSignal.error(new IllegalStateException("Voice-agent receive buffer overflow."))); + WebSocket current = webSocket; + if (current != null) { + current.cancel(); + } + open.set(false); + closed.set(true); + shutdownHttpClient(); + } + } + + private void shutdownHttpClient() { + if (clientShutdown.compareAndSet(false, true)) { + httpClient.dispatcher().executorService().shutdown(); + httpClient.connectionPool().evictAll(); + } + } + + public static OkHttpClient createHttpClient(VoiceAgentWebSocketClientConfiguration configuration, + VoiceAgentWebSocketConnectionOptions options) { + OkHttpClient.Builder builder = new OkHttpClient.Builder(); + builder.connectTimeout(options.getHandshakeTimeout().toMillis(), TimeUnit.MILLISECONDS) + .readTimeout(0, TimeUnit.MILLISECONDS) + .followRedirects(false); + ProxyOptions proxyOptions = configuration.getProxyOptions(); + if (proxyOptions != null) { + Proxy.Type proxyType = proxyOptions.getType() == ProxyOptions.Type.SOCKS4 + || proxyOptions.getType() == ProxyOptions.Type.SOCKS5 ? Proxy.Type.SOCKS : Proxy.Type.HTTP; + Proxy proxy = new Proxy(proxyType, proxyOptions.getAddress()); + if (proxyOptions.getNonProxyHosts() == null) { + builder.proxy(proxy); + } else { + Pattern nonProxyHosts = Pattern.compile(proxyOptions.getNonProxyHosts(), Pattern.CASE_INSENSITIVE); + builder.proxySelector(new ProxySelector() { + @Override + public List select(URI uri) { + return Collections + .singletonList(uri.getHost() != null && nonProxyHosts.matcher(uri.getHost()).matches() + ? Proxy.NO_PROXY + : proxy); + } + + @Override + public void connectFailed(URI uri, SocketAddress address, IOException error) { + LOGGER.atVerbose() + .addKeyValue("uri", uri) + .addKeyValue("proxyAddress", address) + .log("Failed to connect through the configured proxy."); + } + }); + } + if (proxyOptions.getUsername() != null) { + builder.proxyAuthenticator((route, response) -> response.request() + .newBuilder() + .header("Proxy-Authorization", + Credentials.basic(proxyOptions.getUsername(), proxyOptions.getPassword())) + .build()); + } + } + return builder.build(); + } + + private final class Listener extends WebSocketListener { + @Override + public void onOpen(WebSocket webSocket, Response response) { + handshakeSucceeded.set(true); + open.set(true); + handshakeCompleted.countDown(); + } + + @Override + public void onMessage(WebSocket webSocket, String text) { + if (text.getBytes(StandardCharsets.UTF_8).length > options.getMaxMessageSize()) { + fail(new IllegalArgumentException("Voice-agent message exceeds the configured size limit."), null); + webSocket.close(1009, "Message too large"); + return; + } + try { + signal(EventSignal.event(new ReceivedEvent<>(decoder.decode(text), BinaryData.fromString(text)))); + } catch (IOException | RuntimeException error) { + malformedEvent(webSocket, error); + } + } + + @Override + public void onMessage(WebSocket webSocket, ByteString bytes) { + if (bytes.size() > options.getMaxMessageSize()) { + fail(new IllegalArgumentException("Voice-agent message exceeds the configured size limit."), null); + webSocket.close(1009, "Message too large"); + return; + } + try { + onMessage(webSocket, VoiceAgentWebSocketUtils.decodeEvent(bytes.toByteArray())); + } catch (CharacterCodingException error) { + malformedEvent(webSocket, error); + } + } + + private void malformedEvent(WebSocket webSocket, Throwable error) { + if (options.getMalformedEventHandler() != null) { + try { + options.getMalformedEventHandler().accept(error); + return; + } catch (RuntimeException callbackError) { + error = callbackError; + } + } + fail(new IllegalArgumentException("Invalid JSON event.", error), null); + webSocket.close(1007, "Invalid JSON event"); + } + + @Override + public void onClosing(WebSocket webSocket, int code, String reason) { + closeCode = code; + closeReason = reason; + webSocket.close(code == 1005 ? 1000 : code, reason); + } + + @Override + public void onClosed(WebSocket webSocket, int code, String reason) { + closeCode = code; + closeReason = reason; + open.set(false); + closed.set(true); + signal(EventSignal.complete()); + handshakeCompleted.countDown(); + closeCompleted.countDown(); + shutdownHttpClient(); + } + + @Override + public void onFailure(WebSocket webSocket, Throwable error, Response response) { + fail(error, response); + } + } + + static final class ReceivedEvent { + private final T event; + private final BinaryData rawEvent; + + ReceivedEvent(T event, BinaryData rawEvent) { + this.event = event; + this.rawEvent = rawEvent; + } + + T getEvent() { + return event; + } + + BinaryData getRawEvent() { + return rawEvent; + } + } + + private static final class EventSignal { + private final ReceivedEvent event; + private final Throwable error; + private final boolean complete; + + private EventSignal(ReceivedEvent event, Throwable error, boolean complete) { + this.event = event; + this.error = error; + this.complete = complete; + } + + private static EventSignal event(ReceivedEvent event) { + return new EventSignal<>(event, null, false); + } + + private static EventSignal error(Throwable error) { + return new EventSignal<>(null, error, false); + } + + private static EventSignal complete() { + return new EventSignal<>(null, null, true); + } + } + + private static final class EventIterator implements Iterator { + private final BlockingQueue> events; + private final Duration timeout; + private final Function, R> mapper; + private EventSignal next; + + private EventIterator(BlockingQueue> events, Duration timeout, + Function, R> mapper) { + this.events = events; + this.timeout = timeout; + this.mapper = mapper; + } + + @Override + public boolean hasNext() { + if (next == null) { + try { + next = timeout == null ? events.take() : events.poll(timeout.toNanos(), TimeUnit.NANOSECONDS); + if (next == null) { + throw new IllegalStateException("Timed out waiting for a voice-agent event.", + new TimeoutException()); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for a voice-agent event.", error); + } + } + if (next.error != null) { + Throwable error = next.error; + next = null; + if (error instanceof RuntimeException) { + throw (RuntimeException) error; + } + throw new IllegalStateException("Voice-agent event stream failed.", error); + } + return !next.complete; + } + + @Override + public R next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + ReceivedEvent event = next.event; + next = null; + return mapper.apply(event); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClientTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClientTests.java index 0692bfdc257ea..c652b426c2c4f 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClientTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClientTests.java @@ -5,6 +5,7 @@ import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport; import com.azure.ai.agents.models.VoiceAgentTransport; import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.http.ProxyOptions; @@ -48,8 +49,8 @@ public void syncTransportHonorsNonProxyHosts() { URI.create("https://localhost"), request -> Mono.just(new com.azure.core.credential.AccessToken("token", OffsetDateTime.now().plusHours(1))), "v1", "test-user-agent", null, proxyOptions); - OkHttpClient client = BetaVoiceAgentWebSocketSessionClient.createHttpClient(configuration, - new VoiceAgentWebSocketConnectionOptions()); + OkHttpClient client + = WebSocketSessionTransport.createHttpClient(configuration, new VoiceAgentWebSocketConnectionOptions()); List bypassed = client.proxySelector().select(URI.create("https://LOCALHOST/session")); List proxied = client.proxySelector().select(URI.create("https://example.com/session")); diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java index cd7b225d6c60a..540572039a8e7 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java @@ -6,13 +6,21 @@ import com.azure.ai.agents.AgentsClientBuilder; import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.implementation.OpenAIJsonHelper; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.credential.AccessToken; import com.azure.core.exception.ResourceNotFoundException; import com.azure.core.util.BinaryData; import com.azure.core.util.ClientOptions; import com.azure.core.util.Configuration; import com.azure.core.util.Header; +import com.openai.core.JsonValue; +import com.openai.models.live.ClientEvent; +import com.openai.models.live.ResponseCreateEvent; +import com.openai.models.live.ServerEvent; import io.netty.handler.codec.http.HttpHeaderNames; import java.net.URI; import java.net.URLDecoder; @@ -36,6 +44,7 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.netty.DisposableServer; +import reactor.netty.http.server.WebsocketServerSpec; import reactor.test.StepVerifier; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -144,6 +153,7 @@ public void syncNativeEventsAreLosslessAndSessionIsBidirectional() { assertEquals(FUTURE, events.next().toString()); assertEquals(ERROR, events.next().toString()); assertThrows(IllegalStateException.class, session::receiveEvents); + assertThrows(IllegalStateException.class, session::receiveTypedEvents); assertThrows(IllegalArgumentException.class, () -> session.sendEvent(BinaryData.fromString("[]"))); session.sendEvent(BinaryData.fromString(COMMAND)); assertEquals(COMMAND, events.next().toString()); @@ -179,6 +189,7 @@ public void asyncNativeEventsAreLosslessAndSessionIsBidirectional() { .verify(TIMEOUT); assertEquals(Arrays.asList(COMMAND), sent); StepVerifier.create(session.receiveEvents()).expectError(IllegalStateException.class).verify(TIMEOUT); + StepVerifier.create(session.receiveTypedEvents()).expectError(IllegalStateException.class).verify(TIMEOUT); session.closeAsync(1000, "done").block(TIMEOUT); assertFalse(session.isOpen()); StepVerifier.create(session.sendEvent(BinaryData.fromString(COMMAND))) @@ -189,6 +200,150 @@ public void asyncNativeEventsAreLosslessAndSessionIsBidirectional() { } } + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void sameBuilderKeepsLiveAndRealtimeHandshakesIndependent(boolean async) { + List requestUris = new CopyOnWriteArrayList<>(); + List headers = new CopyOnWriteArrayList<>(); + server = VoiceAgentWebSocketSessionTests.tlsServer().host("localhost").port(0).handle((request, response) -> { + requestUris.add(request.uri()); + headers.add(request.requestHeaders().copy()); + return response.sendWebsocket((inbound, outbound) -> inbound.receive().then(), + WebsocketServerSpec.builder().protocols("realtime").build()); + }).bindNow(); + AgentsClientBuilder builder = builder(); + String inputs = "{\"name\":\"value\"}"; + VoiceAgentLiveWebSocketConnectionOptions liveOptions + = new VoiceAgentLiveWebSocketConnectionOptions().setStructuredInputs(inputs); + VoiceAgentWebSocketConnectionOptions realtimeOptions + = new VoiceAgentWebSocketConnectionOptions().setStructuredInputs(inputs); + if (async) { + try (BetaVoiceAgentLiveWebSocketSessionAsyncClient live = builder.beta() + .buildBetaVoiceAgentLiveWebSocketAsyncClient() + .openWebSocketSession("agent", liveOptions) + .block(TIMEOUT)) { + assertNotNull(live); + assertTrue(live.isOpen()); + } + try (BetaVoiceAgentWebSocketSessionAsyncClient realtime = builder.beta() + .buildBetaVoiceAgentWebSocketAsyncClient() + .openWebSocketSession("agent", realtimeOptions) + .block(TIMEOUT)) { + assertNotNull(realtime); + assertTrue(realtime.isOpen()); + } + } else { + try (BetaVoiceAgentLiveWebSocketSessionClient live + = builder.beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent", liveOptions)) { + assertTrue(live.isOpen()); + } + try (BetaVoiceAgentWebSocketSessionClient realtime + = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", realtimeOptions)) { + assertTrue(realtime.isOpen()); + } + } + assertEquals(2, requestUris.size()); + assertEquals("/agents/agent/endpoint/protocols/voice/live/sessions", URI.create(requestUris.get(0)).getPath()); + assertTrue(requestUris.get(0).contains("structured_inputs=")); + assertFalse(requestUris.get(0).contains("transport=")); + assertNull(headers.get(0).get(HttpHeaderNames.SEC_WEBSOCKET_PROTOCOL)); + assertNull(headers.get(0).get("x-ms-voice-structured-inputs")); + assertEquals("/agents/agent/endpoint/protocols/voice", URI.create(requestUris.get(1)).getPath()); + assertTrue(requestUris.get(1).contains("transport=websocket")); + assertFalse(requestUris.get(1).contains("structured_inputs=")); + assertEquals("realtime", headers.get(1).get(HttpHeaderNames.SEC_WEBSOCKET_PROTOCOL)); + assertEquals(inputs, headers.get(1).get("x-ms-voice-structured-inputs")); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void typedEventsRoundTripWithoutLosingUnknownFields(boolean async) { + List sent = new CopyOnWriteArrayList<>(); + String audio = "{\"type\":\"session.output_audio.delta\",\"delta\":\"AAA=\",\"future_field\":true}"; + String backend = "{\"type\":\"response.event\",\"event_id\":\"backend-1\"," + + "\"event\":{\"type\":\"response.completed\",\"response\":{\"id\":\"response-1\",\"future_field\":true}}}"; + server = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, + response) -> response.sendWebsocket((inbound, + outbound) -> outbound.sendString(Flux.concat(Flux.just(STARTED, audio, backend, FUTURE, ERROR), + inbound.receive().asString().doOnNext(sent::add))).then())) + .bindNow(); + ClientEvent command = ClientEvent.ofResponseCreate(ResponseCreateEvent.builder() + .eventId("command-1") + .putAdditionalProperty("future_setting", JsonValue.from(true)) + .build()); + String commandJson = OpenAIJsonHelper.toBinaryData(command).toString(); + if (async) { + BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentLiveWebSocketAsyncClient() + .openWebSocketSession("agent") + .block(TIMEOUT); + assertNotNull(session); + try { + assertThrows(NullPointerException.class, () -> session.sendEvent((ClientEvent) null)); + StepVerifier.create(session.sendEvent(command).thenMany(session.receiveTypedEvents())) + .assertNext(event -> assertStartedEvent(event)) + .assertNext(event -> assertAudioEvent(event, audio)) + .assertNext(event -> assertBackendEvent(event, backend)) + .assertNext(event -> assertEventJson(event, FUTURE)) + .assertNext(event -> { + assertTrue(event.isErrorEvent()); + assertEventJson(event, ERROR); + }) + .assertNext(event -> assertEventJson(event, commandJson)) + .thenCancel() + .verify(TIMEOUT); + StepVerifier.create(session.receiveEvents()).expectError(IllegalStateException.class).verify(TIMEOUT); + } finally { + session.closeAsync().block(TIMEOUT); + } + } else { + try (BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + assertThrows(NullPointerException.class, () -> session.sendEvent((ClientEvent) null)); + assertThrows(IllegalArgumentException.class, () -> session.receiveTypedEvents(Duration.ZERO)); + Iterator events = session.receiveTypedEvents(TIMEOUT).iterator(); + assertStartedEvent(events.next()); + assertAudioEvent(events.next(), audio); + assertBackendEvent(events.next(), backend); + assertEventJson(events.next(), FUTURE); + ServerEvent error = events.next(); + assertTrue(error.isErrorEvent()); + assertEventJson(error, ERROR); + session.sendEvent(command); + assertEventJson(events.next(), commandJson); + assertThrows(IllegalStateException.class, session::receiveEvents); + } + } + assertEquals(Arrays.asList(commandJson), sent); + } + + private static void assertStartedEvent(ServerEvent event) { + assertTrue(event.isSessionStarted()); + assertEquals(JsonValue.from("conversation-1"), + event.asSessionStarted()._additionalProperties().get("conversation_id")); + assertEventJson(event, STARTED); + } + + private static void assertAudioEvent(ServerEvent event, String json) { + assertTrue(event.isSessionOutputAudioDelta()); + assertEquals("AAA=", event.asSessionOutputAudioDelta().delta()); + assertEventJson(event, json); + } + + private static void assertBackendEvent(ServerEvent event, String json) { + assertTrue(event.isResponseEvent()); + assertEquals("backend-1", event.asResponseEvent().eventId()); + assertEventJson(event, json); + } + + private static void assertEventJson(ServerEvent event, String json) { + assertEquals(BinaryData.fromString(json).toObject(Object.class), + OpenAIJsonHelper.toBinaryData(event).toObject(Object.class)); + } + @ParameterizedTest @ValueSource(booleans = { false, true }) public void handshakeFailuresSurfaceAsAzureExceptions(boolean async) { From 016c447e9b22eb1323af580096c76ce2b7df3244 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Mon, 26 Oct 2026 08:27:13 +0800 Subject: [PATCH 06/20] Move voice agent definition helper into realtime sample utilities --- .../voice/VoiceAgentBasicAsyncSample.java | 4 +-- .../agents/voice/VoiceAgentBasicSample.java | 4 +-- .../voice/VoiceAgentRealtimeSampleUtils.java | 20 ++++++++++++ .../agents/voice/VoiceAgentSampleUtils.java | 31 ------------------- .../voice/VoiceAgentVersionsSample.java | 6 ++-- 5 files changed, 27 insertions(+), 38 deletions(-) delete mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentSampleUtils.java diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicAsyncSample.java index d5ee1ea3e8e86..849da1d764423 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicAsyncSample.java @@ -39,7 +39,7 @@ public static void main(String[] args) { .buildAgentsAsyncClient(); client.createAgentVersion(agentName, - new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + new CreateAgentVersionInput(VoiceAgentRealtimeSampleUtils.createDefinition(modelType, model, "You are a friendly voice assistant. Keep replies short and natural."))) .doOnNext(created -> System.out.printf("Created voice agent %s, version %s%n", created.getName(), created.getVersion())) @@ -49,7 +49,7 @@ public static void main(String[] args) { .thenMany(client.listAgents(AgentKind.VOICE, null, null, null, null)) .doOnNext(agent -> System.out.println("Voice agent: " + agent.getName())) .then(client.createAgentVersion(agentName, - new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + new CreateAgentVersionInput(VoiceAgentRealtimeSampleUtils.createDefinition(modelType, model, "You are a friendly voice assistant. Always greet the caller warmly.")) .setDescription("Updated voice-agent instructions."))) .doOnNext(updated -> System.out.println("Created updated version: " + updated.getVersion())) diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicSample.java index e03ab19962386..e5128424c3660 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicSample.java @@ -40,7 +40,7 @@ public static void main(String[] args) { .buildAgentsClient(); try { AgentVersionDetails created = client.createAgentVersion(agentName, - new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + new CreateAgentVersionInput(VoiceAgentRealtimeSampleUtils.createDefinition(modelType, model, "You are a friendly voice assistant. Keep replies short and natural."))); System.out.printf("Created voice agent %s, version %s%n", created.getName(), created.getVersion()); @@ -51,7 +51,7 @@ public static void main(String[] args) { } AgentVersionDetails updated = client.createAgentVersion(agentName, - new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + new CreateAgentVersionInput(VoiceAgentRealtimeSampleUtils.createDefinition(modelType, model, "You are a friendly voice assistant. Always greet the caller warmly.")) .setDescription("Updated voice-agent instructions.")); System.out.println("Created updated version: " + updated.getVersion()); diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java index 0e905c78c1142..aac7dea9530af 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java @@ -11,7 +11,13 @@ import com.azure.ai.agents.models.RealtimeResponseAudioTranscriptDoneEvent; import com.azure.ai.agents.models.RealtimeResponseDoneEvent; import com.azure.ai.agents.models.RealtimeSessionCreatedEvent; +import com.azure.ai.agents.models.VoiceAgentAudioConfiguration; +import com.azure.ai.agents.models.VoiceAgentAudioOutputConfiguration; +import com.azure.ai.agents.models.VoiceAgentDefinition; import com.azure.ai.agents.models.VoiceConversation; +import com.azure.ai.agents.models.VoiceModelType; +import com.azure.ai.agents.models.VoiceOutputModality; +import com.azure.ai.agents.models.VoiceType; import com.azure.core.http.rest.RequestOptions; import com.azure.core.util.BinaryData; import reactor.core.publisher.Mono; @@ -20,6 +26,7 @@ import javax.sound.sampled.AudioSystem; import javax.sound.sampled.LineUnavailableException; import javax.sound.sampled.SourceDataLine; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; @@ -30,6 +37,19 @@ final class VoiceAgentRealtimeSampleUtils { private VoiceAgentRealtimeSampleUtils() { } + static VoiceAgentDefinition createDefinition(VoiceModelType modelType, String model, String instructions) { + VoiceAgentAudioOutputConfiguration output = new VoiceAgentAudioOutputConfiguration() + .setVoice("en-US-AvaNeural") + .setVoiceType(VoiceType.AZURE_STANDARD); + return new VoiceAgentDefinition() + .setModelType(modelType) + .setModel(model) + .setInstructions(instructions) + .setAudio(new VoiceAgentAudioConfiguration().setOutput(output)) + .setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)) + .setStore(true); + } + static boolean handleResponseEvent(RealtimeServerEvent event, AtomicReference conversationId, SpeakerPlayer player) { if (event instanceof RealtimeSessionCreatedEvent) { diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentSampleUtils.java deleted file mode 100644 index 2b37d1c0b1e02..0000000000000 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentSampleUtils.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.ai.agents.voice; - -import com.azure.ai.agents.models.VoiceAgentAudioConfiguration; -import com.azure.ai.agents.models.VoiceAgentAudioOutputConfiguration; -import com.azure.ai.agents.models.VoiceAgentDefinition; -import com.azure.ai.agents.models.VoiceModelType; -import com.azure.ai.agents.models.VoiceOutputModality; -import com.azure.ai.agents.models.VoiceType; - -import java.util.Collections; - -final class VoiceAgentSampleUtils { - private VoiceAgentSampleUtils() { - } - - static VoiceAgentDefinition createDefinition(VoiceModelType modelType, String model, String instructions) { - VoiceAgentAudioOutputConfiguration output = new VoiceAgentAudioOutputConfiguration() - .setVoice("en-US-AvaNeural") - .setVoiceType(VoiceType.AZURE_STANDARD); - return new VoiceAgentDefinition() - .setModelType(modelType) - .setModel(model) - .setInstructions(instructions) - .setAudio(new VoiceAgentAudioConfiguration().setOutput(output)) - .setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)) - .setStore(true); - } -} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentVersionsSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentVersionsSample.java index 724b553e30756..9e9e991c73459 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentVersionsSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentVersionsSample.java @@ -38,14 +38,14 @@ public static void main(String[] args) { .buildAgentsClient(); try { AgentVersionDetails first = client.createAgentVersion(agentName, - new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + new CreateAgentVersionInput(VoiceAgentRealtimeSampleUtils.createDefinition(modelType, model, "You are a helpful voice assistant."))); AgentVersionDetails released = client.createAgentVersion(agentName, - new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + new CreateAgentVersionInput(VoiceAgentRealtimeSampleUtils.createDefinition(modelType, model, "You are a helpful voice assistant. Greet the caller by name.")) .setDescription("Added a personalized greeting.")); AgentVersionDetails draft = client.createAgentVersion(agentName, - new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + new CreateAgentVersionInput(VoiceAgentRealtimeSampleUtils.createDefinition(modelType, model, "You are an experimental voice assistant.")) .setDescription("Candidate persona under review.") .setDraft(true)); From 2ad1dd60b0de1ba55b936d24fc4dd772a449162a Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Mon, 26 Oct 2026 09:39:27 +0800 Subject: [PATCH 07/20] Use typed GPT Live events by default and update samples --- sdk/ai/azure-ai-agents/README.md | 8 +- ...eAgentLiveWebSocketSessionAsyncClient.java | 6 +- ...aVoiceAgentLiveWebSocketSessionClient.java | 14 +- .../ai/agents/voice/GptLiveSampleUtils.java | 137 ++++++++++-------- .../ai/agents/voice/GptLiveSamplesTests.java | 115 ++++++++++----- .../voice/VoiceAgentLiveWebSocketTests.java | 38 ++--- 6 files changed, 189 insertions(+), 129 deletions(-) diff --git a/sdk/ai/azure-ai-agents/README.md b/sdk/ai/azure-ai-agents/README.md index 5f0ab82b32af8..191d59706a57d 100644 --- a/sdk/ai/azure-ai-agents/README.md +++ b/sdk/ai/azure-ai-agents/README.md @@ -1119,10 +1119,10 @@ send the `Foundry-Features: VoiceAgents=V1Preview` header, and **do not** negoti subprotocol. `allowPreview(true)` is not required. Foundry sends the initial native `session.start`. Wait for `session.started` before sending application events; -do not send `session.start` yourself. Use `sendEvent(ClientEvent)` and `receiveTypedEvents()` with the +do not send `session.start` yourself. Use `sendEvent(ClientEvent)` and `receiveEvents()` with the `com.openai.models.live` event models from the SDK's existing `openai-java` dependency. These are distinct -from Realtime event models. The synchronous client also offers `receiveTypedEvents(Duration)` for bounded waits. -The existing `sendEvent(BinaryData)` and `receiveEvents()` APIs expose complete JSON objects when the original +from Realtime event models. The synchronous client also offers `receiveEvents(Duration)` for bounded waits. +The `sendEvent(BinaryData)` and `receiveRawEvents()` APIs expose complete JSON objects when the original wire representation is needed, including unknown properties, nested `response.event` backend events, and the Foundry `conversation_id` on `session.started`. OpenAI typed models retain extra properties and unknown event variants, but typed serialization need not preserve the original JSON formatting. @@ -1136,7 +1136,7 @@ import com.openai.models.live.ClientEvent; import com.openai.models.live.ResponseCreateEvent; import com.openai.models.live.ServerEvent; -for (ServerEvent event : session.receiveTypedEvents()) { +for (ServerEvent event : session.receiveEvents()) { if (event.isSessionStarted()) { session.sendEvent(ClientEvent.ofResponseCreate(ResponseCreateEvent.builder().build())); } else if (event.isSessionOutputTranscriptDelta()) { diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java index 51bd3895dcba6..46241ee2c7e92 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java @@ -66,16 +66,16 @@ public String getCloseReason() { * Service {@code error} events are delivered as JSON; transport and malformed JSON errors terminate the stream. * @return the single-subscriber stream of complete JSON events. */ - public Flux receiveEvents() { + public Flux receiveRawEvents() { return transport.receiveRawEvents(); } /** * Receives typed native Live server events. Typed and raw receivers share the single-subscriber limit. - * Use {@link #receiveEvents()} when the original JSON representation is needed. + * Use {@link #receiveRawEvents()} when the original JSON representation is needed. * @return the typed server event stream. */ - public Flux receiveTypedEvents() { + public Flux receiveEvents() { return transport.receiveEvents(); } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java index ba8e905f7d3f8..05c1f9d4ff95a 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java @@ -66,27 +66,27 @@ public String getCloseReason() { * @return the single-consumer stream of complete JSON events. * @throws IllegalStateException if the receive stream was already claimed. */ - public IterableStream receiveEvents() { - return receiveEvents(null); + public IterableStream receiveRawEvents() { + return receiveRawEvents(null); } /** * Receives typed native Live server events. Typed and raw receivers share the single-consumer limit. * @return the typed server event stream. */ - public IterableStream receiveTypedEvents() { - return receiveTypedEvents(null); + public IterableStream receiveEvents() { + return receiveEvents(null); } /** * Receives typed native Live events with a timeout per wait. Timeouts leave the session open. - * Use {@link #receiveEvents(Duration)} when the original JSON representation is needed. + * Use {@link #receiveRawEvents(Duration)} when the original JSON representation is needed. * @param timeout a positive timeout, or null to wait indefinitely. * @return the single-consumer typed event stream. * @throws IllegalArgumentException if the timeout is zero or negative. * @throws IllegalStateException if the receive stream was already claimed, or a wait times out. */ - public IterableStream receiveTypedEvents(Duration timeout) { + public IterableStream receiveEvents(Duration timeout) { return transport.receiveEvents(timeout); } @@ -107,7 +107,7 @@ public void sendEvent(ClientEvent event) { * @throws IllegalArgumentException if the timeout is zero or negative. * @throws IllegalStateException if the receive stream was already claimed, or a wait times out. */ - public IterableStream receiveEvents(Duration timeout) { + public IterableStream receiveRawEvents(Duration timeout) { return transport.receiveRawEvents(timeout); } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java index 19fd0955a4806..ec88180d33158 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java @@ -20,6 +20,8 @@ import com.azure.core.util.Configuration; import com.azure.json.JsonProviders; import com.azure.json.JsonWriter; +import com.openai.core.JsonValue; +import com.openai.models.live.ServerEvent; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -90,8 +92,8 @@ static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient (audio, error) -> closeAudio(audio), GptLiveSampleUtils::closeAudio); } - static byte[] decodeAudio(Map event) { - byte[] pcm = Base64.getDecoder().decode(string(event, "delta")); + static byte[] decodeAudio(ServerEvent event) { + byte[] pcm = Base64.getDecoder().decode(event.asSessionOutputAudioDelta().delta()); if (pcm.length % 2 != 0) { throw new IllegalArgumentException("GPT Live PCM16 audio must contain complete two-byte samples."); } @@ -183,15 +185,15 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses thread.setDaemon(true); return thread; }); - BlockingQueue pending = new LinkedBlockingQueue<>(256); + BlockingQueue pending = new LinkedBlockingQueue<>(256); AtomicReference receiveFailure = new AtomicReference<>(); AtomicBoolean receiveEnded = new AtomicBoolean(); AudioTranscriptPrinter transcripts = new AudioTranscriptPrinter(System.console() != null, spoken); Future receiver = workers.submit(() -> { try { - for (BinaryData data : session.receiveEvents(TIMEOUT)) { - pending.put(data); - if ("session.closed".equals(type(object(data)))) { + for (ServerEvent event : session.receiveEvents(TIMEOUT)) { + pending.put(event); + if (event.isSessionClosed()) { break; } } @@ -218,21 +220,20 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses BinaryData silence = audio(new byte[960]); try { while (true) { - BinaryData data = pending.poll(20, TimeUnit.MILLISECONDS); + ServerEvent event = pending.poll(20, TimeUnit.MILLISECONDS); long now = System.nanoTime(); - if (data != null) { - Map event = object(data); + if (event != null) { if (spoken) { transcripts.print(event); - if ("session.output_audio.delta".equals(type(event))) { + if (event.isSessionOutputAudioDelta()) { byte[] pcm = decodeAudio(event); playAudio.accept(pcm); if (hasSpeech(pcm)) { greetingAudio = true; quietSince = now; } - } else if ("session.output_transcript.delta".equals(type(event)) - && !string(event, "delta").trim().isEmpty()) { + } else if (event.isSessionOutputTranscriptDelta() + && !event.asSessionOutputTranscriptDelta().delta().trim().isEmpty()) { greetingTranscript = true; quietSince = now; } @@ -240,8 +241,8 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses printEvent(event); } Map backend = backendEvent(event); - String type = type(backend); - if ("session.started".equals(type)) { + String type = event.isResponseEvent() ? type(backend) : ""; + if (event.isSessionStarted()) { if (started) { throw new IllegalStateException("Duplicate session.started event."); } @@ -279,13 +280,14 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses if ("function_call".equals(type(item))) { results.add(toolHandler.apply(item)); } - } else if ("session.closed".equals(type)) { - if (started && ("remote_hangup".equals(event.get("reason")) - || closing && "close_requested".equals(event.get("reason")))) { - System.out.println("Live session closed (" + event.get("reason") + ")."); + } else if (event.isSessionClosed()) { + String reason = event.asSessionClosed().reason().toString(); + if (started && ("remote_hangup".equals(reason) + || closing && "close_requested".equals(reason))) { + System.out.println("Live session closed (" + reason + ")."); return; } - throw new IllegalStateException("GPT Live closed before all turns completed. Event: " + data + throw new IllegalStateException("GPT Live closed before all turns completed. Event: " + event + "; transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason()); } } else if (receiveEnded.get()) { @@ -380,7 +382,7 @@ static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient input, playAudio, drainAudio)); } - static Mono runInteractiveConversation(Flux events, + static Mono runInteractiveConversation(Flux events, Function> sendEvent, Supplier closeDetails, Reader input, Consumer playAudio, Runnable drainAudio) { return Mono.defer(() -> { @@ -400,14 +402,14 @@ private static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyn nextPrompt, playAudio, drainAudio, spoken)); } - private static Mono runConversation(Flux events, + private static Mono runConversation(Flux events, Function> sendEvent, Supplier closeDetails, Function> nextPrompt, Consumer playAudio, Runnable drainAudio, boolean spoken) { return runConversation(events, sendEvent, closeDetails, nextPrompt, playAudio, drainAudio, spoken, null); } - private static Mono runConversation(Flux events, + private static Mono runConversation(Flux events, Function> sendEvent, Supplier closeDetails, Function> nextPrompt, Consumer playAudio, Runnable drainAudio, boolean spoken, AudioProcessor microphone) { @@ -428,24 +430,22 @@ private static Mono runConversation(Flux events, AtomicReference> responseCompleted = new AtomicReference<>(); Mono receive = events .publishOn(Schedulers.boundedElastic(), 1) - .doOnNext(data -> { - Map event = object(data); - String eventType = type(event); + .doOnNext(event -> { if (spoken) { transcripts.print(event); - if ("session.output_audio.delta".equals(eventType)) { + if (event.isSessionOutputAudioDelta()) { byte[] pcm = decodeAudio(event); playAudio.accept(pcm); greeting.onAudio(pcm); - } else if ("session.output_transcript.delta".equals(eventType) - && !string(event, "delta").trim().isEmpty()) { + } else if (event.isSessionOutputTranscriptDelta() + && !event.asSessionOutputTranscriptDelta().delta().trim().isEmpty()) { greeting.onTranscript(); } } else { printEvent(event); } - String type = type(backendEvent(event)); - if ("session.started".equals(type)) { + String type = event.isResponseEvent() ? type(backendEvent(event)) : ""; + if (event.isSessionStarted()) { if (!started.compareAndSet(false, true)) { throw new IllegalStateException("Duplicate session.started event."); } @@ -465,21 +465,22 @@ private static Mono runConversation(Flux events, System.out.println(); } completion.tryEmitEmpty(); - } else if ("session.closed".equals(type)) { - boolean remote = started.get() && "remote_hangup".equals(event.get("reason")); + } else if (event.isSessionClosed()) { + String reason = event.asSessionClosed().reason().toString(); + boolean remote = started.get() && "remote_hangup".equals(reason); boolean requested = microphone != null - || closeRequested.get() && "close_requested".equals(event.get("reason")); + || closeRequested.get() && "close_requested".equals(reason); if (!requested && !remote) { throw new IllegalStateException("GPT Live closed before all turns completed. Event: " - + data + "; " + closeDetails.get()); + + event + "; " + closeDetails.get()); } remoteHangup.set(remote || microphone != null); - System.out.println("Live session closed (" + event.get("reason") + ")."); + System.out.println("Live session closed (" + reason + ")."); finalized.set(true); closed.tryEmitEmpty(); } }) - .takeUntil(data -> "session.closed".equals(type(object(data)))) + .takeUntil(ServerEvent::isSessionClosed) .then(Mono.defer(() -> remoteHangup.get() ? Mono.empty() : finalized.get() ? Mono.never() : Mono.error(new IllegalStateException("GPT Live disconnected without session.closed.")))); Mono audio = ready.asMono() @@ -675,8 +676,28 @@ static BinaryData commentary(String content) { } } - static Map backendEvent(Map event) { - return "response.event".equals(type(event)) ? object(event.get("event")) : event; + static Map backendEvent(ServerEvent event) { + return event.isResponseEvent() + ? JsonValue.from(event.asResponseEvent().event()._additionalProperties()).convert(Map.class) + : java.util.Collections.emptyMap(); + } + + static void checkError(ServerEvent event) { + if (event.isErrorEvent()) { + com.openai.models.live.Error error = event.asErrorEvent().error(); + boolean moderated = "content_filter".equals(error._code().asString().orElse(null)) + || "content_filter".equals(error._type().asString().orElse(null)); + throw new IllegalStateException(moderated + ? "GPT Live service error: generation stopped by service moderation " + + "(content_filter). The call has ended; review the conversation and service diagnostics. Event: " + + event + : "GPT Live service error: " + event); + } + if (event.isResponseEvent()) { + checkError(backendEvent(event)); + } else if (event._json().isPresent()) { + checkError(event._json().get().convert(Map.class)); + } } static void checkError(Map event) { @@ -693,18 +714,19 @@ static void checkError(Map event) { } } - static void printEvent(Map event) { + static void printEvent(ServerEvent event) { checkError(event); - if ("session.started".equals(type(event))) { - Object conversationId = event.get("conversation_id"); + if (event.isSessionStarted()) { + JsonValue conversationId = event.asSessionStarted()._additionalProperties().get("conversation_id"); System.out.println(conversationId == null + || !conversationId.asString().isPresent() ? "Live session started; conversation_id not provided (persistence may be disabled)." - : "Live session started; conversation_id=" + conversationId); - } else if ("session.input_transcript.delta".equals(type(event))) { - System.out.print(string(event, "delta")); - } else if ("session.output_transcript.delta".equals(type(event))) { - System.out.print(string(event, "delta")); - } else if ("response.event".equals(type(event))) { + : "Live session started; conversation_id=" + conversationId.asString().get()); + } else if (event.isSessionInputTranscriptDelta()) { + System.out.print(event.asSessionInputTranscriptDelta().delta()); + } else if (event.isSessionOutputTranscriptDelta()) { + System.out.print(event.asSessionOutputTranscriptDelta().delta()); + } else if (event.isResponseEvent()) { Map backend = backendEvent(event); checkError(backend); if ("response.output_text.delta".equals(type(backend))) { @@ -979,30 +1001,27 @@ synchronized void endUtterance() { write(text); } - synchronized void print(Map event) { + synchronized void print(ServerEvent event) { checkError(event); - String eventType = type(event); - if ("response.event".equals(eventType)) { - checkError(backendEvent(event)); - } else if ("session.input_transcript.delta".equals(eventType) - || "session.output_transcript.delta".equals(eventType)) { - if (completeUtterances && "session.output_transcript.delta".equals(eventType)) { - String delta = string(event, "delta"); + if (event.isSessionInputTranscriptDelta() || event.isSessionOutputTranscriptDelta()) { + String delta = event.isSessionInputTranscriptDelta() + ? event.asSessionInputTranscriptDelta().delta() : event.asSessionOutputTranscriptDelta().delta(); + if (completeUtterances && event.isSessionOutputTranscriptDelta()) { if (delta.length() > MAX_BUFFERED_TRANSCRIPT_CHARACTERS - pending.length() - utterance.length()) { throw new IllegalStateException("Buffered transcript exceeded 65536 characters."); } utterance.append(delta); return; } - String nextSpeaker = "session.input_transcript.delta".equals(eventType) ? "You" : "Assistant"; + String nextSpeaker = event.isSessionInputTranscriptDelta() ? "You" : "Assistant"; if (!nextSpeaker.equals(speaker)) { write((lineOpen ? System.lineSeparator() : "") + nextSpeaker + ": "); speaker = nextSpeaker; } - write(string(event, "delta")); - } else if ("session.started".equals(eventType)) { + write(delta); + } else if (event.isSessionStarted()) { printEvent(event); - } else if ("session.closed".equals(eventType)) { + } else if (event.isSessionClosed()) { endUtterance(); if (lineOpen) { write(System.lineSeparator()); diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java index 664b051b3390a..38c2a6c6e9b88 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java @@ -6,10 +6,12 @@ import com.azure.ai.agents.AgentsClientBuilder; import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; +import com.azure.ai.agents.implementation.OpenAIJsonHelper; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.core.credential.AccessToken; import com.azure.core.util.BinaryData; import com.azure.core.util.Configuration; +import com.openai.models.live.ServerEvent; import java.io.ByteArrayOutputStream; import java.io.FilterReader; import java.io.InputStream; @@ -101,18 +103,15 @@ public void audioTranscriptsLabelSpeakersWithoutPrintingBackendText() { try (PrintStream capture = new PrintStream(output)) { System.setOut(capture); GptLiveSampleUtils.AudioTranscriptPrinter printer = new GptLiveSampleUtils.AudioTranscriptPrinter(); - printer.print(GptLiveSampleUtils.object(BinaryData.fromString(STARTED))); - printer.print(GptLiveSampleUtils - .object(GptLiveSampleUtils.event("session.input_transcript.delta", "delta", "Hello"))); - printer.print(GptLiveSampleUtils - .object(GptLiveSampleUtils.event("session.input_transcript.delta", "delta", " there"))); - printer.print(GptLiveSampleUtils.object(BinaryData.fromString("{\"type\":\"response.event\"," + printer.print(typedEvent(BinaryData.fromString(STARTED))); + printer.print(typedEvent(GptLiveSampleUtils.event("session.input_transcript.delta", "delta", "Hello"))); + printer.print(typedEvent(GptLiveSampleUtils.event("session.input_transcript.delta", "delta", " there"))); + printer.print(typedEvent(BinaryData.fromString("{\"type\":\"response.event\"," + "\"event\":{\"type\":\"response.output_text.delta\",\"delta\":\"backend-only text\"}}"))); - printer.print( - GptLiveSampleUtils.object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Hi!"))); + printer.print(typedEvent(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Hi!"))); printer.beginTurn(); - printer.print(GptLiveSampleUtils - .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "How can I help?"))); + printer.print( + typedEvent(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "How can I help?"))); } finally { System.setOut(original); } @@ -126,7 +125,7 @@ public void audioTranscriptsLabelSpeakersWithoutPrintingBackendText() { @ValueSource(booleans = { false, true }) @ResourceLock("java.lang.System.out") public void startupPrintsFoundryConversationIdInScriptedAndInteractiveModes(boolean interactive) { - Map event = GptLiveSampleUtils.object(BinaryData.fromString( + ServerEvent event = typedEvent(BinaryData.fromString( "{\"type\":\"session.started\",\"conversation_id\":\"conversation-1\",\"session\":{\"id\":\"native-1\"}}")); ByteArrayOutputStream output = new ByteArrayOutputStream(); PrintStream original = System.out; @@ -148,7 +147,7 @@ public void startupPrintsFoundryConversationIdInScriptedAndInteractiveModes(bool @ValueSource(booleans = { false, true }) @ResourceLock("java.lang.System.out") public void startupWithoutPersistenceDoesNotSubstituteNativeSessionId(boolean interactive) { - Map event = GptLiveSampleUtils.object(BinaryData.fromString(STARTED)); + ServerEvent event = typedEvent(BinaryData.fromString(STARTED)); ByteArrayOutputStream output = new ByteArrayOutputStream(); PrintStream original = System.out; try (PrintStream capture = new PrintStream(output)) { @@ -173,8 +172,8 @@ public void transcriptBufferIsBoundedWhileReadingInput() { GptLiveSampleUtils.AudioTranscriptPrinter printer = new GptLiveSampleUtils.AudioTranscriptPrinter(); try (PrintStream capture = new PrintStream(output)) { System.setOut(capture); - printer.print(GptLiveSampleUtils - .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Previous response."))); + printer.print( + typedEvent(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Previous response."))); printer.beginInput(); assertTrue(new String(output.toByteArray(), StandardCharsets.UTF_8) .endsWith("Previous response." + System.lineSeparator() + "You: ")); @@ -182,11 +181,11 @@ public void transcriptBufferIsBoundedWhileReadingInput() { char[] characters = new char[GptLiveSampleUtils.AudioTranscriptPrinter.MAX_BUFFERED_TRANSCRIPT_CHARACTERS - "Assistant: ".length()]; Arrays.fill(characters, 'a'); - printer.print(GptLiveSampleUtils - .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", new String(characters)))); + printer.print(typedEvent( + GptLiveSampleUtils.event("session.output_transcript.delta", "delta", new String(characters)))); assertEquals(promptLength, output.size(), "Transcript output must not overwrite the input line"); - IllegalStateException error = assertThrows(IllegalStateException.class, () -> printer.print( - GptLiveSampleUtils.object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "x")))); + IllegalStateException error = assertThrows(IllegalStateException.class, () -> printer + .print(typedEvent(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "x")))); assertTrue(error.getMessage().contains("65536")); printer.endInput(); assertTrue(output.size() > promptLength, "Buffered text must be flushed after input ends"); @@ -205,21 +204,19 @@ public void consoleTranscriptUsesSingleLineBreaks(boolean inputEchoed) { System.setOut(capture); GptLiveSampleUtils.AudioTranscriptPrinter printer = new GptLiveSampleUtils.AudioTranscriptPrinter(inputEchoed); - printer.print(GptLiveSampleUtils - .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Welcome."))); + printer.print(typedEvent(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Welcome."))); printer.beginInput(); if (inputEchoed) { System.out.println("testing"); } - printer.print(GptLiveSampleUtils - .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Reply."))); + printer.print(typedEvent(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Reply."))); printer.endInput(); printer.beginInput(); if (inputEchoed) { System.out.println("exit"); } printer.endInput(); - printer.print(GptLiveSampleUtils.object(BinaryData.fromString(CLOSED))); + printer.print(typedEvent(BinaryData.fromString(CLOSED))); System.out.println("Live session closed."); } finally { System.setOut(original); @@ -241,13 +238,13 @@ public void partialUtteranceSurvivesInputSubmissionWithoutBeingPrinted() { GptLiveSampleUtils.AudioTranscriptPrinter printer = new GptLiveSampleUtils.AudioTranscriptPrinter(false, true); printer.beginInput(); - printer.print(GptLiveSampleUtils - .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "This sentence "))); + printer.print( + typedEvent(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "This sentence "))); printer.endInput(); assertFalse(new String(output.toByteArray(), StandardCharsets.UTF_8).contains("This sentence")); printer.beginTurn(); printer.beginInput(); - printer.print(GptLiveSampleUtils.object( + printer.print(typedEvent( GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "continues after Enter."))); printer.endUtterance(); assertTrue(new String(output.toByteArray(), StandardCharsets.UTF_8).endsWith("You: ")); @@ -271,17 +268,17 @@ public void completedAndPartialUtterancesShareTheTranscriptBufferLimit() { GptLiveSampleUtils.AudioTranscriptPrinter printer = new GptLiveSampleUtils.AudioTranscriptPrinter(false, true); printer.beginInput(); - printer.print(GptLiveSampleUtils - .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Completed."))); + printer + .print(typedEvent(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "Completed."))); printer.endUtterance(); int queuedLength = ("Assistant: Completed." + System.lineSeparator()).length(); char[] characters = new char[GptLiveSampleUtils.AudioTranscriptPrinter.MAX_BUFFERED_TRANSCRIPT_CHARACTERS - queuedLength]; Arrays.fill(characters, 'a'); - printer.print(GptLiveSampleUtils - .object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", new String(characters)))); - assertThrows(IllegalStateException.class, () -> printer.print( - GptLiveSampleUtils.object(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "x")))); + printer.print(typedEvent( + GptLiveSampleUtils.event("session.output_transcript.delta", "delta", new String(characters)))); + assertThrows(IllegalStateException.class, () -> printer + .print(typedEvent(GptLiveSampleUtils.event("session.output_transcript.delta", "delta", "x")))); printer.endInput(); assertTrue(new String(output.toByteArray(), StandardCharsets.UTF_8).contains("Assistant: Completed.")); assertFalse(new String(output.toByteArray(), StandardCharsets.UTF_8).contains("aaaa")); @@ -333,18 +330,53 @@ public void spokenSentenceIsNotSplitByBackendCompletionOrAnotherTypedTurn(boolea assertEquals(3, transcript.split("You: ", -1).length - 1, "Input must remain available between backend turns"); } + @Test + @ResourceLock("java.lang.System.out") + public void unknownLiveEventsDoNotInterruptTranscriptHandlers() { + ServerEvent event = typedEvent(GptLiveSampleUtils.event("session.future_event", "value", "future")); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream original = System.out; + try (PrintStream capture = new PrintStream(output)) { + System.setOut(capture); + GptLiveSampleUtils.printEvent(event); + new GptLiveSampleUtils.AudioTranscriptPrinter().print(event); + } finally { + System.setOut(original); + } + assertEquals(0, output.size()); + } + + @ParameterizedTest + @ValueSource(strings = { "response.failed", "response.incomplete" }) + public void responseFailuresAreReportedForUnknownAndDelegatedEvents(String type) { + BinaryData failure = GptLiveSampleUtils.event(type, "event_id", "failure-1"); + for (boolean delegated : new boolean[] { false, true }) { + Map wrapper = new LinkedHashMap<>(); + wrapper.put("type", "response.event"); + wrapper.put("event", GptLiveSampleUtils.object(failure)); + ServerEvent event = typedEvent(delegated ? BinaryData.fromObject(wrapper) : failure); + IllegalStateException error + = assertThrows(IllegalStateException.class, () -> GptLiveSampleUtils.printEvent(event)); + assertTrue(error.getMessage().contains(type)); + assertTrue(error.getMessage().contains("failure-1")); + assertThrows(IllegalStateException.class, + () -> new GptLiveSampleUtils.AudioTranscriptPrinter().print(event)); + } + } + @Test public void moderationIsReportedExplicitlyAndPreservesServiceDiagnostics() { Map event = GptLiveSampleUtils.object(BinaryData.fromString(MODERATED)); IllegalStateException error = assertThrows(IllegalStateException.class, - () -> new GptLiveSampleUtils.AudioTranscriptPrinter().print(event)); + () -> new GptLiveSampleUtils.AudioTranscriptPrinter().print(typedEvent(BinaryData.fromString(MODERATED)))); assertTrue(error.getMessage().contains("generation stopped by service moderation")); assertTrue(error.getMessage().contains("content_filter")); assertTrue(error.getMessage().contains("moderated-1")); Map wrapper = new java.util.LinkedHashMap<>(); wrapper.put("type", "response.event"); wrapper.put("event", event); - assertThrows(IllegalStateException.class, () -> new GptLiveSampleUtils.AudioTranscriptPrinter().print(wrapper)); + assertThrows(IllegalStateException.class, + () -> new GptLiveSampleUtils.AudioTranscriptPrinter().print(typedEvent(BinaryData.fromObject(wrapper)))); } @ParameterizedTest @@ -1044,12 +1076,17 @@ public void nativeAudioEncodingAndDecodingPreservePcmBytes() { Map command = GptLiveSampleUtils.object(GptLiveSampleUtils.audio(pcm)); assertEquals("session.input_audio.append", command.get("type")); assertArrayEquals(pcm, Base64.getDecoder().decode((String) command.get("audio"))); - Map event = Collections.singletonMap("delta", Base64.getEncoder().encodeToString(pcm)); + ServerEvent event = typedEvent( + GptLiveSampleUtils.event("session.output_audio.delta", "delta", Base64.getEncoder().encodeToString(pcm))); assertArrayEquals(pcm, GptLiveSampleUtils.decodeAudio(event)); - assertThrows(IllegalArgumentException.class, - () -> GptLiveSampleUtils.decodeAudio(Collections.singletonMap("delta", "%%%"))); - assertThrows(IllegalArgumentException.class, - () -> GptLiveSampleUtils.decodeAudio(Collections.singletonMap("delta", "AA=="))); + assertThrows(IllegalArgumentException.class, () -> GptLiveSampleUtils + .decodeAudio(typedEvent(GptLiveSampleUtils.event("session.output_audio.delta", "delta", "%%%")))); + assertThrows(IllegalArgumentException.class, () -> GptLiveSampleUtils + .decodeAudio(typedEvent(GptLiveSampleUtils.event("session.output_audio.delta", "delta", "AA==")))); + } + + private static ServerEvent typedEvent(BinaryData data) { + return OpenAIJsonHelper.fromBinaryData(data, ServerEvent.class); } private static Stream audioSessionCases() { diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java index 540572039a8e7..31c71d5e0e61f 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java @@ -147,13 +147,13 @@ public void syncNativeEventsAreLosslessAndSessionIsBidirectional() { BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent"); try { - Iterator events = session.receiveEvents(TIMEOUT).iterator(); + Iterator events = session.receiveRawEvents(TIMEOUT).iterator(); assertEquals(STARTED, events.next().toString()); assertEquals(RESPONSE, events.next().toString()); assertEquals(FUTURE, events.next().toString()); assertEquals(ERROR, events.next().toString()); + assertThrows(IllegalStateException.class, session::receiveRawEvents); assertThrows(IllegalStateException.class, session::receiveEvents); - assertThrows(IllegalStateException.class, session::receiveTypedEvents); assertThrows(IllegalArgumentException.class, () -> session.sendEvent(BinaryData.fromString("[]"))); session.sendEvent(BinaryData.fromString(COMMAND)); assertEquals(COMMAND, events.next().toString()); @@ -181,15 +181,15 @@ public void asyncNativeEventsAreLosslessAndSessionIsBidirectional() { .verify(TIMEOUT); StepVerifier .create(session.sendEvent(BinaryData.fromString(COMMAND)) - .thenMany(session.receiveEvents()) + .thenMany(session.receiveRawEvents()) .map(BinaryData::toString)) .expectNext(STARTED, RESPONSE, FUTURE, ERROR) .expectNext(COMMAND) .thenCancel() .verify(TIMEOUT); assertEquals(Arrays.asList(COMMAND), sent); + StepVerifier.create(session.receiveRawEvents()).expectError(IllegalStateException.class).verify(TIMEOUT); StepVerifier.create(session.receiveEvents()).expectError(IllegalStateException.class).verify(TIMEOUT); - StepVerifier.create(session.receiveTypedEvents()).expectError(IllegalStateException.class).verify(TIMEOUT); session.closeAsync(1000, "done").block(TIMEOUT); assertFalse(session.isOpen()); StepVerifier.create(session.sendEvent(BinaryData.fromString(COMMAND))) @@ -283,7 +283,7 @@ public void typedEventsRoundTripWithoutLosingUnknownFields(boolean async) { assertNotNull(session); try { assertThrows(NullPointerException.class, () -> session.sendEvent((ClientEvent) null)); - StepVerifier.create(session.sendEvent(command).thenMany(session.receiveTypedEvents())) + StepVerifier.create(session.sendEvent(command).thenMany(session.receiveEvents())) .assertNext(event -> assertStartedEvent(event)) .assertNext(event -> assertAudioEvent(event, audio)) .assertNext(event -> assertBackendEvent(event, backend)) @@ -295,7 +295,9 @@ public void typedEventsRoundTripWithoutLosingUnknownFields(boolean async) { .assertNext(event -> assertEventJson(event, commandJson)) .thenCancel() .verify(TIMEOUT); - StepVerifier.create(session.receiveEvents()).expectError(IllegalStateException.class).verify(TIMEOUT); + StepVerifier.create(session.receiveRawEvents()) + .expectError(IllegalStateException.class) + .verify(TIMEOUT); } finally { session.closeAsync().block(TIMEOUT); } @@ -303,8 +305,8 @@ public void typedEventsRoundTripWithoutLosingUnknownFields(boolean async) { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { assertThrows(NullPointerException.class, () -> session.sendEvent((ClientEvent) null)); - assertThrows(IllegalArgumentException.class, () -> session.receiveTypedEvents(Duration.ZERO)); - Iterator events = session.receiveTypedEvents(TIMEOUT).iterator(); + assertThrows(IllegalArgumentException.class, () -> session.receiveEvents(Duration.ZERO)); + Iterator events = session.receiveEvents(TIMEOUT).iterator(); assertStartedEvent(events.next()); assertAudioEvent(events.next(), audio); assertBackendEvent(events.next(), backend); @@ -314,7 +316,7 @@ public void typedEventsRoundTripWithoutLosingUnknownFields(boolean async) { assertEventJson(error, ERROR); session.sendEvent(command); assertEventJson(events.next(), commandJson); - assertThrows(IllegalStateException.class, session::receiveEvents); + assertThrows(IllegalStateException.class, session::receiveRawEvents); } } assertEquals(Arrays.asList(commandJson), sent); @@ -379,7 +381,7 @@ public void malformedJsonTerminatesReception(boolean async) { .block(TIMEOUT); assertNotNull(session); try { - StepVerifier.create(session.receiveEvents()) + StepVerifier.create(session.receiveRawEvents()) .expectError(IllegalArgumentException.class) .verify(TIMEOUT); } finally { @@ -388,7 +390,8 @@ public void malformedJsonTerminatesReception(boolean async) { } else { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - assertThrows(IllegalArgumentException.class, () -> session.receiveEvents(TIMEOUT).iterator().hasNext()); + assertThrows(IllegalArgumentException.class, + () -> session.receiveRawEvents(TIMEOUT).iterator().hasNext()); } } } @@ -438,7 +441,7 @@ public void peerClosurePreservesEventsAndCloseMetadata(boolean async) { .block(TIMEOUT); assertNotNull(session); try { - StepVerifier.create(session.receiveEvents().map(BinaryData::toString)) + StepVerifier.create(session.receiveRawEvents().map(BinaryData::toString)) .expectNext(STARTED) .expectComplete() .verify(TIMEOUT); @@ -451,7 +454,7 @@ public void peerClosurePreservesEventsAndCloseMetadata(boolean async) { } else { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - Iterator events = session.receiveEvents(TIMEOUT).iterator(); + Iterator events = session.receiveRawEvents(TIMEOUT).iterator(); assertEquals(STARTED, events.next().toString()); assertFalse(events.hasNext()); assertEquals(1000, session.getCloseCode()); @@ -480,7 +483,7 @@ public void malformedEventCallbackCanSkipInvalidJson(boolean async) { .block(TIMEOUT); assertNotNull(session); try { - StepVerifier.create(session.receiveEvents().map(BinaryData::toString)) + StepVerifier.create(session.receiveRawEvents().map(BinaryData::toString)) .expectNext(STARTED) .thenCancel() .verify(TIMEOUT); @@ -490,7 +493,7 @@ public void malformedEventCallbackCanSkipInvalidJson(boolean async) { } else { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent", options)) { - assertEquals(STARTED, session.receiveEvents(TIMEOUT).iterator().next().toString()); + assertEquals(STARTED, session.receiveRawEvents(TIMEOUT).iterator().next().toString()); } } assertEquals(1, malformed.get()); @@ -516,14 +519,15 @@ public void oversizedEventsFailEvenWhenMalformedCallbackIsConfigured(boolean asy .block(TIMEOUT); assertNotNull(session); try { - StepVerifier.create(session.receiveEvents()).expectError().verify(TIMEOUT); + StepVerifier.create(session.receiveRawEvents()).expectError().verify(TIMEOUT); } finally { session.closeAsync().block(TIMEOUT); } } else { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent", options)) { - assertThrows(IllegalArgumentException.class, () -> session.receiveEvents(TIMEOUT).iterator().hasNext()); + assertThrows(IllegalArgumentException.class, + () -> session.receiveRawEvents(TIMEOUT).iterator().hasNext()); } } assertEquals(0, malformed.get()); From a592a109db5255acfd3438b3da821ac69362b403 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Mon, 26 Oct 2026 10:36:33 +0800 Subject: [PATCH 08/20] Use typed GPT Live client events and preserve explicit JSON nulls --- .../implementation/OpenAIJsonHelper.java | 5 +- .../voice/GptLiveFunctionToolSample.java | 16 +-- .../ai/agents/voice/GptLiveSampleUtils.java | 97 ++++++++++--------- .../ai/agents/voice/GptLiveSamplesTests.java | 31 ++++-- 4 files changed, 87 insertions(+), 62 deletions(-) diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/OpenAIJsonHelper.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/OpenAIJsonHelper.java index 6d7c41624bd0f..e52cc3464af44 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/OpenAIJsonHelper.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/OpenAIJsonHelper.java @@ -114,10 +114,7 @@ public static BinaryData toBinaryData(Object openAIObject) { } try { String json = OPENAI_MODEL_MAPPER.writeValueAsString(openAIObject); - try (JsonReader reader = JsonProviders.createReader(new StringReader(json))) { - reader.nextToken(); - return BinaryData.fromObject(reader.readUntyped()); - } + return BinaryData.fromObject(OPENAI_MODEL_MAPPER.readTree(json)); } catch (IOException e) { throw new RuntimeException("Failed to convert OpenAI type to BinaryData", e); } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java index df9e0beeac618..c44e21e315725 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java @@ -8,6 +8,9 @@ import com.azure.core.util.BinaryData; import com.azure.core.util.Configuration; import com.azure.identity.DefaultAzureCredentialBuilder; +import com.openai.models.live.ClientEvent; +import com.openai.models.live.ResponseItemCreateEvent; +import com.openai.models.responses.ResponseInputItem; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -79,7 +82,7 @@ static BinaryData weatherTool() { return BinaryData.fromObject(tool); } - static BinaryData toolResult(Map call) { + static ClientEvent toolResult(Map call) { if (!"get_weather".equals(GptLiveSampleUtils.string(call, "name"))) { throw new IllegalArgumentException("Unsupported sample tool: " + call.get("name")); } @@ -90,11 +93,12 @@ static BinaryData toolResult(Map call) { result.put("condition", "sunny"); result.put("temperature_f", 72); result.put("simulated", true); - Map item = new LinkedHashMap<>(); - item.put("type", "function_call_output"); - item.put("call_id", GptLiveSampleUtils.string(call, "call_id")); - item.put("output", BinaryData.fromObject(result).toString()); - return GptLiveSampleUtils.event("response.item.create", "item", item); + return ClientEvent.ofResponseItemCreate(ResponseItemCreateEvent.builder() + .item(ResponseInputItem.FunctionCallOutput.builder() + .callId(GptLiveSampleUtils.string(call, "call_id")) + .output(BinaryData.fromObject(result).toString()) + .build()) + .build()); } } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java index ec88180d33158..cfbef7c77543a 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java @@ -18,12 +18,18 @@ import com.azure.ai.agents.models.VoiceAgentSubProtocol; import com.azure.core.util.BinaryData; import com.azure.core.util.Configuration; -import com.azure.json.JsonProviders; -import com.azure.json.JsonWriter; import com.openai.core.JsonValue; +import com.openai.models.live.ClientEvent; +import com.openai.models.live.CommentaryAppendEvent; +import com.openai.models.live.InputAudioAppendEvent; +import com.openai.models.live.ResponseCreateEvent; +import com.openai.models.live.ResponseItemCreateEvent; import com.openai.models.live.ServerEvent; +import com.openai.models.live.SessionCloseEvent; +import com.openai.models.responses.EasyInputMessage; +import com.openai.models.responses.ResponseInputContent; +import com.openai.models.responses.ResponseInputText; import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.Reader; @@ -116,7 +122,7 @@ static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Li * without a tool handler. */ static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, List prompts, - Function, BinaryData> toolHandler) { + Function, ClientEvent> toolHandler) { Iterator pending = prompts.iterator(); runConversation(session, () -> { if (!pending.hasNext()) { @@ -172,14 +178,14 @@ static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient } private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Supplier nextPrompt, - Function, BinaryData> toolHandler) { + Function, ClientEvent> toolHandler) { runConversation(session, transcripts -> nextPrompt.get(), pcm -> { }, () -> { }, false, toolHandler); } private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Function nextPrompt, Consumer playAudio, Runnable drainAudio, boolean spoken, - Function, BinaryData> toolHandler) { + Function, ClientEvent> toolHandler) { ExecutorService workers = Executors.newFixedThreadPool(2, task -> { Thread thread = new Thread(task, "gpt-live-sync-conversation"); thread.setDaemon(true); @@ -205,7 +211,7 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses receiveEnded.set(true); } }); - List results = new ArrayList<>(); + List results = new ArrayList<>(); Future prompt = null; boolean started = false; boolean greetingComplete = !spoken; @@ -217,7 +223,7 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses long deadline = System.nanoTime() + TIMEOUT.toNanos(); long nextAudio = 0; long quietSince = 0; - BinaryData silence = audio(new byte[960]); + ClientEvent silence = audio(new byte[960]); try { while (true) { ServerEvent event = pending.poll(20, TimeUnit.MILLISECONDS); @@ -269,11 +275,11 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses if (++continuations > 8) { throw new IllegalStateException("Sample tool continuation limit exceeded."); } - for (BinaryData result : results) { + for (ClientEvent result : results) { session.sendEvent(result); } results.clear(); - session.sendEvent(event("response.create")); + session.sendEvent(responseCreate()); } } else if (toolHandler != null && "response.output_item.done".equals(type)) { Map item = object(backend.get("item")); @@ -319,14 +325,14 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses deadline = System.nanoTime() + TIMEOUT.toNanos(); if (text == null) { closing = true; - session.sendEvent(event("session.close")); + session.sendEvent(closeSession()); } else { awaitingResponse = true; if (!spoken) { System.out.print("Assistant: "); } session.sendEvent(userMessage(text)); - session.sendEvent(event("response.create")); + session.sendEvent(responseCreate()); } } if ((!started || !greetingComplete || awaitingResponse || closing) && now >= deadline) { @@ -383,7 +389,7 @@ static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient } static Mono runInteractiveConversation(Flux events, - Function> sendEvent, Supplier closeDetails, Reader input, + Function> sendEvent, Supplier closeDetails, Reader input, Consumer playAudio, Runnable drainAudio) { return Mono.defer(() -> { System.out.println("Type a message and press Enter. Blank line, 'exit' or 'quit' ends the session."); @@ -403,14 +409,14 @@ private static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyn } private static Mono runConversation(Flux events, - Function> sendEvent, Supplier closeDetails, + Function> sendEvent, Supplier closeDetails, Function> nextPrompt, Consumer playAudio, Runnable drainAudio, boolean spoken) { return runConversation(events, sendEvent, closeDetails, nextPrompt, playAudio, drainAudio, spoken, null); } private static Mono runConversation(Flux events, - Function> sendEvent, Supplier closeDetails, + Function> sendEvent, Supplier closeDetails, Function> nextPrompt, Consumer playAudio, Runnable drainAudio, boolean spoken, AudioProcessor microphone) { return Mono.defer(() -> { @@ -500,7 +506,7 @@ private static Mono runConversation(Flux events, .then(Mono.fromRunnable(transcripts::endUtterance)) : microphone == null ? Mono.empty() : sendEvent.apply(userMessage("Greet me briefly with: Hi, how can I help you today?")) - .then(sendEvent.apply(event("response.create"))) + .then(sendEvent.apply(responseCreate())) .then(Mono.fromRunnable(() -> System.out.println( "Opening greeting requested. Speak into the microphone. Press Enter to end the call."))))) .then(Mono.defer(() -> prompt(sendEvent, nextPrompt, responseCompleted, spoken, transcripts))) @@ -508,7 +514,7 @@ private static Mono runConversation(Flux events, closeRequested.set(true); stopAudio.tryEmitEmpty(); return (interactive ? audioStopped.asMono() : Mono.empty()) - .then(sendEvent.apply(event("session.close"))) + .then(sendEvent.apply(closeSession())) .then(closed.asMono()).timeout(TIMEOUT); })); return interactive ? Mono.firstWithSignal(receive, audio, renderTranscripts, conversation) @@ -516,7 +522,7 @@ private static Mono runConversation(Flux events, }); } - private static Mono prompt(Function> sendEvent, + private static Mono prompt(Function> sendEvent, Function> nextPrompt, AtomicReference> responseCompleted, boolean spoken, AudioTranscriptPrinter transcripts) { @@ -528,7 +534,7 @@ private static Mono prompt(Function> sendEvent, System.out.flush(); } return sendEvent.apply(userMessage(text)) - .then(sendEvent.apply(event("response.create"))) + .then(sendEvent.apply(responseCreate())) .then(completion.asMono()).timeout(TIMEOUT) .thenReturn(true); }).defaultIfEmpty(false) @@ -536,9 +542,9 @@ private static Mono prompt(Function> sendEvent, ? Mono.defer(() -> prompt(sendEvent, nextPrompt, responseCompleted, spoken, transcripts)) : Mono.empty()); } - static Flux silenceFrames() { + static Flux silenceFrames() { // Twenty milliseconds of 24-kHz mono PCM16LE. - BinaryData silence = audio(new byte[960]); + ClientEvent silence = audio(new byte[960]); return Mono.just(silence) .concatWith(Mono.just(silence).delayElement(Duration.ofMillis(20)).repeat()); } @@ -646,34 +652,35 @@ static BinaryData event(String type, String field, Object value) { return BinaryData.fromObject(event); } - static BinaryData userMessage(String text) { - Map part = new LinkedHashMap<>(); - part.put("type", "input_text"); - part.put("text", text); - Map item = new LinkedHashMap<>(); - item.put("type", "message"); - item.put("role", "user"); - item.put("content", java.util.Collections.singletonList(part)); - return event("response.item.create", "item", item); + static ClientEvent userMessage(String text) { + return ClientEvent.ofResponseItemCreate(ResponseItemCreateEvent.builder() + .item(EasyInputMessage.builder() + .role(EasyInputMessage.Role.USER) + .type(EasyInputMessage.Type.MESSAGE) + .contentOfResponseInputMessageContentList(java.util.Collections.singletonList( + ResponseInputContent.ofInputText(ResponseInputText.builder().text(text).build()))) + .build()) + .build()); } - static BinaryData audio(byte[] pcm) { - return event("session.input_audio.append", "audio", Base64.getEncoder().encodeToString(pcm)); + static ClientEvent audio(byte[] pcm) { + return ClientEvent.ofSessionInputAudioAppend(InputAudioAppendEvent.builder() + .audio(Base64.getEncoder().encodeToString(pcm)).build()); } - static BinaryData commentary(String content) { - try (ByteArrayOutputStream output = new ByteArrayOutputStream(); - JsonWriter writer = JsonProviders.createWriter(output)) { - writer.writeStartObject() - .writeStringField("type", "session.commentary.append") - .writeStringField("content", content) - .writeNullField("delegation_id") - .writeEndObject() - .flush(); - return BinaryData.fromBytes(output.toByteArray()); - } catch (IOException error) { - throw new IllegalStateException("Unable to serialize GPT Live commentary.", error); - } + static ClientEvent commentary(String content) { + return ClientEvent.ofSessionCommentaryAppend(CommentaryAppendEvent.builder() + .content(content) + .delegationId(java.util.Optional.empty()) + .build()); + } + + static ClientEvent responseCreate() { + return ClientEvent.ofResponseCreate(ResponseCreateEvent.builder().build()); + } + + static ClientEvent closeSession() { + return ClientEvent.ofSessionClose(SessionCloseEvent.builder().build()); } static Map backendEvent(ServerEvent event) { diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java index 38c2a6c6e9b88..74453674bf2ca 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java @@ -11,6 +11,7 @@ import com.azure.core.credential.AccessToken; import com.azure.core.util.BinaryData; import com.azure.core.util.Configuration; +import com.openai.models.live.ClientEvent; import com.openai.models.live.ServerEvent; import java.io.ByteArrayOutputStream; import java.io.FilterReader; @@ -904,10 +905,9 @@ public void silentAudioToleratesSendsSlowerThanItsFrameRate() { .verifyComplete(); } - private void assertSilentAudioFrame(BinaryData frame) { - Map event = GptLiveSampleUtils.object(frame); - assertEquals("session.input_audio.append", event.get("type")); - assertArrayEquals(new byte[960], Base64.getDecoder().decode(GptLiveSampleUtils.string(event, "audio"))); + private void assertSilentAudioFrame(ClientEvent frame) { + assertTrue(frame.isSessionInputAudioAppend()); + assertArrayEquals(new byte[960], Base64.getDecoder().decode(frame.asSessionInputAudioAppend().audio())); } @ParameterizedTest @@ -1070,12 +1070,29 @@ public void prematureClosureDoesNotReportSuccessfulConversation(boolean async) { } } + @Test + public void typedCommentaryPreservesExplicitNullDelegationId() throws IOException { + ClientEvent event = GptLiveSampleUtils.commentary("Hello"); + assertTrue(event.isSessionCommentaryAppend()); + BinaryData data = OpenAIJsonHelper.toBinaryData(event); + Map command = GptLiveSampleUtils.object(data); + assertEquals("session.commentary.append", command.get("type")); + assertEquals("Hello", command.get("content")); + assertTrue(command.containsKey("delegation_id")); + assertEquals(null, command.get("delegation_id")); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + try (com.azure.json.JsonWriter writer = com.azure.json.JsonProviders.createWriter(output)) { + data.writeTo(writer); + } + assertEquals(command, GptLiveSampleUtils.object(BinaryData.fromBytes(output.toByteArray()))); + } + @Test public void nativeAudioEncodingAndDecodingPreservePcmBytes() { byte[] pcm = new byte[] { 0, 1, -1, 127 }; - Map command = GptLiveSampleUtils.object(GptLiveSampleUtils.audio(pcm)); - assertEquals("session.input_audio.append", command.get("type")); - assertArrayEquals(pcm, Base64.getDecoder().decode((String) command.get("audio"))); + ClientEvent command = GptLiveSampleUtils.audio(pcm); + assertTrue(command.isSessionInputAudioAppend()); + assertArrayEquals(pcm, Base64.getDecoder().decode(command.asSessionInputAudioAppend().audio())); ServerEvent event = typedEvent( GptLiveSampleUtils.event("session.output_audio.delta", "delta", Base64.getEncoder().encodeToString(pcm))); assertArrayEquals(pcm, GptLiveSampleUtils.decodeAudio(event)); From b6f12171a6e5689149bffead82916ed59136ad7f Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Mon, 26 Oct 2026 14:39:16 +0800 Subject: [PATCH 09/20] Refactor and organize GPT Live sample conversation helpers --- .../GptLiveAudioConversationAsyncSample.java | 2 +- .../voice/GptLiveFunctionToolSample.java | 2 +- .../ai/agents/voice/GptLiveSampleUtils.java | 386 +++++++++--------- .../GptLiveTextConversationAsyncSample.java | 4 +- .../voice/GptLiveTextConversationSample.java | 4 +- .../ai/agents/voice/GptLiveSamplesTests.java | 102 +++-- 6 files changed, 276 insertions(+), 224 deletions(-) diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java index 10f71f7fb2241..a011f6a5ae106 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java @@ -62,7 +62,7 @@ public static void main(String[] args) { = GptLiveSampleUtils.createAgent(builder, model, backendModel, "audio-async", false, keepAgent)) { Mono.usingWhen(agent.builder.beta().buildBetaVoiceAgentLiveWebSocketAsyncClient() .openWebSocketSession(agent.name, new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store)), - GptLiveSampleUtils::runConversation, + GptLiveSampleUtils::runAudioConversationAsync, BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(); diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java index c44e21e315725..74417bfa77622 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java @@ -61,7 +61,7 @@ public static void main(String[] args) { = GptLiveSampleUtils.createAgent(builder, model, backendModel, "tools", true, keepAgent); BetaVoiceAgentLiveWebSocketSessionClient session = agent.builder.beta() .buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession(agent.name)) { - GptLiveSampleUtils.runConversation(session, + GptLiveSampleUtils.runToolConversation(session, Collections.singletonList(args.length == 0 ? "What is the weather in Seattle?" : args[0]), GptLiveFunctionToolSample::toolResult); } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java index cfbef7c77543a..c200b0574f99e 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java @@ -7,7 +7,7 @@ import com.azure.ai.agents.AgentsClientBuilder; import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; +import com.azure.ai.agents.implementation.OpenAIJsonHelper; import com.azure.ai.agents.models.CreateAgentVersionInput; import com.azure.ai.agents.models.LiveInitialSessionAudioParam; import com.azure.ai.agents.models.LiveResponsesDelegationParam; @@ -29,6 +29,8 @@ import com.openai.models.responses.EasyInputMessage; import com.openai.models.responses.ResponseInputContent; import com.openai.models.responses.ResponseInputText; +import com.openai.models.responses.ResponseOutputItem; +import com.openai.models.responses.ResponseStreamEvent; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -50,9 +52,11 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Predicate; import java.util.function.Supplier; import javax.sound.sampled.AudioFormat; import javax.sound.sampled.AudioSystem; @@ -78,80 +82,52 @@ static String requireSetting(Configuration configuration, String name) { return value; } - /** - * Microphone entry point called by {@link GptLiveAudioConversationAsyncSample}. - */ - static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session) { - return Mono.defer(() -> runConversation(session, - new AudioProcessor( - null, null, pcm -> session.sendEvent(audio(pcm.toBytes()))), System.in)); - } - - static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, - AudioProcessor processor, InputStream input) { - return Mono.usingWhen(Mono.just(processor), - audio -> Mono.firstWithSignal(runConversation(session.receiveEvents(), session::sendEvent, - () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), - transcripts -> VoiceAgentLiveAudioConversationAsyncSample.waitForEnter(input) - .then(closeAudio(audio)).then(Mono.empty()), - audio::queueAudio, () -> { }, true, audio), audio.failureSignal()), GptLiveSampleUtils::closeAudio, - (audio, error) -> closeAudio(audio), GptLiveSampleUtils::closeAudio); - } - - static byte[] decodeAudio(ServerEvent event) { - byte[] pcm = Base64.getDecoder().decode(event.asSessionOutputAudioDelta().delta()); - if (pcm.length % 2 != 0) { - throw new IllegalArgumentException("GPT Live PCM16 audio must contain complete two-byte samples."); - } - return pcm; - } - - private static Mono closeAudio(AudioProcessor audio) { - return Mono.fromRunnable(audio::close).subscribeOn(Schedulers.boundedElastic()); - } + // #region Synchronous conversation /** * Scripted-text entry point called by {@link GptLiveTextConversationSample} when command-line prompts are supplied. */ - static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, List prompts) { - runConversation(session, prompts, null); + static void runTextConversation(BetaVoiceAgentLiveWebSocketSessionClient session, List prompts) { + runToolConversation(session, prompts, null); } /** * Called by {@link GptLiveFunctionToolSample}; also serves scripted {@link GptLiveTextConversationSample} calls * without a tool handler. */ - static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, List prompts, + static void runToolConversation(BetaVoiceAgentLiveWebSocketSessionClient session, List prompts, Function, ClientEvent> toolHandler) { Iterator pending = prompts.iterator(); - runConversation(session, () -> { + runConversation(session.receiveEvents(TIMEOUT), session::sendEvent, + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), transcripts -> { if (!pending.hasNext()) { return null; } String prompt = pending.next(); System.out.println("You: " + prompt); return prompt; - }, toolHandler); + }, pcm -> { }, () -> { }, false, toolHandler); } /** * Interactive entry point called by {@link GptLiveTextConversationSample} when no command-line prompts are supplied. */ - static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Reader input) { + static void runInteractiveConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Reader input) { try (VoiceAgentRealtimeSampleUtils.SpeakerPlayer player = new VoiceAgentRealtimeSampleUtils.SpeakerPlayer()) { - runConversation(session, input, player::play, player::drain); + runInteractiveConversation(session, input, player::play, player::drain); player.drain(); System.out.printf("(received %.2fs of reply audio%s)%n", player.getSecondsReceived(), player.isEnabled() ? " and played it" : ""); } } - static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Reader input, + static void runInteractiveConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Reader input, Consumer playAudio, Runnable drainAudio) { System.out.println("Type a message and press Enter. Blank line, 'exit' or 'quit' ends the session."); System.out.println("Interactive voice streams silent audio while you type; voice usage is billable."); BufferedReader console = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); - runConversation(session, transcripts -> { + runConversation(session.receiveEvents(TIMEOUT), session::sendEvent, + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), transcripts -> { try { return readPromptBlocking(console, transcripts); } catch (IOException error) { @@ -160,32 +136,41 @@ static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Re }, playAudio, drainAudio, true, null); } - /** - * Scripted-text entry point called by {@link GptLiveTextConversationAsyncSample} with command-line prompts. - */ - static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, List prompts) { - return Mono.defer(() -> { - Iterator pending = prompts.iterator(); - return runConversation(session, transcripts -> Mono.defer(() -> { - if (!pending.hasNext()) { - return Mono.empty(); + static void runConversation(Iterable events, Consumer sendEvent, + Supplier closeDetails, Function nextPrompt, + Consumer playAudio, Runnable drainAudio, boolean spoken, + Function, ClientEvent> toolHandler) { + List results = new ArrayList<>(); + AtomicInteger continuations = new AtomicInteger(); + runConversationCore(events, sendEvent, closeDetails, nextPrompt, playAudio, drainAudio, spoken, backend -> { + if (toolHandler != null && backend.isOutputItemDone()) { + ResponseOutputItem item = backend.asOutputItemDone().item(); + if (item.isFunctionCall()) { + results.add(toolHandler.apply(object(OpenAIJsonHelper.toBinaryData(item.asFunctionCall())))); } - String prompt = pending.next(); - System.out.println("You: " + prompt); - return Mono.just(prompt); - }), pcm -> { }, () -> { }, false); + } else if (backend.isCompleted()) { + if (results.isEmpty()) { + continuations.set(0); + return false; + } + if (continuations.incrementAndGet() > 8) { + throw new IllegalStateException("Sample tool continuation limit exceeded."); + } + for (ClientEvent result : results) { + sendEvent.accept(result); + } + results.clear(); + sendEvent.accept(responseCreate()); + return true; + } + return false; }); } - private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Supplier nextPrompt, - Function, ClientEvent> toolHandler) { - runConversation(session, transcripts -> nextPrompt.get(), pcm -> { }, () -> { }, false, toolHandler); - } - - private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient session, - Function nextPrompt, Consumer playAudio, Runnable drainAudio, - boolean spoken, - Function, ClientEvent> toolHandler) { + private static void runConversationCore(Iterable events, Consumer sendEvent, + Supplier closeDetails, Function nextPrompt, + Consumer playAudio, Runnable drainAudio, boolean spoken, + Predicate handleResponse) { ExecutorService workers = Executors.newFixedThreadPool(2, task -> { Thread thread = new Thread(task, "gpt-live-sync-conversation"); thread.setDaemon(true); @@ -197,7 +182,7 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses AudioTranscriptPrinter transcripts = new AudioTranscriptPrinter(System.console() != null, spoken); Future receiver = workers.submit(() -> { try { - for (ServerEvent event : session.receiveEvents(TIMEOUT)) { + for (ServerEvent event : events) { pending.put(event); if (event.isSessionClosed()) { break; @@ -211,7 +196,6 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses receiveEnded.set(true); } }); - List results = new ArrayList<>(); Future prompt = null; boolean started = false; boolean greetingComplete = !spoken; @@ -219,7 +203,6 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses boolean greetingTranscript = false; boolean awaitingResponse = false; boolean closing = false; - int continuations = 0; long deadline = System.nanoTime() + TIMEOUT.toNanos(); long nextAudio = 0; long quietSince = 0; @@ -246,46 +229,32 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses } else { printEvent(event); } - Map backend = backendEvent(event); - String type = event.isResponseEvent() ? type(backend) : ""; + ResponseStreamEvent backend = backendEvent(event); if (event.isSessionStarted()) { if (started) { throw new IllegalStateException("Duplicate session.started event."); } started = true; if (spoken) { - session.sendEvent(silence); + sendEvent.accept(silence); nextAudio = now + TimeUnit.MILLISECONDS.toNanos(20); - session.sendEvent(commentary("Hi, how can I help you today?")); + sendEvent.accept(commentary("Hi, how can I help you today?")); } - } else if ("response.completed".equals(type)) { + } else if (backend != null && backend.isCompleted()) { if (!awaitingResponse && spoken && started) { continue; } if (!awaitingResponse || closing) { throw new IllegalStateException("Unexpected GPT Live response lifecycle event."); } - if (results.isEmpty()) { + if (!handleResponse.test(backend)) { awaitingResponse = false; - continuations = 0; if (!spoken) { System.out.println(); } - } else { - if (++continuations > 8) { - throw new IllegalStateException("Sample tool continuation limit exceeded."); - } - for (ClientEvent result : results) { - session.sendEvent(result); - } - results.clear(); - session.sendEvent(responseCreate()); - } - } else if (toolHandler != null && "response.output_item.done".equals(type)) { - Map item = object(backend.get("item")); - if ("function_call".equals(type(item))) { - results.add(toolHandler.apply(item)); } + } else if (backend != null) { + handleResponse.test(backend); } else if (event.isSessionClosed()) { String reason = event.asSessionClosed().reason().toString(); if (started && ("remote_hangup".equals(reason) @@ -294,7 +263,7 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses return; } throw new IllegalStateException("GPT Live closed before all turns completed. Event: " + event - + "; transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason()); + + "; " + closeDetails.get()); } } else if (receiveEnded.get()) { if (receiveFailure.get() != null) { @@ -304,7 +273,7 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses } if (started && spoken && !closing && now >= nextAudio) { - session.sendEvent(silence); + sendEvent.accept(silence); nextAudio = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(20); } if (spoken && greetingAudio && greetingTranscript && quietSince != 0 @@ -325,14 +294,14 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses deadline = System.nanoTime() + TIMEOUT.toNanos(); if (text == null) { closing = true; - session.sendEvent(closeSession()); + sendEvent.accept(closeSession()); } else { awaitingResponse = true; if (!spoken) { System.out.print("Assistant: "); } - session.sendEvent(userMessage(text)); - session.sendEvent(responseCreate()); + sendEvent.accept(userMessage(text)); + sendEvent.accept(responseCreate()); } } if ((!started || !greetingComplete || awaitingResponse || closing) && now >= deadline) { @@ -357,14 +326,75 @@ private static void runConversation(BetaVoiceAgentLiveWebSocketSessionClient ses } } + private static String readPromptBlocking(BufferedReader reader, AudioTranscriptPrinter transcripts) throws IOException { + transcripts.beginInput(); + try { + String line = reader.readLine(); + if (line == null) { + return null; + } + String text = line.trim(); + return text.isEmpty() || "exit".equalsIgnoreCase(text) || "quit".equalsIgnoreCase(text) ? null : text; + } finally { + transcripts.endInput(); + } + } + + // #endregion + + // #region Asynchronous conversation + + /** + * Microphone entry point called by {@link GptLiveAudioConversationAsyncSample}. + */ + static Mono runAudioConversationAsync(BetaVoiceAgentLiveWebSocketSessionAsyncClient session) { + return Mono.defer(() -> runAudioConversationAsync(session, + new AudioProcessor( + null, null, pcm -> session.sendEvent(audio(pcm.toBytes()))), System.in)); + } + + static Mono runAudioConversationAsync(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, + AudioProcessor processor, InputStream input) { + return Mono.usingWhen(Mono.just(processor), + audio -> Mono.firstWithSignal(runConversationAsync(session.receiveEvents(), session::sendEvent, + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), + transcripts -> VoiceAgentLiveAudioConversationAsyncSample.waitForEnter(input) + .then(closeAudioAsync(audio)).then(Mono.empty()), + audio::queueAudio, () -> { }, true, audio), audio.failureSignalAsync()), GptLiveSampleUtils::closeAudioAsync, + (audio, error) -> closeAudioAsync(audio), GptLiveSampleUtils::closeAudioAsync); + } + + private static Mono closeAudioAsync(AudioProcessor audio) { + return Mono.fromRunnable(audio::close).subscribeOn(Schedulers.boundedElastic()); + } + + /** + * Scripted-text entry point called by {@link GptLiveTextConversationAsyncSample} with command-line prompts. + */ + static Mono runTextConversationAsync(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, List prompts) { + return Mono.defer(() -> { + Iterator pending = prompts.iterator(); + return runConversationAsync(session.receiveEvents(), session::sendEvent, + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), + transcripts -> Mono.defer(() -> { + if (!pending.hasNext()) { + return Mono.empty(); + } + String prompt = pending.next(); + System.out.println("You: " + prompt); + return Mono.just(prompt); + }), pcm -> { }, () -> { }, false, null); + }); + } + /** * Interactive entry point called by {@link GptLiveTextConversationAsyncSample} without command-line prompts. */ - static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input) { + static Mono runInteractiveConversationAsync(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input) { return Mono.usingWhen( Mono.fromCallable(VoiceAgentRealtimeSampleUtils.SpeakerPlayer::new) .subscribeOn(Schedulers.boundedElastic()), - player -> runConversation(session, input, player::play, player::drain), + player -> runInteractiveConversationAsync(session, input, player::play, player::drain), player -> Mono.fromRunnable(() -> { boolean speakerEnabled = player.isEnabled(); try { @@ -376,57 +406,54 @@ static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient }).subscribeOn(Schedulers.boundedElastic())); } - static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input, - Consumer playAudio) { - return runConversation(session, input, playAudio, () -> { }); - } - - static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input, + static Mono runInteractiveConversationAsync(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input, Consumer playAudio, Runnable drainAudio) { - return Mono.defer(() -> runInteractiveConversation(session.receiveEvents(), session::sendEvent, + return Mono.defer(() -> runInteractiveConversationAsync(session.receiveEvents(), session::sendEvent, () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), input, playAudio, drainAudio)); } - static Mono runInteractiveConversation(Flux events, + static Mono runInteractiveConversationAsync(Flux events, Function> sendEvent, Supplier closeDetails, Reader input, Consumer playAudio, Runnable drainAudio) { return Mono.defer(() -> { System.out.println("Type a message and press Enter. Blank line, 'exit' or 'quit' ends the session."); System.out.println("Interactive voice streams silent audio while you type; voice usage is billable."); BufferedReader console = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); - return runConversation(events, sendEvent, closeDetails, transcripts -> readPrompt(console, transcripts), - playAudio, drainAudio, true); + return runConversationAsync(events, sendEvent, closeDetails, transcripts -> readPromptAsync(console, transcripts), + playAudio, drainAudio, true, null); }); } - private static Mono runConversation(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, - Function> nextPrompt, - Consumer playAudio, Runnable drainAudio, boolean spoken) { - return Mono.defer(() -> runConversation(session.receiveEvents(), session::sendEvent, - () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), - nextPrompt, playAudio, drainAudio, spoken)); - } - - private static Mono runConversation(Flux events, + private static Mono runConversationAsync(Flux events, Function> sendEvent, Supplier closeDetails, Function> nextPrompt, - Consumer playAudio, Runnable drainAudio, boolean spoken) { - return runConversation(events, sendEvent, closeDetails, nextPrompt, playAudio, drainAudio, spoken, null); + Consumer playAudio, Runnable drainAudio, boolean spoken, AudioProcessor microphone) { + Mono opening = microphone == null ? Mono.empty() + : Mono.defer(() -> sendEvent.apply(userMessage("Greet me briefly with: Hi, how can I help you today?")) + .then(sendEvent.apply(responseCreate())) + .then(Mono.fromRunnable(() -> System.out.println( + "Opening greeting requested. Speak into the microphone. Press Enter to end the call.")))); + return runConversationCoreAsync(events, sendEvent, closeDetails, nextPrompt, playAudio, drainAudio, spoken, + spoken && microphone == null, microphone == null && System.console() != null, + () -> { + if (microphone != null) { + microphone.start(); + } + }, opening, reason -> microphone != null); } - private static Mono runConversation(Flux events, + private static Mono runConversationCoreAsync(Flux events, Function> sendEvent, Supplier closeDetails, Function> nextPrompt, - Consumer playAudio, Runnable drainAudio, boolean spoken, AudioProcessor microphone) { + Consumer playAudio, Runnable drainAudio, boolean spoken, boolean interactive, boolean inputEchoed, + Runnable onStarted, Mono opening, Predicate acceptClose) { return Mono.defer(() -> { - boolean interactive = spoken && microphone == null; AtomicBoolean finalized = new AtomicBoolean(); AtomicBoolean started = new AtomicBoolean(); AtomicBoolean closeRequested = new AtomicBoolean(); AtomicBoolean remoteHangup = new AtomicBoolean(); - AudioTranscriptPrinter transcripts = new AudioTranscriptPrinter( - microphone == null && System.console() != null, interactive); + AudioTranscriptPrinter transcripts = new AudioTranscriptPrinter(inputEchoed, interactive); Greeting greeting = new Greeting(); Sinks.Empty ready = Sinks.empty(); Sinks.Empty closed = Sinks.empty(); @@ -450,16 +477,14 @@ private static Mono runConversation(Flux events, } else { printEvent(event); } - String type = event.isResponseEvent() ? type(backendEvent(event)) : ""; + ResponseStreamEvent backend = backendEvent(event); if (event.isSessionStarted()) { if (!started.compareAndSet(false, true)) { throw new IllegalStateException("Duplicate session.started event."); } - if (microphone != null) { - microphone.start(); - } + onStarted.run(); ready.tryEmitEmpty(); - } else if ("response.completed".equals(type)) { + } else if (backend != null && backend.isCompleted()) { Sinks.Empty completion = responseCompleted.getAndSet(null); if (spoken && started.get() && completion == null) { return; @@ -474,13 +499,14 @@ private static Mono runConversation(Flux events, } else if (event.isSessionClosed()) { String reason = event.asSessionClosed().reason().toString(); boolean remote = started.get() && "remote_hangup".equals(reason); - boolean requested = microphone != null + boolean accepted = acceptClose.test(reason); + boolean requested = accepted || closeRequested.get() && "close_requested".equals(reason); if (!requested && !remote) { throw new IllegalStateException("GPT Live closed before all turns completed. Event: " + event + "; " + closeDetails.get()); } - remoteHangup.set(remote || microphone != null); + remoteHangup.set(remote || accepted); System.out.println("Live session closed (" + reason + ")."); finalized.set(true); closed.tryEmitEmpty(); @@ -490,26 +516,22 @@ private static Mono runConversation(Flux events, .then(Mono.defer(() -> remoteHangup.get() ? Mono.empty() : finalized.get() ? Mono.never() : Mono.error(new IllegalStateException("GPT Live disconnected without session.closed.")))); Mono audio = ready.asMono() - .thenMany(silenceFrames().takeUntilOther(stopAudio.asMono()) + .thenMany(silenceFramesAsync().takeUntilOther(stopAudio.asMono()) .concatMap(frame -> sendEvent.apply(frame).doOnSuccess(ignored -> audioStarted.tryEmitEmpty()), 1)) .then() .doOnSuccess(ignored -> audioStopped.tryEmitEmpty()) .then(Mono.never()); - Mono renderTranscripts = greeting.quietPeriods() + Mono renderTranscripts = greeting.quietPeriodsAsync() .doOnNext(ignored -> transcripts.endUtterance()).then(); Mono conversation = ready.asMono().timeout(TIMEOUT) .then(Mono.defer(() -> interactive ? audioStarted.asMono().timeout(TIMEOUT) : Mono.empty())) .then(Mono.defer(() -> interactive ? sendEvent.apply(commentary("Hi, how can I help you today?")) .timeout(TIMEOUT) - .then(greeting.awaitPlayback(drainAudio)) + .then(greeting.awaitPlaybackAsync(drainAudio)) .then(Mono.fromRunnable(transcripts::endUtterance)) - : microphone == null ? Mono.empty() - : sendEvent.apply(userMessage("Greet me briefly with: Hi, how can I help you today?")) - .then(sendEvent.apply(responseCreate())) - .then(Mono.fromRunnable(() -> System.out.println( - "Opening greeting requested. Speak into the microphone. Press Enter to end the call."))))) - .then(Mono.defer(() -> prompt(sendEvent, nextPrompt, responseCompleted, spoken, transcripts))) + : opening)) + .then(Mono.defer(() -> promptAsync(sendEvent, nextPrompt, responseCompleted, spoken, transcripts))) .then(Mono.defer(() -> { closeRequested.set(true); stopAudio.tryEmitEmpty(); @@ -522,7 +544,7 @@ private static Mono runConversation(Flux events, }); } - private static Mono prompt(Function> sendEvent, + private static Mono promptAsync(Function> sendEvent, Function> nextPrompt, AtomicReference> responseCompleted, boolean spoken, AudioTranscriptPrinter transcripts) { @@ -539,10 +561,10 @@ private static Mono prompt(Function> sendEvent, .thenReturn(true); }).defaultIfEmpty(false) .flatMap(keepGoing -> keepGoing - ? Mono.defer(() -> prompt(sendEvent, nextPrompt, responseCompleted, spoken, transcripts)) : Mono.empty()); + ? Mono.defer(() -> promptAsync(sendEvent, nextPrompt, responseCompleted, spoken, transcripts)) : Mono.empty()); } - static Flux silenceFrames() { + static Flux silenceFramesAsync() { // Twenty milliseconds of 24-kHz mono PCM16LE. ClientEvent silence = audio(new byte[960]); return Mono.just(silence) @@ -567,24 +589,24 @@ void onTranscript() { progress.tryEmitNext(received); } - Mono awaitPlayback(Runnable drainAudio) { + Mono awaitPlaybackAsync(Runnable drainAudio) { // Output can continue with silent PCM after speaking; only speech activity restarts this timer. - return quietPeriods().next() + return quietPeriodsAsync().next() .timeout(TIMEOUT) .then(Mono.fromRunnable(drainAudio).subscribeOn(Schedulers.boundedElastic())); } - Flux quietPeriods() { + Flux quietPeriodsAsync() { return progress.asFlux().filter(parts -> parts == 3) .switchMap(parts -> Mono.delay(Duration.ofSeconds(1))); } } - static Mono readPrompt(Reader input) { - return readPrompt(input, new AudioTranscriptPrinter()); + static Mono readPromptAsync(Reader input) { + return readPromptAsync(input, new AudioTranscriptPrinter()); } - private static Mono readPrompt(Reader input, AudioTranscriptPrinter transcripts) { + private static Mono readPromptAsync(Reader input, AudioTranscriptPrinter transcripts) { return Mono.defer(() -> { BufferedReader reader = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); return Mono.fromCallable(() -> readPromptBlocking(reader, transcripts)).subscribeOn(Schedulers.boundedElastic()) @@ -592,18 +614,16 @@ private static Mono readPrompt(Reader input, AudioTranscriptPrinter tran }); } - private static String readPromptBlocking(BufferedReader reader, AudioTranscriptPrinter transcripts) throws IOException { - transcripts.beginInput(); - try { - String line = reader.readLine(); - if (line == null) { - return null; - } - String text = line.trim(); - return text.isEmpty() || "exit".equalsIgnoreCase(text) || "quit".equalsIgnoreCase(text) ? null : text; - } finally { - transcripts.endInput(); + // #endregion + + // #region Shared utilities + + static byte[] decodeAudio(ServerEvent event) { + byte[] pcm = Base64.getDecoder().decode(event.asSessionOutputAudioDelta().delta()); + if (pcm.length % 2 != 0) { + throw new IllegalArgumentException("GPT Live PCM16 audio must contain complete two-byte samples."); } + return pcm; } private static boolean hasSpeech(byte[] pcm) { @@ -627,10 +647,6 @@ private static boolean hasSpeech(byte[] pcm) { return (Map) value; } - static String type(Map event) { - return string(event, "type"); - } - static String string(Map event, String field) { Object value = event.get(field); if (!(value instanceof String)) { @@ -639,10 +655,6 @@ static String string(Map event, String field) { return (String) value; } - static BinaryData event(String type) { - return event(type, null, null); - } - static BinaryData event(String type, String field, Object value) { Map event = new LinkedHashMap<>(); event.put("type", type); @@ -683,10 +695,10 @@ static ClientEvent closeSession() { return ClientEvent.ofSessionClose(SessionCloseEvent.builder().build()); } - static Map backendEvent(ServerEvent event) { + static ResponseStreamEvent backendEvent(ServerEvent event) { return event.isResponseEvent() - ? JsonValue.from(event.asResponseEvent().event()._additionalProperties()).convert(Map.class) - : java.util.Collections.emptyMap(); + ? JsonValue.from(event.asResponseEvent().event()._additionalProperties()).convert(ResponseStreamEvent.class) + : null; } static void checkError(ServerEvent event) { @@ -703,21 +715,21 @@ static void checkError(ServerEvent event) { if (event.isResponseEvent()) { checkError(backendEvent(event)); } else if (event._json().isPresent()) { - checkError(event._json().get().convert(Map.class)); + checkError(event._json().get().convert(ResponseStreamEvent.class)); } } - static void checkError(Map event) { - if ("error".equals(type(event)) || "response.failed".equals(type(event)) - || "response.incomplete".equals(type(event))) { - Object error = event.get("error"); + static void checkError(ResponseStreamEvent event) { + if (event.isError() || event.isFailed() || event.isIncomplete()) { + BinaryData data = OpenAIJsonHelper.toBinaryData(event); + Object error = object(data).get("error"); if (error instanceof Map && ("content_filter".equals(((Map) error).get("code")) || "content_filter".equals(((Map) error).get("type")))) { throw new IllegalStateException("GPT Live service error: generation stopped by service moderation " + "(content_filter). The call has ended; review the conversation and service diagnostics. Event: " - + BinaryData.fromObject(event)); + + data); } - throw new IllegalStateException("GPT Live service error: " + BinaryData.fromObject(event)); + throw new IllegalStateException("GPT Live service error: " + data); } } @@ -734,10 +746,9 @@ static void printEvent(ServerEvent event) { } else if (event.isSessionOutputTranscriptDelta()) { System.out.print(event.asSessionOutputTranscriptDelta().delta()); } else if (event.isResponseEvent()) { - Map backend = backendEvent(event); - checkError(backend); - if ("response.output_text.delta".equals(type(backend))) { - System.out.print(string(backend, "delta")); + ResponseStreamEvent backend = backendEvent(event); + if (backend.isOutputTextDelta()) { + System.out.print(backend.asOutputTextDelta().delta()); } } } @@ -759,21 +770,13 @@ static final class AudioProcessor implements AutoCloseable { private Thread captureThread; private Thread playbackThread; - AudioProcessor(BetaVoiceAgentWebSocketSessionAsyncClient session) { - this(session, null, null); - } - - AudioProcessor(BetaVoiceAgentWebSocketSessionAsyncClient session, TargetDataLine microphone, SourceDataLine speaker) { - this(microphone, speaker, pcm -> session.appendInputAudio(pcm)); - } - AudioProcessor(TargetDataLine microphone, SourceDataLine speaker, Function> sendAudio) { this.sendAudio = sendAudio; this.microphone = microphone; this.speaker = speaker; } - Mono failureSignal() { + Mono failureSignalAsync() { return failure.asMono(); } @@ -866,13 +869,6 @@ private void fail(Throwable error) { } } - synchronized void skipPendingAudio() { - clearPlayback(); - if (speaker != null) { - speaker.flush(); - } - } - @Override public synchronized void close() { if (closed) { @@ -1100,4 +1096,6 @@ public void close() { } } } + + // #endregion } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java index 0dafec9b90006..238ca16209efd 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java @@ -77,9 +77,9 @@ public static void main(String[] args) { Mono.usingWhen(agent.builder.beta().buildBetaVoiceAgentLiveWebSocketAsyncClient() .openWebSocketSession(agent.name, new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store)), session -> args.length == 0 - ? GptLiveSampleUtils.runConversation(session, + ? GptLiveSampleUtils.runInteractiveConversationAsync(session, console == null ? new InputStreamReader(System.in) : console.reader()) - : GptLiveSampleUtils.runConversation(session, Arrays.asList(args)), + : GptLiveSampleUtils.runTextConversationAsync(session, Arrays.asList(args)), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(); diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java index f02f6bab3d30e..7487985dbcb51 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java @@ -72,10 +72,10 @@ public static void main(String[] args) { .openWebSocketSession(agent.name, new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store))) { if (args.length == 0) { Console console = System.console(); - GptLiveSampleUtils.runConversation(session, + GptLiveSampleUtils.runInteractiveConversation(session, console == null ? new InputStreamReader(System.in) : console.reader()); } else { - GptLiveSampleUtils.runConversation(session, Arrays.asList(args)); + GptLiveSampleUtils.runTextConversation(session, Arrays.asList(args)); } } finally { Schedulers.shutdownNow(); diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java index 74453674bf2ca..0a90488c0ddaf 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java @@ -441,18 +441,55 @@ void closeServer() { } } + @Test + public void blockingConversationConsumesEventsWithoutSession() { + java.util.concurrent.BlockingQueue incoming = new java.util.concurrent.LinkedBlockingQueue<>(); + incoming.add(typedEvent(BinaryData.fromString(STARTED))); + Iterable events = () -> Stream.generate(() -> { + try { + return incoming.take(); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Event producer interrupted.", error); + } + }).iterator(); + java.util.Iterator prompts = Arrays.asList("one", "two").iterator(); + GptLiveSampleUtils.runConversation(events, command -> { + commands.add(GptLiveSampleUtils.object(OpenAIJsonHelper.toBinaryData(command))); + if (command.isResponseCreate()) { + incoming.add(typedEvent(BinaryData.fromString(COMPLETED))); + } else if (command.isSessionClose()) { + incoming.add(typedEvent(BinaryData.fromString(CLOSED))); + } + }, () -> "test close details", transcripts -> prompts.hasNext() ? prompts.next() : null, pcm -> { + }, () -> { + }, false, null); + assertEquals(Arrays.asList("response.item.create", "response.create", "response.item.create", "response.create", + "session.close"), commandTypes()); + } + + @Test + public void blockingConversationReportsCloseDetailsWithoutSession() { + IllegalStateException error = assertThrows(IllegalStateException.class, () -> GptLiveSampleUtils + .runConversation(Collections.singletonList(typedEvent(BinaryData.fromString(CLOSED))), command -> { + }, () -> "transport close code=1006, reason=test disconnect", transcripts -> null, pcm -> { + }, () -> { + }, false, null)); + assertTrue(error.getMessage().contains("transport close code=1006, reason=test disconnect")); + } + @ParameterizedTest @ValueSource(booleans = { false, true }) public void textSamplesWaitForStartupAndCompleteMultipleTurns(boolean async) { startTextServer(false, false); if (async) { Mono.usingWhen(builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), - session -> GptLiveSampleUtils.runConversation(session, Arrays.asList("one", "two")), + session -> GptLiveSampleUtils.runTextConversationAsync(session, Arrays.asList("one", "two")), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(TIMEOUT); } else { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - GptLiveSampleUtils.runConversation(session, Arrays.asList("one", "two")); + GptLiveSampleUtils.runTextConversation(session, Arrays.asList("one", "two")); } } List expected = Arrays.asList("response.item.create", "response.create", "response.item.create", @@ -518,12 +555,12 @@ private void runInteractiveConversation(boolean async, Reader input, Consumer GptLiveSampleUtils.runConversation(session, input, playAudio, drainAudio), + session -> GptLiveSampleUtils.runInteractiveConversationAsync(session, input, playAudio, drainAudio), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(TIMEOUT); } else { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - GptLiveSampleUtils.runConversation(session, input, playAudio, drainAudio); + GptLiveSampleUtils.runInteractiveConversation(session, input, playAudio, drainAudio); } } } @@ -589,7 +626,7 @@ public int read(char[] buffer, int offset, int length) throws IOException { throw new IOException("Console unavailable"); } }; - StepVerifier.create(GptLiveSampleUtils.readPrompt(input)) + StepVerifier.create(GptLiveSampleUtils.readPromptAsync(input)) .expectErrorMatches( error -> error instanceof IOException && "Console unavailable".equals(error.getMessage())) .verify(TIMEOUT); @@ -598,7 +635,7 @@ public int read(char[] buffer, int offset, int length) throws IOException { @Test public void consoleInputIsSubmittedOnlyAfterEnterWithoutLosingCharacters() throws Exception { try (PipedReader input = new PipedReader(); PipedWriter keyboard = new PipedWriter(input)) { - CompletableFuture prompt = GptLiveSampleUtils.readPrompt(input).toFuture(); + CompletableFuture prompt = GptLiveSampleUtils.readPromptAsync(input).toFuture(); try { keyboard.write("testtesttes testing"); keyboard.flush(); @@ -698,7 +735,7 @@ public void greetingWaitsForBothOutputsAndRestartsQuietPeriodForEachChunk() { StepVerifier.withVirtualTime(() -> { GptLiveSampleUtils.Greeting pending = new GptLiveSampleUtils.Greeting(); greeting.set(pending); - return pending.awaitPlayback(() -> drained.set(true)); + return pending.awaitPlaybackAsync(() -> drained.set(true)); }) .expectSubscription() .then(() -> greeting.get().onTranscript()) @@ -726,7 +763,7 @@ public void missingGreetingOutputTimesOutInsteadOfStartingInput(String type) { } else { greeting.onTranscript(); } - return greeting.awaitPlayback(() -> drained.set(true)); + return greeting.awaitPlaybackAsync(() -> drained.set(true)); }).expectSubscription().thenAwait(GptLiveSampleUtils.TIMEOUT).expectError(TimeoutException.class).verify(); assertFalse(drained.get()); } @@ -737,7 +774,7 @@ public void greetingDrainFailureIsReported() { GptLiveSampleUtils.Greeting greeting = new GptLiveSampleUtils.Greeting(); greeting.onAudio(new byte[] { 1, 2 }); greeting.onTranscript(); - return greeting.awaitPlayback(() -> { + return greeting.awaitPlaybackAsync(() -> { throw new IllegalStateException("Speaker drain failed"); }); }).expectSubscription().thenAwait(Duration.ofSeconds(1)).expectErrorMessage("Speaker drain failed").verify(); @@ -754,7 +791,7 @@ public void continuousSilentOutputDoesNotKeepTheGreetingOpen() { .take(50) .doOnNext(tick -> greeting.onAudio(new byte[960])) .then(); - return Mono.when(silence, greeting.awaitPlayback(() -> drained.set(true))); + return Mono.when(silence, greeting.awaitPlaybackAsync(() -> drained.set(true))); }).expectSubscription().thenAwait(Duration.ofSeconds(1)).verifyComplete(); assertTrue(drained.get()); } @@ -794,7 +831,7 @@ public void greetingSpeechDetectionHonorsSignedPcm16PeakThreshold(int peak) { GptLiveSampleUtils.Greeting greeting = new GptLiveSampleUtils.Greeting(); greeting.onTranscript(); greeting.onAudio(new byte[] { (byte) peak, (byte) (peak >> 8) }); - return greeting.awaitPlayback(() -> drained.set(true)); + return greeting.awaitPlaybackAsync(() -> drained.set(true)); }).expectSubscription().thenAwait(audible ? Duration.ofSeconds(1) : GptLiveSampleUtils.TIMEOUT); if (audible) { verification.verifyComplete(); @@ -807,7 +844,7 @@ public void greetingSpeechDetectionHonorsSignedPcm16PeakThreshold(int peak) { @ParameterizedTest @ValueSource(strings = { "", "\r\n", "exit\n", "quit\n" }) public void consoleEofBlankAndExitProduceNoPrompt(String input) { - StepVerifier.create(GptLiveSampleUtils.readPrompt(new StringReader(input))).verifyComplete(); + StepVerifier.create(GptLiveSampleUtils.readPromptAsync(new StringReader(input))).verifyComplete(); } @ParameterizedTest @@ -862,7 +899,7 @@ public void interactiveTextPlaysGreetingAndRepliesWithoutPrintingBackendNotes(bo @Test public void silentAudioFramesArePacedAtRealTimePcmRate() { - StepVerifier.withVirtualTime(() -> GptLiveSampleUtils.silenceFrames().take(3)) + StepVerifier.withVirtualTime(() -> GptLiveSampleUtils.silenceFramesAsync().take(3)) .expectSubscription() .thenAwait(Duration.ZERO) .assertNext(this::assertSilentAudioFrame) @@ -877,7 +914,7 @@ public void silentAudioFramesArePacedAtRealTimePcmRate() { @Test public void silentAudioFramesWaitForDemandWithoutTimerOverflow() { - StepVerifier.withVirtualTime(() -> GptLiveSampleUtils.silenceFrames().take(3), 0) + StepVerifier.withVirtualTime(() -> GptLiveSampleUtils.silenceFramesAsync().take(3), 0) .expectSubscription() .expectNoEvent(Duration.ofSeconds(40)) .thenRequest(1) @@ -896,7 +933,7 @@ public void silentAudioFramesWaitForDemandWithoutTimerOverflow() { @Test public void silentAudioToleratesSendsSlowerThanItsFrameRate() { StepVerifier - .withVirtualTime(() -> GptLiveSampleUtils.silenceFrames() + .withVirtualTime(() -> GptLiveSampleUtils.silenceFramesAsync() .concatMap(frame -> Mono.delay(Duration.ofMillis(75)).thenReturn(frame), 1) .take(500)) .expectSubscription() @@ -964,7 +1001,7 @@ public void close() { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { IllegalStateException error = assertThrows(IllegalStateException.class, - () -> GptLiveSampleUtils.runConversation(session, input, pcm -> { + () -> GptLiveSampleUtils.runInteractiveConversation(session, input, pcm -> { }, () -> { })); assertEquals("Unable to read console input.", error.getMessage()); @@ -980,7 +1017,7 @@ public void serviceErrorsFailBothTextSamples(boolean async) { StepVerifier .create(Mono.usingWhen( builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), - session -> GptLiveSampleUtils.runConversation(session, Collections.singletonList("one")), + session -> GptLiveSampleUtils.runTextConversationAsync(session, Collections.singletonList("one")), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync)) .expectErrorMatches(error -> error instanceof IllegalStateException && error.getMessage().contains("GPT Live service error")) @@ -989,7 +1026,7 @@ public void serviceErrorsFailBothTextSamples(boolean async) { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { assertThrows(IllegalStateException.class, - () -> GptLiveSampleUtils.runConversation(session, Collections.singletonList("one"))); + () -> GptLiveSampleUtils.runTextConversation(session, Collections.singletonList("one"))); } } assertFalse(commandTypes().contains("session.close")); @@ -1000,7 +1037,7 @@ public void toolsUseGranularEventsAndReturnResultsBeforeContinuation() { startTextServer(true, false); try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - GptLiveSampleUtils.runConversation(session, Collections.singletonList("Weather in Seattle?"), + GptLiveSampleUtils.runToolConversation(session, Collections.singletonList("Weather in Seattle?"), GptLiveFunctionToolSample::toolResult); } assertEquals(Arrays.asList("response.item.create", "response.create", "response.item.create", "response.create", @@ -1014,12 +1051,27 @@ public void toolsUseGranularEventsAndReturnResultsBeforeContinuation() { assertFalse(item.containsKey("delegation_id")); } + @Test + public void toolHandlerFailureStopsConversationBeforeContinuation() { + startTextServer(true, false); + IllegalStateException failure = new IllegalStateException("Tool execution failed"); + try (BetaVoiceAgentLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + IllegalStateException error = assertThrows(IllegalStateException.class, () -> GptLiveSampleUtils + .runToolConversation(session, Collections.singletonList("Weather in Seattle?"), call -> { + throw failure; + })); + assertEquals(failure, error); + } + assertEquals(Arrays.asList("response.item.create", "response.create"), commandTypes()); + } + @Test public void toolConversationUsesSharedMultiTurnLoop() { startTextServer(true, false); try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - GptLiveSampleUtils.runConversation(session, Arrays.asList("Weather in Seattle?", "Thanks!"), + GptLiveSampleUtils.runToolConversation(session, Arrays.asList("Weather in Seattle?", "Thanks!"), GptLiveFunctionToolSample::toolResult); } assertEquals(Arrays.asList("response.item.create", "response.create", "response.item.create", "response.create", @@ -1057,7 +1109,7 @@ public void prematureClosureDoesNotReportSuccessfulConversation(boolean async) { StepVerifier .create(Mono.usingWhen( builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), - session -> GptLiveSampleUtils.runConversation(session, Collections.singletonList("one")), + session -> GptLiveSampleUtils.runTextConversationAsync(session, Collections.singletonList("one")), BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync)) .expectError(IllegalStateException.class) .verify(TIMEOUT); @@ -1065,7 +1117,7 @@ public void prematureClosureDoesNotReportSuccessfulConversation(boolean async) { try (BetaVoiceAgentLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { assertThrows(IllegalStateException.class, - () -> GptLiveSampleUtils.runConversation(session, Collections.singletonList("one"))); + () -> GptLiveSampleUtils.runTextConversation(session, Collections.singletonList("one"))); } } } @@ -1189,7 +1241,7 @@ public int read() { try { GptLiveSampleUtils.AudioProcessor processor = new GptLiveSampleUtils.AudioProcessor(microphone, speaker, pcm -> session.sendEvent(GptLiveSampleUtils.audio(pcm.toBytes()))); - Mono conversation = GptLiveSampleUtils.runConversation(session, processor, enterAfterAudio); + Mono conversation = GptLiveSampleUtils.runAudioConversationAsync(session, processor, enterAfterAudio); if (moderated) { IllegalStateException error = assertThrows(IllegalStateException.class, () -> conversation.block(TIMEOUT)); @@ -1219,7 +1271,9 @@ public int read() { } private List commandTypes() { - return commands.stream().map(GptLiveSampleUtils::type).collect(java.util.stream.Collectors.toList()); + return commands.stream() + .map(command -> GptLiveSampleUtils.string(command, "type")) + .collect(java.util.stream.Collectors.toList()); } private AgentsClientBuilder builder() { From 866f73577b3ea6c43854b6060abe43a55008d0bf Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Mon, 26 Oct 2026 16:33:12 +0800 Subject: [PATCH 10/20] Unify GPT Live sync and async lifecycle handling --- .../ai/agents/voice/GptLiveSampleUtils.java | 272 ++++++++++++------ .../ai/agents/voice/GptLiveSamplesTests.java | 134 +++++++++ 2 files changed, 314 insertions(+), 92 deletions(-) diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java index c200b0574f99e..a527ea6eab8db 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java @@ -142,7 +142,7 @@ static void runConversation(Iterable events, Consumer Function, ClientEvent> toolHandler) { List results = new ArrayList<>(); AtomicInteger continuations = new AtomicInteger(); - runConversationCore(events, sendEvent, closeDetails, nextPrompt, playAudio, drainAudio, spoken, backend -> { + runConversationCore(events, sendEvent, closeDetails, nextPrompt, playAudio, drainAudio, spoken, () -> { }, backend -> { if (toolHandler != null && backend.isOutputItemDone()) { ResponseOutputItem item = backend.asOutputItemDone().item(); if (item.isFunctionCall()) { @@ -164,13 +164,40 @@ static void runConversation(Iterable events, Consumer return true; } return false; - }); + }, reason -> false); } - private static void runConversationCore(Iterable events, Consumer sendEvent, + static void runConversationCore(Iterable events, Consumer sendEvent, Supplier closeDetails, Function nextPrompt, Consumer playAudio, Runnable drainAudio, boolean spoken, - Predicate handleResponse) { + Runnable onStarted, Predicate handleResponse, Predicate acceptClose) { + ConversationEventHandler handler + = new ConversationEventHandler(spoken, closeDetails, acceptClose) { + @Override + Boolean onStarted() { + onStarted.run(); + return false; + } + + @Override + Boolean onResponse(ResponseStreamEvent event) { + boolean continuation = handleResponse.test(event); + if (event.isCompleted()) { + completeResponse(continuation); + } + return false; + } + + @Override + Boolean onClosed() { + return true; + } + + @Override + Boolean ignored() { + return false; + } + }; ExecutorService workers = Executors.newFixedThreadPool(2, task -> { Thread thread = new Thread(task, "gpt-live-sync-conversation"); thread.setDaemon(true); @@ -197,12 +224,9 @@ private static void runConversationCore(Iterable events, Consumer prompt = null; - boolean started = false; boolean greetingComplete = !spoken; boolean greetingAudio = false; boolean greetingTranscript = false; - boolean awaitingResponse = false; - boolean closing = false; long deadline = System.nanoTime() + TIMEOUT.toNanos(); long nextAudio = 0; long quietSince = 0; @@ -229,41 +253,15 @@ private static void runConversationCore(Iterable events, Consumer events, Consumer= nextAudio) { + if (handler.canStreamAudio() && now >= nextAudio) { sendEvent.accept(silence); nextAudio = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(20); } @@ -285,7 +283,7 @@ private static void runConversationCore(Iterable events, Consumer nextPrompt.apply(transcripts)); } if (prompt != null && prompt.isDone()) { @@ -293,10 +291,10 @@ private static void runConversationCore(Iterable events, Consumer events, Consumer= deadline) { + if (!handler.canReadPrompt(greetingComplete) && now >= deadline) { throw new IllegalStateException("Timed out waiting for the GPT Live conversation."); } } @@ -436,23 +434,20 @@ private static Mono runConversationAsync(Flux events, "Opening greeting requested. Speak into the microphone. Press Enter to end the call.")))); return runConversationCoreAsync(events, sendEvent, closeDetails, nextPrompt, playAudio, drainAudio, spoken, spoken && microphone == null, microphone == null && System.console() != null, - () -> { + () -> Mono.fromRunnable(() -> { if (microphone != null) { microphone.start(); } - }, opening, reason -> microphone != null); + }), opening, backend -> Mono.just(false), reason -> microphone != null); } - private static Mono runConversationCoreAsync(Flux events, + static Mono runConversationCoreAsync(Flux events, Function> sendEvent, Supplier closeDetails, Function> nextPrompt, Consumer playAudio, Runnable drainAudio, boolean spoken, boolean interactive, boolean inputEchoed, - Runnable onStarted, Mono opening, Predicate acceptClose) { + Supplier> onStarted, Mono opening, + Function> handleResponse, Predicate acceptClose) { return Mono.defer(() -> { - AtomicBoolean finalized = new AtomicBoolean(); - AtomicBoolean started = new AtomicBoolean(); - AtomicBoolean closeRequested = new AtomicBoolean(); - AtomicBoolean remoteHangup = new AtomicBoolean(); AudioTranscriptPrinter transcripts = new AudioTranscriptPrinter(inputEchoed, interactive); Greeting greeting = new Greeting(); Sinks.Empty ready = Sinks.empty(); @@ -461,9 +456,37 @@ private static Mono runConversationCoreAsync(Flux events, Sinks.Empty stopAudio = Sinks.empty(); Sinks.Empty audioStopped = Sinks.empty(); AtomicReference> responseCompleted = new AtomicReference<>(); + ConversationEventHandler> handler + = new ConversationEventHandler>(spoken, closeDetails, acceptClose) { + @Override + Mono onStarted() { + return Mono.defer(onStarted).doOnSuccess(ignored -> ready.tryEmitEmpty()); + } + + @Override + Mono onResponse(ResponseStreamEvent event) { + return Mono.defer(() -> handleResponse.apply(event)).defaultIfEmpty(false) + .doOnNext(continuation -> { + if (event.isCompleted() && completeResponse(continuation)) { + responseCompleted.getAndSet(null).tryEmitEmpty(); + } + }).then(); + } + + @Override + Mono onClosed() { + return Mono.fromRunnable(() -> closed.tryEmitEmpty()); + } + + @Override + Mono ignored() { + return Mono.empty(); + } + }; Mono receive = events .publishOn(Schedulers.boundedElastic(), 1) - .doOnNext(event -> { + .takeUntil(ServerEvent::isSessionClosed) + .concatMap(event -> Mono.defer(() -> { if (spoken) { transcripts.print(event); if (event.isSessionOutputAudioDelta()) { @@ -477,44 +500,9 @@ private static Mono runConversationCoreAsync(Flux events, } else { printEvent(event); } - ResponseStreamEvent backend = backendEvent(event); - if (event.isSessionStarted()) { - if (!started.compareAndSet(false, true)) { - throw new IllegalStateException("Duplicate session.started event."); - } - onStarted.run(); - ready.tryEmitEmpty(); - } else if (backend != null && backend.isCompleted()) { - Sinks.Empty completion = responseCompleted.getAndSet(null); - if (spoken && started.get() && completion == null) { - return; - } - if (!started.get() || closeRequested.get() || completion == null) { - throw new IllegalStateException("Unexpected GPT Live response lifecycle event."); - } - if (!spoken) { - System.out.println(); - } - completion.tryEmitEmpty(); - } else if (event.isSessionClosed()) { - String reason = event.asSessionClosed().reason().toString(); - boolean remote = started.get() && "remote_hangup".equals(reason); - boolean accepted = acceptClose.test(reason); - boolean requested = accepted - || closeRequested.get() && "close_requested".equals(reason); - if (!requested && !remote) { - throw new IllegalStateException("GPT Live closed before all turns completed. Event: " - + event + "; " + closeDetails.get()); - } - remoteHangup.set(remote || accepted); - System.out.println("Live session closed (" + reason + ")."); - finalized.set(true); - closed.tryEmitEmpty(); - } - }) - .takeUntil(ServerEvent::isSessionClosed) - .then(Mono.defer(() -> remoteHangup.get() ? Mono.empty() : finalized.get() ? Mono.never() - : Mono.error(new IllegalStateException("GPT Live disconnected without session.closed.")))); + return handler.process(event); + }), 1) + .then(Mono.defer(() -> handler.awaitConversationAfterReceive() ? Mono.never() : Mono.empty())); Mono audio = ready.asMono() .thenMany(silenceFramesAsync().takeUntilOther(stopAudio.asMono()) .concatMap(frame -> sendEvent.apply(frame).doOnSuccess(ignored -> audioStarted.tryEmitEmpty()), 1)) @@ -531,9 +519,9 @@ private static Mono runConversationCoreAsync(Flux events, .then(greeting.awaitPlaybackAsync(drainAudio)) .then(Mono.fromRunnable(transcripts::endUtterance)) : opening)) - .then(Mono.defer(() -> promptAsync(sendEvent, nextPrompt, responseCompleted, spoken, transcripts))) + .then(Mono.defer(() -> promptAsync(sendEvent, nextPrompt, responseCompleted, spoken, transcripts, handler))) .then(Mono.defer(() -> { - closeRequested.set(true); + handler.requestClose(); stopAudio.tryEmitEmpty(); return (interactive ? audioStopped.asMono() : Mono.empty()) .then(sendEvent.apply(closeSession())) @@ -547,10 +535,11 @@ private static Mono runConversationCoreAsync(Flux events, private static Mono promptAsync(Function> sendEvent, Function> nextPrompt, AtomicReference> responseCompleted, boolean spoken, - AudioTranscriptPrinter transcripts) { + AudioTranscriptPrinter transcripts, ConversationEventHandler handler) { return nextPrompt.apply(transcripts).flatMap(text -> { Sinks.Empty completion = Sinks.empty(); responseCompleted.set(completion); + handler.beginResponse(); if (!spoken) { System.out.print("Assistant: "); System.out.flush(); @@ -561,7 +550,8 @@ private static Mono promptAsync(Function> sendEven .thenReturn(true); }).defaultIfEmpty(false) .flatMap(keepGoing -> keepGoing - ? Mono.defer(() -> promptAsync(sendEvent, nextPrompt, responseCompleted, spoken, transcripts)) : Mono.empty()); + ? Mono.defer(() -> promptAsync(sendEvent, nextPrompt, responseCompleted, spoken, transcripts, handler)) + : Mono.empty()); } static Flux silenceFramesAsync() { @@ -618,6 +608,104 @@ private static Mono readPromptAsync(Reader input, AudioTranscriptPrinter // #region Shared utilities + private abstract static class ConversationEventHandler { + private final boolean spoken; + private final Supplier closeDetails; + private final Predicate acceptClose; + private volatile boolean started; + private volatile boolean awaitingResponse; + private volatile boolean closing; + private volatile boolean finalized; + private volatile boolean remoteHangup; + + ConversationEventHandler(boolean spoken, Supplier closeDetails, Predicate acceptClose) { + this.spoken = spoken; + this.closeDetails = closeDetails; + this.acceptClose = acceptClose; + } + + final boolean canStreamAudio() { + return spoken && started && !closing; + } + + final boolean canReadPrompt(boolean greetingComplete) { + return started && greetingComplete && !awaitingResponse && !closing; + } + + final void beginResponse() { + awaitingResponse = true; + } + + final void requestClose() { + closing = true; + } + + final boolean awaitConversationAfterReceive() { + if (!finalized) { + throw new IllegalStateException("GPT Live disconnected without session.closed."); + } + return !remoteHangup; + } + + final T process(ServerEvent event) { + if (event.isSessionStarted()) { + if (started) { + throw new IllegalStateException("Duplicate session.started event."); + } + started = true; + return onStarted(); + } + ResponseStreamEvent backend = backendEvent(event); + if (backend != null) { + if (backend.isCompleted()) { + // Spoken greetings or unsolicited responses have no pending prompt, even during closure. + if (spoken && started && !awaitingResponse) { + return ignored(); + } + // Otherwise, completion requires a pending response in a started session that is not closing. + if (!started || !awaitingResponse || closing) { + throw new IllegalStateException("Unexpected GPT Live response lifecycle event."); + } + } + return onResponse(backend); + } + if (event.isSessionClosed()) { + String reason = event.asSessionClosed().reason().toString(); + boolean remote = started && "remote_hangup".equals(reason); + boolean accepted = acceptClose.test(reason); + // Allow a remote hangup after startup, a caller-accepted reason, or acknowledgment of our close request. + if (!remote && !accepted && !(closing && "close_requested".equals(reason))) { + throw new IllegalStateException("GPT Live closed before all turns completed. Event: " + event + + "; " + closeDetails.get()); + } + remoteHangup = remote || accepted; + finalized = true; + System.out.println("Live session closed (" + reason + ")."); + return onClosed(); + } + return ignored(); + } + + final boolean completeResponse(boolean continuation) { + if (continuation) { + return false; + } + awaitingResponse = false; + if (!spoken) { + System.out.println(); + } + return true; + } + + abstract T onStarted(); + + abstract T onResponse(ResponseStreamEvent event); + + abstract T onClosed(); + + abstract T ignored(); + } + static byte[] decodeAudio(ServerEvent event) { byte[] pcm = Base64.getDecoder().decode(event.asSessionOutputAudioDelta().delta()); if (pcm.length % 2 != 0) { diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java index 0a90488c0ddaf..c7ebf7cc96ca0 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java @@ -13,6 +13,7 @@ import com.azure.core.util.Configuration; import com.openai.models.live.ClientEvent; import com.openai.models.live.ServerEvent; +import com.openai.models.responses.ResponseStreamEvent; import java.io.ByteArrayOutputStream; import java.io.FilterReader; import java.io.InputStream; @@ -42,6 +43,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.Stream; import javax.sound.sampled.SourceDataLine; import javax.sound.sampled.TargetDataLine; @@ -478,6 +480,138 @@ public void blockingConversationReportsCloseDetailsWithoutSession() { assertTrue(error.getMessage().contains("transport close code=1006, reason=test disconnect")); } + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void conversationCoresUseTheSameLifecycleHooks(boolean async) { + List callbacks = new CopyOnWriteArrayList<>(); + runCoreWithCallbacks(async, () -> callbacks.add("started"), event -> { + assertTrue(event.isCompleted()); + callbacks.add("response"); + return false; + }, reason -> { + callbacks.add("close:" + reason); + return "expired".equals(reason); + }); + assertEquals(Arrays.asList("started", "response", "close:expired"), callbacks); + assertEquals(Arrays.asList("response.item.create", "response.create", "session.close"), commandTypes()); + } + + @ParameterizedTest + @MethodSource("coreCallbackFailureCases") + public void conversationCoresPropagateHookFailures(boolean async, String callback) { + IllegalStateException failure = new IllegalStateException("Failed " + callback); + IllegalStateException actual + = assertThrows(IllegalStateException.class, () -> runCoreWithCallbacks(async, () -> { + if ("started".equals(callback)) { + throw failure; + } + }, event -> { + if ("response".equals(callback)) { + throw failure; + } + return false; + }, reason -> { + throw failure; + })); + assertEquals(failure, actual); + assertEquals("started".equals(callback) + ? Collections.emptyList() + : "response".equals(callback) + ? Arrays.asList("response.item.create", "response.create") + : Arrays.asList("response.item.create", "response.create", "session.close"), + commandTypes()); + } + + private static Stream coreCallbackFailureCases() { + return Stream.of(false, true) + .flatMap( + async -> Stream.of("started", "response", "closed").map(callback -> Arguments.of(async, callback))); + } + + private void runCoreWithCallbacks(boolean async, Runnable onStarted, Predicate handleResponse, + Predicate acceptClose) { + java.util.concurrent.BlockingQueue blocking = new java.util.concurrent.LinkedBlockingQueue<>(); + Sinks.Many reactive = Sinks.many().unicast().onBackpressureBuffer(); + Consumer enqueue + = async ? event -> reactive.emitNext(event, Sinks.EmitFailureHandler.FAIL_FAST) : blocking::add; + enqueue.accept(typedEvent(BinaryData.fromString(STARTED))); + Consumer sendEvent = command -> { + commands.add(GptLiveSampleUtils.object(OpenAIJsonHelper.toBinaryData(command))); + if (command.isResponseCreate()) { + enqueue.accept(typedEvent(BinaryData.fromString(COMPLETED))); + } else if (command.isSessionClose()) { + enqueue + .accept(typedEvent(BinaryData.fromString("{\"type\":\"session.closed\",\"reason\":\"expired\"}"))); + } + }; + AtomicInteger prompts = new AtomicInteger(); + if (async) { + GptLiveSampleUtils.runConversationCoreAsync(reactive.asFlux(), + command -> Mono.fromRunnable(() -> sendEvent.accept(command)), () -> "test close details", + transcripts -> Mono.justOrEmpty(prompts.getAndIncrement() == 0 ? "one" : null), pcm -> { + }, () -> { + }, false, false, false, () -> Mono.fromRunnable(onStarted), Mono.empty(), + event -> Mono.fromSupplier(() -> handleResponse.test(event)), acceptClose).block(TIMEOUT); + } else { + Iterable events = () -> Stream.generate(() -> { + try { + return blocking.take(); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Event producer interrupted.", error); + } + }).iterator(); + GptLiveSampleUtils.runConversationCore(events, sendEvent, () -> "test close details", + transcripts -> prompts.getAndIncrement() == 0 ? "one" : null, pcm -> { + }, () -> { + }, false, onStarted, handleResponse, acceptClose); + } + } + + @Test + public void asyncResponseContinuationWaitsForHandlerBeforeProcessingNextEvent() { + Sinks.Many incoming = Sinks.many().unicast().onBackpressureBuffer(); + Sinks.Empty continuation = Sinks.empty(); + CountDownLatch handling = new CountDownLatch(1); + AtomicInteger responses = new AtomicInteger(); + AtomicInteger prompts = new AtomicInteger(); + incoming.emitNext(typedEvent(BinaryData.fromString(STARTED)), Sinks.EmitFailureHandler.FAIL_FAST); + Mono conversation = GptLiveSampleUtils.runConversationCoreAsync(incoming.asFlux(), command -> { + commands.add(GptLiveSampleUtils.object(OpenAIJsonHelper.toBinaryData(command))); + if (command.isResponseCreate()) { + incoming.emitNext(typedEvent(BinaryData.fromString(COMPLETED)), Sinks.EmitFailureHandler.FAIL_FAST); + } else if (command.isSessionClose()) { + incoming.emitNext(typedEvent(BinaryData.fromString(CLOSED)), Sinks.EmitFailureHandler.FAIL_FAST); + } + return Mono.empty(); + }, () -> "test close details", transcripts -> Mono.justOrEmpty(prompts.getAndIncrement() == 0 ? "one" : null), + pcm -> { + }, () -> { + }, false, false, false, Mono::empty, Mono.empty(), event -> { + if (responses.incrementAndGet() == 1) { + incoming.emitNext(typedEvent(BinaryData.fromString(COMPLETED)), Sinks.EmitFailureHandler.FAIL_FAST); + handling.countDown(); + return continuation.asMono().thenReturn(true); + } + assertEquals(1, prompts.get(), "The next prompt must wait until the continuation completes"); + return Mono.just(false); + }, reason -> false); + StepVerifier.create(conversation).then(() -> { + try { + assertTrue(handling.await(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new AssertionError(error); + } + assertEquals(1, responses.get(), "Later events must wait for the asynchronous handler"); + assertEquals(1, prompts.get()); + continuation.emitEmpty(Sinks.EmitFailureHandler.FAIL_FAST); + }).verifyComplete(); + assertEquals(2, responses.get()); + assertEquals(2, prompts.get()); + assertEquals(Arrays.asList("response.item.create", "response.create", "session.close"), commandTypes()); + } + @ParameterizedTest @ValueSource(booleans = { false, true }) public void textSamplesWaitForStartupAndCompleteMultipleTurns(boolean async) { From b0ae16730d8a22c389051519d1ed9132e8d6a523 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Tue, 27 Oct 2026 09:22:19 +0800 Subject: [PATCH 11/20] Suppress external event type checks for live WebSocket clients --- sdk/ai/azure-ai-agents/checkstyle-suppressions.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml b/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml index 92a667f63b23f..d62c40674576b 100644 --- a/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml +++ b/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml @@ -54,6 +54,8 @@ + + From c26180f44d421558aa8b31b8240a8298a1432e3c Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Tue, 27 Oct 2026 09:47:36 +0800 Subject: [PATCH 12/20] Regenerate agents Checkstyle suppressions --- .../checkstyle-suppressions.xml | 60 +++++++++---------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml b/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml index d62c40674576b..df0c05171b47b 100644 --- a/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml +++ b/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml @@ -3,59 +3,57 @@ - - - - - - + + + + + + + + + + + + - + + + + - - - - + + + - - + + + - + - + - - + - - - - - - - - + - - - - - - + + + From ceee18dc410b73464a02b9303fbea3bf736503c3 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 30 Oct 2026 09:46:17 +0800 Subject: [PATCH 13/20] Move WebSocket transport release note to unreleased section --- sdk/ai/azure-ai-agents/CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/ai/azure-ai-agents/CHANGELOG.md b/sdk/ai/azure-ai-agents/CHANGELOG.md index 65afc1092917b..a40fb360baca6 100644 --- a/sdk/ai/azure-ai-agents/CHANGELOG.md +++ b/sdk/ai/azure-ai-agents/CHANGELOG.md @@ -68,6 +68,8 @@ ### Other Changes +- Extracted shared protocol-neutral WebSocket transports for Live and Realtime, keeping independent blocking + and asynchronous implementations and protocol-specific event decoding in the respective clients. - Regenerated the client from Foundry TypeSpec commit `c547bc946e1b61c5ac0c798b48c22f0b7515ad59`. - Regenerated the client with `@azure-tools/openai-typespec` version `1.34.1`. - Updated post-generation customizations to store fixed wire values in new Live and voice-agent models as class @@ -159,8 +161,6 @@ ### Other Changes -- Extracted shared protocol-neutral WebSocket transports for Live and Realtime, keeping independent blocking - and asynchronous implementations and protocol-specific event decoding in the respective clients. - Regenerated from the updated Foundry TypeSpec specification. - Updated `SessionConfiguration` documentation to reflect the service-supported maximum idle timeout of 14,400 seconds (four hours); the default remains 900 seconds. From 442be563bf830de32dc433db3d8373a4b504dc39 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Tue, 3 Nov 2026 19:23:09 +0800 Subject: [PATCH 14/20] Address voice WebSocket review feedback Align VoiceAgents client names, separate transport construction from connection effects, and move conversation flows and single-example helpers into their samples. Use generated Live function tool models and typed Responses tool calls without internal sample helpers. Validation: 221 focused tests and Spotless passed after a clean rebuild. Explicit SpotBugs reports 42 findings and Checkstyle reports 3 violations in unchanged files. --- sdk/ai/azure-ai-agents/CHANGELOG.md | 8 +- sdk/ai/azure-ai-agents/README.md | 28 +- .../checkstyle-suppressions.xml | 4 +- .../azure/ai/agents/AgentsClientBuilder.java | 32 +- ...aVoiceAgentsLiveWebSocketAsyncClient.java} | 10 +- ...> BetaVoiceAgentsLiveWebSocketClient.java} | 12 +- ...gentsLiveWebSocketSessionAsyncClient.java} | 4 +- ...oiceAgentsLiveWebSocketSessionClient.java} | 4 +- ... BetaVoiceAgentsWebSocketAsyncClient.java} | 12 +- ...va => BetaVoiceAgentsWebSocketClient.java} | 12 +- ...iceAgentsWebSocketSessionAsyncClient.java} | 8 +- ...etaVoiceAgentsWebSocketSessionClient.java} | 10 +- .../realtime/WebSocketSessionTransport.java | 18 +- .../com/azure/ai/agents/ReadmeSamples.java | 4 +- .../GptLiveAudioConversationAsyncSample.java | 152 +++++- .../voice/GptLiveFunctionToolSample.java | 142 ++++- .../ai/agents/voice/GptLiveSampleUtils.java | 506 +----------------- .../GptLiveTextConversationAsyncSample.java | 251 ++++++++- .../voice/GptLiveTextConversationSample.java | 272 +++++++++- ...AgentLiveAudioConversationAsyncSample.java | 16 +- .../VoiceAgentLiveFunctionToolSample.java | 10 +- ...eAgentLiveTextConversationAsyncSample.java | 14 +- .../VoiceAgentLiveTextConversationSample.java | 10 +- ...iceAgentsWebSocketSessionClientTests.java} | 35 +- ...FoundryFeaturesHeaderVerificationTest.java | 6 +- .../ai/agents/voice/GptLiveSamplesTests.java | 215 ++++---- .../VoiceAgentConversationsAsyncTests.java | 8 +- .../voice/VoiceAgentConversationsTests.java | 6 +- .../voice/VoiceAgentLiveWebSocketTests.java | 90 ++-- .../voice/VoiceAgentRealtimeLiveTests.java | 14 +- .../VoiceAgentWebSocketSessionAsyncTests.java | 68 +-- .../VoiceAgentWebSocketSessionTests.java | 82 +-- 32 files changed, 1177 insertions(+), 886 deletions(-) rename sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/{BetaVoiceAgentLiveWebSocketAsyncClient.java => BetaVoiceAgentsLiveWebSocketAsyncClient.java} (87%) rename sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/{BetaVoiceAgentLiveWebSocketClient.java => BetaVoiceAgentsLiveWebSocketClient.java} (85%) rename sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/{BetaVoiceAgentLiveWebSocketSessionAsyncClient.java => BetaVoiceAgentsLiveWebSocketSessionAsyncClient.java} (95%) rename sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/{BetaVoiceAgentLiveWebSocketSessionClient.java => BetaVoiceAgentsLiveWebSocketSessionClient.java} (96%) rename sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/{BetaVoiceAgentWebSocketAsyncClient.java => BetaVoiceAgentsWebSocketAsyncClient.java} (77%) rename sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/{BetaVoiceAgentWebSocketClient.java => BetaVoiceAgentsWebSocketClient.java} (81%) rename sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/{BetaVoiceAgentWebSocketSessionAsyncClient.java => BetaVoiceAgentsWebSocketSessionAsyncClient.java} (95%) rename sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/{BetaVoiceAgentWebSocketSessionClient.java => BetaVoiceAgentsWebSocketSessionClient.java} (95%) rename sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/{BetaVoiceAgentWebSocketSessionClientTests.java => BetaVoiceAgentsWebSocketSessionClientTests.java} (65%) diff --git a/sdk/ai/azure-ai-agents/CHANGELOG.md b/sdk/ai/azure-ai-agents/CHANGELOG.md index a40fb360baca6..076eebfcdd337 100644 --- a/sdk/ai/azure-ai-agents/CHANGELOG.md +++ b/sdk/ai/azure-ai-agents/CHANGELOG.md @@ -19,8 +19,8 @@ represented as `Duration` and outputs as `List`. - Added preview telephony campaign clients and models for recipients, scheduling, progress, and validation errors. - Added preview synchronous and asynchronous GPT Live WebSocket clients through - `AgentsClientBuilder.beta().buildBetaVoiceAgentLiveWebSocketClient()` and - `buildBetaVoiceAgentLiveWebSocketAsyncClient()`. Native Live events are sent and received as lossless JSON + `AgentsClientBuilder.beta().buildBetaVoiceAgentsLiveWebSocketClient()` and + `buildBetaVoiceAgentsLiveWebSocketAsyncClient()`. Native Live events are sent and received as lossless JSON using `BinaryData`, separately from the Realtime protocol clients. - Added typed GPT Live sends through `sendEvent(com.openai.models.live.ClientEvent)` and typed receives through `receiveTypedEvents()`, retaining the raw JSON APIs. The synchronous client also supports per-event receive timeouts. @@ -254,8 +254,8 @@ subagents, persistence, avatars, and animation. - Added `createAgentFromPrompt` and `createAgentFromPromptWithResponse` to `BetaAgentsClient` and `BetaAgentsAsyncClient` for generating and creating agents from kind-specific high-level inputs. -- Added preview `BetaVoiceAgentWebSocketClient`, `BetaVoiceAgentWebSocketAsyncClient`, - `BetaVoiceAgentWebSocketSessionClient`, and `BetaVoiceAgentWebSocketSessionAsyncClient` for authenticated realtime +- Added preview `BetaVoiceAgentsWebSocketClient`, `BetaVoiceAgentsWebSocketAsyncClient`, + `BetaVoiceAgentsWebSocketSessionClient`, and `BetaVoiceAgentsWebSocketSessionAsyncClient` for authenticated realtime `wss://` sessions. The clients support typed and raw events, text and audio input, response cancellation, client-executed function results, persisted conversations, agent-version overrides, and the related realtime session, conversation-item, response, transcription, MCP, audio, avatar, and RTC event models. diff --git a/sdk/ai/azure-ai-agents/README.md b/sdk/ai/azure-ai-agents/README.md index 191d59706a57d..2277fc2e66f80 100644 --- a/sdk/ai/azure-ai-agents/README.md +++ b/sdk/ai/azure-ai-agents/README.md @@ -70,8 +70,8 @@ The Agents client library has the following sub-clients which group the differen - `ResponsesClient` / `ResponsesAsyncClient`: Create responses that require Azure-specific request fields, such as an explicit `AgentReference` or structured inputs. For standard OpenAI Responses API calls through an agent endpoint, use an agent-scoped OpenAI client. See the [OpenAI Responses API documentation][openai_responses_api_docs] for more information. - `BetaMemoryStoresClient` / `BetaMemoryStoresAsyncClient` **(preview)**: Manage memory stores and individual memory items for agents. - `ToolboxesClient` / `ToolboxesAsyncClient`: Manage toolboxes and toolbox versions. -- `BetaVoiceAgentWebSocketClient` / `BetaVoiceAgentWebSocketAsyncClient` **(preview)**: Open typed realtime WebSocket sessions with voice agents. -- `BetaVoiceAgentLiveWebSocketClient` / `BetaVoiceAgentLiveWebSocketAsyncClient` **(preview)**: Open native GPT Live WebSocket sessions with persisted Live voice agents. +- `BetaVoiceAgentsWebSocketClient` / `BetaVoiceAgentsWebSocketAsyncClient` **(preview)**: Open typed realtime WebSocket sessions with voice agents. +- `BetaVoiceAgentsLiveWebSocketClient` / `BetaVoiceAgentsLiveWebSocketAsyncClient` **(preview)**: Open native GPT Live WebSocket sessions with persisted Live voice agents. - `BetaVoiceAgentsTelephonyClient` / `BetaVoiceAgentsTelephonyAsyncClient` **(preview)**: Manage telephony bindings, calls, transfer targets, and outbound call jobs. - `BetaVoiceAgentsConversationsClient` / `BetaVoiceAgentsConversationsAsyncClient` **(preview)**: Read and delete persisted voice-agent conversations and retrieve their responses, items, and audio. @@ -198,8 +198,8 @@ Build clients whose names start with `Beta` from `AgentsClientBuilder.beta()`. T |---|---| | `BetaAgentsClient` | `WorkflowAgents=V1Preview,ExternalAgents=V1Preview,DraftAgents=V1Preview,VoiceAgents=V1Preview,DigitalWorker=V1Preview,GitHubCopilot=V1Preview,Skills=V1Preview,MicrosoftCopilot=V1Preview` | | `BetaMemoryStoresClient` | `MemoryStores=V1Preview` | -| `BetaVoiceAgentWebSocketClient` | `VoiceAgents=V1Preview` | -| `BetaVoiceAgentLiveWebSocketClient` / `BetaVoiceAgentLiveWebSocketAsyncClient` | `VoiceAgents=V1Preview` | +| `BetaVoiceAgentsWebSocketClient` | `VoiceAgents=V1Preview` | +| `BetaVoiceAgentsLiveWebSocketClient` / `BetaVoiceAgentsLiveWebSocketAsyncClient` | `VoiceAgents=V1Preview` | | `BetaVoiceAgentsTelephonyClient` | `VoiceAgents=V1Preview` | | `BetaVoiceAgentsConversationsClient` | `VoiceAgents=V1Preview` | @@ -208,7 +208,7 @@ The async `Beta*AsyncClient` counterparts follow the same behavior. ### Realtime voice-agent sessions Create and manage preview voice agents with `VoiceAgentDefinition` and an `AgentsClient` or `AgentsAsyncClient` built -using `allowPreview(true)`. Use `BetaVoiceAgentWebSocketClient` or `BetaVoiceAgentWebSocketAsyncClient` to open a typed, +using `allowPreview(true)`. Use `BetaVoiceAgentsWebSocketClient` or `BetaVoiceAgentsWebSocketAsyncClient` to open a typed, bidirectional session with an existing voice agent. The client acquires a token for `https://ai.azure.com/.default`, negotiates the `realtime` WebSocket subprotocol, and sends the required `VoiceAgents=V1Preview` feature header automatically. @@ -991,10 +991,10 @@ AgentsClientBuilder builder = new AgentsClientBuilder() .credential(new DefaultAzureCredentialBuilder().build()) .endpoint(endpoint); -BetaVoiceAgentWebSocketClient realtimeClient - = builder.beta().buildBetaVoiceAgentWebSocketClient(); -BetaVoiceAgentWebSocketAsyncClient realtimeAsyncClient - = builder.beta().buildBetaVoiceAgentWebSocketAsyncClient(); +BetaVoiceAgentsWebSocketClient realtimeClient + = builder.beta().buildBetaVoiceAgentsWebSocketClient(); +BetaVoiceAgentsWebSocketAsyncClient realtimeAsyncClient + = builder.beta().buildBetaVoiceAgentsWebSocketAsyncClient(); ``` #### Send a synchronous text turn @@ -1023,7 +1023,7 @@ VoiceAgentWebSocketConnectionOptions options .setReceiveBufferCapacity(512) .setMaxMessageSize(8 * 1024 * 1024) .setOverflowStrategy(VoiceAgentWebSocketOverflowStrategy.ERROR); -try (BetaVoiceAgentWebSocketSessionClient session = realtimeClient.openWebSocketSession(agentName, options)) { +try (BetaVoiceAgentsWebSocketSessionClient session = realtimeClient.openWebSocketSession(agentName, options)) { session.sendEvent(BinaryData.fromString( "{\"type\":\"response.create\",\"event_id\":\"response-1\"}")); for (RealtimeServerEvent event : session.receiveEvents()) { @@ -1044,7 +1044,7 @@ Use `close(code, reason)` or asynchronous `closeAsync(code, reason)` to send a c fit in 123 UTF-8 bytes and close codes must be valid WebSocket codes. The first asynchronous close request wins. ```java -try (BetaVoiceAgentWebSocketSessionClient session = realtimeClient.openWebSocketSession(agentName)) { +try (BetaVoiceAgentsWebSocketSessionClient session = realtimeClient.openWebSocketSession(agentName)) { session.sendText("Hello! Tell me about the services you provide."); session.createResponse(); @@ -1080,9 +1080,9 @@ Mono.usingWhen( }) .takeUntil(event -> event instanceof RealtimeResponseDoneEvent) .then(), - BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync, + BetaVoiceAgentsWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), - BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync) + BetaVoiceAgentsWebSocketSessionAsyncClient::closeAsync) .block(); ``` @@ -1112,7 +1112,7 @@ The live audio example requires a Java Sound-compatible microphone and speaker. ### GPT Live WebSocket examples (preview) GPT Live is a different wire protocol from Realtime. Create a persisted agent with `VoiceAgentLiveDefinition`, -then use `BetaVoiceAgentLiveWebSocketClient` or `BetaVoiceAgentLiveWebSocketAsyncClient` to connect. +then use `BetaVoiceAgentsLiveWebSocketClient` or `BetaVoiceAgentsLiveWebSocketAsyncClient` to connect. These clients use `wss://{project-endpoint}/agents/{agent_name}/endpoint/protocols/voice/live/sessions?api-version=v1`, send the `Foundry-Features: VoiceAgents=V1Preview` header, and **do not** negotiate the `realtime` WebSocket diff --git a/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml b/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml index df0c05171b47b..ca4ab791025a0 100644 --- a/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml +++ b/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml @@ -13,8 +13,8 @@ - - + + diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java index 4a932609eabc8..46a2dbbb115eb 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java @@ -630,14 +630,14 @@ public BetaAgentsClientBuilder beta() { serviceClients = { BetaAgentsClient.class, BetaMemoryStoresClient.class, - BetaVoiceAgentWebSocketClient.class, - BetaVoiceAgentLiveWebSocketClient.class, + BetaVoiceAgentsWebSocketClient.class, + BetaVoiceAgentsLiveWebSocketClient.class, BetaVoiceAgentsTelephonyClient.class, BetaVoiceAgentsConversationsClient.class, BetaAgentsAsyncClient.class, BetaMemoryStoresAsyncClient.class, - BetaVoiceAgentWebSocketAsyncClient.class, - BetaVoiceAgentLiveWebSocketAsyncClient.class, + BetaVoiceAgentsWebSocketAsyncClient.class, + BetaVoiceAgentsLiveWebSocketAsyncClient.class, BetaVoiceAgentsTelephonyAsyncClient.class, BetaVoiceAgentsConversationsAsyncClient.class }) public final class BetaAgentsClientBuilder { @@ -753,8 +753,8 @@ public BetaMemoryStoresClient buildBetaMemoryStoresClient() { * @throws IllegalStateException if unsupported HTTP pipeline configuration is present. */ @Beta - public BetaVoiceAgentWebSocketAsyncClient buildBetaVoiceAgentWebSocketAsyncClient() { - return new BetaVoiceAgentWebSocketAsyncClient(createVoiceAgentWebSocketConfiguration()); + public BetaVoiceAgentsWebSocketAsyncClient buildBetaVoiceAgentsWebSocketAsyncClient() { + return new BetaVoiceAgentsWebSocketAsyncClient(createVoiceAgentWebSocketConfiguration()); } /** @@ -769,8 +769,8 @@ public BetaVoiceAgentWebSocketAsyncClient buildBetaVoiceAgentWebSocketAsyncClien * @throws IllegalStateException if unsupported HTTP pipeline configuration is present. */ @Beta - public BetaVoiceAgentWebSocketClient buildBetaVoiceAgentWebSocketClient() { - return new BetaVoiceAgentWebSocketClient(createVoiceAgentWebSocketConfiguration()); + public BetaVoiceAgentsWebSocketClient buildBetaVoiceAgentsWebSocketClient() { + return new BetaVoiceAgentsWebSocketClient(createVoiceAgentWebSocketConfiguration()); } /** @@ -783,8 +783,8 @@ public BetaVoiceAgentWebSocketClient buildBetaVoiceAgentWebSocketClient() { * @throws IllegalStateException if unsupported HTTP pipeline configuration is present. */ @Beta - public BetaVoiceAgentLiveWebSocketAsyncClient buildBetaVoiceAgentLiveWebSocketAsyncClient() { - return new BetaVoiceAgentLiveWebSocketAsyncClient(createVoiceAgentWebSocketConfiguration()); + public BetaVoiceAgentsLiveWebSocketAsyncClient buildBetaVoiceAgentsLiveWebSocketAsyncClient() { + return new BetaVoiceAgentsLiveWebSocketAsyncClient(createVoiceAgentWebSocketConfiguration()); } /** @@ -797,8 +797,8 @@ public BetaVoiceAgentLiveWebSocketAsyncClient buildBetaVoiceAgentLiveWebSocketAs * @throws IllegalStateException if unsupported HTTP pipeline configuration is present. */ @Beta - public BetaVoiceAgentLiveWebSocketClient buildBetaVoiceAgentLiveWebSocketClient() { - return new BetaVoiceAgentLiveWebSocketClient(createVoiceAgentWebSocketConfiguration()); + public BetaVoiceAgentsLiveWebSocketClient buildBetaVoiceAgentsLiveWebSocketClient() { + return new BetaVoiceAgentsLiveWebSocketClient(createVoiceAgentWebSocketConfiguration()); } /** @@ -983,8 +983,8 @@ private VoiceAgentWebSocketClientConfiguration createVoiceAgentWebSocketConfigur * * @return an asynchronous voice-agent WebSocket client. */ - private BetaVoiceAgentWebSocketAsyncClient buildBetaVoiceAgentWebSocketAsyncClient() { - return new BetaVoiceAgentWebSocketAsyncClient(createVoiceAgentWebSocketConfiguration()); + private BetaVoiceAgentsWebSocketAsyncClient buildBetaVoiceAgentsWebSocketAsyncClient() { + return new BetaVoiceAgentsWebSocketAsyncClient(createVoiceAgentWebSocketConfiguration()); } /** @@ -992,8 +992,8 @@ private BetaVoiceAgentWebSocketAsyncClient buildBetaVoiceAgentWebSocketAsyncClie * * @return a synchronous voice-agent WebSocket client. */ - private BetaVoiceAgentWebSocketClient buildBetaVoiceAgentWebSocketClient() { - return new BetaVoiceAgentWebSocketClient(createVoiceAgentWebSocketConfiguration()); + private BetaVoiceAgentsWebSocketClient buildBetaVoiceAgentsWebSocketClient() { + return new BetaVoiceAgentsWebSocketClient(createVoiceAgentWebSocketConfiguration()); } /** diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java similarity index 87% rename from sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketAsyncClient.java rename to sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java index 386e2fcb0b94a..1668fe1ee92e8 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java @@ -22,10 +22,10 @@ */ @ServiceClient(builder = AgentsClientBuilder.class, isAsync = true) @Beta(warningText = "This class is in preview and may change in future releases.") -public final class BetaVoiceAgentLiveWebSocketAsyncClient { +public final class BetaVoiceAgentsLiveWebSocketAsyncClient { private final VoiceAgentWebSocketClientConfiguration configuration; - BetaVoiceAgentLiveWebSocketAsyncClient(VoiceAgentWebSocketClientConfiguration configuration) { + BetaVoiceAgentsLiveWebSocketAsyncClient(VoiceAgentWebSocketClientConfiguration configuration) { this.configuration = Objects.requireNonNull(configuration, "'configuration' cannot be null."); } @@ -34,7 +34,7 @@ public final class BetaVoiceAgentLiveWebSocketAsyncClient { * @param agentName the Live voice agent name. * @return the connected session. Each subscription opens a new connection. */ - public Mono openWebSocketSession(String agentName) { + public Mono openWebSocketSession(String agentName) { return openWebSocketSession(agentName, new VoiceAgentLiveWebSocketConnectionOptions()); } @@ -45,7 +45,7 @@ public Mono openWebSocketSession( * @param options the connection options, copied when this method is called. * @return the connected session. Each subscription opens a new connection. */ - public Mono openWebSocketSession(String agentName, + public Mono openWebSocketSession(String agentName, VoiceAgentLiveWebSocketConnectionOptions options) { Objects.requireNonNull(agentName, "'agentName' cannot be null."); VoiceAgentWebSocketConnectionOptions snapshot = VoiceAgentWebSocketUtils.liveOptions(options); @@ -55,7 +55,7 @@ public Mono openWebSocketSession( HttpClient.create(), null, token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, token), json -> OpenAIJsonHelper.fromBinaryData(VoiceAgentWebSocketUtils.deserializeRawEvent(json), ServerEvent.class)); - return transport.connect().thenReturn(new BetaVoiceAgentLiveWebSocketSessionAsyncClient(transport)); + return transport.connect().thenReturn(new BetaVoiceAgentsLiveWebSocketSessionAsyncClient(transport)); }); } } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java similarity index 85% rename from sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketClient.java rename to sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java index 4e497d4517914..833360098cd67 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java @@ -21,11 +21,11 @@ */ @ServiceClient(builder = AgentsClientBuilder.class) @Beta(warningText = "This class is in preview and may change in future releases.") -public final class BetaVoiceAgentLiveWebSocketClient { - private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentLiveWebSocketClient.class); +public final class BetaVoiceAgentsLiveWebSocketClient { + private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentsLiveWebSocketClient.class); private final VoiceAgentWebSocketClientConfiguration configuration; - BetaVoiceAgentLiveWebSocketClient(VoiceAgentWebSocketClientConfiguration configuration) { + BetaVoiceAgentsLiveWebSocketClient(VoiceAgentWebSocketClientConfiguration configuration) { this.configuration = Objects.requireNonNull(configuration, "'configuration' cannot be null."); } @@ -34,7 +34,7 @@ public final class BetaVoiceAgentLiveWebSocketClient { * @param agentName the Live voice agent name. * @return the connected session. */ - public BetaVoiceAgentLiveWebSocketSessionClient openWebSocketSession(String agentName) { + public BetaVoiceAgentsLiveWebSocketSessionClient openWebSocketSession(String agentName) { return openWebSocketSession(agentName, new VoiceAgentLiveWebSocketConnectionOptions()); } @@ -46,14 +46,14 @@ public BetaVoiceAgentLiveWebSocketSessionClient openWebSocketSession(String agen * @return the connected session. * @throws IllegalArgumentException if the agent name is empty. */ - public BetaVoiceAgentLiveWebSocketSessionClient openWebSocketSession(String agentName, + public BetaVoiceAgentsLiveWebSocketSessionClient openWebSocketSession(String agentName, VoiceAgentLiveWebSocketConnectionOptions options) { Objects.requireNonNull(agentName, "'agentName' cannot be null."); if (agentName.isEmpty()) { throw LOGGER.logExceptionAsError(new IllegalArgumentException("'agentName' cannot be empty.")); } VoiceAgentWebSocketConnectionOptions snapshot = VoiceAgentWebSocketUtils.liveOptions(options); - return new BetaVoiceAgentLiveWebSocketSessionClient(WebSocketSessionTransport.connect(configuration, + return new BetaVoiceAgentsLiveWebSocketSessionClient(WebSocketSessionTransport.connect(configuration, VoiceAgentWebSocketUtils.buildLiveWebSocketUri(configuration, agentName, snapshot), snapshot, null, token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, token), json -> OpenAIJsonHelper .fromBinaryData(VoiceAgentWebSocketUtils.deserializeRawEvent(json), ServerEvent.class))); diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionAsyncClient.java similarity index 95% rename from sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java rename to sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionAsyncClient.java index 46241ee2c7e92..6bdbc0672b861 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionAsyncClient.java @@ -22,10 +22,10 @@ * Typed and raw receivers share one event subscriber. Close sessions when they are no longer needed. */ @Beta(warningText = "This class is in preview and may change in future releases.") -public final class BetaVoiceAgentLiveWebSocketSessionAsyncClient implements AsyncCloseable, AutoCloseable { +public final class BetaVoiceAgentsLiveWebSocketSessionAsyncClient implements AsyncCloseable, AutoCloseable { private final WebSocketSessionAsyncTransport transport; - BetaVoiceAgentLiveWebSocketSessionAsyncClient(WebSocketSessionAsyncTransport transport) { + BetaVoiceAgentsLiveWebSocketSessionAsyncClient(WebSocketSessionAsyncTransport transport) { this.transport = transport; } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java similarity index 96% rename from sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java rename to sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java index 05c1f9d4ff95a..e70a68b5b7b6a 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentLiveWebSocketSessionClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java @@ -21,10 +21,10 @@ * Typed and raw receivers share one event iterator. Close sessions when they are no longer needed. */ @Beta(warningText = "This class is in preview and may change in future releases.") -public final class BetaVoiceAgentLiveWebSocketSessionClient implements AutoCloseable { +public final class BetaVoiceAgentsLiveWebSocketSessionClient implements AutoCloseable { private final WebSocketSessionTransport transport; - BetaVoiceAgentLiveWebSocketSessionClient(WebSocketSessionTransport transport) { + BetaVoiceAgentsLiveWebSocketSessionClient(WebSocketSessionTransport transport) { this.transport = transport; } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketAsyncClient.java similarity index 77% rename from sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketAsyncClient.java rename to sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketAsyncClient.java index 4b5ed49dece92..ddb5caa844524 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketAsyncClient.java @@ -16,10 +16,10 @@ */ @ServiceClient(builder = AgentsClientBuilder.class, isAsync = true) @Beta(warningText = "This class is in preview and may change in future releases.") -public final class BetaVoiceAgentWebSocketAsyncClient { +public final class BetaVoiceAgentsWebSocketAsyncClient { private final VoiceAgentWebSocketClientConfiguration configuration; - BetaVoiceAgentWebSocketAsyncClient(VoiceAgentWebSocketClientConfiguration configuration) { + BetaVoiceAgentsWebSocketAsyncClient(VoiceAgentWebSocketClientConfiguration configuration) { this.configuration = Objects.requireNonNull(configuration, "'configuration' cannot be null."); } @@ -29,7 +29,7 @@ public final class BetaVoiceAgentWebSocketAsyncClient { * @param agentName the voice agent name. * @return a connected session. */ - public Mono openWebSocketSession(String agentName) { + public Mono openWebSocketSession(String agentName) { return openWebSocketSession(agentName, new VoiceAgentWebSocketConnectionOptions()); } @@ -40,14 +40,14 @@ public Mono openWebSocketSession(Stri * @param options connection options. * @return a connected session. */ - public Mono openWebSocketSession(String agentName, + public Mono openWebSocketSession(String agentName, VoiceAgentWebSocketConnectionOptions options) { Objects.requireNonNull(agentName, "'agentName' cannot be null."); Objects.requireNonNull(options, "'options' cannot be null."); VoiceAgentWebSocketConnectionOptions optionsSnapshot = new VoiceAgentWebSocketConnectionOptions(options); return Mono.defer(() -> { - BetaVoiceAgentWebSocketSessionAsyncClient session - = new BetaVoiceAgentWebSocketSessionAsyncClient(configuration, agentName, optionsSnapshot); + BetaVoiceAgentsWebSocketSessionAsyncClient session + = new BetaVoiceAgentsWebSocketSessionAsyncClient(configuration, agentName, optionsSnapshot); return session.connect().thenReturn(session); }); } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketClient.java similarity index 81% rename from sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketClient.java rename to sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketClient.java index 232c956f84313..5a54c46d1e491 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketClient.java @@ -16,11 +16,11 @@ */ @ServiceClient(builder = AgentsClientBuilder.class) @Beta(warningText = "This class is in preview and may change in future releases.") -public final class BetaVoiceAgentWebSocketClient { - private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentWebSocketClient.class); +public final class BetaVoiceAgentsWebSocketClient { + private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentsWebSocketClient.class); private final VoiceAgentWebSocketClientConfiguration configuration; - BetaVoiceAgentWebSocketClient(VoiceAgentWebSocketClientConfiguration configuration) { + BetaVoiceAgentsWebSocketClient(VoiceAgentWebSocketClientConfiguration configuration) { this.configuration = Objects.requireNonNull(configuration, "'configuration' cannot be null."); } @@ -30,7 +30,7 @@ public final class BetaVoiceAgentWebSocketClient { * @param agentName the voice agent name. * @return a connected session. */ - public BetaVoiceAgentWebSocketSessionClient openWebSocketSession(String agentName) { + public BetaVoiceAgentsWebSocketSessionClient openWebSocketSession(String agentName) { return openWebSocketSession(agentName, new VoiceAgentWebSocketConnectionOptions()); } @@ -42,14 +42,14 @@ public BetaVoiceAgentWebSocketSessionClient openWebSocketSession(String agentNam * @throws IllegalArgumentException if {@code agentName} is empty. * @return a connected session. */ - public BetaVoiceAgentWebSocketSessionClient openWebSocketSession(String agentName, + public BetaVoiceAgentsWebSocketSessionClient openWebSocketSession(String agentName, VoiceAgentWebSocketConnectionOptions options) { Objects.requireNonNull(agentName, "'agentName' cannot be null."); if (agentName.isEmpty()) { throw LOGGER.logExceptionAsError(new IllegalArgumentException("'agentName' cannot be empty.")); } Objects.requireNonNull(options, "'options' cannot be null."); - return BetaVoiceAgentWebSocketSessionClient.connect(configuration, agentName, + return BetaVoiceAgentsWebSocketSessionClient.connect(configuration, agentName, new VoiceAgentWebSocketConnectionOptions(options)); } } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java similarity index 95% rename from sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java rename to sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java index fc3e7d04ce012..a2accd93c6689 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java @@ -34,19 +34,19 @@ /** * An asynchronous bidirectional realtime session connected to a Foundry voice agent. * - *

    Instances are created by {@link BetaVoiceAgentWebSocketAsyncClient#openWebSocketSession(String)}. A session supports one + *

    Instances are created by {@link BetaVoiceAgentsWebSocketAsyncClient#openWebSocketSession(String)}. A session supports one * subscriber to {@link #receiveEvents()}. Close the session when it is no longer needed.

    */ @Beta(warningText = "This class is in preview and may change in future releases.") -public final class BetaVoiceAgentWebSocketSessionAsyncClient implements AsyncCloseable, AutoCloseable { +public final class BetaVoiceAgentsWebSocketSessionAsyncClient implements AsyncCloseable, AutoCloseable { private final WebSocketSessionAsyncTransport transport; - BetaVoiceAgentWebSocketSessionAsyncClient(VoiceAgentWebSocketClientConfiguration configuration, String agentName, + BetaVoiceAgentsWebSocketSessionAsyncClient(VoiceAgentWebSocketClientConfiguration configuration, String agentName, VoiceAgentWebSocketConnectionOptions options) { this(configuration, agentName, options, HttpClient.create()); } - BetaVoiceAgentWebSocketSessionAsyncClient(VoiceAgentWebSocketClientConfiguration configuration, String agentName, + BetaVoiceAgentsWebSocketSessionAsyncClient(VoiceAgentWebSocketClientConfiguration configuration, String agentName, VoiceAgentWebSocketConnectionOptions options, HttpClient httpClient) { VoiceAgentWebSocketConnectionOptions sessionOptions = options == null ? new VoiceAgentWebSocketConnectionOptions() : options; diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java similarity index 95% rename from sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java rename to sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java index 118cf90909cbe..ec844ef8483b4 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java @@ -34,17 +34,17 @@ * A synchronous bidirectional realtime session connected to a Foundry voice agent. */ @Beta(warningText = "This class is in preview and may change in future releases.") -public final class BetaVoiceAgentWebSocketSessionClient implements AutoCloseable { - private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentWebSocketSessionClient.class); +public final class BetaVoiceAgentsWebSocketSessionClient implements AutoCloseable { + private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentsWebSocketSessionClient.class); private final WebSocketSessionTransport transport; - private BetaVoiceAgentWebSocketSessionClient(WebSocketSessionTransport transport) { + private BetaVoiceAgentsWebSocketSessionClient(WebSocketSessionTransport transport) { this.transport = transport; } - static BetaVoiceAgentWebSocketSessionClient connect(VoiceAgentWebSocketClientConfiguration configuration, + static BetaVoiceAgentsWebSocketSessionClient connect(VoiceAgentWebSocketClientConfiguration configuration, String agentName, VoiceAgentWebSocketConnectionOptions options) { - return new BetaVoiceAgentWebSocketSessionClient(WebSocketSessionTransport.connect(configuration, + return new BetaVoiceAgentsWebSocketSessionClient(WebSocketSessionTransport.connect(configuration, VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, options), options, VoiceAgentWebSocketUtils.SUBPROTOCOL, token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, options, token), diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java index 59a2904a85a62..60a6f3b4c2f3e 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java @@ -74,17 +74,20 @@ public interface EventDecoder { } private WebSocketSessionTransport(VoiceAgentWebSocketClientConfiguration configuration, URI websocketUri, - VoiceAgentWebSocketConnectionOptions options, String subprotocol, Function headersFactory, - EventDecoder decoder) { + VoiceAgentWebSocketConnectionOptions options, EventDecoder decoder) { this.options = options; this.decoder = Objects.requireNonNull(decoder, "'decoder' cannot be null."); this.receiveBufferCapacity = options.getReceiveBufferCapacity(); this.events = new ArrayBlockingQueue<>(receiveBufferCapacity + 1); this.websocketUri = Objects.requireNonNull(websocketUri, "'websocketUri' cannot be null."); + this.httpClient = createHttpClient(configuration, options); + } + + private void open(VoiceAgentWebSocketClientConfiguration configuration, String subprotocol, + Function headersFactory) { String token = configuration.getCredential() .getTokenSync(VoiceAgentWebSocketUtils.createTokenRequestContext()) .getToken(); - this.httpClient = createHttpClient(configuration, options); Request.Builder request = new Request.Builder().url(websocketUri.toString()); if (subprotocol != null) { request.header("Sec-WebSocket-Protocol", subprotocol); @@ -98,13 +101,16 @@ private WebSocketSessionTransport(VoiceAgentWebSocketClientConfiguration configu public static WebSocketSessionTransport connect(VoiceAgentWebSocketClientConfiguration configuration, URI websocketUri, VoiceAgentWebSocketConnectionOptions options, String subprotocol, Function headersFactory, EventDecoder decoder) { - WebSocketSessionTransport session = new WebSocketSessionTransport<>(configuration, websocketUri, options, - subprotocol, headersFactory, decoder); + WebSocketSessionTransport session + = new WebSocketSessionTransport<>(configuration, websocketUri, options, decoder); try { + session.open(configuration, subprotocol, headersFactory); session.awaitHandshake(); return session; } catch (RuntimeException error) { - session.webSocket.cancel(); + if (session.webSocket != null) { + session.webSocket.cancel(); + } session.shutdownHttpClient(); throw error; } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java index f7884be7b05e1..3bd1dfd30c2ec 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java @@ -27,14 +27,14 @@ import com.openai.services.blocking.ConversationService; public final class ReadmeSamples { - public void realtimeForwardCompatibility(BetaVoiceAgentWebSocketClient realtimeClient, String agentName) { + public void realtimeForwardCompatibility(BetaVoiceAgentsWebSocketClient realtimeClient, String agentName) { // BEGIN: com.azure.ai.agents.realtime_forward_compatibility VoiceAgentWebSocketConnectionOptions options = new VoiceAgentWebSocketConnectionOptions() .setReceiveBufferCapacity(512) .setMaxMessageSize(8 * 1024 * 1024) .setOverflowStrategy(VoiceAgentWebSocketOverflowStrategy.ERROR); - try (BetaVoiceAgentWebSocketSessionClient session = realtimeClient.openWebSocketSession(agentName, options)) { + try (BetaVoiceAgentsWebSocketSessionClient session = realtimeClient.openWebSocketSession(agentName, options)) { session.sendEvent(BinaryData.fromString( "{\"type\":\"response.create\",\"event_id\":\"response-1\"}")); for (RealtimeServerEvent event : session.receiveEvents()) { diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java index a011f6a5ae106..0a59abaffd26a 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveAudioConversationAsyncSample.java @@ -3,13 +3,25 @@ package com.azure.ai.agents.voice; -import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsLiveWebSocketSessionAsyncClient; import com.azure.ai.agents.AgentsClientBuilder; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.core.util.Configuration; import com.azure.identity.DefaultAzureCredentialBuilder; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; +import com.openai.models.live.ClientEvent; +import com.openai.models.live.ServerEvent; +import com.openai.models.responses.ResponseStreamEvent; +import java.io.InputStream; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Sinks; + /** * Streams microphone and speaker audio over a native GPT Live WebSocket. @@ -60,15 +72,145 @@ public static void main(String[] args) { .allowPreview(true); try (GptLiveSampleUtils.AgentResource agent = GptLiveSampleUtils.createAgent(builder, model, backendModel, "audio-async", false, keepAgent)) { - Mono.usingWhen(agent.builder.beta().buildBetaVoiceAgentLiveWebSocketAsyncClient() + Mono.usingWhen(agent.builder.beta().buildBetaVoiceAgentsLiveWebSocketAsyncClient() .openWebSocketSession(agent.name, new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store)), - GptLiveSampleUtils::runAudioConversationAsync, - BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync, + GptLiveAudioConversationAsyncSample::runAudioConversationAsync, + BetaVoiceAgentsLiveWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), - BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(); + BetaVoiceAgentsLiveWebSocketSessionAsyncClient::closeAsync).block(); } finally { Schedulers.shutdownNow(); } } + static Mono runAudioConversationAsync(BetaVoiceAgentsLiveWebSocketSessionAsyncClient session) { + return Mono.defer(() -> runAudioConversationAsync(session, + new GptLiveSampleUtils.AudioProcessor( + null, null, pcm -> session.sendEvent(GptLiveSampleUtils.audio(pcm.toBytes()))), System.in)); + } + + static Mono runAudioConversationAsync(BetaVoiceAgentsLiveWebSocketSessionAsyncClient session, + GptLiveSampleUtils.AudioProcessor processor, InputStream input) { + return Mono.usingWhen(Mono.just(processor), + audio -> Mono.firstWithSignal(runConversationAsync(session.receiveEvents(), session::sendEvent, + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), + transcripts -> VoiceAgentLiveAudioConversationAsyncSample.waitForEnter(input) + .then(closeAudioAsync(audio)).then(Mono.empty()), + audio::queueAudio, () -> { }, true, audio), audio.failureSignalAsync()), GptLiveAudioConversationAsyncSample::closeAudioAsync, + (audio, error) -> closeAudioAsync(audio), GptLiveAudioConversationAsyncSample::closeAudioAsync); + } + + private static Mono closeAudioAsync(GptLiveSampleUtils.AudioProcessor audio) { + return Mono.fromRunnable(audio::close).subscribeOn(Schedulers.boundedElastic()); + } + + private static Mono runConversationAsync(Flux events, + Function> sendEvent, Supplier closeDetails, + Function> nextPrompt, + Consumer playAudio, Runnable drainAudio, boolean spoken, GptLiveSampleUtils.AudioProcessor microphone) { + Mono opening = microphone == null ? Mono.empty() + : Mono.defer(() -> sendEvent.apply(GptLiveSampleUtils.userMessage("Greet me briefly with: Hi, how can I help you today?")) + .then(sendEvent.apply(GptLiveSampleUtils.responseCreate())) + .then(Mono.fromRunnable(() -> System.out.println( + "Opening greeting requested. Speak into the microphone. Press Enter to end the call.")))); + return runConversationCoreAsync(events, sendEvent, closeDetails, nextPrompt, playAudio, drainAudio, spoken, + spoken && microphone == null, microphone == null && System.console() != null, + () -> Mono.fromRunnable(() -> { + if (microphone != null) { + microphone.start(); + } + }), opening, backend -> Mono.just(false), reason -> microphone != null); + } + + static Mono runConversationCoreAsync(Flux events, + Function> sendEvent, Supplier closeDetails, + Function> nextPrompt, + Consumer playAudio, Runnable drainAudio, boolean spoken, boolean interactive, boolean inputEchoed, + Supplier> onStarted, Mono opening, + Function> handleResponse, Predicate acceptClose) { + return Mono.defer(() -> { + GptLiveSampleUtils.AudioTranscriptPrinter transcripts = new GptLiveSampleUtils.AudioTranscriptPrinter(inputEchoed, interactive); + GptLiveSampleUtils.Greeting greeting = new GptLiveSampleUtils.Greeting(); + Sinks.Empty ready = Sinks.empty(); + Sinks.Empty closed = Sinks.empty(); + Sinks.Empty audioStarted = Sinks.empty(); + Sinks.Empty stopAudio = Sinks.empty(); + Sinks.Empty audioStopped = Sinks.empty(); + AtomicReference> responseCompleted = new AtomicReference<>(); + GptLiveSampleUtils.ConversationEventHandler> handler + = new GptLiveSampleUtils.ConversationEventHandler>(spoken, closeDetails, acceptClose) { + @Override + Mono onStarted() { + return Mono.defer(onStarted).doOnSuccess(ignored -> ready.tryEmitEmpty()); + } + + @Override + Mono onResponse(ResponseStreamEvent event) { + return Mono.defer(() -> handleResponse.apply(event)).defaultIfEmpty(false) + .doOnNext(continuation -> { + if (event.isCompleted() && completeResponse(continuation)) { + responseCompleted.getAndSet(null).tryEmitEmpty(); + } + }).then(); + } + + @Override + Mono onClosed() { + return Mono.fromRunnable(() -> closed.tryEmitEmpty()); + } + + @Override + Mono ignored() { + return Mono.empty(); + } + }; + Mono receive = events + .publishOn(Schedulers.boundedElastic(), 1) + .takeUntil(ServerEvent::isSessionClosed) + .concatMap(event -> Mono.defer(() -> { + if (spoken) { + transcripts.print(event); + if (event.isSessionOutputAudioDelta()) { + byte[] pcm = GptLiveSampleUtils.decodeAudio(event); + playAudio.accept(pcm); + greeting.onAudio(pcm); + } else if (event.isSessionOutputTranscriptDelta() + && !event.asSessionOutputTranscriptDelta().delta().trim().isEmpty()) { + greeting.onTranscript(); + } + } else { + GptLiveSampleUtils.printEvent(event); + } + return handler.process(event); + }), 1) + .then(Mono.defer(() -> handler.awaitConversationAfterReceive() ? Mono.never() : Mono.empty())); + Mono audio = ready.asMono() + .thenMany(GptLiveSampleUtils.silenceFramesAsync().takeUntilOther(stopAudio.asMono()) + .concatMap(frame -> sendEvent.apply(frame).doOnSuccess(ignored -> audioStarted.tryEmitEmpty()), 1)) + .then() + .doOnSuccess(ignored -> audioStopped.tryEmitEmpty()) + .then(Mono.never()); + Mono renderTranscripts = greeting.quietPeriodsAsync() + .doOnNext(ignored -> transcripts.endUtterance()).then(); + Mono conversation = ready.asMono().timeout(GptLiveSampleUtils.TIMEOUT) + .then(Mono.defer(() -> interactive ? audioStarted.asMono().timeout(GptLiveSampleUtils.TIMEOUT) : Mono.empty())) + .then(Mono.defer(() -> interactive + ? sendEvent.apply(GptLiveSampleUtils.commentary("Hi, how can I help you today?")) + .timeout(GptLiveSampleUtils.TIMEOUT) + .then(greeting.awaitPlaybackAsync(drainAudio)) + .then(Mono.fromRunnable(transcripts::endUtterance)) + : opening)) + .then(Mono.defer(() -> GptLiveSampleUtils.promptAsync(sendEvent, nextPrompt, responseCompleted, spoken, transcripts, handler))) + .then(Mono.defer(() -> { + handler.requestClose(); + stopAudio.tryEmitEmpty(); + return (interactive ? audioStopped.asMono() : Mono.empty()) + .then(sendEvent.apply(GptLiveSampleUtils.closeSession())) + .then(closed.asMono()).timeout(GptLiveSampleUtils.TIMEOUT); + })); + return interactive ? Mono.firstWithSignal(receive, audio, renderTranscripts, conversation) + : Mono.firstWithSignal(receive, conversation); + }); + } + } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java index 74417bfa77622..43af2ff27b305 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveFunctionToolSample.java @@ -3,17 +3,25 @@ package com.azure.ai.agents.voice; -import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; +import com.azure.ai.agents.BetaVoiceAgentsLiveWebSocketSessionClient; import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.models.LiveFunctionToolInputParam; import com.azure.core.util.BinaryData; import com.azure.core.util.Configuration; import com.azure.identity.DefaultAzureCredentialBuilder; import com.openai.models.live.ClientEvent; import com.openai.models.live.ResponseItemCreateEvent; +import com.openai.models.live.ServerEvent; import com.openai.models.responses.ResponseInputItem; +import com.openai.models.responses.ResponseFunctionToolCall; +import com.openai.models.responses.ResponseStreamEvent; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Iterator; +import java.util.List; +import java.util.function.Function; /** * Executes a sample weather function requested by the GPT Live Responses backend. @@ -59,46 +67,132 @@ public static void main(String[] args) { .allowPreview(true); try (GptLiveSampleUtils.AgentResource agent = GptLiveSampleUtils.createAgent(builder, model, backendModel, "tools", true, keepAgent); - BetaVoiceAgentLiveWebSocketSessionClient session = agent.builder.beta() - .buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession(agent.name)) { - GptLiveSampleUtils.runToolConversation(session, + BetaVoiceAgentsLiveWebSocketSessionClient session = agent.builder.beta() + .buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession(agent.name)) { + runConversation(session, Collections.singletonList(args.length == 0 ? "What is the weather in Seattle?" : args[0]), GptLiveFunctionToolSample::toolResult); } } - static BinaryData weatherTool() { - Map schema = new LinkedHashMap<>(); - schema.put("type", "object"); - schema.put("properties", Collections.singletonMap("city", Collections.singletonMap("type", "string"))); - schema.put("required", Collections.singletonList("city")); - schema.put("additionalProperties", false); - Map tool = new LinkedHashMap<>(); - tool.put("type", "function"); - tool.put("name", "get_weather"); - tool.put("description", "Get simulated weather for a city. This sample returns demonstration data only."); - tool.put("parameters", schema); - tool.put("strict", true); - return BinaryData.fromObject(tool); + static void runConversation(BetaVoiceAgentsLiveWebSocketSessionClient session, List prompts, + Function toolHandler) { + Iterator pending = prompts.iterator(); + List results = new ArrayList<>(); + boolean started = false; + boolean awaitingResponse = false; + boolean closing = false; + int continuations = 0; + for (ServerEvent event : session.receiveEvents(GptLiveSampleUtils.TIMEOUT)) { + GptLiveSampleUtils.printEvent(event); + boolean nextTurn = false; + if (event.isSessionStarted()) { + if (started) { + throw new IllegalStateException("Duplicate session.started event."); + } + started = true; + nextTurn = true; + } else if (event.isResponseEvent()) { + ResponseStreamEvent response = GptLiveSampleUtils.backendEvent(event); + if (response.isOutputItemDone() && response.asOutputItemDone().item().isFunctionCall()) { + ResponseFunctionToolCall call = response.asOutputItemDone().item().asFunctionCall(); + results.add(toolHandler.apply(call)); + } else if (response.isCompleted()) { + if (!started || !awaitingResponse || closing) { + throw new IllegalStateException("Unexpected GPT Live response lifecycle event."); + } + if (!results.isEmpty()) { + if (++continuations > 8) { + throw new IllegalStateException("Sample tool continuation limit exceeded."); + } + for (ClientEvent result : results) { + session.sendEvent(result); + } + results.clear(); + session.sendEvent(GptLiveSampleUtils.responseCreate()); + } else { + continuations = 0; + awaitingResponse = false; + System.out.println(); + nextTurn = true; + } + } + } else if (event.isSessionClosed()) { + String reason = event.asSessionClosed().reason().toString(); + if (!(started && "remote_hangup".equals(reason)) + && !(closing && "close_requested".equals(reason))) { + throw new IllegalStateException("GPT Live closed before all turns completed. Event: " + event + + "; transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason()); + } + System.out.println("Live session closed (" + reason + ")."); + return; + } + if (nextTurn) { + if (pending.hasNext()) { + String prompt = pending.next(); + System.out.println("You: " + prompt); + System.out.print("Assistant: "); + awaitingResponse = true; + session.sendEvent(GptLiveSampleUtils.userMessage(prompt)); + session.sendEvent(GptLiveSampleUtils.responseCreate()); + } else { + closing = true; + session.sendEvent(GptLiveSampleUtils.closeSession()); + } + } + } + throw new IllegalStateException("GPT Live disconnected without session.closed."); + } + + static LiveFunctionToolInputParam weatherTool() { + Map schema = new LinkedHashMap<>(); + schema.put("type", BinaryData.fromObject("object")); + schema.put("properties", BinaryData.fromObject( + Collections.singletonMap("city", Collections.singletonMap("type", "string")))); + schema.put("required", BinaryData.fromObject(Collections.singletonList("city"))); + schema.put("additionalProperties", BinaryData.fromObject(false)); + return new LiveFunctionToolInputParam("get_weather") + .setDescription("Get simulated weather for a city. This sample returns demonstration data only.") + .setParameters(schema) + .setStrict(true); } - static ClientEvent toolResult(Map call) { - if (!"get_weather".equals(GptLiveSampleUtils.string(call, "name"))) { - throw new IllegalArgumentException("Unsupported sample tool: " + call.get("name")); + static ClientEvent toolResult(ResponseFunctionToolCall call) { + if (!"get_weather".equals(call.name())) { + throw new IllegalArgumentException("Unsupported sample tool: " + call.name()); } - Map arguments = GptLiveSampleUtils.object( - BinaryData.fromString(GptLiveSampleUtils.string(call, "arguments"))); + Map arguments = object( + BinaryData.fromString(call.arguments())); Map result = new LinkedHashMap<>(); - result.put("city", GptLiveSampleUtils.string(arguments, "city")); + result.put("city", string(arguments, "city")); result.put("condition", "sunny"); result.put("temperature_f", 72); result.put("simulated", true); return ClientEvent.ofResponseItemCreate(ResponseItemCreateEvent.builder() .item(ResponseInputItem.FunctionCallOutput.builder() - .callId(GptLiveSampleUtils.string(call, "call_id")) + .callId(call.callId()) .output(BinaryData.fromObject(result).toString()) .build()) .build()); } + static Map object(BinaryData data) { + return data.toObject(Map.class); + } + + static Map object(Object value) { + if (!(value instanceof Map)) { + throw new IllegalArgumentException("Expected a native Live JSON object: " + value); + } + return (Map) value; + } + + static String string(Map event, String field) { + Object value = event.get(field); + if (!(value instanceof String)) { + throw new IllegalArgumentException("Missing or invalid Live event field: " + field); + } + return (String) value; + } + } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java index a527ea6eab8db..dd0793132d320 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java @@ -5,9 +5,6 @@ import com.azure.ai.agents.AgentsClient; import com.azure.ai.agents.AgentsClientBuilder; -import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; -import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; -import com.azure.ai.agents.implementation.OpenAIJsonHelper; import com.azure.ai.agents.models.CreateAgentVersionInput; import com.azure.ai.agents.models.LiveInitialSessionAudioParam; import com.azure.ai.agents.models.LiveResponsesDelegationParam; @@ -29,32 +26,19 @@ import com.openai.models.responses.EasyInputMessage; import com.openai.models.responses.ResponseInputContent; import com.openai.models.responses.ResponseInputText; -import com.openai.models.responses.ResponseOutputItem; import com.openai.models.responses.ResponseStreamEvent; import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; import java.time.Duration; -import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; -import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -82,249 +66,7 @@ static String requireSetting(Configuration configuration, String name) { return value; } - // #region Synchronous conversation - - /** - * Scripted-text entry point called by {@link GptLiveTextConversationSample} when command-line prompts are supplied. - */ - static void runTextConversation(BetaVoiceAgentLiveWebSocketSessionClient session, List prompts) { - runToolConversation(session, prompts, null); - } - - /** - * Called by {@link GptLiveFunctionToolSample}; also serves scripted {@link GptLiveTextConversationSample} calls - * without a tool handler. - */ - static void runToolConversation(BetaVoiceAgentLiveWebSocketSessionClient session, List prompts, - Function, ClientEvent> toolHandler) { - Iterator pending = prompts.iterator(); - runConversation(session.receiveEvents(TIMEOUT), session::sendEvent, - () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), transcripts -> { - if (!pending.hasNext()) { - return null; - } - String prompt = pending.next(); - System.out.println("You: " + prompt); - return prompt; - }, pcm -> { }, () -> { }, false, toolHandler); - } - - /** - * Interactive entry point called by {@link GptLiveTextConversationSample} when no command-line prompts are supplied. - */ - static void runInteractiveConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Reader input) { - try (VoiceAgentRealtimeSampleUtils.SpeakerPlayer player = new VoiceAgentRealtimeSampleUtils.SpeakerPlayer()) { - runInteractiveConversation(session, input, player::play, player::drain); - player.drain(); - System.out.printf("(received %.2fs of reply audio%s)%n", player.getSecondsReceived(), - player.isEnabled() ? " and played it" : ""); - } - } - - static void runInteractiveConversation(BetaVoiceAgentLiveWebSocketSessionClient session, Reader input, - Consumer playAudio, Runnable drainAudio) { - System.out.println("Type a message and press Enter. Blank line, 'exit' or 'quit' ends the session."); - System.out.println("Interactive voice streams silent audio while you type; voice usage is billable."); - BufferedReader console = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); - runConversation(session.receiveEvents(TIMEOUT), session::sendEvent, - () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), transcripts -> { - try { - return readPromptBlocking(console, transcripts); - } catch (IOException error) { - throw new IllegalStateException("Unable to read console input.", error); - } - }, playAudio, drainAudio, true, null); - } - - static void runConversation(Iterable events, Consumer sendEvent, - Supplier closeDetails, Function nextPrompt, - Consumer playAudio, Runnable drainAudio, boolean spoken, - Function, ClientEvent> toolHandler) { - List results = new ArrayList<>(); - AtomicInteger continuations = new AtomicInteger(); - runConversationCore(events, sendEvent, closeDetails, nextPrompt, playAudio, drainAudio, spoken, () -> { }, backend -> { - if (toolHandler != null && backend.isOutputItemDone()) { - ResponseOutputItem item = backend.asOutputItemDone().item(); - if (item.isFunctionCall()) { - results.add(toolHandler.apply(object(OpenAIJsonHelper.toBinaryData(item.asFunctionCall())))); - } - } else if (backend.isCompleted()) { - if (results.isEmpty()) { - continuations.set(0); - return false; - } - if (continuations.incrementAndGet() > 8) { - throw new IllegalStateException("Sample tool continuation limit exceeded."); - } - for (ClientEvent result : results) { - sendEvent.accept(result); - } - results.clear(); - sendEvent.accept(responseCreate()); - return true; - } - return false; - }, reason -> false); - } - - static void runConversationCore(Iterable events, Consumer sendEvent, - Supplier closeDetails, Function nextPrompt, - Consumer playAudio, Runnable drainAudio, boolean spoken, - Runnable onStarted, Predicate handleResponse, Predicate acceptClose) { - ConversationEventHandler handler - = new ConversationEventHandler(spoken, closeDetails, acceptClose) { - @Override - Boolean onStarted() { - onStarted.run(); - return false; - } - - @Override - Boolean onResponse(ResponseStreamEvent event) { - boolean continuation = handleResponse.test(event); - if (event.isCompleted()) { - completeResponse(continuation); - } - return false; - } - - @Override - Boolean onClosed() { - return true; - } - - @Override - Boolean ignored() { - return false; - } - }; - ExecutorService workers = Executors.newFixedThreadPool(2, task -> { - Thread thread = new Thread(task, "gpt-live-sync-conversation"); - thread.setDaemon(true); - return thread; - }); - BlockingQueue pending = new LinkedBlockingQueue<>(256); - AtomicReference receiveFailure = new AtomicReference<>(); - AtomicBoolean receiveEnded = new AtomicBoolean(); - AudioTranscriptPrinter transcripts = new AudioTranscriptPrinter(System.console() != null, spoken); - Future receiver = workers.submit(() -> { - try { - for (ServerEvent event : events) { - pending.put(event); - if (event.isSessionClosed()) { - break; - } - } - } catch (InterruptedException error) { - Thread.currentThread().interrupt(); - } catch (RuntimeException error) { - receiveFailure.set(error); - } finally { - receiveEnded.set(true); - } - }); - Future prompt = null; - boolean greetingComplete = !spoken; - boolean greetingAudio = false; - boolean greetingTranscript = false; - long deadline = System.nanoTime() + TIMEOUT.toNanos(); - long nextAudio = 0; - long quietSince = 0; - ClientEvent silence = audio(new byte[960]); - try { - while (true) { - ServerEvent event = pending.poll(20, TimeUnit.MILLISECONDS); - long now = System.nanoTime(); - if (event != null) { - if (spoken) { - transcripts.print(event); - if (event.isSessionOutputAudioDelta()) { - byte[] pcm = decodeAudio(event); - playAudio.accept(pcm); - if (hasSpeech(pcm)) { - greetingAudio = true; - quietSince = now; - } - } else if (event.isSessionOutputTranscriptDelta() - && !event.asSessionOutputTranscriptDelta().delta().trim().isEmpty()) { - greetingTranscript = true; - quietSince = now; - } - } else { - printEvent(event); - } - if (handler.process(event)) { - return; - } - if (event.isSessionStarted()) { - if (spoken) { - sendEvent.accept(silence); - nextAudio = now + TimeUnit.MILLISECONDS.toNanos(20); - sendEvent.accept(commentary("Hi, how can I help you today?")); - } - } - } else if (receiveEnded.get()) { - if (receiveFailure.get() != null) { - throw receiveFailure.get(); - } - throw new IllegalStateException("GPT Live disconnected without session.closed."); - } - - if (handler.canStreamAudio() && now >= nextAudio) { - sendEvent.accept(silence); - nextAudio = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(20); - } - if (spoken && greetingAudio && greetingTranscript && quietSince != 0 - && now - quietSince >= TimeUnit.SECONDS.toNanos(1)) { - transcripts.endUtterance(); - quietSince = 0; - if (!greetingComplete) { - drainAudio.run(); - greetingComplete = true; - } - } - if (handler.canReadPrompt(greetingComplete) && prompt == null) { - prompt = workers.submit(() -> nextPrompt.apply(transcripts)); - } - if (prompt != null && prompt.isDone()) { - String text = prompt.get(); - prompt = null; - deadline = System.nanoTime() + TIMEOUT.toNanos(); - if (text == null) { - handler.requestClose(); - sendEvent.accept(closeSession()); - } else { - handler.beginResponse(); - if (!spoken) { - System.out.print("Assistant: "); - } - sendEvent.accept(userMessage(text)); - sendEvent.accept(responseCreate()); - } - } - if (!handler.canReadPrompt(greetingComplete) && now >= deadline) { - throw new IllegalStateException("Timed out waiting for the GPT Live conversation."); - } - } - } catch (InterruptedException error) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("GPT Live conversation interrupted.", error); - } catch (ExecutionException error) { - if (error.getCause() instanceof RuntimeException) { - throw (RuntimeException) error.getCause(); - } - throw new IllegalStateException("Unable to read console input.", error.getCause()); - } finally { - receiver.cancel(true); - if (prompt != null) { - prompt.cancel(true); - } - transcripts.endInput(); - workers.shutdownNow(); - } - } - - private static String readPromptBlocking(BufferedReader reader, AudioTranscriptPrinter transcripts) throws IOException { + static String readPromptBlocking(BufferedReader reader, AudioTranscriptPrinter transcripts) throws IOException { transcripts.beginInput(); try { String line = reader.readLine(); @@ -338,201 +80,7 @@ private static String readPromptBlocking(BufferedReader reader, AudioTranscriptP } } - // #endregion - - // #region Asynchronous conversation - - /** - * Microphone entry point called by {@link GptLiveAudioConversationAsyncSample}. - */ - static Mono runAudioConversationAsync(BetaVoiceAgentLiveWebSocketSessionAsyncClient session) { - return Mono.defer(() -> runAudioConversationAsync(session, - new AudioProcessor( - null, null, pcm -> session.sendEvent(audio(pcm.toBytes()))), System.in)); - } - - static Mono runAudioConversationAsync(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, - AudioProcessor processor, InputStream input) { - return Mono.usingWhen(Mono.just(processor), - audio -> Mono.firstWithSignal(runConversationAsync(session.receiveEvents(), session::sendEvent, - () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), - transcripts -> VoiceAgentLiveAudioConversationAsyncSample.waitForEnter(input) - .then(closeAudioAsync(audio)).then(Mono.empty()), - audio::queueAudio, () -> { }, true, audio), audio.failureSignalAsync()), GptLiveSampleUtils::closeAudioAsync, - (audio, error) -> closeAudioAsync(audio), GptLiveSampleUtils::closeAudioAsync); - } - - private static Mono closeAudioAsync(AudioProcessor audio) { - return Mono.fromRunnable(audio::close).subscribeOn(Schedulers.boundedElastic()); - } - - /** - * Scripted-text entry point called by {@link GptLiveTextConversationAsyncSample} with command-line prompts. - */ - static Mono runTextConversationAsync(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, List prompts) { - return Mono.defer(() -> { - Iterator pending = prompts.iterator(); - return runConversationAsync(session.receiveEvents(), session::sendEvent, - () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), - transcripts -> Mono.defer(() -> { - if (!pending.hasNext()) { - return Mono.empty(); - } - String prompt = pending.next(); - System.out.println("You: " + prompt); - return Mono.just(prompt); - }), pcm -> { }, () -> { }, false, null); - }); - } - - /** - * Interactive entry point called by {@link GptLiveTextConversationAsyncSample} without command-line prompts. - */ - static Mono runInteractiveConversationAsync(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input) { - return Mono.usingWhen( - Mono.fromCallable(VoiceAgentRealtimeSampleUtils.SpeakerPlayer::new) - .subscribeOn(Schedulers.boundedElastic()), - player -> runInteractiveConversationAsync(session, input, player::play, player::drain), - player -> Mono.fromRunnable(() -> { - boolean speakerEnabled = player.isEnabled(); - try { - player.close(); - } finally { - System.out.printf("(received %.2fs of reply audio%s)%n", - player.getSecondsReceived(), speakerEnabled ? " and played it" : ""); - } - }).subscribeOn(Schedulers.boundedElastic())); - } - - static Mono runInteractiveConversationAsync(BetaVoiceAgentLiveWebSocketSessionAsyncClient session, Reader input, - Consumer playAudio, Runnable drainAudio) { - return Mono.defer(() -> runInteractiveConversationAsync(session.receiveEvents(), session::sendEvent, - () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), - input, playAudio, drainAudio)); - } - - static Mono runInteractiveConversationAsync(Flux events, - Function> sendEvent, Supplier closeDetails, Reader input, - Consumer playAudio, Runnable drainAudio) { - return Mono.defer(() -> { - System.out.println("Type a message and press Enter. Blank line, 'exit' or 'quit' ends the session."); - System.out.println("Interactive voice streams silent audio while you type; voice usage is billable."); - BufferedReader console = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); - return runConversationAsync(events, sendEvent, closeDetails, transcripts -> readPromptAsync(console, transcripts), - playAudio, drainAudio, true, null); - }); - } - - private static Mono runConversationAsync(Flux events, - Function> sendEvent, Supplier closeDetails, - Function> nextPrompt, - Consumer playAudio, Runnable drainAudio, boolean spoken, AudioProcessor microphone) { - Mono opening = microphone == null ? Mono.empty() - : Mono.defer(() -> sendEvent.apply(userMessage("Greet me briefly with: Hi, how can I help you today?")) - .then(sendEvent.apply(responseCreate())) - .then(Mono.fromRunnable(() -> System.out.println( - "Opening greeting requested. Speak into the microphone. Press Enter to end the call.")))); - return runConversationCoreAsync(events, sendEvent, closeDetails, nextPrompt, playAudio, drainAudio, spoken, - spoken && microphone == null, microphone == null && System.console() != null, - () -> Mono.fromRunnable(() -> { - if (microphone != null) { - microphone.start(); - } - }), opening, backend -> Mono.just(false), reason -> microphone != null); - } - - static Mono runConversationCoreAsync(Flux events, - Function> sendEvent, Supplier closeDetails, - Function> nextPrompt, - Consumer playAudio, Runnable drainAudio, boolean spoken, boolean interactive, boolean inputEchoed, - Supplier> onStarted, Mono opening, - Function> handleResponse, Predicate acceptClose) { - return Mono.defer(() -> { - AudioTranscriptPrinter transcripts = new AudioTranscriptPrinter(inputEchoed, interactive); - Greeting greeting = new Greeting(); - Sinks.Empty ready = Sinks.empty(); - Sinks.Empty closed = Sinks.empty(); - Sinks.Empty audioStarted = Sinks.empty(); - Sinks.Empty stopAudio = Sinks.empty(); - Sinks.Empty audioStopped = Sinks.empty(); - AtomicReference> responseCompleted = new AtomicReference<>(); - ConversationEventHandler> handler - = new ConversationEventHandler>(spoken, closeDetails, acceptClose) { - @Override - Mono onStarted() { - return Mono.defer(onStarted).doOnSuccess(ignored -> ready.tryEmitEmpty()); - } - - @Override - Mono onResponse(ResponseStreamEvent event) { - return Mono.defer(() -> handleResponse.apply(event)).defaultIfEmpty(false) - .doOnNext(continuation -> { - if (event.isCompleted() && completeResponse(continuation)) { - responseCompleted.getAndSet(null).tryEmitEmpty(); - } - }).then(); - } - - @Override - Mono onClosed() { - return Mono.fromRunnable(() -> closed.tryEmitEmpty()); - } - - @Override - Mono ignored() { - return Mono.empty(); - } - }; - Mono receive = events - .publishOn(Schedulers.boundedElastic(), 1) - .takeUntil(ServerEvent::isSessionClosed) - .concatMap(event -> Mono.defer(() -> { - if (spoken) { - transcripts.print(event); - if (event.isSessionOutputAudioDelta()) { - byte[] pcm = decodeAudio(event); - playAudio.accept(pcm); - greeting.onAudio(pcm); - } else if (event.isSessionOutputTranscriptDelta() - && !event.asSessionOutputTranscriptDelta().delta().trim().isEmpty()) { - greeting.onTranscript(); - } - } else { - printEvent(event); - } - return handler.process(event); - }), 1) - .then(Mono.defer(() -> handler.awaitConversationAfterReceive() ? Mono.never() : Mono.empty())); - Mono audio = ready.asMono() - .thenMany(silenceFramesAsync().takeUntilOther(stopAudio.asMono()) - .concatMap(frame -> sendEvent.apply(frame).doOnSuccess(ignored -> audioStarted.tryEmitEmpty()), 1)) - .then() - .doOnSuccess(ignored -> audioStopped.tryEmitEmpty()) - .then(Mono.never()); - Mono renderTranscripts = greeting.quietPeriodsAsync() - .doOnNext(ignored -> transcripts.endUtterance()).then(); - Mono conversation = ready.asMono().timeout(TIMEOUT) - .then(Mono.defer(() -> interactive ? audioStarted.asMono().timeout(TIMEOUT) : Mono.empty())) - .then(Mono.defer(() -> interactive - ? sendEvent.apply(commentary("Hi, how can I help you today?")) - .timeout(TIMEOUT) - .then(greeting.awaitPlaybackAsync(drainAudio)) - .then(Mono.fromRunnable(transcripts::endUtterance)) - : opening)) - .then(Mono.defer(() -> promptAsync(sendEvent, nextPrompt, responseCompleted, spoken, transcripts, handler))) - .then(Mono.defer(() -> { - handler.requestClose(); - stopAudio.tryEmitEmpty(); - return (interactive ? audioStopped.asMono() : Mono.empty()) - .then(sendEvent.apply(closeSession())) - .then(closed.asMono()).timeout(TIMEOUT); - })); - return interactive ? Mono.firstWithSignal(receive, audio, renderTranscripts, conversation) - : Mono.firstWithSignal(receive, conversation); - }); - } - - private static Mono promptAsync(Function> sendEvent, + static Mono promptAsync(Function> sendEvent, Function> nextPrompt, AtomicReference> responseCompleted, boolean spoken, AudioTranscriptPrinter transcripts, ConversationEventHandler handler) { @@ -592,23 +140,7 @@ Flux quietPeriodsAsync() { } } - static Mono readPromptAsync(Reader input) { - return readPromptAsync(input, new AudioTranscriptPrinter()); - } - - private static Mono readPromptAsync(Reader input, AudioTranscriptPrinter transcripts) { - return Mono.defer(() -> { - BufferedReader reader = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); - return Mono.fromCallable(() -> readPromptBlocking(reader, transcripts)).subscribeOn(Schedulers.boundedElastic()) - .doOnCancel(transcripts::endInput); - }); - } - - // #endregion - - // #region Shared utilities - - private abstract static class ConversationEventHandler { + abstract static class ConversationEventHandler { private final boolean spoken; private final Supplier closeDetails; private final Predicate acceptClose; @@ -714,7 +246,7 @@ static byte[] decodeAudio(ServerEvent event) { return pcm; } - private static boolean hasSpeech(byte[] pcm) { + static boolean hasSpeech(byte[] pcm) { for (int offset = 0; offset < pcm.length; offset += 2) { int sample = (short) ((pcm[offset] & 0xff) | (pcm[offset + 1] << 8)); if (sample < -Greeting.SILENCE_PEAK_THRESHOLD || sample > Greeting.SILENCE_PEAK_THRESHOLD) { @@ -724,25 +256,6 @@ private static boolean hasSpeech(byte[] pcm) { return false; } - static Map object(BinaryData data) { - return data.toObject(Map.class); - } - - static Map object(Object value) { - if (!(value instanceof Map)) { - throw new IllegalArgumentException("Expected a native Live JSON object: " + value); - } - return (Map) value; - } - - static String string(Map event, String field) { - Object value = event.get(field); - if (!(value instanceof String)) { - throw new IllegalArgumentException("Missing or invalid Live event field: " + field); - } - return (String) value; - } - static BinaryData event(String type, String field, Object value) { Map event = new LinkedHashMap<>(); event.put("type", type); @@ -809,15 +322,14 @@ static void checkError(ServerEvent event) { static void checkError(ResponseStreamEvent event) { if (event.isError() || event.isFailed() || event.isIncomplete()) { - BinaryData data = OpenAIJsonHelper.toBinaryData(event); - Object error = object(data).get("error"); + Object error = JsonValue.from(event).convert(Map.class).get("error"); if (error instanceof Map && ("content_filter".equals(((Map) error).get("code")) || "content_filter".equals(((Map) error).get("type")))) { throw new IllegalStateException("GPT Live service error: generation stopped by service moderation " + "(content_filter). The call has ended; review the conversation and service diagnostics. Event: " - + data); + + event); } - throw new IllegalStateException("GPT Live service error: " + data); + throw new IllegalStateException("GPT Live service error: " + event); } } @@ -1143,7 +655,7 @@ static AgentResource createAgent(AgentsClientBuilder builder, String model, Stri ? "Use get_weather for weather requests. After receiving its result, answer briefly." : "Answer the user briefly and helpfully."); if (tools) { - responses.setTools(java.util.Collections.singletonList(GptLiveFunctionToolSample.weatherTool())); + responses.setTools(java.util.Collections.singletonList(BinaryData.fromObject(GptLiveFunctionToolSample.weatherTool()))); } VoiceAgentLiveDefinition definition = new VoiceAgentLiveDefinition(VoiceAgentSubProtocol.LIVE, model) .setInstructions("You are a helpful voice assistant. Delegate requests to your Responses backend.") @@ -1184,6 +696,4 @@ public void close() { } } } - - // #endregion } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java index 238ca16209efd..d3a19df7d40ba 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationAsyncSample.java @@ -3,16 +3,31 @@ package com.azure.ai.agents.voice; -import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsLiveWebSocketSessionAsyncClient; import com.azure.ai.agents.AgentsClientBuilder; import com.azure.core.util.Configuration; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.identity.DefaultAzureCredentialBuilder; +import com.openai.models.live.ServerEvent; +import com.openai.models.responses.ResponseStreamEvent; import java.io.Console; import java.io.InputStreamReader; import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import reactor.core.publisher.Sinks; import reactor.core.scheduler.Schedulers; +import com.openai.models.live.ClientEvent; +import java.io.BufferedReader; +import java.io.Reader; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; + /** * Runs a nonblocking, multi-turn GPT Live text conversation using Responses delegation. @@ -74,18 +89,242 @@ public static void main(String[] args) { Console console = System.console(); try (GptLiveSampleUtils.AgentResource agent = GptLiveSampleUtils.createAgent(builder, model, backendModel, "text-async", false, keepAgent)) { - Mono.usingWhen(agent.builder.beta().buildBetaVoiceAgentLiveWebSocketAsyncClient() + Mono.usingWhen(agent.builder.beta().buildBetaVoiceAgentsLiveWebSocketAsyncClient() .openWebSocketSession(agent.name, new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store)), session -> args.length == 0 - ? GptLiveSampleUtils.runInteractiveConversationAsync(session, + ? runInteractiveConversationAsync(session, console == null ? new InputStreamReader(System.in) : console.reader()) - : GptLiveSampleUtils.runTextConversationAsync(session, Arrays.asList(args)), - BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync, + : runConversation(session, Arrays.asList(args)), + BetaVoiceAgentsLiveWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), - BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(); + BetaVoiceAgentsLiveWebSocketSessionAsyncClient::closeAsync).block(); } finally { Schedulers.shutdownNow(); } } + static Mono runConversation(BetaVoiceAgentsLiveWebSocketSessionAsyncClient session, List prompts) { + return Mono.defer(() -> { + Sinks.Empty ready = Sinks.empty(); + Sinks.Empty closed = Sinks.empty(); + AtomicBoolean started = new AtomicBoolean(); + AtomicBoolean closing = new AtomicBoolean(); + AtomicBoolean finalized = new AtomicBoolean(); + AtomicBoolean remoteHangup = new AtomicBoolean(); + AtomicReference> responseCompleted = new AtomicReference<>(); + Mono receive = session.receiveEvents() + .publishOn(Schedulers.boundedElastic(), 1) + .takeUntil(ServerEvent::isSessionClosed) + .doOnNext(event -> { + GptLiveSampleUtils.printEvent(event); + if (event.isSessionStarted()) { + if (!started.compareAndSet(false, true)) { + throw new IllegalStateException("Duplicate session.started event."); + } + ready.tryEmitEmpty(); + } else if (event.isResponseEvent()) { + ResponseStreamEvent response = GptLiveSampleUtils.backendEvent(event); + if (response.isCompleted()) { + Sinks.Empty completion = responseCompleted.getAndSet(null); + if (!started.get() || completion == null || closing.get()) { + throw new IllegalStateException("Unexpected GPT Live response lifecycle event."); + } + System.out.println(); + completion.tryEmitEmpty(); + } + } else if (event.isSessionClosed()) { + String reason = event.asSessionClosed().reason().toString(); + remoteHangup.set(started.get() && "remote_hangup".equals(reason)); + if (!remoteHangup.get() && !(closing.get() && "close_requested".equals(reason))) { + throw new IllegalStateException("GPT Live closed before all turns completed. Event: " + + event + "; transport close code=" + session.getCloseCode() + + ", reason=" + session.getCloseReason()); + } + finalized.set(true); + System.out.println("Live session closed (" + reason + ")."); + closed.tryEmitEmpty(); + } + }) + .then(Mono.defer(() -> { + if (!finalized.get()) { + return Mono.error(new IllegalStateException("GPT Live disconnected without session.closed.")); + } + return remoteHangup.get() ? Mono.empty() : Mono.never(); + })); + Mono conversation = ready.asMono().timeout(GptLiveSampleUtils.TIMEOUT) + .thenMany(Flux.fromIterable(prompts).concatMap(prompt -> Mono.defer(() -> { + Sinks.Empty completion = Sinks.empty(); + responseCompleted.set(completion); + System.out.println("You: " + prompt); + System.out.print("Assistant: "); + return session.sendEvent(GptLiveSampleUtils.userMessage(prompt)) + .then(session.sendEvent(GptLiveSampleUtils.responseCreate())) + .then(completion.asMono()).timeout(GptLiveSampleUtils.TIMEOUT); + }), 1)) + .then(Mono.defer(() -> { + closing.set(true); + return session.sendEvent(GptLiveSampleUtils.closeSession()) + .then(closed.asMono()).timeout(GptLiveSampleUtils.TIMEOUT); + })); + return Mono.firstWithSignal(receive, conversation); + }); + } + + static Mono runInteractiveConversationAsync(BetaVoiceAgentsLiveWebSocketSessionAsyncClient session, Reader input) { + return Mono.usingWhen( + Mono.fromCallable(VoiceAgentRealtimeSampleUtils.SpeakerPlayer::new) + .subscribeOn(Schedulers.boundedElastic()), + player -> runInteractiveConversationAsync(session, input, player::play, player::drain), + player -> Mono.fromRunnable(() -> { + boolean speakerEnabled = player.isEnabled(); + try { + player.close(); + } finally { + System.out.printf("(received %.2fs of reply audio%s)%n", + player.getSecondsReceived(), speakerEnabled ? " and played it" : ""); + } + }).subscribeOn(Schedulers.boundedElastic())); + } + + static Mono runInteractiveConversationAsync(BetaVoiceAgentsLiveWebSocketSessionAsyncClient session, Reader input, + Consumer playAudio, Runnable drainAudio) { + return Mono.defer(() -> runInteractiveConversationAsync(session.receiveEvents(), session::sendEvent, + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), + input, playAudio, drainAudio)); + } + + static Mono runInteractiveConversationAsync(Flux events, + Function> sendEvent, Supplier closeDetails, Reader input, + Consumer playAudio, Runnable drainAudio) { + return Mono.defer(() -> { + System.out.println("Type a message and press Enter. Blank line, 'exit' or 'quit' ends the session."); + System.out.println("Interactive voice streams silent audio while you type; voice usage is billable."); + BufferedReader console = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); + return runConversationAsync(events, sendEvent, closeDetails, transcripts -> readPromptAsync(console, transcripts), + playAudio, drainAudio, true, null); + }); + } + + static Mono readPromptAsync(Reader input) { + return readPromptAsync(input, new GptLiveSampleUtils.AudioTranscriptPrinter()); + } + + private static Mono readPromptAsync(Reader input, GptLiveSampleUtils.AudioTranscriptPrinter transcripts) { + return Mono.defer(() -> { + BufferedReader reader = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); + return Mono.fromCallable(() -> GptLiveSampleUtils.readPromptBlocking(reader, transcripts)) + .subscribeOn(Schedulers.boundedElastic()) + .doOnCancel(transcripts::endInput); + }); + } + + private static Mono runConversationAsync(Flux events, + Function> sendEvent, Supplier closeDetails, + Function> nextPrompt, + Consumer playAudio, Runnable drainAudio, boolean spoken, GptLiveSampleUtils.AudioProcessor microphone) { + Mono opening = microphone == null ? Mono.empty() + : Mono.defer(() -> sendEvent.apply(GptLiveSampleUtils.userMessage("Greet me briefly with: Hi, how can I help you today?")) + .then(sendEvent.apply(GptLiveSampleUtils.responseCreate())) + .then(Mono.fromRunnable(() -> System.out.println( + "Opening greeting requested. Speak into the microphone. Press Enter to end the call.")))); + return runConversationCoreAsync(events, sendEvent, closeDetails, nextPrompt, playAudio, drainAudio, spoken, + spoken && microphone == null, microphone == null && System.console() != null, + () -> Mono.fromRunnable(() -> { + if (microphone != null) { + microphone.start(); + } + }), opening, backend -> Mono.just(false), reason -> microphone != null); + } + + static Mono runConversationCoreAsync(Flux events, + Function> sendEvent, Supplier closeDetails, + Function> nextPrompt, + Consumer playAudio, Runnable drainAudio, boolean spoken, boolean interactive, boolean inputEchoed, + Supplier> onStarted, Mono opening, + Function> handleResponse, Predicate acceptClose) { + return Mono.defer(() -> { + GptLiveSampleUtils.AudioTranscriptPrinter transcripts = new GptLiveSampleUtils.AudioTranscriptPrinter(inputEchoed, interactive); + GptLiveSampleUtils.Greeting greeting = new GptLiveSampleUtils.Greeting(); + Sinks.Empty ready = Sinks.empty(); + Sinks.Empty closed = Sinks.empty(); + Sinks.Empty audioStarted = Sinks.empty(); + Sinks.Empty stopAudio = Sinks.empty(); + Sinks.Empty audioStopped = Sinks.empty(); + AtomicReference> responseCompleted = new AtomicReference<>(); + GptLiveSampleUtils.ConversationEventHandler> handler + = new GptLiveSampleUtils.ConversationEventHandler>(spoken, closeDetails, acceptClose) { + @Override + Mono onStarted() { + return Mono.defer(onStarted).doOnSuccess(ignored -> ready.tryEmitEmpty()); + } + + @Override + Mono onResponse(ResponseStreamEvent event) { + return Mono.defer(() -> handleResponse.apply(event)).defaultIfEmpty(false) + .doOnNext(continuation -> { + if (event.isCompleted() && completeResponse(continuation)) { + responseCompleted.getAndSet(null).tryEmitEmpty(); + } + }).then(); + } + + @Override + Mono onClosed() { + return Mono.fromRunnable(() -> closed.tryEmitEmpty()); + } + + @Override + Mono ignored() { + return Mono.empty(); + } + }; + Mono receive = events + .publishOn(Schedulers.boundedElastic(), 1) + .takeUntil(ServerEvent::isSessionClosed) + .concatMap(event -> Mono.defer(() -> { + if (spoken) { + transcripts.print(event); + if (event.isSessionOutputAudioDelta()) { + byte[] pcm = GptLiveSampleUtils.decodeAudio(event); + playAudio.accept(pcm); + greeting.onAudio(pcm); + } else if (event.isSessionOutputTranscriptDelta() + && !event.asSessionOutputTranscriptDelta().delta().trim().isEmpty()) { + greeting.onTranscript(); + } + } else { + GptLiveSampleUtils.printEvent(event); + } + return handler.process(event); + }), 1) + .then(Mono.defer(() -> handler.awaitConversationAfterReceive() ? Mono.never() : Mono.empty())); + Mono audio = ready.asMono() + .thenMany(GptLiveSampleUtils.silenceFramesAsync().takeUntilOther(stopAudio.asMono()) + .concatMap(frame -> sendEvent.apply(frame).doOnSuccess(ignored -> audioStarted.tryEmitEmpty()), 1)) + .then() + .doOnSuccess(ignored -> audioStopped.tryEmitEmpty()) + .then(Mono.never()); + Mono renderTranscripts = greeting.quietPeriodsAsync() + .doOnNext(ignored -> transcripts.endUtterance()).then(); + Mono conversation = ready.asMono().timeout(GptLiveSampleUtils.TIMEOUT) + .then(Mono.defer(() -> interactive ? audioStarted.asMono().timeout(GptLiveSampleUtils.TIMEOUT) : Mono.empty())) + .then(Mono.defer(() -> interactive + ? sendEvent.apply(GptLiveSampleUtils.commentary("Hi, how can I help you today?")) + .timeout(GptLiveSampleUtils.TIMEOUT) + .then(greeting.awaitPlaybackAsync(drainAudio)) + .then(Mono.fromRunnable(transcripts::endUtterance)) + : opening)) + .then(Mono.defer(() -> GptLiveSampleUtils.promptAsync(sendEvent, nextPrompt, responseCompleted, spoken, transcripts, handler))) + .then(Mono.defer(() -> { + handler.requestClose(); + stopAudio.tryEmitEmpty(); + return (interactive ? audioStopped.asMono() : Mono.empty()) + .then(sendEvent.apply(GptLiveSampleUtils.closeSession())) + .then(closed.asMono()).timeout(GptLiveSampleUtils.TIMEOUT); + })); + return interactive ? Mono.firstWithSignal(receive, audio, renderTranscripts, conversation) + : Mono.firstWithSignal(receive, conversation); + }); + } + } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java index 7487985dbcb51..5f8f20bedbe41 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveTextConversationSample.java @@ -3,15 +3,37 @@ package com.azure.ai.agents.voice; -import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; +import com.azure.ai.agents.BetaVoiceAgentsLiveWebSocketSessionClient; import com.azure.ai.agents.AgentsClientBuilder; import com.azure.core.util.Configuration; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.identity.DefaultAzureCredentialBuilder; +import com.openai.models.live.ClientEvent; +import com.openai.models.live.ServerEvent; +import com.openai.models.responses.ResponseStreamEvent; import java.io.Console; import java.io.InputStreamReader; import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Supplier; import reactor.core.scheduler.Schedulers; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.function.Predicate; + /** * Runs a synchronous, multi-turn GPT Live text conversation using Responses delegation. @@ -21,7 +43,7 @@ * It streams silent PCM while typing; voice usage is billable. Greeting completion uses the same speech-inactivity * heuristic and speaker drain as the asynchronous sample. Assistant transcript fragments are assembled into * utterances independently of typed turns; completed text waits until Enter when a You: line is active. - * Synchronous session I/O runs on workers using the shared voice-loop coordination in GptLiveSampleUtils. + * Synchronous session I/O runs on workers coordinated by this sample. * After a remote disconnect, a pending console read may require Enter to release the input worker. * Scripted command-line prompts print backend text without speaker playback. * A uniquely named agent is deleted on exit unless FOUNDRY_KEEP_GPT_LIVE_AGENT=true. @@ -67,19 +89,257 @@ public static void main(String[] args) { .allowPreview(true); try (GptLiveSampleUtils.AgentResource agent = GptLiveSampleUtils.createAgent(builder, model, backendModel, "text", false, keepAgent); - BetaVoiceAgentLiveWebSocketSessionClient session = agent.builder.beta() - .buildBetaVoiceAgentLiveWebSocketClient() + BetaVoiceAgentsLiveWebSocketSessionClient session = agent.builder.beta() + .buildBetaVoiceAgentsLiveWebSocketClient() .openWebSocketSession(agent.name, new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store))) { if (args.length == 0) { Console console = System.console(); - GptLiveSampleUtils.runInteractiveConversation(session, + runInteractiveConversation(session, console == null ? new InputStreamReader(System.in) : console.reader()); } else { - GptLiveSampleUtils.runTextConversation(session, Arrays.asList(args)); + runConversation(session, Arrays.asList(args)); } } finally { Schedulers.shutdownNow(); } } + static void runConversation(BetaVoiceAgentsLiveWebSocketSessionClient session, List prompts) { + runConversation(session.receiveEvents(GptLiveSampleUtils.TIMEOUT), session::sendEvent, + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), prompts); + } + + static void runConversation(Iterable events, Consumer sendEvent, + Supplier closeDetails, List prompts) { + Iterator pending = prompts.iterator(); + boolean started = false; + boolean awaitingResponse = false; + boolean closing = false; + for (ServerEvent event : events) { + GptLiveSampleUtils.printEvent(event); + boolean nextTurn = false; + if (event.isSessionStarted()) { + if (started) { + throw new IllegalStateException("Duplicate session.started event."); + } + started = true; + nextTurn = true; + } else if (event.isResponseEvent()) { + ResponseStreamEvent response = GptLiveSampleUtils.backendEvent(event); + if (response.isCompleted()) { + if (!started || !awaitingResponse || closing) { + throw new IllegalStateException("Unexpected GPT Live response lifecycle event."); + } + awaitingResponse = false; + System.out.println(); + nextTurn = true; + } + } else if (event.isSessionClosed()) { + String reason = event.asSessionClosed().reason().toString(); + if (!(started && "remote_hangup".equals(reason)) + && !(closing && "close_requested".equals(reason))) { + throw new IllegalStateException("GPT Live closed before all turns completed. Event: " + event + + "; " + closeDetails.get()); + } + System.out.println("Live session closed (" + reason + ")."); + return; + } + if (nextTurn) { + if (pending.hasNext()) { + String prompt = pending.next(); + System.out.println("You: " + prompt); + System.out.print("Assistant: "); + awaitingResponse = true; + sendEvent.accept(GptLiveSampleUtils.userMessage(prompt)); + sendEvent.accept(GptLiveSampleUtils.responseCreate()); + } else { + closing = true; + sendEvent.accept(GptLiveSampleUtils.closeSession()); + } + } + } + throw new IllegalStateException("GPT Live disconnected without session.closed."); + } + + static void runInteractiveConversation(BetaVoiceAgentsLiveWebSocketSessionClient session, Reader input) { + try (VoiceAgentRealtimeSampleUtils.SpeakerPlayer player = new VoiceAgentRealtimeSampleUtils.SpeakerPlayer()) { + runInteractiveConversation(session, input, player::play, player::drain); + player.drain(); + System.out.printf("(received %.2fs of reply audio%s)%n", player.getSecondsReceived(), + player.isEnabled() ? " and played it" : ""); + } + } + + static void runInteractiveConversation(BetaVoiceAgentsLiveWebSocketSessionClient session, Reader input, + Consumer playAudio, Runnable drainAudio) { + System.out.println("Type a message and press Enter. Blank line, 'exit' or 'quit' ends the session."); + System.out.println("Interactive voice streams silent audio while you type; voice usage is billable."); + BufferedReader console = input instanceof BufferedReader ? (BufferedReader) input : new BufferedReader(input); + runConversationCore(session.receiveEvents(GptLiveSampleUtils.TIMEOUT), session::sendEvent, + () -> "transport close code=" + session.getCloseCode() + ", reason=" + session.getCloseReason(), transcripts -> { + try { + return GptLiveSampleUtils.readPromptBlocking(console, transcripts); + } catch (IOException error) { + throw new IllegalStateException("Unable to read console input.", error); + } + }, playAudio, drainAudio, true, () -> { }, backend -> false, reason -> false); + } + + static void runConversationCore(Iterable events, Consumer sendEvent, + Supplier closeDetails, Function nextPrompt, + Consumer playAudio, Runnable drainAudio, boolean spoken, + Runnable onStarted, Predicate handleResponse, Predicate acceptClose) { + GptLiveSampleUtils.ConversationEventHandler handler + = new GptLiveSampleUtils.ConversationEventHandler(spoken, closeDetails, acceptClose) { + @Override + Boolean onStarted() { + onStarted.run(); + return false; + } + + @Override + Boolean onResponse(ResponseStreamEvent event) { + boolean continuation = handleResponse.test(event); + if (event.isCompleted()) { + completeResponse(continuation); + } + return false; + } + + @Override + Boolean onClosed() { + return true; + } + + @Override + Boolean ignored() { + return false; + } + }; + ExecutorService workers = Executors.newFixedThreadPool(2, task -> { + Thread thread = new Thread(task, "gpt-live-sync-conversation"); + thread.setDaemon(true); + return thread; + }); + BlockingQueue pending = new LinkedBlockingQueue<>(256); + AtomicReference receiveFailure = new AtomicReference<>(); + AtomicBoolean receiveEnded = new AtomicBoolean(); + GptLiveSampleUtils.AudioTranscriptPrinter transcripts = new GptLiveSampleUtils.AudioTranscriptPrinter(System.console() != null, spoken); + Future receiver = workers.submit(() -> { + try { + for (ServerEvent event : events) { + pending.put(event); + if (event.isSessionClosed()) { + break; + } + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } catch (RuntimeException error) { + receiveFailure.set(error); + } finally { + receiveEnded.set(true); + } + }); + Future prompt = null; + boolean greetingComplete = !spoken; + boolean greetingAudio = false; + boolean greetingTranscript = false; + long deadline = System.nanoTime() + GptLiveSampleUtils.TIMEOUT.toNanos(); + long nextAudio = 0; + long quietSince = 0; + ClientEvent silence = GptLiveSampleUtils.audio(new byte[960]); + try { + while (true) { + ServerEvent event = pending.poll(20, TimeUnit.MILLISECONDS); + long now = System.nanoTime(); + if (event != null) { + if (spoken) { + transcripts.print(event); + if (event.isSessionOutputAudioDelta()) { + byte[] pcm = GptLiveSampleUtils.decodeAudio(event); + playAudio.accept(pcm); + if (GptLiveSampleUtils.hasSpeech(pcm)) { + greetingAudio = true; + quietSince = now; + } + } else if (event.isSessionOutputTranscriptDelta() + && !event.asSessionOutputTranscriptDelta().delta().trim().isEmpty()) { + greetingTranscript = true; + quietSince = now; + } + } else { + GptLiveSampleUtils.printEvent(event); + } + if (handler.process(event)) { + return; + } + if (event.isSessionStarted()) { + if (spoken) { + sendEvent.accept(silence); + nextAudio = now + TimeUnit.MILLISECONDS.toNanos(20); + sendEvent.accept(GptLiveSampleUtils.commentary("Hi, how can I help you today?")); + } + } + } else if (receiveEnded.get()) { + if (receiveFailure.get() != null) { + throw receiveFailure.get(); + } + throw new IllegalStateException("GPT Live disconnected without session.closed."); + } + + if (handler.canStreamAudio() && now >= nextAudio) { + sendEvent.accept(silence); + nextAudio = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(20); + } + if (spoken && greetingAudio && greetingTranscript && quietSince != 0 + && now - quietSince >= TimeUnit.SECONDS.toNanos(1)) { + transcripts.endUtterance(); + quietSince = 0; + if (!greetingComplete) { + drainAudio.run(); + greetingComplete = true; + } + } + if (handler.canReadPrompt(greetingComplete) && prompt == null) { + prompt = workers.submit(() -> nextPrompt.apply(transcripts)); + } + if (prompt != null && prompt.isDone()) { + String text = prompt.get(); + prompt = null; + deadline = System.nanoTime() + GptLiveSampleUtils.TIMEOUT.toNanos(); + if (text == null) { + handler.requestClose(); + sendEvent.accept(GptLiveSampleUtils.closeSession()); + } else { + handler.beginResponse(); + if (!spoken) { + System.out.print("Assistant: "); + } + sendEvent.accept(GptLiveSampleUtils.userMessage(text)); + sendEvent.accept(GptLiveSampleUtils.responseCreate()); + } + } + if (!handler.canReadPrompt(greetingComplete) && now >= deadline) { + throw new IllegalStateException("Timed out waiting for the GPT Live conversation."); + } + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("GPT Live conversation interrupted.", error); + } catch (ExecutionException error) { + if (error.getCause() instanceof RuntimeException) { + throw (RuntimeException) error.getCause(); + } + throw new IllegalStateException("Unable to read console input.", error.getCause()); + } finally { + receiver.cancel(true); + if (prompt != null) { + prompt.cancel(true); + } + transcripts.endInput(); + workers.shutdownNow(); + } + } + } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java index baaf6ff0f9481..b0c48a48d127c 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java @@ -7,8 +7,8 @@ import com.azure.ai.agents.AgentsClientBuilder; import com.azure.ai.agents.BetaVoiceAgentsConversationsAsyncClient; import com.azure.ai.agents.BetaAgentsAsyncClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketAsyncClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketSessionAsyncClient; import com.azure.ai.agents.models.CreateAgentVersionInput; import com.azure.ai.agents.models.RealtimeConversationItemInputAudioTranscriptionCompletedEvent; import com.azure.ai.agents.models.RealtimeInputAudioBufferSpeechStartedEvent; @@ -82,7 +82,7 @@ public static void main(String[] args) { .allowPreview(true); AgentsAsyncClient agents = builder.buildAgentsAsyncClient(); BetaAgentsAsyncClient betaAgents = builder.beta().buildBetaAgentsAsyncClient(); - BetaVoiceAgentWebSocketAsyncClient realtime = builder.beta().buildBetaVoiceAgentWebSocketAsyncClient(); + BetaVoiceAgentsWebSocketAsyncClient realtime = builder.beta().buildBetaVoiceAgentsWebSocketAsyncClient(); BetaVoiceAgentsConversationsAsyncClient conversations = builder.beta().buildBetaVoiceAgentsConversationsAsyncClient(); @@ -100,9 +100,9 @@ public static void main(String[] args) { }) .then(Mono.usingWhen(realtime.openWebSocketSession(agentName), session -> runConversation(session, conversationId), - BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync, + BetaVoiceAgentsWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), - BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync)) + BetaVoiceAgentsWebSocketSessionAsyncClient::closeAsync)) .then(Mono.defer(() -> conversationId.get() == null ? Mono.fromRunnable(() -> System.out.println("No persisted conversation ID was returned.")) : VoiceAgentRealtimeSampleUtils.readConversation(conversations, agentName, conversationId.get()))) @@ -121,7 +121,7 @@ private static Mono cleanupAgent(AgentsAsyncClient agents, String agentNam .doOnSuccess(ignored -> System.out.println("Deleted voice agent: " + agentName)); } - private static Mono runConversation(BetaVoiceAgentWebSocketSessionAsyncClient session, + private static Mono runConversation(BetaVoiceAgentsWebSocketSessionAsyncClient session, AtomicReference conversationId) { AudioProcessor processor = new AudioProcessor(session); AtomicBoolean responseActive = new AtomicBoolean(); @@ -208,11 +208,11 @@ static final class AudioProcessor implements AutoCloseable { private Thread captureThread; private Thread playbackThread; - AudioProcessor(BetaVoiceAgentWebSocketSessionAsyncClient session) { + AudioProcessor(BetaVoiceAgentsWebSocketSessionAsyncClient session) { this(session, null, null); } - AudioProcessor(BetaVoiceAgentWebSocketSessionAsyncClient session, TargetDataLine microphone, SourceDataLine speaker) { + AudioProcessor(BetaVoiceAgentsWebSocketSessionAsyncClient session, TargetDataLine microphone, SourceDataLine speaker) { this(microphone, speaker, pcm -> session.appendInputAudio(pcm)); } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveFunctionToolSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveFunctionToolSample.java index f51801608fb3d..9ed04a93c7d21 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveFunctionToolSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveFunctionToolSample.java @@ -5,8 +5,8 @@ import com.azure.ai.agents.AgentsClient; import com.azure.ai.agents.AgentsClientBuilder; -import com.azure.ai.agents.BetaVoiceAgentWebSocketClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketSessionClient; import com.azure.ai.agents.models.CreateAgentVersionInput; import com.azure.ai.agents.models.RealtimeConversationItem; import com.azure.ai.agents.models.RealtimeConversationItemType; @@ -64,7 +64,7 @@ public static void main(String[] args) { .endpoint(endpoint) .allowPreview(true); AgentsClient agents = builder.buildAgentsClient(); - BetaVoiceAgentWebSocketClient realtime = builder.beta().buildBetaVoiceAgentWebSocketClient(); + BetaVoiceAgentsWebSocketClient realtime = builder.beta().buildBetaVoiceAgentsWebSocketClient(); Map cityProperty = new LinkedHashMap<>(); cityProperty.put("type", "string"); @@ -89,7 +89,7 @@ public static void main(String[] args) { try { agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition)); System.out.println("Created voice agent: " + agentName); - try (BetaVoiceAgentWebSocketSessionClient session = realtime.openWebSocketSession(agentName)) { + try (BetaVoiceAgentsWebSocketSessionClient session = realtime.openWebSocketSession(agentName)) { ExecutorService receiver = Executors.newSingleThreadExecutor(); Future response = receiver.submit(() -> receiveResponse(session)); try { @@ -114,7 +114,7 @@ public static void main(String[] args) { } } - private static void receiveResponse(BetaVoiceAgentWebSocketSessionClient session) { + private static void receiveResponse(BetaVoiceAgentsWebSocketSessionClient session) { for (RealtimeServerEvent event : session.receiveEvents()) { if (event instanceof RealtimeResponseFunctionCallArgumentsDoneEvent) { RealtimeResponseFunctionCallArgumentsDoneEvent call diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java index c054427a5cc3a..caaf3e36a0c2d 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java @@ -7,8 +7,8 @@ import com.azure.ai.agents.AgentsClientBuilder; import com.azure.ai.agents.BetaVoiceAgentsConversationsAsyncClient; import com.azure.ai.agents.BetaAgentsAsyncClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketAsyncClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketSessionAsyncClient; import com.azure.ai.agents.models.CreateAgentVersionInput; import com.azure.ai.agents.models.VoiceAgentDefinition; import com.azure.core.util.BinaryData; @@ -59,7 +59,7 @@ public static void main(String[] args) { .allowPreview(true); AgentsAsyncClient agents = builder.buildAgentsAsyncClient(); BetaAgentsAsyncClient betaAgents = builder.beta().buildBetaAgentsAsyncClient(); - BetaVoiceAgentWebSocketAsyncClient realtime = builder.beta().buildBetaVoiceAgentWebSocketAsyncClient(); + BetaVoiceAgentsWebSocketAsyncClient realtime = builder.beta().buildBetaVoiceAgentsWebSocketAsyncClient(); BetaVoiceAgentsConversationsAsyncClient conversations = builder.beta().buildBetaVoiceAgentsConversationsAsyncClient(); @@ -80,9 +80,9 @@ public static void main(String[] args) { }) .then(Mono.usingWhen(realtime.openWebSocketSession(agentName), session -> runConversation(session, scanner, conversationId, player), - BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync, + BetaVoiceAgentsWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), - BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync)) + BetaVoiceAgentsWebSocketSessionAsyncClient::closeAsync)) .then(Mono.defer(() -> conversationId.get() == null ? Mono.fromRunnable(() -> System.out.println("No persisted conversation ID was returned.")) : VoiceAgentRealtimeSampleUtils.readConversation(conversations, agentName, conversationId.get()))) @@ -105,7 +105,7 @@ private static Mono cleanupAgent(AgentsAsyncClient agents, String agentNam .doOnSuccess(ignored -> System.out.println("Deleted voice agent: " + agentName)); } - private static Mono runConversation(BetaVoiceAgentWebSocketSessionAsyncClient session, Scanner scanner, + private static Mono runConversation(BetaVoiceAgentsWebSocketSessionAsyncClient session, Scanner scanner, AtomicReference conversationId, VoiceAgentRealtimeSampleUtils.SpeakerPlayer player) { AtomicReference> responseCompleted = new AtomicReference<>(); Disposable receiver = session.receiveEvents().subscribe(event -> { @@ -131,7 +131,7 @@ private static Mono runConversation(BetaVoiceAgentWebSocketSessionAsyncCli }); } - private static Mono prompt(BetaVoiceAgentWebSocketSessionAsyncClient session, Scanner scanner, + private static Mono prompt(BetaVoiceAgentsWebSocketSessionAsyncClient session, Scanner scanner, AtomicReference> responseCompleted) { return Mono.fromCallable(() -> { System.out.print("You: "); diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java index 9903d76639c14..97ac549ca56c0 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java @@ -7,8 +7,8 @@ import com.azure.ai.agents.AgentsClientBuilder; import com.azure.ai.agents.BetaVoiceAgentsConversationsClient; import com.azure.ai.agents.BetaAgentsClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketSessionClient; import com.azure.ai.agents.models.AgentDetails; import com.azure.ai.agents.models.AgentVersionDetails; import com.azure.ai.agents.models.CreateAgentVersionInput; @@ -59,7 +59,7 @@ public static void main(String[] args) { .allowPreview(true); AgentsClient agents = builder.buildAgentsClient(); BetaAgentsClient betaAgents = builder.beta().buildBetaAgentsClient(); - BetaVoiceAgentWebSocketClient realtime = builder.beta().buildBetaVoiceAgentWebSocketClient(); + BetaVoiceAgentsWebSocketClient realtime = builder.beta().buildBetaVoiceAgentsWebSocketClient(); BetaVoiceAgentsConversationsClient conversations = builder.beta().buildBetaVoiceAgentsConversationsClient(); try (SampleAgent agent = createAgent(agents, betaAgents, agentName)) { @@ -70,7 +70,7 @@ public static void main(String[] args) { AtomicReference conversationId = new AtomicReference<>(); try (VoiceAgentRealtimeSampleUtils.SpeakerPlayer player = new VoiceAgentRealtimeSampleUtils.SpeakerPlayer(); - BetaVoiceAgentWebSocketSessionClient session = realtime.openWebSocketSession(agentName); + BetaVoiceAgentsWebSocketSessionClient session = realtime.openWebSocketSession(agentName); Scanner scanner = new Scanner(System.in)) { AtomicReference> responseCompleted = new AtomicReference<>(); ExecutorService receiver = Executors.newSingleThreadExecutor(); @@ -162,7 +162,7 @@ public void close() { } } - private static boolean awaitResponse(BetaVoiceAgentWebSocketSessionClient session, + private static boolean awaitResponse(BetaVoiceAgentsWebSocketSessionClient session, CompletableFuture completion) { try { completion.get(RESPONSE_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClientTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java similarity index 65% rename from sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClientTests.java rename to sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java index c652b426c2c4f..0d96de039f81b 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClientTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java @@ -9,6 +9,7 @@ import com.azure.ai.agents.models.VoiceAgentTransport; import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.http.ProxyOptions; +import com.azure.core.http.HttpHeaders; import java.net.InetSocketAddress; import java.net.Proxy; import java.net.URI; @@ -19,8 +20,10 @@ import reactor.core.publisher.Mono; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; -public class BetaVoiceAgentWebSocketSessionClientTests { +public class BetaVoiceAgentsWebSocketSessionClientTests { @Test public void connectionUriUsesSelectedTransport() { VoiceAgentWebSocketClientConfiguration configuration = new VoiceAgentWebSocketClientConfiguration( @@ -59,6 +62,36 @@ public void syncTransportHonorsNonProxyHosts() { assertEquals(new Proxy(Proxy.Type.HTTP, proxyAddress), proxied.get(0)); } + @Test + public void syncConnectionPreservesTokenFailure() { + IllegalStateException failure = new IllegalStateException("Token unavailable"); + VoiceAgentWebSocketClientConfiguration configuration = new VoiceAgentWebSocketClientConfiguration( + URI.create("https://localhost"), request -> Mono.error(failure), "v1", "test-user-agent", null, null); + + IllegalStateException actual = assertThrows(IllegalStateException.class, + () -> WebSocketSessionTransport.connect(configuration, URI.create("wss://localhost/session"), + new VoiceAgentWebSocketConnectionOptions(), null, token -> new HttpHeaders(), json -> json)); + + assertSame(failure, actual); + } + + @Test + public void syncConnectionPreservesHeaderInitializationFailure() { + IllegalStateException failure = new IllegalStateException("Headers unavailable"); + VoiceAgentWebSocketClientConfiguration configuration = new VoiceAgentWebSocketClientConfiguration( + URI.create("https://localhost"), + request -> Mono.just(new com.azure.core.credential.AccessToken("token", OffsetDateTime.now().plusHours(1))), + "v1", "test-user-agent", null, null); + + IllegalStateException actual + = assertThrows(IllegalStateException.class, () -> WebSocketSessionTransport.connect(configuration, + URI.create("wss://localhost/session"), new VoiceAgentWebSocketConnectionOptions(), null, token -> { + throw failure; + }, json -> json)); + + assertSame(failure, actual); + } + private static String queryParameter(URI uri, String name) { for (String parameter : uri.getQuery().split("&")) { String[] pair = parameter.split("=", 2); diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/FoundryFeaturesHeaderVerificationTest.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/FoundryFeaturesHeaderVerificationTest.java index 948392401a2a8..4bb9e42023995 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/FoundryFeaturesHeaderVerificationTest.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/FoundryFeaturesHeaderVerificationTest.java @@ -412,11 +412,11 @@ public void webSocketClientsRejectUnsupportedHttpConfiguration() { .retryPolicy(new RetryPolicy()); IllegalStateException syncError - = assertThrows(IllegalStateException.class, () -> builder.beta().buildBetaVoiceAgentWebSocketClient()); + = assertThrows(IllegalStateException.class, () -> builder.beta().buildBetaVoiceAgentsWebSocketClient()); assertTrue(syncError.getMessage().contains("httpClient, pipeline, addPolicy, retryOptions, retryPolicy")); - IllegalStateException asyncError - = assertThrows(IllegalStateException.class, () -> builder.beta().buildBetaVoiceAgentWebSocketAsyncClient()); + IllegalStateException asyncError = assertThrows(IllegalStateException.class, + () -> builder.beta().buildBetaVoiceAgentsWebSocketAsyncClient()); assertEquals(syncError.getMessage(), asyncError.getMessage()); } diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java index c7ebf7cc96ca0..54d35e5afef47 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java @@ -4,8 +4,8 @@ package com.azure.ai.agents.voice; import com.azure.ai.agents.AgentsClientBuilder; -import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; -import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; +import com.azure.ai.agents.BetaVoiceAgentsLiveWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsLiveWebSocketSessionClient; import com.azure.ai.agents.implementation.OpenAIJsonHelper; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.core.credential.AccessToken; @@ -14,6 +14,7 @@ import com.openai.models.live.ClientEvent; import com.openai.models.live.ServerEvent; import com.openai.models.responses.ResponseStreamEvent; +import com.openai.models.responses.ResponseFunctionToolCall; import java.io.ByteArrayOutputStream; import java.io.FilterReader; import java.io.InputStream; @@ -300,7 +301,7 @@ public void spokenSentenceIsNotSplitByBackendCompletionOrAnotherTypedTurn(boolea .port(0) .handle((request, response) -> response.sendWebsocket((inbound, outbound) -> outbound .sendString(Flux.concat(Mono.just(STARTED), inbound.receive().asString().concatMap(json -> { - Map command = GptLiveSampleUtils.object(BinaryData.fromString(json)); + Map command = GptLiveFunctionToolSample.object(BinaryData.fromString(json)); commands.add(command); if ("session.commentary.append".equals(command.get("type"))) { return Flux.just("{\"type\":\"session.output_transcript.delta\",\"delta\":\"Welcome.\"}", @@ -356,7 +357,7 @@ public void responseFailuresAreReportedForUnknownAndDelegatedEvents(String type) for (boolean delegated : new boolean[] { false, true }) { Map wrapper = new LinkedHashMap<>(); wrapper.put("type", "response.event"); - wrapper.put("event", GptLiveSampleUtils.object(failure)); + wrapper.put("event", GptLiveFunctionToolSample.object(failure)); ServerEvent event = typedEvent(delegated ? BinaryData.fromObject(wrapper) : failure); IllegalStateException error = assertThrows(IllegalStateException.class, () -> GptLiveSampleUtils.printEvent(event)); @@ -369,7 +370,7 @@ public void responseFailuresAreReportedForUnknownAndDelegatedEvents(String type) @Test public void moderationIsReportedExplicitlyAndPreservesServiceDiagnostics() { - Map event = GptLiveSampleUtils.object(BinaryData.fromString(MODERATED)); + Map event = GptLiveFunctionToolSample.object(BinaryData.fromString(MODERATED)); IllegalStateException error = assertThrows(IllegalStateException.class, () -> new GptLiveSampleUtils.AudioTranscriptPrinter().print(typedEvent(BinaryData.fromString(MODERATED)))); assertTrue(error.getMessage().contains("generation stopped by service moderation")); @@ -455,28 +456,23 @@ public void blockingConversationConsumesEventsWithoutSession() { throw new IllegalStateException("Event producer interrupted.", error); } }).iterator(); - java.util.Iterator prompts = Arrays.asList("one", "two").iterator(); - GptLiveSampleUtils.runConversation(events, command -> { - commands.add(GptLiveSampleUtils.object(OpenAIJsonHelper.toBinaryData(command))); + GptLiveTextConversationSample.runConversation(events, command -> { + commands.add(GptLiveFunctionToolSample.object(OpenAIJsonHelper.toBinaryData(command))); if (command.isResponseCreate()) { incoming.add(typedEvent(BinaryData.fromString(COMPLETED))); } else if (command.isSessionClose()) { incoming.add(typedEvent(BinaryData.fromString(CLOSED))); } - }, () -> "test close details", transcripts -> prompts.hasNext() ? prompts.next() : null, pcm -> { - }, () -> { - }, false, null); + }, () -> "test close details", Arrays.asList("one", "two")); assertEquals(Arrays.asList("response.item.create", "response.create", "response.item.create", "response.create", "session.close"), commandTypes()); } @Test public void blockingConversationReportsCloseDetailsWithoutSession() { - IllegalStateException error = assertThrows(IllegalStateException.class, () -> GptLiveSampleUtils + IllegalStateException error = assertThrows(IllegalStateException.class, () -> GptLiveTextConversationSample .runConversation(Collections.singletonList(typedEvent(BinaryData.fromString(CLOSED))), command -> { - }, () -> "transport close code=1006, reason=test disconnect", transcripts -> null, pcm -> { - }, () -> { - }, false, null)); + }, () -> "transport close code=1006, reason=test disconnect", Collections.emptyList())); assertTrue(error.getMessage().contains("transport close code=1006, reason=test disconnect")); } @@ -536,7 +532,7 @@ private void runCoreWithCallbacks(boolean async, Runnable onStarted, Predicate reactive.emitNext(event, Sinks.EmitFailureHandler.FAIL_FAST) : blocking::add; enqueue.accept(typedEvent(BinaryData.fromString(STARTED))); Consumer sendEvent = command -> { - commands.add(GptLiveSampleUtils.object(OpenAIJsonHelper.toBinaryData(command))); + commands.add(GptLiveFunctionToolSample.object(OpenAIJsonHelper.toBinaryData(command))); if (command.isResponseCreate()) { enqueue.accept(typedEvent(BinaryData.fromString(COMPLETED))); } else if (command.isSessionClose()) { @@ -546,7 +542,7 @@ private void runCoreWithCallbacks(boolean async, Runnable onStarted, Predicate Mono.fromRunnable(() -> sendEvent.accept(command)), () -> "test close details", transcripts -> Mono.justOrEmpty(prompts.getAndIncrement() == 0 ? "one" : null), pcm -> { }, () -> { @@ -561,7 +557,7 @@ private void runCoreWithCallbacks(boolean async, Runnable onStarted, Predicate "test close details", + GptLiveTextConversationSample.runConversationCore(events, sendEvent, () -> "test close details", transcripts -> prompts.getAndIncrement() == 0 ? "one" : null, pcm -> { }, () -> { }, false, onStarted, handleResponse, acceptClose); @@ -576,26 +572,28 @@ public void asyncResponseContinuationWaitsForHandlerBeforeProcessingNextEvent() AtomicInteger responses = new AtomicInteger(); AtomicInteger prompts = new AtomicInteger(); incoming.emitNext(typedEvent(BinaryData.fromString(STARTED)), Sinks.EmitFailureHandler.FAIL_FAST); - Mono conversation = GptLiveSampleUtils.runConversationCoreAsync(incoming.asFlux(), command -> { - commands.add(GptLiveSampleUtils.object(OpenAIJsonHelper.toBinaryData(command))); - if (command.isResponseCreate()) { - incoming.emitNext(typedEvent(BinaryData.fromString(COMPLETED)), Sinks.EmitFailureHandler.FAIL_FAST); - } else if (command.isSessionClose()) { - incoming.emitNext(typedEvent(BinaryData.fromString(CLOSED)), Sinks.EmitFailureHandler.FAIL_FAST); - } - return Mono.empty(); - }, () -> "test close details", transcripts -> Mono.justOrEmpty(prompts.getAndIncrement() == 0 ? "one" : null), - pcm -> { - }, () -> { - }, false, false, false, Mono::empty, Mono.empty(), event -> { - if (responses.incrementAndGet() == 1) { + Mono conversation + = GptLiveTextConversationAsyncSample.runConversationCoreAsync(incoming.asFlux(), command -> { + commands.add(GptLiveFunctionToolSample.object(OpenAIJsonHelper.toBinaryData(command))); + if (command.isResponseCreate()) { incoming.emitNext(typedEvent(BinaryData.fromString(COMPLETED)), Sinks.EmitFailureHandler.FAIL_FAST); - handling.countDown(); - return continuation.asMono().thenReturn(true); + } else if (command.isSessionClose()) { + incoming.emitNext(typedEvent(BinaryData.fromString(CLOSED)), Sinks.EmitFailureHandler.FAIL_FAST); } - assertEquals(1, prompts.get(), "The next prompt must wait until the continuation completes"); - return Mono.just(false); - }, reason -> false); + return Mono.empty(); + }, () -> "test close details", + transcripts -> Mono.justOrEmpty(prompts.getAndIncrement() == 0 ? "one" : null), pcm -> { + }, () -> { + }, false, false, false, Mono::empty, Mono.empty(), event -> { + if (responses.incrementAndGet() == 1) { + incoming.emitNext(typedEvent(BinaryData.fromString(COMPLETED)), + Sinks.EmitFailureHandler.FAIL_FAST); + handling.countDown(); + return continuation.asMono().thenReturn(true); + } + assertEquals(1, prompts.get(), "The next prompt must wait until the continuation completes"); + return Mono.just(false); + }, reason -> false); StepVerifier.create(conversation).then(() -> { try { assertTrue(handling.await(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)); @@ -617,22 +615,23 @@ public void asyncResponseContinuationWaitsForHandlerBeforeProcessingNextEvent() public void textSamplesWaitForStartupAndCompleteMultipleTurns(boolean async) { startTextServer(false, false); if (async) { - Mono.usingWhen(builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), - session -> GptLiveSampleUtils.runTextConversationAsync(session, Arrays.asList("one", "two")), - BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(TIMEOUT); + Mono.usingWhen( + builder().beta().buildBetaVoiceAgentsLiveWebSocketAsyncClient().openWebSocketSession("agent"), + session -> GptLiveTextConversationAsyncSample.runConversation(session, Arrays.asList("one", "two")), + BetaVoiceAgentsLiveWebSocketSessionAsyncClient::closeAsync).block(TIMEOUT); } else { - try (BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - GptLiveSampleUtils.runTextConversation(session, Arrays.asList("one", "two")); + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent")) { + GptLiveTextConversationSample.runConversation(session, Arrays.asList("one", "two")); } } List expected = Arrays.asList("response.item.create", "response.create", "response.item.create", "response.create", "session.close"); assertEquals(expected, commandTypes()); - Map item = GptLiveSampleUtils.object(commands.get(0).get("item")); + Map item = GptLiveFunctionToolSample.object(commands.get(0).get("item")); assertEquals("user", item.get("role")); assertEquals("message", item.get("type")); - assertEquals("one", GptLiveSampleUtils.object(((List) item.get("content")).get(0)).get("text")); + assertEquals("one", GptLiveFunctionToolSample.object(((List) item.get("content")).get(0)).get("text")); } @ParameterizedTest @@ -652,9 +651,9 @@ public void close() { "response.item.create", "response.create", "session.close"), commandsWithoutInputAudio()); List sentPrompts = commands.stream() .filter(command -> "response.item.create".equals(command.get("type"))) - .map(command -> GptLiveSampleUtils.object(command.get("item"))) - .map(item -> GptLiveSampleUtils.object(((List) item.get("content")).get(0))) - .map(part -> GptLiveSampleUtils.string(part, "text")) + .map(command -> GptLiveFunctionToolSample.object(command.get("item"))) + .map(item -> GptLiveFunctionToolSample.object(((List) item.get("content")).get(0))) + .map(part -> GptLiveFunctionToolSample.string(part, "text")) .collect(java.util.stream.Collectors.toList()); assertEquals(Arrays.asList("one", "two"), sentPrompts); assertFalse(inputClosed.get(), "Conversation cleanup must not close standard input"); @@ -688,13 +687,15 @@ private void runInteractiveConversation(boolean async, Reader input) { private void runInteractiveConversation(boolean async, Reader input, Consumer playAudio, Runnable drainAudio) { if (async) { - Mono.usingWhen(builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), - session -> GptLiveSampleUtils.runInteractiveConversationAsync(session, input, playAudio, drainAudio), - BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync).block(TIMEOUT); + Mono.usingWhen( + builder().beta().buildBetaVoiceAgentsLiveWebSocketAsyncClient().openWebSocketSession("agent"), + session -> GptLiveTextConversationAsyncSample.runInteractiveConversationAsync(session, input, playAudio, + drainAudio), + BetaVoiceAgentsLiveWebSocketSessionAsyncClient::closeAsync).block(TIMEOUT); } else { - try (BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - GptLiveSampleUtils.runInteractiveConversation(session, input, playAudio, drainAudio); + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent")) { + GptLiveTextConversationSample.runInteractiveConversation(session, input, playAudio, drainAudio); } } } @@ -708,7 +709,7 @@ public void interactiveTextSampleReceivesServiceErrorsWhileWaitingForInput(boole .port(0) .handle((request, response) -> response.sendWebsocket((inbound, outbound) -> outbound .sendString(Flux.concat(Mono.just(STARTED), inbound.receive().asString().concatMap(json -> { - Map command = GptLiveSampleUtils.object(BinaryData.fromString(json)); + Map command = GptLiveFunctionToolSample.object(BinaryData.fromString(json)); return "session.commentary.append".equals(command.get("type")) ? Flux.just("{\"type\":\"session.output_transcript.delta\",\"delta\":\"Welcome.\"}", "{\"type\":\"session.output_audio.delta\",\"delta\":\"AQI=\"}") @@ -760,7 +761,7 @@ public int read(char[] buffer, int offset, int length) throws IOException { throw new IOException("Console unavailable"); } }; - StepVerifier.create(GptLiveSampleUtils.readPromptAsync(input)) + StepVerifier.create(GptLiveTextConversationAsyncSample.readPromptAsync(input)) .expectErrorMatches( error -> error instanceof IOException && "Console unavailable".equals(error.getMessage())) .verify(TIMEOUT); @@ -769,7 +770,7 @@ public int read(char[] buffer, int offset, int length) throws IOException { @Test public void consoleInputIsSubmittedOnlyAfterEnterWithoutLosingCharacters() throws Exception { try (PipedReader input = new PipedReader(); PipedWriter keyboard = new PipedWriter(input)) { - CompletableFuture prompt = GptLiveSampleUtils.readPromptAsync(input).toFuture(); + CompletableFuture prompt = GptLiveTextConversationAsyncSample.readPromptAsync(input).toFuture(); try { keyboard.write("testtesttes testing"); keyboard.flush(); @@ -978,7 +979,8 @@ public void greetingSpeechDetectionHonorsSignedPcm16PeakThreshold(int peak) { @ParameterizedTest @ValueSource(strings = { "", "\r\n", "exit\n", "quit\n" }) public void consoleEofBlankAndExitProduceNoPrompt(String input) { - StepVerifier.create(GptLiveSampleUtils.readPromptAsync(new StringReader(input))).verifyComplete(); + StepVerifier.create(GptLiveTextConversationAsyncSample.readPromptAsync(new StringReader(input))) + .verifyComplete(); } @ParameterizedTest @@ -1090,7 +1092,7 @@ public void interactiveTextPreservesServerClosureReason(boolean async, String re .port(0) .handle((request, response) -> response.sendWebsocket((inbound, outbound) -> outbound .sendString(Flux.concat(Mono.just(STARTED), inbound.receive().asString().concatMap(json -> { - Map command = GptLiveSampleUtils.object(BinaryData.fromString(json)); + Map command = GptLiveFunctionToolSample.object(BinaryData.fromString(json)); commands.add(command); if ("session.commentary.append".equals(command.get("type"))) { return Flux.just("{\"type\":\"session.output_transcript.delta\",\"delta\":\"Welcome.\"}", @@ -1132,10 +1134,10 @@ public void close() { throw new AssertionError("Standard input must not be closed"); } }; - try (BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent")) { IllegalStateException error = assertThrows(IllegalStateException.class, - () -> GptLiveSampleUtils.runInteractiveConversation(session, input, pcm -> { + () -> GptLiveTextConversationSample.runInteractiveConversation(session, input, pcm -> { }, () -> { })); assertEquals("Unable to read console input.", error.getMessage()); @@ -1150,17 +1152,18 @@ public void serviceErrorsFailBothTextSamples(boolean async) { if (async) { StepVerifier .create(Mono.usingWhen( - builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), - session -> GptLiveSampleUtils.runTextConversationAsync(session, Collections.singletonList("one")), - BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync)) + builder().beta().buildBetaVoiceAgentsLiveWebSocketAsyncClient().openWebSocketSession("agent"), + session -> GptLiveTextConversationAsyncSample.runConversation(session, + Collections.singletonList("one")), + BetaVoiceAgentsLiveWebSocketSessionAsyncClient::closeAsync)) .expectErrorMatches(error -> error instanceof IllegalStateException && error.getMessage().contains("GPT Live service error")) .verify(TIMEOUT); } else { - try (BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent")) { assertThrows(IllegalStateException.class, - () -> GptLiveSampleUtils.runTextConversation(session, Collections.singletonList("one"))); + () -> GptLiveTextConversationSample.runConversation(session, Collections.singletonList("one"))); } } assertFalse(commandTypes().contains("session.close")); @@ -1169,17 +1172,17 @@ public void serviceErrorsFailBothTextSamples(boolean async) { @Test public void toolsUseGranularEventsAndReturnResultsBeforeContinuation() { startTextServer(true, false); - try (BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - GptLiveSampleUtils.runToolConversation(session, Collections.singletonList("Weather in Seattle?"), + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent")) { + GptLiveFunctionToolSample.runConversation(session, Collections.singletonList("Weather in Seattle?"), GptLiveFunctionToolSample::toolResult); } assertEquals(Arrays.asList("response.item.create", "response.create", "response.item.create", "response.create", "session.close"), commandTypes()); - Map item = GptLiveSampleUtils.object(commands.get(2).get("item")); + Map item = GptLiveFunctionToolSample.object(commands.get(2).get("item")); assertEquals("function_call_output", item.get("type")); assertEquals("call-1", item.get("call_id")); - Map result = GptLiveSampleUtils.object(BinaryData.fromString((String) item.get("output"))); + Map result = GptLiveFunctionToolSample.object(BinaryData.fromString((String) item.get("output"))); assertEquals("Seattle", result.get("city")); assertEquals(true, result.get("simulated")); assertFalse(item.containsKey("delegation_id")); @@ -1189,10 +1192,10 @@ public void toolsUseGranularEventsAndReturnResultsBeforeContinuation() { public void toolHandlerFailureStopsConversationBeforeContinuation() { startTextServer(true, false); IllegalStateException failure = new IllegalStateException("Tool execution failed"); - try (BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - IllegalStateException error = assertThrows(IllegalStateException.class, () -> GptLiveSampleUtils - .runToolConversation(session, Collections.singletonList("Weather in Seattle?"), call -> { + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent")) { + IllegalStateException error = assertThrows(IllegalStateException.class, () -> GptLiveFunctionToolSample + .runConversation(session, Collections.singletonList("Weather in Seattle?"), call -> { throw failure; })); assertEquals(failure, error); @@ -1201,32 +1204,34 @@ public void toolHandlerFailureStopsConversationBeforeContinuation() { } @Test - public void toolConversationUsesSharedMultiTurnLoop() { + public void toolSampleHandlesMultipleTurns() { startTextServer(true, false); - try (BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { - GptLiveSampleUtils.runToolConversation(session, Arrays.asList("Weather in Seattle?", "Thanks!"), + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent")) { + GptLiveFunctionToolSample.runConversation(session, Arrays.asList("Weather in Seattle?", "Thanks!"), GptLiveFunctionToolSample::toolResult); } assertEquals(Arrays.asList("response.item.create", "response.create", "response.item.create", "response.create", "response.item.create", "response.create", "session.close"), commandTypes()); - Map followUp = GptLiveSampleUtils.object(commands.get(4).get("item")); + Map followUp = GptLiveFunctionToolSample.object(commands.get(4).get("item")); assertEquals("user", followUp.get("role")); - assertEquals("Thanks!", GptLiveSampleUtils.object(((List) followUp.get("content")).get(0)).get("text")); + assertEquals("Thanks!", + GptLiveFunctionToolSample.object(((List) followUp.get("content")).get(0)).get("text")); } @Test public void toolSchemaAndArgumentsAreValidated() { - Map tool = GptLiveSampleUtils.object(GptLiveFunctionToolSample.weatherTool()); + Map tool + = GptLiveFunctionToolSample.object(BinaryData.fromObject(GptLiveFunctionToolSample.weatherTool())); assertEquals("function", tool.get("type")); assertEquals("get_weather", tool.get("name")); assertEquals(true, tool.get("strict")); - assertEquals(false, GptLiveSampleUtils.object(tool.get("parameters")).get("additionalProperties")); - Map unknown = GptLiveSampleUtils - .object(BinaryData.fromString("{\"name\":\"unknown\",\"call_id\":\"c\",\"arguments\":\"{}\"}")); + assertEquals(false, GptLiveFunctionToolSample.object(tool.get("parameters")).get("additionalProperties")); + ResponseFunctionToolCall unknown + = ResponseFunctionToolCall.builder().name("unknown").callId("c").arguments("{}").build(); assertThrows(IllegalArgumentException.class, () -> GptLiveFunctionToolSample.toolResult(unknown)); - Map invalid = GptLiveSampleUtils - .object(BinaryData.fromString("{\"name\":\"get_weather\",\"call_id\":\"c\",\"arguments\":\"{}\"}")); + ResponseFunctionToolCall invalid + = ResponseFunctionToolCall.builder().name("get_weather").callId("c").arguments("{}").build(); assertThrows(IllegalArgumentException.class, () -> GptLiveFunctionToolSample.toolResult(invalid)); } @@ -1242,16 +1247,17 @@ public void prematureClosureDoesNotReportSuccessfulConversation(boolean async) { if (async) { StepVerifier .create(Mono.usingWhen( - builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("agent"), - session -> GptLiveSampleUtils.runTextConversationAsync(session, Collections.singletonList("one")), - BetaVoiceAgentLiveWebSocketSessionAsyncClient::closeAsync)) + builder().beta().buildBetaVoiceAgentsLiveWebSocketAsyncClient().openWebSocketSession("agent"), + session -> GptLiveTextConversationAsyncSample.runConversation(session, + Collections.singletonList("one")), + BetaVoiceAgentsLiveWebSocketSessionAsyncClient::closeAsync)) .expectError(IllegalStateException.class) .verify(TIMEOUT); } else { - try (BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent")) { assertThrows(IllegalStateException.class, - () -> GptLiveSampleUtils.runTextConversation(session, Collections.singletonList("one"))); + () -> GptLiveTextConversationSample.runConversation(session, Collections.singletonList("one"))); } } } @@ -1261,7 +1267,7 @@ public void typedCommentaryPreservesExplicitNullDelegationId() throws IOExceptio ClientEvent event = GptLiveSampleUtils.commentary("Hello"); assertTrue(event.isSessionCommentaryAppend()); BinaryData data = OpenAIJsonHelper.toBinaryData(event); - Map command = GptLiveSampleUtils.object(data); + Map command = GptLiveFunctionToolSample.object(data); assertEquals("session.commentary.append", command.get("type")); assertEquals("Hello", command.get("content")); assertTrue(command.containsKey("delegation_id")); @@ -1270,7 +1276,7 @@ public void typedCommentaryPreservesExplicitNullDelegationId() throws IOExceptio try (com.azure.json.JsonWriter writer = com.azure.json.JsonProviders.createWriter(output)) { data.writeTo(writer); } - assertEquals(command, GptLiveSampleUtils.object(BinaryData.fromBytes(output.toByteArray()))); + assertEquals(command, GptLiveFunctionToolSample.object(BinaryData.fromBytes(output.toByteArray()))); } @Test @@ -1344,7 +1350,7 @@ public void audioSampleGreetsCapturesPlaysAndReleasesDevices(boolean moderated, outbound) -> outbound.sendString(Flux.concat( Flux.just(started, "{\"type\":\"session.output_audio.delta\",\"delta\":\"AQI=\"}"), inbound.receive().asString().concatMap(json -> { - Map command = GptLiveSampleUtils.object(BinaryData.fromString(json)); + Map command = GptLiveFunctionToolSample.object(BinaryData.fromString(json)); commands.add(command); if ("session.input_audio.append".equals(command.get("type"))) { captured.set(true); @@ -1355,8 +1361,8 @@ public void audioSampleGreetsCapturesPlaysAndReleasesDevices(boolean moderated, return "session.close".equals(command.get("type")) ? Mono.just(CLOSED) : Mono.empty(); }))).then()); }).bindNow(); - BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() - .buildBetaVoiceAgentLiveWebSocketAsyncClient() + BetaVoiceAgentsLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentsLiveWebSocketAsyncClient() .openWebSocketSession("agent", new VoiceAgentLiveWebSocketConnectionOptions().setStoreEnabled(store)) .block(TIMEOUT); assertNotNull(session); @@ -1375,7 +1381,8 @@ public int read() { try { GptLiveSampleUtils.AudioProcessor processor = new GptLiveSampleUtils.AudioProcessor(microphone, speaker, pcm -> session.sendEvent(GptLiveSampleUtils.audio(pcm.toBytes()))); - Mono conversation = GptLiveSampleUtils.runAudioConversationAsync(session, processor, enterAfterAudio); + Mono conversation + = GptLiveAudioConversationAsyncSample.runAudioConversationAsync(session, processor, enterAfterAudio); if (moderated) { IllegalStateException error = assertThrows(IllegalStateException.class, () -> conversation.block(TIMEOUT)); @@ -1396,9 +1403,9 @@ public int read() { .filter(command -> "response.item.create".equals(command.get("type"))) .findFirst() .get(); - Map item = GptLiveSampleUtils.object(greeting.get("item")); + Map item = GptLiveFunctionToolSample.object(greeting.get("item")); assertEquals("Greet me briefly with: Hi, how can I help you today?", - GptLiveSampleUtils.object(((List) item.get("content")).get(0)).get("text")); + GptLiveFunctionToolSample.object(((List) item.get("content")).get(0)).get("text")); } finally { session.closeAsync().block(TIMEOUT); } @@ -1406,7 +1413,7 @@ public int read() { private List commandTypes() { return commands.stream() - .map(command -> GptLiveSampleUtils.string(command, "type")) + .map(command -> GptLiveFunctionToolSample.string(command, "type")) .collect(java.util.stream.Collectors.toList()); } @@ -1435,7 +1442,7 @@ private void startTextServer(boolean tools, boolean fail, Mono beforeGreet response) -> response.sendWebsocket((inbound, outbound) -> outbound .sendString(Flux.concat(Mono.just(STARTED).delayElement(Duration.ofMillis(50)), inbound.receive().asString().concatMap(json -> { - Map command = GptLiveSampleUtils.object(BinaryData.fromString(json)); + Map command = GptLiveFunctionToolSample.object(BinaryData.fromString(json)); commands.add(command); if ("session.commentary.append".equals(command.get("type"))) { return beforeGreeting.thenMany( diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsAsyncTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsAsyncTests.java index 6d684658132ab..c16daa0a8a0f1 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsAsyncTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsAsyncTests.java @@ -7,7 +7,7 @@ import com.azure.ai.agents.AgentsAsyncClient; import com.azure.ai.agents.AgentsClientBuilder; import com.azure.ai.agents.BetaVoiceAgentsConversationsAsyncClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketSessionAsyncClient; import com.azure.ai.agents.models.CreateAgentVersionInput; import com.azure.ai.agents.models.RealtimeResponseDoneEvent; import com.azure.ai.agents.models.RealtimeSessionCreatedEvent; @@ -105,7 +105,7 @@ public void readLivePersistedConversation() { try { agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition)).block(TIMEOUT); created = true; - Mono.usingWhen(builder.beta().buildBetaVoiceAgentWebSocketAsyncClient().openWebSocketSession(agentName), + Mono.usingWhen(builder.beta().buildBetaVoiceAgentsWebSocketAsyncClient().openWebSocketSession(agentName), session -> session.receiveEvents().index().concatMap(indexed -> { if (indexed.getT1() == 0) { assertTrue(indexed.getT2() instanceof RealtimeSessionCreatedEvent, @@ -123,8 +123,8 @@ public void readLivePersistedConversation() { .switchIfEmpty(Mono.error(new AssertionError("Session ended without response.done."))) .timeout(Duration.ofSeconds(45)) .then(), - BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), - BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync).block(Duration.ofSeconds(90)); + BetaVoiceAgentsWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), + BetaVoiceAgentsWebSocketSessionAsyncClient::closeAsync).block(Duration.ofSeconds(90)); Mono.delay(Duration.ofSeconds(30)).block(Duration.ofSeconds(35)); reading = true; assertPersistedConversation(conversations, agentName, conversationId.get()); diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsTests.java index 590ed1b4cceaf..d80518c0c5a52 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsTests.java @@ -6,7 +6,7 @@ import com.azure.ai.agents.AgentsClient; import com.azure.ai.agents.AgentsClientBuilder; import com.azure.ai.agents.BetaVoiceAgentsConversationsClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketSessionClient; import com.azure.ai.agents.models.CreateAgentVersionInput; import com.azure.ai.agents.models.RealtimeServerEvent; import com.azure.ai.agents.models.RealtimeResponseDoneEvent; @@ -107,8 +107,8 @@ public void readLivePersistedConversation() throws InterruptedException { try { agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition)); created = true; - try (BetaVoiceAgentWebSocketSessionClient session - = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession(agentName)) { + try (BetaVoiceAgentsWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentsWebSocketClient().openWebSocketSession(agentName)) { Iterator events = session.receiveEvents(TIMEOUT).iterator(); assertTrue(events.hasNext(), "Expected session.created."); RealtimeServerEvent first = events.next(); diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java index 31c71d5e0e61f..8e405faa76a44 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java @@ -4,10 +4,10 @@ package com.azure.ai.agents.voice; import com.azure.ai.agents.AgentsClientBuilder; -import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionAsyncClient; -import com.azure.ai.agents.BetaVoiceAgentLiveWebSocketSessionClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.BetaVoiceAgentsLiveWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsLiveWebSocketSessionClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketSessionClient; import com.azure.ai.agents.implementation.OpenAIJsonHelper; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; @@ -105,18 +105,18 @@ public void handshakeUsesLiveRouteWithoutRealtimeSubprotocol(boolean async) thro .setAgentVersionOverride("version 1") .setStructuredInputs(inputs); if (async) { - Mono connection = builder.beta() - .buildBetaVoiceAgentLiveWebSocketAsyncClient() + Mono connection = builder.beta() + .buildBetaVoiceAgentsLiveWebSocketAsyncClient() .openWebSocketSession("agent /name", options); options.setStoreEnabled(false).setAgentVersionOverride("changed").setStructuredInputs("{}"); - BetaVoiceAgentLiveWebSocketSessionAsyncClient session = connection.block(TIMEOUT); + BetaVoiceAgentsLiveWebSocketSessionAsyncClient session = connection.block(TIMEOUT); assertNotNull(session); assertTrue(session.isOpen()); session.closeAsync().block(TIMEOUT); assertFalse(session.isOpen()); } else { - try (BetaVoiceAgentLiveWebSocketSessionClient session = builder.beta() - .buildBetaVoiceAgentLiveWebSocketClient() + try (BetaVoiceAgentsLiveWebSocketSessionClient session = builder.beta() + .buildBetaVoiceAgentsLiveWebSocketClient() .openWebSocketSession("agent /name", options)) { assertTrue(session.isOpen()); } @@ -144,8 +144,8 @@ public void handshakeUsesLiveRouteWithoutRealtimeSubprotocol(boolean async) thro public void syncNativeEventsAreLosslessAndSessionIsBidirectional() { List sent = new CopyOnWriteArrayList<>(); startEchoServer(sent); - BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent"); + BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent"); try { Iterator events = session.receiveRawEvents(TIMEOUT).iterator(); assertEquals(STARTED, events.next().toString()); @@ -170,8 +170,8 @@ public void syncNativeEventsAreLosslessAndSessionIsBidirectional() { public void asyncNativeEventsAreLosslessAndSessionIsBidirectional() { List sent = new CopyOnWriteArrayList<>(); startEchoServer(sent); - BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() - .buildBetaVoiceAgentLiveWebSocketAsyncClient() + BetaVoiceAgentsLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentsLiveWebSocketAsyncClient() .openWebSocketSession("agent") .block(TIMEOUT); assertNotNull(session); @@ -218,27 +218,27 @@ public void sameBuilderKeepsLiveAndRealtimeHandshakesIndependent(boolean async) VoiceAgentWebSocketConnectionOptions realtimeOptions = new VoiceAgentWebSocketConnectionOptions().setStructuredInputs(inputs); if (async) { - try (BetaVoiceAgentLiveWebSocketSessionAsyncClient live = builder.beta() - .buildBetaVoiceAgentLiveWebSocketAsyncClient() + try (BetaVoiceAgentsLiveWebSocketSessionAsyncClient live = builder.beta() + .buildBetaVoiceAgentsLiveWebSocketAsyncClient() .openWebSocketSession("agent", liveOptions) .block(TIMEOUT)) { assertNotNull(live); assertTrue(live.isOpen()); } - try (BetaVoiceAgentWebSocketSessionAsyncClient realtime = builder.beta() - .buildBetaVoiceAgentWebSocketAsyncClient() + try (BetaVoiceAgentsWebSocketSessionAsyncClient realtime = builder.beta() + .buildBetaVoiceAgentsWebSocketAsyncClient() .openWebSocketSession("agent", realtimeOptions) .block(TIMEOUT)) { assertNotNull(realtime); assertTrue(realtime.isOpen()); } } else { - try (BetaVoiceAgentLiveWebSocketSessionClient live - = builder.beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent", liveOptions)) { + try (BetaVoiceAgentsLiveWebSocketSessionClient live + = builder.beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent", liveOptions)) { assertTrue(live.isOpen()); } - try (BetaVoiceAgentWebSocketSessionClient realtime - = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", realtimeOptions)) { + try (BetaVoiceAgentsWebSocketSessionClient realtime + = builder.beta().buildBetaVoiceAgentsWebSocketClient().openWebSocketSession("agent", realtimeOptions)) { assertTrue(realtime.isOpen()); } } @@ -276,8 +276,8 @@ public void typedEventsRoundTripWithoutLosingUnknownFields(boolean async) { .build()); String commandJson = OpenAIJsonHelper.toBinaryData(command).toString(); if (async) { - BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() - .buildBetaVoiceAgentLiveWebSocketAsyncClient() + BetaVoiceAgentsLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentsLiveWebSocketAsyncClient() .openWebSocketSession("agent") .block(TIMEOUT); assertNotNull(session); @@ -302,8 +302,8 @@ public void typedEventsRoundTripWithoutLosingUnknownFields(boolean async) { session.closeAsync().block(TIMEOUT); } } else { - try (BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent")) { assertThrows(NullPointerException.class, () -> session.sendEvent((ClientEvent) null)); assertThrows(IllegalArgumentException.class, () -> session.receiveEvents(Duration.ZERO)); Iterator events = session.receiveEvents(TIMEOUT).iterator(); @@ -356,12 +356,12 @@ public void handshakeFailuresSurfaceAsAzureExceptions(boolean async) { .bindNow(); if (async) { StepVerifier - .create(builder().beta().buildBetaVoiceAgentLiveWebSocketAsyncClient().openWebSocketSession("missing")) + .create(builder().beta().buildBetaVoiceAgentsLiveWebSocketAsyncClient().openWebSocketSession("missing")) .expectError(ResourceNotFoundException.class) .verify(TIMEOUT); } else { assertThrows(ResourceNotFoundException.class, - () -> builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("missing")); + () -> builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("missing")); } } @@ -375,8 +375,8 @@ public void malformedJsonTerminatesReception(boolean async) { (inbound, outbound) -> outbound.sendString(Mono.just("[]")).then(inbound.receive().then()))) .bindNow(); if (async) { - BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() - .buildBetaVoiceAgentLiveWebSocketAsyncClient() + BetaVoiceAgentsLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentsLiveWebSocketAsyncClient() .openWebSocketSession("agent") .block(TIMEOUT); assertNotNull(session); @@ -388,8 +388,8 @@ public void malformedJsonTerminatesReception(boolean async) { session.closeAsync().block(TIMEOUT); } } else { - try (BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent")) { assertThrows(IllegalArgumentException.class, () -> session.receiveRawEvents(TIMEOUT).iterator().hasNext()); } @@ -435,8 +435,8 @@ public void peerClosurePreservesEventsAndCloseMetadata(boolean async) { .then(outbound.sendClose(1000, "finished")))) .bindNow(); if (async) { - BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() - .buildBetaVoiceAgentLiveWebSocketAsyncClient() + BetaVoiceAgentsLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentsLiveWebSocketAsyncClient() .openWebSocketSession("agent") .block(TIMEOUT); assertNotNull(session); @@ -452,8 +452,8 @@ public void peerClosurePreservesEventsAndCloseMetadata(boolean async) { session.closeAsync().block(TIMEOUT); } } else { - try (BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent")) { + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent")) { Iterator events = session.receiveRawEvents(TIMEOUT).iterator(); assertEquals(STARTED, events.next().toString()); assertFalse(events.hasNext()); @@ -477,8 +477,8 @@ public void malformedEventCallbackCanSkipInvalidJson(boolean async) { (inbound, outbound) -> outbound.sendString(Flux.just("[]", STARTED)).then(inbound.receive().then()))) .bindNow(); if (async) { - BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() - .buildBetaVoiceAgentLiveWebSocketAsyncClient() + BetaVoiceAgentsLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentsLiveWebSocketAsyncClient() .openWebSocketSession("agent", options) .block(TIMEOUT); assertNotNull(session); @@ -491,8 +491,8 @@ public void malformedEventCallbackCanSkipInvalidJson(boolean async) { session.closeAsync().block(TIMEOUT); } } else { - try (BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent", options)) { + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent", options)) { assertEquals(STARTED, session.receiveRawEvents(TIMEOUT).iterator().next().toString()); } } @@ -513,8 +513,8 @@ public void oversizedEventsFailEvenWhenMalformedCallbackIsConfigured(boolean asy (inbound, outbound) -> outbound.sendString(Mono.just(STARTED)).then(inbound.receive().then()))) .bindNow(); if (async) { - BetaVoiceAgentLiveWebSocketSessionAsyncClient session = builder().beta() - .buildBetaVoiceAgentLiveWebSocketAsyncClient() + BetaVoiceAgentsLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentsLiveWebSocketAsyncClient() .openWebSocketSession("agent", options) .block(TIMEOUT); assertNotNull(session); @@ -524,8 +524,8 @@ public void oversizedEventsFailEvenWhenMalformedCallbackIsConfigured(boolean asy session.closeAsync().block(TIMEOUT); } } else { - try (BetaVoiceAgentLiveWebSocketSessionClient session - = builder().beta().buildBetaVoiceAgentLiveWebSocketClient().openWebSocketSession("agent", options)) { + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent", options)) { assertThrows(IllegalArgumentException.class, () -> session.receiveRawEvents(TIMEOUT).iterator().hasNext()); } @@ -541,9 +541,9 @@ public void unsupportedHttpSettingsAreRejected(boolean async) { .httpClient(com.azure.core.http.HttpClient.createDefault()); if (async) { assertThrows(IllegalStateException.class, - () -> builder.beta().buildBetaVoiceAgentLiveWebSocketAsyncClient()); + () -> builder.beta().buildBetaVoiceAgentsLiveWebSocketAsyncClient()); } else { - assertThrows(IllegalStateException.class, () -> builder.beta().buildBetaVoiceAgentLiveWebSocketClient()); + assertThrows(IllegalStateException.class, () -> builder.beta().buildBetaVoiceAgentsLiveWebSocketClient()); } } diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentRealtimeLiveTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentRealtimeLiveTests.java index 9b05562145d25..613b92cc46172 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentRealtimeLiveTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentRealtimeLiveTests.java @@ -6,8 +6,8 @@ import com.azure.ai.agents.AgentsAsyncClient; import com.azure.ai.agents.AgentsClient; import com.azure.ai.agents.AgentsClientBuilder; -import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketSessionClient; import com.azure.ai.agents.models.CreateAgentVersionInput; import com.azure.ai.agents.models.RealtimeClientEvent; import com.azure.ai.agents.models.RealtimeConversationItemCreateEvent; @@ -87,8 +87,8 @@ public void realtimeLive(Scenario scenario) { try { agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition(scenario))); created = true; - try (BetaVoiceAgentWebSocketSessionClient session - = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession(agentName)) { + try (BetaVoiceAgentsWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentsWebSocketClient().openWebSocketSession(agentName)) { Turn turn = new Turn(scenario); Iterator events = session.receiveEvents(EVENT_TIMEOUT).iterator(); turn.accept(events.next()).forEach(session::sendEvent); @@ -118,7 +118,7 @@ public void realtimeLiveAsync(Scenario scenario) { .block(EVENT_TIMEOUT); created = true; Turn turn = new Turn(scenario); - Mono.usingWhen(builder.beta().buildBetaVoiceAgentWebSocketAsyncClient().openWebSocketSession(agentName), + Mono.usingWhen(builder.beta().buildBetaVoiceAgentsWebSocketAsyncClient().openWebSocketSession(agentName), session -> session.receiveEvents() .timeout(EVENT_TIMEOUT) .concatMap(event -> Flux.fromIterable(turn.accept(event)) @@ -130,8 +130,8 @@ public void realtimeLiveAsync(Scenario scenario) { .timeout(RESPONSE_TIMEOUT) .doOnNext(ignored -> turn.assertComplete()) .then(), - BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), - BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync).block(Duration.ofSeconds(90)); + BetaVoiceAgentsWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), + BetaVoiceAgentsWebSocketSessionAsyncClient::closeAsync).block(Duration.ofSeconds(90)); } finally { if (created) { agents.deleteAgent(agentName).block(EVENT_TIMEOUT); diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentWebSocketSessionAsyncTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentWebSocketSessionAsyncTests.java index 1b6e6ce806156..8a4967c1bcb62 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentWebSocketSessionAsyncTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentWebSocketSessionAsyncTests.java @@ -4,8 +4,8 @@ package com.azure.ai.agents.voice; import com.azure.ai.agents.AgentsClientBuilder; -import com.azure.ai.agents.BetaVoiceAgentWebSocketAsyncClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketSessionAsyncClient; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketHttpResponse; import com.azure.ai.agents.models.RawRealtimeServerEvent; import com.azure.ai.agents.models.RealtimeResponseCreateEvent; @@ -93,8 +93,8 @@ public void handshakeUsesSdkManagedProtocolValues() { new Header("Foundry-Features", "override-feature"), new Header("Sec-WebSocket-Protocol", "override-protocol")))); - BetaVoiceAgentWebSocketSessionAsyncClient session = builder.beta() - .buildBetaVoiceAgentWebSocketAsyncClient() + BetaVoiceAgentsWebSocketSessionAsyncClient session = builder.beta() + .buildBetaVoiceAgentsWebSocketAsyncClient() .openWebSocketSession("agent name") .block(Duration.ofSeconds(5)); session.closeAsync().block(Duration.ofSeconds(5)); @@ -133,8 +133,8 @@ public void handshakeUsesHttpLogApplicationIdFallback() { .httpLogOptions(new HttpLogOptions().setApplicationId("log-app")) .clientOptions(new ClientOptions()); - BetaVoiceAgentWebSocketSessionAsyncClient session = builder.beta() - .buildBetaVoiceAgentWebSocketAsyncClient() + BetaVoiceAgentsWebSocketSessionAsyncClient session = builder.beta() + .buildBetaVoiceAgentsWebSocketAsyncClient() .openWebSocketSession("agent") .block(Duration.ofSeconds(5)); session.closeAsync().block(Duration.ofSeconds(5)); @@ -168,8 +168,8 @@ public void typedStringAndMappingSendsRejectInvalidJson() { AgentsClientBuilder builder = builder(server.port()); String raw = "{\"type\": \"response.create\"}"; BinaryData mapping = BinaryData.fromObject(Collections.singletonMap("type", "response.cancel")); - BetaVoiceAgentWebSocketSessionAsyncClient session = builder.beta() - .buildBetaVoiceAgentWebSocketAsyncClient() + BetaVoiceAgentsWebSocketSessionAsyncClient session = builder.beta() + .buildBetaVoiceAgentsWebSocketAsyncClient() .openWebSocketSession("agent", tlsOptions()) .block(Duration.ofSeconds(5)); try { @@ -196,7 +196,7 @@ public void pingPongFramesAreNotApplicationEvents() { new TextWebSocketFrame("{\"type\":\"session.created\",\"session\":{}}"), new TextWebSocketFrame("{\"type\":\"future.event\",\"foo\":\"bar\"}"))); server = frameWebSocketServer(frames, false); - BetaVoiceAgentWebSocketSessionAsyncClient session + BetaVoiceAgentsWebSocketSessionAsyncClient session = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(Duration.ofSeconds(5)); List events; try { @@ -219,7 +219,7 @@ public void malformedEventsCanBeReportedAndSkipped() { new TextWebSocketFrame(warningJson()))), true); VoiceAgentWebSocketConnectionOptions options = tlsOptions().setMalformedEventHandler(error -> failures.incrementAndGet()); - BetaVoiceAgentWebSocketSessionAsyncClient session + BetaVoiceAgentsWebSocketSessionAsyncClient session = createAsyncClient(server.port()).openWebSocketSession("agent", options).block(Duration.ofSeconds(5)); StepVerifier.create(session.receiveEvents()) .assertNext(this::assertWarningEvent) @@ -236,7 +236,7 @@ public void boundedQueuesHonorOverflowPolicies() { .map(index -> new TextWebSocketFrame("{\"type\":\"future.event\",\"index\":" + index + "}")), true); VoiceAgentWebSocketConnectionOptions options = tlsOptions().setReceiveBufferCapacity(2).setOverflowStrategy(strategy); - BetaVoiceAgentWebSocketSessionAsyncClient session + BetaVoiceAgentsWebSocketSessionAsyncClient session = createAsyncClient(server.port()).openWebSocketSession("agent", options).block(Duration.ofSeconds(5)); boolean overflowError = strategy == VoiceAgentWebSocketOverflowStrategy.ERROR; List received = new ArrayList<>(); @@ -273,7 +273,7 @@ public void messageSizeLimitCannotBeBypassedByRecoveryHandler() { VoiceAgentWebSocketConnectionOptions options = tlsOptions().setMaxMessageSize(16).setMalformedEventHandler(error -> recovered.set(true)); StepVerifier.create(createAsyncClient(server.port()).openWebSocketSession("agent", options) - .flatMapMany(BetaVoiceAgentWebSocketSessionAsyncClient::receiveEvents)).expectError().verify(); + .flatMapMany(BetaVoiceAgentsWebSocketSessionAsyncClient::receiveEvents)).expectError().verify(); assertFalse(recovered.get()); } @@ -284,7 +284,7 @@ public void rawEventRoundTripsAndOptionsValidateBounds() throws Exception { RawRealtimeServerEvent copy = BinaryData.fromObject(event).toObject(RawRealtimeServerEvent.class); assertEquals(payload.toObject(Map.class), copy.getRawEvent().toObject(Map.class)); server = oneShotWebSocketServer("{\"type\":42,\"value\":1}"); - BetaVoiceAgentWebSocketSessionAsyncClient session + BetaVoiceAgentsWebSocketSessionAsyncClient session = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(Duration.ofSeconds(5)); try { StepVerifier.create(session.receiveEvents()) @@ -314,7 +314,7 @@ public void insecureEndpointsAreRejectedBeforeAuthentication() { "https://example.com/#fragment" }) { AgentsClientBuilder clientBuilder = new AgentsClientBuilder().endpoint(endpoint).credential(credential); StepVerifier - .create(clientBuilder.beta().buildBetaVoiceAgentWebSocketAsyncClient().openWebSocketSession("agent")) + .create(clientBuilder.beta().buildBetaVoiceAgentsWebSocketAsyncClient().openWebSocketSession("agent")) .expectError(IllegalArgumentException.class) .verify(); } @@ -332,7 +332,7 @@ public void rawEventsUseCustomizedTlsTransport() { WebsocketServerSpec.builder().protocols("realtime").build())) .bindNow(); BinaryData payload = BinaryData.fromString("{\"type\":\"future.event\",\"value\":42}"); - BetaVoiceAgentWebSocketSessionAsyncClient session + BetaVoiceAgentsWebSocketSessionAsyncClient session = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(Duration.ofSeconds(5)); StepVerifier.create(session.sendEvent(BinaryData.fromString("[]"))) .expectError(IllegalArgumentException.class) @@ -350,7 +350,7 @@ public void rawEventsUseCustomizedTlsTransport() { public void customCloseFrame() { server = startServer(new CopyOnWriteArrayList<>(), new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), false); - BetaVoiceAgentWebSocketSessionAsyncClient session + BetaVoiceAgentsWebSocketSessionAsyncClient session = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(Duration.ofSeconds(5)); assertNotNull(session); StepVerifier.create(session.closeAsync(1006, "invalid")).expectError(IllegalArgumentException.class).verify(); @@ -376,18 +376,18 @@ public void asyncSessionNegotiatesHandshakeAndExchangesTypedEvents() { }; VoiceAgentWebSocketConnectionOptions options = tlsOptions().setStoreEnabled(true).setAgentVersionOverride("version 2"); - BetaVoiceAgentWebSocketAsyncClient client + BetaVoiceAgentsWebSocketAsyncClient client = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") .credential(credential) .configuration(Configuration.NONE) .clientOptions(new ClientOptions().setApplicationId("test-app") .setHeaders(Collections.singletonList(new Header("X-Test-Header", "test-value")))) .beta() - .buildBetaVoiceAgentWebSocketAsyncClient(); - Mono sessionMono + .buildBetaVoiceAgentsWebSocketAsyncClient(); + Mono sessionMono = client.openWebSocketSession("agent name", options); options.setStoreEnabled(false).setAgentVersionOverride("mutated"); - BetaVoiceAgentWebSocketSessionAsyncClient session = sessionMono.block(); + BetaVoiceAgentsWebSocketSessionAsyncClient session = sessionMono.block(); assertTrue(session.isOpen()); StepVerifier.create(session.receiveEvents().take(4)) .then(() -> session.sendText("hello").block()) @@ -428,12 +428,12 @@ public void tokenFailureOccursBeforeNetworkAccess() { return response.send(); }).bindNow(); TokenCredential credential = request -> Mono.error(new IllegalStateException("token unavailable")); - BetaVoiceAgentWebSocketAsyncClient client + BetaVoiceAgentsWebSocketAsyncClient client = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") .credential(credential) .configuration(Configuration.NONE) .beta() - .buildBetaVoiceAgentWebSocketAsyncClient(); + .buildBetaVoiceAgentsWebSocketAsyncClient(); StepVerifier.create(client.openWebSocketSession("agent", tlsOptions())) .expectErrorMatches( error -> error instanceof IllegalStateException && error.getMessage().contains("token unavailable")) @@ -447,12 +447,12 @@ public void tokenAcquisitionDoesNotUseHandshakeTimeout() { new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), false); TokenCredential credential = request -> Mono.delay(Duration.ofMillis(1500)) .map(ignored -> new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); - BetaVoiceAgentWebSocketAsyncClient client + BetaVoiceAgentsWebSocketAsyncClient client = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") .credential(credential) .configuration(Configuration.NONE) .beta() - .buildBetaVoiceAgentWebSocketAsyncClient(); + .buildBetaVoiceAgentsWebSocketAsyncClient(); VoiceAgentWebSocketConnectionOptions options = tlsOptions().setHandshakeTimeout(Duration.ofSeconds(1)); StepVerifier.withVirtualTime(() -> client.openWebSocketSession("agent", options).flatMap(session -> { assertTrue(session.isOpen()); @@ -490,11 +490,11 @@ public void cancellingAsyncConnectCancelsTokenRequest() { AtomicBoolean tokenRequestCancelled = new AtomicBoolean(); TokenCredential credential = request -> Mono.never().doOnCancel(() -> tokenRequestCancelled.set(true)); - BetaVoiceAgentWebSocketAsyncClient client = new AgentsClientBuilder().endpoint("https://example.com") + BetaVoiceAgentsWebSocketAsyncClient client = new AgentsClientBuilder().endpoint("https://example.com") .credential(credential) .configuration(Configuration.NONE) .beta() - .buildBetaVoiceAgentWebSocketAsyncClient(); + .buildBetaVoiceAgentsWebSocketAsyncClient(); StepVerifier.create(client.openWebSocketSession("agent", tlsOptions())).thenCancel().verify(); assertTrue(tokenRequestCancelled.get()); } @@ -502,7 +502,7 @@ public void cancellingAsyncConnectCancelsTokenRequest() { @Test public void unknownEventFallsBackToRealtimeServerEvent() { server = oneShotWebSocketServer("{\"type\":\"future.event\",\"value\":42}"); - BetaVoiceAgentWebSocketSessionAsyncClient session + BetaVoiceAgentsWebSocketSessionAsyncClient session = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(); StepVerifier.create(session.receiveEvents()).assertNext(event -> { assertEquals("future.event", event.getType().toString()); @@ -518,7 +518,7 @@ public void fragmentedTextFrameIsAggregated() { int split = message.length() / 2; server = frameWebSocketServer(Flux.just(new TextWebSocketFrame(false, 0, message.substring(0, split)), new ContinuationWebSocketFrame(true, 0, message.substring(split))), true); - BetaVoiceAgentWebSocketSessionAsyncClient session + BetaVoiceAgentsWebSocketSessionAsyncClient session = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(); StepVerifier.create(session.receiveEvents()).assertNext(this::assertWarningEvent).verifyComplete(); session.close(); @@ -528,7 +528,7 @@ public void fragmentedTextFrameIsAggregated() { public void binaryJsonFrameIsParsed() { server = frameWebSocketServer( Mono.just(new BinaryWebSocketFrame(Unpooled.copiedBuffer(warningJson(), StandardCharsets.UTF_8))), true); - BetaVoiceAgentWebSocketSessionAsyncClient session + BetaVoiceAgentsWebSocketSessionAsyncClient session = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(); StepVerifier.create(session.receiveEvents()).assertNext(this::assertWarningEvent).verifyComplete(); session.close(); @@ -537,7 +537,7 @@ public void binaryJsonFrameIsParsed() { @Test public void malformedJsonTerminatesReceiveStream() { server = oneShotWebSocketServer("{not-json"); - BetaVoiceAgentWebSocketSessionAsyncClient session + BetaVoiceAgentsWebSocketSessionAsyncClient session = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(); StepVerifier.create(session.receiveEvents()).expectError().verify(); assertFalse(session.isOpen()); @@ -547,7 +547,7 @@ public void malformedJsonTerminatesReceiveStream() { public void closeIsIdempotentAndSendAfterCloseFails() { server = startServer(new CopyOnWriteArrayList<>(), new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), false); - BetaVoiceAgentWebSocketSessionAsyncClient session + BetaVoiceAgentsWebSocketSessionAsyncClient session = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(); StepVerifier.create(session.closeAsync().then(session.closeAsync())).verifyComplete(); StepVerifier.create(session.sendText("after close")) @@ -559,7 +559,7 @@ public void closeIsIdempotentAndSendAfterCloseFails() { @Test public void secureSessionUsesWssAndReceivesTypedEvent() throws Exception { server = oneShotWebSocketServer(warningJson()); - BetaVoiceAgentWebSocketSessionAsyncClient session + BetaVoiceAgentsWebSocketSessionAsyncClient session = createAsyncClient(server.port()).openWebSocketSession("secure-agent", tlsOptions()) .block(Duration.ofSeconds(5)); assertEquals("wss", URI.create(session.getEndpoint()).getScheme()); @@ -572,8 +572,8 @@ private AgentsClientBuilder builder(int port) { .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))); } - private BetaVoiceAgentWebSocketAsyncClient createAsyncClient(int port) { - return builder(port).configuration(Configuration.NONE).beta().buildBetaVoiceAgentWebSocketAsyncClient(); + private BetaVoiceAgentsWebSocketAsyncClient createAsyncClient(int port) { + return builder(port).configuration(Configuration.NONE).beta().buildBetaVoiceAgentsWebSocketAsyncClient(); } private static VoiceAgentWebSocketConnectionOptions tlsOptions() { diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentWebSocketSessionTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentWebSocketSessionTests.java index b4acb9796b16a..e7d60fc6d7744 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentWebSocketSessionTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentWebSocketSessionTests.java @@ -4,8 +4,8 @@ package com.azure.ai.agents.voice; import com.azure.ai.agents.AgentsClientBuilder; -import com.azure.ai.agents.BetaVoiceAgentWebSocketClient; -import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketClient; +import com.azure.ai.agents.BetaVoiceAgentsWebSocketSessionClient; import com.azure.ai.agents.models.RawRealtimeServerEvent; import com.azure.ai.agents.models.RealtimeResponseCreateEvent; import com.azure.ai.agents.models.RealtimeSessionCreatedEvent; @@ -111,8 +111,8 @@ public void handshakeUsesSdkManagedProtocolValues() { new Header("Foundry-Features", "override-feature"), new Header("Sec-WebSocket-Protocol", "override-protocol")))); - BetaVoiceAgentWebSocketSessionClient session - = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent name"); + BetaVoiceAgentsWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentsWebSocketClient().openWebSocketSession("agent name"); session.close(); assertFalse(session.isOpen()); @@ -149,8 +149,8 @@ public void handshakeUsesHttpLogApplicationIdFallback() { .httpLogOptions(new HttpLogOptions().setApplicationId("log-app")) .clientOptions(new ClientOptions()); - BetaVoiceAgentWebSocketSessionClient session - = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent"); + BetaVoiceAgentsWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentsWebSocketClient().openWebSocketSession("agent"); session.close(); assertTrue(userAgent.get().startsWith("log-app azsdk-java-azure-ai-agents/"), userAgent.get()); @@ -167,8 +167,8 @@ public void typedStringAndMappingSendsRejectInvalidJson() { .configuration(Configuration.NONE); String raw = "{\"type\": \"response.create\"}"; BinaryData mapping = BinaryData.fromObject(Collections.singletonMap("type", "response.cancel")); - try (BetaVoiceAgentWebSocketSessionClient session - = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", tlsOptions())) { + try (BetaVoiceAgentsWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentsWebSocketClient().openWebSocketSession("agent", tlsOptions())) { assertThrows(IllegalArgumentException.class, () -> session.sendEvent(BinaryData.fromString("not valid json"))); session.sendEvent(new RealtimeResponseCreateEvent()); @@ -200,8 +200,8 @@ public void pingPongFramesAreNotApplicationEvents() { .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))) .configuration(Configuration.NONE); List events = new ArrayList<>(); - try (BetaVoiceAgentWebSocketSessionClient session - = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", tlsOptions())) { + try (BetaVoiceAgentsWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentsWebSocketClient().openWebSocketSession("agent", tlsOptions())) { Iterator iterator = session.receiveEvents(Duration.ofSeconds(5)).iterator(); events.add(iterator.next()); events.add(iterator.next()); @@ -224,8 +224,8 @@ public void malformedEventsCanBeReportedAndSkipped() { AgentsClientBuilder builder = new AgentsClientBuilder().endpoint("https://localhost:" + server.port()) .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))) .configuration(Configuration.NONE); - try (BetaVoiceAgentWebSocketSessionClient session - = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", options)) { + try (BetaVoiceAgentsWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentsWebSocketClient().openWebSocketSession("agent", options)) { Iterator events = session.receiveEvents(Duration.ofSeconds(5)).iterator(); assertWarningEvent(events.next()); assertWarningEvent(events.next()); @@ -246,8 +246,8 @@ public void boundedQueuesHonorOverflowPolicies() { .configuration(Configuration.NONE); List received = new ArrayList<>(); boolean overflowError = strategy == VoiceAgentWebSocketOverflowStrategy.ERROR; - try (BetaVoiceAgentWebSocketSessionClient session - = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", options)) { + try (BetaVoiceAgentsWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentsWebSocketClient().openWebSocketSession("agent", options)) { assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { while (session.isOpen()) { Thread.yield(); @@ -282,8 +282,8 @@ public void messageSizeLimitCannotBeBypassedByRecoveryHandler() { .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))) .configuration(Configuration.NONE); assertThrows(RuntimeException.class, () -> { - try (BetaVoiceAgentWebSocketSessionClient session - = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", options)) { + try (BetaVoiceAgentsWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentsWebSocketClient().openWebSocketSession("agent", options)) { session.receiveEvents(Duration.ofSeconds(5)).iterator().next(); } }); @@ -304,7 +304,7 @@ public void insecureEndpointsAreRejectedBeforeAuthentication() { "https://example.com/#fragment" }) { AgentsClientBuilder builder = new AgentsClientBuilder().endpoint(endpoint).credential(credential); assertThrows(IllegalArgumentException.class, - () -> builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent")); + () -> builder.beta().buildBetaVoiceAgentsWebSocketClient().openWebSocketSession("agent")); } assertFalse(requested.get()); } @@ -322,13 +322,13 @@ private static VoiceAgentWebSocketConnectionOptions tlsOptions() { public void syncReceiveStreamRejectsSecondIterator() { server = startServer(new CopyOnWriteArrayList<>(), new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), false); - BetaVoiceAgentWebSocketClient client + BetaVoiceAgentsWebSocketClient client = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))) .beta() - .buildBetaVoiceAgentWebSocketClient(); + .buildBetaVoiceAgentsWebSocketClient(); - try (BetaVoiceAgentWebSocketSessionClient session = client.openWebSocketSession("agent", tlsOptions())) { + try (BetaVoiceAgentsWebSocketSessionClient session = client.openWebSocketSession("agent", tlsOptions())) { Iterable events = session.receiveEvents(Duration.ofSeconds(5)); events.iterator(); IllegalStateException exception = assertThrows(IllegalStateException.class, events::iterator); @@ -349,8 +349,8 @@ public void rawEventsUseCustomizedTlsTransport() { .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))) .configuration(Configuration.NONE); BinaryData payload = BinaryData.fromString("{\"type\":\"future.event\",\"value\":42}"); - try (BetaVoiceAgentWebSocketSessionClient session - = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", tlsOptions())) { + try (BetaVoiceAgentsWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentsWebSocketClient().openWebSocketSession("agent", tlsOptions())) { assertThrows(IllegalArgumentException.class, () -> session.sendEvent(BinaryData.fromString("[]"))); assertThrows(IllegalArgumentException.class, () -> session.sendEvent(BinaryData.fromString("{} {}"))); session.sendEvent(payload); @@ -368,8 +368,8 @@ public void customCloseFrameAndReceiveTimeout() { AgentsClientBuilder builder = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))); - try (BetaVoiceAgentWebSocketSessionClient session - = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", tlsOptions())) { + try (BetaVoiceAgentsWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentsWebSocketClient().openWebSocketSession("agent", tlsOptions())) { Iterator iterator = session.receiveEvents(Duration.ofMillis(20)).iterator(); IllegalStateException timeout = assertThrows(IllegalStateException.class, iterator::hasNext); assertInstanceOf(TimeoutException.class, timeout.getCause()); @@ -400,12 +400,12 @@ static void restoreTlsConfiguration() { public void syncConnectRejectsNullArguments() { TokenCredential credential = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); - BetaVoiceAgentWebSocketClient client + BetaVoiceAgentsWebSocketClient client = new AgentsClientBuilder().endpoint("https://localhost/api/projects/project") .credential(credential) .configuration(Configuration.NONE) .beta() - .buildBetaVoiceAgentWebSocketClient(); + .buildBetaVoiceAgentsWebSocketClient(); NullPointerException agentNameException = assertThrows(NullPointerException.class, () -> client.openWebSocketSession(null, tlsOptions())); @@ -424,12 +424,12 @@ public void syncTokenFailureOccursBeforeNetworkAccess() { return response.send(); }).bindNow(); TokenCredential credential = request -> Mono.error(new IllegalStateException("token unavailable")); - BetaVoiceAgentWebSocketClient client + BetaVoiceAgentsWebSocketClient client = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") .credential(credential) .configuration(Configuration.NONE) .beta() - .buildBetaVoiceAgentWebSocketClient(); + .buildBetaVoiceAgentsWebSocketClient(); IllegalStateException exception = assertThrows(IllegalStateException.class, () -> client.openWebSocketSession("agent", tlsOptions())); @@ -446,12 +446,12 @@ public void syncRejectedHandshakeMapsConflictToAzureException() { .bindNow(); TokenCredential credential = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); - BetaVoiceAgentWebSocketClient client + BetaVoiceAgentsWebSocketClient client = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") .credential(credential) .configuration(Configuration.NONE) .beta() - .buildBetaVoiceAgentWebSocketClient(); + .buildBetaVoiceAgentsWebSocketClient(); ResourceModifiedException exception = assertThrows(ResourceModifiedException.class, () -> client.openWebSocketSession("disabled-agent", tlsOptions())); @@ -463,11 +463,11 @@ public void syncRejectedHandshakeMapsConflictToAzureException() { public void syncClientRejectsEmptyAgentName() { TokenCredential credential = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); - BetaVoiceAgentWebSocketClient client = new AgentsClientBuilder().endpoint("https://example.com") + BetaVoiceAgentsWebSocketClient client = new AgentsClientBuilder().endpoint("https://example.com") .credential(credential) .configuration(Configuration.NONE) .beta() - .buildBetaVoiceAgentWebSocketClient(); + .buildBetaVoiceAgentsWebSocketClient(); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> client.openWebSocketSession("", tlsOptions())); @@ -480,14 +480,14 @@ public void syncReceiveBufferOverflowFailsTheEventStream() { server = frameWebSocketServer(frames); TokenCredential credential = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); - BetaVoiceAgentWebSocketClient client + BetaVoiceAgentsWebSocketClient client = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") .credential(credential) .configuration(Configuration.NONE) .beta() - .buildBetaVoiceAgentWebSocketClient(); + .buildBetaVoiceAgentsWebSocketClient(); - try (BetaVoiceAgentWebSocketSessionClient session = client.openWebSocketSession("agent", tlsOptions())) { + try (BetaVoiceAgentsWebSocketSessionClient session = client.openWebSocketSession("agent", tlsOptions())) { assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { while (session.isOpen()) { Thread.yield(); @@ -505,14 +505,14 @@ public void syncOrderlyClosePreservesFullReceiveBuffer() { server = frameWebSocketServer(frames); TokenCredential credential = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); - BetaVoiceAgentWebSocketClient client + BetaVoiceAgentsWebSocketClient client = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") .credential(credential) .configuration(Configuration.NONE) .beta() - .buildBetaVoiceAgentWebSocketClient(); + .buildBetaVoiceAgentsWebSocketClient(); - try (BetaVoiceAgentWebSocketSessionClient session = client.openWebSocketSession("agent", tlsOptions())) { + try (BetaVoiceAgentsWebSocketSessionClient session = client.openWebSocketSession("agent", tlsOptions())) { assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { Iterator events = session.receiveEvents().iterator(); int eventCount = 0; @@ -536,14 +536,14 @@ public void syncSessionReceivesTypedEventAndCloses() { new AtomicReference<>(), true); TokenCredential credential = request -> Mono.just(new AccessToken("sync-token", OffsetDateTime.now().plusHours(1))); - BetaVoiceAgentWebSocketClient client + BetaVoiceAgentsWebSocketClient client = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") .credential(credential) .configuration(Configuration.NONE) .beta() - .buildBetaVoiceAgentWebSocketClient(); + .buildBetaVoiceAgentsWebSocketClient(); - try (BetaVoiceAgentWebSocketSessionClient session = client.openWebSocketSession("sync-agent", tlsOptions())) { + try (BetaVoiceAgentsWebSocketSessionClient session = client.openWebSocketSession("sync-agent", tlsOptions())) { Iterator events = session.receiveEvents().iterator(); assertWarningEvent(events.next()); session.sendFunctionCallOutput("call-1", "{\"temperature\":72}"); From 0512a00bf30c241a2995ec65d22c955e41a17061 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Tue, 3 Nov 2026 20:01:41 +0800 Subject: [PATCH 15/20] Address remaining voice WebSocket review comments --- sdk/ai/azure-ai-agents/CHANGELOG.md | 5 +- sdk/ai/azure-ai-agents/README.md | 34 +++--- ...taVoiceAgentsLiveWebSocketAsyncClient.java | 12 +-- .../BetaVoiceAgentsLiveWebSocketClient.java | 12 ++- ...VoiceAgentsLiveWebSocketSessionClient.java | 4 + ...oiceAgentsWebSocketSessionAsyncClient.java | 5 +- ...BetaVoiceAgentsWebSocketSessionClient.java | 5 +- .../implementation/OpenAIJsonHelper.java | 41 ++++++- .../realtime/VoiceAgentWebSocketUtils.java | 34 +++--- .../realtime/WebSocketEventDecoder.java | 15 +++ .../WebSocketSessionAsyncTransport.java | 42 ++++---- .../realtime/WebSocketSessionState.java | 11 ++ .../realtime/WebSocketSessionTransport.java | 55 +++++----- .../realtime/WebSocketTransportOptions.java | 67 ++++++++++++ ...ceAgentLiveWebSocketConnectionOptions.java | 76 +++++++++---- .../com/azure/ai/agents/ReadmeSamples.java | 29 +++++ ...oiceAgentsWebSocketSessionClientTests.java | 100 ++++++++++++++++-- .../ai/agents/voice/GptLiveSamplesTests.java | 2 + 18 files changed, 415 insertions(+), 134 deletions(-) create mode 100644 sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketEventDecoder.java create mode 100644 sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionState.java create mode 100644 sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketTransportOptions.java diff --git a/sdk/ai/azure-ai-agents/CHANGELOG.md b/sdk/ai/azure-ai-agents/CHANGELOG.md index 076eebfcdd337..38021136fb231 100644 --- a/sdk/ai/azure-ai-agents/CHANGELOG.md +++ b/sdk/ai/azure-ai-agents/CHANGELOG.md @@ -23,7 +23,7 @@ `buildBetaVoiceAgentsLiveWebSocketAsyncClient()`. Native Live events are sent and received as lossless JSON using `BinaryData`, separately from the Realtime protocol clients. - Added typed GPT Live sends through `sendEvent(com.openai.models.live.ClientEvent)` and typed receives through - `receiveTypedEvents()`, retaining the raw JSON APIs. The synchronous client also supports per-event receive timeouts. + `receiveEvents()`, retaining the raw JSON APIs. The synchronous client also supports per-event receive timeouts. Live transports use native OpenAI `ServerEvent` models while retaining original JSON for raw receivers. Live and Realtime clients select their own routes and headers without a shared protocol-mode flag. - Added synchronous/asynchronous persisted GPT Live conversation history and audio-download examples with offline tests. @@ -60,6 +60,9 @@ ### Bugs Fixed +- Fixed OpenAI model JSON conversion to use an explicit Jackson serializer, preserving explicit nulls without + depending on the application's Azure JSON serializer provider. +- Removed query values from WebSocket proxy-failure diagnostics to avoid logging structured inputs. - Updated the synchronous live text sample to replace an existing sample agent on a creation conflict and delete the replacement on exit, including when the conversation fails. - Fixed voice-agent definition deserialization through `AgentDefinition` and `VoiceAgentBaseDefinition` to select diff --git a/sdk/ai/azure-ai-agents/README.md b/sdk/ai/azure-ai-agents/README.md index 2277fc2e66f80..b61a8c1b0049a 100644 --- a/sdk/ai/azure-ai-agents/README.md +++ b/sdk/ai/azure-ai-agents/README.md @@ -1129,20 +1129,26 @@ variants, but typed serialization need not preserve the original JSON formatting The existing samples named `VoiceAgentLive*ConversationSample` above demonstrate live connections using the **Realtime** protocol; use the separate GPT Live clients for `VoiceAgentLiveDefinition` agents. -For example, with a connected synchronous Live session configured for Responses delegation: - -```java -import com.openai.models.live.ClientEvent; -import com.openai.models.live.ResponseCreateEvent; -import com.openai.models.live.ServerEvent; - -for (ServerEvent event : session.receiveEvents()) { - if (event.isSessionStarted()) { - session.sendEvent(ClientEvent.ofResponseCreate(ResponseCreateEvent.builder().build())); - } else if (event.isSessionOutputTranscriptDelta()) { - System.out.print(event.asSessionOutputTranscriptDelta().delta()); - } else if (event.isSessionClosed() || event.isErrorEvent()) { - break; +For example, configure and open a synchronous Live session for an agent using Responses delegation: + +```java readme-sample-com.azure.ai.agents.live_connection_options +BetaVoiceAgentsLiveWebSocketClient liveClient = builder.beta().buildBetaVoiceAgentsLiveWebSocketClient(); +VoiceAgentLiveWebSocketConnectionOptions options = new VoiceAgentLiveWebSocketConnectionOptions() + .setReceiveBufferCapacity(512) + .setMaxMessageSize(8 * 1024 * 1024) + .setOverflowStrategy(VoiceAgentWebSocketOverflowStrategy.ERROR) + .setHandshakeTimeout(Duration.ofSeconds(30)) + .setCloseTimeout(Duration.ofSeconds(10)) + .setStoreEnabled(false); +try (BetaVoiceAgentsLiveWebSocketSessionClient session = liveClient.openWebSocketSession(agentName, options)) { + for (ServerEvent event : session.receiveEvents()) { + if (event.isSessionStarted()) { + session.sendEvent(ClientEvent.ofResponseCreate(ResponseCreateEvent.builder().build())); + } else if (event.isSessionOutputTranscriptDelta()) { + System.out.print(event.asSessionOutputTranscriptDelta().delta()); + } else if (event.isSessionClosed() || event.isErrorEvent()) { + break; + } } } ``` diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java index 1668fe1ee92e8..b461e71d3e5d3 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java @@ -6,10 +6,10 @@ import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; import com.azure.ai.agents.implementation.realtime.WebSocketSessionAsyncTransport; +import com.azure.ai.agents.implementation.realtime.WebSocketTransportOptions; import com.azure.ai.agents.implementation.OpenAIJsonHelper; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; -import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.annotation.ServiceClient; import com.openai.models.live.ServerEvent; import java.util.Objects; @@ -48,13 +48,13 @@ public Mono openWebSocketSession public Mono openWebSocketSession(String agentName, VoiceAgentLiveWebSocketConnectionOptions options) { Objects.requireNonNull(agentName, "'agentName' cannot be null."); - VoiceAgentWebSocketConnectionOptions snapshot = VoiceAgentWebSocketUtils.liveOptions(options); + VoiceAgentLiveWebSocketConnectionOptions snapshot = new VoiceAgentLiveWebSocketConnectionOptions(options); return Mono.defer(() -> { WebSocketSessionAsyncTransport transport = new WebSocketSessionAsyncTransport<>(configuration, - VoiceAgentWebSocketUtils.buildLiveWebSocketUri(configuration, agentName, snapshot), snapshot, - HttpClient.create(), null, token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, token), - json -> OpenAIJsonHelper.fromBinaryData(VoiceAgentWebSocketUtils.deserializeRawEvent(json), - ServerEvent.class)); + VoiceAgentWebSocketUtils.buildLiveWebSocketUri(configuration, agentName, snapshot), + new WebSocketTransportOptions(snapshot), HttpClient.create(), null, + token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, token), json -> OpenAIJsonHelper + .fromBinaryData(VoiceAgentWebSocketUtils.deserializeRawEvent(json), ServerEvent.class)); return transport.connect().thenReturn(new BetaVoiceAgentsLiveWebSocketSessionAsyncClient(transport)); }); } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java index 833360098cd67..92de691b32b8e 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java @@ -3,16 +3,17 @@ package com.azure.ai.agents; +import com.azure.ai.agents.implementation.OpenAIJsonHelper; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport; -import com.azure.ai.agents.implementation.OpenAIJsonHelper; -import com.openai.models.live.ServerEvent; +import com.azure.ai.agents.implementation.realtime.WebSocketTransportOptions; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; -import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.annotation.ServiceClient; import com.azure.core.util.logging.ClientLogger; +import com.openai.models.live.ServerEvent; + import java.util.Objects; /** @@ -52,9 +53,10 @@ public BetaVoiceAgentsLiveWebSocketSessionClient openWebSocketSession(String age if (agentName.isEmpty()) { throw LOGGER.logExceptionAsError(new IllegalArgumentException("'agentName' cannot be empty.")); } - VoiceAgentWebSocketConnectionOptions snapshot = VoiceAgentWebSocketUtils.liveOptions(options); + VoiceAgentLiveWebSocketConnectionOptions snapshot = new VoiceAgentLiveWebSocketConnectionOptions(options); return new BetaVoiceAgentsLiveWebSocketSessionClient(WebSocketSessionTransport.connect(configuration, - VoiceAgentWebSocketUtils.buildLiveWebSocketUri(configuration, agentName, snapshot), snapshot, null, + VoiceAgentWebSocketUtils.buildLiveWebSocketUri(configuration, agentName, snapshot), + new WebSocketTransportOptions(snapshot), null, token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, token), json -> OpenAIJsonHelper .fromBinaryData(VoiceAgentWebSocketUtils.deserializeRawEvent(json), ServerEvent.class))); } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java index e70a68b5b7b6a..034908e0a5cc5 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java @@ -92,6 +92,8 @@ public IterableStream receiveEvents(Duration timeout) { /** * Sends a typed native Live command. Foundry supplies {@code session.start}; clients must not send it. + * This method enqueues the event; returning does not mean the frame has been written to the network. + * Later network failures surface through the receive stream and session lifecycle. * @param event the command to send. * @throws IllegalStateException if the session cannot accept the event. */ @@ -114,6 +116,8 @@ public IterableStream receiveRawEvents(Duration timeout) { /** * Sends a native Live JSON event. Foundry supplies {@code session.start}; clients must not send it. * The service validates native command semantics, including delegation restrictions. + * This method enqueues the event; returning does not mean the frame has been written to the network. + * Later network failures surface through the receive stream and session lifecycle. * @param event the complete JSON object. * @throws IllegalArgumentException if the event is not a JSON object. * @throws IllegalStateException if the session cannot accept the event. diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java index a2accd93c6689..e9c95a97993e9 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java @@ -6,6 +6,7 @@ import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; import com.azure.ai.agents.implementation.realtime.WebSocketSessionAsyncTransport; +import com.azure.ai.agents.implementation.realtime.WebSocketTransportOptions; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.RealtimeClientEvent; import com.azure.ai.agents.models.RealtimeConversationItemCreateEvent; @@ -51,8 +52,8 @@ public final class BetaVoiceAgentsWebSocketSessionAsyncClient implements AsyncCl VoiceAgentWebSocketConnectionOptions sessionOptions = options == null ? new VoiceAgentWebSocketConnectionOptions() : options; transport = new WebSocketSessionAsyncTransport<>(configuration, - VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, sessionOptions), sessionOptions, - httpClient, VoiceAgentWebSocketUtils.SUBPROTOCOL, + VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, sessionOptions), + new WebSocketTransportOptions(sessionOptions), httpClient, VoiceAgentWebSocketUtils.SUBPROTOCOL, token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, sessionOptions, token), VoiceAgentWebSocketUtils::deserializeEvent); } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java index ec844ef8483b4..7a5877af7d1ac 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java @@ -6,6 +6,7 @@ import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport; +import com.azure.ai.agents.implementation.realtime.WebSocketTransportOptions; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.RealtimeClientEvent; import com.azure.ai.agents.models.RealtimeConversationItemCreateEvent; @@ -45,8 +46,8 @@ private BetaVoiceAgentsWebSocketSessionClient(WebSocketSessionTransport VoiceAgentWebSocketUtils.buildHeaders(configuration, options, token), VoiceAgentWebSocketUtils::deserializeEvent)); } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/OpenAIJsonHelper.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/OpenAIJsonHelper.java index e52cc3464af44..ed80f517b9b6a 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/OpenAIJsonHelper.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/OpenAIJsonHelper.java @@ -4,6 +4,8 @@ package com.azure.ai.agents.implementation; import com.azure.core.util.BinaryData; +import com.azure.core.util.logging.ClientLogger; +import com.azure.core.util.serializer.JsonSerializer; import com.azure.json.JsonProviders; import com.azure.json.JsonReader; import com.azure.json.JsonSerializable; @@ -16,16 +18,53 @@ import com.openai.models.responses.ResponseCreateParams; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.io.StringReader; +import java.io.UncheckedIOException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import reactor.core.publisher.Mono; public final class OpenAIJsonHelper { + private static final ClientLogger LOGGER = new ClientLogger(OpenAIJsonHelper.class); private static final ObjectMapper OPENAI_MODEL_MAPPER = ObjectMappers.jsonMapper(); + private static final JsonSerializer OPENAI_MODEL_SERIALIZER = new JsonSerializer() { + @Override + public T deserialize(InputStream stream, com.azure.core.util.serializer.TypeReference typeReference) { + try { + return OPENAI_MODEL_MAPPER.readValue(stream, + OPENAI_MODEL_MAPPER.constructType(typeReference.getJavaType())); + } catch (IOException error) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(error)); + } + } + + @Override + public Mono deserializeAsync(InputStream stream, + com.azure.core.util.serializer.TypeReference typeReference) { + return Mono.fromCallable(() -> deserialize(stream, typeReference)); + } + + @Override + public void serialize(OutputStream stream, Object value) { + try { + stream.write(OPENAI_MODEL_MAPPER.writeValueAsBytes(value)); + } catch (IOException error) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(error)); + } + } + + @Override + public Mono serializeAsync(OutputStream stream, Object value) { + return Mono.fromRunnable(() -> serialize(stream, value)); + } + }; + private static final ObjectMapper MAPPER = ObjectMappers.jsonMapper() .rebuild() .configure(MapperFeature.AUTO_DETECT_FIELDS, true) @@ -114,7 +153,7 @@ public static BinaryData toBinaryData(Object openAIObject) { } try { String json = OPENAI_MODEL_MAPPER.writeValueAsString(openAIObject); - return BinaryData.fromObject(OPENAI_MODEL_MAPPER.readTree(json)); + return BinaryData.fromObject(OPENAI_MODEL_MAPPER.readTree(json), OPENAI_MODEL_SERIALIZER); } catch (IOException e) { throw new RuntimeException("Failed to convert OpenAI type to BinaryData", e); } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java index 34d1e80261448..12724e1810591 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java @@ -72,19 +72,6 @@ public static RealtimeServerEvent deserializeEvent(String json) throws IOExcepti private VoiceAgentWebSocketUtils() { } - public static VoiceAgentWebSocketConnectionOptions liveOptions(VoiceAgentLiveWebSocketConnectionOptions options) { - Objects.requireNonNull(options, "'options' cannot be null."); - return new VoiceAgentWebSocketConnectionOptions().setReceiveBufferCapacity(options.getReceiveBufferCapacity()) - .setMaxMessageSize(options.getMaxMessageSize()) - .setOverflowStrategy(options.getOverflowStrategy()) - .setMalformedEventHandler(options.getMalformedEventHandler()) - .setHandshakeTimeout(options.getHandshakeTimeout()) - .setCloseTimeout(options.getCloseTimeout()) - .setStoreEnabled(options.isStoreEnabled()) - .setAgentVersionOverride(options.getAgentVersionOverride()) - .setStructuredInputs(options.getStructuredInputs()); - } - public static BinaryData deserializeRawEvent(String json) { BinaryData event = BinaryData.fromString(json); validateEvent(event); @@ -118,7 +105,8 @@ private static boolean isProtectedHeader(String name) { public static URI buildWebSocketUri(VoiceAgentWebSocketClientConfiguration configuration, String agentName, VoiceAgentWebSocketConnectionOptions options) { - UrlBuilder url = buildSessionUrl(configuration, agentName, "", options); + UrlBuilder url = buildSessionUrl(configuration, agentName, "", options.isStoreEnabled(), + options.getAgentVersionOverride()); VoiceAgentTransport transport = options.getTransport(); url.setQueryParameter("transport", transport == null ? "websocket" : encode(transport.toString())); if (options.getAgentSessionId() != null) { @@ -128,8 +116,9 @@ public static URI buildWebSocketUri(VoiceAgentWebSocketClientConfiguration confi } public static URI buildLiveWebSocketUri(VoiceAgentWebSocketClientConfiguration configuration, String agentName, - VoiceAgentWebSocketConnectionOptions options) { - UrlBuilder url = buildSessionUrl(configuration, agentName, "/live/sessions", options); + VoiceAgentLiveWebSocketConnectionOptions options) { + UrlBuilder url = buildSessionUrl(configuration, agentName, "/live/sessions", options.isStoreEnabled(), + options.getAgentVersionOverride()); if (options.getStructuredInputs() != null) { url.setQueryParameter("structured_inputs", encode(options.getStructuredInputs())); } @@ -137,11 +126,14 @@ public static URI buildLiveWebSocketUri(VoiceAgentWebSocketClientConfiguration c } private static UrlBuilder buildSessionUrl(VoiceAgentWebSocketClientConfiguration configuration, String agentName, - String suffix, VoiceAgentWebSocketConnectionOptions options) { + String suffix, Boolean store, String agentVersionOverride) { Objects.requireNonNull(agentName, "'agentName' cannot be null."); if (agentName.isEmpty()) { throw new IllegalArgumentException("'agentName' cannot be empty."); } + if (!suffix.isEmpty() && !suffix.startsWith("/")) { + throw new IllegalArgumentException("A voice-agent route suffix must start with '/'."); + } URI endpoint = configuration.getEndpoint(); String scheme; if ("https".equalsIgnoreCase(endpoint.getScheme()) || "wss".equalsIgnoreCase(endpoint.getScheme())) { @@ -161,11 +153,11 @@ private static UrlBuilder buildSessionUrl(VoiceAgentWebSocketClientConfiguration UrlBuilder url = UrlBuilder.parse(baseUri.toString()); url.setQueryParameter("api-version", encode(configuration.getApiVersion())); url.setQueryParameter("x-ms-client-sdk", encode(configuration.getUserAgent())); - if (options.isStoreEnabled() != null) { - url.setQueryParameter("store", options.isStoreEnabled().toString()); + if (store != null) { + url.setQueryParameter("store", store.toString()); } - if (options.getAgentVersionOverride() != null) { - url.setQueryParameter("x-agent-version-override", encode(options.getAgentVersionOverride())); + if (agentVersionOverride != null) { + url.setQueryParameter("x-agent-version-override", encode(agentVersionOverride)); } return url; } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketEventDecoder.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketEventDecoder.java new file mode 100644 index 0000000000000..45d834f742173 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketEventDecoder.java @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.implementation.realtime; + +import java.io.IOException; + +/** + * Decodes a protocol event received by either WebSocket transport. + * @param the decoded event type. + */ +@FunctionalInterface +public interface WebSocketEventDecoder { + T decode(String json) throws IOException; +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java index 6d13ca944d239..f21f9d4d7e777 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java @@ -3,7 +3,6 @@ package com.azure.ai.agents.implementation.realtime; -import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport.ReceivedEvent; import com.azure.core.credential.AccessToken; import com.azure.core.exception.ClientAuthenticationException; @@ -51,13 +50,13 @@ public final class WebSocketSessionAsyncTransport implements AutoCloseable { private static final ClientLogger LOGGER = new ClientLogger(WebSocketSessionAsyncTransport.class); private static final int MAX_OUTSTANDING_SENDS = 256; private final VoiceAgentWebSocketClientConfiguration configuration; - private final VoiceAgentWebSocketConnectionOptions options; + private final WebSocketTransportOptions options; private final HttpClient httpClient; private final URI websocketUri; private final String subprotocol; private final Function headersFactory; - private final WebSocketSessionTransport.EventDecoder decoder; - private final AtomicReference state = new AtomicReference<>(State.NEW); + private final WebSocketEventDecoder decoder; + private final AtomicReference state = new AtomicReference<>(WebSocketSessionState.NEW); private final AtomicReference channel = new AtomicReference<>(); private final AtomicReference outbound = new AtomicReference<>(); private final AtomicReference connectionOperation = new AtomicReference<>(); @@ -73,10 +72,10 @@ public final class WebSocketSessionAsyncTransport implements AutoCloseable { private volatile String closeReason; public WebSocketSessionAsyncTransport(VoiceAgentWebSocketClientConfiguration configuration, URI websocketUri, - VoiceAgentWebSocketConnectionOptions options, HttpClient httpClient, String subprotocol, - Function headersFactory, WebSocketSessionTransport.EventDecoder decoder) { + WebSocketTransportOptions options, HttpClient httpClient, String subprotocol, + Function headersFactory, WebSocketEventDecoder decoder) { this.configuration = Objects.requireNonNull(configuration, "'configuration' cannot be null."); - this.options = options == null ? new VoiceAgentWebSocketConnectionOptions() : options; + this.options = Objects.requireNonNull(options, "'options' cannot be null."); this.eventQueue = new ArrayBlockingQueue<>(this.options.getReceiveBufferCapacity()); this.events = Sinks.many().unicast().onBackpressureBuffer(eventQueue); this.httpClient = Objects.requireNonNull(httpClient, "'httpClient' cannot be null."); @@ -88,7 +87,7 @@ public WebSocketSessionAsyncTransport(VoiceAgentWebSocketClientConfiguration con public Mono connect() { return Mono.defer(() -> { - if (!state.compareAndSet(State.NEW, State.CONNECTING)) { + if (!state.compareAndSet(WebSocketSessionState.NEW, WebSocketSessionState.CONNECTING)) { return Mono.error(new IllegalStateException("The voice-agent session has already been started.")); } return configuration.getCredential() @@ -111,7 +110,7 @@ public String getEndpoint() { public boolean isOpen() { Channel current = channel.get(); - return state.get() == State.OPEN && current != null && current.isActive(); + return state.get() == WebSocketSessionState.OPEN && current != null && current.isActive(); } public Integer getCloseCode() { @@ -170,13 +169,13 @@ public Mono closeAsync(int code, String reason) { return existing; } Mono created = Mono.defer(() -> { - State current = state.get(); - if (current == State.CLOSED || current == State.NEW) { - state.set(State.CLOSED); + WebSocketSessionState current = state.get(); + if (current == WebSocketSessionState.CLOSED || current == WebSocketSessionState.NEW) { + state.set(WebSocketSessionState.CLOSED); events.tryEmitComplete(); return Mono.empty(); } - state.set(State.CLOSING); + state.set(WebSocketSessionState.CLOSING); WebsocketOutbound currentOutbound = outbound.get(); Channel currentChannel = channel.get(); Mono graceful = currentOutbound == null ? Mono.empty() : currentOutbound.sendClose(code, reason); @@ -229,7 +228,7 @@ private static int toConnectTimeoutMillis(Duration timeout) { private Mono handleConnection(WebsocketInbound inbound, WebsocketOutbound outbound) { this.outbound.set(outbound); inbound.withConnection(connection -> channel.set(connection.channel())); - state.set(State.OPEN); + state.set(WebSocketSessionState.OPEN); ready.tryEmitEmpty(); inbound.receiveCloseStatus().subscribe(status -> { closeCode = status.code(); @@ -294,8 +293,8 @@ private void closeWithProtocolError(int code, String reason, Throwable error) { private void terminateWithError(Throwable error) { Throwable mappedError = mapHandshakeError(error); - State previous = state.getAndSet(State.CLOSED); - if (previous == State.CLOSED) { + WebSocketSessionState previous = state.getAndSet(WebSocketSessionState.CLOSED); + if (previous == WebSocketSessionState.CLOSED) { return; } ready.tryEmitError(mappedError); @@ -335,11 +334,11 @@ private Throwable mapHandshakeError(Throwable error) { } private void terminateNormally() { - State previous = state.getAndSet(State.CLOSED); - if (previous == State.CLOSED) { + WebSocketSessionState previous = state.getAndSet(WebSocketSessionState.CLOSED); + if (previous == WebSocketSessionState.CLOSED) { return; } - if (previous == State.CONNECTING) { + if (previous == WebSocketSessionState.CONNECTING) { ready.tryEmitError(new IllegalStateException("The WebSocket closed before the handshake completed.")); } events.tryEmitComplete(); @@ -360,7 +359,7 @@ private void disposeReceive() { private Channel requireOpenChannel() { Channel current = channel.get(); - if (state.get() != State.OPEN || current == null || !current.isActive()) { + if (state.get() != WebSocketSessionState.OPEN || current == null || !current.isActive()) { throw LOGGER .logExceptionAsError(new IllegalStateException("The voice-agent WebSocket session is not open.")); } @@ -397,7 +396,4 @@ private HttpClient configureProxy(HttpClient client) { }); } - private enum State { - NEW, CONNECTING, OPEN, CLOSING, CLOSED - } } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionState.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionState.java new file mode 100644 index 0000000000000..9a88118abc8cf --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionState.java @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.implementation.realtime; + +/** + * Connection lifecycle shared by the blocking and asynchronous WebSocket transports. + */ +public enum WebSocketSessionState { + NEW, CONNECTING, OPEN, CLOSING, CLOSED +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java index 60a6f3b4c2f3e..c6f61713421e8 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java @@ -3,7 +3,6 @@ package com.azure.ai.agents.implementation.realtime; -import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.exception.ClientAuthenticationException; import com.azure.core.exception.HttpResponseException; import com.azure.core.exception.ResourceModifiedException; @@ -51,30 +50,24 @@ public final class WebSocketSessionTransport implements AutoCloseable { private static final ClientLogger LOGGER = new ClientLogger(WebSocketSessionTransport.class); private final URI websocketUri; - private final VoiceAgentWebSocketConnectionOptions options; - private final EventDecoder decoder; + private final WebSocketTransportOptions options; + private final WebSocketEventDecoder decoder; private final OkHttpClient httpClient; private final BlockingQueue> events; private final int receiveBufferCapacity; private final CountDownLatch handshakeCompleted = new CountDownLatch(1); private final CountDownLatch closeCompleted = new CountDownLatch(1); private final AtomicReference connectionError = new AtomicReference<>(); - private final AtomicBoolean handshakeSucceeded = new AtomicBoolean(); private final AtomicBoolean receiveClaimed = new AtomicBoolean(); - private final AtomicBoolean open = new AtomicBoolean(); - private final AtomicBoolean closed = new AtomicBoolean(); + private final AtomicReference state + = new AtomicReference<>(WebSocketSessionState.CONNECTING); private final AtomicBoolean clientShutdown = new AtomicBoolean(); private volatile WebSocket webSocket; private volatile Integer closeCode; private volatile String closeReason; - @FunctionalInterface - public interface EventDecoder { - T decode(String json) throws IOException; - } - private WebSocketSessionTransport(VoiceAgentWebSocketClientConfiguration configuration, URI websocketUri, - VoiceAgentWebSocketConnectionOptions options, EventDecoder decoder) { + WebSocketTransportOptions options, WebSocketEventDecoder decoder) { this.options = options; this.decoder = Objects.requireNonNull(decoder, "'decoder' cannot be null."); this.receiveBufferCapacity = options.getReceiveBufferCapacity(); @@ -99,8 +92,8 @@ private void open(VoiceAgentWebSocketClientConfiguration configuration, String s } public static WebSocketSessionTransport connect(VoiceAgentWebSocketClientConfiguration configuration, - URI websocketUri, VoiceAgentWebSocketConnectionOptions options, String subprotocol, - Function headersFactory, EventDecoder decoder) { + URI websocketUri, WebSocketTransportOptions options, String subprotocol, + Function headersFactory, WebSocketEventDecoder decoder) { WebSocketSessionTransport session = new WebSocketSessionTransport<>(configuration, websocketUri, options, decoder); try { @@ -121,7 +114,7 @@ public String getEndpoint() { } public boolean isOpen() { - return open.get() && !closed.get(); + return state.get() == WebSocketSessionState.OPEN; } public Integer getCloseCode() { @@ -177,11 +170,10 @@ public void close() { public void close(int code, String reason) { VoiceAgentWebSocketUtils.validateClose(code, reason); - if (!closed.compareAndSet(false, true)) { + if (!state.compareAndSet(WebSocketSessionState.OPEN, WebSocketSessionState.CLOSING)) { shutdownHttpClient(); return; } - open.set(false); if (!webSocket.close(code, reason)) { webSocket.cancel(); closeCompleted.countDown(); @@ -194,6 +186,7 @@ public void close(int code, String reason) { Thread.currentThread().interrupt(); webSocket.cancel(); } finally { + state.set(WebSocketSessionState.CLOSED); signal(EventSignal.complete()); shutdownHttpClient(); } @@ -212,9 +205,6 @@ private void awaitHandshake() { throw LOGGER.logExceptionAsError( new IllegalStateException("Interrupted while opening the voice-agent WebSocket session.", error)); } - if (handshakeSucceeded.get()) { - return; - } Throwable error = connectionError.get(); if (error instanceof RuntimeException) { throw LOGGER.logExceptionAsError((RuntimeException) error); @@ -234,9 +224,10 @@ private void fail(Throwable error, Response response) { response.close(); } } - connectionError.compareAndSet(null, mapped); - open.set(false); - closed.set(true); + WebSocketSessionState previous = state.getAndSet(WebSocketSessionState.CLOSED); + if (previous == WebSocketSessionState.CONNECTING) { + connectionError.compareAndSet(null, mapped); + } handshakeCompleted.countDown(); signal(EventSignal.error(mapped)); closeCompleted.countDown(); @@ -282,8 +273,7 @@ private synchronized void signal(EventSignal signal) { if (current != null) { current.cancel(); } - open.set(false); - closed.set(true); + state.set(WebSocketSessionState.CLOSED); shutdownHttpClient(); } } @@ -296,7 +286,7 @@ private void shutdownHttpClient() { } public static OkHttpClient createHttpClient(VoiceAgentWebSocketClientConfiguration configuration, - VoiceAgentWebSocketConnectionOptions options) { + WebSocketTransportOptions options) { OkHttpClient.Builder builder = new OkHttpClient.Builder(); builder.connectTimeout(options.getHandshakeTimeout().toMillis(), TimeUnit.MILLISECONDS) .readTimeout(0, TimeUnit.MILLISECONDS) @@ -322,7 +312,8 @@ public List select(URI uri) { @Override public void connectFailed(URI uri, SocketAddress address, IOException error) { LOGGER.atVerbose() - .addKeyValue("uri", uri) + .addKeyValue("host", uri.getHost()) + .addKeyValue("path", uri.getPath()) .addKeyValue("proxyAddress", address) .log("Failed to connect through the configured proxy."); } @@ -342,8 +333,7 @@ public void connectFailed(URI uri, SocketAddress address, IOException error) { private final class Listener extends WebSocketListener { @Override public void onOpen(WebSocket webSocket, Response response) { - handshakeSucceeded.set(true); - open.set(true); + state.compareAndSet(WebSocketSessionState.CONNECTING, WebSocketSessionState.OPEN); handshakeCompleted.countDown(); } @@ -399,8 +389,11 @@ public void onClosing(WebSocket webSocket, int code, String reason) { public void onClosed(WebSocket webSocket, int code, String reason) { closeCode = code; closeReason = reason; - open.set(false); - closed.set(true); + WebSocketSessionState previous = state.getAndSet(WebSocketSessionState.CLOSED); + if (previous == WebSocketSessionState.CONNECTING) { + connectionError.compareAndSet(null, + new IllegalStateException("The WebSocket closed before the handshake completed.")); + } signal(EventSignal.complete()); handshakeCompleted.countDown(); closeCompleted.countDown(); diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketTransportOptions.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketTransportOptions.java new file mode 100644 index 0000000000000..71599f32f3b4a --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketTransportOptions.java @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.implementation.realtime; + +import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; +import com.azure.ai.agents.models.VoiceAgentWebSocketOverflowStrategy; +import java.time.Duration; +import java.util.Objects; +import java.util.function.Consumer; + +/** + * Immutable transport settings captured from a protocol-specific public options bag. + */ +public final class WebSocketTransportOptions { + private final int receiveBufferCapacity; + private final int maxMessageSize; + private final VoiceAgentWebSocketOverflowStrategy overflowStrategy; + private final Consumer malformedEventHandler; + private final Duration handshakeTimeout; + private final Duration closeTimeout; + + public WebSocketTransportOptions(VoiceAgentWebSocketConnectionOptions options) { + Objects.requireNonNull(options, "'options' cannot be null."); + this.receiveBufferCapacity = options.getReceiveBufferCapacity(); + this.maxMessageSize = options.getMaxMessageSize(); + this.overflowStrategy = options.getOverflowStrategy(); + this.malformedEventHandler = options.getMalformedEventHandler(); + this.handshakeTimeout = options.getHandshakeTimeout(); + this.closeTimeout = options.getCloseTimeout(); + } + + public WebSocketTransportOptions(VoiceAgentLiveWebSocketConnectionOptions options) { + Objects.requireNonNull(options, "'options' cannot be null."); + this.receiveBufferCapacity = options.getReceiveBufferCapacity(); + this.maxMessageSize = options.getMaxMessageSize(); + this.overflowStrategy = options.getOverflowStrategy(); + this.malformedEventHandler = options.getMalformedEventHandler(); + this.handshakeTimeout = options.getHandshakeTimeout(); + this.closeTimeout = options.getCloseTimeout(); + } + + public int getReceiveBufferCapacity() { + return receiveBufferCapacity; + } + + public int getMaxMessageSize() { + return maxMessageSize; + } + + public VoiceAgentWebSocketOverflowStrategy getOverflowStrategy() { + return overflowStrategy; + } + + public Consumer getMalformedEventHandler() { + return malformedEventHandler; + } + + public Duration getHandshakeTimeout() { + return handshakeTimeout; + } + + public Duration getCloseTimeout() { + return closeTimeout; + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java index 79fd1507df458..f6a0a6174e4ed 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java @@ -5,6 +5,7 @@ import com.azure.ai.agents.implementation.utils.Beta; import com.azure.core.annotation.Fluent; +import com.azure.core.util.logging.ClientLogger; import java.time.Duration; import java.util.Objects; import java.util.function.Consumer; @@ -16,11 +17,19 @@ @Beta(warningText = "Preview API. VoiceAgents=V1Preview") @Fluent public final class VoiceAgentLiveWebSocketConnectionOptions { - private final VoiceAgentWebSocketConnectionOptions options; + private static final ClientLogger LOGGER = new ClientLogger(VoiceAgentLiveWebSocketConnectionOptions.class); + private int receiveBufferCapacity = 256; + private int maxMessageSize = 32 * 1024 * 1024; + private VoiceAgentWebSocketOverflowStrategy overflowStrategy = VoiceAgentWebSocketOverflowStrategy.ERROR; + private Consumer malformedEventHandler; + private Duration handshakeTimeout = Duration.ofSeconds(30); + private Duration closeTimeout = Duration.ofSeconds(10); + private Boolean store; + private String agentVersionOverride; + private String structuredInputs; /** Creates options with a 30-second handshake timeout and a 256-event receive buffer. */ public VoiceAgentLiveWebSocketConnectionOptions() { - options = new VoiceAgentWebSocketConnectionOptions(); } /** @@ -28,8 +37,16 @@ public VoiceAgentLiveWebSocketConnectionOptions() { * @param source the options to copy. */ public VoiceAgentLiveWebSocketConnectionOptions(VoiceAgentLiveWebSocketConnectionOptions source) { - options = new VoiceAgentWebSocketConnectionOptions( - Objects.requireNonNull(source, "'source' cannot be null.").options); + Objects.requireNonNull(source, "'source' cannot be null."); + this.receiveBufferCapacity = source.receiveBufferCapacity; + this.maxMessageSize = source.maxMessageSize; + this.overflowStrategy = source.overflowStrategy; + this.malformedEventHandler = source.malformedEventHandler; + this.handshakeTimeout = source.handshakeTimeout; + this.closeTimeout = source.closeTimeout; + this.store = source.store; + this.agentVersionOverride = source.agentVersionOverride; + this.structuredInputs = source.structuredInputs; } /** @@ -37,7 +54,7 @@ public VoiceAgentLiveWebSocketConnectionOptions(VoiceAgentLiveWebSocketConnectio * @return the maximum queued events, default 256. */ public int getReceiveBufferCapacity() { - return options.getReceiveBufferCapacity(); + return receiveBufferCapacity; } /** @@ -47,7 +64,11 @@ public int getReceiveBufferCapacity() { * @throws IllegalArgumentException if the capacity is outside the supported range. */ public VoiceAgentLiveWebSocketConnectionOptions setReceiveBufferCapacity(int capacity) { - options.setReceiveBufferCapacity(capacity); + if (capacity < 1 || capacity > 65536) { + throw LOGGER + .logExceptionAsError(new IllegalArgumentException("Receive capacity must be between 1 and 65536.")); + } + this.receiveBufferCapacity = capacity; return this; } @@ -56,7 +77,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setReceiveBufferCapacity(int cap * @return the size in bytes, default 32 MiB. */ public int getMaxMessageSize() { - return options.getMaxMessageSize(); + return maxMessageSize; } /** @@ -66,7 +87,10 @@ public int getMaxMessageSize() { * @throws IllegalArgumentException if the size is not positive. */ public VoiceAgentLiveWebSocketConnectionOptions setMaxMessageSize(int bytes) { - options.setMaxMessageSize(bytes); + if (bytes <= 0) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException("Message size must be positive.")); + } + this.maxMessageSize = bytes; return this; } @@ -75,7 +99,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setMaxMessageSize(int bytes) { * @return the strategy, default ERROR. */ public VoiceAgentWebSocketOverflowStrategy getOverflowStrategy() { - return options.getOverflowStrategy(); + return overflowStrategy; } /** @@ -84,7 +108,7 @@ public VoiceAgentWebSocketOverflowStrategy getOverflowStrategy() { * @return this options instance. */ public VoiceAgentLiveWebSocketConnectionOptions setOverflowStrategy(VoiceAgentWebSocketOverflowStrategy strategy) { - options.setOverflowStrategy(strategy); + this.overflowStrategy = Objects.requireNonNull(strategy, "'strategy' cannot be null."); return this; } @@ -93,7 +117,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setOverflowStrategy(VoiceAgentWe * @return the callback, or null to terminate the session. */ public Consumer getMalformedEventHandler() { - return options.getMalformedEventHandler(); + return malformedEventHandler; } /** @@ -103,7 +127,7 @@ public Consumer getMalformedEventHandler() { * @return this options instance. */ public VoiceAgentLiveWebSocketConnectionOptions setMalformedEventHandler(Consumer handler) { - options.setMalformedEventHandler(handler); + this.malformedEventHandler = handler; return this; } @@ -112,7 +136,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setMalformedEventHandler(Consume * @return the timeout, default 30 seconds. */ public Duration getHandshakeTimeout() { - return options.getHandshakeTimeout(); + return handshakeTimeout; } /** @@ -122,7 +146,8 @@ public Duration getHandshakeTimeout() { * @throws IllegalArgumentException if the timeout is not positive. */ public VoiceAgentLiveWebSocketConnectionOptions setHandshakeTimeout(Duration timeout) { - options.setHandshakeTimeout(timeout); + validatePositive(timeout, "handshakeTimeout"); + this.handshakeTimeout = timeout; return this; } @@ -131,7 +156,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setHandshakeTimeout(Duration tim * @return the timeout, default 10 seconds. */ public Duration getCloseTimeout() { - return options.getCloseTimeout(); + return closeTimeout; } /** @@ -141,7 +166,8 @@ public Duration getCloseTimeout() { * @throws IllegalArgumentException if the timeout is not positive. */ public VoiceAgentLiveWebSocketConnectionOptions setCloseTimeout(Duration timeout) { - options.setCloseTimeout(timeout); + validatePositive(timeout, "closeTimeout"); + this.closeTimeout = timeout; return this; } @@ -150,7 +176,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setCloseTimeout(Duration timeout * @return the override, or null to use the persisted agent configuration. */ public Boolean isStoreEnabled() { - return options.isStoreEnabled(); + return store; } /** @@ -159,7 +185,7 @@ public Boolean isStoreEnabled() { * @return this options instance. */ public VoiceAgentLiveWebSocketConnectionOptions setStoreEnabled(Boolean store) { - options.setStoreEnabled(store); + this.store = store; return this; } @@ -168,7 +194,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setStoreEnabled(Boolean store) { * @return the version override, or null. */ public String getAgentVersionOverride() { - return options.getAgentVersionOverride(); + return agentVersionOverride; } /** @@ -177,7 +203,7 @@ public String getAgentVersionOverride() { * @return this options instance. */ public VoiceAgentLiveWebSocketConnectionOptions setAgentVersionOverride(String version) { - options.setAgentVersionOverride(version); + this.agentVersionOverride = version; return this; } @@ -186,7 +212,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setAgentVersionOverride(String v * @return the JSON inputs, or null. */ public String getStructuredInputs() { - return options.getStructuredInputs(); + return structuredInputs; } /** @@ -196,7 +222,13 @@ public String getStructuredInputs() { * @return this options instance. */ public VoiceAgentLiveWebSocketConnectionOptions setStructuredInputs(String inputs) { - options.setStructuredInputs(inputs); + this.structuredInputs = inputs; return this; } + + private static void validatePositive(Duration duration, String name) { + if (duration == null || duration.isZero() || duration.isNegative()) { + throw new IllegalArgumentException("'" + name + "' must be positive."); + } + } } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java index 3bd1dfd30c2ec..627af952fca51 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java @@ -10,6 +10,7 @@ import com.azure.ai.agents.models.RawRealtimeServerEvent; import com.azure.ai.agents.models.RealtimeServerEvent; import com.azure.ai.agents.models.SessionLogEvent; +import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.ai.agents.models.VoiceAgentWebSocketOverflowStrategy; import com.azure.core.util.BinaryData; @@ -25,8 +26,36 @@ import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; import com.openai.services.blocking.ConversationService; +import com.openai.models.live.ClientEvent; +import com.openai.models.live.ResponseCreateEvent; +import com.openai.models.live.ServerEvent; +import java.time.Duration; public final class ReadmeSamples { + public void liveConnectionOptions(AgentsClientBuilder builder, String agentName) { + // BEGIN: readme-sample-com.azure.ai.agents.live_connection_options + BetaVoiceAgentsLiveWebSocketClient liveClient = builder.beta().buildBetaVoiceAgentsLiveWebSocketClient(); + VoiceAgentLiveWebSocketConnectionOptions options = new VoiceAgentLiveWebSocketConnectionOptions() + .setReceiveBufferCapacity(512) + .setMaxMessageSize(8 * 1024 * 1024) + .setOverflowStrategy(VoiceAgentWebSocketOverflowStrategy.ERROR) + .setHandshakeTimeout(Duration.ofSeconds(30)) + .setCloseTimeout(Duration.ofSeconds(10)) + .setStoreEnabled(false); + try (BetaVoiceAgentsLiveWebSocketSessionClient session = liveClient.openWebSocketSession(agentName, options)) { + for (ServerEvent event : session.receiveEvents()) { + if (event.isSessionStarted()) { + session.sendEvent(ClientEvent.ofResponseCreate(ResponseCreateEvent.builder().build())); + } else if (event.isSessionOutputTranscriptDelta()) { + System.out.print(event.asSessionOutputTranscriptDelta().delta()); + } else if (event.isSessionClosed() || event.isErrorEvent()) { + break; + } + } + } + // END: readme-sample-com.azure.ai.agents.live_connection_options + } + public void realtimeForwardCompatibility(BetaVoiceAgentsWebSocketClient realtimeClient, String agentName) { // BEGIN: com.azure.ai.agents.realtime_forward_compatibility VoiceAgentWebSocketConnectionOptions options diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java index 0d96de039f81b..639dc636634a8 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java @@ -6,15 +6,20 @@ import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport; +import com.azure.ai.agents.implementation.realtime.WebSocketTransportOptions; +import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.ai.agents.models.VoiceAgentTransport; import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; +import com.azure.ai.agents.models.VoiceAgentWebSocketOverflowStrategy; import com.azure.core.http.ProxyOptions; import com.azure.core.http.HttpHeaders; import java.net.InetSocketAddress; import java.net.Proxy; import java.net.URI; +import java.time.Duration; import java.time.OffsetDateTime; import java.util.List; +import java.util.function.Consumer; import okhttp3.OkHttpClient; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; @@ -52,8 +57,8 @@ public void syncTransportHonorsNonProxyHosts() { URI.create("https://localhost"), request -> Mono.just(new com.azure.core.credential.AccessToken("token", OffsetDateTime.now().plusHours(1))), "v1", "test-user-agent", null, proxyOptions); - OkHttpClient client - = WebSocketSessionTransport.createHttpClient(configuration, new VoiceAgentWebSocketConnectionOptions()); + OkHttpClient client = WebSocketSessionTransport.createHttpClient(configuration, + new WebSocketTransportOptions(new VoiceAgentWebSocketConnectionOptions())); List bypassed = client.proxySelector().select(URI.create("https://LOCALHOST/session")); List proxied = client.proxySelector().select(URI.create("https://example.com/session")); @@ -70,7 +75,8 @@ public void syncConnectionPreservesTokenFailure() { IllegalStateException actual = assertThrows(IllegalStateException.class, () -> WebSocketSessionTransport.connect(configuration, URI.create("wss://localhost/session"), - new VoiceAgentWebSocketConnectionOptions(), null, token -> new HttpHeaders(), json -> json)); + new WebSocketTransportOptions(new VoiceAgentWebSocketConnectionOptions()), null, + token -> new HttpHeaders(), json -> json)); assertSame(failure, actual); } @@ -83,15 +89,97 @@ public void syncConnectionPreservesHeaderInitializationFailure() { request -> Mono.just(new com.azure.core.credential.AccessToken("token", OffsetDateTime.now().plusHours(1))), "v1", "test-user-agent", null, null); - IllegalStateException actual - = assertThrows(IllegalStateException.class, () -> WebSocketSessionTransport.connect(configuration, - URI.create("wss://localhost/session"), new VoiceAgentWebSocketConnectionOptions(), null, token -> { + IllegalStateException actual = assertThrows(IllegalStateException.class, + () -> WebSocketSessionTransport.connect(configuration, URI.create("wss://localhost/session"), + new WebSocketTransportOptions(new VoiceAgentWebSocketConnectionOptions()), null, token -> { throw failure; }, json -> json)); assertSame(failure, actual); } + @Test + public void liveOptionsAndTransportSnapshotAreIndependent() { + Consumer handler = error -> { + }; + VoiceAgentLiveWebSocketConnectionOptions options + = new VoiceAgentLiveWebSocketConnectionOptions().setReceiveBufferCapacity(17) + .setMaxMessageSize(4096) + .setOverflowStrategy(VoiceAgentWebSocketOverflowStrategy.DROP_OLDEST) + .setMalformedEventHandler(handler) + .setHandshakeTimeout(Duration.ofSeconds(2)) + .setCloseTimeout(Duration.ofSeconds(3)) + .setStoreEnabled(true) + .setAgentVersionOverride("version-1") + .setStructuredInputs("{\"name\":\"value\"}"); + VoiceAgentLiveWebSocketConnectionOptions copy = new VoiceAgentLiveWebSocketConnectionOptions(options); + WebSocketTransportOptions snapshot = new WebSocketTransportOptions(options); + options.setReceiveBufferCapacity(1) + .setMaxMessageSize(1) + .setOverflowStrategy(VoiceAgentWebSocketOverflowStrategy.ERROR) + .setMalformedEventHandler(null) + .setHandshakeTimeout(Duration.ofSeconds(10)) + .setCloseTimeout(Duration.ofSeconds(10)) + .setStoreEnabled(false) + .setAgentVersionOverride(null) + .setStructuredInputs(null); + + assertTransportSnapshot(snapshot, handler); + assertTransportSnapshot(new WebSocketTransportOptions(copy), handler); + assertEquals(true, copy.isStoreEnabled()); + assertEquals("version-1", copy.getAgentVersionOverride()); + assertEquals("{\"name\":\"value\"}", copy.getStructuredInputs()); + } + + @Test + public void realtimeTransportSnapshotIsIndependent() { + Consumer handler = error -> { + }; + VoiceAgentWebSocketConnectionOptions options + = new VoiceAgentWebSocketConnectionOptions().setReceiveBufferCapacity(17) + .setMaxMessageSize(4096) + .setOverflowStrategy(VoiceAgentWebSocketOverflowStrategy.DROP_OLDEST) + .setMalformedEventHandler(handler) + .setHandshakeTimeout(Duration.ofSeconds(2)) + .setCloseTimeout(Duration.ofSeconds(3)); + WebSocketTransportOptions snapshot = new WebSocketTransportOptions(options); + options.setReceiveBufferCapacity(1) + .setMaxMessageSize(1) + .setOverflowStrategy(VoiceAgentWebSocketOverflowStrategy.ERROR) + .setMalformedEventHandler(null) + .setHandshakeTimeout(Duration.ofSeconds(10)) + .setCloseTimeout(Duration.ofSeconds(10)); + + assertTransportSnapshot(snapshot, handler); + } + + @Test + public void liveOptionsValidateTransportSettings() { + VoiceAgentLiveWebSocketConnectionOptions options = new VoiceAgentLiveWebSocketConnectionOptions(); + assertEquals(256, options.getReceiveBufferCapacity()); + assertEquals(32 * 1024 * 1024, options.getMaxMessageSize()); + assertEquals(VoiceAgentWebSocketOverflowStrategy.ERROR, options.getOverflowStrategy()); + assertEquals(Duration.ofSeconds(30), options.getHandshakeTimeout()); + assertEquals(Duration.ofSeconds(10), options.getCloseTimeout()); + assertThrows(IllegalArgumentException.class, () -> options.setReceiveBufferCapacity(0)); + assertThrows(IllegalArgumentException.class, () -> options.setReceiveBufferCapacity(65537)); + assertThrows(IllegalArgumentException.class, () -> options.setMaxMessageSize(0)); + assertThrows(NullPointerException.class, () -> options.setOverflowStrategy(null)); + assertThrows(IllegalArgumentException.class, () -> options.setHandshakeTimeout(null)); + assertThrows(IllegalArgumentException.class, () -> options.setHandshakeTimeout(Duration.ZERO)); + assertThrows(IllegalArgumentException.class, () -> options.setCloseTimeout(Duration.ofSeconds(-1))); + assertThrows(NullPointerException.class, () -> new VoiceAgentLiveWebSocketConnectionOptions(null)); + } + + private static void assertTransportSnapshot(WebSocketTransportOptions snapshot, Consumer handler) { + assertEquals(17, snapshot.getReceiveBufferCapacity()); + assertEquals(4096, snapshot.getMaxMessageSize()); + assertEquals(VoiceAgentWebSocketOverflowStrategy.DROP_OLDEST, snapshot.getOverflowStrategy()); + assertSame(handler, snapshot.getMalformedEventHandler()); + assertEquals(Duration.ofSeconds(2), snapshot.getHandshakeTimeout()); + assertEquals(Duration.ofSeconds(3), snapshot.getCloseTimeout()); + } + private static String queryParameter(URI uri, String name) { for (String parameter : uri.getQuery().split("&")) { String[] pair = parameter.split("=", 2); diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java index 54d35e5afef47..20859c7d1a10e 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/GptLiveSamplesTests.java @@ -1272,6 +1272,8 @@ public void typedCommentaryPreservesExplicitNullDelegationId() throws IOExceptio assertEquals("Hello", command.get("content")); assertTrue(command.containsKey("delegation_id")); assertEquals(null, command.get("delegation_id")); + assertEquals(command, data.toObject(Map.class)); + assertEquals(command, GptLiveFunctionToolSample.object(BinaryData.fromBytes(data.toBytes()))); ByteArrayOutputStream output = new ByteArrayOutputStream(); try (com.azure.json.JsonWriter writer = com.azure.json.JsonProviders.createWriter(output)) { data.writeTo(writer); From 335e04d2610557b8d2edfcf218ecdb421451658c Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Tue, 3 Nov 2026 21:10:27 +0800 Subject: [PATCH 16/20] Refactor voice WebSocket transports to raw messages --- sdk/ai/azure-ai-agents/CHANGELOG.md | 3 +- sdk/ai/azure-ai-agents/README.md | 5 +- ...taVoiceAgentsLiveWebSocketAsyncClient.java | 11 +-- .../BetaVoiceAgentsLiveWebSocketClient.java | 13 ++- ...AgentsLiveWebSocketSessionAsyncClient.java | 22 +++-- ...VoiceAgentsLiveWebSocketSessionClient.java | 25 +++-- .../BetaVoiceAgentsWebSocketAsyncClient.java | 15 ++- .../BetaVoiceAgentsWebSocketClient.java | 12 ++- ...oiceAgentsWebSocketSessionAsyncClient.java | 34 ++----- ...BetaVoiceAgentsWebSocketSessionClient.java | 24 ++--- .../realtime/VoiceAgentWebSocketUtils.java | 70 ++++++++++++++ .../realtime/WebSocketEventDecoder.java | 2 +- .../WebSocketSessionAsyncTransport.java | 28 ++---- .../realtime/WebSocketSessionTransport.java | 93 ++++++------------ ...ceAgentLiveWebSocketConnectionOptions.java | 4 +- ...oiceAgentsWebSocketSessionClientTests.java | 4 +- .../voice/VoiceAgentLiveWebSocketTests.java | 96 +++++++++++++++++-- 17 files changed, 288 insertions(+), 173 deletions(-) diff --git a/sdk/ai/azure-ai-agents/CHANGELOG.md b/sdk/ai/azure-ai-agents/CHANGELOG.md index 38021136fb231..8cda19bfa43af 100644 --- a/sdk/ai/azure-ai-agents/CHANGELOG.md +++ b/sdk/ai/azure-ai-agents/CHANGELOG.md @@ -24,7 +24,8 @@ using `BinaryData`, separately from the Realtime protocol clients. - Added typed GPT Live sends through `sendEvent(com.openai.models.live.ClientEvent)` and typed receives through `receiveEvents()`, retaining the raw JSON APIs. The synchronous client also supports per-event receive timeouts. - Live transports use native OpenAI `ServerEvent` models while retaining original JSON for raw receivers. + Live session clients decode native OpenAI `ServerEvent` models; shared transports buffer only raw messages. + Raw receivers do not depend on JSON parsing or typed decoding succeeding. Live and Realtime clients select their own routes and headers without a shared protocol-mode flag. - Added synchronous/asynchronous persisted GPT Live conversation history and audio-download examples with offline tests. - Added opt-in persistence to the GPT Live text and microphone/speaker samples using `FOUNDRY_GPT_LIVE_STORE` (default `false`), diff --git a/sdk/ai/azure-ai-agents/README.md b/sdk/ai/azure-ai-agents/README.md index b61a8c1b0049a..52d24e435397d 100644 --- a/sdk/ai/azure-ai-agents/README.md +++ b/sdk/ai/azure-ai-agents/README.md @@ -1155,8 +1155,9 @@ try (BetaVoiceAgentsLiveWebSocketSessionClient session = liveClient.openWebSocke Send only native Live commands valid for the configured delegation mode, whether typed or raw. In particular, `response.item.create` and `response.create` are Responses-delegation-only commands. -Service `error` events arrive as events for the application to inspect; transport and malformed JSON -errors fail the receive stream unless a malformed-event callback was explicitly configured. +Service `error` events arrive as events for the application to inspect. Typed receivers decode JSON and +fail on malformed events unless a malformed-event callback was explicitly configured. Raw receivers do not +parse JSON or construct typed models; transport size limits and UTF-8 validation still apply. Configure timeouts, bounded receive capacity, overflow handling, message size, persistence, agent version, and structured inputs with `VoiceAgentLiveWebSocketConnectionOptions`. Structured inputs are passed as diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java index b461e71d3e5d3..741cabb1208b6 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java @@ -7,11 +7,9 @@ import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; import com.azure.ai.agents.implementation.realtime.WebSocketSessionAsyncTransport; import com.azure.ai.agents.implementation.realtime.WebSocketTransportOptions; -import com.azure.ai.agents.implementation.OpenAIJsonHelper; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.core.annotation.ServiceClient; -import com.openai.models.live.ServerEvent; import java.util.Objects; import reactor.core.publisher.Mono; import reactor.netty.http.client.HttpClient; @@ -50,12 +48,13 @@ public Mono openWebSocketSession Objects.requireNonNull(agentName, "'agentName' cannot be null."); VoiceAgentLiveWebSocketConnectionOptions snapshot = new VoiceAgentLiveWebSocketConnectionOptions(options); return Mono.defer(() -> { - WebSocketSessionAsyncTransport transport = new WebSocketSessionAsyncTransport<>(configuration, + WebSocketSessionAsyncTransport transport = new WebSocketSessionAsyncTransport(configuration, VoiceAgentWebSocketUtils.buildLiveWebSocketUri(configuration, agentName, snapshot), new WebSocketTransportOptions(snapshot), HttpClient.create(), null, - token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, token), json -> OpenAIJsonHelper - .fromBinaryData(VoiceAgentWebSocketUtils.deserializeRawEvent(json), ServerEvent.class)); - return transport.connect().thenReturn(new BetaVoiceAgentsLiveWebSocketSessionAsyncClient(transport)); + token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, token)); + return transport.connect() + .thenReturn( + new BetaVoiceAgentsLiveWebSocketSessionAsyncClient(transport, snapshot.getMalformedEventHandler())); }); } } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java index 92de691b32b8e..247bbcd70a55b 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java @@ -3,7 +3,6 @@ package com.azure.ai.agents; -import com.azure.ai.agents.implementation.OpenAIJsonHelper; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport; @@ -12,7 +11,6 @@ import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.core.annotation.ServiceClient; import com.azure.core.util.logging.ClientLogger; -import com.openai.models.live.ServerEvent; import java.util.Objects; @@ -54,10 +52,11 @@ public BetaVoiceAgentsLiveWebSocketSessionClient openWebSocketSession(String age throw LOGGER.logExceptionAsError(new IllegalArgumentException("'agentName' cannot be empty.")); } VoiceAgentLiveWebSocketConnectionOptions snapshot = new VoiceAgentLiveWebSocketConnectionOptions(options); - return new BetaVoiceAgentsLiveWebSocketSessionClient(WebSocketSessionTransport.connect(configuration, - VoiceAgentWebSocketUtils.buildLiveWebSocketUri(configuration, agentName, snapshot), - new WebSocketTransportOptions(snapshot), null, - token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, token), json -> OpenAIJsonHelper - .fromBinaryData(VoiceAgentWebSocketUtils.deserializeRawEvent(json), ServerEvent.class))); + return new BetaVoiceAgentsLiveWebSocketSessionClient( + WebSocketSessionTransport.connect(configuration, + VoiceAgentWebSocketUtils.buildLiveWebSocketUri(configuration, agentName, snapshot), + new WebSocketTransportOptions(snapshot), null, + token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, token)), + snapshot.getMalformedEventHandler()); } } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionAsyncClient.java index 6bdbc0672b861..66111ef660c6f 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionAsyncClient.java @@ -4,6 +4,7 @@ package com.azure.ai.agents; import com.azure.ai.agents.implementation.OpenAIJsonHelper; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; import com.azure.ai.agents.implementation.realtime.WebSocketSessionAsyncTransport; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.core.util.AsyncCloseable; @@ -11,6 +12,7 @@ import com.openai.models.live.ClientEvent; import com.openai.models.live.ServerEvent; import java.util.Objects; +import java.util.function.Consumer; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -23,10 +25,13 @@ */ @Beta(warningText = "This class is in preview and may change in future releases.") public final class BetaVoiceAgentsLiveWebSocketSessionAsyncClient implements AsyncCloseable, AutoCloseable { - private final WebSocketSessionAsyncTransport transport; + private final WebSocketSessionAsyncTransport transport; + private final Consumer malformedEventHandler; - BetaVoiceAgentsLiveWebSocketSessionAsyncClient(WebSocketSessionAsyncTransport transport) { + BetaVoiceAgentsLiveWebSocketSessionAsyncClient(WebSocketSessionAsyncTransport transport, + Consumer malformedEventHandler) { this.transport = transport; + this.malformedEventHandler = malformedEventHandler; } /** @@ -63,20 +68,23 @@ public String getCloseReason() { /** * Receives native Live server events in wire order, including unknown events. - * Service {@code error} events are delivered as JSON; transport and malformed JSON errors terminate the stream. - * @return the single-subscriber stream of complete JSON events. + * Messages are not parsed or validated as JSON. Transport errors terminate the stream. + * @return the single-subscriber stream of raw messages. */ public Flux receiveRawEvents() { - return transport.receiveRawEvents(); + return transport.receiveEvents(); } /** * Receives typed native Live server events. Typed and raw receivers share the single-subscriber limit. - * Use {@link #receiveRawEvents()} when the original JSON representation is needed. + * Use {@link #receiveRawEvents()} when the original JSON representation is needed. * @return the typed server event stream. */ public Flux receiveEvents() { - return transport.receiveEvents(); + return VoiceAgentWebSocketUtils.decodeEvents(transport.receiveEvents(), + json -> OpenAIJsonHelper.fromBinaryData(VoiceAgentWebSocketUtils.deserializeRawEvent(json), + ServerEvent.class), + malformedEventHandler, () -> transport.closeAsync(1007, "Invalid JSON event").subscribe()); } /** diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java index 034908e0a5cc5..beaffa6f1fc9d 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java @@ -4,6 +4,7 @@ package com.azure.ai.agents; import com.azure.ai.agents.implementation.OpenAIJsonHelper; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.core.util.BinaryData; @@ -12,6 +13,7 @@ import com.openai.models.live.ServerEvent; import java.time.Duration; import java.util.Objects; +import java.util.function.Consumer; /** * A synchronous bidirectional native GPT Live session. @@ -22,10 +24,13 @@ */ @Beta(warningText = "This class is in preview and may change in future releases.") public final class BetaVoiceAgentsLiveWebSocketSessionClient implements AutoCloseable { - private final WebSocketSessionTransport transport; + private final WebSocketSessionTransport transport; + private final Consumer malformedEventHandler; - BetaVoiceAgentsLiveWebSocketSessionClient(WebSocketSessionTransport transport) { + BetaVoiceAgentsLiveWebSocketSessionClient(WebSocketSessionTransport transport, + Consumer malformedEventHandler) { this.transport = transport; + this.malformedEventHandler = malformedEventHandler; } /** @@ -62,8 +67,8 @@ public String getCloseReason() { /** * Receives native Live server events in wire order, including unknown events. - * Service {@code error} events are delivered as JSON; transport and malformed JSON errors terminate iteration. - * @return the single-consumer stream of complete JSON events. + * Messages are not parsed or validated as JSON. Transport errors terminate iteration. + * @return the single-consumer stream of raw messages. * @throws IllegalStateException if the receive stream was already claimed. */ public IterableStream receiveRawEvents() { @@ -80,14 +85,17 @@ public IterableStream receiveEvents() { /** * Receives typed native Live events with a timeout per wait. Timeouts leave the session open. - * Use {@link #receiveRawEvents(Duration)} when the original JSON representation is needed. + * Use {@link #receiveRawEvents(Duration)} when the original JSON representation is needed. * @param timeout a positive timeout, or null to wait indefinitely. * @return the single-consumer typed event stream. * @throws IllegalArgumentException if the timeout is zero or negative. * @throws IllegalStateException if the receive stream was already claimed, or a wait times out. */ public IterableStream receiveEvents(Duration timeout) { - return transport.receiveEvents(timeout); + return VoiceAgentWebSocketUtils.decodeEvents( + transport.receiveEvents(timeout), json -> OpenAIJsonHelper + .fromBinaryData(VoiceAgentWebSocketUtils.deserializeRawEvent(json), ServerEvent.class), + malformedEventHandler, () -> transport.close(1007, "Invalid JSON event")); } /** @@ -103,14 +111,15 @@ public void sendEvent(ClientEvent event) { } /** - * Receives native Live events with a timeout per wait. Timeouts leave the session open so iteration can resume. + * Receives raw Live messages without JSON decoding, with a timeout per wait. + * Timeouts leave the session open so iteration can resume. * @param timeout a positive timeout, or null to wait indefinitely. * @return the single-consumer event stream. * @throws IllegalArgumentException if the timeout is zero or negative. * @throws IllegalStateException if the receive stream was already claimed, or a wait times out. */ public IterableStream receiveRawEvents(Duration timeout) { - return transport.receiveRawEvents(timeout); + return transport.receiveEvents(timeout); } /** diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketAsyncClient.java index ddb5caa844524..f3a4fa41d0fc6 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketAsyncClient.java @@ -4,10 +4,14 @@ package com.azure.ai.agents; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.realtime.WebSocketSessionAsyncTransport; +import com.azure.ai.agents.implementation.realtime.WebSocketTransportOptions; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.annotation.ServiceClient; import reactor.core.publisher.Mono; +import reactor.netty.http.client.HttpClient; import java.util.Objects; @@ -46,9 +50,14 @@ public Mono openWebSocketSession(Str Objects.requireNonNull(options, "'options' cannot be null."); VoiceAgentWebSocketConnectionOptions optionsSnapshot = new VoiceAgentWebSocketConnectionOptions(options); return Mono.defer(() -> { - BetaVoiceAgentsWebSocketSessionAsyncClient session - = new BetaVoiceAgentsWebSocketSessionAsyncClient(configuration, agentName, optionsSnapshot); - return session.connect().thenReturn(session); + WebSocketSessionAsyncTransport transport = new WebSocketSessionAsyncTransport(configuration, + VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, optionsSnapshot), + new WebSocketTransportOptions(optionsSnapshot), HttpClient.create(), + VoiceAgentWebSocketUtils.SUBPROTOCOL, + token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, optionsSnapshot, token)); + return transport.connect() + .thenReturn(new BetaVoiceAgentsWebSocketSessionAsyncClient(transport, + optionsSnapshot.getMalformedEventHandler())); }); } } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketClient.java index 5a54c46d1e491..c3c6fe7da459f 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketClient.java @@ -4,6 +4,9 @@ package com.azure.ai.agents; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport; +import com.azure.ai.agents.implementation.realtime.WebSocketTransportOptions; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.annotation.ServiceClient; @@ -49,7 +52,12 @@ public BetaVoiceAgentsWebSocketSessionClient openWebSocketSession(String agentNa throw LOGGER.logExceptionAsError(new IllegalArgumentException("'agentName' cannot be empty.")); } Objects.requireNonNull(options, "'options' cannot be null."); - return BetaVoiceAgentsWebSocketSessionClient.connect(configuration, agentName, - new VoiceAgentWebSocketConnectionOptions(options)); + VoiceAgentWebSocketConnectionOptions snapshot = new VoiceAgentWebSocketConnectionOptions(options); + return new BetaVoiceAgentsWebSocketSessionClient( + WebSocketSessionTransport.connect(configuration, + VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, snapshot), + new WebSocketTransportOptions(snapshot), VoiceAgentWebSocketUtils.SUBPROTOCOL, + token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, snapshot, token)), + snapshot.getMalformedEventHandler()); } } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java index e9c95a97993e9..1f399eb550e69 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java @@ -3,10 +3,8 @@ package com.azure.ai.agents; -import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; import com.azure.ai.agents.implementation.realtime.WebSocketSessionAsyncTransport; -import com.azure.ai.agents.implementation.realtime.WebSocketTransportOptions; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.RealtimeClientEvent; import com.azure.ai.agents.models.RealtimeConversationItemCreateEvent; @@ -20,7 +18,6 @@ import com.azure.ai.agents.models.RealtimeConversationItemUserMessage; import com.azure.ai.agents.models.RealtimeServerEvent; import com.azure.ai.agents.models.VoiceAgentResponseCreateOptions; -import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.util.AsyncCloseable; import com.azure.core.util.BinaryData; import com.openai.models.realtime.RealtimeConversationItemUserMessage.Content; @@ -28,9 +25,9 @@ import java.util.Base64; import java.util.Collections; import java.util.Objects; +import java.util.function.Consumer; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import reactor.netty.http.client.HttpClient; /** * An asynchronous bidirectional realtime session connected to a Foundry voice agent. @@ -40,26 +37,13 @@ */ @Beta(warningText = "This class is in preview and may change in future releases.") public final class BetaVoiceAgentsWebSocketSessionAsyncClient implements AsyncCloseable, AutoCloseable { - private final WebSocketSessionAsyncTransport transport; + private final WebSocketSessionAsyncTransport transport; + private final Consumer malformedEventHandler; - BetaVoiceAgentsWebSocketSessionAsyncClient(VoiceAgentWebSocketClientConfiguration configuration, String agentName, - VoiceAgentWebSocketConnectionOptions options) { - this(configuration, agentName, options, HttpClient.create()); - } - - BetaVoiceAgentsWebSocketSessionAsyncClient(VoiceAgentWebSocketClientConfiguration configuration, String agentName, - VoiceAgentWebSocketConnectionOptions options, HttpClient httpClient) { - VoiceAgentWebSocketConnectionOptions sessionOptions - = options == null ? new VoiceAgentWebSocketConnectionOptions() : options; - transport = new WebSocketSessionAsyncTransport<>(configuration, - VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, sessionOptions), - new WebSocketTransportOptions(sessionOptions), httpClient, VoiceAgentWebSocketUtils.SUBPROTOCOL, - token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, sessionOptions, token), - VoiceAgentWebSocketUtils::deserializeEvent); - } - - Mono connect() { - return transport.connect(); + BetaVoiceAgentsWebSocketSessionAsyncClient(WebSocketSessionAsyncTransport transport, + Consumer malformedEventHandler) { + this.transport = transport; + this.malformedEventHandler = malformedEventHandler; } /** @@ -104,7 +88,9 @@ public String getCloseReason() { * @return the server event stream. */ public Flux receiveEvents() { - return transport.receiveEvents(); + return VoiceAgentWebSocketUtils.decodeEvents(transport.receiveEvents(), + VoiceAgentWebSocketUtils::deserializeEvent, malformedEventHandler, + () -> transport.closeAsync(1007, "Invalid JSON event").subscribe()); } /** diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java index 7a5877af7d1ac..df8193e324de1 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java @@ -3,10 +3,8 @@ package com.azure.ai.agents; -import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport; -import com.azure.ai.agents.implementation.realtime.WebSocketTransportOptions; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.RealtimeClientEvent; import com.azure.ai.agents.models.RealtimeConversationItemCreateEvent; @@ -20,7 +18,6 @@ import com.azure.ai.agents.models.RealtimeConversationItemUserMessage; import com.azure.ai.agents.models.RealtimeServerEvent; import com.azure.ai.agents.models.VoiceAgentResponseCreateOptions; -import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.util.BinaryData; import com.openai.models.realtime.RealtimeConversationItemUserMessage.Content; import com.azure.core.util.IterableStream; @@ -30,6 +27,7 @@ import java.util.Base64; import java.util.Collections; import java.util.Objects; +import java.util.function.Consumer; /** * A synchronous bidirectional realtime session connected to a Foundry voice agent. @@ -37,19 +35,13 @@ @Beta(warningText = "This class is in preview and may change in future releases.") public final class BetaVoiceAgentsWebSocketSessionClient implements AutoCloseable { private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentsWebSocketSessionClient.class); - private final WebSocketSessionTransport transport; + private final WebSocketSessionTransport transport; + private final Consumer malformedEventHandler; - private BetaVoiceAgentsWebSocketSessionClient(WebSocketSessionTransport transport) { + BetaVoiceAgentsWebSocketSessionClient(WebSocketSessionTransport transport, + Consumer malformedEventHandler) { this.transport = transport; - } - - static BetaVoiceAgentsWebSocketSessionClient connect(VoiceAgentWebSocketClientConfiguration configuration, - String agentName, VoiceAgentWebSocketConnectionOptions options) { - return new BetaVoiceAgentsWebSocketSessionClient(WebSocketSessionTransport.connect(configuration, - VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, options), - new WebSocketTransportOptions(options), VoiceAgentWebSocketUtils.SUBPROTOCOL, - token -> VoiceAgentWebSocketUtils.buildHeaders(configuration, options, token), - VoiceAgentWebSocketUtils::deserializeEvent)); + this.malformedEventHandler = malformedEventHandler; } /** @@ -106,7 +98,9 @@ public IterableStream receiveEvents() { * @throws IllegalStateException if another receiver exists or a wait times out (with a TimeoutException cause). */ public IterableStream receiveEvents(Duration timeout) { - return transport.receiveEvents(timeout); + return VoiceAgentWebSocketUtils.decodeEvents(transport.receiveEvents(timeout), + VoiceAgentWebSocketUtils::deserializeEvent, malformedEventHandler, + () -> transport.close(1007, "Invalid JSON event")); } /** diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java index 12724e1810591..211acb6b39b63 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java @@ -14,7 +14,9 @@ import com.azure.core.http.HttpHeaderName; import com.azure.core.http.HttpHeaders; import com.azure.core.util.BinaryData; +import com.azure.core.util.IterableStream; import com.azure.core.util.UrlBuilder; +import com.azure.core.util.logging.ClientLogger; import com.azure.json.JsonProviders; import com.azure.json.JsonReader; import com.azure.json.JsonToken; @@ -27,9 +29,77 @@ import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; import java.util.Locale; +import java.util.Iterator; +import java.util.NoSuchElementException; import java.util.Objects; +import java.util.function.Consumer; +import reactor.core.publisher.Flux; public final class VoiceAgentWebSocketUtils { + private static final ClientLogger LOGGER = new ClientLogger(VoiceAgentWebSocketUtils.class); + + public static IterableStream decodeEvents(IterableStream messages, + WebSocketEventDecoder decoder, Consumer malformedEventHandler, Runnable closeOnError) { + return IterableStream.of(() -> new Iterator() { + private final Iterator source = messages.iterator(); + private T next; + private boolean terminated; + + @Override + public boolean hasNext() { + while (next == null && !terminated && source.hasNext()) { + BinaryData message = source.next(); + try { + next = decodeEvent(message, decoder, malformedEventHandler); + } catch (RuntimeException error) { + terminated = true; + closeOnError.run(); + throw LOGGER.logExceptionAsError(error); + } + } + return next != null; + } + + @Override + public T next() { + if (!hasNext()) { + throw LOGGER.logExceptionAsError(new NoSuchElementException()); + } + T event = next; + next = null; + return event; + } + }); + } + + public static Flux decodeEvents(Flux messages, WebSocketEventDecoder decoder, + Consumer malformedEventHandler, Runnable closeOnError) { + return messages.handle((message, sink) -> { + try { + T event = decodeEvent(message, decoder, malformedEventHandler); + if (event != null) { + sink.next(event); + } + } catch (RuntimeException error) { + closeOnError.run(); + sink.error(error); + } + }); + } + + private static T decodeEvent(BinaryData message, WebSocketEventDecoder decoder, + Consumer malformedEventHandler) { + try { + return decoder.decode(message.toString()); + } catch (IOException | RuntimeException error) { + if (malformedEventHandler != null) { + malformedEventHandler.accept(error); + return null; + } + throw LOGGER.logExceptionAsError(new IllegalArgumentException("Invalid JSON event.", error)); + } + } + public static String decodeEvent(byte[] bytes) throws CharacterCodingException { return StandardCharsets.UTF_8.newDecoder() .onMalformedInput(CodingErrorAction.REPORT) diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketEventDecoder.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketEventDecoder.java index 45d834f742173..487f459f81e66 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketEventDecoder.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketEventDecoder.java @@ -6,7 +6,7 @@ import java.io.IOException; /** - * Decodes a protocol event received by either WebSocket transport. + * Decodes a raw message in a protocol session client. * @param the decoded event type. */ @FunctionalInterface diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java index f21f9d4d7e777..fb41390c789f9 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java @@ -3,7 +3,6 @@ package com.azure.ai.agents.implementation.realtime; -import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport.ReceivedEvent; import com.azure.core.credential.AccessToken; import com.azure.core.exception.ClientAuthenticationException; import com.azure.core.exception.HttpResponseException; @@ -43,10 +42,9 @@ import reactor.netty.transport.ProxyProvider; /** - * Shared asynchronous WebSocket connection and I/O. Protocol clients supply the incoming message decoder. - * @param the decoded message type. + * Shared asynchronous WebSocket connection and raw message I/O. */ -public final class WebSocketSessionAsyncTransport implements AutoCloseable { +public final class WebSocketSessionAsyncTransport implements AutoCloseable { private static final ClientLogger LOGGER = new ClientLogger(WebSocketSessionAsyncTransport.class); private static final int MAX_OUTSTANDING_SENDS = 256; private final VoiceAgentWebSocketClientConfiguration configuration; @@ -55,7 +53,6 @@ public final class WebSocketSessionAsyncTransport implements AutoCloseable { private final URI websocketUri; private final String subprotocol; private final Function headersFactory; - private final WebSocketEventDecoder decoder; private final AtomicReference state = new AtomicReference<>(WebSocketSessionState.NEW); private final AtomicReference channel = new AtomicReference<>(); private final AtomicReference outbound = new AtomicReference<>(); @@ -63,8 +60,8 @@ public final class WebSocketSessionAsyncTransport implements AutoCloseable { private final AtomicReference lifecycle = new AtomicReference<>(); private final AtomicBoolean receiveClaimed = new AtomicBoolean(); private final Semaphore sendPermits = new Semaphore(MAX_OUTSTANDING_SENDS); - private final Queue> eventQueue; - private final Sinks.Many> events; + private final Queue eventQueue; + private final Sinks.Many events; private final Sinks.One ready = Sinks.one(); private final Sinks.One closeSignal = Sinks.one(); private final AtomicReference> closeOperation = new AtomicReference<>(); @@ -73,7 +70,7 @@ public final class WebSocketSessionAsyncTransport implements AutoCloseable { public WebSocketSessionAsyncTransport(VoiceAgentWebSocketClientConfiguration configuration, URI websocketUri, WebSocketTransportOptions options, HttpClient httpClient, String subprotocol, - Function headersFactory, WebSocketEventDecoder decoder) { + Function headersFactory) { this.configuration = Objects.requireNonNull(configuration, "'configuration' cannot be null."); this.options = Objects.requireNonNull(options, "'options' cannot be null."); this.eventQueue = new ArrayBlockingQueue<>(this.options.getReceiveBufferCapacity()); @@ -82,7 +79,6 @@ public WebSocketSessionAsyncTransport(VoiceAgentWebSocketClientConfiguration con this.websocketUri = Objects.requireNonNull(websocketUri, "'websocketUri' cannot be null."); this.subprotocol = subprotocol; this.headersFactory = Objects.requireNonNull(headersFactory, "'headersFactory' cannot be null."); - this.decoder = Objects.requireNonNull(decoder, "'decoder' cannot be null."); } public Mono connect() { @@ -121,15 +117,7 @@ public String getCloseReason() { return closeReason; } - public Flux receiveEvents() { - return receiveMessages().map(ReceivedEvent::getEvent); - } - - public Flux receiveRawEvents() { - return receiveMessages().map(ReceivedEvent::getRawEvent); - } - - private Flux> receiveMessages() { + public Flux receiveEvents() { return Flux.defer(() -> receiveClaimed.compareAndSet(false, true) ? events.asFlux() : Flux.error(new IllegalStateException("Only one receiveEvents subscriber is supported per session."))); @@ -247,7 +235,7 @@ private void handleFrame(WebSocketFrame frame) { byte[] bytes = new byte[frame.content().readableBytes()]; frame.content().getBytes(frame.content().readerIndex(), bytes); String json = VoiceAgentWebSocketUtils.decodeEvent(bytes); - ReceivedEvent event = new ReceivedEvent<>(decoder.decode(json), BinaryData.fromString(json)); + BinaryData event = BinaryData.fromString(json); Sinks.EmitResult result = events.tryEmitNext(event); if (result == Sinks.EmitResult.FAIL_OVERFLOW || result == Sinks.EmitResult.FAIL_ZERO_SUBSCRIBER) { switch (options.getOverflowStrategy()) { @@ -276,7 +264,7 @@ private void handleFrame(WebSocketFrame frame) { failure = callbackError; } } - closeWithProtocolError(1007, "Invalid JSON event", failure); + closeWithProtocolError(1007, "Invalid UTF-8 message", failure); } } } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java index c6f61713421e8..1ba561662e8fe 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java @@ -45,15 +45,13 @@ /** * Shared blocking WebSocket connection and I/O, independent of the asynchronous transport. - * @param the decoded message type. */ -public final class WebSocketSessionTransport implements AutoCloseable { +public final class WebSocketSessionTransport implements AutoCloseable { private static final ClientLogger LOGGER = new ClientLogger(WebSocketSessionTransport.class); private final URI websocketUri; private final WebSocketTransportOptions options; - private final WebSocketEventDecoder decoder; private final OkHttpClient httpClient; - private final BlockingQueue> events; + private final BlockingQueue events; private final int receiveBufferCapacity; private final CountDownLatch handshakeCompleted = new CountDownLatch(1); private final CountDownLatch closeCompleted = new CountDownLatch(1); @@ -67,9 +65,8 @@ public final class WebSocketSessionTransport implements AutoCloseable { private volatile String closeReason; private WebSocketSessionTransport(VoiceAgentWebSocketClientConfiguration configuration, URI websocketUri, - WebSocketTransportOptions options, WebSocketEventDecoder decoder) { + WebSocketTransportOptions options) { this.options = options; - this.decoder = Objects.requireNonNull(decoder, "'decoder' cannot be null."); this.receiveBufferCapacity = options.getReceiveBufferCapacity(); this.events = new ArrayBlockingQueue<>(receiveBufferCapacity + 1); this.websocketUri = Objects.requireNonNull(websocketUri, "'websocketUri' cannot be null."); @@ -91,11 +88,10 @@ private void open(VoiceAgentWebSocketClientConfiguration configuration, String s this.webSocket = httpClient.newWebSocket(request.build(), new Listener()); } - public static WebSocketSessionTransport connect(VoiceAgentWebSocketClientConfiguration configuration, + public static WebSocketSessionTransport connect(VoiceAgentWebSocketClientConfiguration configuration, URI websocketUri, WebSocketTransportOptions options, String subprotocol, - Function headersFactory, WebSocketEventDecoder decoder) { - WebSocketSessionTransport session - = new WebSocketSessionTransport<>(configuration, websocketUri, options, decoder); + Function headersFactory) { + WebSocketSessionTransport session = new WebSocketSessionTransport(configuration, websocketUri, options); try { session.open(configuration, subprotocol, headersFactory); session.awaitHandshake(); @@ -125,15 +121,7 @@ public String getCloseReason() { return closeReason; } - public IterableStream receiveEvents(Duration timeout) { - return receiveMessages(timeout, ReceivedEvent::getEvent); - } - - public IterableStream receiveRawEvents(Duration timeout) { - return receiveMessages(timeout, ReceivedEvent::getRawEvent); - } - - private IterableStream receiveMessages(Duration timeout, Function, R> mapper) { + public IterableStream receiveEvents(Duration timeout) { if (timeout != null && (timeout.isZero() || timeout.isNegative())) { throw LOGGER.logExceptionAsError(new IllegalArgumentException("Receive timeout must be positive.")); } @@ -147,7 +135,7 @@ private IterableStream receiveMessages(Duration timeout, Function(events, timeout, mapper); + return new EventIterator(events, timeout); }); } @@ -252,7 +240,7 @@ private Throwable mapHandshakeError(Throwable error, Response response) { } } - private synchronized void signal(EventSignal signal) { + private synchronized void signal(EventSignal signal) { if (signal.event != null && events.size() >= receiveBufferCapacity) { switch (options.getOverflowStrategy()) { case DROP_LATEST: @@ -344,11 +332,7 @@ public void onMessage(WebSocket webSocket, String text) { webSocket.close(1009, "Message too large"); return; } - try { - signal(EventSignal.event(new ReceivedEvent<>(decoder.decode(text), BinaryData.fromString(text)))); - } catch (IOException | RuntimeException error) { - malformedEvent(webSocket, error); - } + signal(EventSignal.event(BinaryData.fromString(text))); } @Override @@ -374,8 +358,8 @@ private void malformedEvent(WebSocket webSocket, Throwable error) { error = callbackError; } } - fail(new IllegalArgumentException("Invalid JSON event.", error), null); - webSocket.close(1007, "Invalid JSON event"); + fail(new IllegalArgumentException("Invalid UTF-8 message.", error), null); + webSocket.close(1007, "Invalid UTF-8 message"); } @Override @@ -406,59 +390,38 @@ public void onFailure(WebSocket webSocket, Throwable error, Response response) { } } - static final class ReceivedEvent { - private final T event; - private final BinaryData rawEvent; - - ReceivedEvent(T event, BinaryData rawEvent) { - this.event = event; - this.rawEvent = rawEvent; - } - - T getEvent() { - return event; - } - - BinaryData getRawEvent() { - return rawEvent; - } - } - - private static final class EventSignal { - private final ReceivedEvent event; + private static final class EventSignal { + private final BinaryData event; private final Throwable error; private final boolean complete; - private EventSignal(ReceivedEvent event, Throwable error, boolean complete) { + private EventSignal(BinaryData event, Throwable error, boolean complete) { this.event = event; this.error = error; this.complete = complete; } - private static EventSignal event(ReceivedEvent event) { - return new EventSignal<>(event, null, false); + private static EventSignal event(BinaryData event) { + return new EventSignal(event, null, false); } - private static EventSignal error(Throwable error) { - return new EventSignal<>(null, error, false); + private static EventSignal error(Throwable error) { + return new EventSignal(null, error, false); } - private static EventSignal complete() { - return new EventSignal<>(null, null, true); + private static EventSignal complete() { + return new EventSignal(null, null, true); } } - private static final class EventIterator implements Iterator { - private final BlockingQueue> events; + private static final class EventIterator implements Iterator { + private final BlockingQueue events; private final Duration timeout; - private final Function, R> mapper; - private EventSignal next; + private EventSignal next; - private EventIterator(BlockingQueue> events, Duration timeout, - Function, R> mapper) { + private EventIterator(BlockingQueue events, Duration timeout) { this.events = events; this.timeout = timeout; - this.mapper = mapper; } @Override @@ -487,13 +450,13 @@ public boolean hasNext() { } @Override - public R next() { + public BinaryData next() { if (!hasNext()) { throw new NoSuchElementException(); } - ReceivedEvent event = next.event; + BinaryData event = next.event; next = null; - return mapper.apply(event); + return event; } } } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java index f6a0a6174e4ed..12af0b3e2b939 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java @@ -121,7 +121,9 @@ public Consumer getMalformedEventHandler() { } /** - * Sets a callback to report and skip malformed events. It runs on the receive thread and must not block. + * Sets a callback to report and skip malformed events. It runs while receiving events and must not block. + * JSON decoding failures are reported only by typed receivers; raw receivers do not parse JSON. + * Invalid UTF-8 is reported by the transport for either receiver type. * If the callback throws, the session terminates. Transport errors and oversized messages are not skipped. * @param handler the callback, or null to terminate on malformed events. * @return this options instance. diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java index 639dc636634a8..680c54e9f41e0 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java @@ -76,7 +76,7 @@ public void syncConnectionPreservesTokenFailure() { IllegalStateException actual = assertThrows(IllegalStateException.class, () -> WebSocketSessionTransport.connect(configuration, URI.create("wss://localhost/session"), new WebSocketTransportOptions(new VoiceAgentWebSocketConnectionOptions()), null, - token -> new HttpHeaders(), json -> json)); + token -> new HttpHeaders())); assertSame(failure, actual); } @@ -93,7 +93,7 @@ public void syncConnectionPreservesHeaderInitializationFailure() { () -> WebSocketSessionTransport.connect(configuration, URI.create("wss://localhost/session"), new WebSocketTransportOptions(new VoiceAgentWebSocketConnectionOptions()), null, token -> { throw failure; - }, json -> json)); + })); assertSame(failure, actual); } diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java index 8e405faa76a44..88526d703823c 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveWebSocketTests.java @@ -166,6 +166,45 @@ public void syncNativeEventsAreLosslessAndSessionIsBidirectional() { } } + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void rawReceptionDoesNotRequireTypedDecoding(boolean async) { + String message = "not-json"; + String unusualPayload = "{\"type\":\"session.started\",\"session\":7}"; + AtomicInteger malformed = new AtomicInteger(); + VoiceAgentLiveWebSocketConnectionOptions options = new VoiceAgentLiveWebSocketConnectionOptions() + .setMalformedEventHandler(error -> malformed.incrementAndGet()); + server = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, response) -> response.sendWebsocket((inbound, + outbound) -> outbound.sendString(Flux.just(message, unusualPayload)).then(inbound.receive().then()))) + .bindNow(); + if (async) { + BetaVoiceAgentsLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentsLiveWebSocketAsyncClient() + .openWebSocketSession("agent", options) + .block(TIMEOUT); + assertNotNull(session); + try { + StepVerifier.create(session.receiveRawEvents().take(2)) + .assertNext(event -> assertEquals(message, event.toString())) + .assertNext(event -> assertEquals(unusualPayload, event.toString())) + .verifyComplete(); + } finally { + session.closeAsync().block(TIMEOUT); + } + } else { + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent", options)) { + Iterator events = session.receiveRawEvents(TIMEOUT).iterator(); + assertEquals(message, events.next().toString()); + assertEquals(unusualPayload, events.next().toString()); + } + } + assertEquals(0, malformed.get()); + } + @Test public void asyncNativeEventsAreLosslessAndSessionIsBidirectional() { List sent = new CopyOnWriteArrayList<>(); @@ -381,7 +420,7 @@ public void malformedJsonTerminatesReception(boolean async) { .block(TIMEOUT); assertNotNull(session); try { - StepVerifier.create(session.receiveRawEvents()) + StepVerifier.create(session.receiveEvents()) .expectError(IllegalArgumentException.class) .verify(TIMEOUT); } finally { @@ -390,8 +429,7 @@ public void malformedJsonTerminatesReception(boolean async) { } else { try (BetaVoiceAgentsLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent")) { - assertThrows(IllegalArgumentException.class, - () -> session.receiveRawEvents(TIMEOUT).iterator().hasNext()); + assertThrows(IllegalArgumentException.class, () -> session.receiveEvents(TIMEOUT).iterator().hasNext()); } } } @@ -424,6 +462,46 @@ public void connectionOptionsValidateBoundsAndCopyAllSettings() { assertEquals("{\"name\":\"value\"}", copy.getStructuredInputs()); } + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void throwingMalformedEventCallbackTerminatesTypedReception(boolean async) { + IllegalStateException failure = new IllegalStateException("Malformed event callback failed."); + VoiceAgentLiveWebSocketConnectionOptions options + = new VoiceAgentLiveWebSocketConnectionOptions().setMalformedEventHandler(error -> { + throw failure; + }); + server + = VoiceAgentWebSocketSessionTests.tlsServer() + .host("localhost") + .port(0) + .handle((request, response) -> response.sendWebsocket((inbound, + outbound) -> outbound.sendString(Flux.just("not-json", STARTED)).then(inbound.receive().then()))) + .bindNow(); + if (async) { + BetaVoiceAgentsLiveWebSocketSessionAsyncClient session = builder().beta() + .buildBetaVoiceAgentsLiveWebSocketAsyncClient() + .openWebSocketSession("agent", options) + .block(TIMEOUT); + assertNotNull(session); + try { + StepVerifier.create(session.receiveEvents()) + .expectErrorMatches(error -> error == failure) + .verify(TIMEOUT); + assertFalse(session.isOpen()); + } finally { + session.closeAsync().block(TIMEOUT); + } + } else { + try (BetaVoiceAgentsLiveWebSocketSessionClient session + = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent", options)) { + Iterator events = session.receiveEvents(TIMEOUT).iterator(); + assertEquals(failure, assertThrows(IllegalStateException.class, events::hasNext)); + assertFalse(events.hasNext()); + assertFalse(session.isOpen()); + } + } + } + @ParameterizedTest @ValueSource(booleans = { false, true }) public void peerClosurePreservesEventsAndCloseMetadata(boolean async) { @@ -473,8 +551,8 @@ public void malformedEventCallbackCanSkipInvalidJson(boolean async) { server = VoiceAgentWebSocketSessionTests.tlsServer() .host("localhost") .port(0) - .handle((request, response) -> response.sendWebsocket( - (inbound, outbound) -> outbound.sendString(Flux.just("[]", STARTED)).then(inbound.receive().then()))) + .handle((request, response) -> response.sendWebsocket((inbound, + outbound) -> outbound.sendString(Flux.just("[]", "not-json", STARTED)).then(inbound.receive().then()))) .bindNow(); if (async) { BetaVoiceAgentsLiveWebSocketSessionAsyncClient session = builder().beta() @@ -483,8 +561,8 @@ public void malformedEventCallbackCanSkipInvalidJson(boolean async) { .block(TIMEOUT); assertNotNull(session); try { - StepVerifier.create(session.receiveRawEvents().map(BinaryData::toString)) - .expectNext(STARTED) + StepVerifier.create(session.receiveEvents()) + .assertNext(VoiceAgentLiveWebSocketTests::assertStartedEvent) .thenCancel() .verify(TIMEOUT); } finally { @@ -493,10 +571,10 @@ public void malformedEventCallbackCanSkipInvalidJson(boolean async) { } else { try (BetaVoiceAgentsLiveWebSocketSessionClient session = builder().beta().buildBetaVoiceAgentsLiveWebSocketClient().openWebSocketSession("agent", options)) { - assertEquals(STARTED, session.receiveRawEvents(TIMEOUT).iterator().next().toString()); + assertStartedEvent(session.receiveEvents(TIMEOUT).iterator().next()); } } - assertEquals(1, malformed.get()); + assertEquals(2, malformed.get()); } @ParameterizedTest From 6b21fb6d75263436216822106bc476911426bd94 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Wed, 4 Nov 2026 09:15:28 +0800 Subject: [PATCH 17/20] Address voice WebSocket URI and lifecycle review feedback --- sdk/ai/azure-ai-agents/README.md | 1 + .../realtime/VoiceAgentWebSocketUtils.java | 11 +++--- .../realtime/WebSocketSessionTransport.java | 4 +-- .../com/azure/ai/agents/ReadmeSamples.java | 1 + ...oiceAgentsWebSocketSessionClientTests.java | 36 ++++++++++++++++--- 5 files changed, 41 insertions(+), 12 deletions(-) diff --git a/sdk/ai/azure-ai-agents/README.md b/sdk/ai/azure-ai-agents/README.md index 52d24e435397d..e3fff2b159dc4 100644 --- a/sdk/ai/azure-ai-agents/README.md +++ b/sdk/ai/azure-ai-agents/README.md @@ -1140,6 +1140,7 @@ VoiceAgentLiveWebSocketConnectionOptions options = new VoiceAgentLiveWebSocketCo .setHandshakeTimeout(Duration.ofSeconds(30)) .setCloseTimeout(Duration.ofSeconds(10)) .setStoreEnabled(false); + try (BetaVoiceAgentsLiveWebSocketSessionClient session = liveClient.openWebSocketSession(agentName, options)) { for (ServerEvent event : session.receiveEvents()) { if (event.isSessionStarted()) { diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java index 211acb6b39b63..9487420a34e97 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java @@ -205,10 +205,7 @@ private static UrlBuilder buildSessionUrl(VoiceAgentWebSocketClientConfiguration throw new IllegalArgumentException("A voice-agent route suffix must start with '/'."); } URI endpoint = configuration.getEndpoint(); - String scheme; - if ("https".equalsIgnoreCase(endpoint.getScheme()) || "wss".equalsIgnoreCase(endpoint.getScheme())) { - scheme = "wss"; - } else { + if (!"https".equalsIgnoreCase(endpoint.getScheme()) && !"wss".equalsIgnoreCase(endpoint.getScheme())) { throw new IllegalArgumentException( "Voice-agent WebSocket endpoints must use https or wss to protect credentials."); } @@ -219,8 +216,10 @@ private static UrlBuilder buildSessionUrl(VoiceAgentWebSocketClientConfiguration String basePath = endpoint.getRawPath() == null ? "" : endpoint.getRawPath().replaceAll("/$", ""); String path = basePath + "/agents/" + encode(agentName) + "/endpoint/protocols/voice" + suffix; - URI baseUri = URI.create(scheme + "://" + endpoint.getRawAuthority() + path); - UrlBuilder url = UrlBuilder.parse(baseUri.toString()); + UrlBuilder url = UrlBuilder.parse(endpoint.toString()).setScheme("wss").setPath(path); + // Initialize the lazy query before clearing it; the pinned Azure Core version otherwise retains it. + url.getQuery(); + url.clearQuery(); url.setQueryParameter("api-version", encode(configuration.getApiVersion())); url.setQueryParameter("x-ms-client-sdk", encode(configuration.getUserAgent())); if (store != null) { diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java index 1ba561662e8fe..43ccdbdb5c9a5 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java @@ -57,8 +57,7 @@ public final class WebSocketSessionTransport implements AutoCloseable { private final CountDownLatch closeCompleted = new CountDownLatch(1); private final AtomicReference connectionError = new AtomicReference<>(); private final AtomicBoolean receiveClaimed = new AtomicBoolean(); - private final AtomicReference state - = new AtomicReference<>(WebSocketSessionState.CONNECTING); + private final AtomicReference state = new AtomicReference<>(WebSocketSessionState.NEW); private final AtomicBoolean clientShutdown = new AtomicBoolean(); private volatile WebSocket webSocket; private volatile Integer closeCode; @@ -75,6 +74,7 @@ private WebSocketSessionTransport(VoiceAgentWebSocketClientConfiguration configu private void open(VoiceAgentWebSocketClientConfiguration configuration, String subprotocol, Function headersFactory) { + state.set(WebSocketSessionState.CONNECTING); String token = configuration.getCredential() .getTokenSync(VoiceAgentWebSocketUtils.createTokenRequestContext()) .getToken(); diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java index 627af952fca51..4cf121f591473 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java @@ -42,6 +42,7 @@ public void liveConnectionOptions(AgentsClientBuilder builder, String agentName) .setHandshakeTimeout(Duration.ofSeconds(30)) .setCloseTimeout(Duration.ofSeconds(10)) .setStoreEnabled(false); + try (BetaVoiceAgentsLiveWebSocketSessionClient session = liveClient.openWebSocketSession(agentName, options)) { for (ServerEvent event : session.receiveEvents()) { if (event.isSessionStarted()) { diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java index 680c54e9f41e0..380f3232f9513 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClientTests.java @@ -16,6 +16,8 @@ import java.net.InetSocketAddress; import java.net.Proxy; import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.OffsetDateTime; import java.util.List; @@ -30,7 +32,7 @@ public class BetaVoiceAgentsWebSocketSessionClientTests { @Test - public void connectionUriUsesSelectedTransport() { + public void connectionUriUsesSelectedTransport() throws Exception { VoiceAgentWebSocketClientConfiguration configuration = new VoiceAgentWebSocketClientConfiguration( URI.create("https://localhost"), request -> Mono.just(new com.azure.core.credential.AccessToken("token", OffsetDateTime.now().plusHours(1))), @@ -48,6 +50,32 @@ public void connectionUriUsesSelectedTransport() { assertEquals("webrtc", queryParameter(webRtcUri, "transport")); } + @Test + public void connectionUrisPreserveEncodedPathsAndReplaceEndpointQuery() throws Exception { + VoiceAgentWebSocketClientConfiguration configuration = new VoiceAgentWebSocketClientConfiguration( + URI.create("https://localhost:8443/project%20name/?discard=old&api-version=old"), + request -> Mono.just(new com.azure.core.credential.AccessToken("token", OffsetDateTime.now().plusHours(1))), + "v1", "test-user-agent", null, null); + + URI realtimeUri = VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, "agent/name", + new VoiceAgentWebSocketConnectionOptions().setAgentSessionId("session & id")); + URI liveUri = VoiceAgentWebSocketUtils.buildLiveWebSocketUri(configuration, "agent/name", + new VoiceAgentLiveWebSocketConnectionOptions().setStructuredInputs("{\"name\":\"a & b\"}")); + + for (URI uri : new URI[] { realtimeUri, liveUri }) { + assertEquals("wss", uri.getScheme()); + assertEquals("localhost", uri.getHost()); + assertEquals(8443, uri.getPort()); + assertEquals("v1", queryParameter(uri, "api-version")); + assertEquals(null, queryParameter(uri, "discard")); + } + assertEquals("/project%20name/agents/agent%2Fname/endpoint/protocols/voice", realtimeUri.getRawPath()); + assertEquals("/project%20name/agents/agent%2Fname/endpoint/protocols/voice/live/sessions", + liveUri.getRawPath()); + assertEquals("session & id", queryParameter(realtimeUri, "agent_session_id")); + assertEquals("{\"name\":\"a & b\"}", queryParameter(liveUri, "structured_inputs")); + } + @Test public void syncTransportHonorsNonProxyHosts() { InetSocketAddress proxyAddress = new InetSocketAddress("localhost", 8080); @@ -180,11 +208,11 @@ private static void assertTransportSnapshot(WebSocketTransportOptions snapshot, assertEquals(Duration.ofSeconds(3), snapshot.getCloseTimeout()); } - private static String queryParameter(URI uri, String name) { - for (String parameter : uri.getQuery().split("&")) { + private static String queryParameter(URI uri, String name) throws Exception { + for (String parameter : uri.getRawQuery().split("&")) { String[] pair = parameter.split("=", 2); if (name.equals(pair[0])) { - return pair.length == 2 ? pair[1] : ""; + return pair.length == 2 ? URLDecoder.decode(pair[1], StandardCharsets.UTF_8.name()) : ""; } } return null; From c48bbbfb3b654c7c16f20361048199ce26828c3a Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Wed, 4 Nov 2026 14:43:16 +0800 Subject: [PATCH 18/20] Normalize voice WebSocket imports and Javadoc formatting --- ...taVoiceAgentsLiveWebSocketAsyncClient.java | 5 ++- .../BetaVoiceAgentsLiveWebSocketClient.java | 2 + ...AgentsLiveWebSocketSessionAsyncClient.java | 19 ++++++++-- ...VoiceAgentsLiveWebSocketSessionClient.java | 16 +++++++- .../BetaVoiceAgentsWebSocketClient.java | 2 +- ...oiceAgentsWebSocketSessionAsyncClient.java | 13 ++++--- ...BetaVoiceAgentsWebSocketSessionClient.java | 38 +++++++++++++------ ...oiceAgentWebSocketClientConfiguration.java | 2 +- .../realtime/VoiceAgentWebSocketUtils.java | 7 ++-- .../WebSocketSessionAsyncTransport.java | 21 +++++----- .../realtime/WebSocketSessionTransport.java | 17 +++++---- ...ceAgentLiveWebSocketConnectionOptions.java | 30 +++++++++++++-- .../VoiceAgentWebSocketConnectionOptions.java | 17 ++++++++- 13 files changed, 138 insertions(+), 51 deletions(-) diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java index 741cabb1208b6..90fca1b92c29e 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketAsyncClient.java @@ -10,10 +10,11 @@ import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; import com.azure.core.annotation.ServiceClient; -import java.util.Objects; import reactor.core.publisher.Mono; import reactor.netty.http.client.HttpClient; +import java.util.Objects; + /** * An asynchronous client for opening native GPT Live sessions with persisted Foundry Live voice agents. * Create instances with {@link AgentsClientBuilder#beta()}. @@ -29,6 +30,7 @@ public final class BetaVoiceAgentsLiveWebSocketAsyncClient { /** * Opens a WebSocket session using the persisted Live agent configuration. + * * @param agentName the Live voice agent name. * @return the connected session. Each subscription opens a new connection. */ @@ -39,6 +41,7 @@ public Mono openWebSocketSession /** * Opens a WebSocket session. Foundry sends the initial {@code session.start}; consume {@code session.started} * from the event stream before sending application events. No Realtime subprotocol is negotiated. + * * @param agentName the Live voice agent name. * @param options the connection options, copied when this method is called. * @return the connected session. Each subscription opens a new connection. diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java index 247bbcd70a55b..9044c117f70a1 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketClient.java @@ -30,6 +30,7 @@ public final class BetaVoiceAgentsLiveWebSocketClient { /** * Opens a WebSocket session using the persisted Live agent configuration. + * * @param agentName the Live voice agent name. * @return the connected session. */ @@ -40,6 +41,7 @@ public BetaVoiceAgentsLiveWebSocketSessionClient openWebSocketSession(String age /** * Opens a WebSocket session. Foundry sends the initial {@code session.start}; consume {@code session.started} * from the event stream before sending application events. No Realtime subprotocol is negotiated. + * * @param agentName the Live voice agent name. * @param options the connection options, copied before connecting. * @return the connected session. diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionAsyncClient.java index 66111ef660c6f..352e9034d3723 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionAsyncClient.java @@ -11,11 +11,12 @@ import com.azure.core.util.BinaryData; import com.openai.models.live.ClientEvent; import com.openai.models.live.ServerEvent; -import java.util.Objects; -import java.util.function.Consumer; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import java.util.Objects; +import java.util.function.Consumer; + /** * An asynchronous bidirectional native GPT Live session. *

    @@ -36,6 +37,7 @@ public final class BetaVoiceAgentsLiveWebSocketSessionAsyncClient implements Asy /** * Gets the WebSocket endpoint. It may contain structured inputs; do not log sensitive query values. + * * @return the endpoint. */ public String getEndpoint() { @@ -44,6 +46,7 @@ public String getEndpoint() { /** * Determines whether the transport is open, not whether {@code session.started} has been received. + * * @return whether the WebSocket is open. */ public boolean isOpen() { @@ -52,6 +55,7 @@ public boolean isOpen() { /** * Gets the peer close code. + * * @return the code, or null if no close frame has been received. */ public Integer getCloseCode() { @@ -60,6 +64,7 @@ public Integer getCloseCode() { /** * Gets the peer close reason. + * * @return the reason, or null if no close frame has been received. */ public String getCloseReason() { @@ -69,6 +74,7 @@ public String getCloseReason() { /** * Receives native Live server events in wire order, including unknown events. * Messages are not parsed or validated as JSON. Transport errors terminate the stream. + * * @return the single-subscriber stream of raw messages. */ public Flux receiveRawEvents() { @@ -78,6 +84,7 @@ public Flux receiveRawEvents() { /** * Receives typed native Live server events. Typed and raw receivers share the single-subscriber limit. * Use {@link #receiveRawEvents()} when the original JSON representation is needed. + * * @return the typed server event stream. */ public Flux receiveEvents() { @@ -89,6 +96,7 @@ public Flux receiveEvents() { /** * Sends a typed native Live command. Foundry supplies {@code session.start}; clients must not send it. + * * @param event the command to send. * @return completion after the frame is written. */ @@ -100,6 +108,7 @@ public Mono sendEvent(ClientEvent event) { /** * Sends a native Live JSON event. Foundry supplies {@code session.start}; clients must not send it. * The service validates native command semantics, including delegation restrictions. + * * @param event the complete JSON object. * @return completion after the frame is written, or an error for invalid JSON or a closed session. */ @@ -109,6 +118,7 @@ public Mono sendEvent(BinaryData event) { /** * Closes the WebSocket normally. + * * @return completion when the session is closed. */ @Override @@ -118,6 +128,7 @@ public Mono closeAsync() { /** * Closes the WebSocket with the specified close frame. The first asynchronous close request wins. + * * @param code a valid WebSocket close code. * @param reason a non-null reason of at most 123 UTF-8 bytes. * @return completion when the session is closed. @@ -127,7 +138,9 @@ public Mono closeAsync(int code, String reason) { return transport.closeAsync(code, reason); } - /** Closes the session, blocking until closure or the configured close timeout. */ + /** + * Closes the session, blocking until closure or the configured close timeout. + */ @Override public void close() { transport.close(); diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java index beaffa6f1fc9d..0607e5e888035 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsLiveWebSocketSessionClient.java @@ -11,6 +11,7 @@ import com.azure.core.util.IterableStream; import com.openai.models.live.ClientEvent; import com.openai.models.live.ServerEvent; + import java.time.Duration; import java.util.Objects; import java.util.function.Consumer; @@ -35,6 +36,7 @@ public final class BetaVoiceAgentsLiveWebSocketSessionClient implements AutoClos /** * Gets the WebSocket endpoint. It may contain structured inputs; do not log sensitive query values. + * * @return the endpoint. */ public String getEndpoint() { @@ -43,6 +45,7 @@ public String getEndpoint() { /** * Determines whether the transport is open, not whether {@code session.started} has been received. + * * @return whether the WebSocket is open. */ public boolean isOpen() { @@ -51,6 +54,7 @@ public boolean isOpen() { /** * Gets the peer close code. + * * @return the code, or null if no close frame has been received. */ public Integer getCloseCode() { @@ -59,6 +63,7 @@ public Integer getCloseCode() { /** * Gets the peer close reason. + * * @return the reason, or null if no close frame has been received. */ public String getCloseReason() { @@ -68,6 +73,7 @@ public String getCloseReason() { /** * Receives native Live server events in wire order, including unknown events. * Messages are not parsed or validated as JSON. Transport errors terminate iteration. + * * @return the single-consumer stream of raw messages. * @throws IllegalStateException if the receive stream was already claimed. */ @@ -77,6 +83,7 @@ public IterableStream receiveRawEvents() { /** * Receives typed native Live server events. Typed and raw receivers share the single-consumer limit. + * * @return the typed server event stream. */ public IterableStream receiveEvents() { @@ -86,6 +93,7 @@ public IterableStream receiveEvents() { /** * Receives typed native Live events with a timeout per wait. Timeouts leave the session open. * Use {@link #receiveRawEvents(Duration)} when the original JSON representation is needed. + * * @param timeout a positive timeout, or null to wait indefinitely. * @return the single-consumer typed event stream. * @throws IllegalArgumentException if the timeout is zero or negative. @@ -102,6 +110,7 @@ public IterableStream receiveEvents(Duration timeout) { * Sends a typed native Live command. Foundry supplies {@code session.start}; clients must not send it. * This method enqueues the event; returning does not mean the frame has been written to the network. * Later network failures surface through the receive stream and session lifecycle. + * * @param event the command to send. * @throws IllegalStateException if the session cannot accept the event. */ @@ -113,6 +122,7 @@ public void sendEvent(ClientEvent event) { /** * Receives raw Live messages without JSON decoding, with a timeout per wait. * Timeouts leave the session open so iteration can resume. + * * @param timeout a positive timeout, or null to wait indefinitely. * @return the single-consumer event stream. * @throws IllegalArgumentException if the timeout is zero or negative. @@ -127,6 +137,7 @@ public IterableStream receiveRawEvents(Duration timeout) { * The service validates native command semantics, including delegation restrictions. * This method enqueues the event; returning does not mean the frame has been written to the network. * Later network failures surface through the receive stream and session lifecycle. + * * @param event the complete JSON object. * @throws IllegalArgumentException if the event is not a JSON object. * @throws IllegalStateException if the session cannot accept the event. @@ -135,7 +146,9 @@ public void sendEvent(BinaryData event) { transport.sendEvent(event); } - /** Closes the WebSocket normally. */ + /** + * Closes the WebSocket normally. + */ @Override public void close() { transport.close(); @@ -143,6 +156,7 @@ public void close() { /** * Closes the WebSocket with the specified close frame. + * * @param code a valid WebSocket close code. * @param reason a non-null reason of at most 123 UTF-8 bytes. * @throws IllegalArgumentException if the code or reason is invalid. diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketClient.java index c3c6fe7da459f..b895f9de9f48e 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketClient.java @@ -42,8 +42,8 @@ public BetaVoiceAgentsWebSocketSessionClient openWebSocketSession(String agentNa * * @param agentName the voice agent name. * @param options connection options. - * @throws IllegalArgumentException if {@code agentName} is empty. * @return a connected session. + * @throws IllegalArgumentException if {@code agentName} is empty. */ public BetaVoiceAgentsWebSocketSessionClient openWebSocketSession(String agentName, VoiceAgentWebSocketConnectionOptions options) { diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java index 1f399eb550e69..b5c58bacf8bbd 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionAsyncClient.java @@ -7,27 +7,28 @@ import com.azure.ai.agents.implementation.realtime.WebSocketSessionAsyncTransport; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.RealtimeClientEvent; +import com.azure.ai.agents.models.RealtimeConversationItem; import com.azure.ai.agents.models.RealtimeConversationItemCreateEvent; +import com.azure.ai.agents.models.RealtimeConversationItemFunctionCallOutput; +import com.azure.ai.agents.models.RealtimeConversationItemUserMessage; import com.azure.ai.agents.models.RealtimeInputAudioBufferAppendEvent; import com.azure.ai.agents.models.RealtimeInputAudioBufferClearEvent; import com.azure.ai.agents.models.RealtimeInputAudioBufferCommitEvent; import com.azure.ai.agents.models.RealtimeResponseCancelEvent; import com.azure.ai.agents.models.RealtimeResponseCreateEvent; -import com.azure.ai.agents.models.RealtimeConversationItem; -import com.azure.ai.agents.models.RealtimeConversationItemFunctionCallOutput; -import com.azure.ai.agents.models.RealtimeConversationItemUserMessage; import com.azure.ai.agents.models.RealtimeServerEvent; import com.azure.ai.agents.models.VoiceAgentResponseCreateOptions; import com.azure.core.util.AsyncCloseable; import com.azure.core.util.BinaryData; import com.openai.models.realtime.RealtimeConversationItemUserMessage.Content; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + import java.io.IOException; import java.util.Base64; import java.util.Collections; import java.util.Objects; import java.util.function.Consumer; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; /** * An asynchronous bidirectional realtime session connected to a Foundry voice agent. @@ -113,6 +114,7 @@ public Mono sendEvent(RealtimeClientEvent event) { /** * Sends a JSON object, including event types and fields unknown to this SDK. + * * @param event the complete JSON event. * @return completion after the frame is written, or an error for invalid JSON or a closed session. */ @@ -250,6 +252,7 @@ public Mono closeAsync() { /** * Closes the session with an application-selected WebSocket close frame. * The first requested close frame wins when close is called more than once. + * * @param code a valid WebSocket close code. * @param reason non-null reason of at most 123 UTF-8 bytes. * @return a completion signal, or an error if the code or reason is invalid. diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java index df8193e324de1..422090b0aae86 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentsWebSocketSessionClient.java @@ -7,21 +7,22 @@ import com.azure.ai.agents.implementation.realtime.WebSocketSessionTransport; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.ai.agents.models.RealtimeClientEvent; +import com.azure.ai.agents.models.RealtimeConversationItem; import com.azure.ai.agents.models.RealtimeConversationItemCreateEvent; +import com.azure.ai.agents.models.RealtimeConversationItemFunctionCallOutput; +import com.azure.ai.agents.models.RealtimeConversationItemUserMessage; import com.azure.ai.agents.models.RealtimeInputAudioBufferAppendEvent; import com.azure.ai.agents.models.RealtimeInputAudioBufferClearEvent; import com.azure.ai.agents.models.RealtimeInputAudioBufferCommitEvent; import com.azure.ai.agents.models.RealtimeResponseCancelEvent; import com.azure.ai.agents.models.RealtimeResponseCreateEvent; -import com.azure.ai.agents.models.RealtimeConversationItem; -import com.azure.ai.agents.models.RealtimeConversationItemFunctionCallOutput; -import com.azure.ai.agents.models.RealtimeConversationItemUserMessage; import com.azure.ai.agents.models.RealtimeServerEvent; import com.azure.ai.agents.models.VoiceAgentResponseCreateOptions; import com.azure.core.util.BinaryData; -import com.openai.models.realtime.RealtimeConversationItemUserMessage.Content; import com.azure.core.util.IterableStream; import com.azure.core.util.logging.ClientLogger; +import com.openai.models.realtime.RealtimeConversationItemUserMessage.Content; + import java.io.IOException; import java.time.Duration; import java.util.Base64; @@ -83,8 +84,8 @@ public String getCloseReason() { /** * Receives typed server events in wire order. The returned stream may be iterated once. * - * @throws IllegalStateException if the event stream has already been claimed. * @return the server event stream. + * @throws IllegalStateException if the event stream has already been claimed. */ public IterableStream receiveEvents() { return receiveEvents(null); @@ -92,6 +93,7 @@ public IterableStream receiveEvents() { /** * Receives events with a timeout for each wait. A timeout leaves the session open and the iterator can be retried. + * * @param timeout positive per-event timeout, or null to wait indefinitely. * @return the single-consumer event stream. * @throws IllegalArgumentException if timeout is zero or negative. @@ -107,8 +109,8 @@ public IterableStream receiveEvents(Duration timeout) { * Sends a typed realtime client event. * * @param event the event to send. - * @throws IllegalArgumentException if the event cannot be serialized. - * @throws IllegalStateException if the session is closed or cannot accept another event. + * @throws IllegalArgumentException if the event cannot be serialized. + * @throws IllegalStateException if the session is closed or cannot accept another event. */ public void sendEvent(RealtimeClientEvent event) { Objects.requireNonNull(event, "'event' cannot be null."); @@ -122,6 +124,7 @@ public void sendEvent(RealtimeClientEvent event) { /** * Sends a JSON object, including event types and fields unknown to this SDK. + * * @param event the complete JSON event. * @throws IllegalArgumentException if the event is not a JSON object. * @throws IllegalStateException if the session cannot accept the event. @@ -171,17 +174,23 @@ public void appendInputAudio(BinaryData audio) { sendEvent(new RealtimeInputAudioBufferAppendEvent(Base64.getEncoder().encodeToString(audio.toBytes()))); } - /** Clears the input audio buffer. */ + /** + * Clears the input audio buffer. + */ public void clearInputAudio() { sendEvent(new RealtimeInputAudioBufferClearEvent()); } - /** Commits the input audio buffer. */ + /** + * Commits the input audio buffer. + */ public void commitInputAudio() { sendEvent(new RealtimeInputAudioBufferCommitEvent()); } - /** Requests a response using the voice agent's configuration. */ + /** + * Requests a response using the voice agent's configuration. + */ public void createResponse() { sendEvent(new RealtimeResponseCreateEvent()); } @@ -196,7 +205,9 @@ public void createResponse(VoiceAgentResponseCreateOptions responseOptions) { sendEvent(new RealtimeResponseCreateEvent().setResponse(responseOptions)); } - /** Cancels the active response. */ + /** + * Cancels the active response. + */ public void cancelResponse() { sendEvent(new RealtimeResponseCancelEvent()); } @@ -222,7 +233,9 @@ public void sendFunctionCallOutput(String callId, String output) { createResponse(); } - /** Closes the session. */ + /** + * Closes the session. + */ @Override public void close() { transport.close(); @@ -230,6 +243,7 @@ public void close() { /** * Closes the session with an application-selected WebSocket close frame. + * * @param code a valid WebSocket close code. * @param reason non-null reason of at most 123 UTF-8 bytes. * @throws IllegalArgumentException if the code or reason is invalid. diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java index d123c4d966303..3b154026caa09 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java @@ -80,7 +80,7 @@ public String getUserAgent() { /** * Gets safe additional headers. * - * @return safe additional headers, or {@code null}. + * @return safe additional headers, or {@code null}. */ public HttpHeaders getHeaders() { return headers; diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java index 9487420a34e97..d444e9e50a283 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java @@ -6,8 +6,8 @@ import com.azure.ai.agents.implementation.models.AgentDefinitionOptInKeys; import com.azure.ai.agents.models.RawRealtimeServerEvent; import com.azure.ai.agents.models.RealtimeServerEvent; -import com.azure.ai.agents.models.VoiceAgentTransport; import com.azure.ai.agents.models.VoiceAgentLiveWebSocketConnectionOptions; +import com.azure.ai.agents.models.VoiceAgentTransport; import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; import com.azure.core.credential.TokenRequestContext; import com.azure.core.http.HttpHeader; @@ -20,6 +20,8 @@ import com.azure.json.JsonProviders; import com.azure.json.JsonReader; import com.azure.json.JsonToken; +import reactor.core.publisher.Flux; + import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URI; @@ -28,12 +30,11 @@ import java.nio.charset.CharacterCodingException; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; -import java.util.Locale; import java.util.Iterator; +import java.util.Locale; import java.util.NoSuchElementException; import java.util.Objects; import java.util.function.Consumer; -import reactor.core.publisher.Flux; public final class VoiceAgentWebSocketUtils { private static final ClientLogger LOGGER = new ClientLogger(VoiceAgentWebSocketUtils.class); diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java index fb41390c789f9..2e867c929c615 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionAsyncTransport.java @@ -19,6 +19,17 @@ import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; import io.netty.handler.codec.http.websocketx.WebSocketClientHandshakeException; import io.netty.handler.codec.http.websocketx.WebSocketFrame; +import reactor.core.Disposable; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.publisher.Sinks; +import reactor.netty.Connection; +import reactor.netty.http.client.HttpClient; +import reactor.netty.http.client.WebsocketClientSpec; +import reactor.netty.http.websocket.WebsocketInbound; +import reactor.netty.http.websocket.WebsocketOutbound; +import reactor.netty.transport.ProxyProvider; + import java.io.IOException; import java.net.URI; import java.time.Duration; @@ -30,16 +41,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; -import reactor.core.Disposable; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; -import reactor.core.publisher.Sinks; -import reactor.netty.Connection; -import reactor.netty.http.client.HttpClient; -import reactor.netty.http.client.WebsocketClientSpec; -import reactor.netty.http.websocket.WebsocketInbound; -import reactor.netty.http.websocket.WebsocketOutbound; -import reactor.netty.transport.ProxyProvider; /** * Shared asynchronous WebSocket connection and raw message I/O. diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java index 43ccdbdb5c9a5..1992a3fcc74f7 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/WebSocketSessionTransport.java @@ -13,6 +13,14 @@ import com.azure.core.util.BinaryData; import com.azure.core.util.IterableStream; import com.azure.core.util.logging.ClientLogger; +import okhttp3.Credentials; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.WebSocket; +import okhttp3.WebSocketListener; +import okio.ByteString; + import java.io.IOException; import java.net.Proxy; import java.net.ProxySelector; @@ -33,15 +41,8 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Pattern; import java.util.function.Function; -import okhttp3.Credentials; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.WebSocket; -import okhttp3.WebSocketListener; -import okio.ByteString; +import java.util.regex.Pattern; /** * Shared blocking WebSocket connection and I/O, independent of the asynchronous transport. diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java index 12af0b3e2b939..06bc43c606477 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentLiveWebSocketConnectionOptions.java @@ -6,6 +6,7 @@ import com.azure.ai.agents.implementation.utils.Beta; import com.azure.core.annotation.Fluent; import com.azure.core.util.logging.ClientLogger; + import java.time.Duration; import java.util.Objects; import java.util.function.Consumer; @@ -28,12 +29,15 @@ public final class VoiceAgentLiveWebSocketConnectionOptions { private String agentVersionOverride; private String structuredInputs; - /** Creates options with a 30-second handshake timeout and a 256-event receive buffer. */ + /** + * Creates options with a 30-second handshake timeout and a 256-event receive buffer. + */ public VoiceAgentLiveWebSocketConnectionOptions() { } /** * Copies connection options. + * * @param source the options to copy. */ public VoiceAgentLiveWebSocketConnectionOptions(VoiceAgentLiveWebSocketConnectionOptions source) { @@ -51,6 +55,7 @@ public VoiceAgentLiveWebSocketConnectionOptions(VoiceAgentLiveWebSocketConnectio /** * Gets the receive queue capacity. + * * @return the maximum queued events, default 256. */ public int getReceiveBufferCapacity() { @@ -59,6 +64,7 @@ public int getReceiveBufferCapacity() { /** * Sets the receive queue capacity. + * * @param capacity the capacity, between 1 and 65536. * @return this options instance. * @throws IllegalArgumentException if the capacity is outside the supported range. @@ -74,6 +80,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setReceiveBufferCapacity(int cap /** * Gets the maximum accepted message size. + * * @return the size in bytes, default 32 MiB. */ public int getMaxMessageSize() { @@ -82,6 +89,7 @@ public int getMaxMessageSize() { /** * Sets the maximum accepted message size. + * * @param bytes the positive maximum message size in bytes. * @return this options instance. * @throws IllegalArgumentException if the size is not positive. @@ -96,6 +104,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setMaxMessageSize(int bytes) { /** * Gets the receive queue overflow strategy. + * * @return the strategy, default ERROR. */ public VoiceAgentWebSocketOverflowStrategy getOverflowStrategy() { @@ -104,6 +113,7 @@ public VoiceAgentWebSocketOverflowStrategy getOverflowStrategy() { /** * Sets the overflow strategy. Drop strategies explicitly permit event loss. + * * @param strategy the non-null strategy. * @return this options instance. */ @@ -114,6 +124,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setOverflowStrategy(VoiceAgentWe /** * Gets the callback for malformed events. + * * @return the callback, or null to terminate the session. */ public Consumer getMalformedEventHandler() { @@ -121,10 +132,11 @@ public Consumer getMalformedEventHandler() { } /** - * Sets a callback to report and skip malformed events. It runs while receiving events and must not block. - * JSON decoding failures are reported only by typed receivers; raw receivers do not parse JSON. - * Invalid UTF-8 is reported by the transport for either receiver type. + * Sets a callback to report and skip malformed events. It runs while receiving events and must not block. + * JSON decoding failures are reported only by typed receivers; raw receivers do not parse JSON. + * Invalid UTF-8 is reported by the transport for either receiver type. * If the callback throws, the session terminates. Transport errors and oversized messages are not skipped. + * * @param handler the callback, or null to terminate on malformed events. * @return this options instance. */ @@ -135,6 +147,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setMalformedEventHandler(Consume /** * Gets the handshake timeout. + * * @return the timeout, default 30 seconds. */ public Duration getHandshakeTimeout() { @@ -143,6 +156,7 @@ public Duration getHandshakeTimeout() { /** * Sets the handshake timeout. + * * @param timeout a non-null positive timeout. * @return this options instance. * @throws IllegalArgumentException if the timeout is not positive. @@ -155,6 +169,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setHandshakeTimeout(Duration tim /** * Gets the close timeout. + * * @return the timeout, default 10 seconds. */ public Duration getCloseTimeout() { @@ -163,6 +178,7 @@ public Duration getCloseTimeout() { /** * Sets the close timeout. + * * @param timeout a non-null positive timeout. * @return this options instance. * @throws IllegalArgumentException if the timeout is not positive. @@ -175,6 +191,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setCloseTimeout(Duration timeout /** * Gets the persistence override. + * * @return the override, or null to use the persisted agent configuration. */ public Boolean isStoreEnabled() { @@ -183,6 +200,7 @@ public Boolean isStoreEnabled() { /** * Overrides Foundry conversation and audio persistence for this session. + * * @param store the override, or null to use the persisted agent configuration. * @return this options instance. */ @@ -193,6 +211,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setStoreEnabled(Boolean store) { /** * Gets the agent version override. + * * @return the version override, or null. */ public String getAgentVersionOverride() { @@ -201,6 +220,7 @@ public String getAgentVersionOverride() { /** * Sets the agent version to connect to. + * * @param version the version override, or null to use the default version. * @return this options instance. */ @@ -211,6 +231,7 @@ public VoiceAgentLiveWebSocketConnectionOptions setAgentVersionOverride(String v /** * Gets structured inputs for instruction and greeting templates. + * * @return the JSON inputs, or null. */ public String getStructuredInputs() { @@ -220,6 +241,7 @@ public String getStructuredInputs() { /** * Sets the JSON inputs for instruction and greeting templates. * The SDK URL-encodes this value into the {@code structured_inputs} query parameter. + * * @param inputs the unencoded JSON inputs, or null. * @return this options instance. */ diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentWebSocketConnectionOptions.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentWebSocketConnectionOptions.java index 8b392338f9a1b..e8bee1e5f2dcf 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentWebSocketConnectionOptions.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentWebSocketConnectionOptions.java @@ -6,6 +6,7 @@ import com.azure.ai.agents.implementation.utils.Beta; import com.azure.core.annotation.Fluent; import com.azure.core.util.logging.ClientLogger; + import java.time.Duration; import java.util.Objects; import java.util.function.Consumer; @@ -24,6 +25,7 @@ public final class VoiceAgentWebSocketConnectionOptions { /** * Gets the maximum number of queued events. + * * @return the capacity, default 256. */ public int getReceiveBufferCapacity() { @@ -32,9 +34,10 @@ public int getReceiveBufferCapacity() { /** * Sets the bounded receive queue capacity. Configure before connecting. + * * @param capacity number of events, between 1 and 65536. * @return this options instance. - * @throws IllegalArgumentException if capacity is outside the supported range. + * @throws IllegalArgumentException if capacity is outside the supported range. */ public VoiceAgentWebSocketConnectionOptions setReceiveBufferCapacity(int capacity) { if (capacity < 1 || capacity > 65536) { @@ -47,6 +50,7 @@ public VoiceAgentWebSocketConnectionOptions setReceiveBufferCapacity(int capacit /** * Gets the maximum accepted JSON message size. + * * @return the size in bytes, default 32 MiB. */ public int getMaxMessageSize() { @@ -55,9 +59,10 @@ public int getMaxMessageSize() { /** * Sets the maximum accepted JSON message size. Oversized messages terminate the connection. + * * @param bytes positive size in bytes. * @return this options instance. - * @throws IllegalArgumentException if bytes is not positive. + * @throws IllegalArgumentException if bytes is not positive. */ public VoiceAgentWebSocketConnectionOptions setMaxMessageSize(int bytes) { if (bytes <= 0) { @@ -69,6 +74,7 @@ public VoiceAgentWebSocketConnectionOptions setMaxMessageSize(int bytes) { /** * Gets the receive queue overflow action. + * * @return the strategy, default ERROR. */ public VoiceAgentWebSocketOverflowStrategy getOverflowStrategy() { @@ -77,6 +83,7 @@ public VoiceAgentWebSocketOverflowStrategy getOverflowStrategy() { /** * Sets the receive queue overflow action. Drop strategies explicitly permit data loss. + * * @param strategy the non-null strategy. * @return this options instance. */ @@ -87,6 +94,7 @@ public VoiceAgentWebSocketConnectionOptions setOverflowStrategy(VoiceAgentWebSoc /** * Gets the callback for skipping malformed events. + * * @return the callback, or null to terminate on malformed events. */ public Consumer getMalformedEventHandler() { @@ -97,6 +105,7 @@ public Consumer getMalformedEventHandler() { * Sets a callback that reports and skips malformed events without terminating reception. * The callback runs on the receive thread and must not block. If it throws, the session terminates. * This does not recover from transport errors or oversized messages. + * * @param handler callback, or null to terminate on malformed events (the default). * @return this options instance. */ @@ -142,6 +151,7 @@ public VoiceAgentWebSocketConnectionOptions(VoiceAgentWebSocketConnectionOptions /** * Gets the session correlation identifier. + * * @return the session identifier, or null. */ public String getAgentSessionId() { @@ -150,6 +160,7 @@ public String getAgentSessionId() { /** * Sets the session correlation identifier. + * * @param agentSessionId the session identifier, or null. * @return this options instance. */ @@ -160,6 +171,7 @@ public VoiceAgentWebSocketConnectionOptions setAgentSessionId(String agentSessio /** * Gets the structured inputs JSON object sent in the handshake header. + * * @return the structured inputs, or null. */ public String getStructuredInputs() { @@ -168,6 +180,7 @@ public String getStructuredInputs() { /** * Sets the structured inputs JSON object sent in the handshake header. + * * @param structuredInputs the JSON object, or null. * @return this options instance. */ From 621f1aedb5857f596a63bb3e74a1cdc0d4282ff5 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Thu, 5 Nov 2026 10:49:26 +0800 Subject: [PATCH 19/20] Regenerate Checkstyle suppressions after GPT Live rebase --- sdk/ai/azure-ai-agents/checkstyle-suppressions.xml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml b/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml index ca4ab791025a0..31911326b715b 100644 --- a/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml +++ b/sdk/ai/azure-ai-agents/checkstyle-suppressions.xml @@ -3,13 +3,9 @@ + - - - - - @@ -47,11 +43,13 @@ + + From c4e6a2e66b0ac01ec605b7062c88e7f297b66929 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Thu, 5 Nov 2026 10:54:13 +0800 Subject: [PATCH 20/20] Adapt GPT Live samples to regenerated definition and delegation APIs --- .../java/com/azure/ai/agents/voice/GptLiveSampleUtils.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java index dd0793132d320..6adda9a28bab6 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/GptLiveSampleUtils.java @@ -7,12 +7,11 @@ import com.azure.ai.agents.AgentsClientBuilder; import com.azure.ai.agents.models.CreateAgentVersionInput; import com.azure.ai.agents.models.LiveInitialSessionAudioParam; -import com.azure.ai.agents.models.LiveResponsesDelegationParam; import com.azure.ai.agents.models.LiveResponsesDelegationSettingsInputParam; import com.azure.ai.agents.models.LiveSessionAudioFormatPCMParam; import com.azure.ai.agents.models.LiveSessionAudioFormatPCMParamRate; import com.azure.ai.agents.models.VoiceAgentLiveDefinition; -import com.azure.ai.agents.models.VoiceAgentSubProtocol; +import com.azure.ai.agents.models.VoiceAgentLiveResponsesDelegation; import com.azure.core.util.BinaryData; import com.azure.core.util.Configuration; import com.openai.core.JsonValue; @@ -657,11 +656,11 @@ static AgentResource createAgent(AgentsClientBuilder builder, String model, Stri if (tools) { responses.setTools(java.util.Collections.singletonList(BinaryData.fromObject(GptLiveFunctionToolSample.weatherTool()))); } - VoiceAgentLiveDefinition definition = new VoiceAgentLiveDefinition(VoiceAgentSubProtocol.LIVE, model) + VoiceAgentLiveDefinition definition = new VoiceAgentLiveDefinition(model) .setInstructions("You are a helpful voice assistant. Delegate requests to your Responses backend.") .setAudio(new LiveInitialSessionAudioParam().setFormat(new LiveSessionAudioFormatPCMParam( LiveSessionAudioFormatPCMParamRate.TWO_FOUR_ZERO_ZERO_ZERO))) - .setDelegation(BinaryData.fromObject(new LiveResponsesDelegationParam(responses))); + .setDelegation(new VoiceAgentLiveResponsesDelegation(responses)); String agentName = createAgentName(scenario); AgentsClient agents = builder.buildAgentsClient(); agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition));