Skip to content

Commit a1a9384

Browse files
kvmiloscopybara-github
authored andcommitted
fix(live): replay stored transcriptions as conversation history
PiperOrigin-RevId: 1000843268
1 parent 97f22a2 commit a1a9384

3 files changed

Lines changed: 792 additions & 51 deletions

File tree

‎core/src/main/java/com/google/adk/flows/llmflows/Contents.java‎

Lines changed: 175 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import com.google.genai.types.FunctionCall;
3636
import com.google.genai.types.FunctionResponse;
3737
import com.google.genai.types.Part;
38+
import com.google.genai.types.Transcription;
3839
import io.reactivex.rxjava3.core.Single;
3940
import java.util.ArrayList;
4041
import java.util.Collections;
@@ -111,14 +112,54 @@ private ImmutableList<Content> getCurrentTurnContents(
111112
// Find the latest event that starts the current turn and process from there.
112113
for (int i = events.size() - 1; i >= 0; i--) {
113114
Event event = events.get(i);
114-
if (event.author().equals(Role.USER) || isOtherAgentReply(agentName, event)) {
115+
TranscriptionKind kind = transcriptionKind(event);
116+
int start = -1;
117+
if (kind != null) {
118+
start = spokenTurnStart(currentBranch, events, agentName, i, kind);
119+
} else if (shouldIncludeEventInContext(currentBranch, event)
120+
&& (event.author().equals(Role.USER) || isOtherAgentReply(agentName, event))) {
121+
start = i;
122+
}
123+
if (start >= 0) {
115124
return getContents(
116-
currentBranch, events.subList(i, events.size()), agentName, groupFunctionResponses);
125+
currentBranch, events.subList(start, events.size()), agentName, groupFunctionResponses);
117126
}
118127
}
119128
return ImmutableList.of();
120129
}
121130

131+
/**
132+
* Returns where the spoken run ending at {@code end} starts, so the turn keeps the whole run
133+
* rather than its last chunk. Returns -1 when the run does not start a turn: the agent's own
134+
* speech, a chunk outside the context, or a blank run, which the merge drops.
135+
*/
136+
private int spokenTurnStart(
137+
@Nullable String currentBranch,
138+
List<Event> events,
139+
String agentName,
140+
int end,
141+
TranscriptionKind kind) {
142+
Event event = events.get(end);
143+
boolean startsTurn = kind == TranscriptionKind.INPUT || isOtherAgentReply(agentName, event);
144+
if (!startsTurn || !shouldIncludeEventInContext(currentBranch, event)) {
145+
return -1;
146+
}
147+
int start = end;
148+
boolean blank = kind.text(event).isBlank();
149+
for (int i = end - 1; i >= 0; i--) {
150+
// Events left out of the context do not break the run.
151+
if (!shouldIncludeEventInContext(currentBranch, events.get(i))) {
152+
continue;
153+
}
154+
if (!continuesRun(kind, events.get(start), events.get(i))) {
155+
break;
156+
}
157+
start = i;
158+
blank &= kind.text(events.get(i)).isBlank();
159+
}
160+
return blank ? -1 : start;
161+
}
162+
122163
private ImmutableList<Content> getContents(
123164
@Nullable String currentBranch,
124165
List<Event> events,
@@ -127,47 +168,31 @@ private ImmutableList<Content> getContents(
127168
List<Event> filteredEvents = new ArrayList<>();
128169
boolean hasCompactEvent = false;
129170

130-
// Filter the events, leaving the contents and the function calls and responses from the current
131-
// agent.
132171
for (Event event : events) {
133-
if (event.actions().compaction().isPresent()) {
134-
// Always include the compaction event for the later processCompactionEvent call.
135-
// The compaction event is used to filter out normal events that are covered by the
136-
// compaction event.
137-
hasCompactEvent = true;
172+
if (shouldIncludeEventInContext(currentBranch, event)) {
173+
hasCompactEvent |= event.actions().compaction().isPresent();
138174
filteredEvents.add(event);
139-
continue;
140-
}
141-
142-
// Skip events without content, or generated neither by user nor by model or has empty text.
143-
// E.g. events purely for mutating session states.
144-
if (isEmptyContent(event)) {
145-
continue;
146-
}
147-
if (!isEventBelongsToBranch(currentBranch, event)) {
148-
continue;
149-
}
150-
if (isRequestConfirmationEvent(event)) {
151-
continue;
152175
}
176+
}
153177

154-
// TODO: Skip auth events.
178+
if (hasCompactEvent) {
179+
filteredEvents = processCompactionEvent(filteredEvents, agentName);
180+
}
155181

182+
// Merge after compaction, which may end mid-run, and before other agents are narrated.
183+
List<Event> resultEvents = new ArrayList<>();
184+
for (Event event : mergeTranscriptions(filteredEvents)) {
156185
if (isOtherAgentReply(agentName, event)) {
157186
Event foreignEvent = convertForeignEvent(event);
158187
if (foreignEvent != null) {
159-
filteredEvents.add(foreignEvent);
188+
resultEvents.add(foreignEvent);
160189
}
161190
} else {
162-
filteredEvents.add(event);
191+
resultEvents.add(event);
163192
}
164193
}
165194

166-
if (hasCompactEvent) {
167-
filteredEvents = processCompactionEvent(filteredEvents);
168-
}
169-
170-
List<Event> resultEvents = rearrangeEventsForLatestFunctionResponse(filteredEvents);
195+
resultEvents = rearrangeEventsForLatestFunctionResponse(resultEvents);
171196
resultEvents =
172197
rearrangeEventsForAsyncFunctionResponsesInHistory(resultEvents, groupFunctionResponses);
173198

@@ -177,11 +202,25 @@ private ImmutableList<Content> getContents(
177202
.collect(toImmutableList());
178203
}
179204

205+
/**
206+
* Whether {@code event} goes into the LLM context. Compaction events always do, so {@link
207+
* #processCompactionEvent} can apply them.
208+
*/
209+
private boolean shouldIncludeEventInContext(@Nullable String currentBranch, Event event) {
210+
if (event.actions().compaction().isPresent()) {
211+
return true;
212+
}
213+
// TODO: Skip auth events.
214+
return !isEmptyContent(event)
215+
&& isEventBelongsToBranch(currentBranch, event)
216+
&& !isRequestConfirmationEvent(event);
217+
}
218+
180219
/**
181220
* Check if an event has missing or empty content.
182221
*
183-
* <p>This can happen to the events that only changed session state. When both content and
184-
* transcriptions are empty, the event will be considered as empty. The content is considered
222+
* <p>This can happen to the events that only changed session state. When the content is empty and
223+
* no transcription has text, the event will be considered as empty. The content is considered
185224
* empty if none of its parts contain text, inline data, file data, function call, function
186225
* response, server-side tool call, or server-side tool response. Parts with only thoughts are
187226
* also considered empty.
@@ -190,15 +229,104 @@ private ImmutableList<Content> getContents(
190229
* @return {@code true} if the event is considered to have empty content, {@code false} otherwise.
191230
*/
192231
private boolean isEmptyContent(Event event) {
193-
if (event.content().isEmpty()) {
194-
return true;
232+
return !hasVisibleContent(event) && transcriptionKind(event) == null;
233+
}
234+
235+
/** Whether {@code event} has a role and at least one part that is visible in the LLM context. */
236+
private boolean hasVisibleContent(Event event) {
237+
return event
238+
.content()
239+
.filter(content -> !content.role().orElse("").isEmpty())
240+
.flatMap(Content::parts)
241+
.map(parts -> parts.stream().anyMatch(part -> !isPartInvisible(part)))
242+
.orElse(false);
243+
}
244+
245+
/**
246+
* Merges each spoken run into one text event, so later invocations replay what was said: input as
247+
* a user turn, output as a model turn. A blank run is dropped.
248+
*/
249+
private ImmutableList<Event> mergeTranscriptions(List<Event> events) {
250+
ImmutableList.Builder<Event> merged = ImmutableList.builder();
251+
StringBuilder text = new StringBuilder();
252+
for (int i = 0; i < events.size(); i++) {
253+
Event event = events.get(i);
254+
TranscriptionKind kind = transcriptionKind(event);
255+
if (kind == null) {
256+
merged.add(event);
257+
continue;
258+
}
259+
text.append(kind.text(event));
260+
if (i + 1 < events.size() && continuesRun(kind, event, events.get(i + 1))) {
261+
continue;
262+
}
263+
// A blank run says nothing, while a space chunk inside a run still joins the words around it.
264+
String runText = text.toString();
265+
if (!runText.isBlank()) {
266+
merged.add(kind.toTextEvent(event, runText));
267+
}
268+
text.setLength(0);
269+
}
270+
return merged.build();
271+
}
272+
273+
/**
274+
* Whether {@code other} belongs to the {@code kind} run that {@code event} is in. An output run
275+
* is one agent's speech; an input run is the user's even when stored under different agents.
276+
*/
277+
private boolean continuesRun(TranscriptionKind kind, Event event, Event other) {
278+
return transcriptionKind(other) == kind
279+
&& (kind == TranscriptionKind.INPUT || event.author().equals(other.author()));
280+
}
281+
282+
/**
283+
* The kind of transcription {@code event} is merged as, or null if it has visible content or no
284+
* transcription text.
285+
*/
286+
private @Nullable TranscriptionKind transcriptionKind(Event event) {
287+
if (hasVisibleContent(event)) {
288+
return null;
289+
}
290+
if (!TranscriptionKind.INPUT.text(event).isEmpty()) {
291+
return TranscriptionKind.INPUT;
292+
}
293+
if (!TranscriptionKind.OUTPUT.text(event).isEmpty()) {
294+
return TranscriptionKind.OUTPUT;
295+
}
296+
return null;
297+
}
298+
299+
/** Which side of a live conversation a transcription records. */
300+
private enum TranscriptionKind {
301+
INPUT,
302+
OUTPUT;
303+
304+
String text(Event event) {
305+
Optional<Transcription> transcription =
306+
switch (this) {
307+
case INPUT -> event.inputTranscription();
308+
case OUTPUT -> event.outputTranscription();
309+
};
310+
return transcription.flatMap(Transcription::text).orElse("");
311+
}
312+
313+
/** The run's last {@code event}, carrying the run's {@code text} instead of a transcription. */
314+
Event toTextEvent(Event event, String text) {
315+
return switch (this) {
316+
// A live run stores the user's speech under the listening agent's name.
317+
case INPUT ->
318+
event.toBuilder()
319+
.author(Role.USER)
320+
.content(Content.builder().role(Role.USER).parts(Part.fromText(text)).build())
321+
.inputTranscription(null)
322+
.build();
323+
case OUTPUT ->
324+
event.toBuilder()
325+
.content(Content.builder().role("model").parts(Part.fromText(text)).build())
326+
.outputTranscription(null)
327+
.build();
328+
};
195329
}
196-
var content = event.content().get();
197-
return (content.role().isEmpty()
198-
|| content.role().get().isEmpty()
199-
|| content.parts().isEmpty()
200-
|| content.parts().get().isEmpty()
201-
|| content.parts().get().stream().allMatch(this::isPartInvisible));
202330
}
203331

204332
/**
@@ -314,9 +442,10 @@ private boolean isPartInvisible(Part part) {
314442
* </pre>
315443
*
316444
* @param events the list of event to filter.
445+
* @param agentName the current agent, which authors each summary.
317446
* @return a new list with compaction applied.
318447
*/
319-
private List<Event> processCompactionEvent(List<Event> events) {
448+
private List<Event> processCompactionEvent(List<Event> events, String agentName) {
320449
// Step 1: Split events into compaction events and regular events.
321450
List<Event> compactionEvents = new ArrayList<>();
322451
List<Event> regularEvents = new ArrayList<>();
@@ -350,13 +479,13 @@ private List<Event> processCompactionEvent(List<Event> events) {
350479
e--;
351480
} else {
352481
// Otherwise the event is before the compaction, let's move to the next compaction event;
353-
result.add(createCompactionEvent(compactionEvents.get(c)));
482+
result.add(createCompactionEvent(compactionEvents.get(c), agentName));
354483
c--;
355484
}
356485
}
357486
// Flush any remaining compactions.
358487
while (c >= 0) {
359-
result.add(createCompactionEvent(compactionEvents.get(c)));
488+
result.add(createCompactionEvent(compactionEvents.get(c), agentName));
360489
c--;
361490
}
362491
// Flush any remaining regular events.
@@ -390,11 +519,12 @@ private static List<Event> removeOverlappingCompactions(List<Event> events) {
390519
return Lists.reverse(result);
391520
}
392521

393-
private static Event createCompactionEvent(Event event) {
522+
private static Event createCompactionEvent(Event event, String agentName) {
394523
EventCompaction compaction = event.actions().compaction().get();
395524
return event.toBuilder()
396525
.timestamp(compaction.endTimestamp())
397-
.author("model")
526+
// Authored by this agent, so convertForeignEvent leaves the summary alone.
527+
.author(agentName.isEmpty() ? "model" : agentName)
398528
.content(compaction.compactedContent())
399529
.build();
400530
}

0 commit comments

Comments
 (0)