From 9868c7b12c296f6ae6e14bf192edbf155f030830 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 18 Nov 2026 18:39:05 -0500 Subject: [PATCH] feat(livekit): trace end-of-utterance waits --- .changeset/livekit-eou-wait.md | 5 + .../livekit-v1-audio.span-tree.json | 3 +- .../livekit-v1-audio.span-tree.txt | 3 +- .../livekit-v1-diarized.span-tree.json | 10 +- .../livekit-v1-diarized.span-tree.txt | 10 +- .../livekit-v1-disabled.span-tree.json | 3 +- .../livekit-v1-disabled.span-tree.txt | 3 +- .../livekit-v1-interrupted.span-tree.json | 3 +- .../livekit-v1-interrupted.span-tree.txt | 3 +- .../livekit-v1-latest-audio.span-tree.json | 3 +- .../livekit-v1-latest-audio.span-tree.txt | 3 +- .../livekit-v1-latest-diarized.span-tree.json | 10 +- .../livekit-v1-latest-diarized.span-tree.txt | 10 +- .../livekit-v1-latest-disabled.span-tree.json | 3 +- .../livekit-v1-latest-disabled.span-tree.txt | 3 +- ...vekit-v1-latest-interrupted.span-tree.json | 3 +- ...ivekit-v1-latest-interrupted.span-tree.txt | 3 +- .../livekit-v1-latest-order.span-tree.json | 3 +- .../livekit-v1-latest-order.span-tree.txt | 3 +- ...livekit-v1-latest-recovered.span-tree.json | 3 +- .../livekit-v1-latest-recovered.span-tree.txt | 3 +- ...v1-latest-streaming-cascade.span-tree.json | 10 +- ...-v1-latest-streaming-cascade.span-tree.txt | 10 +- .../livekit-v1-order.span-tree.json | 3 +- .../livekit-v1-order.span-tree.txt | 3 +- .../livekit-v1-recovered.span-tree.json | 3 +- .../livekit-v1-recovered.span-tree.txt | 3 +- ...ivekit-v1-streaming-cascade.span-tree.json | 10 +- ...livekit-v1-streaming-cascade.span-tree.txt | 10 +- .../livekit-instrumentation/assertions.ts | 2 +- .../livekit-instrumentation/lifecycle.mjs | 176 +++++ .../livekit-instrumentation/scenario.test.ts | 182 ++++- .../auto-instrumentations/configs/livekit.ts | 30 + .../providers/livekit-interceptors.ts | 21 + .../providers/livekit-plugin.test.ts | 681 +++++++++++++++++- .../providers/livekit-plugin.ts | 507 +++++++++++-- js/src/vendor-sdk-types/livekit.ts | 27 + js/src/wrappers/livekit.ts | 25 +- 38 files changed, 1672 insertions(+), 124 deletions(-) create mode 100644 .changeset/livekit-eou-wait.md create mode 100644 e2e/scenarios/livekit-instrumentation/lifecycle.mjs diff --git a/.changeset/livekit-eou-wait.md b/.changeset/livekit-eou-wait.md new file mode 100644 index 000000000..116d04c95 --- /dev/null +++ b/.changeset/livekit-eou-wait.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat(livekit): trace end-of-utterance waits diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.json index 10e8c6628..0b4e97ef8 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.json @@ -165,6 +165,7 @@ } ], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -198,7 +199,7 @@ } }, "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.txt index 83602463d..7c476f8ea 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-audio.span-tree.txt @@ -11,7 +11,7 @@ span_tree: } } metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { @@ -197,6 +197,7 @@ span_tree: } └── livekit.eou_wait [task] metadata: { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-diarized.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-diarized.span-tree.json index 8e70dc183..478ef44d5 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-diarized.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-diarized.span-tree.json @@ -253,16 +253,21 @@ } }, { - "name": "livekit.eou_wait", + "name": "eou_wait", "type": "task", "children": [], "metadata": { + "audio.source_id": "session-input", "contrib.livekit.eou.endpointing_delay": 0, + "contrib.livekit.eou.native_end_unix_ms": 0, + "contrib.livekit.eou.native_start_unix_ms": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", "contrib.livekit.eou.rearm_count": 0, "contrib.livekit.eou.source": "manual", "contrib.livekit.eou.wait_duration": 0, + "eou.outcome": "committed", + "eou.speech_end_source": "vad", "scenario": "livekit-instrumentation", "testRunId": "", "turn.id": "" @@ -321,6 +326,7 @@ "type": "task", "children": [], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -951,7 +957,7 @@ } }, "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-diarized.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-diarized.span-tree.txt index 0e7bb2e5a..97f267bfa 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-diarized.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-diarized.span-tree.txt @@ -11,7 +11,7 @@ span_tree: } } metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { @@ -405,14 +405,19 @@ span_tree: │ │ "testRunId": "", │ │ "turn.id": "" │ │ } - │ ├── livekit.eou_wait [task] + │ ├── eou_wait [task] │ │ metadata: { + │ │ "audio.source_id": "session-input", │ │ "contrib.livekit.eou.endpointing_delay": 0, + │ │ "contrib.livekit.eou.native_end_unix_ms": 0, + │ │ "contrib.livekit.eou.native_start_unix_ms": 0, │ │ "contrib.livekit.eou.not_committed_count": 0, │ │ "contrib.livekit.eou.outcome": "committed", │ │ "contrib.livekit.eou.rearm_count": 0, │ │ "contrib.livekit.eou.source": "manual", │ │ "contrib.livekit.eou.wait_duration": 0, + │ │ "eou.outcome": "committed", + │ │ "eou.speech_end_source": "vad", │ │ "scenario": "livekit-instrumentation", │ │ "testRunId": "", │ │ "turn.id": "" @@ -439,6 +444,7 @@ span_tree: │ } │ └── livekit.eou_wait [task] │ metadata: { + │ "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", │ "contrib.livekit.eou.endpointing_delay": 0, │ "contrib.livekit.eou.not_committed_count": 0, │ "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-disabled.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-disabled.span-tree.json index a9d18c0e3..42adc548f 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-disabled.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-disabled.span-tree.json @@ -165,6 +165,7 @@ } ], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -188,7 +189,7 @@ } ], "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.sources": { "agent-output": { "label": "Agent audio" diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-disabled.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-disabled.span-tree.txt index bc99eeece..37158e8fe 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-disabled.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-disabled.span-tree.txt @@ -1,7 +1,7 @@ span_tree: └── session [task] metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.sources": { "agent-output": { "label": "Agent audio" @@ -152,6 +152,7 @@ span_tree: } └── livekit.eou_wait [task] metadata: { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.json index 305997453..2652f2d57 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.json @@ -544,6 +544,7 @@ } ], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -577,7 +578,7 @@ } }, "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.txt index df75fb160..417635c39 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-interrupted.span-tree.txt @@ -11,7 +11,7 @@ span_tree: } } metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { @@ -538,6 +538,7 @@ span_tree: } └── livekit.eou_wait [task] metadata: { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.json index 10e8c6628..0b4e97ef8 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.json @@ -165,6 +165,7 @@ } ], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -198,7 +199,7 @@ } }, "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.txt index 83602463d..7c476f8ea 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-audio.span-tree.txt @@ -11,7 +11,7 @@ span_tree: } } metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { @@ -197,6 +197,7 @@ span_tree: } └── livekit.eou_wait [task] metadata: { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-diarized.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-diarized.span-tree.json index 2d76ab834..311398f11 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-diarized.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-diarized.span-tree.json @@ -253,16 +253,21 @@ } }, { - "name": "livekit.eou_wait", + "name": "eou_wait", "type": "task", "children": [], "metadata": { + "audio.source_id": "session-input", "contrib.livekit.eou.endpointing_delay": 0, + "contrib.livekit.eou.native_end_unix_ms": 0, + "contrib.livekit.eou.native_start_unix_ms": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", "contrib.livekit.eou.rearm_count": 0, "contrib.livekit.eou.source": "manual", "contrib.livekit.eou.wait_duration": 0, + "eou.outcome": "committed", + "eou.speech_end_source": "vad", "scenario": "livekit-instrumentation", "testRunId": "", "turn.id": "" @@ -393,6 +398,7 @@ "type": "task", "children": [], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -1005,7 +1011,7 @@ } }, "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-diarized.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-diarized.span-tree.txt index eba1ca536..4c9078187 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-diarized.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-diarized.span-tree.txt @@ -11,7 +11,7 @@ span_tree: } } metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { @@ -477,14 +477,19 @@ span_tree: │ │ "testRunId": "", │ │ "turn.id": "" │ │ } - │ ├── livekit.eou_wait [task] + │ ├── eou_wait [task] │ │ metadata: { + │ │ "audio.source_id": "session-input", │ │ "contrib.livekit.eou.endpointing_delay": 0, + │ │ "contrib.livekit.eou.native_end_unix_ms": 0, + │ │ "contrib.livekit.eou.native_start_unix_ms": 0, │ │ "contrib.livekit.eou.not_committed_count": 0, │ │ "contrib.livekit.eou.outcome": "committed", │ │ "contrib.livekit.eou.rearm_count": 0, │ │ "contrib.livekit.eou.source": "manual", │ │ "contrib.livekit.eou.wait_duration": 0, + │ │ "eou.outcome": "committed", + │ │ "eou.speech_end_source": "vad", │ │ "scenario": "livekit-instrumentation", │ │ "testRunId": "", │ │ "turn.id": "" @@ -511,6 +516,7 @@ span_tree: │ } │ └── livekit.eou_wait [task] │ metadata: { + │ "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", │ "contrib.livekit.eou.endpointing_delay": 0, │ "contrib.livekit.eou.not_committed_count": 0, │ "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-disabled.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-disabled.span-tree.json index a9d18c0e3..42adc548f 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-disabled.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-disabled.span-tree.json @@ -165,6 +165,7 @@ } ], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -188,7 +189,7 @@ } ], "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.sources": { "agent-output": { "label": "Agent audio" diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-disabled.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-disabled.span-tree.txt index bc99eeece..37158e8fe 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-disabled.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-disabled.span-tree.txt @@ -1,7 +1,7 @@ span_tree: └── session [task] metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.sources": { "agent-output": { "label": "Agent audio" @@ -152,6 +152,7 @@ span_tree: } └── livekit.eou_wait [task] metadata: { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.json index 10d9f796b..918511ba2 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.json @@ -526,6 +526,7 @@ } ], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -559,7 +560,7 @@ } }, "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.txt index 0a406977c..234d6b361 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-interrupted.span-tree.txt @@ -11,7 +11,7 @@ span_tree: } } metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { @@ -520,6 +520,7 @@ span_tree: } └── livekit.eou_wait [task] metadata: { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.json index 0c10915b4..e98062639 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.json @@ -531,6 +531,7 @@ } ], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -564,7 +565,7 @@ } }, "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.txt index ee59bf0a0..43587989a 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-order.span-tree.txt @@ -11,7 +11,7 @@ span_tree: } } metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { @@ -525,6 +525,7 @@ span_tree: } └── livekit.eou_wait [task] metadata: { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.json index b72080ce0..1be8a3393 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.json @@ -1046,6 +1046,7 @@ } ], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -1079,7 +1080,7 @@ } }, "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.txt index ad8387670..105515d0d 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-recovered.span-tree.txt @@ -11,7 +11,7 @@ span_tree: } } metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { @@ -1009,6 +1009,7 @@ span_tree: } └── livekit.eou_wait [task] metadata: { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.json index 07dc9fc36..95009c536 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.json @@ -94,16 +94,21 @@ } }, { - "name": "livekit.eou_wait", + "name": "eou_wait", "type": "task", "children": [], "metadata": { + "audio.source_id": "session-input", "contrib.livekit.eou.endpointing_delay": 0, + "contrib.livekit.eou.native_end_unix_ms": 0, + "contrib.livekit.eou.native_start_unix_ms": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", "contrib.livekit.eou.rearm_count": 0, "contrib.livekit.eou.source": "manual", "contrib.livekit.eou.wait_duration": 0, + "eou.outcome": "committed", + "eou.speech_end_source": "vad", "scenario": "livekit-instrumentation", "testRunId": "", "turn.id": "" @@ -219,6 +224,7 @@ "type": "task", "children": [], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -751,7 +757,7 @@ } }, "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.txt index 0a21d9871..25fee29a8 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-latest-streaming-cascade.span-tree.txt @@ -11,7 +11,7 @@ span_tree: } } metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { @@ -269,14 +269,19 @@ span_tree: │ │ "testRunId": "", │ │ "turn.id": "" │ │ } - │ ├── livekit.eou_wait [task] + │ ├── eou_wait [task] │ │ metadata: { + │ │ "audio.source_id": "session-input", │ │ "contrib.livekit.eou.endpointing_delay": 0, + │ │ "contrib.livekit.eou.native_end_unix_ms": 0, + │ │ "contrib.livekit.eou.native_start_unix_ms": 0, │ │ "contrib.livekit.eou.not_committed_count": 0, │ │ "contrib.livekit.eou.outcome": "committed", │ │ "contrib.livekit.eou.rearm_count": 0, │ │ "contrib.livekit.eou.source": "manual", │ │ "contrib.livekit.eou.wait_duration": 0, + │ │ "eou.outcome": "committed", + │ │ "eou.speech_end_source": "vad", │ │ "scenario": "livekit-instrumentation", │ │ "testRunId": "", │ │ "turn.id": "" @@ -303,6 +308,7 @@ span_tree: │ } │ └── livekit.eou_wait [task] │ metadata: { + │ "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", │ "contrib.livekit.eou.endpointing_delay": 0, │ "contrib.livekit.eou.not_committed_count": 0, │ "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.json index f7df7aec1..dcb9fd15f 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.json @@ -549,6 +549,7 @@ } ], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -582,7 +583,7 @@ } }, "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.txt index e04372ca9..ff3a665e4 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-order.span-tree.txt @@ -11,7 +11,7 @@ span_tree: } } metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { @@ -543,6 +543,7 @@ span_tree: } └── livekit.eou_wait [task] metadata: { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.json index da034f96b..879d38e95 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.json @@ -1082,6 +1082,7 @@ } ], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -1115,7 +1116,7 @@ } }, "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.txt index bf9403dbe..7a6117f0f 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-recovered.span-tree.txt @@ -11,7 +11,7 @@ span_tree: } } metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { @@ -1045,6 +1045,7 @@ span_tree: } └── livekit.eou_wait [task] metadata: { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.json b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.json index 9d06a70c4..7058b7794 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.json +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.json @@ -94,16 +94,21 @@ } }, { - "name": "livekit.eou_wait", + "name": "eou_wait", "type": "task", "children": [], "metadata": { + "audio.source_id": "session-input", "contrib.livekit.eou.endpointing_delay": 0, + "contrib.livekit.eou.native_end_unix_ms": 0, + "contrib.livekit.eou.native_start_unix_ms": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", "contrib.livekit.eou.rearm_count": 0, "contrib.livekit.eou.source": "manual", "contrib.livekit.eou.wait_duration": 0, + "eou.outcome": "committed", + "eou.speech_end_source": "vad", "scenario": "livekit-instrumentation", "testRunId": "", "turn.id": "" @@ -162,6 +167,7 @@ "type": "task", "children": [], "metadata": { + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", "contrib.livekit.eou.endpointing_delay": 0, "contrib.livekit.eou.not_committed_count": 0, "contrib.livekit.eou.outcome": "committed", @@ -712,7 +718,7 @@ } }, "metadata": { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { diff --git a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.txt b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.txt index 5a0ff29e5..c11b6310a 100644 --- a/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.txt +++ b/e2e/scenarios/livekit-instrumentation/__snapshots__/livekit-v1-streaming-cascade.span-tree.txt @@ -11,7 +11,7 @@ span_tree: } } metadata: { - "audio.schema.version": "3.0.0", + "audio.schema.version": "3.1.0", "audio.segments": { "part-0": { "attachment": { @@ -212,14 +212,19 @@ span_tree: │ │ "testRunId": "", │ │ "turn.id": "" │ │ } - │ ├── livekit.eou_wait [task] + │ ├── eou_wait [task] │ │ metadata: { + │ │ "audio.source_id": "session-input", │ │ "contrib.livekit.eou.endpointing_delay": 0, + │ │ "contrib.livekit.eou.native_end_unix_ms": 0, + │ │ "contrib.livekit.eou.native_start_unix_ms": 0, │ │ "contrib.livekit.eou.not_committed_count": 0, │ │ "contrib.livekit.eou.outcome": "committed", │ │ "contrib.livekit.eou.rearm_count": 0, │ │ "contrib.livekit.eou.source": "manual", │ │ "contrib.livekit.eou.wait_duration": 0, + │ │ "eou.outcome": "committed", + │ │ "eou.speech_end_source": "vad", │ │ "scenario": "livekit-instrumentation", │ │ "testRunId": "", │ │ "turn.id": "" @@ -246,6 +251,7 @@ span_tree: │ } │ └── livekit.eou_wait [task] │ metadata: { + │ "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", │ "contrib.livekit.eou.endpointing_delay": 0, │ "contrib.livekit.eou.not_committed_count": 0, │ "contrib.livekit.eou.outcome": "committed", diff --git a/e2e/scenarios/livekit-instrumentation/assertions.ts b/e2e/scenarios/livekit-instrumentation/assertions.ts index 3f7331261..404032913 100644 --- a/e2e/scenarios/livekit-instrumentation/assertions.ts +++ b/e2e/scenarios/livekit-instrumentation/assertions.ts @@ -111,7 +111,7 @@ export function assertAudioTrace(raw: CapturedLogEvent[]) { for (const event of events.filter((e) => e.span.name === "session")) { const { session, segments } = audioSession(event); if (!session) continue; - expect(event.metadata?.["audio.schema.version"]).toBe("3.0.0"); + expect(event.metadata?.["audio.schema.version"]).toBe("3.1.0"); expect(Number.isInteger(session.timeline.origin_unix_ms)).toBe(true); for (const member of [ ...Object.values(session.participants ?? {}), diff --git a/e2e/scenarios/livekit-instrumentation/lifecycle.mjs b/e2e/scenarios/livekit-instrumentation/lifecycle.mjs new file mode 100644 index 000000000..f5fa88108 --- /dev/null +++ b/e2e/scenarios/livekit-instrumentation/lifecycle.mjs @@ -0,0 +1,176 @@ +// Synthetic lifecycle inputs on the installed framework classes. No provider +// calls or fabricated provider responses; this is private-hook regression coverage. +import "./configure.mjs"; +import assert from "node:assert/strict"; +import { setImmediate as tick } from "node:timers/promises"; +import { initLogger, wrapLiveKit } from "braintrust"; +import { scopedName } from "../../helpers/provider-runtime.mjs"; +import { dispose } from "@livekit/rtc-node"; +const agents = await import("@livekit/agents"); +const { voice, initializeLogger, llm } = + process.env.LIVEKIT_WRAP === "1" ? wrapLiveKit(agents) : agents; +initializeLogger({ pretty: false, level: "silent" }); +const logger = initLogger({ projectName: scopedName("livekit-eou-lifecycle") }); +const realNow = Date.now; +let now = realNow() + 10000; +const cases = []; + +async function probe(name, run) { + const session = new voice.AgentSession({ + minEndpointingDelay: 0, + maxEndpointingDelay: 0, + }); + let input; + session.input.audio = new (class extends voice.AudioInput { + source = new ReadableStream({ + start(controller) { + input = controller; + }, + }); + get stream() { + return this.source; + } + async close() { + input.close(); + } + })(); + await session.start({ + agent: new voice.Agent({ instructions: "Lifecycle probe" }), + record: false, + }); + const recognition = session.activity.audioRecognition; + assert.ok(recognition, "exercise the installed AudioRecognition"); + session.sessionSpan.setAttribute("lk.probe.case", name); + // Replace application acceptance/model callbacks, leaving all framework + // timestamp assignments, VAD dispatch, wait factories, and cleanup intact. + recognition.hooks.onEndOfTurn = async () => true; + recognition.hooks.onPreemptiveGeneration = () => {}; + recognition.hooks.onFinalTranscript = () => {}; + recognition.hooks.retrieveChatCtx = () => new llm.ChatContext(); + recognition.stt = () => {}; + Date.now = () => now; + try { + await run(recognition); + } finally { + input.close(); + await session.close(); + Date.now = realNow; + } + cases.push(name); + now += 10000; +} + +try { + await probe("synchronous-stt", async (recognition) => { + const at = now - 2000; + recognition.processSTTEvent({ + type: 2, + speechEndTime: at, + alternatives: [ + { + text: "Complete turn", + startTime: 0, + endTime: 0, + language: "en", + confidence: 1, + }, + ], + }); + await tick(); + assert.equal( + recognition.eouWaitSpan, + undefined, + "framework committed the wait", + ); + console.log(JSON.stringify({ probe: "synchronous-stt", start: at / 1000 })); + }); + + for (const ending of ["commit", "shutdown", "drop"]) { + await probe(`transcription-pending-${ending}`, async (recognition) => { + const base = now - 5000; + let feed; + const queue = new ReadableStream({ + start(controller) { + feed = controller; + }, + }); + const vad = { + stream: () => ({ + updateInputStream() {}, + detachInputStream() {}, + close() {}, + [Symbol.asyncIterator]: () => queue[Symbol.asyncIterator](), + }), + }; + recognition.vad = vad; + const task = recognition.createVadTask(vad, new AbortController().signal); + const emit = async (type, at, fields = {}) => { + now = at; + feed.enqueue({ + type, + frames: [], + speechDuration: 0, + silenceDuration: 0, + inferenceDuration: 0, + ...fields, + }); + await tick(); + }; + await emit(0, base + 1000); + await emit(2, base + 2100, { + silenceDuration: 80, + inferenceDuration: 20, + }); + // Repeat the actual evaluation: the same transcription-gated pause remains open. + recognition.runEOUDetection(new llm.ChatContext(), "vad"); + assert.equal( + recognition.eouWaitSpan, + undefined, + "native transcript guard skipped factory", + ); + await emit(0, base + 2400, { speechDuration: 80, inferenceDuration: 20 }); + await emit(2, base + 3600, { + silenceDuration: 80, + inferenceDuration: 20, + }); + assert.equal(recognition.eouWaitSpan, undefined); + if (ending === "commit") { + now = base + 4000; + recognition.processSTTEvent({ + type: 2, + alternatives: [ + { + text: "Later transcript", + startTime: 0, + endTime: 0, + language: "en", + confidence: 1, + }, + ], + }); + await tick(); + assert.equal( + recognition.eouWaitSpan, + undefined, + "later transcript committed existing pause", + ); + } else if (ending === "drop") { + now = base + 4000; + recognition.clearUserTurn(); + } + feed.close(); + await task; + console.log( + JSON.stringify({ + probe: `transcription-pending-${ending}`, + base: base / 1000, + }), + ); + }); + } +} finally { + Date.now = realNow; + await logger.flush(); + await dispose(); +} +assert.equal(cases.length, 4); diff --git a/e2e/scenarios/livekit-instrumentation/scenario.test.ts b/e2e/scenarios/livekit-instrumentation/scenario.test.ts index 01b327b29..80457ac9a 100644 --- a/e2e/scenarios/livekit-instrumentation/scenario.test.ts +++ b/e2e/scenarios/livekit-instrumentation/scenario.test.ts @@ -175,6 +175,98 @@ const spoken = (event: CapturedLogEvent | undefined, pattern: RegExp) => ); for (const variant of variants) { + // Real installed framework classes with synthetic lifecycle events, not providers. + for (const mode of ["auto", "wrapped", "wrapped-auto"] as const) + test(`${variant} ${mode} transcription lifecycle`, async () => { + await withScenarioHarness(async (harness) => { + const result = await harness.runNodeScenarioDir( + scenarioOptions(variant, { + mode, + entry: "lifecycle.mjs", + recording: false, + }), + ); + const events = harness.events(); + const roots = findAllSpans(events, "session"); + expect(roots).toHaveLength(4); + const probes = result.stdout + .split("\n") + .filter((line) => line.startsWith('{"probe":')) + .map((line) => JSON.parse(line)); + for (const probe of probes) { + const root = roots.find( + (row) => + row.metadata?.["contrib.livekit.probe.case"] === probe.probe, + )!; + expect(root).toBeDefined(); + const turns = findAllSpans(events, "user_turn").filter( + (row) => row.span.rootId === root.span.rootId, + ); + expect(turns).toHaveLength(1); + const waits = findAllSpans(events, "eou_wait") + .filter((row) => row.span.rootId === root.span.rootId) + .sort( + (a, b) => Number(a.metrics?.start) - Number(b.metrics?.start), + ); + expect(waits).toHaveLength(probe.start ? 1 : 2); + expect( + findAllSpans(events, "livekit.eou_wait") + .filter((row) => row.span.rootId === root.span.rootId) + // The selector filters by name before merging. An earlier flush + // can retain the diagnostic name for the same subsequently + // normalized row; count identities, not historical names. + .filter( + (row) => !waits.some((wait) => wait.span.id === row.span.id), + ) + .map((row) => ({ + probe: probe.probe, + metadata: row.metadata, + metrics: row.metrics, + })), + ).toEqual([]); + if (probe.start) { + expect(waits[0].metrics?.start).toBe(probe.start); + expect(waits[0].metadata?.["eou.outcome"]).toBe("committed"); + expect(waits[0].metadata).not.toHaveProperty( + "eou.speech_end_source", + ); + } else { + expect(Number(waits[0].metrics?.start) - probe.base).toBeCloseTo(2); + expect(Number(waits[0].metrics?.end) - probe.base).toBeCloseTo(2.3); + expect(Number(waits[1].metrics?.start) - probe.base).toBeCloseTo( + 3.5, + ); + if (!probe.probe.endsWith("shutdown")) + expect(Number(waits[1].metrics?.end) - probe.base).toBeCloseTo(4); + expect(waits[0].metadata?.["eou.outcome"]).toBe("user_resumed"); + expect(waits[0].metadata).not.toHaveProperty( + "contrib.livekit.eou.native_start_unix_ms", + ); + for (const row of waits) + expect(row.metadata?.["eou.speech_end_source"]).toBe("vad"); + if (probe.probe.endsWith("shutdown")) { + expect(waits[1].metadata).not.toHaveProperty("eou.outcome"); + expect(waits[1].metadata?.["contrib.livekit.incomplete"]).toBe( + true, + ); + } else + expect(waits[1].metadata?.["eou.outcome"]).toBe( + probe.probe.endsWith("drop") ? "dropped" : "committed", + ); + } + for (const row of waits) { + expect(row.span.parentIds).toEqual([turns[0].span.id]); + expect(row.metadata?.["turn.id"]).toBe(turns[0].span.id); + expect(row.metadata?.["audio.source_id"]).toBe("session-input"); + expect(row.metadata).not.toHaveProperty("eou.end_reason"); + expect(row.metadata).not.toHaveProperty("eou.detection_methods"); + expect(row.metrics?.end).toEqual(expect.any(Number)); + } + } + expect(probes).toHaveLength(4); + }); + }, 120000); + // Fixture playback through each setup path, including idempotent double wrapping. for (const mode of ["auto", "wrapped", "wrapped-auto"] as const) test(`${variant} ${mode} fixture recording`, async () => { @@ -411,42 +503,60 @@ for (const variant of variants) { }); }, 120000); - test(`${variant} streaming-cascade order conversation`, async () => { - await withScenarioHarness(async (harness) => { - await harness.runNodeScenarioDir( - scenarioOptions(variant, { - cassette: "streaming-cascade", - env: { - LIVEKIT_STREAMING_STT: "1", - LIVEKIT_WS_PATH: `${originalScenarioDir}/__cassettes__/${variant}-streaming-cascade.websocket.json`, - BRAINTRUST_E2E_CASSETTE_MODE: - process.env.BRAINTRUST_E2E_CASSETTE_MODE ?? "replay", - OPENAI_API_KEY: process.env.OPENAI_API_KEY ?? "replay", - }, - }), - ); - const events = assertAudioTrace(harness.events()); - const user = findLatestSpan(events, "user_speech"); - // User speech is observed audio: its source and its place on the - // timeline, from LiveKit's local VAD, but no text. - expect(user?.metadata?.["audio.source_id"]).toBe("session-input"); - expect(user?.row.input).toBeUndefined(); - expect(selectionsOf(user)).toHaveLength(1); - expect( - selectionsOf( - findAllSpans(events, "user_turn").find((e) => e.row.input), - ), - ).toHaveLength(1); - // LiveKit has no native STT span; each final transcript gets one, and the - // model's context links to the transcripts of the turn it answers. - const stt = findAllSpans(events, "stt"); - expect(stt.length).toBeGreaterThan(0); - expect(contextLinks(findAllSpans(events, "llm_response")[0])).toEqual( - stt.map((span) => span.span.id), - ); - await snapshotTrace(events, `${variant}-streaming-cascade`); - }); - }, 120000); + for (const mode of ["auto", "wrapped", "wrapped-auto"] as const) + test(`${variant} ${mode} streaming-cascade order conversation`, async () => { + await withScenarioHarness(async (harness) => { + await harness.runNodeScenarioDir( + scenarioOptions(variant, { + cassette: "streaming-cascade", + mode, + env: { + LIVEKIT_STREAMING_STT: "1", + LIVEKIT_WS_PATH: `${originalScenarioDir}/__cassettes__/${variant}-streaming-cascade.websocket.json`, + BRAINTRUST_E2E_CASSETTE_MODE: + process.env.BRAINTRUST_E2E_CASSETTE_MODE ?? "replay", + OPENAI_API_KEY: process.env.OPENAI_API_KEY ?? "replay", + }, + }), + ); + const events = assertAudioTrace(harness.events()); + const user = findLatestSpan(events, "user_speech"); + // User speech is observed audio: its source and its place on the + // timeline, from LiveKit's local VAD, but no text. + expect(user?.metadata?.["audio.source_id"]).toBe("session-input"); + expect(user?.row.input).toBeUndefined(); + expect(selectionsOf(user)).toHaveLength(1); + expect( + selectionsOf( + findAllSpans(events, "user_turn").find((e) => e.row.input), + ), + ).toHaveLength(1); + // LiveKit has no native STT span; each final transcript gets one, and the + // model's context links to the transcripts of the turn it answers. + const stt = findAllSpans(events, "stt"); + expect(stt.length).toBeGreaterThan(0); + expect(contextLinks(findAllSpans(events, "llm_response")[0])).toEqual( + stt.map((span) => span.span.id), + ); + const waits = findAllSpans(events, "eou_wait"); + expect(waits.length).toBeGreaterThan(0); + for (const wait of waits) { + expect(wait.span.type).toBe("task"); + expect(wait.metadata).toMatchObject({ + "eou.outcome": "committed", + "eou.speech_end_source": "vad", + "audio.source_id": "session-input", + }); + expect(wait.metadata?.["turn.id"]).toBe(wait.span.parentIds[0]); + expect(wait.metadata).not.toHaveProperty("eou.end_reason"); + expect(wait.metadata).not.toHaveProperty("eou.detection_methods"); + expect(wait.metrics?.end).toBeGreaterThanOrEqual( + wait.metrics?.start as number, + ); + } + await snapshotTrace(events, `${variant}-streaming-cascade`); + }); + }, 120000); // Two callers share one microphone; diarized STT tells their voices apart. test(`${variant} diarized two-caller conversation`, async () => { diff --git a/js/src/auto-instrumentations/configs/livekit.ts b/js/src/auto-instrumentations/configs/livekit.ts index 7aab73304..00224026b 100644 --- a/js/src/auto-instrumentations/configs/livekit.ts +++ b/js/src/auto-instrumentations/configs/livekit.ts @@ -53,6 +53,36 @@ const targets: [ "voice/agent_activity", { className: "AgentActivity", methodName: "userTurnCompleted" }, ], + [ + "vadInference", + "voice/agent_activity", + { className: "AgentActivity", methodName: "onVADInferenceDone" }, + ], + [ + "speechEnd", + "voice/agent_activity", + { className: "AgentActivity", methodName: "onEndOfSpeech" }, + ], + [ + "recognitionClose", + "voice/audio_recognition", + { className: "AudioRecognition", methodName: "close" }, + ], + [ + "eouWait", + "voice/audio_recognition", + { className: "AudioRecognition", methodName: "ensureEouWaitSpan" }, + ], + [ + "eouEvaluation", + "voice/audio_recognition", + { className: "AudioRecognition", methodName: "runEOUDetection" }, + ], + [ + "eouEnd", + "voice/audio_recognition", + { className: "AudioRecognition", methodName: "endEouWaitSpan" }, + ], [ "stt", "voice/audio_recognition", diff --git a/js/src/instrumentation/providers/livekit-interceptors.ts b/js/src/instrumentation/providers/livekit-interceptors.ts index 62b0721f4..148fa8aca 100644 --- a/js/src/instrumentation/providers/livekit-interceptors.ts +++ b/js/src/instrumentation/providers/livekit-interceptors.ts @@ -6,6 +6,9 @@ import type { TraceOptions, PipelineOptions, SpeechEvent, + SpeechEndEvent, + VadInferenceEvent, + EouWaitOptions, } from "../../vendor-sdk-types/livekit"; import { channel, defineInterceptor } from "../core/interceptor-definitions"; @@ -51,6 +54,24 @@ export const livekitChannels = defineInterceptor("@livekit/agents", { speechStart: channel<[unknown?], void>({ channelName: "activity.speechStart", }), + vadInference: channel<[VadInferenceEvent], void>({ + channelName: "activity.vadInference", + }), + speechEnd: channel<[SpeechEndEvent?], void>({ + channelName: "activity.speechEnd", + }), + recognitionClose: channel<[], Promise>({ + channelName: "recognition.close", + }), + eouWait: channel<[LiveKitSpan, EouWaitOptions], LiveKitSpan>({ + channelName: "recognition.eouWait", + }), + eouEvaluation: channel<[unknown, string?], void>({ + channelName: "recognition.eouEvaluation", + }), + eouEnd: channel<[string, number?], void>({ + channelName: "recognition.eouEnd", + }), stt: channel<[SpeechEvent], void>({ channelName: "recognition.stt", }), diff --git a/js/src/instrumentation/providers/livekit-plugin.test.ts b/js/src/instrumentation/providers/livekit-plugin.test.ts index f3acfab09..29600ea02 100644 --- a/js/src/instrumentation/providers/livekit-plugin.test.ts +++ b/js/src/instrumentation/providers/livekit-plugin.test.ts @@ -91,6 +91,26 @@ function start( }); return span; } +/** Synthetic lifecycle fixtures, independently observed VAD then native wait. */ +function vadWait(parent: NativeSpan, at: number, recognition: any = {}) { + tracing.speechEnd( + { agentSession: {}, audioRecognition: recognition }, + { + type: 2, + silenceDuration: now - at, + inferenceDuration: 0, + }, + ); + const wait = start("eou_wait", parent, { "lk.eou.source": "vad" }, at); + const rearm = (anchor = at, trigger = "vad") => + tracing.eouWait(recognition, wait, { + lastSpeakingTime: anchor, + trigger, + endpointingDelay: 500, + }); + rearm(); + return { wait, rearm, recognition }; +} async function rows() { const all = (await logger.drain()) as any[]; const named = (name: string) => @@ -629,7 +649,7 @@ describe("tracing", () => { activity.end(); sessionStart.end(); const turn = start("user_turn", root); - const wait = start("eou_wait", turn); + const { wait } = vadWait(turn, 1000); start("eou_detection", wait, { "lk.eou.probability": 0.9 }).end(); wait.end(); turn.end(); @@ -655,8 +675,8 @@ describe("tracing", () => { "livekit.setup_toolsets": "livekit.start_agent_activity", "livekit.on_enter": "livekit.start_agent_activity", user_turn: "session", - "livekit.eou_wait": "user_turn", - "livekit.eou_detection": "livekit.eou_wait", + eou_wait: "user_turn", + "livekit.eou_detection": "eou_wait", tts: "session", "livekit.tts_request": "tts", "livekit.tts_request_run": "livekit.tts_request", @@ -742,8 +762,8 @@ describe("tracing", () => { const tool = start("function_tool", turn, { "lk.function_tool.name": "late_tool", }); - // LiveKit can leave internal spans such as eou_wait open. They don't hold - // the session, but still export if LiveKit ends them later. + // Unfinished waits do not hold the session. Close their exported rows as + // incomplete when the last outstanding mapped operation ends. const wait = start("eou_wait", turn); turn.end([2, 0]); root.end([2, 0]); @@ -759,8 +779,12 @@ describe("tracing", () => { output: { ok: true }, }); expect(one("livekit.eou_wait")).toMatchObject({ - span_parents: [one("agent_turn").span_id], - metrics: { end: 4 }, + span_parents: [one("session").span_id], + metrics: { end: 2 }, + metadata: { + "contrib.livekit.incomplete": true, + "contrib.livekit.termination": "session_closed", + }, }); }); @@ -828,6 +852,647 @@ describe("tracing", () => { }); }); +describe("eou_wait audio schema", () => { + function pendingRecognition(parent: NativeSpan, at = 2000): any { + const recognition = { + userTurnSpan: parent, + lastSpeakingTime: at, + vad: {}, + stt: {}, + audioTranscript: "", + vadBaseTurnDetection: true, + }; + tracing.speechEnd( + { agentSession: {}, audioRecognition: recognition }, + { + type: 2, + silenceDuration: now - at, + inferenceDuration: 0, + }, + ); + return recognition; + } + + test("a synchronous STT factory is reconciled only after the framework adopts its timestamp", async () => { + const root = start("agent_session"); + const turn = start("user_turn", root); + now = 2500; + const recognition: any = { userTurnSpan: turn }; + let wait!: NativeSpan; + tracing.processSTT( + recognition, + { type: 2, speechEndTime: 2000, alternatives: [{ text: "hello" }] }, + () => { + recognition.lastSpeakingTime = 2000; + wait = start("eou_wait", turn, {}, 2000); + tracing.eouWait(recognition, wait, { + trigger: "stt", + lastSpeakingTime: 2000, + endpointingDelay: 0, + }); + }, + ); + wait.setAttribute("lk.eou.outcome", "committed"); + wait.end(2600); + const { one, named } = await rows(); + expect(one("eou_wait").metrics).toMatchObject({ start: 2, end: 2.6 }); + expect(named("livekit.eou_wait")).toHaveLength(0); + }); + + test("a transcript-gated pause exports once on resume without any native wait", async () => { + const root = start("agent_session"); + const turn = start("user_turn", root); + now = 2200; + const recognition = pendingRecognition(turn); + tracing.eouEvaluation(recognition); + tracing.eouEvaluation(recognition); + tracing.eouEnd(recognition, "user_resumed", 2300); + tracing.eouEnd(recognition, "user_resumed", 2300); + turn.end(2400); + root.end(2400); + await tracing.closeSession(root); + const { one, named } = await rows(); + expect(one("eou_wait")).toMatchObject({ + metrics: { start: 2, end: 2.3 }, + span_parents: [one("user_turn").span_id], + metadata: { + "eou.outcome": "user_resumed", + "eou.speech_end_source": "vad", + }, + }); + expect(one("eou_wait").metadata).not.toHaveProperty( + "contrib.livekit.eou.native_start_unix_ms", + ); + expect(named("livekit.eou_wait")).toHaveLength(0); + expect(tracing.sessions.size).toBe(0); + }); + + test("a later native factory attaches to the pending pause before detector children start", async () => { + const root = start("agent_session"); + const turn = start("user_turn", root); + now = 2200; + const recognition = pendingRecognition(turn); + tracing.eouEvaluation(recognition); + const wait = tracing.waitFactory(recognition, () => + start("eou_wait", turn, { "lk.eou.source": "stt" }, 2100), + ); + tracing.eouWait(recognition, wait, { + trigger: "stt", + lastSpeakingTime: 2000, + endpointingDelay: 0, + }); + const detector = start("eou_detection", wait, {}, 2200); + detector.end(2400); + wait.setAttribute("lk.eou.outcome", "committed"); + wait.end(2500); + const { one } = await rows(); + expect(one("eou_wait").metrics).toMatchObject({ start: 2, end: 2.5 }); + expect(one("eou_wait").metadata).toMatchObject({ + "contrib.livekit.eou.native_start_unix_ms": 2100, + "contrib.livekit.eou.source": "stt", + "eou.outcome": "committed", + }); + expect(one("livekit.eou_detection").span_parents).toEqual([ + one("eou_wait").span_id, + ]); + }); + + test.each([ + "manual", + "no-stt", + "has-transcript", + "invalid-anchor", + "no-evidence", + "manual-flush", + ])( + "does not invent a transcription-gated wait with %s", + async (condition) => { + const root = start("agent_session"); + now = 2200; + const recognition = pendingRecognition(root); + if (condition === "manual") recognition.turnDetectionMode = "manual"; + if (condition === "no-stt") recognition.stt = undefined; + if (condition === "has-transcript") recognition.audioTranscript = "hello"; + if (condition === "invalid-anchor") recognition.lastSpeakingTime = NaN; + if (condition === "no-evidence") + tracing.vadSpeech({ agentSession: {}, audioRecognition: recognition }); + if (condition === "manual-flush") + recognition.sttPipeline = { manualFlush: true }; + tracing.eouEvaluation(recognition); + tracing.eouEnd(recognition, "user_resumed", 2300); + expect((await rows()).named("eou_wait")).toHaveLength(0); + }, + ); + + test("a pending pause without turn membership uses the session and shutdown leaves its outcome unknown", async () => { + const root = start("agent_session"); + now = 2200; + const recognition = pendingRecognition(root); + recognition.userTurnSpan = undefined; + recognition.rootSpanContext = { getValue: () => root }; + tracing.eouEvaluation(recognition); + tracing.recognitionClose(recognition); + tracing.eouEnd(recognition, "dropped", 2500); + root.end(2600); + await tracing.closeSession(root); + const { one } = await rows(); + expect(one("eou_wait").span_parents).toEqual([one("session").span_id]); + expect(one("eou_wait").metadata).not.toHaveProperty("turn.id"); + expect(one("eou_wait").metadata).not.toHaveProperty("eou.outcome"); + expect(one("eou_wait").metadata["contrib.livekit.incomplete"]).toBe(true); + expect(tracing.sessions.size).toBe(0); + }); + + test.each(["session-close", "processor-shutdown"])( + "an unfinished pending pause does not hold open %s", + async (cleanup) => { + const root = start("agent_session"); + now = 2200; + const recognition = pendingRecognition(root); + recognition.userTurnSpan = undefined; + recognition.rootSpanContext = { getValue: () => root }; + tracing.eouEvaluation(recognition); + now = 2500; + if (cleanup === "processor-shutdown") await tracing.shutdown(); + else { + root.end(2500); + await tracing.closeSession(root); + } + const { one } = await rows(); + expect(one("eou_wait").metrics).toMatchObject({ start: 2, end: 2.5 }); + expect(one("eou_wait").metadata).not.toHaveProperty("eou.outcome"); + expect(one("eou_wait").metadata["contrib.livekit.incomplete"]).toBe(true); + expect(tracing.sessions.size).toBe(0); + }, + ); + + test("an absolute end on an STT event establishes timing but cannot distinguish provider from VAD-adapter provenance", async () => { + const root = start("agent_session"); + now = 2500; + const recognition = { lastSpeakingTime: 2000 }; + tracing.processSTT( + recognition, + { type: 2, speechEndTime: 2000, alternatives: [{ text: "hello" }] }, + () => {}, + ); + const wait = start("eou_wait", root, {}, 2000); + tracing.eouWait(recognition, wait, { + lastSpeakingTime: 2000, + trigger: "stt", + endpointingDelay: 0, + }); + wait.setAttribute("lk.eou.outcome", "committed"); + wait.end(2600); + const { one } = await rows(); + expect(one("eou_wait").metrics).toMatchObject({ start: 2, end: 2.6 }); + expect(one("eou_wait").metadata).not.toHaveProperty( + "eou.speech_end_source", + ); + }); + + test("a failed STT callback preserves the exception and cannot establish anchor provenance", async () => { + const root = start("agent_session"); + now = 2500; + const recognition = { lastSpeakingTime: 2000 }; + const error = new Error("callback failed"); + expect(() => + tracing.processSTT( + recognition, + { type: 2, speechEndTime: 2000, alternatives: [{ text: "hello" }] }, + () => { + throw error; + }, + ), + ).toThrow(error); + const wait = start("eou_wait", root, {}, 2000); + tracing.eouWait(recognition, wait, { + lastSpeakingTime: 2000, + trigger: "stt", + endpointingDelay: 0, + }); + wait.end(2600); + expect((await rows()).named("eou_wait")).toHaveLength(0); + }); + + test("combined wrapping and auto instrumentation observe an STT invocation only once", async () => { + const root = start("agent_session"); + const turn = start("user_turn", root); + const recognition = { userTurnSpan: turn }; + const event = { type: 2, alternatives: [{ text: "hello" }] }; + const target = vi.fn(() => "result"); + expect( + tracing.processSTT(recognition, event, () => + tracing.processSTT(recognition, event, target), + ), + ).toBe("result"); + expect(target).toHaveBeenCalledTimes(1); + expect((await rows()).named("stt")).toHaveLength(1); + }); + + test.each(["dropped", "committed"])( + "native %s during recognition close requires an actual decision", + async (outcome) => { + const root = start("agent_session"); + now = 2200; + const { wait } = vadWait(root, 2000); + tracing.recognitionClose({ eouWaitSpan: wait }); + wait.setAttribute("lk.eou.outcome", outcome); + wait.end(2600); + const { one } = await rows(); + expect(one("eou_wait").metadata["contrib.livekit.eou.outcome"]).toBe( + outcome, + ); + expect(one("eou_wait").metadata["eou.outcome"]).toBe( + outcome === "committed" ? "committed" : undefined, + ); + if (outcome === "dropped") + expect(one("eou_wait").metadata).toMatchObject({ + "contrib.livekit.incomplete": true, + "contrib.livekit.termination": "recognition_closed", + }); + }, + ); + + test("resume timing survives the diagnostic event cap and requires an explicit timestamp", async () => { + const root = start("agent_session"); + now = 2200; + const first = vadWait(root, 2000).wait; + for (let i = 0; i < 300; i++) first.addEvent("rearmed", {}, 2200); + first.addEvent("user_resumed", {}, 2300); + first.setAttribute("lk.eou.outcome", "user_resumed"); + first.end(2600); + const second = vadWait(root, 2000).wait; + second.addEvent("user_resumed", {}); + second.setAttribute("lk.eou.outcome", "user_resumed"); + second.end(2600); + const { one } = await rows(); + expect(one("eou_wait").metrics.end).toBe(2.3); + expect( + one("livekit.eou_wait").metadata["contrib.livekit.eou.canonical_omitted"], + ).toBe("unobserved_wait_end"); + }); + + test("an STT evaluation trigger retaining a VAD anchor does not change its provenance", async () => { + const root = start("agent_session"); + now = 2200; + const recognition = { vad: {}, lastSpeakingTime: 2000 }; + const { wait, rearm } = vadWait(root, 2000, recognition); + tracing.processSTT( + recognition, + { type: 2, speechEndTime: 2000, alternatives: [{ text: "hello" }] }, + () => {}, + ); + rearm(2000, "stt"); + wait.setAttributes({ + "lk.eou.outcome": "committed", + "lk.eou.source": "stt", + }); + wait.end(2500); + expect((await rows()).one("eou_wait").metadata).toMatchObject({ + "eou.speech_end_source": "vad", + "contrib.livekit.eou.source": "stt", + }); + }); + + test("VAD provenance uses the exact framework correction even across separate clock reads", async () => { + const root = start("agent_session"); + now = 2200; + const { wait, rearm } = vadWait(root, 2000); + rearm(2001); + wait.end(2500); + expect((await rows()).one("eou_wait").metrics.start).toBe(2.001); + }); + + test("VAD speech activity invalidates an earlier pause anchor before another wait", async () => { + const root = start("agent_session"); + now = 2200; + const { wait, rearm, recognition } = vadWait(root, 2000); + tracing.vadSpeech( + { agentSession: {}, audioRecognition: recognition }, + { rawAccumulatedSpeech: 20 }, + ); + rearm(2100); + wait.end(2500); + expect((await rows()).named("eou_wait")).toHaveLength(0); + }); + + test.each([NaN, Infinity, 1900])( + "invalid native end %s cannot establish a canonical interval", + async (end) => { + const root = start("agent_session"); + now = 2200; + const { wait } = vadWait(root, 2000); + wait.setAttribute("lk.eou.outcome", "committed"); + wait.end(end); + expect((await rows()).named("eou_wait")).toHaveLength(0); + }, + ); + + test("commit uses observed VAD timing, owning turn and established audio identities", async () => { + const root = start("agent_session"); + const turn = start("user_turn", root, { + "lk.participant_id": "PA_caller", + "lk.participant_kind": "STANDARD", + }); + now = 2300; + const { wait } = vadWait(turn, 2000); + wait.setAttributes({ + "lk.eou.outcome": "committed", + "lk.eou.source": "stt", + "lk.eou.endpointing_delay": 0.5, + }); + wait.end([2, 500_000_000]); + turn.end(2500); + root.end(2600); + const { one } = await rows(); + expect(one("eou_wait")).toMatchObject({ + span_attributes: { name: "eou_wait", type: "task" }, + span_parents: [one("user_turn").span_id], + metrics: { start: 2, end: 2.5 }, + metadata: { + "turn.id": one("user_turn").span_id, + "audio.source_id": "session-input", + "audio.participant_ids": ["participant-1"], + "eou.outcome": "committed", + "eou.speech_end_source": "vad", + "contrib.livekit.eou.source": "stt", + "contrib.livekit.eou.endpointing_delay": 0.5, + }, + }); + expect(one("eou_wait").metadata).not.toHaveProperty("eou.end_reason"); + expect(one("eou_wait").metadata).not.toHaveProperty( + "eou.detection_methods", + ); + expect(one("session").metadata["audio.schema.version"]).toBe("3.1.0"); + }); + + test("resume then later pause commits; rearms and speculative work do not create or end waits", async () => { + const root = start("agent_session"); + const turn = start("user_turn", root); + start("user_speaking", turn, {}, 1000).end(2000); + now = 2200; + const first = vadWait(turn, 2000); + first.wait.addEvent("user_resumed", {}, new Date(2300)); + first.wait.setAttribute("lk.eou.outcome", "user_resumed"); + first.wait.end(2300); + start("user_speaking", turn, {}, 2300).end(3500); + now = 3600; + const second = vadWait(turn, 3500); + for (let i = 0; i < 4; i++) { + second.wait.addEvent( + "rearmed", + { "lk.eou.source": "stt" }, + 3600 + i * 10, + ); + start( + "eou_detection", + second.wait, + { "lk.eou.probability": 0.9 }, + 3600 + i * 10, + ).end(3605 + i * 10); + second.rearm(3500, "stt"); + } + const model = start("llm_node", root, {}, 3700); + second.wait.setAttributes({ + "lk.eou.outcome": "committed", + "lk.eou.rearm_count": 4, + }); + second.wait.end(4000); + model.end(4500); + turn.end(4000); + root.end(4500); + const { named, one } = await rows(); + expect( + named("eou_wait").map((row) => [ + row.metrics.start, + row.metrics.end, + row.metadata["eou.outcome"], + ]), + ).toEqual([ + [2, 2.3, "user_resumed"], + [3.5, 4, "committed"], + ]); + expect(named("livekit.eou_wait")).toHaveLength(0); + expect(one("llm_response").metrics).toMatchObject({ start: 3.7, end: 4.5 }); + expect(named("livekit.eou_detection")).toHaveLength(4); + expect( + named("eou_wait").every( + (row) => row.span_parents[0] === one("user_turn").span_id, + ), + ).toBe(true); + await tracing.closeSession(root); + expect(tracing.sessions.size).toBe(0); + }); + + test("late STT with multiple native gates corrects the anchor without claiming final gate or a complete method set", async () => { + const root = start("agent_session"); + const turn = start("user_turn", root); + now = 2200; + const recognition = { + inputStartedAt: 1000, + lastSpeakingTime: 2000, + vad: {}, + turnDetectionMode: "stt", + }; + const { wait, rearm } = vadWait(turn, 2000, recognition); + start( + "eou_detection", + wait, + { "lk.eou.probability": 0.8, "lk.eou.unlikely_threshold": 0.5 }, + 2200, + ).end(2400); + now = 2800; + // Real STT-origin end timestamp corrects an earlier VAD anchor. The + // final's receipt (2.8 s) is deliberately different from speech end (2.1 s). + const event = { type: 2, alternatives: [{ text: "hello", endTime: 1.1 }] }; + tracing.processSTT(recognition, event, () => { + recognition.lastSpeakingTime = 2100; + }); + tracing.stt( + { ...recognition, userTurnSpan: turn }, + { ...event, createdAt: 2800 }, + ); + rearm(2100, "stt"); + wait.setAttributes({ + "lk.eou.outcome": "committed", + "lk.eou.source": "stt", + "lk.eou.endpointing_delay": 0.5, + }); + wait.end(2900); + const { one } = await rows(); + expect(one("eou_wait").metrics).toMatchObject({ start: 2.1, end: 2.9 }); + expect(one("eou_wait").metadata).toMatchObject({ + "eou.speech_end_source": "stt", + "eou.outcome": "committed", + "contrib.livekit.eou.native_start_unix_ms": 2000, + }); + expect(one("eou_wait").metadata).not.toHaveProperty("eou.end_reason"); + expect(one("eou_wait").metadata).not.toHaveProperty( + "eou.detection_methods", + ); + expect(one("stt").metrics.end).toBe(2.8); + }); + + test("resume event bounds canonical wait while detector cleanup retains its native interval", async () => { + const root = start("agent_session"); + const turn = start("user_turn", root); + now = 2200; + const { wait } = vadWait(turn, 2000); + const detector = start("eou_detection", wait, {}, 2200); + detector.addEvent("superseded", { "lk.eou.outcome": "user_resumed" }, 2600); + detector.end(2600); + wait.addEvent("user_resumed", {}, [2, 300_000_000]); + wait.setAttributes({ + "lk.eou.outcome": "user_resumed", + "lk.eou.wait_duration": 0.6, + }); + wait.end(2600); + const { one } = await rows(); + expect(one("eou_wait").metrics).toMatchObject({ start: 2, end: 2.3 }); + expect(one("livekit.eou_detection")).toMatchObject({ + span_parents: [one("eou_wait").span_id], + metrics: { start: 2.2, end: 2.6 }, + }); + expect(one("eou_wait").metadata).toMatchObject({ + "contrib.livekit.eou.native_end_unix_ms": 2600, + "contrib.livekit.eou.wait_duration": 0.6, + }); + expect(one("eou_wait").metadata["contrib.livekit.events"]).toContainEqual({ + name: "user_resumed", + time_unix_ms: 2300, + attributes: {}, + }); + }); + + test.each(["dropped", "committed", "unknown"])( + "native %s only maps known outcomes", + async (outcome) => { + const root = start("agent_session"); + now = 2200; + const { wait } = vadWait(root, 2000); + wait.setAttribute("lk.eou.outcome", outcome); + wait.end(2500); + const { one } = await rows(); + expect(one("eou_wait").span_parents).toEqual([one("session").span_id]); + expect(one("eou_wait").metadata).not.toHaveProperty("turn.id"); + expect(one("eou_wait").metadata).not.toHaveProperty( + "audio.participant_ids", + ); + expect(one("eou_wait").metadata["eou.outcome"]).toBe( + outcome === "unknown" ? undefined : outcome, + ); + expect(one("eou_wait").metadata["contrib.livekit.eou.outcome"]).toBe( + outcome, + ); + expect(one("eou_wait").metadata).not.toHaveProperty("eou.end_reason"); + }, + ); + + test.each(["session", "processor"])( + "%s shutdown closes unfinished waits and detectors without inventing outcomes", + async (mode) => { + const root = start("agent_session"); + const turn = start("user_turn", root); + now = 2200; + const { wait } = vadWait(turn, 2000); + start("eou_detection", wait, {}, 2200); + now = 2600; + if (mode === "session") { + turn.end(2600); + root.end(2600); + await tracing.closeSession(root); + } else await tracing.shutdown(); + expect(tracing.sessions.size).toBe(0); + // A late native result cannot turn the observed incomplete boundary into commitment. + wait.setAttribute("lk.eou.outcome", "committed"); + wait.end(2800); + const { one } = await rows(); + expect(one("eou_wait")).toMatchObject({ + metrics: { start: 2, end: 2.6 }, + metadata: { "contrib.livekit.incomplete": true }, + }); + expect(one("eou_wait").metadata).not.toHaveProperty("eou.outcome"); + expect(one("livekit.eou_detection").metrics.end).toBe(2.6); + }, + ); + + test.each([undefined, NaN, Infinity, -1, 0, 3000])( + "missing/invalid speech end %s stays a native diagnostic", + async (at) => { + now = 2500; + const root = start("agent_session"); + const wait = start("eou_wait", root, { "lk.eou.source": "stt" }, 2500); + const recognition = { lastSpeakingTime: at }; + tracing.processSTT( + recognition, + { type: 2, speechEndTime: at, alternatives: [{ text: "hello" }] }, + () => {}, + ); + tracing.eouWait(recognition, wait, { + lastSpeakingTime: at, + trigger: "stt", + endpointingDelay: 0, + }); + wait.setAttribute("lk.eou.outcome", "committed"); + wait.end(2800); + const { one, named } = await rows(); + expect(named("eou_wait")).toHaveLength(0); + expect(one("livekit.eou_wait").metadata).toMatchObject({ + "contrib.livekit.eou.outcome": "committed", + "contrib.livekit.eou.canonical_omitted": "unobserved_speech_end", + }); + for (const key of [ + "eou.outcome", + "eou.speech_end_source", + "eou.end_reason", + "eou.detection_methods", + ]) + expect(one("livekit.eou_wait").metadata).not.toHaveProperty(key); + }, + ); + + test("STT receipt time and synthetic VAD notifications never establish a speech-end anchor", async () => { + const root = start("agent_session"); + now = 2500; + const recognition = { lastSpeakingTime: 2500, turnDetectionMode: "stt" }; + const wait = start("eou_wait", root, {}, 2500); + const result = tracing.processSTT(recognition, { type: 3 }, () => { + tracing.speechEnd( + { agentSession: {}, audioRecognition: recognition }, + { type: 2, silenceDuration: 0, inferenceDuration: 0 }, + ); + return "original result"; + }); + expect(result).toBe("original result"); + tracing.eouWait(recognition, wait, { + lastSpeakingTime: 2500, + trigger: "stt", + endpointingDelay: 0, + }); + wait.end(2800); + expect((await rows()).named("eou_wait")).toHaveLength(0); + }); + + test.each([undefined, 1900, 2900])( + "invalid/missing resume time %s cannot normalize an extended native wait", + async (at) => { + const root = start("agent_session"); + now = 2200; + const { wait } = vadWait(root, 2000); + if (at !== undefined) wait.addEvent("user_resumed", {}, at); + wait.setAttribute("lk.eou.outcome", "user_resumed"); + wait.end(2600); + const { one, named } = await rows(); + expect(named("eou_wait")).toHaveLength(0); + expect( + one("livekit.eou_wait").metadata[ + "contrib.livekit.eou.canonical_omitted" + ], + ).toBe("unobserved_wait_end"); + }, + ); +}); + describe("speech", () => { test("speech keeps native timing; transcripts never move it or claim it", async () => { const root = start("agent_session"); @@ -1542,7 +2207,7 @@ describe("recording", () => { await playOneFrame(sink, origin); // The session is still described; no audio is recorded. expect((await close()).audio).toEqual({ - "schema.version": "3.0.0", + "schema.version": "3.1.0", sources: SESSION_SOURCES, timeline: { origin_unix_ms: origin }, }); diff --git a/js/src/instrumentation/providers/livekit-plugin.ts b/js/src/instrumentation/providers/livekit-plugin.ts index 258730b39..9810b7fef 100644 --- a/js/src/instrumentation/providers/livekit-plugin.ts +++ b/js/src/instrumentation/providers/livekit-plugin.ts @@ -15,6 +15,9 @@ import type { PlaybackProgressedEvent, SpanTime, SpeechEvent, + SpeechEndEvent, + VadInferenceEvent, + EouWaitOptions, TraceOptions, } from "../../vendor-sdk-types/livekit"; import iso from "#braintrust-platform"; @@ -83,6 +86,7 @@ const AUDIO_DEFAULTS: AudioEncodingDefaults = { // Native spans with a Braintrust name. Other session spans export as `livekit.`. const SPANS = new Map([ ["agent_session", { name: "session", type: "task" }], + ["eou_wait", { name: "eou_wait", type: "task" }], ["user_turn", { name: "user_turn", type: "task" }], ["agent_turn", { name: "agent_turn", type: "task" }], ["user_speaking", { name: "user_speech", type: "task" }], @@ -169,17 +173,58 @@ class LiveKitInstrumentationConsumer { ); return Reflect.apply(target, self, args); }); - livekitChannels.speechStart.intercept((target, self, args) => - tracing.scope( + livekitChannels.speechStart.intercept((target, self, args) => { + runInstrumentation(() => tracing.vadSpeech(self as AgentActivity)); + return tracing.scope( (self as AgentActivity).audioRecognition?.userTurnSpan, () => Reflect.apply(target, self, args), - ), - ); + ); + }); livekitChannels.stt.intercept((target, self, args) => { - runInstrumentation(() => tracing.stt(self as AudioRecognition, args[0])); - return tracing.scope((self as AudioRecognition).userTurnSpan, () => + return tracing.processSTT(self as AudioRecognition, args[0], () => + tracing.scope((self as AudioRecognition).userTurnSpan, () => + Reflect.apply(target, self, args), + ), + ); + }); + livekitChannels.vadInference.intercept((target, self, args) => { + runInstrumentation(() => + tracing.vadSpeech(self as AgentActivity, args[0]), + ); + return Reflect.apply(target, self, args); + }); + livekitChannels.speechEnd.intercept((target, self, args) => { + const result = Reflect.apply(target, self, args); + runInstrumentation(() => + tracing.speechEnd(self as AgentActivity, args[0]), + ); + return result; + }); + livekitChannels.recognitionClose.intercept((target, self, args) => { + runInstrumentation(() => + tracing.recognitionClose(self as AudioRecognition), + ); + return Reflect.apply(target, self, args); + }); + livekitChannels.eouWait.intercept((target, self, args) => { + const span = tracing.waitFactory(self as AudioRecognition, () => Reflect.apply(target, self, args), ); + runInstrumentation(() => + tracing.eouWait(self as AudioRecognition, span, args[1]), + ); + return span; + }); + livekitChannels.eouEvaluation.intercept((target, self, args) => { + const result = Reflect.apply(target, self, args); + runInstrumentation(() => tracing.eouEvaluation(self as AudioRecognition)); + return result; + }); + livekitChannels.eouEnd.intercept((target, self, args) => { + runInstrumentation(() => + tracing.eouEnd(self as AudioRecognition, args[0], args[1]), + ); + return Reflect.apply(target, self, args); }); livekitChannels.userCompleted.intercept((target, self, args) => tracing.scope(args[0].userTurnSpan, () => @@ -224,6 +269,16 @@ type Row = { * of what was heard, so LiveKit's text for it may include unplayed words. */ unheard?: boolean; + /** A wait is canonical only with independently observed speech-end evidence. */ + speechEnd?: { at: number; source?: "vad" | "stt" }; + incomplete?: string; + /** Explicit native resume time; retained even if diagnostic events hit their cap. */ + resumedAt?: number; + recognitionClosing?: boolean; + /** Present only once a native wait is associated with this pause. Seconds. */ + nativeWaitStart?: number; + /** Observed wait completion when LiveKit skipped its native span. */ + waitOutcome?: string; /** Confirmed playout on the session timeline, for this speech's selections. */ played?: AudioInterval[]; messages?: Map; @@ -281,6 +336,17 @@ class LiveKitTracing { jobContext?: () => JobContext | undefined; // Callbacks already handed to LiveKit can outlive interceptor unsubscription. private disabled = false; + private speechEnds = new WeakMap< + AudioRecognition, + { source: "vad" } | { at: number; source?: "stt" } + >(); + // processSTTEvent synchronously calls onEndOfSpeech with a synthetic VAD event. + private processingSTT = new WeakMap< + AudioRecognition, + { waits: Map; evaluated: boolean } + >(); + private pauses = new WeakMap(); + private creatingWait = iso.newAsyncLocalStorage(); constructor( private streamingAttachmentBufferBytes: number, @@ -362,16 +428,37 @@ class LiveKitTracing { // LiveKit's span processors stamp its session metadata when a span starts. // Honor its redaction flag before starting publishes any of the span's content. const stamped = span.attributes?.[REDACTION_ENABLED] === true; - const row = this.start( - stamped - ? { - ...options, - attributes: { ...options.attributes, [REDACTION_ENABLED]: true }, - } - : options, - this.rowOf(parent), - ); + const recognition = this.creatingWait.getStore(); + const pause = recognition && this.pauses.get(recognition); + // A transcript-delayed pause may already have an exported row. Associate + // the native span before observing its children, without exporting a duplicate. + const pending = + options.name === "eou_wait" && + pause && + !pause.ended && + pause.nativeWaitStart === undefined + ? pause + : undefined; + const row = + pending ?? + this.start( + stamped + ? { + ...options, + attributes: { ...options.attributes, [REDACTION_ENABLED]: true }, + } + : options, + this.rowOf(parent), + ); if (!row) return; + if (options.name === "eou_wait") { + row.nativeWaitStart = spanTimeSeconds(options.startTime); + if (pending) { + for (const [key, value] of Object.entries(options.attributes ?? {})) + row.attributes.set(key, value); + if (stamped) row.attributes.set(REDACTION_ENABLED, true); + } + } observedRows.set(span, row); this.observeRedaction(row); // Only model requests inside an llm_node contribute to their exported owner. @@ -435,18 +522,25 @@ class LiveKitTracing { time?: SpanTime, ) { const result = addEvent.call(this, name, attributesOrTime, time); - if (!row.ended && row.events.length < MAX_EVENTS) { + if (!row.ended) { const attributes = isObject(attributesOrTime) && !(attributesOrTime instanceof Date) ? attributesOrTime : undefined; - row.events.push({ - name, - attributes, - time: spanTimeSeconds( - time ?? (attributes ? undefined : (attributesOrTime as SpanTime)), - ), - }); + const at = + time ?? (attributes ? undefined : (attributesOrTime as SpanTime)); + if ( + row.name === "eou_wait" && + name === "user_resumed" && + at !== undefined + ) + row.resumedAt = spanTimeSeconds(at); + if (row.events.length < MAX_EVENTS) + row.events.push({ + name, + attributes, + time: spanTimeSeconds(at), + }); } return result; }, @@ -476,7 +570,13 @@ class LiveKitTracing { return; } } - const kind = SPANS.get(name) ?? { + // A native wait can start at notification time when no speech-end anchor exists. + // Retain it as a framework diagnostic until evidence permits normalization at end. + if (name === "eou_wait" && parentState) { + const turn = parentState.rows.get(parent?.turn ?? ""); + parent = turn?.name === "user_turn" ? turn : parentState.root; + } + const kind = (name === "eou_wait" ? undefined : SPANS.get(name)) ?? { name: `livekit.${name}`, type: "task" as const, }; @@ -521,7 +621,9 @@ class LiveKitTracing { const capturing = captures.input || captures.output; const origin = Date.now(); const audio = this.audioSession(row.span, origin, capturing); - row.span.log({ metadata: audio.session() }); + row.span.log({ + metadata: { ...audio.session(), "audio.schema.version": "3.1.0" }, + }); const transcripts: SessionState["transcripts"] = new Map(); this.sessions.set(id, { root: row, @@ -602,6 +704,20 @@ class LiveKitTracing { } private end(row: Row): void { + // Native close() also labels an undecided wait "dropped". Teardown alone + // establishes no abandonment; preserve that outcome only as a diagnostic. + if ( + row.recognitionClosing && + row.attributes.get("lk.eou.outcome") === "dropped" + ) { + row.incomplete = "recognition_closed"; + row.span.log({ + metadata: { + "contrib.livekit.incomplete": true, + "contrib.livekit.termination": row.incomplete, + }, + }); + } // Native identities resolve links in memory; only redacted values are logged. const identities = nativeFields(row.attributes); const native = this.redact(row); @@ -609,11 +725,13 @@ class LiveKitTracing { const events = this.redactEvents(row); if (row.status.message && !this.piiAllowed(row)) row.status = { ...row.status, message: REDACTED_EXCEPTION_MESSAGE }; - row.attributes = new Map(); - row.events = []; const state = this.sessions.get(row.session); if (!row.owner) { row.span.log(endEvent(row, attrs, events)); + if (state && row.name === "eou_wait") { + const wait = this.waitEvent(row, native); + if (wait) row.span.log(wait); + } if (state) state.relationships.batch(row.span, () => correlate(row, identities, state.relationships), @@ -630,13 +748,19 @@ class LiveKitTracing { }); if (state && row.name === "agent_speaking") row.span.log({ metadata: participantIds(state, native) }); - row.span.end({ endTime: row.end }); + row.span.end({ endTime: this.waitEnd(row) ?? row.end }); } else { // Native request spans duplicate the public inference lifecycle. row.span.log(modelFields(attrs)); } + row.attributes = new Map(); + row.events = []; if (row.name === "agent_session") void this.closeSession(row.id); - if (state?.closed) this.clearSession(row.session); + if ( + state?.closed && + (!row.incomplete || row.incomplete === "recognition_closed") + ) + this.clearSession(row.session); } /** @@ -679,6 +803,286 @@ class LiveKitTracing { })); } + /** Invalidate pause evidence before VAD writes a speech-activity clock value. */ + vadSpeech(activity: AgentActivity, event?: VadInferenceEvent): void { + if (activity.audioRecognition && (!event || event.rawAccumulatedSpeech > 0)) + this.speechEnds.delete(activity.audioRecognition); + } + + /** + * In 1.9.1/1.10.0 the VAD loop writes its corrected lastSpeakingTime immediately + * after this callback. Keep provenance, then read that exact value at the wait + * factory. STT and VAD speech-activity writes invalidate or replace it below. + */ + speechEnd(activity: AgentActivity, event?: SpeechEndEvent): void { + const recognition = activity.audioRecognition; + if (!recognition || this.processingSTT.has(recognition)) return; + this.speechEnds.delete(recognition); + if ( + !event || + event.type !== 2 || + !Number.isFinite(event.silenceDuration) || + event.silenceDuration < 0 || + !Number.isFinite(event.inferenceDuration) || + event.inferenceDuration < 0 + ) + return; + this.speechEnds.set(recognition, { source: "vad" }); + } + + processSTT( + recognition: AudioRecognition, + event: SpeechEvent, + fn: () => T, + ): T { + // Manual wrapping can surround an already transformed method. Observe the + // outer invocation once; its synchronous callback owns provenance tracking. + if (this.processingSTT.has(recognition)) return fn(); + runInstrumentation(() => this.stt(recognition, event)); + const observation = runInstrumentation(() => { + const alternative = event.alternatives?.[0]; + // Match the supported framework's assignment conditions, including explicit + // STT timestamps taking precedence over VAD only in STT turn detection. + const hasTime = + event.speechEndTime !== undefined || + (alternative?.endTime !== undefined && + alternative.endTime > 0 && + recognition.inputStartedAt !== undefined); + const updates = + event.type === 0 + ? recognition.turnDetectionMode === "stt" + : event.type === 3 + ? recognition.turnDetectionMode === "stt" + : (event.type === 2 || event.type === 5) && + !!alternative?.text && + (recognition.vad === undefined || + recognition.lastSpeakingTime === undefined || + (recognition.turnDetectionMode === "stt" && hasTime)); + const at = + event.speechEndTime ?? + (alternative?.endTime !== undefined && + alternative.endTime > 0 && + recognition.inputStartedAt !== undefined + ? recognition.inputStartedAt + alternative.endTime * 1000 + : undefined); + // StreamAdapter can supply a VAD-derived speechEndTime on STT events. + // That absolute timestamp is a valid anchor, but its origin is unknown + // here. Only an unaugmented provider-relative end establishes STT origin. + const source = + event.speechEndTime === undefined ? ("stt" as const) : undefined; + return { updates, at, source }; + }); + const invocation = { + waits: new Map(), + evaluated: false, + }; + this.processingSTT.set(recognition, invocation); + let completed = false; + try { + const result = fn(); + completed = true; + return result; + } finally { + this.processingSTT.delete(recognition); + runInstrumentation(() => { + if (!completed) { + this.speechEnds.delete(recognition); + return; + } + if (observation?.updates) { + this.speechEnds.delete(recognition); + // START_OF_SPEECH sets a clock value, but does not observe speech end. + const { at } = observation; + // Reject the framework's notification-time and future-clock fallbacks. + if ( + event.type !== 0 && + validSpeechEnd(at) && + recognition.lastSpeakingTime === at + ) + this.speechEnds.set(recognition, { + at, + source: observation.source, + }); + } + // 1.9.1 creates the native wait synchronously inside processSTTEvent; + // 1.10.0 defers creation. Trust the anchor only after the callback adopts it. + for (const [span, options] of invocation.waits) + this.eouWait(recognition, span, options); + if (invocation.evaluated) this.eouEvaluation(recognition); + }); + } + } + + recognitionClose(recognition: AudioRecognition): void { + const row = + this.rowOf(recognition.eouWaitSpan) ?? this.pauses.get(recognition); + if (row && !row.ended) row.recognitionClosing = true; + } + + waitFactory(recognition: AudioRecognition, fn: () => T): T { + return this.creatingWait.run(recognition, fn); + } + + /** Observe the framework's actual no-transcript guard, after its VAD clock write. */ + eouEvaluation(recognition: AudioRecognition): void { + const invocation = this.processingSTT.get(recognition); + if (invocation) { + invocation.evaluated = true; + return; + } + const pause = this.pauses.get(recognition); + if ( + recognition.closed || + (pause && !pause.ended) || + recognition.eouWaitSpan?.isRecording() || + !recognition.stt || + recognition.audioTranscript !== "" || + recognition.turnDetectionMode === "manual" + ) + return; + // 1.10.0 allows a first evaluation to flush a manual-flush STT pipeline. + // That path did not take the missing-transcript guard; let its native factory run. + if ( + recognition.vadBaseTurnDetection && + recognition.sttPipeline?.manualFlush === true && + !recognition.sttFlushRequested + ) + return; + const at = recognition.lastSpeakingTime; + const evidence = this.speechEnds.get(recognition); + if ( + !validSpeechEnd(at) || + !evidence || + (evidence.source !== "vad" && evidence.at !== at) + ) + return; + const parent = + this.openTurn(recognition) ?? + this.rowOf( + recognition.rootSpanContext?.getValue(OTEL_SPAN_KEY) as + | LiveKitSpan + | undefined, + ); + const row = this.start({ name: "eou_wait", startTime: at }, parent); + if (!row) return; + row.speechEnd = { at, source: evidence.source }; + row.span.log({ + metadata: { "contrib.livekit.eou.transcription_pending": true }, + }); + this.pauses.set(recognition, row); + } + + /** Native endEouWaitSpan is called on resume/abandonment even without a native wait. */ + eouEnd( + recognition: AudioRecognition, + outcome: string, + endTime?: number, + ): void { + const row = this.pauses.get(recognition); + if (!row || row.ended || row.nativeWaitStart !== undefined) return; + row.ended = true; + row.end = (endTime ?? Date.now()) / 1000; + if (row.recognitionClosing || recognition.closed) { + row.incomplete = "recognition_closed"; + row.span.log({ + metadata: { + "contrib.livekit.incomplete": true, + "contrib.livekit.termination": row.incomplete, + }, + }); + } else { + row.waitOutcome = outcome; + if (outcome === "user_resumed") row.resumedAt = row.end; + } + this.end(row); + } + + /** Called after ensureEouWaitSpan: rearms update evidence on the same row. */ + eouWait( + recognition: AudioRecognition, + span: LiveKitSpan, + options: EouWaitOptions, + ): void { + const row = this.rowOf(span); + if (!row || row.ended || row.name !== "eou_wait") return; + this.pauses.set(recognition, row); + this.processingSTT.get(recognition)?.waits.set(span, options); + row.recognitionClosing ||= recognition.closed === true; + const evidence = this.speechEnds.get(recognition); + const state = this.sessions.get(row.session)!; + const turn = state.rows.get(row.turn ?? ""); + if (turn) + row.span.log({ metadata: participantIds(state, turn.attributes) }); + row.speechEnd = + validSpeechEnd(options.lastSpeakingTime) && + evidence && + (evidence.source === "vad" || evidence.at === options.lastSpeakingTime) + ? { at: options.lastSpeakingTime, source: evidence.source } + : undefined; + } + + private waitEnd(row: Row): number | undefined { + if (row.name !== "eou_wait" || !row.speechEnd) return; + const start = row.speechEnd.at / 1000; + if (row.end === undefined || !Number.isFinite(row.end) || row.end < start) + return; + if ( + !row.incomplete && + (row.waitOutcome ?? row.attributes.get("lk.eou.outcome")) === + "user_resumed" + ) { + const resumed = row.resumedAt; + return resumed !== undefined && + Number.isFinite(resumed) && + resumed >= start && + resumed <= row.end + ? resumed + : undefined; + } + return row.end; + } + + private waitEvent(row: Row, native: Map) { + // waitEnd needs the native outcome even after the row's attributes are released. + const outcome = row.waitOutcome ?? native.get("lk.eou.outcome"); + const end = this.waitEnd(row); + if (end === undefined || !row.speechEnd) + return { + metadata: { + "contrib.livekit.eou.canonical_omitted": !row.speechEnd + ? "unobserved_speech_end" + : "unobserved_wait_end", + }, + }; + const state = this.sessions.get(row.session)!; + const turn = state.rows.get(row.turn ?? ""); + return { + span_attributes: SPANS.get("eou_wait")!, + metrics: { start: row.speechEnd.at / 1000 }, + metadata: { + "audio.source_id": "session-input", + ...participantIds( + state, + turn?.name === "user_turn" ? turn.attributes : native, + ), + ...(row.speechEnd.source + ? { "eou.speech_end_source": row.speechEnd.source } + : {}), + ...(!row.incomplete && + ["committed", "user_resumed", "dropped"].includes(outcome as string) + ? { "eou.outcome": outcome } + : {}), + ...(row.nativeWaitStart !== undefined + ? { + "contrib.livekit.eou.native_start_unix_ms": + row.nativeWaitStart * 1000, + "contrib.livekit.eou.native_end_unix_ms": row.end! * 1000, + } + : {}), + }, + }; + } + scope( native: LiveKitSpan | undefined, fn: () => T, @@ -933,17 +1337,41 @@ class LiveKitTracing { const state = this.sessions.get(id); // Native children (notably tools) can outlive session end. Retain ownership until // every mapped operation finishes, even after the recording has finalized. - // Other native spans don't hold the session: LiveKit can leave them open. - // Their rows still log and end on their own if LiveKit ends them later. + // Waits and other native diagnostics don't hold the session: LiveKit can + // leave them open. Finalize their exported rows as incomplete on cleanup. if ( !state || - [...state.rows.values()].some((row) => !row.ended && SPANS.has(row.name)) + [...state.rows.values()].some( + (row) => !row.ended && SPANS.has(row.name) && row.name !== "eou_wait", + ) ) return; + for (const row of state.rows.values()) { + if (!row.ended) this.endIncomplete(row, state.root.end, "session_closed"); + } state.speech.finish(state.root.span); this.sessions.delete(id); } + private endIncomplete( + row: Row, + end: number | undefined, + termination: string, + ): void { + row.ended = true; + row.end = Math.max(row.start, end ?? Date.now() / 1000); + row.incomplete = termination; + row.span.log({ + metadata: { + [row.turn === row.id + ? "turn.incomplete" + : "contrib.livekit.incomplete"]: true, + "contrib.livekit.termination": termination, + }, + }); + this.end(row); + } + async shutdown(): Promise { this.disabled = true; await Promise.all( @@ -952,16 +1380,7 @@ class LiveKitTracing { for (const state of this.sessions.values()) { for (const row of state.rows.values()) { if (row.ended) continue; - row.ended = true; - row.span.log({ - metadata: { - [row.turn === row.id - ? "turn.incomplete" - : "contrib.livekit.incomplete"]: true, - "contrib.livekit.termination": "processor_shutdown", - }, - }); - row.span.end(); + this.endIncomplete(row, undefined, "processor_shutdown"); } state.speech.finish(state.root.span); } @@ -969,6 +1388,10 @@ class LiveKitTracing { } } +function validSpeechEnd(at: number | undefined): at is number { + return at !== undefined && Number.isFinite(at) && at > 0 && at <= Date.now(); +} + function spanTimeSeconds(time: SpanTime = Date.now()): number { if (Array.isArray(time)) return time[0] + time[1] / 1e9; return (time instanceof Date ? time.getTime() : time) / 1000; diff --git a/js/src/vendor-sdk-types/livekit.ts b/js/src/vendor-sdk-types/livekit.ts index cbedf52ad..ac76f5b39 100644 --- a/js/src/vendor-sdk-types/livekit.ts +++ b/js/src/vendor-sdk-types/livekit.ts @@ -24,9 +24,19 @@ export interface AgentActivity { currentSpeech?: SpeechHandle; } export interface AudioRecognition { + eouWaitSpan?: LiveKitSpan; + stt?: unknown; + audioTranscript?: string; + vadBaseTurnDetection?: boolean; + sttPipeline?: { manualFlush?: boolean }; + sttFlushRequested?: boolean; + closed?: boolean; userTurnSpan?: LiveKitSpan; rootSpanContext?: { getValue(key: symbol): unknown }; inputStartedAt?: number; + lastSpeakingTime?: number; + vad?: unknown; + turnDetectionMode?: string; sttModel?: string; sttProvider?: string; sttIdentity?: () => { model?: string; provider?: string } | undefined; @@ -38,11 +48,14 @@ export interface SpeechEvent { confidence?: number; language?: string; startTime?: number; + endTime?: number; /** A diarizing STT's label for the voice, scoped to its input. */ speakerId?: string | null; }[]; requestId?: string; createdAt?: number; + /** Absolute observed speech end, in Unix milliseconds. */ + speechEndTime?: number; } export interface PlaybackStartedEvent { createdAt: number; @@ -77,6 +90,20 @@ export interface AudioOutput { ): unknown; off(event: string, callback: (...args: never[]) => void): unknown; } +/** Only the VAD timing fields observed by instrumentation. Durations are ms. */ +export interface SpeechEndEvent { + type: number; + silenceDuration: number; + inferenceDuration: number; +} +export interface VadInferenceEvent { + rawAccumulatedSpeech: number; +} +export interface EouWaitOptions { + trigger: string; + lastSpeakingTime: number | undefined; + endpointingDelay: number; +} export interface EndOfTurnInfo { userTurnSpan?: LiveKitSpan; startedSpeakingAt?: number; diff --git a/js/src/wrappers/livekit.ts b/js/src/wrappers/livekit.ts index 32ef749ca..ac73f08fb 100644 --- a/js/src/wrappers/livekit.ts +++ b/js/src/wrappers/livekit.ts @@ -43,16 +43,39 @@ function wrapActivity(activity: AgentActivity | undefined) { } // The recognizer exists once the activity starts, before it receives audio. wrapMethod(activity, "attachAudioInput", livekitChannels.input, () => { - if (activity.audioRecognition) + if (activity.audioRecognition) { + wrapMethod( + activity.audioRecognition, + "close", + livekitChannels.recognitionClose, + ); + wrapMethod( + activity.audioRecognition, + "ensureEouWaitSpan", + livekitChannels.eouWait, + ); + wrapMethod( + activity.audioRecognition, + "runEOUDetection", + livekitChannels.eouEvaluation, + ); + wrapMethod( + activity.audioRecognition, + "endEouWaitSpan", + livekitChannels.eouEnd, + ); wrapMethod( activity.audioRecognition, "processSTTEvent", livekitChannels.stt, ); + } }); wrapMethod(activity, "_startSessionImpl", livekitChannels.activity); wrapMethod(activity, "onEndOfTurn", livekitChannels.userTurn); wrapMethod(activity, "onStartOfSpeech", livekitChannels.speechStart); + wrapMethod(activity, "onEndOfSpeech", livekitChannels.speechEnd); + wrapMethod(activity, "onVADInferenceDone", livekitChannels.vadInference); wrapMethod(activity, "userTurnCompleted", livekitChannels.userCompleted); wrapMethod(activity, "_pipelineReplyTaskImpl", livekitChannels.pipeline); }