Skip to content

Commit eb59da8

Browse files
authored
feat(audio): Add AudioSessionRecorder for audio schema v3 sessions (#2725)
Based on #2724 ## Why Every voice integration has to describe its session the same way under audio schema v3: sources, a shared timeline, participants, speakers, recorded segments, and each span's place on that timeline. `AudioSessionRecorder` owns that shape once, so integrations only translate their framework's events into calls. - **Provides common implementation for session shape** to reduce duplication per integration. - **Provides session-local IDs** (`participant-N`, `speaker-N`) derived from native values. - **With audio capture disabled, it can still describes the session**, so timing and identities can still be relayed on the trace. In sum, this enables LiveKit's move to schema v3 (next PR), and future voice integrations. ## Example How an integration uses it: ```ts const audio = new AudioSessionRecorder({ origin: Date.now(), sources: { "session-input": { label: "Agent input audio" }, "agent-output": { label: "Agent audio" }, }, audio: { sampleRate: 48_000, segmentBytes }, // omit to describe without recording async onFile({ id, data, mimeType, metadata }) { // Upload `data` as input.audio[id], then log it with its audio.segments entry. session.log({ input: { audio: { [id]: await upload(data, mimeType) } }, metadata }); }, }); session.log({ metadata: audio.session() }); // schema version, sources, timeline const { id, added } = audio.participant("PA_x7Kq", { type: "user", attributes: { "contrib.livekit.participant_id": "PA_x7Kq" }, }); if (added) session.log({ metadata: added }); // audio.participants, first time only span.log({ metadata: { "audio.participant_ids": [id] } }); audio.record("session-input", pcm, { sampleRate: 16_000, channelCount: 1, offsetMs }); span.log({ metadata: { "audio.selections": audioSelections([{ start, end }]) } }); await audio.finish(); ``` ## Design notes ``` integration ──calls──► AudioSessionRecorder ──returns──► metadata to log │ └─ AudioRecorder (when recording) ─► onFile(segment + entry) ``` - The `AudioSessionRecorder` is responsible for managing the session, resolving references, etc but it is not responsible for instrumentation or trace data itself... it merely returns metadata that instrumentation can log. - It tracks commonly referenced objects like `participant` and `speaker`, and helps spans map to a common, PII-less ID. (e.g. `participant-1`)
1 parent 70b861b commit eb59da8

5 files changed

Lines changed: 570 additions & 5 deletions

File tree

‎js/src/audio-recorder.ts‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,15 @@ type AudioRecorderOptions = {
1919
middleware?: AudioMiddlewareStack;
2020
/** Receives each transformed segment; the recorder waits for it to settle. */
2121
onFlush(segment: Uint8Array, meta: AudioSegmentMeta): void | Promise<void>;
22-
/** Receives the original metadata of a segment that failed, and the stop reason. */
23-
onError(meta: AudioSegmentMeta, reason: string, error: unknown): void;
22+
/**
23+
* Receives the original metadata of a segment that failed, and the stop
24+
* reason; the recorder waits for it to settle.
25+
*/
26+
onError(
27+
meta: AudioSegmentMeta,
28+
reason: string,
29+
error: unknown,
30+
): void | Promise<void>;
2431
};
2532

2633
const PCM_ATTRIBUTES = {
@@ -216,14 +223,14 @@ export class AudioRecorder {
216223
try {
217224
await this.options.onFlush(result.segment, result.meta);
218225
} catch (error) {
219-
this.fail("recording_export_failed", error, meta);
226+
await this.fail("recording_export_failed", error, meta);
220227
}
221228
}
222229

223-
private fail(reason: string, error: unknown, meta: AudioSegmentMeta) {
230+
private async fail(reason: string, error: unknown, meta: AudioSegmentMeta) {
224231
this.stop(reason);
225232
try {
226-
this.options.onError(meta, reason, error);
233+
await this.options.onError(meta, reason, error);
227234
} catch {
228235
// Reporting must not break draining.
229236
}

‎js/src/audio-selections.test.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import { expect, test } from "vitest";
2+
import { audioSelections } from "./audio-selections";
3+
4+
test("intervals become whole-millisecond selections, in timeline order", () => {
5+
expect(
6+
audioSelections([
7+
{ start: 280.4, end: 290.6 },
8+
{ start: 10, end: 20 },
9+
]),
10+
).toEqual({
11+
"selection-0": { start_offset_ms: 10, end_offset_ms: 20 },
12+
"selection-1": { start_offset_ms: 280, end_offset_ms: 291 },
13+
});
14+
});
15+
16+
test("overlapping and touching intervals merge; empty ones are dropped", () => {
17+
expect(
18+
audioSelections([
19+
{ start: 50.4, end: 120.2 },
20+
{ start: 120.2, end: 250.6 }, // Touches the previous one.
21+
{ start: 200, end: 230 }, // Inside the merged one.
22+
{ start: 260, end: 260 }, // Empty.
23+
{ start: 300.2, end: 300.4 }, // Empty once rounded.
24+
]),
25+
).toEqual({ "selection-0": { start_offset_ms: 50, end_offset_ms: 251 } });
26+
});
27+
28+
test("no intervals means no selections", () => {
29+
expect(audioSelections([])).toBeUndefined();
30+
expect(audioSelections([{ start: 5, end: 5 }])).toBeUndefined();
31+
});

‎js/src/audio-selections.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/** An observed interval on the session timeline, in milliseconds from its origin. */
2+
type AudioInterval = { start: number; end: number };
3+
4+
type AudioSelection = { start_offset_ms: number; end_offset_ms: number };
5+
6+
/**
7+
* A span's `audio.selections`: its intervals on the session timeline in whole
8+
* milliseconds, merged where they overlap or touch, in timeline order.
9+
*/
10+
export function audioSelections(
11+
intervals: AudioInterval[],
12+
): Record<string, AudioSelection> | undefined {
13+
const merged: AudioSelection[] = [];
14+
for (const interval of intervals
15+
.map(({ start, end }) => ({
16+
start_offset_ms: Math.round(start),
17+
end_offset_ms: Math.round(end),
18+
}))
19+
.filter((s) => s.end_offset_ms > s.start_offset_ms)
20+
.sort((a, b) => a.start_offset_ms - b.start_offset_ms)) {
21+
const last = merged.at(-1);
22+
if (last && interval.start_offset_ms <= last.end_offset_ms)
23+
last.end_offset_ms = Math.max(last.end_offset_ms, interval.end_offset_ms);
24+
else merged.push(interval);
25+
}
26+
if (!merged.length) return;
27+
return Object.fromEntries(
28+
merged.map((selection, index) => [`selection-${index}`, selection]),
29+
);
30+
}
Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
import { expect, test, vi } from "vitest";
2+
import { AudioSessionRecorder, type AudioFile } from "./audio-session-recorder";
3+
4+
const SOURCES = {
5+
mic: { label: "User audio" },
6+
speaker: { label: "Agent audio" },
7+
};
8+
/** 1 kHz recordings with 100 ms (200 byte stereo) segments; records every file handed off. */
9+
function recorder(
10+
store: (file: AudioFile) => Promise<void> = async () => {},
11+
audio = { sampleRate: 1000, segmentBytes: 400 },
12+
) {
13+
const files: AudioFile[] = [];
14+
const audioSession = new AudioSessionRecorder({
15+
origin: 1_790_000_000_000,
16+
sources: SOURCES,
17+
audio,
18+
onFile: async (file) => {
19+
await store(file);
20+
files.push(file);
21+
},
22+
});
23+
return { audioSession, files };
24+
}
25+
const mono = (offsetMs: number) => ({
26+
sampleRate: 1000,
27+
channelCount: 1,
28+
offsetMs,
29+
});
30+
const ramp = (length: number) => Int16Array.from({ length }, (_, i) => i + 1);
31+
/** Every segment entry handed off, merged in order as a session span would. */
32+
const segments = (files: AudioFile[]) =>
33+
Object.assign(
34+
{},
35+
...files.map(
36+
(file) => file.metadata["audio.segments"] as Record<string, unknown>,
37+
),
38+
);
39+
40+
test("describes a session without audio: schema, sources and timeline, and no files", async () => {
41+
const onFile = vi.fn();
42+
const audioSession = new AudioSessionRecorder({
43+
origin: 5,
44+
sources: SOURCES,
45+
onFile,
46+
});
47+
expect(audioSession.session()).toEqual({
48+
"audio.schema.version": "3.0.0",
49+
"audio.sources": SOURCES,
50+
"audio.timeline": { origin_unix_ms: 5 },
51+
});
52+
expect(audioSession.record("mic", ramp(10), mono(0))).toBe(false);
53+
expect(audioSession.active).toBe(false);
54+
await audioSession.finish();
55+
expect(onFile).not.toHaveBeenCalled();
56+
});
57+
58+
test("a participant's entry is returned on first sight only; its ID is stable", () => {
59+
const { audioSession } = recorder();
60+
const caller = {
61+
type: "user" as const,
62+
attributes: { "contrib.example.id": "PA_caller" },
63+
};
64+
expect(audioSession.participant("PA_caller", caller)).toEqual({
65+
id: "participant-1",
66+
added: { "audio.participants": { "participant-1": caller } },
67+
});
68+
expect(audioSession.participant("PA_agent", { type: "agent" }).added).toEqual(
69+
{ "audio.participants": { "participant-2": { type: "agent" } } },
70+
);
71+
// Seen again: the same ID, and nothing new to publish.
72+
expect(audioSession.participant("PA_caller", caller)).toEqual({
73+
id: "participant-1",
74+
});
75+
});
76+
77+
test("speakers are scoped to their source and kept apart from participants", () => {
78+
const { audioSession } = recorder();
79+
const voice = (source: string, nativeId: string) =>
80+
audioSession.speaker(source, nativeId, { type: "user" });
81+
expect(voice("mic", "0")).toEqual({
82+
id: "speaker-1",
83+
added: { "audio.speakers": { "speaker-1": { type: "user" } } },
84+
});
85+
expect(voice("mic", "1").id).toBe("speaker-2");
86+
expect(voice("mic", "0")).toEqual({ id: "speaker-1" });
87+
// The same label on another source is a different voice.
88+
expect(voice("speaker", "0").id).toBe("speaker-3");
89+
// A participant with the same native ID is a different member.
90+
expect(audioSession.participant("0", { type: "user" }).id).toBe(
91+
"participant-1",
92+
);
93+
});
94+
95+
test("each recorded file is handed off with its ready entry", async () => {
96+
let release!: () => void;
97+
const gate = new Promise<void>((resolve) => (release = resolve));
98+
const { audioSession, files } = recorder(() => gate);
99+
audioSession.record("mic", ramp(150), mono(0));
100+
audioSession.record("speaker", ramp(150), mono(0));
101+
audioSession.advance(150);
102+
const finishing = audioSession.finish();
103+
await new Promise((resolve) => setTimeout(resolve, 5));
104+
// Nothing is described before it is stored.
105+
expect(files).toEqual([]);
106+
release();
107+
await finishing;
108+
expect(files.map((f) => [f.id, f.mimeType, f.data?.byteLength])).toEqual([
109+
["part-0", "audio/pcm", 400],
110+
["part-1", "audio/pcm", 200],
111+
]);
112+
expect(files[0].metadata).toEqual({
113+
"audio.segments": {
114+
"part-0": {
115+
state: "ready",
116+
start_offset_ms: 0,
117+
duration_ms: 100,
118+
attachment: { ref: "/input/audio/part-0" },
119+
mime_type: "audio/pcm",
120+
encoding: "pcm",
121+
sample_rate_hz: 1000,
122+
channels: { "0": { sources: ["mic"] }, "1": { sources: ["speaker"] } },
123+
attributes: {
124+
"pcm.sample_format": "s16le",
125+
"pcm.channel_layout": "interleaved",
126+
},
127+
},
128+
},
129+
});
130+
// The final partial segment.
131+
expect(segments(files)["part-1"]).toMatchObject({
132+
state: "ready",
133+
start_offset_ms: 100,
134+
duration_ms: 50,
135+
});
136+
});
137+
138+
test("whole-millisecond segments tile without drift", async () => {
139+
// 24 kHz, 1001 frames per segment: 41.708… ms each.
140+
const { audioSession, files } = recorder(undefined, {
141+
sampleRate: 24000,
142+
segmentBytes: 1001 * 4,
143+
});
144+
audioSession.record("mic", ramp(1001 * 5), {
145+
sampleRate: 24000,
146+
channelCount: 1,
147+
offsetMs: 0,
148+
});
149+
await audioSession.finish();
150+
const placed = Object.values(segments(files)) as {
151+
start_offset_ms: number;
152+
duration_ms: number;
153+
}[];
154+
expect(placed).toHaveLength(5);
155+
for (let i = 1; i < placed.length; i++)
156+
expect(placed[i].start_offset_ms).toBe(
157+
placed[i - 1].start_offset_ms + placed[i - 1].duration_ms,
158+
);
159+
expect(placed.at(-1)!.start_offset_ms + placed.at(-1)!.duration_ms).toBe(
160+
Math.round((1001 * 5) / 24),
161+
);
162+
});
163+
164+
test("a failed store is offered again as omitted and stops new audio; accepted audio still stores", async () => {
165+
let calls = 0;
166+
const offered: AudioFile[] = [];
167+
const audioSession = new AudioSessionRecorder({
168+
origin: 0,
169+
sources: SOURCES,
170+
audio: { sampleRate: 1000, segmentBytes: 400 },
171+
onFile: async (file) => {
172+
offered.push(file);
173+
if (file.data && ++calls === 2) throw new Error("offline");
174+
},
175+
});
176+
audioSession.record("mic", ramp(300), mono(0));
177+
audioSession.advance(300);
178+
await audioSession.finish();
179+
expect(offered.map((f) => [f.id, f.data !== undefined])).toEqual([
180+
["part-0", true],
181+
["part-1", true],
182+
["part-1", false],
183+
["part-2", true],
184+
]);
185+
expect(offered[2].metadata).toEqual({
186+
"audio.segments": {
187+
"part-1": {
188+
state: "omitted",
189+
start_offset_ms: 100,
190+
duration_ms: 100,
191+
reason: "recording_export_failed",
192+
},
193+
},
194+
});
195+
expect(audioSession.reason).toBe("recording_export_failed");
196+
expect(audioSession.record("mic", ramp(10), mono(300))).toBe(false);
197+
});
198+
199+
test("failing middleware omits the segment with the recorder's reason", async () => {
200+
vi.spyOn(console, "warn").mockImplementation(() => {});
201+
const files: AudioFile[] = [];
202+
const audioSession = new AudioSessionRecorder({
203+
origin: 0,
204+
sources: SOURCES,
205+
audio: {
206+
sampleRate: 1000,
207+
segmentBytes: 400,
208+
middleware: [
209+
() => {
210+
throw new Error("encoder unavailable");
211+
},
212+
],
213+
},
214+
onFile: async (file) => void files.push(file),
215+
});
216+
audioSession.record("mic", ramp(50), mono(0));
217+
await audioSession.finish();
218+
expect(files).toEqual([
219+
{
220+
id: "part-0",
221+
metadata: {
222+
"audio.segments": {
223+
"part-0": {
224+
state: "omitted",
225+
start_offset_ms: 0,
226+
duration_ms: 50,
227+
reason: "recording_transform_failed",
228+
},
229+
},
230+
},
231+
},
232+
]);
233+
vi.restoreAllMocks();
234+
});
235+
236+
test("finish is idempotent and later audio is refused", async () => {
237+
const { audioSession } = recorder();
238+
audioSession.record("mic", ramp(10), mono(0));
239+
const finishing = audioSession.finish();
240+
expect(audioSession.finish()).toBe(finishing);
241+
await finishing;
242+
expect(audioSession.record("mic", ramp(10), mono(10))).toBe(false);
243+
});

0 commit comments

Comments
 (0)