Skip to content

Commit 70b861b

Browse files
authored
fix(livekit): Respect LiveKit's PII and project redaction settings (#2724)
## Problem LiveKit can be configured to withhold PII from its telemetry, but Braintrust's LiveKit traces ignored that setting. The plugin reads LiveKit's span attributes and events before LiveKit's own redaction runs, so traces still carried transcripts, prompts, model responses, tool arguments and results, and exception details. LiveKit withholds PII when any of these apply: - `LIVEKIT_TELEMETRY_ALLOW_PII` is falsy (`0`, `false`, `no`, `off`) - the job's LiveKit Cloud project enables redaction - a span carries `lk.redaction.enabled` ## Solution When LiveKit's configuration withholds PII, the plugin applies LiveKit's own redaction rule before anything is logged. **When PII is disallowed it omits:** - `lk.*pii*` attributes, gen_ai content attributes, prompt variables and message events - transcripts, assistant and heard text, and tool definitions - exception messages and stack traces (errors show `exception details redacted`) **When PII is disallowed it retains:** - span structure, timing, models, usage, tool choice, participant IDs - opt-in audio (governed by `BRAINTRUST` flags). NOTE: `setTracerProvider(..., { allowPii: false })`, is **not** covered because LiveKit doesn't expose it to the plugin. Separately, this also fixes the LiveKit e2e websocket cassette being saved after the scenario exited.
1 parent 2a4b1d1 commit 70b861b

12 files changed

Lines changed: 565 additions & 22 deletions

File tree

‎.changeset/livekit-pii.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"braintrust": patch
3+
---
4+
5+
fix(livekit): Respect LiveKit's PII and project redaction settings
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
// Runs the scenario inside a LiveKit job whose project enables redaction, as
2+
// LiveKit Cloud sets it, so instrumentation must honor LiveKit's PII setting.
3+
// The job is minimal: only the members LiveKit reads outside a real worker.
4+
import { runWithJobContextAsync } from "@livekit/agents";
5+
6+
const job = {
7+
_redactionEnabled: true,
8+
job: { id: "job_redacted", agentName: "", enableRecording: false },
9+
room: { isConnected: false },
10+
simulationContext: () => undefined,
11+
_otelMetadata: () => ({}),
12+
};
13+
await runWithJobContextAsync(job, () => import("./scenario.mjs"));

‎e2e/scenarios/livekit-instrumentation/scenario.test.ts‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,17 @@ type RunOptions = {
8585
cassette?: string;
8686
recording?: boolean;
8787
};
88+
/** The span IDs a span links to with this relationship, in order. */
89+
function linkTargets(span: CapturedLogEvent | undefined, type: string) {
90+
return (
91+
(span?.metadata?.links ?? []) as {
92+
context: { spanId: string };
93+
attributes: Record<string, string>;
94+
}[]
95+
)
96+
.filter((l) => l.attributes["braintrust.relationship.type"] === type)
97+
.map((l) => l.context.spanId);
98+
}
8899
/** What the scenario reported about each uploaded audio file, in order. */
89100
function uploadedAudio(
90101
stdout: string,
@@ -319,6 +330,58 @@ for (const variant of variants) {
319330
120000,
320331
);
321332

333+
// A LiveKit Cloud project with redaction: LiveKit's own PII setting wins.
334+
for (const mode of ["auto", "wrapped"] as const)
335+
test.sequential(
336+
`${variant} ${mode} order conversation in a redacted LiveKit job`,
337+
async () => {
338+
await withScenarioHarness(async (harness) => {
339+
await harness.runNodeScenarioDir(
340+
scenarioOptions(variant, {
341+
mode,
342+
entry: "redacted.mjs",
343+
cassette: "order",
344+
recording: false,
345+
}),
346+
);
347+
const events = assertAudioTrace(harness.events());
348+
// Nothing the caller or the model said, and no tool payloads.
349+
for (const name of ["llm_response", "agent_turn", "tts"])
350+
for (const span of findAllSpans(events, name)) {
351+
expect(span.row.input, name).toBeUndefined();
352+
if (name !== "agent_turn")
353+
expect(span.row.output, name).toBeUndefined();
354+
}
355+
expect(
356+
findLatestSpan(events, "lookup_order")?.row.input,
357+
).toBeUndefined();
358+
expect(
359+
findLatestSpan(events, "lookup_order")?.row.output,
360+
).toBeUndefined();
361+
expect(JSON.stringify(events.map((e) => e.row))).not.toMatch(
362+
/order 1042|Friday/i,
363+
);
364+
// Links still connect the call, holding only span IDs.
365+
const tool = findLatestSpan(events, "lookup_order");
366+
expect(linkTargets(tool, "requested_by")).toHaveLength(1);
367+
expect(
368+
linkTargets(
369+
findAllSpans(events, "llm_response")[1],
370+
"consumes_tool_result",
371+
),
372+
).toEqual([tool?.span.id]);
373+
// The call's shape, models and timing remain.
374+
expect(
375+
events.filter((e) => e.span.type === "llm").map((e) => e.span.name),
376+
).toEqual(["llm_response", "llm_response"]);
377+
expect(
378+
findLatestSpan(events, "llm_response")?.metadata,
379+
).toMatchObject({ model: expect.any(String) });
380+
});
381+
},
382+
120000,
383+
);
384+
322385
test.sequential(
323386
`${variant} streaming-cascade order conversation`,
324387
async () => {

‎e2e/scenarios/livekit-instrumentation/websocket-cassette.mjs‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,24 @@ import { readFile, writeFile, mkdir } from "node:fs/promises";
44
import { setImmediate } from "node:timers/promises";
55
import { dirname } from "node:path";
66
import assert from "node:assert/strict";
7-
export async function websocketCassette(path, record) {
7+
/**
8+
* Records or replays one realtime websocket session. `upstream` is the
9+
* provider's origin; `authorization` builds its Authorization header from the
10+
* client's upgrade request.
11+
*/
12+
export async function websocketCassette(
13+
path,
14+
record,
15+
{
16+
upstream = "wss://api.openai.com",
17+
authorization = () => `Bearer ${process.env.OPENAI_API_KEY}`,
18+
} = {},
19+
) {
820
const events = record ? [] : JSON.parse(await readFile(path, "utf8"));
921
let index = 0,
1022
failure,
1123
socket,
12-
upstream;
24+
upstreamSocket;
1325
const ids = new Map();
1426
const server = new WebSocketServer({ port: 0, host: "127.0.0.1" });
1527
await new Promise((resolve) => server.once("listening", resolve));
@@ -73,13 +85,13 @@ export async function websocketCassette(path, record) {
7385
}
7486
socket = client;
7587
if (record) {
76-
const url = new URL(request.url, "wss://api.openai.com");
88+
const url = new URL(request.url, upstream);
7789
// LiveKit adds model routing for gateways, but OpenAI transcription
7890
// selects its model in session.update rather than the upgrade URL.
7991
if (url.searchParams.get("intent") === "transcription")
8092
url.searchParams.delete("model");
81-
const remote = (upstream = new WebSocket(url, {
82-
headers: { Authorization: `Bearer ${process.env.OPENAI_API_KEY}` },
93+
const remote = (upstreamSocket = new WebSocket(url, {
94+
headers: { Authorization: authorization(request) },
8395
}));
8496
const pending = [];
8597
client.on("message", (data) => {
@@ -126,19 +138,21 @@ export async function websocketCassette(path, record) {
126138
baseURL: `http://127.0.0.1:${server.address().port}/v1`,
127139
async close() {
128140
socket?.terminate();
129-
upstream?.terminate();
141+
upstreamSocket?.terminate();
130142
await new Promise((resolve) => server.close(resolve));
131143
if (failure) throw failure;
132144
if (record) {
133145
await mkdir(dirname(path), { recursive: true });
134146
// Provider session IDs can resemble access tokens. Preserve references
135147
// with synthetic IDs in the cassette, without changing live traffic.
136148
const sessionIds = new Map();
137-
for (const event of events) {
138-
const id = event.message?.session?.id;
139-
if (typeof id === "string" && !sessionIds.has(id))
140-
sessionIds.set(id, `session_fixture_${sessionIds.size + 1}`);
141-
}
149+
for (const event of events)
150+
for (const id of [
151+
event.message?.session?.id,
152+
event.message?.session_id,
153+
])
154+
if (typeof id === "string" && !sessionIds.has(id))
155+
sessionIds.set(id, `session_fixture_${sessionIds.size + 1}`);
142156
const serialized = JSON.stringify(
143157
events,
144158
(_key, value) => sessionIds.get(value) ?? value,

‎js/src/auto-instrumentations/configs/livekit.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ const targets: [
5858
"voice/audio_recognition",
5959
{ className: "AudioRecognition", methodName: "processSTTEvent" },
6060
],
61+
// LiveKit reads its project redaction setting from the job context.
62+
["jobContext", "job", { functionName: "getJobContext" }],
6163
// These factories create the async inference workers inside the scoped call.
6264
// Suppressing here leaves tool execution and other session work instrumented.
6365
["inference", "voice/generation", { functionName: "performLLMInference" }],

‎js/src/instrumentation/plugins/livekit-interceptors.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,4 +54,7 @@ export const livekitChannels = defineInterceptor("@livekit/agents", {
5454
stt: channel<[SpeechEvent], void>({
5555
channelName: "recognition.stt",
5656
}),
57+
jobContext: channel<[boolean?], unknown>({
58+
channelName: "job.context",
59+
}),
5760
});
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import iso from "../../isomorph";
2+
3+
/*
4+
* LiveKit's own PII rule for its telemetry (telemetry/redaction.js and pii.js
5+
* in @livekit/agents 1.9.x), so the trace omits exactly what LiveKit would.
6+
*/
7+
8+
/** The span attribute with which LiveKit marks a span of a redacted job. */
9+
export const REDACTION_ENABLED = "lk.redaction.enabled";
10+
11+
/** The text LiveKit substitutes for exception details when PII is not allowed. */
12+
export const REDACTED_EXCEPTION_MESSAGE = "exception details redacted";
13+
14+
const PII_SEGMENT = /(^|\.)pii(\.|$)/i;
15+
// Flagged by the gen_ai semantic conventions as likely to contain user data.
16+
const GEN_AI_PII_ATTRIBUTES = new Set([
17+
"gen_ai.input.messages",
18+
"gen_ai.output.messages",
19+
"gen_ai.system_instructions",
20+
"gen_ai.tool.call.arguments",
21+
"gen_ai.tool.call.result",
22+
"gen_ai.tool.description",
23+
"gen_ai.tool.definitions",
24+
"gen_ai.retrieval.query.text",
25+
"gen_ai.retrieval.documents",
26+
"gen_ai.memory.query.text",
27+
"gen_ai.memory.records",
28+
"gen_ai.evaluation.explanation",
29+
]);
30+
const PII_EVENTS = new Set([
31+
"gen_ai.system.message",
32+
"gen_ai.user.message",
33+
"gen_ai.assistant.message",
34+
"gen_ai.tool.message",
35+
"gen_ai.choice",
36+
"gen_ai.client.inference.operation.details",
37+
]);
38+
const EXCEPTION_MESSAGE = "exception.message";
39+
const EXCEPTION_STACKTRACE = "exception.stacktrace";
40+
const FALSY = new Set(["0", "false", "no", "off"]);
41+
42+
/**
43+
* Whether LiveKit's configuration allows PII in its telemetry: allowed unless
44+
* LIVEKIT_TELEMETRY_ALLOW_PII is falsy, or the job enables LiveKit Cloud
45+
* project redaction (which wins over the environment).
46+
*/
47+
export function livekitPiiAllowed(job?: { _redactionEnabled?: boolean }) {
48+
if (job?._redactionEnabled) return false;
49+
const allow = iso.getEnv("LIVEKIT_TELEMETRY_ALLOW_PII");
50+
return allow === undefined || !FALSY.has(allow.trim().toLowerCase());
51+
}
52+
53+
export function isPiiEvent(name: string): boolean {
54+
return PII_EVENTS.has(name);
55+
}
56+
57+
/** Native attributes without PII, and with exception details replaced, as LiveKit does. */
58+
export function withoutPii(
59+
attributes: Iterable<[string, unknown]>,
60+
): Map<string, unknown> {
61+
const kept = new Map<string, unknown>();
62+
for (const [key, value] of attributes) {
63+
if (key === EXCEPTION_MESSAGE) kept.set(key, REDACTED_EXCEPTION_MESSAGE);
64+
else if (
65+
key !== EXCEPTION_STACKTRACE &&
66+
!PII_SEGMENT.test(key) &&
67+
!GEN_AI_PII_ATTRIBUTES.has(key) &&
68+
!key.startsWith("gen_ai.prompt.variable")
69+
)
70+
kept.set(key, value);
71+
}
72+
return kept;
73+
}

0 commit comments

Comments
 (0)