From 3c6689a20871fe02256a45105eef554d3b46aff6 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:37:55 +0000 Subject: [PATCH 1/3] feat(langchain): Add capture controls --- .../langchain-anthropic/capture-config.mjs | 10 + .../langchain-anthropic/scenario.mjs | 8 +- .../langchain-anthropic/scenario.test.ts | 79 +++++++ .../capture-config.mjs | 10 + .../scenario.test.ts | 75 +++++++ js/src/instrumentation/braintrust-plugin.ts | 8 +- js/src/instrumentation/config.ts | 25 +++ .../plugins/langchain-plugin.test.ts | 56 ++++- .../plugins/langchain-plugin.ts | 23 +- js/src/instrumentation/registry.ts | 1 + js/src/vendor-sdk-types/langchain.ts | 29 ++- .../langchain/callback-handler.test.ts | 196 +++++++++++++++++- js/src/wrappers/langchain/callback-handler.ts | 34 ++- 13 files changed, 529 insertions(+), 25 deletions(-) create mode 100644 e2e/scenarios/langchain-anthropic/capture-config.mjs create mode 100644 e2e/scenarios/langgraph-auto-instrumentation/capture-config.mjs diff --git a/e2e/scenarios/langchain-anthropic/capture-config.mjs b/e2e/scenarios/langchain-anthropic/capture-config.mjs new file mode 100644 index 000000000..3f8f0483b --- /dev/null +++ b/e2e/scenarios/langchain-anthropic/capture-config.mjs @@ -0,0 +1,10 @@ +import { configureInstrumentation } from "braintrust/instrumentation"; + +export const captureOptions = { + captureChainInputs: false, + captureChainOutputs: false, + captureModelInputs: false, + captureModelOutputs: false, + captureModelTokenUsage: false, +}; +configureInstrumentation({ langchain: captureOptions }); diff --git a/e2e/scenarios/langchain-anthropic/scenario.mjs b/e2e/scenarios/langchain-anthropic/scenario.mjs index c730483a2..8620562e8 100644 --- a/e2e/scenarios/langchain-anthropic/scenario.mjs +++ b/e2e/scenarios/langchain-anthropic/scenario.mjs @@ -9,6 +9,10 @@ import { const { ChatAnthropic } = await import( process.env.LANGCHAIN_ANTHROPIC_PACKAGE_NAME ); +const captureOptions = + process.env.DISABLE_LANGCHAIN_CAPTURE === "1" + ? (await import("./capture-config.mjs")).captureOptions + : {}; const MODEL = "claude-haiku-4-5-20251001"; runMain(async () => { @@ -20,7 +24,9 @@ runMain(async () => { for (const mode of ["manual", "auto"]) { // The hook respects an explicit handler and injects one when absent. const callbacks = - mode === "manual" ? [new BraintrustLangChainCallbackHandler()] : []; + mode === "manual" + ? [new BraintrustLangChainCallbackHandler(captureOptions)] + : []; for (const operation of ["invoke", "streaming-invoke", "stream"]) { await runOperation( `anthropic-${mode}-${operation}`, diff --git a/e2e/scenarios/langchain-anthropic/scenario.test.ts b/e2e/scenarios/langchain-anthropic/scenario.test.ts index 2b312ae20..f1617608f 100644 --- a/e2e/scenarios/langchain-anthropic/scenario.test.ts +++ b/e2e/scenarios/langchain-anthropic/scenario.test.ts @@ -90,3 +90,82 @@ for (const dependencyName of [ }, ); } + +for (const dependencyName of [ + "langchain-anthropic-v1", + "langchain-anthropic-v1-latest", +]) { + test( + `Anthropic retains streaming timing without model payloads or usage (${dependencyName})`, + { timeout: TIMEOUT_MS }, + async () => { + await withScenarioHarness(async ({ events, runNodeScenarioDir }) => { + await runNodeScenarioDir({ + scenarioDir, + entry: "scenario.mjs", + timeoutMs: TIMEOUT_MS, + nodeArgs: [ + "--import", + "./capture-config.mjs", + "--import", + "braintrust/hook.mjs", + ], + env: { + LANGCHAIN_ANTHROPIC_PACKAGE_NAME: dependencyName, + DISABLE_LANGCHAIN_CAPTURE: "1", + }, + runContext: { variantKey: dependencyName, originalScenarioDir }, + }); + const captured = events(); + for (const mode of ["manual", "auto"]) { + for (const operation of ["invoke", "streaming-invoke", "stream"]) { + const parent = findLatestSpan( + captured, + `anthropic-${mode}-${operation}`, + ); + const spans = findChildSpans( + captured, + "ChatAnthropic", + parent?.span.id, + ); + expect(spans).toHaveLength(1); + const span = spans[0]; + expect(span.input).toBeUndefined(); + expect(span.output).toBeUndefined(); + expect(span.row.metadata).toMatchObject({ + model: "claude-haiku-4-5-20251001", + }); + expect( + Object.keys(span.metrics ?? {}).filter((key) => + key.includes("tokens"), + ), + ).toEqual([]); + expect(span.metrics?.end).toBeTypeOf("number"); + if (operation !== "invoke") { + expect(span.metrics?.time_to_first_token).toBeGreaterThanOrEqual( + 0, + ); + } + const provider = captured.filter( + (event) => + event.span.name === "anthropic.messages.create" && + event.span.parentIds.includes(parent!.span.id!) && + typeof event.metrics?.tokens === "number" && + event.metrics.tokens > 0, + ); + expect(provider).toHaveLength(1); + expect(provider[0].input).toBeDefined(); + expect(provider[0].output).toBeDefined(); + } + } + expect( + captured.filter( + (event) => + typeof event.metrics?.tokens === "number" && + event.metrics.tokens > 0, + ), + ).toHaveLength(6); + }); + }, + ); +} diff --git a/e2e/scenarios/langgraph-auto-instrumentation/capture-config.mjs b/e2e/scenarios/langgraph-auto-instrumentation/capture-config.mjs new file mode 100644 index 000000000..3f8f0483b --- /dev/null +++ b/e2e/scenarios/langgraph-auto-instrumentation/capture-config.mjs @@ -0,0 +1,10 @@ +import { configureInstrumentation } from "braintrust/instrumentation"; + +export const captureOptions = { + captureChainInputs: false, + captureChainOutputs: false, + captureModelInputs: false, + captureModelOutputs: false, + captureModelTokenUsage: false, +}; +configureInstrumentation({ langchain: captureOptions }); diff --git a/e2e/scenarios/langgraph-auto-instrumentation/scenario.test.ts b/e2e/scenarios/langgraph-auto-instrumentation/scenario.test.ts index 4b4997d02..3dc6fccf7 100644 --- a/e2e/scenarios/langgraph-auto-instrumentation/scenario.test.ts +++ b/e2e/scenarios/langgraph-auto-instrumentation/scenario.test.ts @@ -9,6 +9,7 @@ import { resolveScenarioDir, withScenarioHarness, } from "../../helpers/scenario-harness"; +import { findLatestSpan } from "../../helpers/trace-selectors"; import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { assertLangGraphAutoInstrumentation } from "./assertions"; @@ -90,3 +91,77 @@ for (const scenario of scenarios) { }, ); } + +for (const scenario of scenarios) { + test( + `langgraph retains spans with payload capture disabled (${scenario.version})`, + { timeout: TIMEOUT_MS }, + async () => { + await withScenarioHarness(async ({ events, runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: "scenario.mjs", + env: { + LANGCHAIN_CORE_PACKAGE_NAME: scenario.coreDependencyName, + LANGCHAIN_OPENAI_PACKAGE_NAME: scenario.openAIDependencyName, + LANGGRAPH_PACKAGE_NAME: scenario.langgraphDependencyName, + }, + nodeArgs: [ + "--import", + "./capture-config.mjs", + "--import", + "braintrust/hook.mjs", + ], + runContext: { variantKey: scenario.variantKey, originalScenarioDir }, + scenarioDir, + timeoutMs: TIMEOUT_MS, + }); + const captured = events(); + const chainSpans = captured.filter((event) => + ["LangGraph", "sayHello", "sayBye", "ChatOpenAI"].includes( + event.span.name ?? "", + ), + ); + expect(chainSpans).toHaveLength(4); + for (const event of chainSpans) { + expect(event.input).toBeUndefined(); + expect(event.output).toBeUndefined(); + expect(event.row.metadata).not.toHaveProperty("inputs"); + expect(event.span.parentIds).toHaveLength(1); + expect(event.metrics?.start).toBeTypeOf("number"); + expect(event.metrics?.end).toBeTypeOf("number"); + expect( + Object.keys(event.metrics ?? {}).filter((key) => + key.includes("tokens"), + ), + ).toEqual([]); + } + const model = findLatestSpan(captured, "ChatOpenAI"); + expect(model).toBeDefined(); + const root = findLatestSpan( + captured, + "langgraph-auto-instrumentation-root", + )!; + const graph = findLatestSpan(captured, "LangGraph")!; + const hello = findLatestSpan(captured, "sayHello")!; + const bye = findLatestSpan(captured, "sayBye")!; + expect(graph.span.parentIds).toEqual([root.span.id]); + expect(hello.span.parentIds).toEqual([graph.span.id]); + expect(bye.span.parentIds).toEqual([graph.span.id]); + expect(model?.span.parentIds).toEqual([hello.span.id]); + const provider = findLatestSpan(captured, "Chat Completion"); + // Callback spans use run IDs; provider spans retain the active SDK parent. + expect(provider?.span.parentIds).toEqual([root.span.id]); + expect(provider?.input).toBeDefined(); + expect(provider?.output).toBeDefined(); + expect(provider?.metrics?.tokens).toBeGreaterThan(0); + expect( + captured.filter( + (event) => + typeof event.metrics?.tokens === "number" && + event.metrics.tokens > 0, + ), + ).toHaveLength(1); + }); + }, + ); +} diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index 85714e3cd..f91d6690a 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -37,9 +37,13 @@ import { WorkersAIPlugin } from "./plugins/workers-ai-plugin"; import { TypeSafePlugin } from "./plugins/typesafe-plugin"; import { CloudflareAIChatPlugin } from "./plugins/cloudflare-ai-chat-plugin"; import { CloudflareAgentsPlugin } from "./plugins/cloudflare-agents-plugin"; -import type { InstrumentationIntegrationsConfig } from "./config"; +import type { + InstrumentationConfig, + InstrumentationIntegrationsConfig, +} from "./config"; export interface BraintrustPluginConfig { + langchain?: InstrumentationConfig["langchain"]; integrations?: InstrumentationIntegrationsConfig; streamingAttachmentBufferBytes?: number; } @@ -318,7 +322,7 @@ export class BraintrustPlugin extends BasePlugin { } if (integrations.langchain !== false && integrations.langgraph !== false) { - this.langChainPlugin = new LangChainPlugin(); + this.langChainPlugin = new LangChainPlugin(this.config.langchain); this.langChainPlugin.enable(); } diff --git a/js/src/instrumentation/config.ts b/js/src/instrumentation/config.ts index bfb0baf65..f2109deb1 100644 --- a/js/src/instrumentation/config.ts +++ b/js/src/instrumentation/config.ts @@ -1,3 +1,5 @@ +import type { LangChainCaptureOptions } from "../vendor-sdk-types/langchain"; + export type SpanExportData = Record; export interface SpanCustomizer { @@ -122,6 +124,29 @@ export interface AudioEncodingConfig { export const DEFAULT_STREAMING_ATTACHMENT_BUFFER_BYTES = 4 * 1024 * 1024; export interface InstrumentationConfig { + /** + * Capture settings for automatically injected LangChain/LangGraph handlers. + * All fields default to true. Disabled payloads are omitted before copying. + * Spans, timing, errors, and metadata remain available. + * Configure before initializing the SDK or loading braintrust/hook.mjs. + * Use provider instrumentation to capture model payloads and usage when + * disabling their capture here. + * + * @example + * // capture-config.mjs, loaded before --import braintrust/hook.mjs + * import { configureInstrumentation } from "braintrust/instrumentation"; + * configureInstrumentation({ + * langchain: { + * captureChainInputs: false, + * captureChainOutputs: false, + * captureModelInputs: false, + * captureModelOutputs: false, + * captureModelTokenUsage: false, + * }, + * }); + */ + langchain?: LangChainCaptureOptions; + /** * Target attachment payload size in bytes for each segmentable media stream. * Defaults to 4 MiB. Must be a positive safe integer. diff --git a/js/src/instrumentation/plugins/langchain-plugin.test.ts b/js/src/instrumentation/plugins/langchain-plugin.test.ts index 6ead73b39..7415d7eed 100644 --- a/js/src/instrumentation/plugins/langchain-plugin.test.ts +++ b/js/src/instrumentation/plugins/langchain-plugin.test.ts @@ -1,6 +1,11 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { langChainChannels } from "./langchain-interceptors"; import { LangChainPlugin } from "./langchain-plugin"; +import type { + LangChainStartSpanArgs, + LangChainEndSpanArgs, +} from "../../vendor-sdk-types/langchain"; +import { BraintrustLangChainCallbackHandler } from "../../wrappers/langchain/callback-handler"; function createManager(handlers: unknown[] = []) { return { @@ -124,3 +129,52 @@ describe("LangChainPlugin", () => { plugin.disable(); }); }); + +it.each(["arguments", "result"] as const)( + "applies capture options to the handler injected into %s", + async (path) => { + const plugin = new LangChainPlugin({ + captureChainInputs: false, + captureChainOutputs: false, + captureModelInputs: false, + captureModelOutputs: false, + captureModelTokenUsage: false, + }); + plugin.enable(); + let handler: BraintrustLangChainCallbackHandler; + try { + if (path === "arguments") { + handler = traceConfigureArguments([])[0][0]; + } else { + const manager = createManager(); + traceConfigureResult(manager); + handler = manager.handlers[0] as BraintrustLangChainCallbackHandler; + } + } finally { + plugin.disable(); + } + // Observe the boundary before any logger copying happens. + const boundary = handler as unknown as { + startSpan(args: LangChainStartSpanArgs): void; + endSpan(args: LangChainEndSpanArgs): void; + }; + const start = vi.spyOn(boundary, "startSpan").mockImplementation(() => {}); + const end = vi.spyOn(boundary, "endSpan").mockImplementation(() => {}); + await handler.handleChainStart({}, { state: "input" }, "chain"); + await handler.handleChainEnd({ state: "output" }, "chain"); + await handler.handleChatModelStart({}, [[{ content: "hello" }]], "model"); + await handler.handleLLMEnd( + { llmOutput: { tokenUsage: { totalTokens: 10 } } }, + "model", + ); + for (const [args] of start.mock.calls) { + expect(args).not.toHaveProperty("event.input"); + } + for (const [args] of end.mock.calls) { + expect(args).not.toHaveProperty("output"); + expect(args).not.toHaveProperty("metrics.tokens"); + } + expect(start).toHaveBeenCalledTimes(2); + expect(end).toHaveBeenCalledTimes(2); + }, +); diff --git a/js/src/instrumentation/plugins/langchain-plugin.ts b/js/src/instrumentation/plugins/langchain-plugin.ts index 5cd352210..513ba49bb 100644 --- a/js/src/instrumentation/plugins/langchain-plugin.ts +++ b/js/src/instrumentation/plugins/langchain-plugin.ts @@ -1,7 +1,10 @@ import { BasePlugin } from "../core"; import { runInstrumentation } from "../core/observe-result"; -import type { LangChainCallbackManager } from "../../vendor-sdk-types/langchain"; +import type { + LangChainCaptureOptions, + LangChainCallbackManager, +} from "../../vendor-sdk-types/langchain"; import { BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME, BraintrustLangChainCallbackHandler, @@ -16,6 +19,10 @@ type LangChainConfigureChannel = export class LangChainPlugin extends BasePlugin { private injectedManagers = new WeakSet(); + constructor(private readonly captureOptions: LangChainCaptureOptions = {}) { + super(); + } + protected onEnable(): void { this.subscribeToConfigure(langChainChannels.configure); this.subscribeToConfigure(langChainChannels.configureSync); @@ -40,7 +47,7 @@ export class LangChainPlugin extends BasePlugin { self: receiver, } as ChannelMessage; const prepare = (event: ChannelMessage) => { - injectHandlerIntoArguments(event.arguments); + injectHandlerIntoArguments(event.arguments, this.captureOptions); }; const returned = (event: ChannelMessage) => { this.injectHandler(event.result); @@ -70,7 +77,10 @@ export class LangChainPlugin extends BasePlugin { } try { - result.addHandler(new BraintrustLangChainCallbackHandler(), true); + result.addHandler( + new BraintrustLangChainCallbackHandler(this.captureOptions), + true, + ); this.injectedManagers.add(result); } catch { // Instrumentation must never break LangChain user code. @@ -102,13 +112,16 @@ function hasBraintrustHandler(manager: LangChainCallbackManager): boolean { ); } -function injectHandlerIntoArguments(args: ArrayLike): void { +function injectHandlerIntoArguments( + args: ArrayLike, + captureOptions: LangChainCaptureOptions, +): void { if (!isWritableArgumentsObject(args)) { return; } const inheritedHandlers = Reflect.get(args, "0"); - const handler = new BraintrustLangChainCallbackHandler(); + const handler = new BraintrustLangChainCallbackHandler(captureOptions); if (inheritedHandlers === undefined || inheritedHandlers === null) { Reflect.set(args, "0", [handler]); diff --git a/js/src/instrumentation/registry.ts b/js/src/instrumentation/registry.ts index f9ab5a498..4a09146ae 100644 --- a/js/src/instrumentation/registry.ts +++ b/js/src/instrumentation/registry.ts @@ -112,6 +112,7 @@ class PluginRegistry { // Read config from environment variables const envConfig = this.readEnvConfig(); const finalConfig = { + langchain: this.config.langchain, streamingAttachmentBufferBytes: this.config.streamingAttachmentBufferBytes, integrations: { diff --git a/js/src/vendor-sdk-types/langchain.ts b/js/src/vendor-sdk-types/langchain.ts index a4ffdc909..5505372a0 100644 --- a/js/src/vendor-sdk-types/langchain.ts +++ b/js/src/vendor-sdk-types/langchain.ts @@ -27,12 +27,29 @@ export type LangChainCallbackManagerConfigureArgs = [ options?: unknown, ]; -export type LangChainCallbackHandlerOptions = { - debug: boolean; - excludeMetadataProps: RegExp; - logger?: Logger | Span; - parent?: Span | (() => Span); -}; +export interface LangChainCaptureOptions { + /** Capture chain inputs, including end/error callback inputs. Defaults to true. */ + captureChainInputs?: boolean; + /** Capture chain outputs. Defaults to true. */ + captureChainOutputs?: boolean; + /** Capture chat messages and LLM prompts. Defaults to true. */ + captureModelInputs?: boolean; + /** Capture chat model and LLM results. Defaults to true. */ + captureModelOutputs?: boolean; + /** + * Capture model token usage, including cache and reasoning tokens. + * Defaults to true. Does not affect duration or time to first token. + */ + captureModelTokenUsage?: boolean; +} + +export type LangChainCallbackHandlerOptions = + LangChainCaptureOptions & { + debug: boolean; + excludeMetadataProps: RegExp; + logger?: Logger | Span; + parent?: Span | (() => Span); + }; export type LangChainStartSpanArgs = StartSpanArgs & { parentRunId?: string; diff --git a/js/src/wrappers/langchain/callback-handler.test.ts b/js/src/wrappers/langchain/callback-handler.test.ts index 53f61cbc2..4f0d89797 100644 --- a/js/src/wrappers/langchain/callback-handler.test.ts +++ b/js/src/wrappers/langchain/callback-handler.test.ts @@ -1,18 +1,26 @@ import { describe, expect, it, vi } from "vitest"; import { BraintrustLangChainCallbackHandler } from "./callback-handler"; -function createHarness() { +import { deepCopyEvent, type StartSpanArgs } from "../../logger"; +import type { LangChainCaptureOptions } from "../../vendor-sdk-types/langchain"; + +function createHarness(options: LangChainCaptureOptions = {}) { const logs: unknown[] = []; const span = { log: vi.fn((event: unknown) => { + deepCopyEvent(event as Parameters[0]); logs.push(event); }), end: vi.fn(), }; const parent = { - startSpan: vi.fn(() => span), + startSpan: vi.fn((args: StartSpanArgs) => { + deepCopyEvent(args.event ?? {}); + return span; + }), }; const handler = new BraintrustLangChainCallbackHandler({ + ...options, parent: parent as never, }); @@ -404,3 +412,187 @@ describe("BraintrustLangChainCallbackHandler model attribution", () => { }, ); }); + +describe("LangChain capture options", () => { + it.each([undefined, true, false])( + "controls chain input and output independently (%s)", + async (capture) => { + for (const option of [ + "captureChainInputs", + "captureChainOutputs", + ] as const) { + const { handler, parent, logs, span } = createHarness({ + [option]: capture, + }); + const input = { messages: ["history"] }; + const output = { messages: ["answer"] }; + const kwargs = { inputs: input, retained: "metadata" }; + await handler.handleChainStart({ name: "graph" }, input, "chain"); + await handler.handleChainEnd( + output, + "chain", + undefined, + ["tag"], + kwargs, + ); + const startEvent = parent.startSpan.mock.calls[0][0].event!; + const endEvent = logs[0]; + if (option === "captureChainInputs" && capture === false) { + expect(startEvent).not.toHaveProperty("input"); + expect(endEvent).not.toHaveProperty("metadata.inputs"); + } else { + expect(startEvent.input).toBe(input); + expect(endEvent).toHaveProperty("metadata.inputs", input); + } + if (option === "captureChainOutputs" && capture === false) { + expect(endEvent).not.toHaveProperty("output"); + } else { + expect(endEvent).toHaveProperty("output", output); + } + expect(endEvent).toHaveProperty("metadata.retained", "metadata"); + expect(kwargs.inputs).toBe(input); + expect(span.end).toHaveBeenCalledOnce(); + } + }, + ); + + it.each(["end", "error"])( + "never serializes suppressed chain payloads on %s", + async (completion) => { + const toJSON = vi.fn(() => { + throw new Error("must not copy state"); + }); + const state = { toJSON }; + const { handler, logs, span } = createHarness({ + captureChainInputs: false, + captureChainOutputs: false, + }); + await handler.handleChainStart({ name: "graph" }, state, "chain"); + const kwargs = { inputs: state }; + if (completion === "error") { + const error = new Error("chain failed"); + await handler.handleChainError(error, "chain", undefined, [], kwargs); + expect(logs[0]).toHaveProperty("error", error); + } else { + await handler.handleChainEnd(state, "chain", undefined, [], kwargs); + } + expect(toJSON).not.toHaveBeenCalled(); + expect(kwargs.inputs).toBe(state); + expect(span.end).toHaveBeenCalledOnce(); + }, + ); + + it.each(["handleChatModelStart", "handleLLMStart"] as const)( + "%s suppresses inputs before serialization and preserves errors", + async (method) => { + const toJSON = vi.fn(() => { + throw new Error("must not copy messages"); + }); + const { handler, parent, logs, span } = createHarness({ + captureModelInputs: false, + }); + await handler[method]( + { name: "model" }, + Object.assign([], { toJSON }), + "model", + ); + expect(parent.startSpan.mock.calls[0][0].event).not.toHaveProperty( + "input", + ); + const error = new Error("model failed"); + await handler.handleLLMError(error, "model"); + expect(logs[0]).toHaveProperty("error", error); + expect(toJSON).not.toHaveBeenCalled(); + expect(span.end).toHaveBeenCalledOnce(); + }, + ); + + it.each([undefined, true, false])( + "controls model input, output and usage independently (%s)", + async (capture) => { + for (const option of [ + "captureModelInputs", + "captureModelOutputs", + "captureModelTokenUsage", + ] as const) { + const { handler, parent, logs, span } = createHarness({ + [option]: capture, + }); + const messages = [[{ content: "hello" }]]; + const response = { + llmOutput: { tokenUsage: { promptTokens: 10, completionTokens: 2 } }, + }; + await handler.handleChatModelStart( + { name: "model" }, + messages, + "model", + ); + await handler.handleLLMNewToken( + "hi", + { prompt: 0, completion: 0 }, + "model", + ); + await handler.handleLLMEnd(response, "model"); + const startEvent = parent.startSpan.mock.calls[0][0].event!; + if (option === "captureModelInputs" && capture === false) { + expect(startEvent).not.toHaveProperty("input"); + } else { + expect(startEvent.input).toBe(messages); + } + if (option === "captureModelOutputs" && capture === false) { + expect(logs[0]).not.toHaveProperty("output"); + } else { + expect(logs[0]).toHaveProperty("output", response); + } + expect(logs[0]).toHaveProperty( + "metrics.time_to_first_token", + expect.any(Number), + ); + if (option === "captureModelTokenUsage" && capture === false) { + expect(logs[0]).toHaveProperty("metrics", { + time_to_first_token: expect.any(Number), + }); + } else { + expect(logs[0]).toHaveProperty("metrics.tokens", 12); + } + expect(span.end).toHaveBeenCalledOnce(); + } + }, + ); + + it("skips result serialization and token extraction while retaining model attribution", async () => { + const toJSON = vi.fn(() => { + throw new Error("must not copy result"); + }); + const usage = vi.fn(() => { + throw new Error("must not extract usage"); + }); + const response = { + toJSON, + generations: [ + [ + { + message: { + get usage_metadata() { + return usage(); + }, + response_metadata: { model: "resolved-model" }, + }, + }, + ], + ], + }; + const { handler, logs } = createHarness({ + captureModelOutputs: false, + captureModelTokenUsage: false, + }); + await handler.handleChatModelStart({ name: "model" }, [], "model"); + await handler.handleLLMEnd(response, "model"); + expect(toJSON).not.toHaveBeenCalled(); + expect(usage).not.toHaveBeenCalled(); + expect(logs[0]).toMatchObject({ + metadata: { model: "resolved-model" }, + metrics: {}, + }); + }); +}); diff --git a/js/src/wrappers/langchain/callback-handler.ts b/js/src/wrappers/langchain/callback-handler.ts index 039e1f234..53648be51 100644 --- a/js/src/wrappers/langchain/callback-handler.ts +++ b/js/src/wrappers/langchain/callback-handler.ts @@ -33,6 +33,7 @@ export class BraintrustLangChainCallbackHandler< ) { this.parent = options?.parent; this.options = { + ...options, debug: options?.debug ?? false, excludeMetadataProps: options?.excludeMetadataProps ?? @@ -147,7 +148,9 @@ export class BraintrustLangChainCallbackHandler< name: runName ?? getSerializedName(llm) ?? "LLM", type: "llm", event: { - input: prompts, + ...(this.options.captureModelInputs !== false + ? { input: prompts } + : {}), tags, metadata: { serialized: llm, @@ -175,7 +178,10 @@ export class BraintrustLangChainCallbackHandler< parentRunId?: string, tags?: string[], ): Promise { - const metrics = getMetricsFromResponse(output); + const metrics = + this.options.captureModelTokenUsage !== false + ? getMetricsFromResponse(output) + : {}; const modelName = getModelNameFromResponse(output); const ttft = this.ttftMs.get(runId); if (ttft !== undefined) { @@ -189,7 +195,7 @@ export class BraintrustLangChainCallbackHandler< this.endSpan({ runId, parentRunId, - output, + ...(this.options.captureModelOutputs !== false ? { output } : {}), metrics, tags, metadata: modelName !== undefined ? { model: modelName } : undefined, @@ -216,7 +222,9 @@ export class BraintrustLangChainCallbackHandler< name: runName ?? getSerializedName(llm) ?? "Chat Model", type: "llm", event: { - input: messages, + ...(this.options.captureModelInputs !== false + ? { input: messages } + : {}), tags, metadata: { serialized: llm, @@ -249,7 +257,7 @@ export class BraintrustLangChainCallbackHandler< parentRunId, name: runName ?? getSerializedName(chain) ?? "Chain", event: { - input: inputs, + ...(this.options.captureChainInputs !== false ? { input: inputs } : {}), tags, metadata: { serialized: chain, @@ -270,7 +278,11 @@ export class BraintrustLangChainCallbackHandler< inputs?: Record; }, ): Promise { - this.endSpan({ runId, parentRunId, error: err, tags, metadata: kwargs }); + const metadata = { ...kwargs }; + if (this.options.captureChainInputs === false) { + delete metadata.inputs; + } + this.endSpan({ runId, parentRunId, error: err, tags, metadata }); } async handleChainEnd( @@ -280,12 +292,18 @@ export class BraintrustLangChainCallbackHandler< tags?: string[], kwargs?: { inputs?: Record }, ): Promise { + const metadata = { ...kwargs }; + if (this.options.captureChainInputs === false) { + delete metadata.inputs; + } this.endSpan({ runId, parentRunId, tags, - output: outputs, - metadata: { ...kwargs }, + ...(this.options.captureChainOutputs !== false + ? { output: outputs } + : {}), + metadata, }); } From 97e78339d7f415cd3811fc9df6b7ab7b2d4b27b5 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:59:36 +0000 Subject: [PATCH 2/3] changest --- .changeset/langchain-capture-controls.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/langchain-capture-controls.md diff --git a/.changeset/langchain-capture-controls.md b/.changeset/langchain-capture-controls.md new file mode 100644 index 000000000..85aacbf7b --- /dev/null +++ b/.changeset/langchain-capture-controls.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat(langchain): Add capture controls From 9a3e35a0a933e3a1c32051217f1c52347de08f45 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:14:25 +0000 Subject: [PATCH 3/3] Update PR #2763 --- .../langchain/callback-handler.test.ts | 29 ++++++++++++++++++- js/src/wrappers/langchain/callback-handler.ts | 7 ++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/js/src/wrappers/langchain/callback-handler.test.ts b/js/src/wrappers/langchain/callback-handler.test.ts index 4f0d89797..df9e264e1 100644 --- a/js/src/wrappers/langchain/callback-handler.test.ts +++ b/js/src/wrappers/langchain/callback-handler.test.ts @@ -456,7 +456,28 @@ describe("LangChain capture options", () => { }, ); - it.each(["end", "error"])( + it.each([undefined, true, false])( + "controls chain output when an agent completes first (%s)", + async (captureChainOutputs) => { + const { handler, logs, span } = createHarness({ captureChainOutputs }); + const output = { output: "answer" }; + const action = { returnValues: output, log: "agent finished" }; + await handler.handleChainStart({ name: "AgentExecutor" }, {}, "chain"); + await handler.handleAgentEnd(action, "chain", undefined, ["tag"]); + await handler.handleChainEnd(output, "chain"); + + expect(logs).toHaveLength(1); + if (captureChainOutputs === false) { + expect(logs[0]).not.toHaveProperty("output"); + } else { + expect(logs[0]).toHaveProperty("output", action); + } + expect(logs[0]).toHaveProperty("metadata.tags", ["tag"]); + expect(span.end).toHaveBeenCalledOnce(); + }, + ); + + it.each(["end", "error", "agent"])( "never serializes suppressed chain payloads on %s", async (completion) => { const toJSON = vi.fn(() => { @@ -474,6 +495,12 @@ describe("LangChain capture options", () => { await handler.handleChainError(error, "chain", undefined, [], kwargs); expect(logs[0]).toHaveProperty("error", error); } else { + if (completion === "agent") { + await handler.handleAgentEnd( + { returnValues: state, log: "agent finished" }, + "chain", + ); + } await handler.handleChainEnd(state, "chain", undefined, [], kwargs); } expect(toJSON).not.toHaveBeenCalled(); diff --git a/js/src/wrappers/langchain/callback-handler.ts b/js/src/wrappers/langchain/callback-handler.ts index 53648be51..8a220328a 100644 --- a/js/src/wrappers/langchain/callback-handler.ts +++ b/js/src/wrappers/langchain/callback-handler.ts @@ -377,7 +377,12 @@ export class BraintrustLangChainCallbackHandler< parentRunId?: string, tags?: string[], ): Promise { - this.endSpan({ runId, parentRunId, output: action, tags }); + this.endSpan({ + runId, + parentRunId, + ...(this.options.captureChainOutputs !== false ? { output: action } : {}), + tags, + }); } async handleRetrieverStart(