From b76466410f8b7e34f11a05f3bf5deb20e970d917 Mon Sep 17 00:00:00 2001 From: Dylan Pulver Date: Wed, 18 Nov 2026 13:19:30 -0600 Subject: [PATCH 1/2] fix(openai-agents): Record reasoning tokens as metric --- .../openai-agents-reasoning-tokens-metric.md | 5 ++ .../openai-agents-v0-latest.span-tree.json | 2 + .../openai-agents-v0-latest.span-tree.txt | 2 + .../openai-agents-v0.span-tree.json | 2 + .../openai-agents-v0.span-tree.txt | 2 + .../openai-agents-instrumentation.test.ts | 69 +++++++++++++++++++ .../openai-agents-trace-processor.ts | 8 +++ 7 files changed, 90 insertions(+) create mode 100644 .changeset/openai-agents-reasoning-tokens-metric.md diff --git a/.changeset/openai-agents-reasoning-tokens-metric.md b/.changeset/openai-agents-reasoning-tokens-metric.md new file mode 100644 index 000000000..77c246809 --- /dev/null +++ b/.changeset/openai-agents-reasoning-tokens-metric.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix(openai-agents): Record reasoning tokens as metric diff --git a/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0-latest.span-tree.json b/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0-latest.span-tree.json index 8c402febf..f419ce565 100644 --- a/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0-latest.span-tree.json +++ b/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0-latest.span-tree.json @@ -130,6 +130,7 @@ "user": null }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 7, "prompt_cached_tokens": 0, "prompt_tokens": 81, @@ -299,6 +300,7 @@ "user": null }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 6, "prompt_cached_tokens": 0, "prompt_tokens": 96, diff --git a/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0-latest.span-tree.txt b/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0-latest.span-tree.txt index 9c2b5c3b1..c4f649407 100644 --- a/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0-latest.span-tree.txt +++ b/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0-latest.span-tree.txt @@ -171,6 +171,7 @@ span_tree: │ │ │ "user": null │ │ │ } │ │ │ metrics: { + │ │ │ "completion_reasoning_tokens": 0, │ │ │ "completion_tokens": 7, │ │ │ "prompt_cached_tokens": 0, │ │ │ "prompt_tokens": 81, @@ -327,6 +328,7 @@ span_tree: │ "user": null │ } │ metrics: { + │ "completion_reasoning_tokens": 0, │ "completion_tokens": 6, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 96, diff --git a/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0.span-tree.json b/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0.span-tree.json index cd8d3b667..0a3b5b4b4 100644 --- a/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0.span-tree.json +++ b/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0.span-tree.json @@ -122,6 +122,7 @@ "user": null }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 7, "prompt_cached_tokens": 0, "prompt_tokens": 81, @@ -273,6 +274,7 @@ "user": null }, "metrics": { + "completion_reasoning_tokens": 0, "completion_tokens": 6, "prompt_cached_tokens": 0, "prompt_tokens": 96, diff --git a/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0.span-tree.txt b/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0.span-tree.txt index 2a8ac150c..c0cce6777 100644 --- a/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0.span-tree.txt +++ b/e2e/scenarios/openai-agents-instrumentation/__snapshots__/openai-agents-v0.span-tree.txt @@ -149,6 +149,7 @@ span_tree: │ │ "user": null │ │ } │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, │ │ "completion_tokens": 7, │ │ "prompt_cached_tokens": 0, │ │ "prompt_tokens": 81, @@ -292,6 +293,7 @@ span_tree: │ "user": null │ } │ metrics: { + │ "completion_reasoning_tokens": 0, │ "completion_tokens": 6, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 96, diff --git a/js/src/instrumentation/providers/openai-agents-instrumentation.test.ts b/js/src/instrumentation/providers/openai-agents-instrumentation.test.ts index bb3c6a5e6..e6317e9b1 100644 --- a/js/src/instrumentation/providers/openai-agents-instrumentation.test.ts +++ b/js/src/instrumentation/providers/openai-agents-instrumentation.test.ts @@ -204,4 +204,73 @@ describe("registerOpenAIAgentsInstrumentation", () => { tokens: 6, }); }); + + it("records reasoning tokens from response span usage", async () => { + const trace = { + type: "trace", + traceId: "trace-openai-agents-reasoning", + name: "Agent workflow", + }; + const span = { + type: "trace.span", + traceId: trace.traceId, + spanId: "span-response-reasoning", + parentId: null, + startedAt: new Date(Date.now() - 100).toISOString(), + endedAt: new Date().toISOString(), + error: null, + spanData: { + type: "response", + response_id: "resp_reasoning", + _input: "What is 2+2?", + _response: { + id: "resp_reasoning", + output: [], + usage: { + input_tokens: 10, + input_tokens_details: { cached_tokens: 3 }, + output_tokens: 8, + output_tokens_details: { reasoning_tokens: 6 }, + total_tokens: 18, + }, + }, + }, + }; + + await openAIAgentsCoreChannels.onTraceStart.invoke( + async () => undefined, + undefined, + [trace as any], + {}, + ); + await openAIAgentsCoreChannels.onSpanStart.invoke( + async () => undefined, + undefined, + [span as any], + {}, + ); + await openAIAgentsCoreChannels.onSpanEnd.invoke( + async () => undefined, + undefined, + [span as any], + {}, + ); + await openAIAgentsCoreChannels.onTraceEnd.invoke( + async () => undefined, + undefined, + [trace as any], + {}, + ); + + const spans = (await backgroundLogger.drain()) as any[]; + const responseSpan = spans.find((s) => s.span_attributes?.type === "llm"); + + expect(responseSpan?.metrics).toMatchObject({ + prompt_tokens: 10, + prompt_cached_tokens: 3, + completion_tokens: 8, + completion_reasoning_tokens: 6, + tokens: 18, + }); + }); }); diff --git a/js/src/instrumentation/providers/openai-agents-trace-processor.ts b/js/src/instrumentation/providers/openai-agents-trace-processor.ts index 5fa394956..4b492e0fe 100644 --- a/js/src/instrumentation/providers/openai-agents-trace-processor.ts +++ b/js/src/instrumentation/providers/openai-agents-trace-processor.ts @@ -178,6 +178,14 @@ function parseUsageMetrics(usage: unknown): Record { metrics.prompt_cache_creation_tokens = cacheWriteTokens; } + const reasoningTokens = getNumberProperty( + usage.output_tokens_details, + "reasoning_tokens", + ); + if (reasoningTokens !== undefined) { + metrics.completion_reasoning_tokens = reasoningTokens; + } + return metrics; } From 95546b8d59581778a1e6cdbc08199b9a235972e4 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 18 Nov 2026 14:48:40 -0500 Subject: [PATCH 2/2] test(openai-agents): Simplify reasoning token coverage --- .../openai-agents-instrumentation.test.ts | 216 ++++++++---------- 1 file changed, 92 insertions(+), 124 deletions(-) diff --git a/js/src/instrumentation/providers/openai-agents-instrumentation.test.ts b/js/src/instrumentation/providers/openai-agents-instrumentation.test.ts index e6317e9b1..74f3853a7 100644 --- a/js/src/instrumentation/providers/openai-agents-instrumentation.test.ts +++ b/js/src/instrumentation/providers/openai-agents-instrumentation.test.ts @@ -10,9 +10,11 @@ import { import { _exportsForTestingOnly } from "../../logger/testing"; import { initLogger } from "../../logger/core"; import { configureNode } from "../../node/config"; +import type { LoggingEvent } from "../../../util/object"; import type { OpenAIAgentsSpan, OpenAIAgentsSpanData, + OpenAIAgentsTrace, } from "../../vendor-sdk-types/openai-agents"; import { openAIAgentsCoreChannels } from "./openai-agents-interceptors"; import { registerOpenAIAgentsInstrumentation } from "./openai-agents-instrumentation"; @@ -127,104 +129,25 @@ describe("registerOpenAIAgentsInstrumentation", () => { }, ); - it("records Braintrust spans from OpenAI Agents trace processor lifecycle events", async () => { - const trace = { - type: "trace", - traceId: "trace-openai-agents-auto", - name: "Agent workflow", - groupId: "group-openai-agents-auto", - metadata: { workflow: "test" }, - }; - const startedAt = new Date(Date.now() - 100).toISOString(); - const endedAt = new Date().toISOString(); - const span = { - type: "trace.span", - traceId: trace.traceId, - spanId: "span-generation", - parentId: null, - startedAt, - endedAt, - error: null, + it.each([ + { + spanName: "Generation", spanData: { type: "generation", + model: "gpt-4.1-mini", input: [{ role: "user", content: "What is 2+2?" }], output: [{ role: "assistant", content: "4" }], - model: "gpt-4.1-mini", - usage: { - prompt_tokens: 5, - completion_tokens: 1, - total_tokens: 6, - }, + usage: { prompt_tokens: 5, completion_tokens: 1, total_tokens: 6 }, }, - }; - - await openAIAgentsCoreChannels.onTraceStart.invoke( - async () => undefined, - undefined, - [trace as any], - {}, - ); - await openAIAgentsCoreChannels.onSpanStart.invoke( - async () => undefined, - undefined, - [span as any], - {}, - ); - await openAIAgentsCoreChannels.onSpanEnd.invoke( - async () => undefined, - undefined, - [span as any], - {}, - ); - await openAIAgentsCoreChannels.onTraceEnd.invoke( - async () => undefined, - undefined, - [trace as any], - {}, - ); - - const spans = (await backgroundLogger.drain()) as any[]; - const taskSpan = spans.find( - (s) => s.span_attributes?.name === "Agent workflow", - ); - const generationSpan = spans.find( - (s) => s.span_attributes?.name === "Generation", - ); - - expect(taskSpan?.span_attributes?.type).toBe("task"); - expect(generationSpan?.span_attributes?.type).toBe("llm"); - for (const instrumentedSpan of [taskSpan, generationSpan]) { - expect(instrumentedSpan?.context?.span_origin).toMatchObject({ - instrumentation: { name: "openai-agents" }, - }); - } - expect(generationSpan?.metrics).toMatchObject({ - prompt_tokens: 5, - completion_tokens: 1, - tokens: 6, - }); - }); - - it("records reasoning tokens from response span usage", async () => { - const trace = { - type: "trace", - traceId: "trace-openai-agents-reasoning", - name: "Agent workflow", - }; - const span = { - type: "trace.span", - traceId: trace.traceId, - spanId: "span-response-reasoning", - parentId: null, - startedAt: new Date(Date.now() - 100).toISOString(), - endedAt: new Date().toISOString(), - error: null, + metrics: { prompt_tokens: 5, completion_tokens: 1, tokens: 6 }, + reasoningTokens: undefined, + }, + { + spanName: "Response", spanData: { type: "response", - response_id: "resp_reasoning", _input: "What is 2+2?", _response: { - id: "resp_reasoning", output: [], usage: { input_tokens: 10, @@ -235,42 +158,87 @@ describe("registerOpenAIAgentsInstrumentation", () => { }, }, }, - }; + metrics: { + prompt_tokens: 10, + prompt_cached_tokens: 3, + completion_tokens: 8, + tokens: 18, + }, + reasoningTokens: 6, + }, + ] satisfies { + spanName: string; + spanData: OpenAIAgentsSpanData; + metrics: Record; + reasoningTokens: number | undefined; + }[])( + "records $spanName spans from OpenAI Agents trace processor lifecycle events", + async ({ spanName, spanData, metrics, reasoningTokens }) => { + const trace: OpenAIAgentsTrace = { + type: "trace", + traceId: "trace-openai-agents-auto", + name: "Agent workflow", + groupId: "group-openai-agents-auto", + metadata: { workflow: "test" }, + }; + const span: OpenAIAgentsSpan = { + type: "trace.span", + traceId: trace.traceId, + spanId: "span-model", + parentId: null, + startedAt: null, + endedAt: null, + error: null, + spanData, + }; - await openAIAgentsCoreChannels.onTraceStart.invoke( - async () => undefined, - undefined, - [trace as any], - {}, - ); - await openAIAgentsCoreChannels.onSpanStart.invoke( - async () => undefined, - undefined, - [span as any], - {}, - ); - await openAIAgentsCoreChannels.onSpanEnd.invoke( - async () => undefined, - undefined, - [span as any], - {}, - ); - await openAIAgentsCoreChannels.onTraceEnd.invoke( - async () => undefined, - undefined, - [trace as any], - {}, - ); + await openAIAgentsCoreChannels.onTraceStart.invoke( + async () => undefined, + undefined, + [trace], + {}, + ); + await openAIAgentsCoreChannels.onSpanStart.invoke( + async () => undefined, + undefined, + [span], + {}, + ); + await openAIAgentsCoreChannels.onSpanEnd.invoke( + async () => undefined, + undefined, + [span], + {}, + ); + await openAIAgentsCoreChannels.onTraceEnd.invoke( + async () => undefined, + undefined, + [trace], + {}, + ); - const spans = (await backgroundLogger.drain()) as any[]; - const responseSpan = spans.find((s) => s.span_attributes?.type === "llm"); + const spans = (await backgroundLogger.drain()).filter( + (row): row is LoggingEvent => + "log_id" in row && row.log_id === "g" && "span_attributes" in row, + ); + const taskSpan = spans.find( + (row) => row.span_attributes?.name === trace.name, + ); + const modelSpan = spans.find( + (row) => row.span_attributes?.name === spanName, + ); - expect(responseSpan?.metrics).toMatchObject({ - prompt_tokens: 10, - prompt_cached_tokens: 3, - completion_tokens: 8, - completion_reasoning_tokens: 6, - tokens: 18, - }); - }); + expect(taskSpan?.span_attributes?.type).toBe("task"); + expect(modelSpan?.span_attributes?.type).toBe("llm"); + for (const instrumentedSpan of [taskSpan, modelSpan]) { + expect(instrumentedSpan?.context?.span_origin).toMatchObject({ + instrumentation: { name: "openai-agents" }, + }); + } + expect(modelSpan?.metrics).toMatchObject(metrics); + expect(modelSpan?.metrics?.completion_reasoning_tokens).toBe( + reasoningTokens, + ); + }, + ); });