Skip to content

Commit c11a384

Browse files
committed
feat!: Add BRAINTRUST_CAPTURE_ATTACHMENTS env var option and captureAttachments logger option for all integrations
1 parent 5b037de commit c11a384

44 files changed

Lines changed: 2766 additions & 331 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.changeset/light-plants-return.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"@braintrust/deepseek-harness": major
3+
"braintrust": major
4+
---
5+
6+
feat!: Add `BRAINTRUST_CAPTURE_ATTACHMENTS` env var option and `captureAttachments` logger option for all integrations
Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import { expect, test } from "vitest";
2+
import { withScenarioHarness } from "./scenario-harness";
3+
import { findLatestSpan } from "./trace-selectors";
4+
import { captureCases } from "./attachment-capture-scenario.mjs";
5+
6+
export function defineAttachmentCaptureTests(options: {
7+
scenarioDir: string;
8+
originalScenarioDir: string;
9+
provider: "openai" | "anthropic";
10+
packageName: string;
11+
variantKey: string;
12+
}) {
13+
for (const mode of ["wrapped", "auto"]) {
14+
test(`attachment capture policies (${options.variantKey}, ${mode})`, async () => {
15+
await withScenarioHarness(async (harness) => {
16+
await harness.runNodeScenarioDir({
17+
scenarioDir: options.scenarioDir,
18+
entry: "scenario.capture.mjs",
19+
nodeArgs: mode === "auto" ? ["--import", "braintrust/hook.mjs"] : [],
20+
env: {
21+
CAPTURE_PACKAGE_NAME: options.packageName,
22+
CAPTURE_WRAPPED: String(mode === "wrapped"),
23+
BRAINTRUST_CAPTURE_ATTACHMENTS: "",
24+
},
25+
runContext: {
26+
variantKey: options.variantKey,
27+
originalScenarioDir: options.originalScenarioDir,
28+
// The same Chat Completions request works on all supported SDK majors.
29+
// Its real-provider recording lives in the existing v6 media cassette.
30+
...(options.provider === "openai"
31+
? { cassette: { variantKey: "openai-v6" } }
32+
: {}),
33+
},
34+
timeoutMs: 180_000,
35+
});
36+
const events = harness.events();
37+
for (const testCase of captureCases) {
38+
const root = findLatestSpan(
39+
events,
40+
`attachment-capture-${testCase.name}`,
41+
);
42+
expect(root).toBeDefined();
43+
const children = events.filter(
44+
(event) =>
45+
event.row.root_span_id === root?.row.root_span_id &&
46+
event.span.id !== root?.span.id,
47+
);
48+
expect(children.length).toBeGreaterThan(0);
49+
const payload = JSON.stringify(children);
50+
if (testCase.enabled)
51+
expect(payload).toContain("braintrust_attachment");
52+
else {
53+
expect(payload).not.toContain("braintrust_attachment");
54+
expect(payload).not.toContain('"file_data":');
55+
expect(payload).not.toContain('"b64_json":');
56+
expect(payload).not.toContain('"data":');
57+
expect(payload).not.toContain("data:image/png;base64,");
58+
}
59+
expect(payload).toContain("completion_tokens");
60+
expect(children.some((event) => event.row.output !== undefined)).toBe(
61+
true,
62+
);
63+
}
64+
});
65+
}, 240_000);
66+
}
67+
}
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
import assert from "node:assert/strict";
2+
import { readFile } from "node:fs/promises";
3+
import { initLogger, wrapOpenAI, wrapAnthropic } from "braintrust";
4+
import { MINIMAL_PNG_BASE64 } from "./media-fixtures.mjs";
5+
import { scopedName } from "./provider-runtime.mjs";
6+
7+
export const captureCases = [
8+
{ name: "default", env: "", enabled: false },
9+
{ name: "local-default", env: "", global: true, local: true, enabled: false },
10+
{
11+
name: "local-enabled",
12+
env: "false",
13+
global: false,
14+
local: true,
15+
option: true,
16+
enabled: true,
17+
},
18+
{
19+
name: "local-disabled",
20+
env: "true",
21+
global: true,
22+
local: true,
23+
option: false,
24+
enabled: false,
25+
},
26+
{
27+
name: "local-environment",
28+
env: "true",
29+
global: false,
30+
local: true,
31+
enabled: true,
32+
},
33+
{ name: "global-enabled", env: "false", option: true, enabled: true },
34+
{ name: "global-disabled", env: "true", option: false, enabled: false },
35+
];
36+
37+
export async function runAttachmentCaptureScenario(provider, scenarioUrl, sdk) {
38+
const wrapped = process.env.CAPTURE_WRAPPED === "true";
39+
let request;
40+
if (provider === "anthropic") {
41+
const raw = new sdk.default({
42+
apiKey: process.env.ANTHROPIC_API_KEY,
43+
baseURL: process.env.ANTHROPIC_BASE_URL,
44+
});
45+
const client = wrapped ? wrapAnthropic(raw) : raw;
46+
const data = (
47+
await readFile(new URL("./test-image.png", scenarioUrl))
48+
).toString("base64");
49+
request = async () => {
50+
const result = await client.messages.create({
51+
model: "claude-haiku-4-5",
52+
max_tokens: 32,
53+
temperature: 0,
54+
messages: [
55+
{
56+
role: "user",
57+
content: [
58+
{
59+
type: "text",
60+
text: "Describe the attached image in one short sentence.",
61+
},
62+
{
63+
type: "image",
64+
source: { type: "base64", media_type: "image/png", data },
65+
},
66+
],
67+
},
68+
],
69+
});
70+
assert.ok(result.content.length > 0);
71+
};
72+
} else if (provider === "openai") {
73+
const raw = new sdk.default({
74+
apiKey: process.env.OPENAI_API_KEY,
75+
baseURL: process.env.OPENAI_BASE_URL,
76+
});
77+
const client = wrapped ? wrapOpenAI(raw) : raw;
78+
request = async () => {
79+
const result = await client.chat.completions.create({
80+
model: "gpt-4o-mini-2024-07-18",
81+
messages: [
82+
{
83+
role: "user",
84+
content: [
85+
{
86+
type: "text",
87+
text: "Describe this image in three words or fewer.",
88+
},
89+
{
90+
type: "image_url",
91+
image_url: {
92+
url: `data:image/png;base64,${MINIMAL_PNG_BASE64}`,
93+
},
94+
},
95+
],
96+
},
97+
],
98+
max_tokens: 24,
99+
temperature: 0,
100+
});
101+
assert.ok(result.choices[0].message.content);
102+
};
103+
}
104+
const projectName = scopedName("tmp-luca-attachment-capture");
105+
for (const testCase of captureCases) {
106+
process.env.BRAINTRUST_CAPTURE_ATTACHMENTS = testCase.env;
107+
if (testCase.local)
108+
initLogger({ projectName, captureAttachments: testCase.global });
109+
const logger = initLogger({
110+
projectName,
111+
setCurrent: !testCase.local,
112+
captureAttachments: testCase.option,
113+
});
114+
await logger.traced(request, {
115+
name: `attachment-capture-${testCase.name}`,
116+
event: {
117+
metadata: {
118+
scenario: `${provider}-instrumentation`,
119+
testRunId: process.env.BRAINTRUST_E2E_RUN_ID,
120+
},
121+
},
122+
});
123+
await logger.flush();
124+
}
125+
}

‎e2e/scenarios/ai-sdk-instrumentation/scenario.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ describeVariants("variants", () => {
5555
await runScenarioDir({
5656
entry: scenario.wrapperEntry,
5757
env: {
58+
BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
5859
AI_SDK_PACKAGE_NAME: scenario.packageName,
5960
AI_SDK_OPENAI_PACKAGE_NAME: scenario.openaiModuleName,
6061
...(scenario.anthropicModuleName
@@ -110,6 +111,7 @@ describeVariants("variants", () => {
110111
await runNodeScenarioDir({
111112
entry: scenario.autoEntry,
112113
env: {
114+
BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
113115
AI_SDK_PACKAGE_NAME: scenario.packageName,
114116
AI_SDK_OPENAI_PACKAGE_NAME: scenario.openaiModuleName,
115117
...(scenario.anthropicModuleName
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import { runMain } from "../../helpers/provider-runtime.mjs";
2+
import { runAttachmentCaptureScenario } from "../../helpers/attachment-capture-scenario.mjs";
3+
4+
runMain(async () =>
5+
runAttachmentCaptureScenario(
6+
"anthropic",
7+
import.meta.url,
8+
await import(process.env.CAPTURE_PACKAGE_NAME),
9+
),
10+
);

‎e2e/scenarios/anthropic-instrumentation/scenario.test.ts‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { defineAttachmentCaptureTests } from "../../helpers/attachment-capture-assertions";
12
import { describe } from "vitest";
23
import {
34
prepareScenarioDir,
@@ -52,7 +53,10 @@ describe.concurrent("variants", () => {
5253
runScenario: async ({ runScenarioDir }) => {
5354
await runScenarioDir({
5455
entry: scenario.wrapperEntry,
55-
env: { ANTHROPIC_PACKAGE_NAME: scenario.dependencyName },
56+
env: {
57+
BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
58+
ANTHROPIC_PACKAGE_NAME: scenario.dependencyName,
59+
},
5660
runContext: {
5761
variantKey: scenario.snapshotName,
5862
originalScenarioDir,
@@ -78,7 +82,10 @@ describe.concurrent("variants", () => {
7882
runScenario: async ({ runNodeScenarioDir }) => {
7983
await runNodeScenarioDir({
8084
entry: scenario.autoEntry,
81-
env: { ANTHROPIC_PACKAGE_NAME: scenario.dependencyName },
85+
env: {
86+
BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
87+
ANTHROPIC_PACKAGE_NAME: scenario.dependencyName,
88+
},
8289
nodeArgs: ["--import", "braintrust/hook.mjs"],
8390
runContext: {
8491
variantKey: scenario.snapshotName,
@@ -100,3 +107,13 @@ describe.concurrent("variants", () => {
100107
});
101108
}
102109
});
110+
111+
for (const scenario of anthropicScenarios) {
112+
defineAttachmentCaptureTests({
113+
scenarioDir,
114+
originalScenarioDir,
115+
provider: "anthropic",
116+
packageName: scenario.dependencyName,
117+
variantKey: scenario.snapshotName,
118+
});
119+
}

‎e2e/scenarios/google-genai-instrumentation/scenario.test.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,10 @@ describe.concurrent("variants", () => {
5454
runScenario: async ({ runScenarioDir }) => {
5555
await runScenarioDir({
5656
entry: scenario.wrapperEntry,
57-
env: { GOOGLE_GENAI_PACKAGE_NAME: scenario.dependencyName },
57+
env: {
58+
BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
59+
GOOGLE_GENAI_PACKAGE_NAME: scenario.dependencyName,
60+
},
5861
runContext: {
5962
variantKey: scenario.snapshotName,
6063
originalScenarioDir,
@@ -73,7 +76,10 @@ describe.concurrent("variants", () => {
7376
runScenario: async ({ runNodeScenarioDir }) => {
7477
await runNodeScenarioDir({
7578
entry: scenario.autoEntry,
76-
env: { GOOGLE_GENAI_PACKAGE_NAME: scenario.dependencyName },
79+
env: {
80+
BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
81+
GOOGLE_GENAI_PACKAGE_NAME: scenario.dependencyName,
82+
},
7783
nodeArgs: ["--import", "braintrust/hook.mjs"],
7884
runContext: {
7985
variantKey: scenario.snapshotName,

‎e2e/scenarios/groq-instrumentation/scenario.test.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,10 @@ describe.concurrent("variants", () => {
3838
runScenario: async ({ runScenarioDir }) => {
3939
await runScenarioDir({
4040
entry: "scenario.ts",
41-
env: { GROQ_PACKAGE_NAME: scenario.dependencyName },
41+
env: {
42+
BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
43+
GROQ_PACKAGE_NAME: scenario.dependencyName,
44+
},
4245
runContext: {
4346
variantKey: scenario.snapshotName,
4447
originalScenarioDir,
@@ -57,7 +60,10 @@ describe.concurrent("variants", () => {
5760
runScenario: async ({ runNodeScenarioDir }) => {
5861
await runNodeScenarioDir({
5962
entry: "scenario.mjs",
60-
env: { GROQ_PACKAGE_NAME: scenario.dependencyName },
63+
env: {
64+
BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
65+
GROQ_PACKAGE_NAME: scenario.dependencyName,
66+
},
6167
nodeArgs: ["--import", "braintrust/hook.mjs"],
6268
runContext: {
6369
variantKey: scenario.snapshotName,
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import { runMain } from "../../helpers/provider-runtime.mjs";
2+
import { runAttachmentCaptureScenario } from "../../helpers/attachment-capture-scenario.mjs";
3+
4+
runMain(async () =>
5+
runAttachmentCaptureScenario(
6+
"openai",
7+
import.meta.url,
8+
await import(process.env.CAPTURE_PACKAGE_NAME),
9+
),
10+
);

‎e2e/scenarios/openai-instrumentation/scenario.test.ts‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { defineAttachmentCaptureTests } from "../../helpers/attachment-capture-assertions";
12
import { describe, it } from "vitest";
23
import {
34
prepareScenarioDir,
@@ -92,7 +93,10 @@ describe.concurrent("variants", () => {
9293
runScenario: async ({ runScenarioDir }) => {
9394
await runScenarioDir({
9495
entry: scenario.wrapperEntry,
95-
env: { OPENAI_PACKAGE_NAME: scenario.dependencyName },
96+
env: {
97+
BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
98+
OPENAI_PACKAGE_NAME: scenario.dependencyName,
99+
},
96100
runContext: {
97101
variantKey: scenario.snapshotName,
98102
originalScenarioDir,
@@ -113,7 +117,10 @@ describe.concurrent("variants", () => {
113117
runScenario: async ({ runNodeScenarioDir }) => {
114118
await runNodeScenarioDir({
115119
entry: scenario.autoEntry,
116-
env: { OPENAI_PACKAGE_NAME: scenario.dependencyName },
120+
env: {
121+
BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
122+
OPENAI_PACKAGE_NAME: scenario.dependencyName,
123+
},
117124
nodeArgs: ["--import", "braintrust/hook.mjs"],
118125
runContext: {
119126
variantKey: scenario.snapshotName,
@@ -132,3 +139,13 @@ describe.concurrent("variants", () => {
132139
});
133140
}
134141
});
142+
143+
for (const scenario of openaiScenarios) {
144+
defineAttachmentCaptureTests({
145+
scenarioDir,
146+
originalScenarioDir,
147+
provider: "openai",
148+
packageName: scenario.dependencyName,
149+
variantKey: scenario.snapshotName,
150+
});
151+
}

0 commit comments

Comments
 (0)