Skip to content

Commit 6d0956a

Browse files
committed
Update PR #2519
1 parent 6c0c0c4 commit 6d0956a

11 files changed

Lines changed: 866 additions & 63 deletions

File tree

‎integrations/openai-agents-js/src/attachment-capture.test.ts‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,3 +171,174 @@ it.each([true, false])(
171171
}
172172
},
173173
);
174+
175+
it.each([false, true])(
176+
"applies capture=%s to native Chat Completions generation inputs",
177+
async (captureAttachments) => {
178+
const options = {
179+
projectId: "test-project-id",
180+
projectName: "tmp-luca-agents-chat-capture",
181+
};
182+
const logger = initLogger({
183+
...options,
184+
captureAttachments,
185+
setCurrent: false,
186+
});
187+
const processor = new OpenAIAgentsTraceProcessor({ logger });
188+
const trace: AgentsTrace = {
189+
type: "trace",
190+
traceId: "chat-input",
191+
name: "chat input",
192+
groupId: null,
193+
};
194+
const content = [
195+
{ type: "text", text: "Describe this media." },
196+
{
197+
type: "image_url",
198+
image_url: { url: "data:image/png;base64,AQID", detail: "low" },
199+
},
200+
{ type: "image_url", image_url: { url: "data:image/png;base64,AQID" } },
201+
{ type: "input_audio", input_audio: { data: "AQID", format: "mp3" } },
202+
{ type: "file", file: { file_data: "AQID", filename: "document.pdf" } },
203+
{
204+
type: "image_url",
205+
image_url: { url: "https://example.com/image.png" },
206+
},
207+
{ type: "file", file: { file_id: "file-123" } },
208+
];
209+
const originalContent = structuredClone(content);
210+
const span: AgentsSpan = {
211+
type: "trace.span",
212+
traceId: trace.traceId,
213+
spanId: "generation",
214+
parentId: null,
215+
startedAt: null,
216+
endedAt: null,
217+
error: null,
218+
spanData: {
219+
type: "generation",
220+
input: [{ role: "user", content }],
221+
},
222+
};
223+
await processor.onTraceStart(trace);
224+
await processor.onSpanStart(span);
225+
initLogger({ ...options, captureAttachments: !captureAttachments });
226+
await processor.onSpanEnd(span);
227+
await processor.onTraceEnd(trace);
228+
const rows = await background.drain();
229+
const inputRows = rows.filter((row) => "input" in row);
230+
expect(inputRows).toHaveLength(2);
231+
for (const row of inputRows) {
232+
expect(row).toHaveProperty("input.0.content", [
233+
content[0],
234+
{
235+
type: "image_url",
236+
image_url: {
237+
detail: "low",
238+
...(captureAttachments ? { url: expect.any(Attachment) } : {}),
239+
},
240+
},
241+
...(captureAttachments
242+
? [{ type: "image_url", image_url: { url: expect.any(Attachment) } }]
243+
: []),
244+
{
245+
type: "input_audio",
246+
input_audio: {
247+
format: "mp3",
248+
...(captureAttachments ? { data: expect.any(Attachment) } : {}),
249+
},
250+
},
251+
{
252+
type: "file",
253+
file: {
254+
filename: "document.pdf",
255+
...(captureAttachments
256+
? { file_data: expect.any(Attachment) }
257+
: {}),
258+
},
259+
},
260+
...content.slice(5),
261+
]);
262+
}
263+
expect(content).toEqual(originalContent);
264+
expect(JSON.stringify(inputRows)).not.toContain("AQID");
265+
},
266+
);
267+
268+
it.each([false, true])(
269+
"captures generation and protocol audio with the owning policy (%s)",
270+
async (captureAttachments) => {
271+
const options = {
272+
projectId: "test-project-id",
273+
projectName: "tmp-luca-agents-audio-output",
274+
};
275+
const logger = initLogger({
276+
...options,
277+
captureAttachments,
278+
setCurrent: false,
279+
});
280+
const processor = new OpenAIAgentsTraceProcessor({ logger });
281+
const trace: AgentsTrace = {
282+
type: "trace",
283+
traceId: "audio-output",
284+
name: "audio output",
285+
groupId: null,
286+
};
287+
const audio = {
288+
data: "AQID",
289+
transcript: "Hello",
290+
id: "audio-1",
291+
expires_at: 123,
292+
};
293+
const output = [{ choices: [{ message: { role: "assistant", audio } }] }];
294+
const span: AgentsSpan = {
295+
type: "trace.span",
296+
traceId: trace.traceId,
297+
spanId: "generation",
298+
parentId: null,
299+
startedAt: null,
300+
endedAt: null,
301+
error: null,
302+
spanData: {
303+
type: "generation",
304+
output,
305+
input: [
306+
{
307+
role: "user",
308+
content: [
309+
{
310+
type: "audio",
311+
audio: "AQID",
312+
format: "mp3",
313+
transcript: "Hello",
314+
},
315+
{ type: "audio", audio: { id: "file-123" } },
316+
],
317+
},
318+
],
319+
},
320+
};
321+
await processor.onTraceStart(trace);
322+
await processor.onSpanStart(span);
323+
initLogger({ ...options, captureAttachments: !captureAttachments });
324+
await processor.onSpanEnd(span);
325+
const rows = await background.drain();
326+
const row = rows.find((row) => "output" in row && row.output !== undefined);
327+
expect(row).toHaveProperty("output.0.choices.0.message.audio", {
328+
transcript: "Hello",
329+
id: "audio-1",
330+
expires_at: 123,
331+
...(captureAttachments ? { data: expect.any(Attachment) } : {}),
332+
});
333+
expect(row).toHaveProperty("input.0.content", [
334+
{
335+
type: "audio",
336+
format: "mp3",
337+
transcript: "Hello",
338+
...(captureAttachments ? { audio: expect.any(Attachment) } : {}),
339+
},
340+
{ type: "audio", audio: { id: "file-123" } },
341+
]);
342+
expect(audio.data).toBe("AQID");
343+
},
344+
);

‎integrations/openai-agents-js/src/index.ts‎

Lines changed: 82 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
currentSpan,
88
NOOP_SPAN,
99
Attachment,
10+
BaseAttachment,
1011
_internalGetGlobalState,
1112
getSpanParentObject,
1213
withCurrent,
@@ -107,13 +108,21 @@ function getTimeElapsed(end?: string, start?: string): number | undefined {
107108
return (endTime - startTime) / 1000;
108109
}
109110

110-
function processAudioAttachment(audio: { data: string; format: string }) {
111+
function processAudioAttachment(audio: {
112+
data: string;
113+
format?: string;
114+
[key: string]: unknown;
115+
}) {
111116
if (
112117
!_internalGetGlobalState()._internalCaptureAttachmentsEnabled(
113118
getSpanParentObject(),
114119
)
115120
) {
116-
return { format: audio.format };
121+
return Object.fromEntries(
122+
Object.keys(audio)
123+
.filter((key) => key !== "data")
124+
.map((key) => [key, audio[key]]),
125+
);
117126
}
118127

119128
const format = audio.format || "wav";
@@ -169,14 +178,49 @@ export class OpenAIAgentsTraceProcessor {
169178
options.maxTraces ?? OpenAIAgentsTraceProcessor.DEFAULT_MAX_TRACES;
170179
}
171180

172-
private processInputMedia(input: any): any {
181+
private processMedia(input: any): any {
182+
if (input instanceof BaseAttachment) return input;
173183
if (Array.isArray(input)) {
174184
return input
175-
.map((item) => this.processInputMedia(item))
185+
.map((item) => this.processMedia(item))
176186
.filter((item) => item !== undefined);
177187
}
178188

179189
if (input && typeof input === "object") {
190+
if (input.type === "audio" && typeof input.audio === "string") {
191+
const audio = processAudioAttachment({
192+
data: input.audio,
193+
format: input.format,
194+
});
195+
const { audio: _audio, ...metadata } = input;
196+
return audio.data === undefined
197+
? metadata
198+
: { ...metadata, audio: audio.data };
199+
}
200+
if (
201+
input.type === "input_audio" &&
202+
input.input_audio &&
203+
typeof input.input_audio === "object"
204+
) {
205+
return {
206+
...input,
207+
input_audio: processAudioAttachment(input.input_audio),
208+
};
209+
}
210+
if (
211+
input.role === "assistant" &&
212+
input.audio &&
213+
typeof input.audio === "object"
214+
) {
215+
return { ...input, audio: processAudioAttachment(input.audio) };
216+
}
217+
const nestedField =
218+
input.type === "image_url"
219+
? "image_url"
220+
: input.type === "file"
221+
? "file"
222+
: undefined;
223+
const media = nestedField ? input[nestedField] : input;
180224
const field =
181225
input.type === "input_image"
182226
? "image_url" in input
@@ -186,41 +230,58 @@ export class OpenAIAgentsTraceProcessor {
186230
? "file_data" in input
187231
? "file_data"
188232
: "file"
189-
: undefined;
190-
if (field && typeof input[field] === "string") {
191-
const data = input[field];
233+
: input.type === "image_url"
234+
? "url"
235+
: input.type === "file"
236+
? "file_data"
237+
: undefined;
238+
if (field && media && typeof media[field] === "string") {
239+
const data = media[field];
192240
if (/^https?:\/\//i.test(data)) return input;
193241
if (
194242
!_internalGetGlobalState()._internalCaptureAttachmentsEnabled(
195243
getSpanParentObject(),
196244
)
197245
) {
198-
const { [field]: _data, ...metadata } = input;
199-
return Object.keys(metadata).some((key) => key !== "type")
200-
? metadata
246+
const { [field]: _data, ...metadata } = media;
247+
const remaining = nestedField
248+
? {
249+
...input,
250+
[nestedField]: Object.keys(metadata).length
251+
? metadata
252+
: undefined,
253+
}
254+
: metadata;
255+
return Object.keys(remaining).some(
256+
(key) => key !== "type" && remaining[key] !== undefined,
257+
)
258+
? remaining
201259
: undefined;
202260
}
203261
const dataUriMatch = data.match(/^data:([^;,]+);base64,([\s\S]*)$/);
204262
const contentType =
205263
dataUriMatch?.[1] ??
206-
(input.type === "input_image"
264+
(input.type === "input_image" || input.type === "image_url"
207265
? "image/png"
208266
: "application/octet-stream");
209267
const extension =
210268
contentType === "application/octet-stream"
211269
? "bin"
212270
: contentType.split("/")[1];
213271
try {
214-
return {
215-
...input,
272+
const processed = {
273+
...media,
216274
[field]: new Attachment({
217275
data: new Uint8Array(
218276
base64ToUint8Array(dataUriMatch?.[2] ?? data),
219277
).buffer,
220-
filename: input.filename || `${input.type}.${extension}`,
278+
filename: media.filename || `${input.type}.${extension}`,
221279
contentType,
222280
}),
223281
};
282+
return nestedField
283+
? { ...input, [nestedField]: processed }
284+
: processed;
224285
} catch {
225286
return input;
226287
}
@@ -229,7 +290,7 @@ export class OpenAIAgentsTraceProcessor {
229290
// Recursively process nested objects
230291
const result: any = {};
231292
for (const [key, value] of Object.entries(input)) {
232-
const processed = this.processInputMedia(value);
293+
const processed = this.processMedia(value);
233294
if (processed !== undefined) result[key] = processed;
234295
}
235296
return result;
@@ -422,11 +483,13 @@ export class OpenAIAgentsTraceProcessor {
422483
}
423484

424485
if (spanData._input !== undefined) {
425-
data.input = this.processInputMedia(spanData._input);
486+
data.input = this.processMedia(spanData._input);
426487
}
427488

428489
if (spanData._response !== undefined) {
429-
data.output = this.processOutputImages(spanData._response.output);
490+
data.output = this.processMedia(
491+
this.processOutputImages(spanData._response.output),
492+
);
430493
}
431494

432495
if (spanData._response) {
@@ -547,8 +610,8 @@ export class OpenAIAgentsTraceProcessor {
547610
metrics.prompt_cached_tokens = usage.input_tokens_details.cached_tokens;
548611

549612
return {
550-
input: spanData.input,
551-
output: spanData.output,
613+
input: this.processMedia(spanData.input),
614+
output: this.processMedia(this.processOutputImages(spanData.output)),
552615
metadata: {
553616
model: spanData.model,
554617
model_config: spanData.model_config,

0 commit comments

Comments
 (0)