Skip to content

Commit 6fd6fad

Browse files
committed
fix(openai-native): send body prompt_cache_key for cache affinity
What: the OpenAI-native handler now sends a task-scoped `prompt_cache_key` in the Responses request body on both body builders (streaming and completePrompt), and no longer sends the `session_id` request header. The key is `metadata.taskId`, falling back to the handler session id, clamped to OpenAI's 64-character limit, and omitted for models whose catalog entry has `supportsPromptCache: false`. Why: OpenAI keys prompt-cache routing off the body `prompt_cache_key` field. The `session_id` header we were sending is not part of the prompt-caching API and is ignored on the public Responses API, so every OpenAI-native task was leaving cache hit-rate on the table. The header also carries an underscore in its name, which hardened gateways (Envoy/nginx `underscores_in_headers`) reject outright, so it was a latent 400 for anyone behind such a proxy with a custom base URL. `prompt_cache_retention: "24h"` is untouched: it is the supported lifetime control on the pre-5.6 models that set it in the catalog, and the existing retention specs pass unmodified. Impact: cache affinity now follows the documented body field, stable across turns of one task and distinct across tasks. No usage accounting, cost, or other request field changes. openai-codex.ts and xai.ts are untouched: codex already sends a body key, and xai never had the header.
1 parent 159c42e commit 6fd6fad

3 files changed

Lines changed: 197 additions & 9 deletions

File tree

‎src/api/providers/__tests__/openai-native-usage.spec.ts‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -440,6 +440,70 @@ describe("OpenAiNativeHandler - normalizeUsage", () => {
440440
})
441441
})
442442

443+
describe("OpenAiNativeHandler - prompt cache key", () => {
444+
let handler: OpenAiNativeHandler
445+
446+
beforeEach(() => {
447+
handler = new OpenAiNativeHandler({
448+
openAiNativeApiKey: "test-key",
449+
})
450+
})
451+
452+
const buildRequestBody = (modelId: string, taskId?: string) => {
453+
handler["options"].apiModelId = modelId
454+
const model = handler.getModel()
455+
const metadata = taskId === undefined ? undefined : { taskId }
456+
return handler["buildRequestBody"](model, [], "", model.verbosity, undefined, metadata)
457+
}
458+
459+
it("should send the task id as the body prompt_cache_key", () => {
460+
const body = buildRequestBody("gpt-5.1", "task-abc")
461+
expect(body.prompt_cache_key).toBe("task-abc")
462+
})
463+
464+
it("should keep the key stable across turns of the same task", () => {
465+
const firstTurn = buildRequestBody("gpt-5.1", "task-stable")
466+
const secondTurn = buildRequestBody("gpt-5.1", "task-stable")
467+
expect(firstTurn.prompt_cache_key).toBe("task-stable")
468+
expect(secondTurn.prompt_cache_key).toBe(firstTurn.prompt_cache_key)
469+
})
470+
471+
it("should give distinct keys to distinct tasks", () => {
472+
const first = buildRequestBody("gpt-5.1", "task-one")
473+
const second = buildRequestBody("gpt-5.1", "task-two")
474+
expect(first.prompt_cache_key).toBe("task-one")
475+
expect(second.prompt_cache_key).toBe("task-two")
476+
expect(first.prompt_cache_key).not.toBe(second.prompt_cache_key)
477+
})
478+
479+
it("should fall back to the handler session id when no task id is supplied", () => {
480+
const body = buildRequestBody("gpt-5.1")
481+
expect(body.prompt_cache_key).toBe(handler["sessionId"])
482+
expect(body.prompt_cache_key).toEqual(expect.any(String))
483+
})
484+
485+
it("should omit the key when the model does not support prompt caching", () => {
486+
const modelId = "codex-mini-latest"
487+
expect(openAiNativeModels[modelId as keyof typeof openAiNativeModels].supportsPromptCache).toBe(false)
488+
489+
const body = buildRequestBody(modelId, "task-abc")
490+
expect(body.prompt_cache_key).toBeUndefined()
491+
})
492+
493+
it("should clamp an over-length task id to OpenAI's 64-character limit", () => {
494+
const longTaskId = "t".repeat(80)
495+
const body = buildRequestBody("gpt-5.1", longTaskId)
496+
expect(body.prompt_cache_key).toHaveLength(64)
497+
expect(body.prompt_cache_key).toBe(longTaskId.slice(0, 64))
498+
})
499+
500+
it("should keep prompt_cache_retention alongside the key for 24h models", () => {
501+
const body = buildRequestBody("gpt-5.1", "task-abc")
502+
expect(body.prompt_cache_key).toBe("task-abc")
503+
expect(body.prompt_cache_retention).toBe("24h")
504+
})
505+
})
506+
443507
describe("cost calculation", () => {
444508
it.each([
445509
{ tier: OpenAiServiceTier.Default, expectedCost: 4.525 },

‎src/api/providers/__tests__/openai-native.spec.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,6 +534,26 @@ describe("OpenAiNativeHandler", () => {
534534

535535
expect(result).toBe("")
536536
})
537+
538+
it("should scope the prompt_cache_key to the handler session", async () => {
539+
mockResponsesCreate.mockResolvedValue({ output: [] })
540+
541+
await handler.completePrompt("Test prompt")
542+
543+
const [request] = mockResponsesCreate.mock.calls[0]
544+
expect(request.prompt_cache_key).toBe(handler["sessionId"])
545+
expect(request.prompt_cache_key).toEqual(expect.any(String))
546+
})
547+
548+
it("should omit the prompt_cache_key when the model does not support prompt caching", async () => {
549+
mockResponsesCreate.mockResolvedValue({ output: [] })
550+
handler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "codex-mini-latest" })
551+
552+
await handler.completePrompt("Test prompt")
553+
554+
const [request] = mockResponsesCreate.mock.calls[0]
555+
expect(request).not.toHaveProperty("prompt_cache_key")
556+
})
537557
})
538558

539559
describe("getModel", () => {
@@ -1446,6 +1466,64 @@ describe("OpenAiNativeHandler", () => {
14461466
expect(callBody.previous_response_id).toBeUndefined()
14471467
})
14481468

1469+
it("sends a task-scoped prompt_cache_key in the body and no session_id header", async () => {
1470+
const mockFetch = vitest.fn().mockResolvedValue({
1471+
ok: true,
1472+
body: new ReadableStream({
1473+
start(controller) {
1474+
controller.enqueue(new TextEncoder().encode("data: [DONE]\n\n"))
1475+
controller.close()
1476+
},
1477+
}),
1478+
})
1479+
global.fetch = mockFetch as typeof fetch
1480+
// Force the hand-built SSE path so the wire body and headers are inspectable.
1481+
mockResponsesCreate.mockRejectedValue(new Error("SDK not available"))
1482+
1483+
const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" })
1484+
1485+
for (const taskId of ["task-cache-one", "task-cache-one", "task-cache-two"]) {
1486+
for await (const _chunk of cacheHandler.createMessage(systemPrompt, messages, { taskId })) {
1487+
// consume
1488+
}
1489+
}
1490+
1491+
const bodies = mockFetch.mock.calls.map(([, init]) => JSON.parse(init.body))
1492+
expect(bodies).toHaveLength(3)
1493+
expect(bodies[0].prompt_cache_key).toBe("task-cache-one")
1494+
// Same task, second turn: the key must not drift, or the growing prefix loses affinity.
1495+
expect(bodies[1].prompt_cache_key).toBe(bodies[0].prompt_cache_key)
1496+
// A different task must not share the bucket.
1497+
expect(bodies[2].prompt_cache_key).toBe("task-cache-two")
1498+
expect(bodies[2].prompt_cache_key).not.toBe(bodies[0].prompt_cache_key)
1499+
1500+
// The retention control is untouched by this change.
1501+
expect(bodies[0].prompt_cache_retention).toBe("24h")
1502+
1503+
// The affinity header is gone: OpenAI ignores it for caching and hardened
1504+
// gateways reject underscore header names outright.
1505+
for (const [, init] of mockFetch.mock.calls) {
1506+
expect(init.headers).not.toHaveProperty("session_id")
1507+
}
1508+
})
1509+
1510+
it("sends the task-scoped prompt_cache_key through the SDK path", async () => {
1511+
mockResponsesCreate.mockResolvedValue(asyncStreamFrom([]))
1512+
1513+
const cacheHandler = new OpenAiNativeHandler({ ...mockOptions, apiModelId: "gpt-5.1" })
1514+
1515+
for (const taskId of ["sdk-task-one", "sdk-task-one", "sdk-task-two"]) {
1516+
await collectStream(cacheHandler.createMessage(systemPrompt, messages, { taskId }))
1517+
}
1518+
1519+
const bodies = mockResponsesCreate.mock.calls.map(([body]) => body)
1520+
expect(bodies[0].prompt_cache_key).toBe("sdk-task-one")
1521+
expect(bodies[1].prompt_cache_key).toBe(bodies[0].prompt_cache_key)
1522+
expect(bodies[2].prompt_cache_key).toBe("sdk-task-two")
1523+
expect(bodies[0].prompt_cache_retention).toBe("24h")
1524+
expect(mockResponsesCreate.mock.calls[0][1].headers).not.toHaveProperty("session_id")
1525+
})
1526+
14491527
it("should provide helpful error messages for different error codes", async () => {
14501528
const testCases = [
14511529
{ status: 400, expectedMessage: "Invalid request to Responses API" },

‎src/api/providers/openai-native.ts‎

Lines changed: 55 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ import { sanitizeOpenAiCallId } from "../../utils/tool-id"
3434

3535
export type OpenAiNativeModel = ReturnType<OpenAiNativeHandler["getModel"]>
3636

37+
/**
38+
* OpenAI caps `prompt_cache_key` at 64 characters; longer values are rejected with a 400.
39+
* Task ids are uuidv7 today (36 chars), so this is a guard rather than a routine truncation.
40+
*/
41+
const PROMPT_CACHE_KEY_MAX_LENGTH = 64
42+
3743
export class OpenAiNativeHandler extends BaseProvider implements SingleCompletionHandler {
3844
protected options: ApiHandlerOptions
3945
private client: OpenAI
@@ -96,14 +102,16 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
96102
this.options.enableResponsesReasoningSummary = true
97103
}
98104
const apiKey = this.options.openAiNativeApiKey ?? NOT_PROVIDED
99-
// Include originator, session_id, and User-Agent headers for API tracking and debugging
105+
// Include originator and User-Agent headers for API tracking and debugging.
106+
// Cache affinity rides on the body `prompt_cache_key`, not on a header: OpenAI documents
107+
// the `session_id` header as ignored on the public Responses API, and underscore header
108+
// names are rejected outright by hardened gateways (Envoy/nginx `underscores_in_headers`).
100109
const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}`
101110
this.client = new OpenAI({
102111
baseURL: this.options.openAiNativeBaseUrl || undefined,
103112
apiKey,
104113
defaultHeaders: {
105114
originator: "zoo-code",
106-
session_id: this.sessionId,
107115
"User-Agent": userAgent,
108116
},
109117
timeout: this.timeoutMs,
@@ -330,6 +338,11 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
330338
include?: string[]
331339
/** Prompt cache retention policy: "in_memory" (default) or "24h" for extended caching */
332340
prompt_cache_retention?: "in_memory" | "24h"
341+
/**
342+
* Cache routing affinity key. OpenAI documents this body field as the routing lever;
343+
* the `session_id` request header is not part of the prompt-caching API.
344+
*/
345+
prompt_cache_key?: string
333346
tools?: Array<{
334347
type: "function"
335348
name: string
@@ -347,6 +360,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
347360
// Decide whether to enable extended prompt cache retention for this request
348361
const promptCacheRetention = this.getPromptCacheRetention(model)
349362

363+
// Cache routing affinity: a stable, task-scoped key lets OpenAI route every turn of the
364+
// same task to the machine holding its cached prefix.
365+
const promptCacheKey = this.getPromptCacheKey(model, metadata)
366+
350367
const body: ResponsesRequestBody = {
351368
model: model.id,
352369
input: formattedInput,
@@ -379,6 +396,8 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
379396
// Enable extended prompt cache retention for models that support it.
380397
// This uses the OpenAI Responses API `prompt_cache_retention` parameter.
381398
...(promptCacheRetention ? { prompt_cache_retention: promptCacheRetention } : {}),
399+
// Cache routing affinity key (body field, not the inert `session_id` header).
400+
...(promptCacheKey ? { prompt_cache_key: promptCacheKey } : {}),
382401
tools: (metadata?.tools ?? [])
383402
.filter((tool) => tool.type === "function")
384403
.map((tool) => {
@@ -418,12 +437,10 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
418437
// Create AbortController for cancellation
419438
this.abortController = new AbortController()
420439

421-
// Build per-request headers using taskId when available, falling back to sessionId
422-
const taskId = metadata?.taskId
440+
// Build per-request headers for API tracking and debugging
423441
const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}`
424442
const requestHeaders: Record<string, string> = {
425443
originator: "zoo-code",
426-
session_id: taskId || this.sessionId,
427444
"User-Agent": userAgent,
428445
}
429446

@@ -557,7 +574,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
557574
private async *makeResponsesApiRequest(
558575
requestBody: any,
559576
model: OpenAiNativeModel,
560-
metadata?: ApiHandlerCreateMessageMetadata,
577+
_metadata?: ApiHandlerCreateMessageMetadata,
561578
_systemPrompt?: string,
562579
_messages?: Anthropic.Messages.MessageParam[],
563580
): ApiStream {
@@ -568,8 +585,7 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
568585
// Create AbortController for cancellation
569586
this.abortController = new AbortController()
570587

571-
// Build per-request headers using taskId when available, falling back to sessionId
572-
const taskId = metadata?.taskId
588+
// Build per-request headers for API tracking and debugging
573589
const userAgent = `zoo-code/${Package.version} (${os.platform()} ${os.release()}; ${os.arch()}) node/${process.version.slice(1)}`
574590

575591
try {
@@ -579,7 +595,6 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
579595
"Content-Type": "application/json",
580596
Authorization: `Bearer ${apiKey}`,
581597
originator: "zoo-code",
582-
session_id: taskId || this.sessionId,
583598
"User-Agent": userAgent,
584599
},
585600
body: JSON.stringify(requestBody),
@@ -1420,6 +1435,30 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
14201435
return undefined
14211436
}
14221437

1438+
/**
1439+
* Returns the prompt cache routing key for this request, if the model caches prompts.
1440+
*
1441+
* OpenAI keys cache affinity off the `prompt_cache_key` body field; the `session_id`
1442+
* request header is not part of the prompt-caching API. The key must stay identical
1443+
* across every turn of one task (so its growing prefix keeps landing on the same
1444+
* machine) and differ between tasks (so unrelated prefixes do not compete for one
1445+
* routing bucket). `metadata.taskId` is the longest-lived identifier the handler
1446+
* receives: it is the persisted task id, stable across turns, restarts and
1447+
* rehydration, whereas `this.sessionId` is regenerated per handler instance.
1448+
* The handler session id is the fallback for callers that send no task id.
1449+
*/
1450+
private getPromptCacheKey(
1451+
model: OpenAiNativeModel,
1452+
metadata?: ApiHandlerCreateMessageMetadata,
1453+
): string | undefined {
1454+
if (!model.info.supportsPromptCache) return undefined
1455+
1456+
const key = metadata?.taskId || this.sessionId
1457+
if (!key) return undefined
1458+
1459+
return key.length > PROMPT_CACHE_KEY_MAX_LENGTH ? key.slice(0, PROMPT_CACHE_KEY_MAX_LENGTH) : key
1460+
}
1461+
14231462
/**
14241463
* Returns a shallow-cloned ModelInfo with pricing overridden for the given tier, if available.
14251464
* If no tier or no overrides exist, the original ModelInfo is returned.
@@ -1531,6 +1570,13 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
15311570
requestBody[SERVICE_TIER_KEY] = serviceTier
15321571
}
15331572

1573+
// Cache routing affinity key. `completePrompt` has no task metadata, so the handler
1574+
// session id scopes the key: one bucket per handler instance, never shared across tasks.
1575+
const promptCacheKey = this.getPromptCacheKey(model)
1576+
if (promptCacheKey) {
1577+
requestBody.prompt_cache_key = promptCacheKey
1578+
}
1579+
15341580
// Add reasoning if supported
15351581
if (reasoningEffort) {
15361582
requestBody.reasoning = {

0 commit comments

Comments
 (0)