Skip to content

Commit e44f211

Browse files
gitcommit90gitcommit90
committed
Preserve Claude cache prefixes across turns
1 parent ccbaf7c commit e44f211

2 files changed

Lines changed: 148 additions & 44 deletions

File tree

‎src/lib/providers/claude.js‎

Lines changed: 92 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ function wrapSystemReminder(text) {
135135
);
136136
}
137137

138-
function prependToFirstUserMessage(messages, text) {
138+
function prependToFirstUserMessage(messages, text, cacheControl) {
139139
if (!Array.isArray(messages) || !messages.length) return messages;
140140
const out = messages.map((m) => ({
141141
...m,
@@ -144,15 +144,17 @@ function prependToFirstUserMessage(messages, text) {
144144
const idx = out.findIndex((m) => m.role === "user");
145145
if (idx < 0) return out;
146146
const m = out[idx];
147+
const reminder = {
148+
type: "text",
149+
text,
150+
...(cacheControl ? { cache_control: cacheControl } : {}),
151+
};
147152
if (typeof m.content === "string") {
148-
out[idx] = { ...m, content: text + m.content };
153+
out[idx] = { ...m, content: [reminder, { type: "text", text: m.content }] };
149154
} else if (Array.isArray(m.content)) {
150-
out[idx] = {
151-
...m,
152-
content: [{ type: "text", text }, ...m.content],
153-
};
155+
out[idx] = { ...m, content: [reminder, ...m.content] };
154156
} else {
155-
out[idx] = { ...m, content: [{ type: "text", text: text + String(m.content ?? "") }] };
157+
out[idx] = { ...m, content: [reminder, { type: "text", text: String(m.content ?? "") }] };
156158
}
157159
return out;
158160
}
@@ -179,11 +181,14 @@ function applyCloaking(body, accessToken, sessionId) {
179181

180182
// Capture original client system for move to user message
181183
let userSystemText = "";
184+
let userSystemCacheControl;
182185
if (Array.isArray(result.system)) {
183186
userSystemText = result.system
184187
.map((b) => (typeof b === "string" ? b : b?.text || ""))
185188
.filter(Boolean)
186189
.join("\n\n");
190+
userSystemCacheControl = result.system.findLast?.((block) => block?.cache_control)?.cache_control
191+
|| [...result.system].reverse().find((block) => block?.cache_control)?.cache_control;
187192
} else if (typeof result.system === "string") {
188193
userSystemText = result.system;
189194
}
@@ -198,7 +203,8 @@ function applyCloaking(body, accessToken, sessionId) {
198203
if (sanitized) {
199204
result.messages = prependToFirstUserMessage(
200205
result.messages || [],
201-
wrapSystemReminder(sanitized)
206+
wrapSystemReminder(sanitized),
207+
userSystemCacheControl
202208
);
203209
}
204210
}
@@ -402,35 +408,74 @@ function hasExplicitCacheControl(payload) {
402408

403409
/** Apply Claude's four-breakpoint policy only after the router has selected
404410
* Claude. Explicit client-authored breakpoints remain authoritative. */
405-
function applyAutomaticCacheControl(payload) {
411+
function applyAutomaticCacheControl(payload, preferredBaseBlocks = []) {
406412
if (hasExplicitCacheControl(payload)) return payload;
407413

408-
let baseBlock = null;
409-
for (const message of payload.messages || []) {
410-
if (message.role !== "user" || !Array.isArray(message.content)) continue;
411-
if (message.content.some((block) => block?.type === "tool_result")) continue;
412-
for (let index = message.content.length - 1; index >= 0; index--) {
413-
const block = message.content[index];
414-
if (block?.type === "text" || block?.type === "image") {
415-
baseBlock = block;
416-
break;
414+
const baseBlocks = [...new Set((Array.isArray(preferredBaseBlocks) ? preferredBaseBlocks : [preferredBaseBlocks]).filter(Boolean))];
415+
if (!baseBlocks.length) {
416+
let baseBlock = null;
417+
for (const message of payload.messages || []) {
418+
if (message.role !== "user" || !Array.isArray(message.content)) continue;
419+
if (message.content.some((block) => block?.type === "tool_result")) continue;
420+
for (let index = message.content.length - 1; index >= 0; index--) {
421+
const block = message.content[index];
422+
if (block?.type === "text" || block?.type === "image") {
423+
baseBlock = block;
424+
break;
425+
}
417426
}
418427
}
428+
if (baseBlock) baseBlocks.push(baseBlock);
419429
}
420-
if (baseBlock) baseBlock.cache_control = { type: "ephemeral" };
430+
for (const block of baseBlocks) block.cache_control = { type: "ephemeral" };
421431

422432
const toolResults = [];
423433
for (const message of payload.messages || []) {
424434
for (const block of Array.isArray(message.content) ? message.content : []) {
425435
if (block?.type === "tool_result") toolResults.push(block);
426436
}
427437
}
428-
for (const block of toolResults.slice(-3)) {
429-
block.cache_control = { type: "ephemeral" };
438+
const availableToolBreakpoints = Math.max(0, 4 - baseBlocks.length);
439+
if (availableToolBreakpoints > 0) {
440+
for (const block of toolResults.slice(-availableToolBreakpoints)) {
441+
block.cache_control = { type: "ephemeral" };
442+
}
430443
}
431444
return payload;
432445
}
433446

447+
function systemCacheScope(message) {
448+
return message?.extra_content?.openai?.cache_scope || "";
449+
}
450+
451+
function systemTextBlocks(message) {
452+
const nativeSystem = message?.[ANTHROPIC_METADATA]?.content;
453+
if (Array.isArray(nativeSystem)) return JSON.parse(JSON.stringify(nativeSystem));
454+
if (typeof nativeSystem === "string") return [{ type: "text", text: nativeSystem }];
455+
if (typeof message?.content === "string") return [{ type: "text", text: message.content }];
456+
if (Array.isArray(message?.content)) {
457+
return message.content
458+
.filter((part) => part?.type === "text" && part.text != null)
459+
.map((part) => ({
460+
type: "text",
461+
text: String(part.text),
462+
...(part.cache_control ? { cache_control: part.cache_control } : {}),
463+
}));
464+
}
465+
return message?.content != null ? [{ type: "text", text: String(message.content) }] : [];
466+
}
467+
468+
function lastCacheableMessageBlock(messages) {
469+
for (let messageIndex = messages.length - 1; messageIndex >= 0; messageIndex--) {
470+
const blocks = Array.isArray(messages[messageIndex]?.content) ? messages[messageIndex].content : [];
471+
for (let blockIndex = blocks.length - 1; blockIndex >= 0; blockIndex--) {
472+
const block = blocks[blockIndex];
473+
if (["text", "image", "tool_result"].includes(block?.type)) return block;
474+
}
475+
}
476+
return null;
477+
}
478+
434479
/**
435480
* Convert OpenAI chat messages → Anthropic messages + system + tools.
436481
* Critical: tool_result must be in a user message; tool_use only on assistant.
@@ -439,6 +484,13 @@ function applyAutomaticCacheControl(payload) {
439484
function toAnthropicBody(body, model, stream) {
440485
const systemParts = [];
441486
const messages = [];
487+
const sourceMessages = body.messages || [];
488+
const hasScopedSystem = sourceMessages.some((message) => message.role === "system" && systemCacheScope(message));
489+
const deferredDynamicSystemParts = [];
490+
const lastDynamicSystemIndex = sourceMessages.findLastIndex?.(
491+
(message) => message.role === "system" && systemCacheScope(message) === "dynamic_context"
492+
) ?? -1;
493+
const preferredCacheBases = [];
442494

443495
const pushMessage = (role, blocks) => {
444496
if (!blocks.length) return;
@@ -459,32 +511,28 @@ function toAnthropicBody(body, model, stream) {
459511
messages.push({ role, content: blocks });
460512
};
461513

462-
for (const m of body.messages || []) {
514+
for (let sourceIndex = 0; sourceIndex < sourceMessages.length; sourceIndex++) {
515+
const m = sourceMessages[sourceIndex];
463516
if (m.role === "system") {
464-
const nativeSystem = m[ANTHROPIC_METADATA]?.content;
465-
if (Array.isArray(nativeSystem)) {
466-
systemParts.push(...JSON.parse(JSON.stringify(nativeSystem)));
517+
const parts = systemTextBlocks(m);
518+
const scope = systemCacheScope(m);
519+
if (!hasScopedSystem || scope === "stable_instruction" || !scope) {
520+
systemParts.push(...parts);
467521
continue;
468522
}
469-
if (typeof nativeSystem === "string") {
470-
systemParts.push({ type: "text", text: nativeSystem });
523+
if (scope === "dynamic_context") {
524+
deferredDynamicSystemParts.push(...parts);
525+
if (sourceIndex !== lastDynamicSystemIndex) continue;
526+
const stableBase = [...systemParts].reverse().find((block) => block?.type === "text" || block?.type === "image") || null;
527+
const rollingBase = lastCacheableMessageBlock(messages);
528+
if (stableBase) preferredCacheBases.push(stableBase);
529+
if (rollingBase) preferredCacheBases.push(rollingBase);
530+
const text = deferredDynamicSystemParts.map((part) => part?.text || "").filter(Boolean).join("\n\n");
531+
if (text) pushMessage("user", [{ type: "text", text: wrapSystemReminder(text) }]);
471532
continue;
472533
}
473-
if (typeof m.content === "string") {
474-
systemParts.push({ type: "text", text: m.content });
475-
} else if (Array.isArray(m.content)) {
476-
for (const part of m.content) {
477-
if (part?.type === "text" && part.text != null) {
478-
systemParts.push({
479-
type: "text",
480-
text: String(part.text),
481-
...(part.cache_control ? { cache_control: part.cache_control } : {}),
482-
});
483-
}
484-
}
485-
} else if (m.content != null) {
486-
systemParts.push({ type: "text", text: String(m.content) });
487-
}
534+
const text = parts.map((part) => part?.text || "").filter(Boolean).join("\n\n");
535+
if (text) pushMessage("user", [{ type: "text", text: wrapSystemReminder(text) }]);
488536
continue;
489537
}
490538
const blocks = contentBlocksFromMessage(m);
@@ -515,7 +563,7 @@ function toAnthropicBody(body, model, stream) {
515563
stream: !!stream,
516564
};
517565
if (systemParts.length) {
518-
out.system = systemParts.some((part) => part.cache_control)
566+
out.system = hasScopedSystem || systemParts.some((part) => part.cache_control)
519567
? systemParts
520568
: systemParts.map((part) => part.text).join("\n\n");
521569
}
@@ -560,7 +608,7 @@ function toAnthropicBody(body, model, stream) {
560608
for (const [key, value] of Object.entries(body[ANTHROPIC_METADATA]?.options || {})) {
561609
out[key] = JSON.parse(JSON.stringify(value));
562610
}
563-
return applyAutomaticCacheControl(applyClaudeEffort(out, body, model));
611+
return applyAutomaticCacheControl(applyClaudeEffort(out, body, model), preferredCacheBases);
564612
}
565613

566614
function mapStopReason(stopReason) {

‎tests/cache-shaping.test.js‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,62 @@ describe("provider-selected cache shaping", () => {
5959
}
6060
});
6161

62+
it("keeps Claude volatile context behind durable history and preserves OAuth cache boundaries", () => {
63+
const payload = claude.toAnthropicBody({
64+
messages: [
65+
scoped("stable identity", "stable_instruction"),
66+
scoped("volatile 12:34:56", "dynamic_context"),
67+
{ role: "user", content: "old task" },
68+
{ role: "assistant", content: "old answer" },
69+
scoped("prior invocation boundary", "inline_context"),
70+
scoped("current invocation 42", "dynamic_context"),
71+
{ role: "user", content: "current task" },
72+
],
73+
}, "claude-opus-5-5", true);
74+
75+
const cloaked = claude.applyCloaking(payload, "sk-ant-oat-test", "00000000-0000-4000-8000-000000000000");
76+
const blocks = cloaked.messages.flatMap((message) => message.content);
77+
const stableIndex = blocks.findIndex((block) => block.text?.includes("stable identity"));
78+
const oldTaskIndex = blocks.findIndex((block) => block.text === "old task");
79+
const oldAnswerIndex = blocks.findIndex((block) => block.text === "old answer");
80+
const inlineIndex = blocks.findIndex((block) => block.text?.includes("prior invocation boundary"));
81+
const volatileIndex = blocks.findIndex((block) => block.text?.includes("volatile 12:34:56"));
82+
const currentIndex = blocks.findIndex((block) => block.text === "current task");
83+
84+
assert.ok(stableIndex >= 0 && stableIndex < oldTaskIndex);
85+
assert.ok(oldTaskIndex < oldAnswerIndex && oldAnswerIndex < inlineIndex);
86+
assert.ok(inlineIndex < volatileIndex && volatileIndex < currentIndex);
87+
assert.equal(blocks[stableIndex].cache_control.type, "ephemeral", "stable identity remains a reusable OAuth breakpoint");
88+
assert.equal(blocks[inlineIndex].cache_control.type, "ephemeral", "the durable history tail is cached before volatile context");
89+
assert.equal(blocks[volatileIndex].cache_control, undefined);
90+
assert.doesNotMatch(blocks.slice(0, inlineIndex + 1).map((block) => block.text || "").join("\n"), /volatile 12:34:56|current invocation 42/);
91+
});
92+
93+
it("reserves Claude breakpoints for stable/history prefixes before rolling tool results", () => {
94+
const messages = [
95+
scoped("stable identity", "stable_instruction"),
96+
scoped("volatile", "dynamic_context"),
97+
{ role: "user", content: "old task" },
98+
{ role: "assistant", content: "old answer" },
99+
scoped("current invocation", "dynamic_context"),
100+
{ role: "user", content: "current task" },
101+
...[1, 2, 3].flatMap((number) => [
102+
{ role: "assistant", content: "", tool_calls: [{ id: `t${number}`, type: "function", function: { name: "shell", arguments: "{}" } }] },
103+
{ role: "tool", tool_call_id: `t${number}`, content: `result ${number}` },
104+
]),
105+
];
106+
const payload = claude.toAnthropicBody({ messages }, "claude-opus-5-5", true);
107+
const blocks = [...(Array.isArray(payload.system) ? payload.system : []), ...payload.messages.flatMap((message) => message.content)];
108+
const marked = blocks.filter((block) => block.cache_control);
109+
assert.equal(marked.length, 4);
110+
assert.equal(blocks.find((block) => block.text === "stable identity").cache_control.type, "ephemeral");
111+
assert.equal(blocks.find((block) => block.text === "old answer").cache_control.type, "ephemeral");
112+
assert.equal(blocks.find((block) => block.type === "tool_result" && block.tool_use_id === "t1").cache_control, undefined);
113+
for (const id of ["t2", "t3"]) {
114+
assert.equal(blocks.find((block) => block.type === "tool_result" && block.tool_use_id === id).cache_control.type, "ephemeral");
115+
}
116+
});
117+
62118
it("does not leak provider-selected cache keys to arbitrary compatible servers", async () => {
63119
let sent;
64120
await openaiCompat.chat({ baseUrl: "https://custom.test/v1", apiKey: "key" }, {

0 commit comments

Comments
 (0)