Skip to content

Commit 88b210f

Browse files
authored
[llm_server] Fix named-session prefix-cache reuse and report cached tokens (pytorch#22799)
Agentic multi-turn workloads got zero KV-cache reuse on named sessions: every turn re-prefilled the full prompt while llama.cpp and SGLang reused shared prefixes, and the reuse was unobservable (usage never reported cached tokens). The cache failure is a framing duplication: Glimmer generates its own to=self/to=user framing, but the splice retained the template's reconstructed framing and then appended generated ids that already contained framing, shifting every later token id (worker 'mismatch' -> full re-prefill). The fix drops the reconstructed framing so the stored raw ids extend the resident prefix exactly. Replay fidelity comes from the stored ids: the transcript fingerprint covers content + tool calls (not reasoning), and splicing replaces echoed reasoning with stored ids. Changes: 1. openai_transcript.py: drop the sentinel re-render's framing ahead of spliced ids. Truncation requires a positively recognized boundary -- an empty tail or a recipient line closed by the message marker -- else falls back to plain text. Templates terminate turns in unbounded ways ([/INST], <|eot|>, <|im_end|>, ...), so no delimiter blocklist can verify the boundary; a header-like literal inside message content can therefore never cause deletion of real conversation text. 2. serve.py (muse-glimmer): join multiple thinking blocks with a newline (SGLang parity) so no protocol framing leaks into reasoning_content. 3. serving_chat.py + protocol.py: populate usage.prompt_tokens_details.cached_tokens from reused_prompt_tokens (streaming and non-streaming), matching the OpenAI/SGLang usage contract. Scope note: this repairs named-session continuation only. Unlike llama.cpp/SGLang, the worker still resets all resident state on any prefix mismatch (worker_prefill_plan.h) -- broader prefix caching is future work. Tests: hermetic contract tests over the real HTTP surface with a fake worker (no model/GPU). Reasoning-returning tests request reasoning explicitly, so this commit lands independently of the return_reasoning default. Recorded ids are the worker's non-terminal generated ids (worker_loop.h:805); the real-tokenizer test asserts the complete assembled prompt under the worker's segment encoding plus realism pins so a duplicated terminator or reordered ids fail. Each fix verified fail-before/pass-after. 135 passed, 5 skipped (real-tokenizer vector needs absent tokenizer dirs), 0 failed. Bench evidence (Terminal-Bench build-cython-ext, same model, temp 0.8, reasoning on): prefix reuse 97.7% (1,174,734/1,202,984), at parity with llama.cpp 98.0% and SGLang 97.1%.
1 parent fd5226a commit 88b210f

9 files changed

Lines changed: 685 additions & 22 deletions

File tree

‎examples/llm_server/python/openai_transcript.py‎

Lines changed: 53 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,18 @@
3434

3535
# The assistant header that precedes a turn's generation scaffold + content.
3636
_ASSIST_HDR = "<|im_start|>assistant\n"
37+
# A tail past an assistant-header match is template framing only if it is
38+
# positively recognized: empty, or a recipient line closed by the message
39+
# marker (e.g. ` to=user<|message|>`). Templates terminate turns in
40+
# unbounded ways (`[/INST]`, `<|eot|>`, `<|im_end|>`, ...), so no blocklist
41+
# of delimiters can enumerate turn structure exhaustively -- and anything
42+
# unrecognized means the match sits inside message content (e.g. a user
43+
# message quoting a header literal), where truncating would delete real
44+
# conversation text. Unknown framing falls back to plain text (correct
45+
# output, just no reuse).
46+
_FRAMING_RE = re.compile(
47+
r"\A ?to=[A-Za-z0-9_.]{1,64}(?: constrain=[A-Za-z0-9_]+)?<\|message\|>\Z"
48+
)
3749
# A scaffold region is exactly empty (history strips it before the last user) or
3850
# one of the Qwen3 think scaffolds (history preserves the empty block after the
3951
# last user; the open form is the think-mode generation preamble). Anything else
@@ -92,11 +104,13 @@ def _find_gemma_tool_call_span(rendered: str, search_pos: int):
92104
class OpenAITranscriptState:
93105
def __init__(self, template: ChatTemplate):
94106
self._template = template
95-
self._assist_hdr = (
96-
template.assistant_header()
97-
if hasattr(template, "assistant_header")
98-
else _ASSIST_HDR
99-
)
107+
# Note: the method's mere presence says nothing about whether the
108+
# header is right for the template -- production adapters always
109+
# provide it, even when it returns the default for a template that
110+
# renders different framing. The tail-shape check below is the only
111+
# boundary verification.
112+
_header_fn = getattr(template, "assistant_header", None)
113+
self._assist_hdr = _header_fn() if _header_fn else _ASSIST_HDR
100114
# session_id -> [{"fp": str, "ids": list[int] | None}, ...] (one per
101115
# assistant turn we produced, in order). Cleared on reset/close.
102116
self._turns: dict[str, list[dict]] = {}
@@ -127,23 +141,50 @@ def _normalize_scaffold(self, text_chunk: str, preamble: str) -> Optional[str]:
127141
the exact resident scaffold. The region is empty (history stripped it ->
128142
insert) or a think scaffold (history preserved it -> replace). Returns the
129143
adjusted text, or None if it isn't a recognized scaffold (-> text fallback)."""
130-
# No scaffold for this turn's mode/template: nothing to reproduce, so
131-
# leave the chunk untouched -- and don't require the Qwen/ChatML header,
132-
# so token-id splicing still works for templates with a different
133-
# assistant header (the fix stays a true no-op for non-think models).
134-
if not preamble:
135-
return text_chunk
136144
h = text_chunk.rfind(self._assist_hdr)
137145
if h == -1:
138-
return None
146+
# No assistant header: with a scaffold to reproduce this is
147+
# ambiguous (-> text fallback); without one there is nothing to
148+
# normalize, so splicing still works for templates with a different
149+
# assistant header.
150+
return None if preamble else text_chunk
139151
base = h + len(self._assist_hdr)
152+
if not preamble:
153+
# No generation scaffold: the worker prefills nothing ahead of the
154+
# turn's raw tokens, so the spliced ids begin immediately after the
155+
# header. Drop the template's re-rendered framing (recipient /
156+
# message markers), which describes the echoed message rather than
157+
# the raw generation -- e.g. a thinking turn was generated as
158+
# ` to=self<|message|>...` but re-renders as ` to=user<|message|>`,
159+
# and keeping that framing shifts every later token id (worker
160+
# "mismatch" -> full re-prefill). Also exact for non-thinking
161+
# turns, whose raw opening the spliced ids reproduce verbatim.
162+
tail = text_chunk[base:]
163+
if tail and not self._is_template_framing(tail):
164+
# Unverifiable boundary: the header match sits inside message
165+
# content (e.g. a user message quoting a header literal), so
166+
# truncating here would delete real conversation text while
167+
# splicing proceeds on the corrupted prompt. Fall back to
168+
# plain text (correct output, just no reuse).
169+
return None
170+
return text_chunk[:base]
140171
region = text_chunk[base:]
141172
if region == preamble:
142173
return text_chunk
143174
if not _THINK_SCAFFOLD_RE.match(region):
144175
return None
145176
return text_chunk[:base] + preamble
146177

178+
def _is_template_framing(self, tail: str) -> bool:
179+
"""Whether `tail` (text between an assistant-header match and the
180+
sentinel) is positively recognized template framing: empty, or a
181+
recipient line closed by the message marker. Anything else --
182+
message content after a quoted header literal, a second header,
183+
turn terminators of any shape -- falls back to plain text."""
184+
if not tail:
185+
return True
186+
return _FRAMING_RE.match(tail) is not None
187+
147188
def _split_on_sentinels(
148189
self, rendered: str, sub: dict[str, dict]
149190
) -> Optional[list[dict]]:

‎examples/llm_server/python/protocol.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,17 @@ def resolved_max_tokens(self) -> int:
9595
return -1
9696

9797

98+
class PromptTokensDetails(BaseModel):
99+
# Warm-resume accounting, SGLang-compatible shape: tokens served from the
100+
# session's resident state instead of prefetched this request.
101+
cached_tokens: int = 0
102+
103+
98104
class Usage(BaseModel):
99105
prompt_tokens: int = 0
100106
completion_tokens: int = 0
101107
total_tokens: int = 0
108+
prompt_tokens_details: Optional[PromptTokensDetails] = None
102109

103110

104111
class ResponseMessage(BaseModel):

‎examples/llm_server/python/serving_chat.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
ChunkChoice,
3434
DeltaMessage,
3535
FunctionCall,
36+
PromptTokensDetails,
3637
ResponseMessage,
3738
ToolCall,
3839
Usage,
@@ -580,6 +581,9 @@ async def _complete(
580581
prompt_tokens=stats.prompt_tokens,
581582
completion_tokens=stats.completion_tokens,
582583
total_tokens=stats.prompt_tokens + stats.completion_tokens,
584+
prompt_tokens_details=PromptTokensDetails(
585+
cached_tokens=stats.reused_prompt_tokens
586+
),
583587
),
584588
)
585589

@@ -763,6 +767,9 @@ def chunk(delta: DeltaMessage, finish=None) -> str:
763767
prompt_tokens=stats.prompt_tokens,
764768
completion_tokens=stats.completion_tokens,
765769
total_tokens=stats.prompt_tokens + stats.completion_tokens,
770+
prompt_tokens_details=PromptTokensDetails(
771+
cached_tokens=stats.reused_prompt_tokens
772+
),
766773
),
767774
)
768775
yield f"data: {usage_chunk.model_dump_json(exclude_none=True)}\n\n"

‎examples/llm_server/python/tests/conftest.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,19 @@ class FakeRunner:
4040
matching the real worker's contract."""
4141

4242
def __init__(
43-
self, tokens, fail=False, finish_reason=None, max_named_sessions=0, gen_ids=None
43+
self,
44+
tokens,
45+
fail=False,
46+
finish_reason=None,
47+
max_named_sessions=0,
48+
gen_ids=None,
49+
reuse=0,
4450
):
4551
self._tokens = list(tokens)
4652
self._fail = fail
4753
self._finish_reason = finish_reason # worker-reported stop reason, if any
4854
self._gen_ids = list(gen_ids or []) # ids reported per turn
55+
self._reuse = reuse # reused_prompt_tokens reported in done stats
4956
self.captured_config = None
5057
self.stopped = False
5158
self.reset_count = 0
@@ -89,6 +96,7 @@ def generate(self, prompt, config, token_callback=None, stats_callback=None):
8996
stats.num_generated_tokens = len(self._tokens)
9097
stats.finish_reason = self._finish_reason
9198
stats.generated_token_ids = list(self._gen_ids)
99+
stats.reused_prompt_tokens = self._reuse
92100
stats_callback(stats)
93101

94102

@@ -119,13 +127,15 @@ def _make(
119127
finish_reason=None,
120128
max_named_sessions=0,
121129
gen_ids=None,
130+
reuse=0,
122131
):
123132
fake = FakeRunner(
124133
tokens,
125134
fail=fail,
126135
finish_reason=finish_reason,
127136
max_named_sessions=max_named_sessions,
128137
gen_ids=gen_ids,
138+
reuse=reuse,
129139
)
130140
runtime = SessionRuntime(fake) # one fake worker
131141
template = ChatTemplate(hf_tokenizer_path=None, allow_fallback=True)

‎examples/llm_server/python/tests/test_contract.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,35 @@ def test_chat_nonstreaming_shape(make_client):
7373
)
7474

7575

76+
def test_usage_reports_cached_tokens(make_client):
77+
# Warm-resume accounting must be visible in-band (SGLang-compatible
78+
# `prompt_tokens_details.cached_tokens`), not just in server logs, so
79+
# benches can measure reuse without scraping.
80+
client, _ = make_client(tokens=["Hello"], reuse=3)
81+
body = client.post(
82+
"/v1/chat/completions",
83+
json={"model": "test-model", "messages": [{"role": "user", "content": "hi"}]},
84+
).json()
85+
assert body["usage"]["prompt_tokens_details"]["cached_tokens"] == 3
86+
87+
88+
def test_streaming_usage_reports_cached_tokens(make_client):
89+
client, _ = make_client(tokens=["a"], reuse=2)
90+
resp = client.post(
91+
"/v1/chat/completions",
92+
json={
93+
"model": "test-model",
94+
"messages": [{"role": "user", "content": "hi"}],
95+
"stream": True,
96+
"stream_options": {"include_usage": True},
97+
},
98+
)
99+
chunks, _ = _sse_chunks(resp.text)
100+
usage_chunks = [c for c in chunks if c.get("usage")]
101+
assert usage_chunks, "expected a chunk carrying usage"
102+
assert usage_chunks[-1]["usage"]["prompt_tokens_details"]["cached_tokens"] == 2
103+
104+
76105
def test_unknown_model_is_rejected(make_client):
77106
client, _ = make_client()
78107
resp = client.post(

0 commit comments

Comments
 (0)