|
34 | 34 |
|
35 | 35 | # The assistant header that precedes a turn's generation scaffold + content. |
36 | 36 | _ASSIST_HDR = "<|im_start|>assistant\n" |
| 37 | +# A tail past an assistant-header match is template framing only if it is |
| 38 | +# positively recognized: empty, or a recipient line closed by the message |
| 39 | +# marker (e.g. ` to=user<|message|>`). Templates terminate turns in |
| 40 | +# unbounded ways (`[/INST]`, `<|eot|>`, `<|im_end|>`, ...), so no blocklist |
| 41 | +# of delimiters can enumerate turn structure exhaustively -- and anything |
| 42 | +# unrecognized means the match sits inside message content (e.g. a user |
| 43 | +# message quoting a header literal), where truncating would delete real |
| 44 | +# conversation text. Unknown framing falls back to plain text (correct |
| 45 | +# output, just no reuse). |
| 46 | +_FRAMING_RE = re.compile( |
| 47 | + r"\A ?to=[A-Za-z0-9_.]{1,64}(?: constrain=[A-Za-z0-9_]+)?<\|message\|>\Z" |
| 48 | +) |
37 | 49 | # A scaffold region is exactly empty (history strips it before the last user) or |
38 | 50 | # one of the Qwen3 think scaffolds (history preserves the empty block after the |
39 | 51 | # last user; the open form is the think-mode generation preamble). Anything else |
@@ -92,11 +104,13 @@ def _find_gemma_tool_call_span(rendered: str, search_pos: int): |
92 | 104 | class OpenAITranscriptState: |
93 | 105 | def __init__(self, template: ChatTemplate): |
94 | 106 | self._template = template |
95 | | - self._assist_hdr = ( |
96 | | - template.assistant_header() |
97 | | - if hasattr(template, "assistant_header") |
98 | | - else _ASSIST_HDR |
99 | | - ) |
| 107 | + # Note: the method's mere presence says nothing about whether the |
| 108 | + # header is right for the template -- production adapters always |
| 109 | + # provide it, even when it returns the default for a template that |
| 110 | + # renders different framing. The tail-shape check below is the only |
| 111 | + # boundary verification. |
| 112 | + _header_fn = getattr(template, "assistant_header", None) |
| 113 | + self._assist_hdr = _header_fn() if _header_fn else _ASSIST_HDR |
100 | 114 | # session_id -> [{"fp": str, "ids": list[int] | None}, ...] (one per |
101 | 115 | # assistant turn we produced, in order). Cleared on reset/close. |
102 | 116 | self._turns: dict[str, list[dict]] = {} |
@@ -127,23 +141,50 @@ def _normalize_scaffold(self, text_chunk: str, preamble: str) -> Optional[str]: |
127 | 141 | the exact resident scaffold. The region is empty (history stripped it -> |
128 | 142 | insert) or a think scaffold (history preserved it -> replace). Returns the |
129 | 143 | adjusted text, or None if it isn't a recognized scaffold (-> text fallback).""" |
130 | | - # No scaffold for this turn's mode/template: nothing to reproduce, so |
131 | | - # leave the chunk untouched -- and don't require the Qwen/ChatML header, |
132 | | - # so token-id splicing still works for templates with a different |
133 | | - # assistant header (the fix stays a true no-op for non-think models). |
134 | | - if not preamble: |
135 | | - return text_chunk |
136 | 144 | h = text_chunk.rfind(self._assist_hdr) |
137 | 145 | if h == -1: |
138 | | - return None |
| 146 | + # No assistant header: with a scaffold to reproduce this is |
| 147 | + # ambiguous (-> text fallback); without one there is nothing to |
| 148 | + # normalize, so splicing still works for templates with a different |
| 149 | + # assistant header. |
| 150 | + return None if preamble else text_chunk |
139 | 151 | base = h + len(self._assist_hdr) |
| 152 | + if not preamble: |
| 153 | + # No generation scaffold: the worker prefills nothing ahead of the |
| 154 | + # turn's raw tokens, so the spliced ids begin immediately after the |
| 155 | + # header. Drop the template's re-rendered framing (recipient / |
| 156 | + # message markers), which describes the echoed message rather than |
| 157 | + # the raw generation -- e.g. a thinking turn was generated as |
| 158 | + # ` to=self<|message|>...` but re-renders as ` to=user<|message|>`, |
| 159 | + # and keeping that framing shifts every later token id (worker |
| 160 | + # "mismatch" -> full re-prefill). Also exact for non-thinking |
| 161 | + # turns, whose raw opening the spliced ids reproduce verbatim. |
| 162 | + tail = text_chunk[base:] |
| 163 | + if tail and not self._is_template_framing(tail): |
| 164 | + # Unverifiable boundary: the header match sits inside message |
| 165 | + # content (e.g. a user message quoting a header literal), so |
| 166 | + # truncating here would delete real conversation text while |
| 167 | + # splicing proceeds on the corrupted prompt. Fall back to |
| 168 | + # plain text (correct output, just no reuse). |
| 169 | + return None |
| 170 | + return text_chunk[:base] |
140 | 171 | region = text_chunk[base:] |
141 | 172 | if region == preamble: |
142 | 173 | return text_chunk |
143 | 174 | if not _THINK_SCAFFOLD_RE.match(region): |
144 | 175 | return None |
145 | 176 | return text_chunk[:base] + preamble |
146 | 177 |
|
| 178 | + def _is_template_framing(self, tail: str) -> bool: |
| 179 | + """Whether `tail` (text between an assistant-header match and the |
| 180 | + sentinel) is positively recognized template framing: empty, or a |
| 181 | + recipient line closed by the message marker. Anything else -- |
| 182 | + message content after a quoted header literal, a second header, |
| 183 | + turn terminators of any shape -- falls back to plain text.""" |
| 184 | + if not tail: |
| 185 | + return True |
| 186 | + return _FRAMING_RE.match(tail) is not None |
| 187 | + |
147 | 188 | def _split_on_sentinels( |
148 | 189 | self, rendered: str, sub: dict[str, dict] |
149 | 190 | ) -> Optional[list[dict]]: |
|
0 commit comments