Skip to content

Commit a05a924

Browse files
authored
Fix reasoning routing and cross-turn carry
Differential Revision: D113436993 Pull Request resolved: pytorch#21434
1 parent 14dab10 commit a05a924

2 files changed

Lines changed: 10 additions & 1 deletion

File tree

‎examples/llm_server/python/protocol.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ class ChatMessage(BaseModel):
3838
role: str
3939
content: Optional[Union[str, list[dict[str, Any]]]] = None
4040
name: Optional[str] = None
41+
# Optional prior-turn reasoning (chain-of-thought). Mirrors
42+
# ResponseMessage.reasoning_content so a multi-turn client can echo an assistant
43+
# turn's reasoning back into the request; a chat template that renders prior
44+
# reasoning needs it here, and without the field it is dropped at parse.
45+
reasoning_content: Optional[str] = None
4146
tool_calls: Optional[list[ToolCall]] = None
4247
tool_call_id: Optional[str] = None
4348

‎examples/llm_server/python/serving_chat.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,11 @@ async def _collect_until_stop(self, stream: AsyncIterator[str], stops: list[str]
189189

190190
def _extract_response(self, req: ChatCompletionRequest, text: str):
191191
"""Return tool calls, optional reasoning, and user-visible content."""
192+
# Split the reasoning channel out BEFORE tool parsing so the detector only
193+
# sees the visible (non-reasoning) text. Parsing the full raw output would let
194+
# a tool-call marker the model emitted inside its private reasoning become a
195+
# phantom tool call (and drop the real answer).
196+
reasoning, text = self._split_reasoning(text)
192197
tool_calls = None
193198
if self._tools_active(req):
194199
parsed = self._tool_detector_cls().detect_and_parse(
@@ -197,7 +202,6 @@ def _extract_response(self, req: ChatCompletionRequest, text: str):
197202
if parsed.calls:
198203
tool_calls = [self._to_openai_tool_call(c) for c in parsed.calls]
199204
text = parsed.normal_text
200-
reasoning, text = self._split_reasoning(text)
201205
if not self._return_reasoning(req):
202206
reasoning = None
203207
return tool_calls, reasoning, self._visible_content(text) or None

0 commit comments

Comments
 (0)