Skip to content

Commit edad0a7

Browse files
authored
[llm_server] Default return_reasoning to True (pytorch#22800)
Separate API choice, not needed for the cache fix: thinking models bill reasoning tokens either way, so return them unless the client explicitly opts out with {"return_reasoning": False}. Matches SGLang/llama.cpp, which both return reasoning by default.
1 parent 88b210f commit edad0a7

2 files changed

Lines changed: 51 additions & 1 deletion

File tree

‎examples/llm_server/python/serving_chat.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,8 +139,11 @@ def _split_reasoning(self, text: str) -> tuple[Optional[str], str]:
139139

140140
@staticmethod
141141
def _return_reasoning(req: ChatCompletionRequest) -> bool:
142+
# Default ON: thinking models bill reasoning tokens either way;
143+
# return them unless the client explicitly opts out, matching
144+
# SGLang/llama.cpp. Explicit {"return_reasoning": False} opts out.
142145
kwargs = req.chat_template_kwargs or {}
143-
value = kwargs.get("return_reasoning", False)
146+
value = kwargs.get("return_reasoning", True)
144147
return value if isinstance(value, bool) else False
145148

146149
@staticmethod

‎examples/llm_server/python/tests/test_tool_calls.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,3 +219,50 @@ def test_parallel_calls_in_one_message(make_client):
219219
).json()
220220
calls = body["choices"][0]["message"]["tool_calls"]
221221
assert [json.loads(c["function"]["arguments"])["city"] for c in calls] == ["A", "B"]
222+
223+
224+
# --- reasoning_content default -------------------------------------------
225+
226+
227+
def _reasoning_serving():
228+
from executorch.examples.llm_server.python.serving_chat import ServingChat
229+
230+
def _split(text):
231+
head, _, tail = text.partition("ANSWER:")
232+
return head.strip() or None, tail.strip()
233+
234+
class _FakeTemplate:
235+
def turn_stop_sequences(self):
236+
return []
237+
238+
def special_tokens(self):
239+
return []
240+
241+
return ServingChat(None, _FakeTemplate(), "m", reasoning_extractor=_split)
242+
243+
244+
def _reasoning_req(**kw):
245+
from executorch.examples.llm_server.python.protocol import ChatCompletionRequest
246+
247+
return ChatCompletionRequest(messages=[{"role": "user", "content": "hi"}], **kw)
248+
249+
250+
def test_reasoning_returned_by_default():
251+
serving = _reasoning_serving()
252+
calls, reasoning, content = serving._extract_response(
253+
_reasoning_req(), "my plan ANSWER: visible text"
254+
)
255+
assert calls is None
256+
assert reasoning == "my plan"
257+
assert content == "visible text"
258+
259+
260+
def test_reasoning_explicit_opt_out():
261+
serving = _reasoning_serving()
262+
calls, reasoning, content = serving._extract_response(
263+
_reasoning_req(chat_template_kwargs={"return_reasoning": False}),
264+
"my plan ANSWER: visible text",
265+
)
266+
assert calls is None
267+
assert reasoning is None
268+
assert content == "visible text"

0 commit comments

Comments
 (0)