Skip to content

Commit eddd8e8

Browse files
committed
fix(openai): keep n>1 stream choices separate in recorded output
_extract_streamed_openai_response used a single accumulator for every choice in a chat completion stream, so an n>1 stream with interleaved choices merged content and tool-call arguments from separate choices into one corrupted output. Accumulate per choice index and return one response per choice when the stream carries multiple choices; single-choice streams keep the existing output shape.
1 parent 19490f1 commit eddd8e8

2 files changed

Lines changed: 135 additions & 4 deletions

File tree

‎langfuse/openai.py‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -760,7 +760,11 @@ def _extract_streamed_response_api_response(chunks: Any) -> Any:
760760

761761

762762
def _extract_streamed_openai_response(resource: Any, chunks: Any) -> Any:
763-
completion: Any = defaultdict(lambda: None) if resource.type == "chat" else ""
763+
# Per-choice accumulators keyed by the provider's choice index: n>1
764+
# streams interleave choices within a single chunk, so a shared
765+
# accumulator would merge content and tool calls across choices.
766+
chat_completions: dict[int, defaultdict] = {}
767+
completion_texts: dict[int, str] = {}
764768
model, usage, finish_reason, service_tier = None, None, None, None
765769

766770
for chunk in chunks:
@@ -778,7 +782,15 @@ def _extract_streamed_openai_response(resource: Any, chunks: Any) -> Any:
778782
for choice in choices:
779783
if _is_openai_v1():
780784
choice = choice.__dict__
785+
786+
choice_index = choice.get("index", None)
787+
if not isinstance(choice_index, int):
788+
choice_index = 0
789+
781790
if resource.type == "chat":
791+
completion = chat_completions.setdefault(
792+
choice_index, defaultdict(lambda: None)
793+
)
782794
delta = choice.get("delta", None)
783795
choice_finish_reason = choice.get("finish_reason", None)
784796
if choice_finish_reason is not None:
@@ -868,9 +880,11 @@ def _extract_streamed_openai_response(resource: Any, chunks: Any) -> Any:
868880
) + tool_arguments
869881

870882
if resource.type == "completion":
871-
completion += choice.get("text", "")
883+
completion_texts[choice_index] = completion_texts.get(
884+
choice_index, ""
885+
) + (choice.get("text", "") or "")
872886

873-
def get_response_for_chat() -> Any:
887+
def get_response_for_chat(completion: Any) -> Any:
874888
content = completion["content"]
875889

876890
if completion["tool_calls"]:
@@ -897,9 +911,26 @@ def get_response_for_chat() -> Any:
897911

898912
return content or None
899913

914+
response: Any
915+
if resource.type == "chat":
916+
if len(chat_completions) > 1:
917+
response = [
918+
get_response_for_chat(chat_completions[index])
919+
for index in sorted(chat_completions)
920+
]
921+
else:
922+
response = get_response_for_chat(
923+
chat_completions.get(0, defaultdict(lambda: None))
924+
)
925+
else:
926+
if len(completion_texts) > 1:
927+
response = [completion_texts[index] for index in sorted(completion_texts)]
928+
else:
929+
response = completion_texts.get(0, "")
930+
900931
return (
901932
model,
902-
get_response_for_chat() if resource.type == "chat" else completion,
933+
response,
903934
usage,
904935
{"finish_reason": finish_reason} if finish_reason is not None else None,
905936
service_tier,

‎tests/unit/test_openai.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,55 @@ def _make_single_chunk_stream():
251251
)
252252

253253

254+
def _make_chat_stream_chunks_two_choices():
255+
# Interleaved n=2 stream: choice indexes arrive as [0, 1], [1, 0], [0, 1].
256+
# Choice 0 streams content "A" then "0"; choice 1 streams "B" then "1".
257+
# Each choice carries one tool call at tool index 0.
258+
def _choice(index, content, tool_args, finish_reason=None):
259+
return SimpleNamespace(
260+
index=index,
261+
delta=SimpleNamespace(
262+
role="assistant" if content is not None else None,
263+
content=content,
264+
function_call=None,
265+
tool_calls=[
266+
SimpleNamespace(
267+
index=0,
268+
id=f"call_{index}",
269+
type="function",
270+
function=SimpleNamespace(
271+
name=f"tool_{index}", arguments=tool_args
272+
),
273+
)
274+
]
275+
if tool_args is not None
276+
else None,
277+
),
278+
finish_reason=finish_reason,
279+
)
280+
281+
return [
282+
SimpleNamespace(
283+
model="gpt-4o-mini",
284+
choices=[_choice(0, "A", '{"x":'), _choice(1, "B", '{"y":')],
285+
usage=None,
286+
),
287+
SimpleNamespace(
288+
model="gpt-4o-mini",
289+
choices=[_choice(1, "1", "1}"), _choice(0, "0", "0}")],
290+
usage=None,
291+
),
292+
SimpleNamespace(
293+
model="gpt-4o-mini",
294+
choices=[
295+
_choice(0, None, None, finish_reason="stop"),
296+
_choice(1, None, None, finish_reason="stop"),
297+
],
298+
usage=SimpleNamespace(prompt_tokens=3, completion_tokens=2, total_tokens=5),
299+
),
300+
]
301+
302+
254303
def test_chat_completion_exports_generation_span(
255304
langfuse_memory_client, get_span, json_attr
256305
):
@@ -401,6 +450,57 @@ def test_streaming_chat_completion_preserves_tool_calls_after_content():
401450
assert metadata == {"finish_reason": "tool_calls"}
402451

403452

453+
def test_streaming_chat_completion_multiple_choices_are_not_merged():
454+
model, completion, usage, metadata, _service_tier = (
455+
lf_openai_module._extract_streamed_openai_response(
456+
SimpleNamespace(type="chat"),
457+
_make_chat_stream_chunks_two_choices(),
458+
)
459+
)
460+
461+
assert model == "gpt-4o-mini"
462+
assert completion == [
463+
{
464+
"role": "assistant",
465+
"content": "A0",
466+
"tool_calls": [
467+
{
468+
"id": "call_0",
469+
"type": "function",
470+
"function": {"name": "tool_0", "arguments": '{"x":0}'},
471+
}
472+
],
473+
},
474+
{
475+
"role": "assistant",
476+
"content": "B1",
477+
"tool_calls": [
478+
{
479+
"id": "call_1",
480+
"type": "function",
481+
"function": {"name": "tool_1", "arguments": '{"y":1}'},
482+
}
483+
],
484+
},
485+
]
486+
assert usage.total_tokens == 5
487+
assert metadata == {"finish_reason": "stop"}
488+
489+
490+
def test_streaming_chat_completion_single_choice_output_unchanged():
491+
model, completion, usage, metadata, _service_tier = (
492+
lf_openai_module._extract_streamed_openai_response(
493+
SimpleNamespace(type="chat"),
494+
_make_chat_stream_chunks(),
495+
)
496+
)
497+
498+
assert model == "gpt-4o-mini"
499+
assert completion == "2"
500+
assert usage.total_tokens == 4
501+
assert metadata == {"finish_reason": "stop"}
502+
503+
404504
def test_response_api_output_serializes_openai_parsed_response_objects():
405505
class ParsedOutput(BaseModel):
406506
name: str

0 commit comments

Comments
 (0)