From fde307539e0bb36b8ed66a11bcd73fc65caa06f5 Mon Sep 17 00:00:00 2001 From: Akshay Sasi Date: Tue, 25 Aug 2026 04:23:30 +0530 Subject: [PATCH] fix: tolerate stream chunks without a choices key in stream_chunk_builder (#34382) The Responses-API to /chat/completions bridge yields ModelResponseStream chunks that carry choices followed by a trailing event object that has no choices key. stream_chunk_builder assumed every chunk was subscriptable at "choices", so assembling those chunks raised KeyError('choices') and was re-wrapped as a 500 APIError building the streaming usage. Guard each choices access with .get("choices") so choices-less chunks are skipped instead of crashing. Behavior is unchanged for chunks that do carry choices, since .get("choices") is truthy only for a non-empty choices list. Adds a regression test that assembles content across chunks followed by a trailing chunk with no choices key. Co-authored-by: mubashir1osmani --- litellm/main.py | 22 ++++++++-------- .../test_streaming_chunk_builder_utils.py | 26 +++++++++++++++++++ 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 84931c63544..d3967473f99 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8568,7 +8568,7 @@ def stream_chunk_builder( if len(chunks) == 0: return None ## Route to the text completion logic - first_chunk_with_choices: Final = next((c for c in chunks if c["choices"]), None) + first_chunk_with_choices: Final = next((c for c in chunks if c.get("choices")), None) if first_chunk_with_choices is not None and isinstance( first_chunk_with_choices["choices"][0], litellm.utils.TextChoices ): # route to the text completion logic @@ -8583,7 +8583,7 @@ def stream_chunk_builder( simple_content_parts: Final[list[str]] = [] is_simple_text_stream = True for chunk in chunks: - if len(chunk["choices"]) == 0: + if not chunk.get("choices"): continue choice = chunk["choices"][0] @@ -8649,7 +8649,7 @@ def stream_chunk_builder( tool_call_chunks: Final = [ chunk for chunk in chunks - if len(chunk["choices"]) > 0 + if chunk.get("choices") and "tool_calls" in chunk["choices"][0]["delta"] and chunk["choices"][0]["delta"]["tool_calls"] is not None ] @@ -8663,7 +8663,7 @@ def stream_chunk_builder( function_call_chunks: Final = [ chunk for chunk in chunks - if len(chunk["choices"]) > 0 + if chunk.get("choices") and "function_call" in chunk["choices"][0]["delta"] and chunk["choices"][0]["delta"]["function_call"] is not None ] @@ -8676,7 +8676,7 @@ def stream_chunk_builder( content_chunks: Final = [ chunk for chunk in chunks - if len(chunk["choices"]) > 0 + if chunk.get("choices") and "content" in chunk["choices"][0]["delta"] and chunk["choices"][0]["delta"]["content"] is not None ] @@ -8687,7 +8687,7 @@ def stream_chunk_builder( thinking_blocks: Final = [ chunk for chunk in chunks - if len(chunk["choices"]) > 0 + if chunk.get("choices") and "thinking_blocks" in chunk["choices"][0]["delta"] and chunk["choices"][0]["delta"]["thinking_blocks"] is not None ] @@ -8700,7 +8700,7 @@ def stream_chunk_builder( reasoning_chunks: Final = [ chunk for chunk in chunks - if len(chunk["choices"]) > 0 + if chunk.get("choices") and "reasoning_content" in chunk["choices"][0]["delta"] and chunk["choices"][0]["delta"]["reasoning_content"] is not None ] @@ -8713,7 +8713,7 @@ def stream_chunk_builder( annotation_chunks: Final = [ chunk for chunk in chunks - if len(chunk["choices"]) > 0 + if chunk.get("choices") and "annotations" in chunk["choices"][0]["delta"] and chunk["choices"][0]["delta"]["annotations"] is not None ] @@ -8730,7 +8730,7 @@ def stream_chunk_builder( audio_chunks: Final = [ chunk for chunk in chunks - if len(chunk["choices"]) > 0 + if chunk.get("choices") and "audio" in chunk["choices"][0]["delta"] and chunk["choices"][0]["delta"]["audio"] is not None ] @@ -8744,7 +8744,7 @@ def stream_chunk_builder( image_chunks: Final = [ chunk for chunk in chunks - if len(chunk["choices"]) > 0 + if chunk.get("choices") and "images" in chunk["choices"][0]["delta"] and chunk["choices"][0]["delta"]["images"] is not None ] @@ -8761,7 +8761,7 @@ def stream_chunk_builder( provider_specific_chunks: Final = [ chunk for chunk in chunks - if len(chunk["choices"]) > 0 + if chunk.get("choices") and "provider_specific_fields" in chunk["choices"][0]["delta"] and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None ] diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 44e77506b3f..4b5b51cb4b8 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -989,6 +989,32 @@ def test_cost_field_in_usage_chunks(): assert usage.completion_tokens == 5 +def test_stream_chunk_builder_tolerates_trailing_chunk_without_choices(): + """Regression for https://github.com/BerriAI/litellm/issues/32051 + + The Responses-API bridge yields ModelResponseStream chunks with choices + followed by a trailing event object that has no ``choices`` key. Building + those chunks used to raise ``KeyError('choices')`` (surfaced as a 500 + APIError); it must now skip the choices-less chunk and assemble content. + """ + from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject + + content_chunks = [ + ModelResponseStream( + model="gpt-4o", + choices=[StreamingChoices(index=0, delta=Delta(content=part))], + ) + for part in ("Hello", " world") + ] + trailing_chunk = BaseLiteLLMOpenAIResponseObject() + assert "choices" not in trailing_chunk + + response = stream_chunk_builder(chunks=content_chunks + [trailing_chunk]) + + assert response is not None + assert response.choices[0].message.content == "Hello world" + + def test_anthropic_speed_and_geo_survive_stream_assembly(): """Anthropic prices fast mode and non-global regions with a multiplier read off ``usage.speed`` / ``usage.inference_geo``. Dropping them while reassembling a stream