From 317bb1ea4d86a90dbaac47b77fe16129fdc1fe68 Mon Sep 17 00:00:00 2001 From: shivam Date: Fri, 24 Jul 2026 01:34:10 +0000 Subject: [PATCH 1/2] fix(streaming): guard empty-choices chunks in Responses bridge and Anthropic adapter iterators Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../adapters/streaming_iterator.py | 6 +- .../adapters/transformation.py | 2 +- .../streaming_iterator.py | 6 ++ .../test_streaming_iterator_combined_chunk.py | 54 +++++++++++ .../test_empty_choices_streaming_iterator.py | 91 +++++++++++++++++++ 5 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 tests/test_litellm/responses/litellm_completion_transformation/test_empty_choices_streaming_iterator.py diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index f02333c34c8..367f2585a9b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -423,7 +423,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): will_merge_into_held = ( self.holding_stop_reason_chunk is not None and getattr(chunk, "usage", None) is not None ) - is_final_chunk = chunk.choices[0].finish_reason is not None + is_final_chunk = bool(chunk.choices) and chunk.choices[0].finish_reason is not None processed_chunk = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_response_to_anthropic( response=chunk, current_content_block_index=self.current_content_block_index, @@ -646,7 +646,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): will_merge_into_held = ( self.holding_stop_reason_chunk is not None and getattr(chunk, "usage", None) is not None ) - is_final_chunk = chunk.choices[0].finish_reason is not None + is_final_chunk = bool(chunk.choices) and chunk.choices[0].finish_reason is not None processed_chunk = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_response_to_anthropic( response=chunk, current_content_block_index=self.current_content_block_index, @@ -889,6 +889,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Example logic - customize based on your needs: # If chunk indicates a tool call + if not chunk.choices: + return False if chunk.choices[0].finish_reason is not None: return False diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 4b6617fbeac..91efc39670e 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1476,7 +1476,7 @@ class LiteLLMAnthropicMessagesAdapter: applied_edits: Optional[List[AppliedEdit]] = None, ) -> Union[ContentBlockDelta, MessageBlockDelta]: ## base case - final chunk w/ finish reason - if response.choices[0].finish_reason is not None: + if response.choices and response.choices[0].finish_reason is not None: delta = MessageDelta( stop_reason=self._translate_openai_finish_reason_to_anthropic(response.choices[0].finish_reason), ) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index cf69654d15d..439c4715506 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -125,6 +125,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return None def _is_reasoning_end(self, chunk): + if not chunk.choices: + return False delta = chunk.choices[0].delta # if this indicates reasoning content, don't consider reasoning ended @@ -722,6 +724,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): # Change: Never return a value, just enqueue output item events if self.sent_output_item_added_event: return + if not chunk.choices: + return delta = chunk.choices[0].delta self._sequence_number += 1 @@ -1033,6 +1037,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): It's unclear how users expect litellm to translate multiple-choices-per-chunk to the responses API output. """ + if not choices: + return "" choice = choices[0] chat_completion_delta: ChatCompletionDelta = choice.delta return chat_completion_delta.content or "" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_combined_chunk.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_combined_chunk.py index 6973340101e..a6523f5a39d 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_combined_chunk.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_combined_chunk.py @@ -143,6 +143,60 @@ def test_delayed_usage_chunk_preserves_cache_tokens(): assert message_delta["usage"]["cache_creation_input_tokens"] == 20 +def test_trailing_empty_choices_usage_chunk_emits_message_delta_usage(): + """Regression for LIT-4767. + + The trailing usage-only chunk an OpenAI-compatible provider sends when + ``include_usage`` is set has ``choices: []``. The adapter used to index + ``choices[0]`` unguarded (``is_final_chunk`` / ``_should_start_new_content_block``) + and crash with IndexError. It must instead merge the usage into the held + stop-reason chunk so ``message_delta`` still reports it. + """ + chunks = [ + ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content="Two."), finish_reason=None)], + ), + ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")], + ), + ModelResponseStream( + choices=[], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ), + ] + wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="gpt-4o") + events = list(wrapper) + + message_delta = next(event for event in events if event.get("type") == "message_delta") + assert message_delta["usage"]["input_tokens"] == 10 + assert message_delta["usage"]["output_tokens"] == 5 + + +def test_leading_empty_choices_chunk_does_not_crash_stream(): + """Azure emits a leading ``prompt_filter_results`` chunk with ``choices: []`` + before any content. It must be tolerated and the following content emitted.""" + chunks = [ + ModelResponseStream(choices=[]), + ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content="Hi"), finish_reason=None)], + ), + ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")], + usage=Usage(prompt_tokens=3, completion_tokens=1, total_tokens=4), + ), + ] + + async def _aiter() -> "AsyncIterator[ModelResponseStream]": + for chunk in chunks: + yield chunk + + wrapper = AnthropicStreamWrapper(completion_stream=_aiter(), model="gpt-4o") + sse = _collect_async(wrapper) + + assert "Hi" in sse + assert "message_stop" in sse + + def test_splitter_passes_through_non_combined_chunks(): """A chunk with content but no finish_reason is not split.""" chunk = ModelResponseStream( diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_empty_choices_streaming_iterator.py b/tests/test_litellm/responses/litellm_completion_transformation/test_empty_choices_streaming_iterator.py new file mode 100644 index 00000000000..b7a3611501d --- /dev/null +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_empty_choices_streaming_iterator.py @@ -0,0 +1,91 @@ +""" +Regression tests for LIT-4767. + +When an upstream OpenAI-compatible provider emits a chunk with ``choices: []`` +(the trailing usage-only chunk every provider sends when ``include_usage`` is +set, or Azure's leading ``prompt_filter_results`` chunk), the Responses bridge +iterator used to index ``choices[0]`` unguarded and die with +``IndexError: list index out of range``, killing the whole stream. + +The empty-choices chunk must be tolerated without crashing, and the usage it +carries must still reach ``response.completed``. +""" + +from unittest.mock import AsyncMock + +from litellm.responses.litellm_completion_transformation.streaming_iterator import ( + LiteLLMCompletionStreamingIterator, +) +from litellm.types.llms.openai import ResponsesAPIStreamEvents +from litellm.types.utils import ( + Delta, + ModelResponseStream, + StreamingChoices, + Usage, +) + + +def _iterator() -> LiteLLMCompletionStreamingIterator: + return LiteLLMCompletionStreamingIterator( + model="gpt-4o", + litellm_custom_stream_wrapper=AsyncMock(), + request_input="hi", + responses_api_request={}, + custom_llm_provider="openai", + ) + + +def _empty_choices_usage_chunk() -> ModelResponseStream: + chunk = ModelResponseStream(id="chunk-usage", model="gpt-4o", choices=[]) + chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + return chunk + + +def test_ensure_output_item_for_empty_choices_chunk_does_not_crash(): + """First chunk with no choices must not raise (traceback frame in the ticket).""" + iterator = _iterator() + # Would raise IndexError before the fix. + assert iterator._ensure_output_item_for_chunk(_empty_choices_usage_chunk()) is None + assert iterator.sent_output_item_added_event is False + + +def test_transform_empty_choices_chunk_returns_no_delta(): + """The mid/trailing usage chunk flows through transform without crashing.""" + iterator = _iterator() + # Would raise IndexError in _get_delta_string_from_streaming_choices before the fix. + assert iterator._transform_chat_completion_chunk_to_response_api_chunk(_empty_choices_usage_chunk()) is None + + +def test_is_reasoning_end_false_for_empty_choices_chunk(): + iterator = _iterator() + assert iterator._is_reasoning_end(_empty_choices_usage_chunk()) is False + + +def test_empty_choices_usage_chunk_still_reaches_response_completed(): + """End-to-end: a text chunk followed by a choices=[] usage chunk must emit + response.completed carrying the usage rather than dying mid-stream.""" + + class _SyncWrapper: + def __init__(self, chunks): + self._it = iter(chunks) + self.logging_obj = None + self.stream_options = {"include_usage": True} + + def __next__(self): + return next(self._it) + + text_chunk = ModelResponseStream( + id="chunk-1", + model="gpt-4o", + choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="Hi"), finish_reason=None)], + ) + iterator = _iterator() + iterator.litellm_logging_obj = None + iterator.litellm_custom_stream_wrapper = _SyncWrapper([text_chunk, _empty_choices_usage_chunk()]) + + events = list(iterator) + + completed = [e for e in events if getattr(e, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED] + assert len(completed) == 1 + assert completed[0].response.usage is not None + assert completed[0].response.usage.total_tokens == 15 From 69e9edb91a9bc8efcadf876917f11cce1b31eae6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:31:38 -0700 Subject: [PATCH 2/2] test(responses): fold the empty-choices regression tests into the mapped streaming iterator test file --- .../test_empty_choices_streaming_iterator.py | 91 ------------------- .../test_streaming_iterator_transformation.py | 43 +++++++++ 2 files changed, 43 insertions(+), 91 deletions(-) delete mode 100644 tests/test_litellm/responses/litellm_completion_transformation/test_empty_choices_streaming_iterator.py diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_empty_choices_streaming_iterator.py b/tests/test_litellm/responses/litellm_completion_transformation/test_empty_choices_streaming_iterator.py deleted file mode 100644 index b7a3611501d..00000000000 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_empty_choices_streaming_iterator.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -Regression tests for LIT-4767. - -When an upstream OpenAI-compatible provider emits a chunk with ``choices: []`` -(the trailing usage-only chunk every provider sends when ``include_usage`` is -set, or Azure's leading ``prompt_filter_results`` chunk), the Responses bridge -iterator used to index ``choices[0]`` unguarded and die with -``IndexError: list index out of range``, killing the whole stream. - -The empty-choices chunk must be tolerated without crashing, and the usage it -carries must still reach ``response.completed``. -""" - -from unittest.mock import AsyncMock - -from litellm.responses.litellm_completion_transformation.streaming_iterator import ( - LiteLLMCompletionStreamingIterator, -) -from litellm.types.llms.openai import ResponsesAPIStreamEvents -from litellm.types.utils import ( - Delta, - ModelResponseStream, - StreamingChoices, - Usage, -) - - -def _iterator() -> LiteLLMCompletionStreamingIterator: - return LiteLLMCompletionStreamingIterator( - model="gpt-4o", - litellm_custom_stream_wrapper=AsyncMock(), - request_input="hi", - responses_api_request={}, - custom_llm_provider="openai", - ) - - -def _empty_choices_usage_chunk() -> ModelResponseStream: - chunk = ModelResponseStream(id="chunk-usage", model="gpt-4o", choices=[]) - chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) - return chunk - - -def test_ensure_output_item_for_empty_choices_chunk_does_not_crash(): - """First chunk with no choices must not raise (traceback frame in the ticket).""" - iterator = _iterator() - # Would raise IndexError before the fix. - assert iterator._ensure_output_item_for_chunk(_empty_choices_usage_chunk()) is None - assert iterator.sent_output_item_added_event is False - - -def test_transform_empty_choices_chunk_returns_no_delta(): - """The mid/trailing usage chunk flows through transform without crashing.""" - iterator = _iterator() - # Would raise IndexError in _get_delta_string_from_streaming_choices before the fix. - assert iterator._transform_chat_completion_chunk_to_response_api_chunk(_empty_choices_usage_chunk()) is None - - -def test_is_reasoning_end_false_for_empty_choices_chunk(): - iterator = _iterator() - assert iterator._is_reasoning_end(_empty_choices_usage_chunk()) is False - - -def test_empty_choices_usage_chunk_still_reaches_response_completed(): - """End-to-end: a text chunk followed by a choices=[] usage chunk must emit - response.completed carrying the usage rather than dying mid-stream.""" - - class _SyncWrapper: - def __init__(self, chunks): - self._it = iter(chunks) - self.logging_obj = None - self.stream_options = {"include_usage": True} - - def __next__(self): - return next(self._it) - - text_chunk = ModelResponseStream( - id="chunk-1", - model="gpt-4o", - choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="Hi"), finish_reason=None)], - ) - iterator = _iterator() - iterator.litellm_logging_obj = None - iterator.litellm_custom_stream_wrapper = _SyncWrapper([text_chunk, _empty_choices_usage_chunk()]) - - events = list(iterator) - - completed = [e for e in events if getattr(e, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED] - assert len(completed) == 1 - assert completed[0].response.usage is not None - assert completed[0].response.usage.total_tokens == 15 diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 5d97b0531d6..343fc873fa4 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -752,6 +752,49 @@ def test_completed_event_restores_usage_hidden_by_stream_options_none(): assert completed.response.usage.output_tokens == 5 +def _empty_choices_chunk(usage: Usage | None = None) -> ModelResponseStream: + return ModelResponseStream(id=CHAT_COMPLETION_ID, model="claude-haiku-4-5", choices=[], usage=usage) + + +@pytest.mark.asyncio +async def test_leading_empty_choices_chunk_does_not_kill_the_stream(): + """ + Azure leads some streams with a `prompt_filter_results` chunk whose `choices` is empty. + The bridge used to index `choices[0]` on it and die before the first token. + """ + iterator = _build_iterator([_empty_choices_chunk(), _chunk("Hello"), _chunk("!", finish_reason="stop")]) + + events = [event async for event in iterator] + + event_types = [getattr(event, "type", None) for event in events] + assert event_types.count(ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED) == 1 + assert "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!" + assert event_types[-1] == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + + +@pytest.mark.asyncio +async def test_trailing_empty_choices_usage_chunk_reaches_response_completed(): + """ + With `stream_options.include_usage` (which the bridge always sets) the last upstream chunk + carries only usage and an empty `choices`. It must not crash the stream, and its usage must + still land on `response.completed`. + """ + usage: Final = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) + iterator = _build_iterator([_chunk("Hello"), _chunk("", finish_reason="stop"), _empty_choices_chunk(usage)]) + + events = [event async for event in iterator] + + completed = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert completed.response.usage.input_tokens == 10 + assert completed.response.usage.output_tokens == 5 + + +def test_is_reasoning_end_ignores_empty_choices_chunk(): + assert _build_iterator([])._is_reasoning_end(_empty_choices_chunk()) is False + + def test_object_tool_call_arguments_stream_as_valid_json(): """A provider that sends decoded object arguments must still stream valid JSON.