From c120e641feea48b98846e00bf06d3b48fb26c250 Mon Sep 17 00:00:00 2001 From: Abhisek Das Date: Sat, 16 May 2026 01:09:51 +0530 Subject: [PATCH] fix(streaming): preserve Gemini thought-only chunks in is_chunk_non_empty `is_chunk_non_empty` inspects the fresh `model_response` created at the top of `chunk_creator`, which never carries reasoning_content. Gemini parses thinking parts into `response_obj["original_chunk"]` via the openai-compatible code path, so a chunk where delta.content is None but delta.reasoning_content is set was being treated as empty and dropped before reaching the caller. Once the chunk is recognised as non-empty, `return_processed_chunk_logic` already rebuilds choices from `original_chunk.model_dump()`, so no further changes are needed. Adds a regression test that fails without the fix and a negative test covering an empty original_chunk to lock in the existing behaviour. Fixes #28000. --- .../litellm_core_utils/streaming_handler.py | 22 +++++ .../test_streaming_handler.py | 86 +++++++++++++++++-- 2 files changed, 99 insertions(+), 9 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index fa7faf3035d..bb8300c8d87 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -95,6 +95,27 @@ def print_verbose(print_statement): pass +def _original_chunk_has_reasoning_content(response_obj: Dict[str, Any]) -> bool: + """Return True when the parsed upstream chunk carries reasoning_content. + + `is_chunk_non_empty` only inspects the fresh `model_response` created at + the top of `chunk_creator`, so providers that surface reasoning_content via + `response_obj["original_chunk"]` (e.g. Gemini thinking parts under the + openai-compatible code path) need this fallback to avoid silent drops. + See https://github.com/BerriAI/litellm/issues/28000. + """ + original_chunk = response_obj.get("original_chunk") + if original_chunk is None: + return False + choices = getattr(original_chunk, "choices", None) or [] + if len(choices) == 0: + return False + delta = getattr(choices[0], "delta", None) + if delta is None: + return False + return getattr(delta, "reasoning_content", None) is not None + + class CustomStreamWrapper: def __init__( self, @@ -818,6 +839,7 @@ class CustomStreamWrapper: "reasoning_content" in model_response.choices[0].delta and model_response.choices[0].delta.reasoning_content is not None ) + or _original_chunk_has_reasoning_content(response_obj) or (model_response.choices[0].delta.provider_specific_fields is not None) or ( "provider_specific_fields" in model_response diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 49d3c51e340..c88d30bdf0e 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -158,6 +158,78 @@ def test_is_chunk_non_empty(initialized_custom_stream_wrapper: CustomStreamWrapp ) +def test_is_chunk_non_empty_with_original_chunk_reasoning_content( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """ + Regression test for https://github.com/BerriAI/litellm/issues/28000. + + When Gemini emits a streaming chunk that carries only thinking content + (delta.content is None, delta.reasoning_content is set), the upstream + handler builds an `original_chunk` ModelResponseStream that holds the + reasoning_content, but `model_response` passed into is_chunk_non_empty + is a fresh empty object. The thought chunk must still be considered + non-empty so it propagates to the caller. + """ + original_chunk = ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + content=None, + reasoning_content="I need to add 2 and 2...", + ), + finish_reason=None, + ) + ], + ) + empty_model_response = ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(), finish_reason=None)], + ) + + assert ( + initialized_custom_stream_wrapper.is_chunk_non_empty( + completion_obj={"content": ""}, + model_response=empty_model_response, + response_obj={"original_chunk": original_chunk}, + ) + is True + ) + + +def test_is_chunk_non_empty_ignores_original_chunk_without_reasoning( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """ + Companion to test_is_chunk_non_empty_with_original_chunk_reasoning_content. + + An empty completion_obj plus an original_chunk that carries no + reasoning_content (and no other non-empty delta fields) must still be + treated as empty so existing pass-through behaviour is preserved. + """ + original_chunk = ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta(content=None), + finish_reason=None, + ) + ], + ) + empty_model_response = ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(), finish_reason=None)], + ) + + assert ( + initialized_custom_stream_wrapper.is_chunk_non_empty( + completion_obj={"content": ""}, + model_response=empty_model_response, + response_obj={"original_chunk": original_chunk}, + ) + is False + ) + + def test_is_chunk_non_empty_with_annotations( initialized_custom_stream_wrapper: CustomStreamWrapper, ): @@ -2036,23 +2108,19 @@ async def test_azure_streaming_role_preserved_with_include_usage(sync_mode: bool chunks.append(chunk) # The prompt_filter chunk should be forwarded with choices=[] - assert len(chunks[0].choices) == 0, ( - f"Expected prompt_filter chunk with choices=[], got {len(chunks[0].choices)} choices" - ) + assert ( + len(chunks[0].choices) == 0 + ), f"Expected prompt_filter chunk with choices=[], got {len(chunks[0].choices)} choices" # At least one chunk must have role='assistant' in its delta has_role = any( - len(c.choices) > 0 - and getattr(c.choices[0].delta, "role", None) == "assistant" + len(c.choices) > 0 and getattr(c.choices[0].delta, "role", None) == "assistant" for c in chunks ) assert has_role, ( "No chunk contained role='assistant' in delta (issue #24221). " "Chunk deltas: " - + str([ - c.choices[0].delta if c.choices else "no choices" - for c in chunks - ]) + + str([c.choices[0].delta if c.choices else "no choices" for c in chunks]) )