fix(streaming): preserve Gemini thought-only chunks in is_chunk_non_empty

`is_chunk_non_empty` inspects the fresh `model_response` created at the
top of `chunk_creator`, which never carries reasoning_content. Gemini
parses thinking parts into `response_obj["original_chunk"]` via the
openai-compatible code path, so a chunk where delta.content is None but
delta.reasoning_content is set was being treated as empty and dropped
before reaching the caller. Once the chunk is recognised as non-empty,
`return_processed_chunk_logic` already rebuilds choices from
`original_chunk.model_dump()`, so no further changes are needed.

Adds a regression test that fails without the fix and a negative test
covering an empty original_chunk to lock in the existing behaviour.

Fixes #28000.
This commit is contained in:
Abhisek Das 2026-05-16 01:09:51 +05:30
parent 50df072d95
commit c120e641fe
2 changed files with 99 additions and 9 deletions

View file

@ -95,6 +95,27 @@ def print_verbose(print_statement):
pass
def _original_chunk_has_reasoning_content(response_obj: Dict[str, Any]) -> bool:
"""Return True when the parsed upstream chunk carries reasoning_content.
`is_chunk_non_empty` only inspects the fresh `model_response` created at
the top of `chunk_creator`, so providers that surface reasoning_content via
`response_obj["original_chunk"]` (e.g. Gemini thinking parts under the
openai-compatible code path) need this fallback to avoid silent drops.
See https://github.com/BerriAI/litellm/issues/28000.
"""
original_chunk = response_obj.get("original_chunk")
if original_chunk is None:
return False
choices = getattr(original_chunk, "choices", None) or []
if len(choices) == 0:
return False
delta = getattr(choices[0], "delta", None)
if delta is None:
return False
return getattr(delta, "reasoning_content", None) is not None
class CustomStreamWrapper:
def __init__(
self,
@ -818,6 +839,7 @@ class CustomStreamWrapper:
"reasoning_content" in model_response.choices[0].delta
and model_response.choices[0].delta.reasoning_content is not None
)
or _original_chunk_has_reasoning_content(response_obj)
or (model_response.choices[0].delta.provider_specific_fields is not None)
or (
"provider_specific_fields" in model_response

View file

@ -158,6 +158,78 @@ def test_is_chunk_non_empty(initialized_custom_stream_wrapper: CustomStreamWrapp
)
def test_is_chunk_non_empty_with_original_chunk_reasoning_content(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""
Regression test for https://github.com/BerriAI/litellm/issues/28000.
When Gemini emits a streaming chunk that carries only thinking content
(delta.content is None, delta.reasoning_content is set), the upstream
handler builds an `original_chunk` ModelResponseStream that holds the
reasoning_content, but `model_response` passed into is_chunk_non_empty
is a fresh empty object. The thought chunk must still be considered
non-empty so it propagates to the caller.
"""
original_chunk = ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(
content=None,
reasoning_content="I need to add 2 and 2...",
),
finish_reason=None,
)
],
)
empty_model_response = ModelResponseStream(
choices=[StreamingChoices(index=0, delta=Delta(), finish_reason=None)],
)
assert (
initialized_custom_stream_wrapper.is_chunk_non_empty(
completion_obj={"content": ""},
model_response=empty_model_response,
response_obj={"original_chunk": original_chunk},
)
is True
)
def test_is_chunk_non_empty_ignores_original_chunk_without_reasoning(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""
Companion to test_is_chunk_non_empty_with_original_chunk_reasoning_content.
An empty completion_obj plus an original_chunk that carries no
reasoning_content (and no other non-empty delta fields) must still be
treated as empty so existing pass-through behaviour is preserved.
"""
original_chunk = ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(content=None),
finish_reason=None,
)
],
)
empty_model_response = ModelResponseStream(
choices=[StreamingChoices(index=0, delta=Delta(), finish_reason=None)],
)
assert (
initialized_custom_stream_wrapper.is_chunk_non_empty(
completion_obj={"content": ""},
model_response=empty_model_response,
response_obj={"original_chunk": original_chunk},
)
is False
)
def test_is_chunk_non_empty_with_annotations(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
@ -2036,23 +2108,19 @@ async def test_azure_streaming_role_preserved_with_include_usage(sync_mode: bool
chunks.append(chunk)
# The prompt_filter chunk should be forwarded with choices=[]
assert len(chunks[0].choices) == 0, (
f"Expected prompt_filter chunk with choices=[], got {len(chunks[0].choices)} choices"
)
assert (
len(chunks[0].choices) == 0
), f"Expected prompt_filter chunk with choices=[], got {len(chunks[0].choices)} choices"
# At least one chunk must have role='assistant' in its delta
has_role = any(
len(c.choices) > 0
and getattr(c.choices[0].delta, "role", None) == "assistant"
len(c.choices) > 0 and getattr(c.choices[0].delta, "role", None) == "assistant"
for c in chunks
)
assert has_role, (
"No chunk contained role='assistant' in delta (issue #24221). "
"Chunk deltas: "
+ str([
c.choices[0].delta if c.choices else "no choices"
for c in chunks
])
+ str([c.choices[0].delta if c.choices else "no choices" for c in chunks])
)