mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-22 00:31:44 +00:00
fix(streaming): preserve Gemini thought-only chunks in is_chunk_non_empty
`is_chunk_non_empty` inspects the fresh `model_response` created at the top of `chunk_creator`, which never carries reasoning_content. Gemini parses thinking parts into `response_obj["original_chunk"]` via the openai-compatible code path, so a chunk where delta.content is None but delta.reasoning_content is set was being treated as empty and dropped before reaching the caller. Once the chunk is recognised as non-empty, `return_processed_chunk_logic` already rebuilds choices from `original_chunk.model_dump()`, so no further changes are needed. Adds a regression test that fails without the fix and a negative test covering an empty original_chunk to lock in the existing behaviour. Fixes #28000.
This commit is contained in:
parent
50df072d95
commit
c120e641fe
2 changed files with 99 additions and 9 deletions
|
|
@ -95,6 +95,27 @@ def print_verbose(print_statement):
|
|||
pass
|
||||
|
||||
|
||||
def _original_chunk_has_reasoning_content(response_obj: Dict[str, Any]) -> bool:
|
||||
"""Return True when the parsed upstream chunk carries reasoning_content.
|
||||
|
||||
`is_chunk_non_empty` only inspects the fresh `model_response` created at
|
||||
the top of `chunk_creator`, so providers that surface reasoning_content via
|
||||
`response_obj["original_chunk"]` (e.g. Gemini thinking parts under the
|
||||
openai-compatible code path) need this fallback to avoid silent drops.
|
||||
See https://github.com/BerriAI/litellm/issues/28000.
|
||||
"""
|
||||
original_chunk = response_obj.get("original_chunk")
|
||||
if original_chunk is None:
|
||||
return False
|
||||
choices = getattr(original_chunk, "choices", None) or []
|
||||
if len(choices) == 0:
|
||||
return False
|
||||
delta = getattr(choices[0], "delta", None)
|
||||
if delta is None:
|
||||
return False
|
||||
return getattr(delta, "reasoning_content", None) is not None
|
||||
|
||||
|
||||
class CustomStreamWrapper:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -818,6 +839,7 @@ class CustomStreamWrapper:
|
|||
"reasoning_content" in model_response.choices[0].delta
|
||||
and model_response.choices[0].delta.reasoning_content is not None
|
||||
)
|
||||
or _original_chunk_has_reasoning_content(response_obj)
|
||||
or (model_response.choices[0].delta.provider_specific_fields is not None)
|
||||
or (
|
||||
"provider_specific_fields" in model_response
|
||||
|
|
|
|||
|
|
@ -158,6 +158,78 @@ def test_is_chunk_non_empty(initialized_custom_stream_wrapper: CustomStreamWrapp
|
|||
)
|
||||
|
||||
|
||||
def test_is_chunk_non_empty_with_original_chunk_reasoning_content(
|
||||
initialized_custom_stream_wrapper: CustomStreamWrapper,
|
||||
):
|
||||
"""
|
||||
Regression test for https://github.com/BerriAI/litellm/issues/28000.
|
||||
|
||||
When Gemini emits a streaming chunk that carries only thinking content
|
||||
(delta.content is None, delta.reasoning_content is set), the upstream
|
||||
handler builds an `original_chunk` ModelResponseStream that holds the
|
||||
reasoning_content, but `model_response` passed into is_chunk_non_empty
|
||||
is a fresh empty object. The thought chunk must still be considered
|
||||
non-empty so it propagates to the caller.
|
||||
"""
|
||||
original_chunk = ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(
|
||||
content=None,
|
||||
reasoning_content="I need to add 2 and 2...",
|
||||
),
|
||||
finish_reason=None,
|
||||
)
|
||||
],
|
||||
)
|
||||
empty_model_response = ModelResponseStream(
|
||||
choices=[StreamingChoices(index=0, delta=Delta(), finish_reason=None)],
|
||||
)
|
||||
|
||||
assert (
|
||||
initialized_custom_stream_wrapper.is_chunk_non_empty(
|
||||
completion_obj={"content": ""},
|
||||
model_response=empty_model_response,
|
||||
response_obj={"original_chunk": original_chunk},
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def test_is_chunk_non_empty_ignores_original_chunk_without_reasoning(
|
||||
initialized_custom_stream_wrapper: CustomStreamWrapper,
|
||||
):
|
||||
"""
|
||||
Companion to test_is_chunk_non_empty_with_original_chunk_reasoning_content.
|
||||
|
||||
An empty completion_obj plus an original_chunk that carries no
|
||||
reasoning_content (and no other non-empty delta fields) must still be
|
||||
treated as empty so existing pass-through behaviour is preserved.
|
||||
"""
|
||||
original_chunk = ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(content=None),
|
||||
finish_reason=None,
|
||||
)
|
||||
],
|
||||
)
|
||||
empty_model_response = ModelResponseStream(
|
||||
choices=[StreamingChoices(index=0, delta=Delta(), finish_reason=None)],
|
||||
)
|
||||
|
||||
assert (
|
||||
initialized_custom_stream_wrapper.is_chunk_non_empty(
|
||||
completion_obj={"content": ""},
|
||||
model_response=empty_model_response,
|
||||
response_obj={"original_chunk": original_chunk},
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def test_is_chunk_non_empty_with_annotations(
|
||||
initialized_custom_stream_wrapper: CustomStreamWrapper,
|
||||
):
|
||||
|
|
@ -2036,23 +2108,19 @@ async def test_azure_streaming_role_preserved_with_include_usage(sync_mode: bool
|
|||
chunks.append(chunk)
|
||||
|
||||
# The prompt_filter chunk should be forwarded with choices=[]
|
||||
assert len(chunks[0].choices) == 0, (
|
||||
f"Expected prompt_filter chunk with choices=[], got {len(chunks[0].choices)} choices"
|
||||
)
|
||||
assert (
|
||||
len(chunks[0].choices) == 0
|
||||
), f"Expected prompt_filter chunk with choices=[], got {len(chunks[0].choices)} choices"
|
||||
|
||||
# At least one chunk must have role='assistant' in its delta
|
||||
has_role = any(
|
||||
len(c.choices) > 0
|
||||
and getattr(c.choices[0].delta, "role", None) == "assistant"
|
||||
len(c.choices) > 0 and getattr(c.choices[0].delta, "role", None) == "assistant"
|
||||
for c in chunks
|
||||
)
|
||||
assert has_role, (
|
||||
"No chunk contained role='assistant' in delta (issue #24221). "
|
||||
"Chunk deltas: "
|
||||
+ str([
|
||||
c.choices[0].delta if c.choices else "no choices"
|
||||
for c in chunks
|
||||
])
|
||||
+ str([c.choices[0].delta if c.choices else "no choices" for c in chunks])
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue