mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-22 00:31:44 +00:00
fix: detect Gemini thinking chunks in is_chunk_non_empty via original_chunk
When Gemini streams thinking/reasoning chunks, the parsed reasoning_content lives in response_obj["original_chunk"] — not in the fresh model_response created by model_response_creator(). The existing reasoning_content check (line 817-820) only inspects the empty model_response, so thinking-only chunks are silently dropped. Add a fallback check that inspects response_obj["original_chunk"].choices[0].delta.reasoning_content. Once is_chunk_non_empty returns True, return_processed_chunk_logic already rebuilds choices from original_chunk.model_dump(), which preserves reasoning_content — no further changes needed. Closes #28000 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
e58a561caa
commit
1dc524d79c
2 changed files with 43 additions and 9 deletions
|
|
@ -818,6 +818,16 @@ class CustomStreamWrapper:
|
|||
"reasoning_content" in model_response.choices[0].delta
|
||||
and model_response.choices[0].delta.reasoning_content is not None
|
||||
)
|
||||
or (
|
||||
response_obj.get("original_chunk") is not None
|
||||
and len(getattr(response_obj["original_chunk"], "choices", [])) > 0
|
||||
and getattr(
|
||||
response_obj["original_chunk"].choices[0].delta,
|
||||
"reasoning_content",
|
||||
None,
|
||||
)
|
||||
is not None
|
||||
)
|
||||
or (model_response.choices[0].delta.provider_specific_fields is not None)
|
||||
or (
|
||||
"provider_specific_fields" in model_response
|
||||
|
|
|
|||
|
|
@ -192,6 +192,34 @@ def test_is_chunk_non_empty_with_annotations(
|
|||
)
|
||||
|
||||
|
||||
def test_is_chunk_non_empty_gemini_thinking_in_original_chunk(
|
||||
initialized_custom_stream_wrapper: CustomStreamWrapper,
|
||||
):
|
||||
"""Gemini thinking chunks: model_response is a fresh empty object but
|
||||
response_obj['original_chunk'] carries reasoning_content. The chunk
|
||||
must be considered non-empty so it reaches the caller."""
|
||||
original_chunk = ModelResponseStream(
|
||||
id="gemini-chunk-1",
|
||||
choices=[
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": None, "reasoning_content": "I need to add 2+2"},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
)
|
||||
# model_response is a fresh empty object (as created by model_response_creator)
|
||||
empty_model_response = ModelResponseStream(
|
||||
id="gemini-chunk-1",
|
||||
choices=[{"index": 0, "delta": {"content": None}, "finish_reason": None}],
|
||||
)
|
||||
assert initialized_custom_stream_wrapper.is_chunk_non_empty(
|
||||
completion_obj={"content": ""},
|
||||
model_response=empty_model_response,
|
||||
response_obj={"original_chunk": original_chunk},
|
||||
)
|
||||
|
||||
|
||||
def test_optional_combine_thinking_block_in_choices(
|
||||
initialized_custom_stream_wrapper: CustomStreamWrapper,
|
||||
):
|
||||
|
|
@ -2036,23 +2064,19 @@ async def test_azure_streaming_role_preserved_with_include_usage(sync_mode: bool
|
|||
chunks.append(chunk)
|
||||
|
||||
# The prompt_filter chunk should be forwarded with choices=[]
|
||||
assert len(chunks[0].choices) == 0, (
|
||||
f"Expected prompt_filter chunk with choices=[], got {len(chunks[0].choices)} choices"
|
||||
)
|
||||
assert (
|
||||
len(chunks[0].choices) == 0
|
||||
), f"Expected prompt_filter chunk with choices=[], got {len(chunks[0].choices)} choices"
|
||||
|
||||
# At least one chunk must have role='assistant' in its delta
|
||||
has_role = any(
|
||||
len(c.choices) > 0
|
||||
and getattr(c.choices[0].delta, "role", None) == "assistant"
|
||||
len(c.choices) > 0 and getattr(c.choices[0].delta, "role", None) == "assistant"
|
||||
for c in chunks
|
||||
)
|
||||
assert has_role, (
|
||||
"No chunk contained role='assistant' in delta (issue #24221). "
|
||||
"Chunk deltas: "
|
||||
+ str([
|
||||
c.choices[0].delta if c.choices else "no choices"
|
||||
for c in chunks
|
||||
])
|
||||
+ str([c.choices[0].delta if c.choices else "no choices" for c in chunks])
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue