mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-02 02:11:58 +00:00
fix(gemini): don't emit empty choices on metadata-only stream chunks
web_search + reasoning makes Gemini stream mid-chunks that carry only grounding/thought metadata — no content part, no finishReason. _process_candidates skips content-less candidates and the existing fallback only ran when finishReason was set, so choices stayed empty and the downstream streaming handler raised IndexError on choices[0]. Emit an empty-delta choice for content-less chunks regardless of finishReason. Fixes #28884
This commit is contained in:
parent
ed073d382d
commit
447deebc5b
2 changed files with 51 additions and 11 deletions
|
|
@ -3351,14 +3351,18 @@ class ModelResponseIterator:
|
|||
self.has_seen_tool_calls = True
|
||||
break
|
||||
|
||||
# Handle final chunk with finishReason but no content.
|
||||
# _process_candidates skips candidates without "content",
|
||||
# so the finish_reason from the final chunk is lost.
|
||||
# _process_candidates skips candidates without a "content" part, so a
|
||||
# content-less chunk leaves choices empty and the downstream streaming
|
||||
# handler hits IndexError on choices[0]. This covers the final chunk
|
||||
# (finishReason, no content) and mid-stream metadata-only chunks
|
||||
# (grounding/web-search/thought, no content and no finishReason — seen
|
||||
# with web_search + reasoning) by emitting an empty-delta choice.
|
||||
if not model_response.choices and _candidates:
|
||||
from litellm.types.utils import Delta, StreamingChoices
|
||||
|
||||
for candidate in _candidates:
|
||||
finish_reason_str = candidate.get("finishReason")
|
||||
mapped_finish_reason = None
|
||||
if finish_reason_str is not None:
|
||||
if self.has_seen_tool_calls:
|
||||
mapped_finish_reason = "tool_calls"
|
||||
|
|
@ -3366,14 +3370,14 @@ class ModelResponseIterator:
|
|||
mapped_finish_reason = VertexGeminiConfig._check_finish_reason(
|
||||
None, finish_reason_str
|
||||
)
|
||||
choice = StreamingChoices(
|
||||
finish_reason=mapped_finish_reason,
|
||||
index=candidate.get("index", 0),
|
||||
delta=Delta(content=None, role=None),
|
||||
logprobs=None,
|
||||
enhancements=None,
|
||||
)
|
||||
model_response.choices.append(choice)
|
||||
choice = StreamingChoices(
|
||||
finish_reason=mapped_finish_reason,
|
||||
index=candidate.get("index", 0),
|
||||
delta=Delta(content=None, role=None),
|
||||
logprobs=None,
|
||||
enhancements=None,
|
||||
)
|
||||
model_response.choices.append(choice)
|
||||
|
||||
# Also handle the case where the final chunk has empty
|
||||
# content (e.g. text:"") WITH finishReason. In this case
|
||||
|
|
|
|||
|
|
@ -302,3 +302,39 @@ def test_streaming_tool_call_finish_reason_with_empty_content_in_final_chunk():
|
|||
assert len(response2.choices) == 1
|
||||
# Must be "tool_calls", NOT "stop"
|
||||
assert response2.choices[0].finish_reason == "tool_calls"
|
||||
|
||||
|
||||
def test_streaming_metadata_only_chunk_does_not_yield_empty_choices():
|
||||
"""
|
||||
web_search + reasoning makes Gemini emit mid-stream chunks that carry only
|
||||
grounding/thought metadata — no content part and no finishReason.
|
||||
_process_candidates skips content-less candidates, so without a fallback
|
||||
`choices` is empty and the downstream streaming handler hits
|
||||
`IndexError: list index out of range` on choices[0].
|
||||
|
||||
Ref: https://github.com/BerriAI/litellm/issues/28884
|
||||
"""
|
||||
logging_obj = _make_logging_obj()
|
||||
iterator = ModelResponseIterator(
|
||||
streaming_response=iter([]),
|
||||
sync_stream=True,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
# Grounding-only chunk: a candidate with groundingMetadata but no content
|
||||
# part and no finishReason (what web_search + reasoning produces mid-stream).
|
||||
metadata_only_chunk = {
|
||||
"candidates": [
|
||||
{
|
||||
"index": 0,
|
||||
"groundingMetadata": {"webSearchQueries": ["weather boston"]},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
response = iterator.chunk_parser(metadata_only_chunk)
|
||||
assert response is not None
|
||||
# Must expose at least one choice so downstream choices[0] is safe.
|
||||
assert len(response.choices) == 1
|
||||
assert response.choices[0].finish_reason is None
|
||||
assert response.choices[0].delta.content is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue