fix(gemini): don't emit empty choices on metadata-only stream chunks

web_search + reasoning makes Gemini stream mid-chunks that carry only
grounding/thought metadata — no content part, no finishReason.
_process_candidates skips content-less candidates and the existing
fallback only ran when finishReason was set, so choices stayed empty
and the downstream streaming handler raised IndexError on choices[0].
Emit an empty-delta choice for content-less chunks regardless of
finishReason.

Fixes #28884
This commit is contained in:
Chenglun Hu 2026-05-28 22:31:41 +08:00
parent ed073d382d
commit 447deebc5b
2 changed files with 51 additions and 11 deletions

View file

@ -3351,14 +3351,18 @@ class ModelResponseIterator:
self.has_seen_tool_calls = True
break
# Handle final chunk with finishReason but no content.
# _process_candidates skips candidates without "content",
# so the finish_reason from the final chunk is lost.
# _process_candidates skips candidates without a "content" part, so a
# content-less chunk leaves choices empty and the downstream streaming
# handler hits IndexError on choices[0]. This covers the final chunk
# (finishReason, no content) and mid-stream metadata-only chunks
# (grounding/web-search/thought, no content and no finishReason — seen
# with web_search + reasoning) by emitting an empty-delta choice.
if not model_response.choices and _candidates:
from litellm.types.utils import Delta, StreamingChoices
for candidate in _candidates:
finish_reason_str = candidate.get("finishReason")
mapped_finish_reason = None
if finish_reason_str is not None:
if self.has_seen_tool_calls:
mapped_finish_reason = "tool_calls"
@ -3366,14 +3370,14 @@ class ModelResponseIterator:
mapped_finish_reason = VertexGeminiConfig._check_finish_reason(
None, finish_reason_str
)
choice = StreamingChoices(
finish_reason=mapped_finish_reason,
index=candidate.get("index", 0),
delta=Delta(content=None, role=None),
logprobs=None,
enhancements=None,
)
model_response.choices.append(choice)
choice = StreamingChoices(
finish_reason=mapped_finish_reason,
index=candidate.get("index", 0),
delta=Delta(content=None, role=None),
logprobs=None,
enhancements=None,
)
model_response.choices.append(choice)
# Also handle the case where the final chunk has empty
# content (e.g. text:"") WITH finishReason. In this case

View file

@ -302,3 +302,39 @@ def test_streaming_tool_call_finish_reason_with_empty_content_in_final_chunk():
assert len(response2.choices) == 1
# Must be "tool_calls", NOT "stop"
assert response2.choices[0].finish_reason == "tool_calls"
def test_streaming_metadata_only_chunk_does_not_yield_empty_choices():
"""
web_search + reasoning makes Gemini emit mid-stream chunks that carry only
grounding/thought metadata — no content part and no finishReason.
_process_candidates skips content-less candidates, so without a fallback
`choices` is empty and the downstream streaming handler hits
`IndexError: list index out of range` on choices[0].
Ref: https://github.com/BerriAI/litellm/issues/28884
"""
logging_obj = _make_logging_obj()
iterator = ModelResponseIterator(
streaming_response=iter([]),
sync_stream=True,
logging_obj=logging_obj,
)
# Grounding-only chunk: a candidate with groundingMetadata but no content
# part and no finishReason (what web_search + reasoning produces mid-stream).
metadata_only_chunk = {
"candidates": [
{
"index": 0,
"groundingMetadata": {"webSearchQueries": ["weather boston"]},
}
]
}
response = iterator.chunk_parser(metadata_only_chunk)
assert response is not None
# Must expose at least one choice so downstream choices[0] is safe.
assert len(response.choices) == 1
assert response.choices[0].finish_reason is None
assert response.choices[0].delta.content is None