fix(vertex): streaming finish_reason="stop" instead of "tool_calls" for gemini-3.1-flash-lite-preview

Models like gemini-3.1-flash-lite-preview send the final streaming chunk
with empty content (text:"") alongside finishReason:"STOP", instead of
omitting content entirely. The existing fix (PR #21577) only handled
chunks without content, so this case was missed.

Now, after processing candidates, if tool_calls were seen in earlier
chunks and a choice has finish_reason="stop", it is overridden to
"tool_calls" to match the OpenAI spec.

Fixes #22900
This commit is contained in:
Chesars 2026-03-17 17:47:01 -03:00
parent 5acceaed32
commit 0c28b47057
2 changed files with 82 additions and 0 deletions

View file

@ -3001,6 +3001,16 @@ class ModelResponseIterator:
)
model_response.choices.append(choice)
# Also handle the case where the final chunk has empty
# content (e.g. text:"") WITH finishReason. In this case
# _process_candidates DOES create a choice, but maps
# finishReason="STOP" to "stop" because the current chunk
# has no tool_calls. Override if we saw tool_calls earlier.
if self.has_seen_tool_calls:
for choice in model_response.choices:
if choice.finish_reason == "stop":
choice.finish_reason = "tool_calls"
setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata) # type: ignore
setattr(model_response, "vertex_ai_url_context_metadata", url_context_metadata) # type: ignore
setattr(model_response, "vertex_ai_safety_ratings", safety_ratings) # type: ignore

View file

@ -230,3 +230,75 @@ def test_streaming_content_filter_finish_reason_preserved():
assert response is not None
assert len(response.choices) == 1
assert response.choices[0].finish_reason == "content_filter"
def test_streaming_tool_call_finish_reason_with_empty_content_in_final_chunk():
"""
When Gemini streams tool calls and the final chunk has BOTH empty content
(e.g. parts: [{text: ""}]) AND finishReason="STOP", the finish_reason
must still be "tool_calls".
This covers models like gemini-3.1-flash-lite-preview that send the
final chunk with content (empty text) instead of omitting it entirely.
Ref: https://github.com/BerriAI/litellm/issues/22900
"""
logging_obj = _make_logging_obj()
iterator = ModelResponseIterator(
streaming_response=iter([]),
sync_stream=True,
logging_obj=logging_obj,
)
# Chunk 1: tool call with no finishReason
chunk_with_tool_calls = {
"candidates": [
{
"content": {
"parts": [
{
"functionCall": {
"name": "get_weather",
"args": {"location": "San Francisco"},
}
}
],
"role": "model",
},
"index": 0,
}
],
}
# Chunk 2: finishReason="STOP" WITH empty content (text: "")
chunk_with_empty_content_and_finish = {
"candidates": [
{
"content": {
"parts": [{"text": ""}],
"role": "model",
},
"finishReason": "STOP",
"index": 0,
}
],
"usageMetadata": {
"promptTokenCount": 50,
"candidatesTokenCount": 20,
"totalTokenCount": 70,
},
}
# Process chunk 1
response1 = iterator.chunk_parser(chunk_with_tool_calls)
assert response1 is not None
assert len(response1.choices) == 1
assert response1.choices[0].delta.tool_calls is not None
assert iterator.has_seen_tool_calls is True
# Process chunk 2 (final chunk with empty content)
response2 = iterator.chunk_parser(chunk_with_empty_content_and_finish)
assert response2 is not None
assert len(response2.choices) == 1
# Must be "tool_calls", NOT "stop"
assert response2.choices[0].finish_reason == "tool_calls"