mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-21 00:21:49 +00:00
fix(gemini): map every documented finishReason and reset per-candidate state
A content-less candidate is now kept as a choice whenever it carries a finishReason, with the raw value on the choice's provider_specific_fields. NO_IMAGE, IMAGE_RECITATION, IMAGE_OTHER and ESCALATION map to content_filter; UNEXPECTED_TOOL_CALL and MISSING_THOUGHT_SIGNATURE map to stop. The /v1/responses bridge reports content_filter and refusal as incomplete with incomplete_details, and tool calls and reasoning no longer leak from one candidate into the next.
This commit is contained in:
parent
9d39d7efaa
commit
cf05466a27
6 changed files with 117 additions and 26 deletions
|
|
@ -225,6 +225,11 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
|
|||
"TOO_MANY_TOOL_CALLS": "stop",
|
||||
"MALFORMED_RESPONSE": "stop",
|
||||
"NO_IMAGE": "content_filter",
|
||||
"IMAGE_RECITATION": "content_filter",
|
||||
"IMAGE_OTHER": "content_filter",
|
||||
"ESCALATION": "content_filter",
|
||||
"UNEXPECTED_TOOL_CALL": "stop",
|
||||
"MISSING_THOUGHT_SIGNATURE": "stop",
|
||||
# Zhipu GLM
|
||||
"network_error": "stop",
|
||||
"sensitive": "content_filter",
|
||||
|
|
|
|||
|
|
@ -1348,6 +1348,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
"TOO_MANY_TOOL_CALLS",
|
||||
"MALFORMED_RESPONSE",
|
||||
"NO_IMAGE",
|
||||
"IMAGE_RECITATION",
|
||||
"IMAGE_OTHER",
|
||||
"ESCALATION",
|
||||
"UNEXPECTED_TOOL_CALL",
|
||||
"MISSING_THOUGHT_SIGNATURE",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -2243,7 +2248,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
image_response: list[ImageURLListItem] | None = None
|
||||
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
|
||||
chat_completion_logprobs: ChoiceLogprobs | None = None
|
||||
tools: list[ChatCompletionToolCallChunk] | None = []
|
||||
tools: list[ChatCompletionToolCallChunk] | None = None
|
||||
functions: ChatCompletionToolCallFunctionChunk | None = None
|
||||
thinking_blocks: list[ChatCompletionThinkingBlock] | None = None
|
||||
reasoning_content: str | None = None
|
||||
|
|
@ -2358,11 +2363,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
tool_invocation_fields["server_side_tool_invocations"] = server_side_tool_invocations
|
||||
chat_completion_message["provider_specific_fields"] = tool_invocation_fields
|
||||
|
||||
if candidate.get("finishReason"):
|
||||
finish_reason_fields = chat_completion_message.get("provider_specific_fields") or {}
|
||||
finish_reason_fields["native_finish_reason"] = candidate.get("finishReason")
|
||||
chat_completion_message["provider_specific_fields"] = finish_reason_fields
|
||||
|
||||
if isinstance(model_response, ModelResponseStream):
|
||||
choice = VertexGeminiConfig._create_streaming_choice(
|
||||
chat_completion_message=chat_completion_message,
|
||||
|
|
@ -2375,15 +2375,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
)
|
||||
model_response.choices.append(choice)
|
||||
elif isinstance(model_response, ModelResponse):
|
||||
native_finish_reason = candidate.get("finishReason")
|
||||
choice = litellm.Choices(
|
||||
finish_reason=VertexGeminiConfig._check_finish_reason(
|
||||
chat_completion_message, candidate.get("finishReason")
|
||||
chat_completion_message, native_finish_reason
|
||||
),
|
||||
index=candidate.get("index", idx),
|
||||
message=chat_completion_message,
|
||||
logprobs=chat_completion_logprobs,
|
||||
enhancements=None,
|
||||
provider_specific_fields=chat_completion_message.get("provider_specific_fields"),
|
||||
provider_specific_fields=(
|
||||
{"native_finish_reason": native_finish_reason} if native_finish_reason is not None else None
|
||||
),
|
||||
)
|
||||
model_response.choices.append(choice)
|
||||
|
||||
|
|
@ -3181,12 +3184,10 @@ class ModelResponseIterator:
|
|||
self.has_seen_tool_calls = True
|
||||
break
|
||||
|
||||
# _process_candidates skips candidates without a "content" part, so a
|
||||
# content-less chunk leaves choices empty and the downstream streaming
|
||||
# handler hits IndexError on choices[0]. This covers the final chunk
|
||||
# (finishReason, no content) and mid-stream metadata-only chunks
|
||||
# (grounding/web-search/thought, no content and no finishReason — seen
|
||||
# with web_search + reasoning) by emitting an empty-delta choice.
|
||||
# _process_candidates skips candidates with neither "content" nor
|
||||
# "finishReason", so a metadata-only chunk (grounding/web-search/thought,
|
||||
# seen with web_search + reasoning) leaves choices empty and the downstream
|
||||
# streaming handler hits IndexError on choices[0]. Emit an empty-delta choice.
|
||||
if not model_response.choices and _candidates:
|
||||
from litellm.types.utils import Delta, StreamingChoices
|
||||
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ from litellm.types.llms.openai import (
|
|||
ChatCompletionToolParamFunctionChunk,
|
||||
ChatCompletionUserMessage,
|
||||
GenericChatCompletionMessage,
|
||||
IncompleteDetails,
|
||||
InputTokensDetails,
|
||||
OpenAIChatCompletionTextObject,
|
||||
OpenAIMcpServerTool,
|
||||
|
|
@ -2295,6 +2296,21 @@ class LiteLLMCompletionResponsesConfig:
|
|||
# Default to completed for unknown finish reasons
|
||||
return "completed"
|
||||
|
||||
@staticmethod
|
||||
def _incomplete_details_for_finish_reason(
|
||||
finish_reason: str | None,
|
||||
existing: IncompleteDetails | None,
|
||||
) -> IncompleteDetails | None:
|
||||
if existing is not None:
|
||||
return existing
|
||||
match finish_reason:
|
||||
case "length":
|
||||
return IncompleteDetails(reason="max_output_tokens")
|
||||
case "content_filter" | "refusal":
|
||||
return IncompleteDetails(reason="content_filter")
|
||||
case _:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _tool_call_id_from_responses_item(item_id: str | None, call_id: str | None) -> str:
|
||||
"""Bedrock Mantle returns a non-unique, index-based ``call_id`` (``call_0``,
|
||||
|
|
@ -2411,17 +2427,10 @@ class LiteLLMCompletionResponsesConfig:
|
|||
if choices and len(choices) > 0:
|
||||
finish_reason = choices[0].finish_reason
|
||||
|
||||
status: Final[ResponsesAPIStatus] = (
|
||||
LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(finish_reason)
|
||||
incomplete_details: Final = LiteLLMCompletionResponsesConfig._incomplete_details_for_finish_reason(
|
||||
finish_reason=finish_reason,
|
||||
existing=getattr(chat_completion_response, "incomplete_details", None),
|
||||
)
|
||||
incomplete_details = getattr(chat_completion_response, "incomplete_details", None)
|
||||
if incomplete_details is None and status == "incomplete":
|
||||
from openai.types.responses.response import IncompleteDetails
|
||||
|
||||
if finish_reason == "length":
|
||||
incomplete_details = IncompleteDetails(reason="max_output_tokens")
|
||||
elif finish_reason in ["content_filter", "refusal"]:
|
||||
incomplete_details = IncompleteDetails(reason="content_filter")
|
||||
|
||||
responses_api_response: Final[ResponsesAPIResponse] = ResponsesAPIResponse(
|
||||
id=chat_completion_response.id,
|
||||
|
|
@ -2447,7 +2456,9 @@ class LiteLLMCompletionResponsesConfig:
|
|||
max_output_tokens=getattr(chat_completion_response, "max_output_tokens", None),
|
||||
previous_response_id=getattr(chat_completion_response, "previous_response_id", None),
|
||||
reasoning=None,
|
||||
status=status,
|
||||
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
|
||||
finish_reason
|
||||
),
|
||||
text={},
|
||||
truncation=getattr(chat_completion_response, "truncation", None),
|
||||
usage=LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(
|
||||
|
|
|
|||
|
|
@ -151,6 +151,12 @@ class TestMapFinishReasonGemini:
|
|||
("IMAGE_PROHIBITED_CONTENT", "content_filter"),
|
||||
("TOO_MANY_TOOL_CALLS", "stop"),
|
||||
("MALFORMED_RESPONSE", "stop"),
|
||||
("NO_IMAGE", "content_filter"),
|
||||
("IMAGE_RECITATION", "content_filter"),
|
||||
("IMAGE_OTHER", "content_filter"),
|
||||
("ESCALATION", "content_filter"),
|
||||
("UNEXPECTED_TOOL_CALL", "stop"),
|
||||
("MISSING_THOUGHT_SIGNATURE", "stop"),
|
||||
],
|
||||
)
|
||||
def test_gemini_finish_reasons(self, gemini_reason, expected):
|
||||
|
|
|
|||
|
|
@ -968,6 +968,12 @@ def test_finish_reason_unspecified_and_malformed_function_call():
|
|||
# Test new Gemini finish reasons
|
||||
assert finish_reason_mappings["TOO_MANY_TOOL_CALLS"] == "stop"
|
||||
assert finish_reason_mappings["MALFORMED_RESPONSE"] == "stop"
|
||||
assert finish_reason_mappings["NO_IMAGE"] == "content_filter"
|
||||
assert finish_reason_mappings["IMAGE_RECITATION"] == "content_filter"
|
||||
assert finish_reason_mappings["IMAGE_OTHER"] == "content_filter"
|
||||
assert finish_reason_mappings["ESCALATION"] == "content_filter"
|
||||
assert finish_reason_mappings["UNEXPECTED_TOOL_CALL"] == "stop"
|
||||
assert finish_reason_mappings["MISSING_THOUGHT_SIGNATURE"] == "stop"
|
||||
|
||||
|
||||
def test_vertex_ai_usage_metadata_response_token_count():
|
||||
|
|
@ -6219,3 +6225,65 @@ def test_gemini_candidate_other_finish_reasons_no_content():
|
|||
)
|
||||
assert responses_length.status == "incomplete"
|
||||
assert responses_length.incomplete_details.reason == "max_output_tokens"
|
||||
|
||||
|
||||
def test_gemini_candidate_with_finish_reason_no_content_streaming_chunk():
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
ModelResponseIterator,
|
||||
)
|
||||
|
||||
chunk: Final = {
|
||||
"candidates": [{"finishReason": "NO_IMAGE", "index": 0}],
|
||||
"usageMetadata": {"promptTokenCount": 19, "candidatesTokenCount": 0, "totalTokenCount": 19},
|
||||
}
|
||||
iterator: Final = ModelResponseIterator(streaming_response=[], sync_stream=True, logging_obj=MagicMock())
|
||||
|
||||
streaming_chunk: Final = iterator.chunk_parser(chunk)
|
||||
|
||||
assert len(streaming_chunk.choices) == 1
|
||||
assert streaming_chunk.choices[0].finish_reason == "content_filter"
|
||||
assert streaming_chunk.choices[0].delta.content is None
|
||||
assert streaming_chunk.choices[0].delta.tool_calls is None
|
||||
|
||||
|
||||
def test_gemini_multi_candidate_messages_do_not_share_state():
|
||||
config: Final = VertexGeminiConfig()
|
||||
completion_response: Final = {
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"role": "model",
|
||||
"parts": [
|
||||
{"text": "Let me check the weather.", "thought": True},
|
||||
{"functionCall": {"name": "get_weather", "args": {"city": "Paris"}}},
|
||||
],
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
},
|
||||
{
|
||||
"content": {"role": "model", "parts": [{"text": "It is sunny in Paris."}]},
|
||||
"finishReason": "STOP",
|
||||
"index": 1,
|
||||
},
|
||||
],
|
||||
"usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 20, "totalTokenCount": 30},
|
||||
}
|
||||
|
||||
resp: Final = config._transform_google_generate_content_to_openai_model_response(
|
||||
completion_response=completion_response,
|
||||
model_response=ModelResponse(),
|
||||
model="gemini-2.5-flash",
|
||||
logging_obj=MagicMock(),
|
||||
raw_response=MagicMock(headers={}),
|
||||
)
|
||||
|
||||
assert len(resp.choices) == 2
|
||||
assert resp.choices[0].finish_reason == "tool_calls"
|
||||
assert resp.choices[0].message.tool_calls[0].function.name == "get_weather"
|
||||
assert resp.choices[0].message.reasoning_content == "Let me check the weather."
|
||||
assert resp.choices[1].finish_reason == "stop"
|
||||
assert resp.choices[1].message.content == "It is sunny in Paris."
|
||||
assert resp.choices[1].message.tool_calls is None
|
||||
assert getattr(resp.choices[1].message, "reasoning_content", None) is None
|
||||
assert resp.choices[1].provider_specific_fields["native_finish_reason"] == "STOP"
|
||||
|
|
|
|||
|
|
@ -4909,7 +4909,7 @@ class TestStreamingSnapshotItemIds:
|
|||
|
||||
|
||||
def test_transform_chat_completion_response_incomplete_details():
|
||||
from openai.types.responses.response import IncompleteDetails
|
||||
from litellm.types.llms.openai import IncompleteDetails
|
||||
|
||||
resp_length = ModelResponse(
|
||||
id="resp-length",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue