fix(gemini): parse tool call arguments with the shared helper (#50020)

Gemini tool call arguments now go through parse_tool_call_arguments, the shared helper the other providers use, so malformed or empty argument strings are repaired or rejected the same way instead of raising a raw JSONDecodeError
This commit is contained in:
Stanley Zhou 2026-11-16 22:31:25 -05:00 • committed by GitHub
parent 5008a9a11d
commit 326702cc94
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 334 additions and 18 deletions

View file

@ -1216,23 +1216,40 @@ def infer_protocol_value(
return "unknown"
def _parsed_gemini_function_call(name: str, arguments: object) -> VertexFunctionCall:
text: Final = arguments.decode() if isinstance(arguments, (bytes, bytearray)) else arguments
if text == "": # pass empty dict, if arguments is empty string - prevents call from failing
return VertexFunctionCall(name=name, args={"type": "object"})
if text is not None and not isinstance(text, str):
raise ValueError(f"Tool call arguments for tool '{name}' must be a JSON string, got {type(text).__name__}")
args: Final = _DECODED_JSON.validate_python(
parse_tool_call_arguments(text, tool_name=name, context="Gemini tool invoke")
)
mapping: Final = as_str_mapping(args)
if args is not None and mapping is None:
raise ValueError(f"Tool call arguments for tool '{name}' must be a JSON object, got {type(args).__name__}")
return VertexFunctionCall(name=name, args=None if mapping is None else dict(mapping))
def _gemini_function_call(name: str, arguments: object, *, drop_params: bool) -> VertexFunctionCall:
try:
return _parsed_gemini_function_call(name, arguments)
except ValueError:
if not drop_params:
raise
return VertexFunctionCall(name=name, args={})
def _gemini_tool_call_invoke_helper(
function_call_params: ChatCompletionToolCallFunctionChunk,
tool_call_id: str | None = None,
*,
drop_params: bool = False,
) -> VertexFunctionCall | None:
name: Final = function_call_params.get("name", "") or ""
arguments: Final = function_call_params.get("arguments", "")
if (
isinstance(arguments, str) and len(arguments) == 0
): # pass empty dict, if arguments is empty string - prevents call from failing
arguments_dict = {
"type": "object",
}
else:
arguments_dict = json.loads(arguments)
function_call: Final = VertexFunctionCall(
name=name,
args=arguments_dict,
function_call: Final = _gemini_function_call(
function_call_params.get("name", "") or "",
function_call_params.get("arguments", ""),
drop_params=drop_params,
)
if tool_call_id:
clean_id: Final = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0]
@ -1358,6 +1375,8 @@ def convert_to_gemini_tool_call_invoke(
message: ChatCompletionAssistantMessage,
model: str | None = None,
forward_function_call_id: bool = False,
custom_llm_provider: str | None = None,
drop_params: bool = False,
) -> list[VertexPartType]:
"""
OpenAI tool invokes:
@ -1415,6 +1434,7 @@ def convert_to_gemini_tool_call_invoke(
gemini_function_call: VertexFunctionCall | None = _gemini_tool_call_invoke_helper(
function_call_params=tool["function"],
tool_call_id=(tool.get("id") if forward_function_call_id else None),
drop_params=drop_params,
)
if gemini_function_call is not None:
part_dict: VertexPartType = {"function_call": gemini_function_call}
@ -1434,7 +1454,9 @@ def convert_to_gemini_tool_call_invoke(
f"function_call missing. Received tool call with 'type': 'function'. No function call in argument - {tool}"
)
elif function_call is not None:
gemini_function_call = _gemini_tool_call_invoke_helper(function_call_params=function_call)
gemini_function_call = _gemini_tool_call_invoke_helper(
function_call_params=function_call, drop_params=drop_params
)
if gemini_function_call is not None:
part_dict_function: Final[VertexPartType] = {"function_call": gemini_function_call}
@ -1460,7 +1482,11 @@ def convert_to_gemini_tool_call_invoke(
)
return _parts_list
except Exception as e:
raise Exception(f"Unable to convert openai tool calls={message} to gemini tool calls. Received error={e}")
raise litellm.BadRequestError(
message=f"Unable to convert openai tool calls={message} to gemini tool calls. Received error={e}",
model=model or "",
llm_provider=custom_llm_provider or "vertex_ai",
) from e
def _json_children(node: object) -> Iterable[object]:

View file

@ -10,6 +10,7 @@ from datetime import datetime, timezone
from types import MappingProxyType
from typing import Final, Literal
from litellm.litellm_core_utils.prompt_templates.factory import convert_to_gemini_tool_call_invoke
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.vertex_ai import CachedContentRequestBody
from litellm.utils import is_cached_message
@ -143,6 +144,19 @@ def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageVa
return non_system_messages[-1].get("role") not in ("assistant", "tool", "function")
def reject_unusable_tool_call_arguments(
cached_messages: Sequence[AllMessageValues],
model: str,
custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"],
drop_params: bool,
) -> None:
if drop_params:
return
for message in cached_messages:
if message["role"] == "assistant":
convert_to_gemini_tool_call_invoke(message, model=model, custom_llm_provider=custom_llm_provider)
def transform_openai_messages_to_gemini_context_caching(
model: str,
messages: list[AllMessageValues],
@ -150,6 +164,7 @@ def transform_openai_messages_to_gemini_context_caching(
cache_key: str,
vertex_project: str | None,
vertex_location: str | None,
drop_params: bool = False,
) -> CachedContentRequestBody:
# Extract TTL from cached messages BEFORE system message transformation
ttl: Final = extract_ttl_from_cached_messages(messages)
@ -163,6 +178,7 @@ def transform_openai_messages_to_gemini_context_caching(
transformed_messages: Final = gemini_convert_messages_with_history(
messages=new_messages,
model=model,
litellm_params={"drop_params": drop_params},
custom_llm_provider=custom_llm_provider,
)

View file

@ -24,6 +24,7 @@ from ..vertex_llm_base import VertexBase, publisher_model_location
from .known_cached_contents import KnownCachedContents, known_cached_contents
from .transformation import (
cached_messages_end_on_supported_turn,
reject_unusable_tool_call_arguments,
separate_cached_messages,
transform_openai_messages_to_gemini_context_caching,
)
@ -426,6 +427,7 @@ class ContextCachingEndpoints(VertexBase):
extra_headers: dict | None = None,
cached_content: str | None = None,
context_caching: bool = True,
drop_params: bool = False,
) -> tuple[list[AllMessageValues], dict, str | None]:
"""
Receives
@ -454,6 +456,7 @@ class ContextCachingEndpoints(VertexBase):
if skip_reason is not None:
verbose_logger.debug("%s Skipping context caching.", skip_reason)
return messages, optional_params, None
reject_unusable_tool_call_arguments(cached_messages, target.model, custom_llm_provider, drop_params)
## AUTHORIZATION ##
token, url = self._get_token_and_url_context_caching(
@ -512,6 +515,7 @@ class ContextCachingEndpoints(VertexBase):
custom_llm_provider=custom_llm_provider,
vertex_project=target.project,
vertex_location=target.location,
drop_params=drop_params,
)
cached_content_request_body["tools"] = tools
@ -575,6 +579,7 @@ class ContextCachingEndpoints(VertexBase):
extra_headers: dict | None = None,
cached_content: str | None = None,
context_caching: bool = True,
drop_params: bool = False,
) -> tuple[list[AllMessageValues], dict, str | None]:
"""
Receives
@ -603,6 +608,7 @@ class ContextCachingEndpoints(VertexBase):
if skip_reason is not None:
verbose_logger.debug("%s Skipping context caching.", skip_reason)
return messages, optional_params, None
reject_unusable_tool_call_arguments(cached_messages, target.model, custom_llm_provider, drop_params)
## AUTHORIZATION ##
token, url = self._get_token_and_url_context_caching(
@ -664,6 +670,7 @@ class ContextCachingEndpoints(VertexBase):
custom_llm_provider=custom_llm_provider,
vertex_project=target.project,
vertex_location=target.location,
drop_params=drop_params,
)
cached_content_request_body["tools"] = tools

View file

@ -798,6 +798,8 @@ def gemini_convert_messages_with_history(
tool_call_responses = []
vertex_project = None
vertex_credentials = None
request_settings: Final = cast(Mapping[str, object], litellm_params or {}) # cast-ok: untyped shared signature
drop_params: Final = drops_unsupported_params(request_settings.get("drop_params"))
if litellm_params:
vertex_project = litellm_params.get("vertex_project") or litellm_params.get("vertex_ai_project")
vertex_credentials = litellm_params.get("vertex_credentials") or litellm_params.get("vertex_ai_credentials")
@ -948,6 +950,8 @@ def gemini_convert_messages_with_history(
assistant_msg,
model=model,
forward_function_call_id=forward_function_call_id,
custom_llm_provider=custom_llm_provider,
drop_params=drop_params,
)
## check if gemini_tool_call already exists in assistant_content
for gemini_tool_call_part in gemini_tool_call_parts:
@ -1309,6 +1313,7 @@ def sync_transform_request_body(
from ..context_caching.vertex_ai_context_caching import ContextCachingEndpoints
context_caching_endpoints: Final = ContextCachingEndpoints()
request_settings: Final = cast(Mapping[str, object], litellm_params) # cast-ok: untyped shared signature
(
messages,
@ -1330,6 +1335,7 @@ def sync_transform_request_body(
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_auth_header=vertex_auth_header,
drop_params=drops_unsupported_params(request_settings.get("drop_params")),
)
return _with_default_output_budget(
@ -1445,6 +1451,7 @@ async def async_transform_request_body(
from ..context_caching.vertex_ai_context_caching import ContextCachingEndpoints
context_caching_endpoints: Final = ContextCachingEndpoints()
request_settings: Final = cast(Mapping[str, object], litellm_params) # cast-ok: untyped shared signature
(
messages,
@ -1466,6 +1473,7 @@ async def async_transform_request_body(
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_auth_header=vertex_auth_header,
drop_params=drops_unsupported_params(request_settings.get("drop_params")),
)
inlined_messages: Final = await async_inline_remote_media(

View file

@ -2089,6 +2089,110 @@ def test_vertex_ai_transform_empty_function_call_arguments():
}
@pytest.mark.parametrize(
"arguments, expected_args",
[
(None, {}),
('{"location": "Boston"', {"location": "Boston"}),
(b'{"location": "Boston"}', {"location": "Boston"}),
],
ids=["null_arguments", "truncated_arguments", "bytes_arguments"],
)
def test_convert_to_gemini_tool_call_invoke_tolerates_null_truncated_and_bytes_arguments(
arguments: str | bytes | None, expected_args: dict[str, str]
) -> None:
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_gemini_tool_call_invoke,
)
message: Final = {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": arguments},
}
],
}
parts: Final = convert_to_gemini_tool_call_invoke(message)
assert len(parts) == 1
assert parts[0]["function_call"]["name"] == "get_weather"
assert parts[0]["function_call"]["args"] == expected_args
@pytest.mark.parametrize(
"arguments",
[{"location": "Boston"}, ["Boston"], 5, False],
ids=["object", "list", "number", "false"],
)
def test_convert_to_gemini_tool_call_invoke_rejects_arguments_that_are_not_a_json_string(
arguments: dict[str, str] | list[str] | int | bool,
) -> None:
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_gemini_tool_call_invoke,
)
message: Final = {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": arguments},
}
],
}
with pytest.raises(litellm.BadRequestError, match="must be a JSON string"):
convert_to_gemini_tool_call_invoke(message)
@pytest.mark.parametrize("arguments", ['["Boston"]', '"Boston"', "5"], ids=["list", "string", "number"])
def test_convert_to_gemini_tool_call_invoke_rejects_json_text_that_is_not_an_object(arguments: str) -> None:
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_gemini_tool_call_invoke,
)
message: Final = {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": arguments},
}
],
}
with pytest.raises(litellm.BadRequestError, match="must be a JSON object"):
convert_to_gemini_tool_call_invoke(message)
@pytest.mark.parametrize(
"arguments",
[{"location": "Boston"}, ["Boston"], '["Boston"]', "not json"],
ids=["object", "list", "json_list_text", "not_json"],
)
@pytest.mark.parametrize("shape", ["tool_calls", "function_call"])
def test_convert_to_gemini_tool_call_invoke_drops_unusable_arguments_under_drop_params(
arguments: object, shape: str
) -> None:
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_gemini_tool_call_invoke,
)
function: Final = {"name": "get_weather", "arguments": arguments}
message: Final = (
{"role": "assistant", "content": None, "tool_calls": [{"id": "call_1", "type": "function", "function": function}]}
if shape == "tool_calls"
else {"role": "assistant", "content": None, "function_call": function}
)
parts: Final = convert_to_gemini_tool_call_invoke(message, drop_params=True)
assert parts == [{"function_call": {"name": "get_weather", "args": {}}}]
@pytest.mark.asyncio
async def test_bedrock_process_image_async_factory():
"""

View file

@ -1,4 +1,5 @@
import json
from collections.abc import Awaitable, Callable, Mapping
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
@ -2510,11 +2511,23 @@ def test_check_and_create_cache_clamps_negative_creation_tokens() -> None:
assert logging_obj.vertex_ai_cache_creation_input_tokens == 0
def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler:
def _caches_created_so_far(calls: list[httpx.Request]) -> dict[str, list[dict[str, str]]]:
return {
"cachedContents": [
{"name": "cachedContents/created", "displayName": json.loads(call.content)["displayName"]}
for call in calls
if call.method == "POST" and call.url.path.endswith("/cachedContents")
]
}
def _recording_gemini_route(calls: list[httpx.Request]) -> Callable[[httpx.Request], httpx.Response]:
def route(request: httpx.Request) -> httpx.Response:
calls.append(request)
if request.url.path.endswith("/cachedContents"):
return httpx.Response(200, json={} if request.method == "GET" else _created_cache_body(3000))
return httpx.Response(
200, json=_caches_created_so_far(calls) if request.method == "GET" else _created_cache_body(3000)
)
return httpx.Response(
200,
json={
@ -2523,7 +2536,11 @@ def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler:
},
)
return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(route)))
return route
def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler:
return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_recording_gemini_route(calls))))
def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(local_model_cost_map) -> None:
@ -2551,3 +2568,73 @@ def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(l
assert created["system_instruction"] == {"parts": [{"text": "You are terse."}]}
assert generated["cachedContent"] == "cachedContents/created"
assert "system_instruction" not in generated
def _cached_conversation_with_tool_arguments(arguments: str) -> list[Mapping[str, object]]:
return [
{
"role": "user",
"content": [{"type": "text", "text": "cached prefix " * 3000, "cache_control": {"type": "ephemeral"}}],
},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": arguments}}
],
"cache_control": {"type": "ephemeral"},
},
{"role": "tool", "tool_call_id": "call_1", "content": "Sunny", "cache_control": {"type": "ephemeral"}},
{"role": "user", "content": [{"type": "text", "text": "Thanks.", "cache_control": {"type": "ephemeral"}}]},
{"role": "user", "content": "Reply with OK only."},
]
async def _complete_sync(request: Mapping[str, object], calls: list[httpx.Request]) -> None:
litellm.completion(**request, client=_recording_gemini_client(calls))
async def _complete_async(request: Mapping[str, object], calls: list[httpx.Request]) -> None:
await litellm.acompletion(
**request, client=AsyncHTTPHandler(transport=httpx.MockTransport(_recording_gemini_route(calls)))
)
@pytest.mark.asyncio
@pytest.mark.parametrize("complete", [_complete_sync, _complete_async])
async def test_cache_creation_drops_tool_call_arguments_gemini_cannot_use_when_drop_params_is_set(
local_model_cost_map: None, complete: Callable[[Mapping[str, object], list[httpx.Request]], Awaitable[None]]
) -> None:
calls: Final[list[httpx.Request]] = []
await complete(
{
"model": "gemini/gemini-flash-latest",
"messages": _cached_conversation_with_tool_arguments('["Boston"]'),
"api_key": "test-key",
"drop_params": True,
},
calls,
)
created: Final = next(
json.loads(call.content) for call in calls if call.method == "POST" and call.url.path.endswith("/cachedContents")
)
function_call: Final = created["contents"][1]["parts"][0]["function_call"]
assert (function_call["name"], function_call["args"]) == ("get_weather", {})
@pytest.mark.asyncio
@pytest.mark.parametrize("complete", [_complete_sync, _complete_async])
async def test_a_strict_request_rejects_tool_call_arguments_even_when_a_lenient_request_cached_that_history(
local_model_cost_map: None, complete: Callable[[Mapping[str, object], list[httpx.Request]], Awaitable[None]]
) -> None:
calls: Final[list[httpx.Request]] = []
request: Final = {
"model": "gemini/gemini-flash-latest",
"messages": _cached_conversation_with_tool_arguments('["Boston"]'),
"api_key": "test-key",
}
await complete({**request, "drop_params": True}, calls)
with pytest.raises(litellm.BadRequestError, match="get_weather"):
await complete({**request, "drop_params": False}, calls)

View file

@ -487,6 +487,74 @@ def test_image_only_user_content_logs_the_blank_text_fixup_at_debug(caplog: pyte
assert contents[0]["parts"][-1] == {"text": " "}
@pytest.mark.parametrize("custom_llm_provider, expected_provider", [("gemini", "gemini"), (None, "vertex_ai")])
def test_tool_call_conversion_error_names_the_calling_provider(
custom_llm_provider: str | None, expected_provider: str
) -> None:
messages: Final = [
{"role": "user", "content": "weather?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": ["Boston"]}}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
]
with pytest.raises(litellm.BadRequestError) as exc_info:
transformation.gemini_convert_messages_with_history(messages, custom_llm_provider=custom_llm_provider)
assert exc_info.value.llm_provider == expected_provider
def test_tool_call_arguments_gemini_cannot_use_are_dropped_when_drop_params_is_set() -> None:
messages: Final = [
{"role": "user", "content": "weather?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": ["Boston"]}}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
]
contents: Final = transformation.gemini_convert_messages_with_history(
messages, litellm_params={"drop_params": True}, custom_llm_provider="gemini"
)
assert contents[1]["parts"] == [{"function_call": {"name": "get_weather", "args": {}}}]
def test_gemini_completion_reports_a_tool_call_it_cannot_convert_as_a_gemini_bad_request() -> None:
with pytest.raises(litellm.BadRequestError) as exc_info:
litellm.completion(
model="gemini/gemini-flash-latest",
api_key="test-key",
messages=[
{"role": "user", "content": "weather?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": {"location": "Boston"}},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
],
)
assert exc_info.value.llm_provider == "gemini"
assert "must be a JSON string, got dict" in str(exc_info.value)
_LOOKUP_TOOL: Final = {
"type": "function",
"function": {