diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 1aa4e289b79..06a722376e7 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1216,23 +1216,40 @@ def infer_protocol_value( return "unknown" +def _parsed_gemini_function_call(name: str, arguments: object) -> VertexFunctionCall: + text: Final = arguments.decode() if isinstance(arguments, (bytes, bytearray)) else arguments + if text == "": # pass empty dict, if arguments is empty string - prevents call from failing + return VertexFunctionCall(name=name, args={"type": "object"}) + if text is not None and not isinstance(text, str): + raise ValueError(f"Tool call arguments for tool '{name}' must be a JSON string, got {type(text).__name__}") + args: Final = _DECODED_JSON.validate_python( + parse_tool_call_arguments(text, tool_name=name, context="Gemini tool invoke") + ) + mapping: Final = as_str_mapping(args) + if args is not None and mapping is None: + raise ValueError(f"Tool call arguments for tool '{name}' must be a JSON object, got {type(args).__name__}") + return VertexFunctionCall(name=name, args=None if mapping is None else dict(mapping)) + + +def _gemini_function_call(name: str, arguments: object, *, drop_params: bool) -> VertexFunctionCall: + try: + return _parsed_gemini_function_call(name, arguments) + except ValueError: + if not drop_params: + raise + return VertexFunctionCall(name=name, args={}) + + def _gemini_tool_call_invoke_helper( function_call_params: ChatCompletionToolCallFunctionChunk, tool_call_id: str | None = None, + *, + drop_params: bool = False, ) -> VertexFunctionCall | None: - name: Final = function_call_params.get("name", "") or "" - arguments: Final = function_call_params.get("arguments", "") - if ( - isinstance(arguments, str) and len(arguments) == 0 - ): # pass empty dict, if arguments is empty string - prevents call from failing - arguments_dict = { - "type": "object", - } - else: - arguments_dict = json.loads(arguments) - function_call: Final = VertexFunctionCall( - name=name, - args=arguments_dict, + function_call: Final = _gemini_function_call( + function_call_params.get("name", "") or "", + function_call_params.get("arguments", ""), + drop_params=drop_params, ) if tool_call_id: clean_id: Final = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0] @@ -1358,6 +1375,8 @@ def convert_to_gemini_tool_call_invoke( message: ChatCompletionAssistantMessage, model: str | None = None, forward_function_call_id: bool = False, + custom_llm_provider: str | None = None, + drop_params: bool = False, ) -> list[VertexPartType]: """ OpenAI tool invokes: @@ -1415,6 +1434,7 @@ def convert_to_gemini_tool_call_invoke( gemini_function_call: VertexFunctionCall | None = _gemini_tool_call_invoke_helper( function_call_params=tool["function"], tool_call_id=(tool.get("id") if forward_function_call_id else None), + drop_params=drop_params, ) if gemini_function_call is not None: part_dict: VertexPartType = {"function_call": gemini_function_call} @@ -1434,7 +1454,9 @@ def convert_to_gemini_tool_call_invoke( f"function_call missing. Received tool call with 'type': 'function'. No function call in argument - {tool}" ) elif function_call is not None: - gemini_function_call = _gemini_tool_call_invoke_helper(function_call_params=function_call) + gemini_function_call = _gemini_tool_call_invoke_helper( + function_call_params=function_call, drop_params=drop_params + ) if gemini_function_call is not None: part_dict_function: Final[VertexPartType] = {"function_call": gemini_function_call} @@ -1460,7 +1482,11 @@ def convert_to_gemini_tool_call_invoke( ) return _parts_list except Exception as e: - raise Exception(f"Unable to convert openai tool calls={message} to gemini tool calls. Received error={e}") + raise litellm.BadRequestError( + message=f"Unable to convert openai tool calls={message} to gemini tool calls. Received error={e}", + model=model or "", + llm_provider=custom_llm_provider or "vertex_ai", + ) from e def _json_children(node: object) -> Iterable[object]: diff --git a/litellm/llms/vertex_ai/context_caching/transformation.py b/litellm/llms/vertex_ai/context_caching/transformation.py index a4601af944b..6a8ed6910f2 100644 --- a/litellm/llms/vertex_ai/context_caching/transformation.py +++ b/litellm/llms/vertex_ai/context_caching/transformation.py @@ -10,6 +10,7 @@ from datetime import datetime, timezone from types import MappingProxyType from typing import Final, Literal +from litellm.litellm_core_utils.prompt_templates.factory import convert_to_gemini_tool_call_invoke from litellm.types.llms.openai import AllMessageValues from litellm.types.llms.vertex_ai import CachedContentRequestBody from litellm.utils import is_cached_message @@ -143,6 +144,19 @@ def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageVa return non_system_messages[-1].get("role") not in ("assistant", "tool", "function") +def reject_unusable_tool_call_arguments( + cached_messages: Sequence[AllMessageValues], + model: str, + custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], + drop_params: bool, +) -> None: + if drop_params: + return + for message in cached_messages: + if message["role"] == "assistant": + convert_to_gemini_tool_call_invoke(message, model=model, custom_llm_provider=custom_llm_provider) + + def transform_openai_messages_to_gemini_context_caching( model: str, messages: list[AllMessageValues], @@ -150,6 +164,7 @@ def transform_openai_messages_to_gemini_context_caching( cache_key: str, vertex_project: str | None, vertex_location: str | None, + drop_params: bool = False, ) -> CachedContentRequestBody: # Extract TTL from cached messages BEFORE system message transformation ttl: Final = extract_ttl_from_cached_messages(messages) @@ -163,6 +178,7 @@ def transform_openai_messages_to_gemini_context_caching( transformed_messages: Final = gemini_convert_messages_with_history( messages=new_messages, model=model, + litellm_params={"drop_params": drop_params}, custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 4603f612926..7df10eaeb6c 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -24,6 +24,7 @@ from ..vertex_llm_base import VertexBase, publisher_model_location from .known_cached_contents import KnownCachedContents, known_cached_contents from .transformation import ( cached_messages_end_on_supported_turn, + reject_unusable_tool_call_arguments, separate_cached_messages, transform_openai_messages_to_gemini_context_caching, ) @@ -426,6 +427,7 @@ class ContextCachingEndpoints(VertexBase): extra_headers: dict | None = None, cached_content: str | None = None, context_caching: bool = True, + drop_params: bool = False, ) -> tuple[list[AllMessageValues], dict, str | None]: """ Receives @@ -454,6 +456,7 @@ class ContextCachingEndpoints(VertexBase): if skip_reason is not None: verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None + reject_unusable_tool_call_arguments(cached_messages, target.model, custom_llm_provider, drop_params) ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( @@ -512,6 +515,7 @@ class ContextCachingEndpoints(VertexBase): custom_llm_provider=custom_llm_provider, vertex_project=target.project, vertex_location=target.location, + drop_params=drop_params, ) cached_content_request_body["tools"] = tools @@ -575,6 +579,7 @@ class ContextCachingEndpoints(VertexBase): extra_headers: dict | None = None, cached_content: str | None = None, context_caching: bool = True, + drop_params: bool = False, ) -> tuple[list[AllMessageValues], dict, str | None]: """ Receives @@ -603,6 +608,7 @@ class ContextCachingEndpoints(VertexBase): if skip_reason is not None: verbose_logger.debug("%s Skipping context caching.", skip_reason) return messages, optional_params, None + reject_unusable_tool_call_arguments(cached_messages, target.model, custom_llm_provider, drop_params) ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( @@ -664,6 +670,7 @@ class ContextCachingEndpoints(VertexBase): custom_llm_provider=custom_llm_provider, vertex_project=target.project, vertex_location=target.location, + drop_params=drop_params, ) cached_content_request_body["tools"] = tools diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 97a36403c75..fef278e4875 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -798,6 +798,8 @@ def gemini_convert_messages_with_history( tool_call_responses = [] vertex_project = None vertex_credentials = None + request_settings: Final = cast(Mapping[str, object], litellm_params or {}) # cast-ok: untyped shared signature + drop_params: Final = drops_unsupported_params(request_settings.get("drop_params")) if litellm_params: vertex_project = litellm_params.get("vertex_project") or litellm_params.get("vertex_ai_project") vertex_credentials = litellm_params.get("vertex_credentials") or litellm_params.get("vertex_ai_credentials") @@ -948,6 +950,8 @@ def gemini_convert_messages_with_history( assistant_msg, model=model, forward_function_call_id=forward_function_call_id, + custom_llm_provider=custom_llm_provider, + drop_params=drop_params, ) ## check if gemini_tool_call already exists in assistant_content for gemini_tool_call_part in gemini_tool_call_parts: @@ -1309,6 +1313,7 @@ def sync_transform_request_body( from ..context_caching.vertex_ai_context_caching import ContextCachingEndpoints context_caching_endpoints: Final = ContextCachingEndpoints() + request_settings: Final = cast(Mapping[str, object], litellm_params) # cast-ok: untyped shared signature ( messages, @@ -1330,6 +1335,7 @@ def sync_transform_request_body( vertex_project=vertex_project, vertex_location=vertex_location, vertex_auth_header=vertex_auth_header, + drop_params=drops_unsupported_params(request_settings.get("drop_params")), ) return _with_default_output_budget( @@ -1445,6 +1451,7 @@ async def async_transform_request_body( from ..context_caching.vertex_ai_context_caching import ContextCachingEndpoints context_caching_endpoints: Final = ContextCachingEndpoints() + request_settings: Final = cast(Mapping[str, object], litellm_params) # cast-ok: untyped shared signature ( messages, @@ -1466,6 +1473,7 @@ async def async_transform_request_body( vertex_project=vertex_project, vertex_location=vertex_location, vertex_auth_header=vertex_auth_header, + drop_params=drops_unsupported_params(request_settings.get("drop_params")), ) inlined_messages: Final = await async_inline_remote_media( diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index f029bce61ef..37778fc43b3 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -2089,6 +2089,110 @@ def test_vertex_ai_transform_empty_function_call_arguments(): } +@pytest.mark.parametrize( + "arguments, expected_args", + [ + (None, {}), + ('{"location": "Boston"', {"location": "Boston"}), + (b'{"location": "Boston"}', {"location": "Boston"}), + ], + ids=["null_arguments", "truncated_arguments", "bytes_arguments"], +) +def test_convert_to_gemini_tool_call_invoke_tolerates_null_truncated_and_bytes_arguments( + arguments: str | bytes | None, expected_args: dict[str, str] +) -> None: + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + message: Final = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": arguments}, + } + ], + } + parts: Final = convert_to_gemini_tool_call_invoke(message) + assert len(parts) == 1 + assert parts[0]["function_call"]["name"] == "get_weather" + assert parts[0]["function_call"]["args"] == expected_args + + +@pytest.mark.parametrize( + "arguments", + [{"location": "Boston"}, ["Boston"], 5, False], + ids=["object", "list", "number", "false"], +) +def test_convert_to_gemini_tool_call_invoke_rejects_arguments_that_are_not_a_json_string( + arguments: dict[str, str] | list[str] | int | bool, +) -> None: + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + message: Final = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": arguments}, + } + ], + } + with pytest.raises(litellm.BadRequestError, match="must be a JSON string"): + convert_to_gemini_tool_call_invoke(message) + + +@pytest.mark.parametrize("arguments", ['["Boston"]', '"Boston"', "5"], ids=["list", "string", "number"]) +def test_convert_to_gemini_tool_call_invoke_rejects_json_text_that_is_not_an_object(arguments: str) -> None: + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + message: Final = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": arguments}, + } + ], + } + with pytest.raises(litellm.BadRequestError, match="must be a JSON object"): + convert_to_gemini_tool_call_invoke(message) + + +@pytest.mark.parametrize( + "arguments", + [{"location": "Boston"}, ["Boston"], '["Boston"]', "not json"], + ids=["object", "list", "json_list_text", "not_json"], +) +@pytest.mark.parametrize("shape", ["tool_calls", "function_call"]) +def test_convert_to_gemini_tool_call_invoke_drops_unusable_arguments_under_drop_params( + arguments: object, shape: str +) -> None: + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + function: Final = {"name": "get_weather", "arguments": arguments} + message: Final = ( + {"role": "assistant", "content": None, "tool_calls": [{"id": "call_1", "type": "function", "function": function}]} + if shape == "tool_calls" + else {"role": "assistant", "content": None, "function_call": function} + ) + parts: Final = convert_to_gemini_tool_call_invoke(message, drop_params=True) + assert parts == [{"function_call": {"name": "get_weather", "args": {}}}] + + @pytest.mark.asyncio async def test_bedrock_process_image_async_factory(): """ diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py index bc8421c80f8..40a44beda59 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -1,4 +1,5 @@ import json +from collections.abc import Awaitable, Callable, Mapping from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -2510,11 +2511,23 @@ def test_check_and_create_cache_clamps_negative_creation_tokens() -> None: assert logging_obj.vertex_ai_cache_creation_input_tokens == 0 -def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler: +def _caches_created_so_far(calls: list[httpx.Request]) -> dict[str, list[dict[str, str]]]: + return { + "cachedContents": [ + {"name": "cachedContents/created", "displayName": json.loads(call.content)["displayName"]} + for call in calls + if call.method == "POST" and call.url.path.endswith("/cachedContents") + ] + } + + +def _recording_gemini_route(calls: list[httpx.Request]) -> Callable[[httpx.Request], httpx.Response]: def route(request: httpx.Request) -> httpx.Response: calls.append(request) if request.url.path.endswith("/cachedContents"): - return httpx.Response(200, json={} if request.method == "GET" else _created_cache_body(3000)) + return httpx.Response( + 200, json=_caches_created_so_far(calls) if request.method == "GET" else _created_cache_body(3000) + ) return httpx.Response( 200, json={ @@ -2523,7 +2536,11 @@ def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler: }, ) - return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(route))) + return route + + +def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler: + return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_recording_gemini_route(calls)))) def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(local_model_cost_map) -> None: @@ -2551,3 +2568,73 @@ def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(l assert created["system_instruction"] == {"parts": [{"text": "You are terse."}]} assert generated["cachedContent"] == "cachedContents/created" assert "system_instruction" not in generated + + +def _cached_conversation_with_tool_arguments(arguments: str) -> list[Mapping[str, object]]: + return [ + { + "role": "user", + "content": [{"type": "text", "text": "cached prefix " * 3000, "cache_control": {"type": "ephemeral"}}], + }, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": arguments}} + ], + "cache_control": {"type": "ephemeral"}, + }, + {"role": "tool", "tool_call_id": "call_1", "content": "Sunny", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": [{"type": "text", "text": "Thanks.", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": "Reply with OK only."}, + ] + + +async def _complete_sync(request: Mapping[str, object], calls: list[httpx.Request]) -> None: + litellm.completion(**request, client=_recording_gemini_client(calls)) + + +async def _complete_async(request: Mapping[str, object], calls: list[httpx.Request]) -> None: + await litellm.acompletion( + **request, client=AsyncHTTPHandler(transport=httpx.MockTransport(_recording_gemini_route(calls))) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("complete", [_complete_sync, _complete_async]) +async def test_cache_creation_drops_tool_call_arguments_gemini_cannot_use_when_drop_params_is_set( + local_model_cost_map: None, complete: Callable[[Mapping[str, object], list[httpx.Request]], Awaitable[None]] +) -> None: + calls: Final[list[httpx.Request]] = [] + await complete( + { + "model": "gemini/gemini-flash-latest", + "messages": _cached_conversation_with_tool_arguments('["Boston"]'), + "api_key": "test-key", + "drop_params": True, + }, + calls, + ) + + created: Final = next( + json.loads(call.content) for call in calls if call.method == "POST" and call.url.path.endswith("/cachedContents") + ) + function_call: Final = created["contents"][1]["parts"][0]["function_call"] + assert (function_call["name"], function_call["args"]) == ("get_weather", {}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("complete", [_complete_sync, _complete_async]) +async def test_a_strict_request_rejects_tool_call_arguments_even_when_a_lenient_request_cached_that_history( + local_model_cost_map: None, complete: Callable[[Mapping[str, object], list[httpx.Request]], Awaitable[None]] +) -> None: + calls: Final[list[httpx.Request]] = [] + request: Final = { + "model": "gemini/gemini-flash-latest", + "messages": _cached_conversation_with_tool_arguments('["Boston"]'), + "api_key": "test-key", + } + await complete({**request, "drop_params": True}, calls) + + with pytest.raises(litellm.BadRequestError, match="get_weather"): + await complete({**request, "drop_params": False}, calls) diff --git a/tests/unit/llms/vertex_ai/gemini/test_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_transformation.py index 946f7c1cc6c..22589dc38e0 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini/test_transformation.py @@ -487,6 +487,74 @@ def test_image_only_user_content_logs_the_blank_text_fixup_at_debug(caplog: pyte assert contents[0]["parts"][-1] == {"text": " "} +@pytest.mark.parametrize("custom_llm_provider, expected_provider", [("gemini", "gemini"), (None, "vertex_ai")]) +def test_tool_call_conversion_error_names_the_calling_provider( + custom_llm_provider: str | None, expected_provider: str +) -> None: + messages: Final = [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": ["Boston"]}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + ] + + with pytest.raises(litellm.BadRequestError) as exc_info: + transformation.gemini_convert_messages_with_history(messages, custom_llm_provider=custom_llm_provider) + + assert exc_info.value.llm_provider == expected_provider + + +def test_tool_call_arguments_gemini_cannot_use_are_dropped_when_drop_params_is_set() -> None: + messages: Final = [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": ["Boston"]}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + ] + + contents: Final = transformation.gemini_convert_messages_with_history( + messages, litellm_params={"drop_params": True}, custom_llm_provider="gemini" + ) + + assert contents[1]["parts"] == [{"function_call": {"name": "get_weather", "args": {}}}] + + +def test_gemini_completion_reports_a_tool_call_it_cannot_convert_as_a_gemini_bad_request() -> None: + with pytest.raises(litellm.BadRequestError) as exc_info: + litellm.completion( + model="gemini/gemini-flash-latest", + api_key="test-key", + messages=[ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": {"location": "Boston"}}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + ], + ) + + assert exc_info.value.llm_provider == "gemini" + assert "must be a JSON string, got dict" in str(exc_info.value) + + _LOOKUP_TOOL: Final = { "type": "function", "function": {