diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 0aed2956b0a..3bc8dc4e23e 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1490,6 +1490,7 @@ def convert_to_gemini_tool_call_result( message: ChatCompletionToolMessage | ChatCompletionFunctionMessage, last_message_with_tool_calls: dict | None, forward_function_call_id: bool = False, + supports_multimodal_function_responses: bool = True, ) -> VertexPartType | list[VertexPartType]: """ OpenAI message with a tool result looks like: @@ -1652,9 +1653,9 @@ def convert_to_gemini_tool_call_result( _part: Final[VertexPartType] = {"function_response": _function_response} - # For multimodal function responses, Gemini expects media parts nested - # inside functionResponse.parts instead of sibling content parts. if inline_data_list: + if not supports_multimodal_function_responses: + return [_part, *(VertexPartType(inline_data=inline_data) for inline_data in inline_data_list)] _function_response["parts"] = [{"inline_data": inline_data} for inline_data in inline_data_list] return [_part] diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 71bb05ee155..ed86934db63 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -726,6 +726,9 @@ def gemini_convert_messages_with_history( from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig forward_function_call_id: Final = VertexGeminiConfig.forward_gemini_function_call_id(model or "") + supports_multimodal_function_responses: Final = model is None or ( + VertexGeminiConfig.supports_multimodal_function_responses(model) + ) try: while msg_i < len(messages): @@ -1029,6 +1032,7 @@ def gemini_convert_messages_with_history( messages[msg_i], last_message_with_tool_calls, forward_function_call_id=forward_function_call_id, + supports_multimodal_function_responses=supports_multimodal_function_responses, ) msg_i += 1 # Handle both single part and list of parts (for Computer Use with images) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index d788ccb0f1d..9acbc9c459c 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -689,6 +689,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) -> bool: return cls._forward_gemini_function_call_id(model) + @classmethod + def supports_multimodal_function_responses(cls, model: str) -> bool: + return cls.is_gemini_3_or_newer(model) + def _supports_penalty_parameters(self, model: str) -> bool: # Gemini 3 models do not support penalty parameters if VertexGeminiConfig._is_gemini_3_or_newer(model): diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index b97724319c7..75449eacfd3 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -2,7 +2,7 @@ import base64 from collections.abc import Sequence from itertools import chain from pathlib import Path -from typing import Final +from typing import Final, Literal import httpx import pytest @@ -1650,10 +1650,28 @@ def test_convert_tool_response_with_base64_image(): assert inline_data["data"] == test_image_base64 -def test_gemini_history_nests_multimodal_tool_response_parts(): - """Full history conversion should not emit sibling inline_data tool result parts.""" - test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" - messages = [ +@pytest.mark.parametrize("provider", ["gemini", "vertex_ai", "vertex_ai_beta"]) +@pytest.mark.parametrize( + "model,nested", + [ + (None, True), + ("gemini-2.5-pro", False), + ("gemini/gemini-2.5-flash", False), + ("vertex_ai/gemini-2.5-flash-lite", False), + ("123456789", False), + ("gemini-3.8-flash", True), + ("gemini/gemini-3.8-flash", True), + ("vertex_ai/gemini-3.8-flash", True), + ("gemini-flash-latest", True), + ], +) +def test_gemini_history_places_tool_media_for_model( + model: str | None, + nested: bool, + provider: Literal["gemini", "vertex_ai", "vertex_ai_beta"], +) -> None: + test_image_base64: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + messages: Final = [ {"role": "user", "content": "Get me an image"}, { "role": "assistant", @@ -1679,24 +1697,53 @@ def test_gemini_history_nests_multimodal_tool_response_parts(): "data": test_image_base64, }, }, + { + "type": "file", + "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQ="}, + }, ], }, + {"role": "user", "content": "Describe the image and file again"}, ] - contents = gemini_convert_messages_with_history(messages=messages) - - tool_response_parts = contents[-1]["parts"] - assert len(tool_response_parts) == 1 - assert "inline_data" not in tool_response_parts[0] - function_response = tool_response_parts[0]["function_response"] - assert function_response["parts"] == [ + contents: Final = ( + gemini_convert_messages_with_history(messages=messages) + if model is None + else transform_request_body( + messages=messages, + model=model, + custom_llm_provider=provider, + optional_params={}, + litellm_params={}, + cached_content=None, + )["contents"] + ) + media_parts: Final = [ { "inline_data": { "data": test_image_base64, "mime_type": "image/png", } - } + }, + {"inline_data": {"data": "JVBERi0xLjQ=", "mime_type": "application/pdf"}}, ] + function_response: Final = { + "name": "get_image", + "response": {"image_ref": "inline"}, + **({"id": "call_get_image"} if model is not None and nested else {}), + **({"parts": media_parts} if nested else {}), + } + assert contents[-2] == { + "role": "user", + "parts": [ + {"function_response": function_response}, + *([] if nested else media_parts), + ], + } + assert contents[-1] == { + "role": "user", + "parts": [{"text": "Describe the image and file again"}], + } def test_convert_tool_response_text_only():