fix(gemini): preserve tool media for older models (#48428)

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1490,6 +1490,7 @@ def convert_to_gemini_tool_call_result(
     message: ChatCompletionToolMessage | ChatCompletionFunctionMessage,
     last_message_with_tool_calls: dict | None,
     forward_function_call_id: bool = False,
+    supports_multimodal_function_responses: bool = True,
 ) -> VertexPartType | list[VertexPartType]:
     """
     OpenAI message with a tool result looks like:
@@ -1652,9 +1653,9 @@ def convert_to_gemini_tool_call_result(
 
     _part: Final[VertexPartType] = {"function_response": _function_response}
 
-    # For multimodal function responses, Gemini expects media parts nested
-    # inside functionResponse.parts instead of sibling content parts.
     if inline_data_list:
+        if not supports_multimodal_function_responses:
+            return [_part, *(VertexPartType(inline_data=inline_data) for inline_data in inline_data_list)]
         _function_response["parts"] = [{"inline_data": inline_data} for inline_data in inline_data_list]
         return [_part]
 

litellm/llms/vertex_ai/gemini/transformation.py

@@ -726,6 +726,9 @@ def gemini_convert_messages_with_history(
     from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
 
     forward_function_call_id: Final = VertexGeminiConfig.forward_gemini_function_call_id(model or "")
+    supports_multimodal_function_responses: Final = model is None or (
+        VertexGeminiConfig.supports_multimodal_function_responses(model)
+    )
 
     try:
         while msg_i < len(messages):
@@ -1029,6 +1032,7 @@ def gemini_convert_messages_with_history(
                     messages[msg_i],
                     last_message_with_tool_calls,
                     forward_function_call_id=forward_function_call_id,
+                    supports_multimodal_function_responses=supports_multimodal_function_responses,
                 )
                 msg_i += 1
                 # Handle both single part and list of parts (for Computer Use with images)

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -689,6 +689,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
     ) -> bool:
         return cls._forward_gemini_function_call_id(model)
 
+    @classmethod
+    def supports_multimodal_function_responses(cls, model: str) -> bool:
+        return cls.is_gemini_3_or_newer(model)
+
     def _supports_penalty_parameters(self, model: str) -> bool:
         # Gemini 3 models do not support penalty parameters
         if VertexGeminiConfig._is_gemini_3_or_newer(model):

tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py

@@ -2,7 +2,7 @@ import base64
 from collections.abc import Sequence
 from itertools import chain
 from pathlib import Path
-from typing import Final
+from typing import Final, Literal
 
 import httpx
 import pytest
@@ -1650,10 +1650,28 @@ def test_convert_tool_response_with_base64_image():
     assert inline_data["data"] == test_image_base64
 
 
-def test_gemini_history_nests_multimodal_tool_response_parts():
-    """Full history conversion should not emit sibling inline_data tool result parts."""
-    test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
-    messages = [
+@pytest.mark.parametrize("provider", ["gemini", "vertex_ai", "vertex_ai_beta"])
+@pytest.mark.parametrize(
+    "model,nested",
+    [
+        (None, True),
+        ("gemini-2.5-pro", False),
+        ("gemini/gemini-2.5-flash", False),
+        ("vertex_ai/gemini-2.5-flash-lite", False),
+        ("123456789", False),
+        ("gemini-3.8-flash", True),
+        ("gemini/gemini-3.8-flash", True),
+        ("vertex_ai/gemini-3.8-flash", True),
+        ("gemini-flash-latest", True),
+    ],
+)
+def test_gemini_history_places_tool_media_for_model(
+    model: str | None,
+    nested: bool,
+    provider: Literal["gemini", "vertex_ai", "vertex_ai_beta"],
+) -> None:
+    test_image_base64: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+    messages: Final = [
         {"role": "user", "content": "Get me an image"},
         {
             "role": "assistant",
@@ -1679,24 +1697,53 @@ def test_gemini_history_nests_multimodal_tool_response_parts():
                         "data": test_image_base64,
                     },
                 },
+                {
+                    "type": "file",
+                    "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQ="},
+                },
             ],
         },
+        {"role": "user", "content": "Describe the image and file again"},
     ]
 
-    contents = gemini_convert_messages_with_history(messages=messages)
-
-    tool_response_parts = contents[-1]["parts"]
-    assert len(tool_response_parts) == 1
-    assert "inline_data" not in tool_response_parts[0]
-    function_response = tool_response_parts[0]["function_response"]
-    assert function_response["parts"] == [
+    contents: Final = (
+        gemini_convert_messages_with_history(messages=messages)
+        if model is None
+        else transform_request_body(
+            messages=messages,
+            model=model,
+            custom_llm_provider=provider,
+            optional_params={},
+            litellm_params={},
+            cached_content=None,
+        )["contents"]
+    )
+    media_parts: Final = [
         {
             "inline_data": {
                 "data": test_image_base64,
                 "mime_type": "image/png",
             }
-        }
+        },
+        {"inline_data": {"data": "JVBERi0xLjQ=", "mime_type": "application/pdf"}},
     ]
+    function_response: Final = {
+        "name": "get_image",
+        "response": {"image_ref": "inline"},
+        **({"id": "call_get_image"} if model is not None and nested else {}),
+        **({"parts": media_parts} if nested else {}),
+    }
+    assert contents[-2] == {
+        "role": "user",
+        "parts": [
+            {"function_response": function_response},
+            *([] if nested else media_parts),
+        ],
+    }
+    assert contents[-1] == {
+        "role": "user",
+        "parts": [{"text": "Describe the image and file again"}],
+    }
 
 
 def test_convert_tool_response_text_only():