From 89f6bf9287bb3adae8f322340f3e89761da8b1c2 Mon Sep 17 00:00:00 2001 From: tjongchiewch Date: Tue, 2 Jun 2026 02:11:17 +0700 Subject: [PATCH] feat(gemini): use responseFormat for googleMaps + structured output The Gemini API rejects googleMaps tool combined with responseMimeType: 'application/json' (400 error). This converts to the newer responseFormat structure which supports googleMaps + structured JSON output with full grounding metadata. Only applies to the gemini/ provider path (generativelanguage.googleapis.com), not vertex_ai/ which does not support the responseFormat field. --- .../gemini/google_genai/transformation.py | 61 +++++++ .../llms/vertex_ai/gemini/transformation.py | 28 ++++ litellm/types/llms/vertex_ai.py | 1 + .../test_google_genai_transformation.py | 155 ++++++++++++++++++ 4 files changed, 245 insertions(+) diff --git a/litellm/llms/gemini/google_genai/transformation.py b/litellm/llms/gemini/google_genai/transformation.py index ee201af7e1a..0692a092f43 100644 --- a/litellm/llms/gemini/google_genai/transformation.py +++ b/litellm/llms/gemini/google_genai/transformation.py @@ -95,6 +95,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): "automatic_function_calling", "thinking_config", "image_config", + "response_format", ] def map_generate_content_optional_params( @@ -353,6 +354,63 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): parameters=deepcopy(value), add_property_ordering=True ) + @staticmethod + def _has_google_maps_tool(tools: Optional[Any]) -> bool: + """Check if googleMaps is present in the tools list.""" + from litellm.llms.vertex_ai.gemini.transformation import _has_google_maps_tool + + return _has_google_maps_tool(tools) + + @staticmethod + def _use_response_format_for_google_maps( + generate_content_config_dict: Dict, + ) -> None: + """ + Convert responseMimeType + responseJsonSchema/responseSchema to the newer + responseFormat structure when googleMaps tool is present. + + The Gemini API rejects googleMaps + responseMimeType: 'application/json' + with error "Google Maps tool with a response mime type: 'application/json' + is unsupported". The newer responseFormat structure works correctly. + """ + mime_key = next( + ( + k + for k in ("responseMimeType", "response_mime_type") + if k in generate_content_config_dict + ), + None, + ) + if mime_key is None: + return + + mime_value = generate_content_config_dict[mime_key] + if mime_value != "application/json": + return + + schema_key = next( + ( + k + for k in ( + "responseJsonSchema", + "response_json_schema", + "responseSchema", + "response_schema", + ) + if k in generate_content_config_dict + ), + None, + ) + + schema = generate_content_config_dict.pop(schema_key) if schema_key else None + generate_content_config_dict.pop(mime_key) + + response_format: Dict[str, Any] = {"text": {"mimeType": "APPLICATION_JSON"}} + if schema is not None: + response_format["text"]["schema"] = schema + + generate_content_config_dict["responseFormat"] = response_format + def transform_generate_content_request( self, model: str, @@ -368,6 +426,9 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): self._normalize_response_schema(generate_content_config_dict, model) + if self._has_google_maps_tool(tools): + self._use_response_format_for_google_maps(generate_content_config_dict) + typed_generate_content_request = GenerateContentRequestDict( model=model, contents=contents, diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 4f5846cc5b6..9e053cc5cac 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -1109,6 +1109,17 @@ def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None: data_dict[k] = v +def _has_google_maps_tool(tools: Optional[Any]) -> bool: + """Check if googleMaps is present in the tools list.""" + if not tools: + return False + if isinstance(tools, list): + for tool in tools: + if isinstance(tool, dict) and "googleMaps" in tool: + return True + return False + + def _transform_request_body( # noqa: PLR0915 messages: List[AllMessageValues], model: str, @@ -1197,6 +1208,23 @@ def _transform_request_body( # noqa: PLR0915 "level" ] + # Gemini API rejects googleMaps + responseMimeType: 'application/json'. + # Convert to the newer responseFormat structure which supports this combination. + if ( + custom_llm_provider == "gemini" + and generation_config is not None + and _has_google_maps_tool(tools) + and generation_config.get("response_mime_type") == "application/json" + ): + schema = generation_config.pop("response_json_schema", None) + if schema is None: + schema = generation_config.pop("response_schema", None) + generation_config.pop("response_mime_type", None) + response_format_value: dict = {"text": {"mimeType": "APPLICATION_JSON"}} + if schema is not None: + response_format_value["text"]["schema"] = schema + generation_config["responseFormat"] = response_format_value + data = RequestBody(contents=content) # Vertex rejects system_instruction/tools/toolConfig alongside cachedContent. # Treat dropping these fields as a request mutation guarded by modify_params. diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index a1d53978761..9a237009a1c 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -240,6 +240,7 @@ class GenerationConfig(TypedDict, total=False): response_mime_type: Literal["text/plain", "application/json"] response_schema: dict response_json_schema: dict + responseFormat: dict seed: int responseLogprobs: bool logprobs: int diff --git a/tests/test_litellm/google_genai/test_google_genai_transformation.py b/tests/test_litellm/google_genai/test_google_genai_transformation.py index 6b0cd500a82..feaa9cc819a 100644 --- a/tests/test_litellm/google_genai/test_google_genai_transformation.py +++ b/tests/test_litellm/google_genai/test_google_genai_transformation.py @@ -2,6 +2,7 @@ """ Test to verify the Google GenAI transformation logic for generateContent parameters """ + import os import sys @@ -644,3 +645,157 @@ def test_validate_environment_with_extra_headers(): assert "X-Custom-Header" in result, "Extra headers should be merged" assert result["X-Custom-Header"] == "custom-value" assert "Content-Type" in result + + +def test_google_maps_tool_uses_response_format(): + """ + Test that when googleMaps is in tools and structured output is requested, + the config uses responseFormat instead of responseMimeType + responseJsonSchema. + + The Gemini API rejects googleMaps + responseMimeType: 'application/json' with: + "Google Maps tool with a response mime type: 'application/json' is unsupported" + """ + config = GoogleGenAIConfig() + + generate_content_config_dict = { + "responseMimeType": "application/json", + "responseJsonSchema": { + "type": "object", + "properties": { + "places": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "cid": {"type": "number"}, + "rating": {"type": "string"}, + }, + "required": ["title", "rating", "cid"], + }, + } + }, + "required": ["places"], + }, + "temperature": 1.0, + } + + tools = [{"googleMaps": {}}] + + result = config.transform_generate_content_request( + model="gemini/gemini-3.1-flash-lite", + contents=[{"role": "user", "parts": [{"text": "Find restaurants"}]}], + tools=tools, + generate_content_config_dict=generate_content_config_dict, + ) + + gen_config = result["generationConfig"] + + assert "responseMimeType" not in gen_config + assert "responseJsonSchema" not in gen_config + assert "responseFormat" in gen_config + assert gen_config["responseFormat"]["text"]["mimeType"] == "APPLICATION_JSON" + assert "schema" in gen_config["responseFormat"]["text"] + assert gen_config["responseFormat"]["text"]["schema"]["type"] == "object" + assert gen_config["temperature"] == 1.0 + + +def test_google_maps_tool_with_enable_widget_uses_response_format(): + """Test responseFormat is used when googleMaps has enableWidget: true.""" + config = GoogleGenAIConfig() + + generate_content_config_dict = { + "responseMimeType": "application/json", + "responseJsonSchema": { + "type": "object", + "properties": {"places": {"type": "array"}}, + }, + } + + tools = [{"googleMaps": {"enableWidget": True}}] + + result = config.transform_generate_content_request( + model="gemini/gemini-3.1-flash-lite", + contents=[{"role": "user", "parts": [{"text": "Find restaurants"}]}], + tools=tools, + generate_content_config_dict=generate_content_config_dict, + ) + + gen_config = result["generationConfig"] + assert "responseMimeType" not in gen_config + assert "responseFormat" in gen_config + assert gen_config["responseFormat"]["text"]["mimeType"] == "APPLICATION_JSON" + + +def test_no_google_maps_keeps_response_mime_type(): + """Test that without googleMaps, responseMimeType is preserved unchanged.""" + config = GoogleGenAIConfig() + + generate_content_config_dict = { + "responseMimeType": "application/json", + "responseJsonSchema": { + "type": "object", + "properties": {"name": {"type": "string"}}, + }, + "temperature": 0.5, + } + + tools = [{"googleSearch": {}}] + + result = config.transform_generate_content_request( + model="gemini/gemini-3.1-flash-lite", + contents=[{"role": "user", "parts": [{"text": "Search something"}]}], + tools=tools, + generate_content_config_dict=generate_content_config_dict, + ) + + gen_config = result["generationConfig"] + assert "responseMimeType" in gen_config + assert gen_config["responseMimeType"] == "application/json" + assert "responseFormat" not in gen_config + + +def test_google_maps_without_structured_output_unchanged(): + """Test that googleMaps without response schema doesn't add responseFormat.""" + config = GoogleGenAIConfig() + + generate_content_config_dict = { + "temperature": 1.0, + } + + tools = [{"googleMaps": {}}] + + result = config.transform_generate_content_request( + model="gemini/gemini-3.1-flash-lite", + contents=[{"role": "user", "parts": [{"text": "Find restaurants"}]}], + tools=tools, + generate_content_config_dict=generate_content_config_dict, + ) + + gen_config = result["generationConfig"] + assert "responseFormat" not in gen_config + assert "responseMimeType" not in gen_config + assert gen_config["temperature"] == 1.0 + + +def test_google_maps_with_text_plain_mime_type_unchanged(): + """Test that googleMaps + text/plain responseMimeType is not converted.""" + config = GoogleGenAIConfig() + + generate_content_config_dict = { + "responseMimeType": "text/plain", + "temperature": 1.0, + } + + tools = [{"googleMaps": {}}] + + result = config.transform_generate_content_request( + model="gemini/gemini-3.1-flash-lite", + contents=[{"role": "user", "parts": [{"text": "Find restaurants"}]}], + tools=tools, + generate_content_config_dict=generate_content_config_dict, + ) + + gen_config = result["generationConfig"] + assert gen_config["responseMimeType"] == "text/plain" + assert "responseFormat" not in gen_config