From 3d8d9b23a7af78f615942ff02dd6f1241e3c3ae5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:13:07 +0000 Subject: [PATCH] feat(vertex_ai): add escape hatch to opt out of automatic responseJsonSchema --- litellm/__init__.py | 1 + .../gemini/google_genai/transformation.py | 4 +- litellm/llms/vertex_ai/common_utils.py | 16 +++++ .../vertex_and_google_ai_studio_gemini.py | 4 +- .../test_google_genai_transformation.py | 29 +++++++++ .../vertex_ai/test_vertex_ai_common_utils.py | 65 +++++++++++++++++++ 6 files changed, 115 insertions(+), 4 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 3f8c742c5a2..b359f3c1399 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -551,6 +551,7 @@ organization = None project = None config_path = None vertex_ai_safety_settings: Optional[dict] = None +vertex_ai_use_response_json_schema: Optional[bool] = None ####### COMPLETION MODELS ################### from typing import Set diff --git a/litellm/llms/gemini/google_genai/transformation.py b/litellm/llms/gemini/google_genai/transformation.py index 68f30308621..d441d8b179b 100644 --- a/litellm/llms/gemini/google_genai/transformation.py +++ b/litellm/llms/gemini/google_genai/transformation.py @@ -14,7 +14,7 @@ from litellm.llms.base_llm.google_genai.transformation import ( ) from litellm.llms.vertex_ai.common_utils import ( _build_vertex_schema, - supports_response_json_schema, + should_use_response_json_schema, ) from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM from litellm.types.router import GenericLiteLLMParams @@ -317,7 +317,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): if not isinstance(value, dict): return - if supports_response_json_schema(model): + if should_use_response_json_schema(model): if json_schema_key is not None: generate_content_config_dict.pop(schema_key) return diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 7dcb4dcf2e8..27f161ee42e 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -270,6 +270,22 @@ def supports_response_json_schema(model: str) -> bool: return bool(gemini_2_plus_pattern.search(model_lower)) +def should_use_response_json_schema(model: str) -> bool: + """ + Resolve which Vertex/Gemini structured-output channel to use for a json_schema response_format. + + Defaults to responseJsonSchema for models that support it, but `litellm.vertex_ai_use_response_json_schema` + (e.g. `litellm_settings: vertex_ai_use_response_json_schema: false` on the proxy) forces the channel: + `False` sends the natively converted `responseSchema` (flattened nullable unions, hoisted constraints, + `propertyOrdering`), `True` sends the client schema verbatim as `responseJsonSchema` + """ + override = litellm.vertex_ai_use_response_json_schema + if override is not None: + return override + + return supports_response_json_schema(model) + + from typing import Literal, Optional all_gemini_url_modes = Literal["chat", "embedding", "batch_embedding", "image_generation", "count_tokens"] diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 8dd0dc19b81..ae7a90a776d 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -99,7 +99,7 @@ from ..common_utils import ( VertexAIError, _build_json_schema, _build_vertex_schema, - supports_response_json_schema, + should_use_response_json_schema, ) from ..vertex_llm_base import VertexBase from .transformation import ( @@ -779,7 +779,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # Automatically use responseJsonSchema for Gemini 2.0+ models # responseJsonSchema uses standard JSON Schema format and supports additionalProperties # For older models (Gemini 1.5), fall back to responseSchema (OpenAPI format) - use_json_schema = supports_response_json_schema(model) + use_json_schema = should_use_response_json_schema(model) if not use_json_schema: # For responseSchema, remove 'additionalProperties' (not supported) diff --git a/tests/test_litellm/google_genai/test_google_genai_transformation.py b/tests/test_litellm/google_genai/test_google_genai_transformation.py index 6b0cd500a82..022c525b1d5 100644 --- a/tests/test_litellm/google_genai/test_google_genai_transformation.py +++ b/tests/test_litellm/google_genai/test_google_genai_transformation.py @@ -426,6 +426,35 @@ def test_transform_generate_content_request_flattens_response_schema_1_5(): assert items["properties"]["title"]["type"].lower() == "string" +def test_transform_generate_content_request_response_json_schema_opt_out(monkeypatch): + """With ``litellm.vertex_ai_use_response_json_schema = False``, a Gemini 2.x request keeps the + natively converted ``responseSchema`` instead of being promoted to ``responseJsonSchema``.""" + import litellm + + monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False) + config = GoogleGenAIConfig() + + schema = { + "type": "object", + "properties": {"barcode": {"anyOf": [{"type": "string"}, {"type": "null"}]}}, + "required": ["barcode"], + } + + result = config.transform_generate_content_request( + model="gemini-2.5-flash", + contents=[{"role": "user", "parts": [{"text": "hi"}]}], + tools=None, + generate_content_config_dict={"responseSchema": schema}, + system_instruction=None, + ) + + gen_config = result["generationConfig"] + assert "responseJsonSchema" not in gen_config + normalized = gen_config["responseSchema"] + assert normalized["propertyOrdering"] == ["barcode"] + assert normalized["properties"]["barcode"]["anyOf"] == [{"type": "string", "nullable": True}] + + def test_transform_generate_content_request_passes_through_response_json_schema(): """If the caller already used ``responseJsonSchema``, it should be preserved (Gemini 2.0+ accepts standard JSON Schema as-is).""" diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index b83d4742b64..f225714a8ec 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -1,9 +1,12 @@ import os import sys +from copy import deepcopy +from typing import Optional from unittest.mock import patch import pytest +import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH sys.path.insert( @@ -17,6 +20,7 @@ from litellm.llms.vertex_ai.common_utils import ( get_vertex_project_id_from_url, pop_vertex_request_labels, set_schema_property_ordering, + should_use_response_json_schema, supports_response_json_schema, validate_vertex_location, vertex_request_labels_from_litellm_params, @@ -183,6 +187,67 @@ def test_supports_response_json_schema(model: str, expected: bool): assert supports_response_json_schema(model) == expected +@pytest.mark.parametrize( + "override, model, expected", + [ + (None, "gemini-2.5-flash", True), + (None, "gemini-1.5-pro", False), + (False, "gemini-2.5-flash", False), + (True, "gemini-1.5-pro", True), + ], +) +def test_should_use_response_json_schema_honors_override( + monkeypatch: pytest.MonkeyPatch, override: Optional[bool], model: str, expected: bool +): + """`litellm.vertex_ai_use_response_json_schema` overrides the model-name heuristic; None keeps it""" + monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", override) + + assert should_use_response_json_schema(model) == expected + + +def test_response_json_schema_opt_out_uses_native_response_schema(monkeypatch: pytest.MonkeyPatch): + """Opting out sends the natively converted `responseSchema` (flattened nullable union, propertyOrdering) + instead of the verbatim `responseJsonSchema`, for a model that would otherwise get the json schema path""" + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + response_format = { + "type": "json_schema", + "json_schema": { + "name": "invoice", + "schema": { + "type": "object", + "properties": { + "total": {"type": "number"}, + "barcode": {"anyOf": [{"type": "string"}, {"type": "null"}]}, + }, + "required": ["total", "barcode"], + }, + }, + } + + monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False) + opted_out: dict = {} + VertexGeminiConfig().apply_response_schema_transformation( + value=deepcopy(response_format), optional_params=opted_out, model="gemini-2.5-flash" + ) + + assert "response_json_schema" not in opted_out + native_schema = opted_out["response_schema"] + assert native_schema["propertyOrdering"] == ["total", "barcode"] + assert native_schema["properties"]["barcode"]["anyOf"] == [{"type": "string", "nullable": True}] + + monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", None) + default: dict = {} + VertexGeminiConfig().apply_response_schema_transformation( + value=deepcopy(response_format), optional_params=default, model="gemini-2.5-flash" + ) + + assert "response_schema" not in default + assert default["response_json_schema"] == response_format["json_schema"]["schema"] + + def test_set_schema_property_ordering_with_excessive_nesting(): """Test set_schema_property_ordering with excessive nesting > max levels +1 deep.""" # generate a schema with excessive nesting