From 83e3458f8a5d3ef3e01965e8bf4f457fcf6896b9 Mon Sep 17 00:00:00 2001 From: ArthurAAM <100235777+ArthurAAM@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:15:04 -0300 Subject: [PATCH 1/5] feat(vertex_ai): make the Gemini structured output channel selectable Gemini 2.x+ requests always went out on responseJsonSchema, with no setting, config or request param able to reach Vertex's native responseSchema, which flattens nullable unions, keeps the constraints inside them and orders properties. litellm.vertex_ai_use_response_json_schema now picks the channel, vertex_ai_use_response_json_schema on a request or on a deployment's litellm_params beats it, and the model name heuristic stays the default, so nothing changes for anyone who sets neither. Native generateContent requests read the same setting, including the ones that carry only a JSON Schema, which the old check skipped. --- litellm/__init__.py | 1 + .../gemini/google_genai/transformation.py | 36 ++++--- litellm/llms/vertex_ai/common_utils.py | 83 ++++++++++++++++ .../llms/vertex_ai/gemini/transformation.py | 15 ++- .../vertex_and_google_ai_studio_gemini.py | 17 ++-- .../test_google_genai_transformation.py | 37 +++++++ .../vertex_ai/gemini/test_transformation.py | 93 ++++++++++++++++++ ...test_vertex_and_google_ai_studio_gemini.py | 43 +++++++- .../vertex_ai/test_vertex_ai_common_utils.py | 97 +++++++++++++++++++ 9 files changed, 396 insertions(+), 26 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index ee2c551481c..76df0f4eb6e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -564,6 +564,7 @@ organization = None project = None config_path = None vertex_ai_safety_settings: Optional[dict] = None +vertex_ai_use_response_json_schema: Optional[bool] = None ####### COMPLETION MODELS ################### from typing import Set diff --git a/litellm/llms/gemini/google_genai/transformation.py b/litellm/llms/gemini/google_genai/transformation.py index d220742b92b..e53dcb4fd24 100644 --- a/litellm/llms/gemini/google_genai/transformation.py +++ b/litellm/llms/gemini/google_genai/transformation.py @@ -14,7 +14,7 @@ from litellm.llms.base_llm.google_genai.transformation import ( ) from litellm.llms.vertex_ai.common_utils import ( _build_vertex_schema, - supports_response_json_schema, + should_use_response_json_schema, ) from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM from litellm.types.router import GenericLiteLLMParams @@ -310,26 +310,34 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): None, ) - if schema_key is None: + source_key: Final = schema_key if schema_key is not None else json_schema_key + if source_key is None: return - value: Final = generate_content_config_dict[schema_key] + value: Final = generate_content_config_dict[source_key] if not isinstance(value, dict): return - if supports_response_json_schema(model): + if should_use_response_json_schema(model): if json_schema_key is not None: - generate_content_config_dict.pop(schema_key) + if schema_key is not None: + generate_content_config_dict.pop(schema_key) return - generate_content_config_dict.pop(schema_key) - new_json_schema_key = "response_json_schema" if schema_key == "response_schema" else "responseJsonSchema" - generate_content_config_dict[new_json_schema_key] = value - else: - if json_schema_key is not None: - generate_content_config_dict.pop(json_schema_key) - generate_content_config_dict[schema_key] = _build_vertex_schema( - parameters=deepcopy(value), add_property_ordering=True - ) + generate_content_config_dict.pop(source_key) + json_target_key: Final = "response_json_schema" if source_key == "response_schema" else "responseJsonSchema" + generate_content_config_dict[json_target_key] = value + return + + if json_schema_key is not None: + generate_content_config_dict.pop(json_schema_key) + native_target_key: Final = ( + source_key + if schema_key is not None + else ("response_schema" if source_key == "response_json_schema" else "responseSchema") + ) + generate_content_config_dict[native_target_key] = _build_vertex_schema( + parameters=deepcopy(value), add_property_ordering=True + ) def transform_generate_content_request( self, diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 1de2337d8eb..03f28b1fb66 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1,6 +1,8 @@ import re +from collections.abc import Mapping from copy import deepcopy from enum import Enum +from types import MappingProxyType from typing import Any, Final, Literal, get_type_hints import httpx @@ -269,6 +271,27 @@ def supports_response_json_schema(model: str) -> bool: return bool(gemini_2_plus_pattern.search(model_lower)) +VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM: Final = "vertex_ai_use_response_json_schema" +VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM: Final = "litellm_param_vertex_ai_verbatim_response_schema" + + +def should_use_response_json_schema(model: str, request_override: bool | None = None) -> bool: + """ + Resolve which structured output channel a json_schema response_format goes to. + + True sends the client schema verbatim as ``responseJsonSchema``, False sends the + natively converted ``responseSchema`` (nullable unions flattened, constraints + hoisted, ``propertyOrdering`` added). Precedence: per request + ``vertex_ai_use_response_json_schema``, then + ``litellm.vertex_ai_use_response_json_schema``, then the model heuristic + """ + if request_override is not None: + return request_override + if litellm.vertex_ai_use_response_json_schema is not None: + return litellm.vertex_ai_use_response_json_schema + return supports_response_json_schema(model) + + from typing import Literal all_gemini_url_modes = Literal["chat", "embedding", "batch_embedding", "image_generation", "count_tokens"] @@ -651,6 +674,66 @@ def _build_json_schema(parameters: dict) -> dict: return parameters +def _response_json_schema_override( + optional_params: Mapping[str, object], litellm_params: Mapping[str, object] +) -> bool | None: + candidates: Final = ( + optional_params.get(VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM), + litellm_params.get(VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM), + ) + return next((candidate for candidate in candidates if isinstance(candidate, bool)), None) + + +def _swap_response_schema_key( + optional_params: Mapping[str, object], dropped_key: str, added_key: str, schema: Mapping[str, object] +) -> Mapping[str, object]: + surviving: Final = MappingProxyType({k: v for k, v in optional_params.items() if k != dropped_key}) + return MappingProxyType({**surviving, added_key: schema}) + + +def resolve_response_schema_channel( + optional_params: Mapping[str, object], litellm_params: Mapping[str, object], model: str +) -> Mapping[str, object]: + """ + Move an already mapped response schema onto the channel this request asks for. + + ``vertex_ai_use_response_json_schema`` on the request or on the deployment's + ``litellm_params`` beats ``litellm.vertex_ai_use_response_json_schema`` and the model + heuristic, and only the request build sees both, so the mapped channel is settled here + """ + override: Final = _response_json_schema_override(optional_params, litellm_params) + if override is None: + return optional_params + + if should_use_response_json_schema(model, override): + if "response_json_schema" in optional_params: + return optional_params + verbatim_schema: Final = optional_params.get(VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM) or litellm_params.get( + VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM + ) + if isinstance(verbatim_schema, dict): + return _swap_response_schema_key( + optional_params, "response_schema", "response_json_schema", verbatim_schema + ) + if "response_schema" in optional_params: + verbose_logger.warning( + "vertex_ai_use_response_json_schema=True is ignored for model=%s, whose schema was already " + "converted for responseSchema. Set litellm.vertex_ai_use_response_json_schema instead", + model, + ) + return optional_params + + json_schema: Final = optional_params.get("response_json_schema") + if not isinstance(json_schema, dict): + return optional_params + return _swap_response_schema_key( + optional_params, + "response_json_schema", + "response_schema", + _build_vertex_schema(parameters=deepcopy(json_schema), add_property_ordering=True), + ) + + def _filter_anyof_fields(schema_dict: dict[str, Any]) -> dict[str, Any]: """ When anyof is present, only keep the anyof field and its contents - otherwise VertexAI will throw an error - https://github.com/BerriAI/litellm/issues/11164 diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 11c026010ee..695e5908fdd 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -28,7 +28,11 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( response_schema_prompt, ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels +from litellm.llms.vertex_ai.common_utils import ( + VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM, + pop_vertex_request_labels, + resolve_response_schema_channel, +) from litellm.types.files import ( get_file_mime_type_for_file_type, get_file_type_from_extension, @@ -1178,7 +1182,14 @@ def _transform_request_body( litellm_params.update({k: v}) remove_keys.append(k) - optional_params = {k: v for k, v in optional_params.items() if k not in remove_keys} + resolved_params: Final = resolve_response_schema_channel( + optional_params=optional_params, litellm_params=litellm_params, model=model + ) + optional_params = { + k: v + for k, v in resolved_params.items() + if k not in remove_keys and k != VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM + } try: if custom_llm_provider == "gemini": diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index d298670aa7a..af1fe84ceab 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -82,9 +82,11 @@ from litellm.utils import ( from ....utils import _remove_additional_properties, _remove_strict_from_schema from ..common_utils import ( + VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM, VertexAIError, _build_json_schema, _build_vertex_schema, + should_use_response_json_schema, supports_response_json_schema, ) from ..vertex_llm_base import VertexBase @@ -754,14 +756,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # remove 'strict' from json schema (not supported by Gemini) new_value = _remove_strict_from_schema(new_value) - # Automatically use responseJsonSchema for Gemini 2.0+ models - # responseJsonSchema uses standard JSON Schema format and supports additionalProperties - # For older models (Gemini 1.5), fall back to responseSchema (OpenAPI format) - use_json_schema: Final = supports_response_json_schema(model) - - if not use_json_schema: - # For responseSchema, remove 'additionalProperties' (not supported) - new_value = _remove_additional_properties(new_value) + use_json_schema: Final = should_use_response_json_schema(model) # Handle response type if new_value.get("type") == "json_object": @@ -791,7 +786,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # - OpenAPI-style format (uppercase types) # - No additionalProperties support # - Requires propertyOrdering - optional_params["response_schema"] = self._map_response_schema(value=schema) + if supports_response_json_schema(model): + optional_params[VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM] = deepcopy(schema) + optional_params["response_schema"] = self._map_response_schema( + value=_remove_additional_properties(schema) + ) @staticmethod def _map_reasoning_effort_to_thinking_budget( diff --git a/tests/test_litellm/google_genai/test_google_genai_transformation.py b/tests/test_litellm/google_genai/test_google_genai_transformation.py index f0d0fc6126d..84bb8407e98 100644 --- a/tests/test_litellm/google_genai/test_google_genai_transformation.py +++ b/tests/test_litellm/google_genai/test_google_genai_transformation.py @@ -6,6 +6,7 @@ Test to verify the Google GenAI transformation logic for generateContent paramet import pytest +import litellm from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, @@ -445,6 +446,42 @@ def test_transform_generate_content_request_passes_through_response_json_schema( assert "responseSchema" not in gen_config +@pytest.mark.parametrize( + "json_schema_key, expected_schema_key", + [("responseJsonSchema", "responseSchema"), ("response_json_schema", "response_schema")], +) +def test_transform_generate_content_request_opt_out_converts_response_json_schema( + monkeypatch, json_schema_key, expected_schema_key +): + """ + litellm.vertex_ai_use_response_json_schema=False also reaches generateContent requests that + only carry a JSON Schema, which the caller's key style decides where to land + """ + monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False) + config = GoogleGenAIConfig() + + schema = { + "type": "object", + "properties": {"barcode": {"anyOf": [{"type": "string"}, {"type": "null"}]}}, + "required": ["barcode"], + } + + result = config.transform_generate_content_request( + model="gemini-2.5-flash", + contents=[{"role": "user", "parts": [{"text": "hi"}]}], + tools=None, + generate_content_config_dict={json_schema_key: schema}, + system_instruction=None, + ) + + gen_config = result["generationConfig"] + assert json_schema_key not in gen_config + assert gen_config[expected_schema_key]["propertyOrdering"] == ["barcode"] + assert gen_config[expected_schema_key]["properties"]["barcode"]["anyOf"] == [ + {"type": "string", "nullable": True} + ] + + def test_transform_generate_content_request_preserves_response_json_schema_when_response_schema_co_present(): """When both ``responseJsonSchema`` and ``responseSchema`` are supplied on Gemini 2.0+, the caller's ``responseJsonSchema`` must win — the diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py index fad310fc5c0..5ffcb76bfff 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py @@ -1,6 +1,9 @@ +import json + import pytest +import litellm from litellm.llms.vertex_ai.gemini import transformation from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, @@ -338,3 +341,93 @@ def test_map_function_enterprise_web_search_snake_case(): assert len(result) == 1 assert "enterpriseWebSearch" in result[0] + + +RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA = { + "type": "object", + "additionalProperties": False, + "properties": { + "total": {"type": "number"}, + "barcode": {"anyOf": [{"type": "string", "maxLength": 10}, {"type": "null"}]}, + }, + "required": ["total", "barcode"], +} + + +def _gemini_request_body(model: str, litellm_params: dict, **completion_kwargs) -> RequestBody: + optional_params = litellm.utils.get_optional_params( + model=model, + custom_llm_provider="vertex_ai", + response_format={ + "type": "json_schema", + "json_schema": { + "name": "invoice", + "schema": json.loads(json.dumps(RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA)), + }, + }, + **completion_kwargs, + ) + return transformation._transform_request_body( + messages=[{"role": "user", "content": "extract it"}], + model=model, + optional_params=optional_params, + custom_llm_provider="vertex_ai", + litellm_params=litellm_params, + cached_content=None, + ) + + +def test__transform_request_body_per_request_response_json_schema_opt_out(): + """ + vertex_ai_use_response_json_schema=False on the request puts the schema on Vertex's native + responseSchema channel, and the knob itself never reaches the provider body + """ + body = _gemini_request_body( + "gemini-2.5-flash", {}, vertex_ai_use_response_json_schema=False + ) + + generation_config = body["generationConfig"] + assert "response_json_schema" not in generation_config + assert generation_config["response_schema"]["propertyOrdering"] == ["total", "barcode"] + assert generation_config["response_schema"]["properties"]["barcode"]["anyOf"] == [ + {"type": "string", "maxLength": 10, "nullable": True} + ] + assert "vertex_ai_use_response_json_schema" not in json.dumps(body) + assert "litellm_param" not in json.dumps(body) + + +def test__transform_request_body_deployment_response_json_schema_opt_out(): + """A deployment's litellm_params opts every request routed to it out of responseJsonSchema""" + body = _gemini_request_body( + "gemini-2.5-flash", {"vertex_ai_use_response_json_schema": False} + ) + + generation_config = body["generationConfig"] + assert "response_json_schema" not in generation_config + assert generation_config["response_schema"]["propertyOrdering"] == ["total", "barcode"] + + +def test__transform_request_body_per_request_opt_in_beats_global_opt_out(monkeypatch): + """ + With the global opted out, vertex_ai_use_response_json_schema=True on the request sends the + client schema verbatim again, additionalProperties included + """ + monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False) + + body = _gemini_request_body( + "gemini-2.5-flash", {}, vertex_ai_use_response_json_schema=True + ) + + generation_config = body["generationConfig"] + assert "response_schema" not in generation_config + assert generation_config["response_json_schema"] == RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA + assert "litellm_param" not in json.dumps(body) + + +def test__transform_request_body_keeps_response_json_schema_by_default(): + """Without any override, Gemini 2.x keeps sending the verbatim responseJsonSchema""" + body = _gemini_request_body("gemini-2.5-flash", {}) + + generation_config = body["generationConfig"] + assert "response_schema" not in generation_config + assert generation_config["response_json_schema"] == RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 3d882deeb52..0b66df76da5 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -11,7 +11,10 @@ from pydantic import BaseModel import litellm from litellm import ModelResponse, completion from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig -from litellm.llms.vertex_ai.common_utils import VertexAIError +from litellm.llms.vertex_ai.common_utils import ( + VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM, + VertexAIError, +) from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) @@ -337,6 +340,44 @@ def test_vertex_ai_response_json_schema_for_gemini_2(): ) +def test_vertex_ai_response_json_schema_opt_out_uses_native_schema(monkeypatch): + """ + litellm.vertex_ai_use_response_json_schema=False sends Gemini 2.x the natively converted + responseSchema, so nullable unions are flattened, constraints survive the flattening and + propertyOrdering is set. The client schema is kept verbatim for a per request opt in. + """ + monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False) + client_schema = { + "type": "object", + "additionalProperties": False, + "properties": { + "total": {"type": "number"}, + "barcode": {"anyOf": [{"type": "string", "maxLength": 10}, {"type": "null"}]}, + }, + "required": ["total", "barcode"], + } + + transformed_request = VertexGeminiConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": {"name": "invoice", "schema": deepcopy(client_schema)}, + }, + }, + optional_params={}, + model="gemini-2.5-flash", + drop_params=False, + ) + + assert "response_json_schema" not in transformed_request + assert transformed_request["response_schema"]["propertyOrdering"] == ["total", "barcode"] + assert transformed_request["response_schema"]["properties"]["barcode"]["anyOf"] == [ + {"type": "string", "maxLength": 10, "nullable": True} + ] + assert "additionalProperties" not in transformed_request["response_schema"] + assert transformed_request[VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM] == client_schema + + def test_vertex_ai_response_schema_for_old_models(): """ Test that older models (Gemini 1.5) automatically use responseSchema. diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index cc923f05831..897dd23cdc1 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -1,3 +1,4 @@ +from copy import deepcopy from unittest.mock import patch import pytest @@ -5,13 +6,17 @@ import pytest from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +import litellm from litellm.llms.vertex_ai.common_utils import ( + VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM, _get_vertex_url, convert_anyof_null_to_nullable, get_vertex_location_from_url, get_vertex_project_id_from_url, pop_vertex_request_labels, + resolve_response_schema_channel, set_schema_property_ordering, + should_use_response_json_schema, supports_response_json_schema, validate_vertex_location, vertex_request_labels_from_litellm_params, @@ -178,6 +183,98 @@ def test_supports_response_json_schema(model: str, expected: bool): assert supports_response_json_schema(model) == expected +CLIENT_SCHEMA = { + "type": "object", + "additionalProperties": False, + "properties": { + "total": {"type": "number"}, + "barcode": {"anyOf": [{"type": "string", "maxLength": 10}, {"type": "null"}]}, + }, + "required": ["total", "barcode"], +} + + +@pytest.mark.parametrize( + "global_setting, request_override, model, expected", + [ + (None, None, "gemini-2.5-flash", True), + (None, None, "gemini-1.5-pro", False), + (False, None, "gemini-2.5-flash", False), + (True, None, "gemini-1.5-pro", True), + (False, True, "gemini-2.5-flash", True), + (True, False, "gemini-2.5-flash", False), + ], +) +def test_should_use_response_json_schema_precedence( + monkeypatch, global_setting, request_override, model, expected +): + """Per request override beats litellm.vertex_ai_use_response_json_schema, which beats the model heuristic""" + monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", global_setting) + + assert should_use_response_json_schema(model, request_override) is expected + + +def test_resolve_response_schema_channel_opt_out_converts_to_native_schema(): + """Opting out per request moves the verbatim JSON Schema onto the native responseSchema channel""" + resolved = resolve_response_schema_channel( + optional_params={ + "response_mime_type": "application/json", + "response_json_schema": deepcopy(CLIENT_SCHEMA), + "vertex_ai_use_response_json_schema": False, + }, + litellm_params={}, + model="gemini-2.5-flash", + ) + + assert "response_json_schema" not in resolved + assert resolved["response_mime_type"] == "application/json" + assert resolved["response_schema"]["propertyOrdering"] == ["total", "barcode"] + assert resolved["response_schema"]["properties"]["barcode"]["anyOf"] == [ + {"type": "string", "maxLength": 10, "nullable": True} + ] + + +def test_resolve_response_schema_channel_reads_deployment_litellm_params(): + """A deployment's litellm_params can opt out for every request routed to it""" + resolved = resolve_response_schema_channel( + optional_params={"response_json_schema": deepcopy(CLIENT_SCHEMA)}, + litellm_params={"vertex_ai_use_response_json_schema": False}, + model="gemini-2.5-flash", + ) + + assert "response_json_schema" not in resolved + assert resolved["response_schema"]["propertyOrdering"] == ["total", "barcode"] + + +def test_resolve_response_schema_channel_opt_in_restores_verbatim_schema(monkeypatch): + """With the global opted out, a per request opt in sends the client schema verbatim again""" + monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False) + + resolved = resolve_response_schema_channel( + optional_params={ + "response_schema": {"type": "object", "propertyOrdering": ["total", "barcode"]}, + VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM: deepcopy(CLIENT_SCHEMA), + "vertex_ai_use_response_json_schema": True, + }, + litellm_params={}, + model="gemini-2.5-flash", + ) + + assert resolved["response_json_schema"] == CLIENT_SCHEMA + assert "response_schema" not in resolved + + +def test_resolve_response_schema_channel_without_override_keeps_channel(): + """No override leaves the mapped channel untouched""" + optional_params = {"response_json_schema": deepcopy(CLIENT_SCHEMA)} + + resolved = resolve_response_schema_channel( + optional_params=optional_params, litellm_params={}, model="gemini-2.5-flash" + ) + + assert resolved is optional_params + + def test_set_schema_property_ordering_with_excessive_nesting(): """Test set_schema_property_ordering with excessive nesting > max levels +1 deep.""" # generate a schema with excessive nesting From 729cb0f2445b96888ca68ffaf1bb6a56d6bb8b3c Mon Sep 17 00:00:00 2001 From: ArthurAAM <100235777+ArthurAAM@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:15:04 -0300 Subject: [PATCH 2/5] fix(vertex_ai): keep Gemini 1.x off the responseJsonSchema channel Neither the global setting nor the per request override can select a channel the model has no field for, so asking for the JSON Schema channel on a Gemini 1.x model now logs a warning and keeps the natively converted responseSchema. --- litellm/llms/vertex_ai/common_utils.py | 25 ++++++++++++----- .../vertex_ai/gemini/test_transformation.py | 27 ++++++++++--------- .../vertex_ai/test_vertex_ai_common_utils.py | 9 +++++-- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 03f28b1fb66..90ac9d2a061 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -271,10 +271,16 @@ def supports_response_json_schema(model: str) -> bool: return bool(gemini_2_plus_pattern.search(model_lower)) +GEMINI_1_MODEL_PATTERN: Final = re.compile(r"gemini-1(?:\.|-)") VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM: Final = "vertex_ai_use_response_json_schema" VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM: Final = "litellm_param_vertex_ai_verbatim_response_schema" +def _rejects_response_json_schema(model: str) -> bool: + """Gemini 1.x generateContent has no responseJsonSchema field, so no override can reach it""" + return bool(GEMINI_1_MODEL_PATTERN.search(model.lower())) + + def should_use_response_json_schema(model: str, request_override: bool | None = None) -> bool: """ Resolve which structured output channel a json_schema response_format goes to. @@ -283,13 +289,20 @@ def should_use_response_json_schema(model: str, request_override: bool | None = natively converted ``responseSchema`` (nullable unions flattened, constraints hoisted, ``propertyOrdering`` added). Precedence: per request ``vertex_ai_use_response_json_schema``, then - ``litellm.vertex_ai_use_response_json_schema``, then the model heuristic + ``litellm.vertex_ai_use_response_json_schema``, then the model heuristic. Neither + override can select a channel the model has no field for """ - if request_override is not None: - return request_override - if litellm.vertex_ai_use_response_json_schema is not None: - return litellm.vertex_ai_use_response_json_schema - return supports_response_json_schema(model) + override: Final = request_override if request_override is not None else litellm.vertex_ai_use_response_json_schema + if override is None: + return supports_response_json_schema(model) + if override and _rejects_response_json_schema(model): + verbose_logger.warning( + "vertex_ai_use_response_json_schema=True ignored for model=%s: it has no responseJsonSchema field, " + "so the schema stays on responseSchema", + model, + ) + return False + return override from typing import Literal diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py index 5ffcb76bfff..a1a7ae16500 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py @@ -1,5 +1,7 @@ import json +from collections.abc import Mapping +from typing import Optional import pytest @@ -354,7 +356,14 @@ RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA = { } -def _gemini_request_body(model: str, litellm_params: dict, **completion_kwargs) -> RequestBody: +def _gemini_request_body( + model: str, + litellm_params: Mapping[str, bool], + request_override: Optional[bool] = None, +) -> RequestBody: + override_kwargs: dict[str, bool] = ( + {} if request_override is None else {"vertex_ai_use_response_json_schema": request_override} + ) optional_params = litellm.utils.get_optional_params( model=model, custom_llm_provider="vertex_ai", @@ -365,14 +374,14 @@ def _gemini_request_body(model: str, litellm_params: dict, **completion_kwargs) "schema": json.loads(json.dumps(RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA)), }, }, - **completion_kwargs, + **override_kwargs, ) return transformation._transform_request_body( messages=[{"role": "user", "content": "extract it"}], model=model, optional_params=optional_params, custom_llm_provider="vertex_ai", - litellm_params=litellm_params, + litellm_params=dict(litellm_params), cached_content=None, ) @@ -382,9 +391,7 @@ def test__transform_request_body_per_request_response_json_schema_opt_out(): vertex_ai_use_response_json_schema=False on the request puts the schema on Vertex's native responseSchema channel, and the knob itself never reaches the provider body """ - body = _gemini_request_body( - "gemini-2.5-flash", {}, vertex_ai_use_response_json_schema=False - ) + body = _gemini_request_body("gemini-2.5-flash", {}, request_override=False) generation_config = body["generationConfig"] assert "response_json_schema" not in generation_config @@ -398,9 +405,7 @@ def test__transform_request_body_per_request_response_json_schema_opt_out(): def test__transform_request_body_deployment_response_json_schema_opt_out(): """A deployment's litellm_params opts every request routed to it out of responseJsonSchema""" - body = _gemini_request_body( - "gemini-2.5-flash", {"vertex_ai_use_response_json_schema": False} - ) + body = _gemini_request_body("gemini-2.5-flash", {"vertex_ai_use_response_json_schema": False}) generation_config = body["generationConfig"] assert "response_json_schema" not in generation_config @@ -414,9 +419,7 @@ def test__transform_request_body_per_request_opt_in_beats_global_opt_out(monkeyp """ monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False) - body = _gemini_request_body( - "gemini-2.5-flash", {}, vertex_ai_use_response_json_schema=True - ) + body = _gemini_request_body("gemini-2.5-flash", {}, request_override=True) generation_config = body["generationConfig"] assert "response_schema" not in generation_config diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index 897dd23cdc1..eef69722830 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -200,7 +200,9 @@ CLIENT_SCHEMA = { (None, None, "gemini-2.5-flash", True), (None, None, "gemini-1.5-pro", False), (False, None, "gemini-2.5-flash", False), - (True, None, "gemini-1.5-pro", True), + (True, None, "gemini-flash-latest", True), + (True, None, "gemini-1.5-pro", False), + (None, True, "gemini-1.5-pro", False), (False, True, "gemini-2.5-flash", True), (True, False, "gemini-2.5-flash", False), ], @@ -208,7 +210,10 @@ CLIENT_SCHEMA = { def test_should_use_response_json_schema_precedence( monkeypatch, global_setting, request_override, model, expected ): - """Per request override beats litellm.vertex_ai_use_response_json_schema, which beats the model heuristic""" + """ + Per request override beats litellm.vertex_ai_use_response_json_schema, which beats the model + heuristic, and neither can put a schema on a channel Gemini 1.x has no field for + """ monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", global_setting) assert should_use_response_json_schema(model, request_override) is expected From 7c0219a1c276248e55ddd7ff720efcc9b0d52305 Mon Sep 17 00:00:00 2001 From: ArthurAAM <100235777+ArthurAAM@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:15:05 -0300 Subject: [PATCH 3/5] fix(vertex_ai): let an override pick only channels the model has An override asking for responseJsonSchema on a model that is not known to accept it now logs a warning and keeps the natively converted responseSchema, so no setting can send a field the provider will reject. --- litellm/llms/vertex_ai/common_utils.py | 16 +++++----------- .../vertex_ai/test_vertex_ai_common_utils.py | 4 ++-- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 90ac9d2a061..f2b991a5e4b 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -271,16 +271,10 @@ def supports_response_json_schema(model: str) -> bool: return bool(gemini_2_plus_pattern.search(model_lower)) -GEMINI_1_MODEL_PATTERN: Final = re.compile(r"gemini-1(?:\.|-)") VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM: Final = "vertex_ai_use_response_json_schema" VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM: Final = "litellm_param_vertex_ai_verbatim_response_schema" -def _rejects_response_json_schema(model: str) -> bool: - """Gemini 1.x generateContent has no responseJsonSchema field, so no override can reach it""" - return bool(GEMINI_1_MODEL_PATTERN.search(model.lower())) - - def should_use_response_json_schema(model: str, request_override: bool | None = None) -> bool: """ Resolve which structured output channel a json_schema response_format goes to. @@ -289,16 +283,16 @@ def should_use_response_json_schema(model: str, request_override: bool | None = natively converted ``responseSchema`` (nullable unions flattened, constraints hoisted, ``propertyOrdering`` added). Precedence: per request ``vertex_ai_use_response_json_schema``, then - ``litellm.vertex_ai_use_response_json_schema``, then the model heuristic. Neither - override can select a channel the model has no field for + ``litellm.vertex_ai_use_response_json_schema``, then the model heuristic. An + override picks between the channels the model has, it never adds one """ override: Final = request_override if request_override is not None else litellm.vertex_ai_use_response_json_schema if override is None: return supports_response_json_schema(model) - if override and _rejects_response_json_schema(model): + if override and not supports_response_json_schema(model): verbose_logger.warning( - "vertex_ai_use_response_json_schema=True ignored for model=%s: it has no responseJsonSchema field, " - "so the schema stays on responseSchema", + "vertex_ai_use_response_json_schema=True ignored for model=%s: it is not known to accept " + "responseJsonSchema, so the schema stays on responseSchema", model, ) return False diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index eef69722830..a8f747e52b8 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -200,7 +200,7 @@ CLIENT_SCHEMA = { (None, None, "gemini-2.5-flash", True), (None, None, "gemini-1.5-pro", False), (False, None, "gemini-2.5-flash", False), - (True, None, "gemini-flash-latest", True), + (True, None, "gemini-flash-latest", False), (True, None, "gemini-1.5-pro", False), (None, True, "gemini-1.5-pro", False), (False, True, "gemini-2.5-flash", True), @@ -212,7 +212,7 @@ def test_should_use_response_json_schema_precedence( ): """ Per request override beats litellm.vertex_ai_use_response_json_schema, which beats the model - heuristic, and neither can put a schema on a channel Gemini 1.x has no field for + heuristic, and neither can select responseJsonSchema for a model not known to accept it """ monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", global_setting) From ef9a6e7141886176df55d261d204d5315aba0b42 Mon Sep 17 00:00:00 2001 From: ArthurAAM <100235777+ArthurAAM@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:15:05 -0300 Subject: [PATCH 4/5] refactor(vertex_ai): stop reassigning optional_params in the Gemini request build The resolved channel goes into a local instead of the parameter, and the test helper takes a typed immutable mapping instead of a bare dict and variadic kwargs. --- .../llms/vertex_ai/gemini/transformation.py | 20 +++++++++---------- .../vertex_ai/gemini/test_transformation.py | 5 +++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 695e5908fdd..95992fc3fa2 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -1185,7 +1185,7 @@ def _transform_request_body( resolved_params: Final = resolve_response_schema_channel( optional_params=optional_params, litellm_params=litellm_params, model=model ) - optional_params = { + request_params: Final = { k: v for k, v in resolved_params.items() if k not in remove_keys and k != VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM @@ -1200,18 +1200,18 @@ def _transform_request_body( content = litellm.VertexGeminiConfig()._transform_messages( messages=messages, model=model, litellm_params=litellm_params ) - tools: Final[Tools | None] = optional_params.pop("tools", None) - tool_choice: Final[ToolConfig | None] = optional_params.pop("tool_choice", None) - include_server_side_tool_invocations: bool = optional_params.pop("include_server_side_tool_invocations", False) - safety_settings: list[SafetSettingsConfig] | None = optional_params.pop("safety_settings", None) + tools: Final[Tools | None] = request_params.pop("tools", None) + tool_choice: Final[ToolConfig | None] = request_params.pop("tool_choice", None) + include_server_side_tool_invocations: bool = request_params.pop("include_server_side_tool_invocations", False) + safety_settings: list[SafetSettingsConfig] | None = request_params.pop("safety_settings", None) # Drop output_config as it's not supported by Vertex AI - optional_params.pop("output_config", None) + request_params.pop("output_config", None) config_fields: Final = GenerationConfig.__annotations__.keys() # labels: optional explicit param and/or metadata.requester_metadata (OpenAI metadata) - labels: Final = pop_vertex_request_labels(optional_params, litellm_params) + labels: Final = pop_vertex_request_labels(request_params, litellm_params) - filtered_params = {k: v for k, v in optional_params.items() if _get_equivalent_key(k, set(config_fields))} + filtered_params = {k: v for k, v in request_params.items() if _get_equivalent_key(k, set(config_fields))} generation_config: Final[GenerationConfig | None] = GenerationConfig(**filtered_params) @@ -1247,7 +1247,7 @@ def _transform_request_body( if cached_content is not None: data["cachedContent"] = cached_content - if service_tier := optional_params.pop("service_tier", None): + if service_tier := request_params.pop("service_tier", None): if isinstance(service_tier, str): if service_tier.lower() == "default": data["serviceTier"] = "standard" @@ -1259,7 +1259,7 @@ def _transform_request_body( # Only add labels for Vertex AI endpoints (not Google GenAI/AI Studio) and only if non-empty if labels and custom_llm_provider != LlmProviders.GEMINI: data["labels"] = labels - _pop_and_merge_extra_body(data, optional_params) + _pop_and_merge_extra_body(data, request_params) _rewrite_google_maps_response_format(data) except Exception as e: raise e diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py index a1a7ae16500..7f30085b8e7 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py @@ -1,7 +1,8 @@ import json from collections.abc import Mapping -from typing import Optional +from types import MappingProxyType +from typing import Final, Optional import pytest @@ -361,7 +362,7 @@ def _gemini_request_body( litellm_params: Mapping[str, bool], request_override: Optional[bool] = None, ) -> RequestBody: - override_kwargs: dict[str, bool] = ( + override_kwargs: Final[Mapping[str, bool]] = MappingProxyType( {} if request_override is None else {"vertex_ai_use_response_json_schema": request_override} ) optional_params = litellm.utils.get_optional_params( From 7cbebbf02bdc7005f03ce674be7110be52b6d094 Mon Sep 17 00:00:00 2001 From: ArthurAAM <100235777+ArthurAAM@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:15:05 -0300 Subject: [PATCH 5/5] revert(vertex_ai): keep the optional_params reassignment in the request build Naming the resolved channel into a local types its values as object, which puts reportArgumentType 20 errors over its budget because the surrounding request build still reads an untyped dict. Typing that whole flow belongs in its own PR. --- .../llms/vertex_ai/gemini/transformation.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 95992fc3fa2..695e5908fdd 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -1185,7 +1185,7 @@ def _transform_request_body( resolved_params: Final = resolve_response_schema_channel( optional_params=optional_params, litellm_params=litellm_params, model=model ) - request_params: Final = { + optional_params = { k: v for k, v in resolved_params.items() if k not in remove_keys and k != VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM @@ -1200,18 +1200,18 @@ def _transform_request_body( content = litellm.VertexGeminiConfig()._transform_messages( messages=messages, model=model, litellm_params=litellm_params ) - tools: Final[Tools | None] = request_params.pop("tools", None) - tool_choice: Final[ToolConfig | None] = request_params.pop("tool_choice", None) - include_server_side_tool_invocations: bool = request_params.pop("include_server_side_tool_invocations", False) - safety_settings: list[SafetSettingsConfig] | None = request_params.pop("safety_settings", None) + tools: Final[Tools | None] = optional_params.pop("tools", None) + tool_choice: Final[ToolConfig | None] = optional_params.pop("tool_choice", None) + include_server_side_tool_invocations: bool = optional_params.pop("include_server_side_tool_invocations", False) + safety_settings: list[SafetSettingsConfig] | None = optional_params.pop("safety_settings", None) # Drop output_config as it's not supported by Vertex AI - request_params.pop("output_config", None) + optional_params.pop("output_config", None) config_fields: Final = GenerationConfig.__annotations__.keys() # labels: optional explicit param and/or metadata.requester_metadata (OpenAI metadata) - labels: Final = pop_vertex_request_labels(request_params, litellm_params) + labels: Final = pop_vertex_request_labels(optional_params, litellm_params) - filtered_params = {k: v for k, v in request_params.items() if _get_equivalent_key(k, set(config_fields))} + filtered_params = {k: v for k, v in optional_params.items() if _get_equivalent_key(k, set(config_fields))} generation_config: Final[GenerationConfig | None] = GenerationConfig(**filtered_params) @@ -1247,7 +1247,7 @@ def _transform_request_body( if cached_content is not None: data["cachedContent"] = cached_content - if service_tier := request_params.pop("service_tier", None): + if service_tier := optional_params.pop("service_tier", None): if isinstance(service_tier, str): if service_tier.lower() == "default": data["serviceTier"] = "standard" @@ -1259,7 +1259,7 @@ def _transform_request_body( # Only add labels for Vertex AI endpoints (not Google GenAI/AI Studio) and only if non-empty if labels and custom_llm_provider != LlmProviders.GEMINI: data["labels"] = labels - _pop_and_merge_extra_body(data, request_params) + _pop_and_merge_extra_body(data, optional_params) _rewrite_google_maps_response_format(data) except Exception as e: raise e