feat(vertex_ai): make the Gemini structured output channel selectable

Gemini 2.x+ requests always went out on responseJsonSchema, with no setting,
config or request param able to reach Vertex's native responseSchema, which
flattens nullable unions, keeps the constraints inside them and orders
properties. litellm.vertex_ai_use_response_json_schema now picks the channel,
vertex_ai_use_response_json_schema on a request or on a deployment's
litellm_params beats it, and the model name heuristic stays the default, so
nothing changes for anyone who sets neither.

Native generateContent requests read the same setting, including the ones that
carry only a JSON Schema, which the old check skipped.
This commit is contained in:
ArthurAAM 2026-08-25 18:15:04 -03:00
parent c3bcb6f64f
commit 83e3458f8a
9 changed files with 396 additions and 26 deletions

View file

@ -564,6 +564,7 @@ organization = None
project = None
config_path = None
vertex_ai_safety_settings: Optional[dict] = None
vertex_ai_use_response_json_schema: Optional[bool] = None
####### COMPLETION MODELS ###################
from typing import Set

View file

@ -14,7 +14,7 @@ from litellm.llms.base_llm.google_genai.transformation import (
)
from litellm.llms.vertex_ai.common_utils import (
_build_vertex_schema,
supports_response_json_schema,
should_use_response_json_schema,
)
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.types.router import GenericLiteLLMParams
@ -310,26 +310,34 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
None,
)
if schema_key is None:
source_key: Final = schema_key if schema_key is not None else json_schema_key
if source_key is None:
return
value: Final = generate_content_config_dict[schema_key]
value: Final = generate_content_config_dict[source_key]
if not isinstance(value, dict):
return
if supports_response_json_schema(model):
if should_use_response_json_schema(model):
if json_schema_key is not None:
generate_content_config_dict.pop(schema_key)
if schema_key is not None:
generate_content_config_dict.pop(schema_key)
return
generate_content_config_dict.pop(schema_key)
new_json_schema_key = "response_json_schema" if schema_key == "response_schema" else "responseJsonSchema"
generate_content_config_dict[new_json_schema_key] = value
else:
if json_schema_key is not None:
generate_content_config_dict.pop(json_schema_key)
generate_content_config_dict[schema_key] = _build_vertex_schema(
parameters=deepcopy(value), add_property_ordering=True
)
generate_content_config_dict.pop(source_key)
json_target_key: Final = "response_json_schema" if source_key == "response_schema" else "responseJsonSchema"
generate_content_config_dict[json_target_key] = value
return
if json_schema_key is not None:
generate_content_config_dict.pop(json_schema_key)
native_target_key: Final = (
source_key
if schema_key is not None
else ("response_schema" if source_key == "response_json_schema" else "responseSchema")
)
generate_content_config_dict[native_target_key] = _build_vertex_schema(
parameters=deepcopy(value), add_property_ordering=True
)
def transform_generate_content_request(
self,

View file

@ -1,6 +1,8 @@
import re
from collections.abc import Mapping
from copy import deepcopy
from enum import Enum
from types import MappingProxyType
from typing import Any, Final, Literal, get_type_hints
import httpx
@ -269,6 +271,27 @@ def supports_response_json_schema(model: str) -> bool:
return bool(gemini_2_plus_pattern.search(model_lower))
VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM: Final = "vertex_ai_use_response_json_schema"
VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM: Final = "litellm_param_vertex_ai_verbatim_response_schema"
def should_use_response_json_schema(model: str, request_override: bool | None = None) -> bool:
"""
Resolve which structured output channel a json_schema response_format goes to.
True sends the client schema verbatim as ``responseJsonSchema``, False sends the
natively converted ``responseSchema`` (nullable unions flattened, constraints
hoisted, ``propertyOrdering`` added). Precedence: per request
``vertex_ai_use_response_json_schema``, then
``litellm.vertex_ai_use_response_json_schema``, then the model heuristic
"""
if request_override is not None:
return request_override
if litellm.vertex_ai_use_response_json_schema is not None:
return litellm.vertex_ai_use_response_json_schema
return supports_response_json_schema(model)
from typing import Literal
all_gemini_url_modes = Literal["chat", "embedding", "batch_embedding", "image_generation", "count_tokens"]
@ -651,6 +674,66 @@ def _build_json_schema(parameters: dict) -> dict:
return parameters
def _response_json_schema_override(
optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> bool | None:
candidates: Final = (
optional_params.get(VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM),
litellm_params.get(VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM),
)
return next((candidate for candidate in candidates if isinstance(candidate, bool)), None)
def _swap_response_schema_key(
optional_params: Mapping[str, object], dropped_key: str, added_key: str, schema: Mapping[str, object]
) -> Mapping[str, object]:
surviving: Final = MappingProxyType({k: v for k, v in optional_params.items() if k != dropped_key})
return MappingProxyType({**surviving, added_key: schema})
def resolve_response_schema_channel(
optional_params: Mapping[str, object], litellm_params: Mapping[str, object], model: str
) -> Mapping[str, object]:
"""
Move an already mapped response schema onto the channel this request asks for.
``vertex_ai_use_response_json_schema`` on the request or on the deployment's
``litellm_params`` beats ``litellm.vertex_ai_use_response_json_schema`` and the model
heuristic, and only the request build sees both, so the mapped channel is settled here
"""
override: Final = _response_json_schema_override(optional_params, litellm_params)
if override is None:
return optional_params
if should_use_response_json_schema(model, override):
if "response_json_schema" in optional_params:
return optional_params
verbatim_schema: Final = optional_params.get(VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM) or litellm_params.get(
VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM
)
if isinstance(verbatim_schema, dict):
return _swap_response_schema_key(
optional_params, "response_schema", "response_json_schema", verbatim_schema
)
if "response_schema" in optional_params:
verbose_logger.warning(
"vertex_ai_use_response_json_schema=True is ignored for model=%s, whose schema was already "
"converted for responseSchema. Set litellm.vertex_ai_use_response_json_schema instead",
model,
)
return optional_params
json_schema: Final = optional_params.get("response_json_schema")
if not isinstance(json_schema, dict):
return optional_params
return _swap_response_schema_key(
optional_params,
"response_json_schema",
"response_schema",
_build_vertex_schema(parameters=deepcopy(json_schema), add_property_ordering=True),
)
def _filter_anyof_fields(schema_dict: dict[str, Any]) -> dict[str, Any]:
"""
When anyof is present, only keep the anyof field and its contents - otherwise VertexAI will throw an error - https://github.com/BerriAI/litellm/issues/11164

View file

@ -28,7 +28,11 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
response_schema_prompt,
)
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels
from litellm.llms.vertex_ai.common_utils import (
VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM,
pop_vertex_request_labels,
resolve_response_schema_channel,
)
from litellm.types.files import (
get_file_mime_type_for_file_type,
get_file_type_from_extension,
@ -1178,7 +1182,14 @@ def _transform_request_body(
litellm_params.update({k: v})
remove_keys.append(k)
optional_params = {k: v for k, v in optional_params.items() if k not in remove_keys}
resolved_params: Final = resolve_response_schema_channel(
optional_params=optional_params, litellm_params=litellm_params, model=model
)
optional_params = {
k: v
for k, v in resolved_params.items()
if k not in remove_keys and k != VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM
}
try:
if custom_llm_provider == "gemini":

View file

@ -82,9 +82,11 @@ from litellm.utils import (
from ....utils import _remove_additional_properties, _remove_strict_from_schema
from ..common_utils import (
VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM,
VertexAIError,
_build_json_schema,
_build_vertex_schema,
should_use_response_json_schema,
supports_response_json_schema,
)
from ..vertex_llm_base import VertexBase
@ -754,14 +756,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# remove 'strict' from json schema (not supported by Gemini)
new_value = _remove_strict_from_schema(new_value)
# Automatically use responseJsonSchema for Gemini 2.0+ models
# responseJsonSchema uses standard JSON Schema format and supports additionalProperties
# For older models (Gemini 1.5), fall back to responseSchema (OpenAPI format)
use_json_schema: Final = supports_response_json_schema(model)
if not use_json_schema:
# For responseSchema, remove 'additionalProperties' (not supported)
new_value = _remove_additional_properties(new_value)
use_json_schema: Final = should_use_response_json_schema(model)
# Handle response type
if new_value.get("type") == "json_object":
@ -791,7 +786,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# - OpenAPI-style format (uppercase types)
# - No additionalProperties support
# - Requires propertyOrdering
optional_params["response_schema"] = self._map_response_schema(value=schema)
if supports_response_json_schema(model):
optional_params[VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM] = deepcopy(schema)
optional_params["response_schema"] = self._map_response_schema(
value=_remove_additional_properties(schema)
)
@staticmethod
def _map_reasoning_effort_to_thinking_budget(

View file

@ -6,6 +6,7 @@ Test to verify the Google GenAI transformation logic for generateContent paramet
import pytest
import litellm
from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
@ -445,6 +446,42 @@ def test_transform_generate_content_request_passes_through_response_json_schema(
assert "responseSchema" not in gen_config
@pytest.mark.parametrize(
"json_schema_key, expected_schema_key",
[("responseJsonSchema", "responseSchema"), ("response_json_schema", "response_schema")],
)
def test_transform_generate_content_request_opt_out_converts_response_json_schema(
monkeypatch, json_schema_key, expected_schema_key
):
"""
litellm.vertex_ai_use_response_json_schema=False also reaches generateContent requests that
only carry a JSON Schema, which the caller's key style decides where to land
"""
monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False)
config = GoogleGenAIConfig()
schema = {
"type": "object",
"properties": {"barcode": {"anyOf": [{"type": "string"}, {"type": "null"}]}},
"required": ["barcode"],
}
result = config.transform_generate_content_request(
model="gemini-2.5-flash",
contents=[{"role": "user", "parts": [{"text": "hi"}]}],
tools=None,
generate_content_config_dict={json_schema_key: schema},
system_instruction=None,
)
gen_config = result["generationConfig"]
assert json_schema_key not in gen_config
assert gen_config[expected_schema_key]["propertyOrdering"] == ["barcode"]
assert gen_config[expected_schema_key]["properties"]["barcode"]["anyOf"] == [
{"type": "string", "nullable": True}
]
def test_transform_generate_content_request_preserves_response_json_schema_when_response_schema_co_present():
"""When both ``responseJsonSchema`` and ``responseSchema`` are supplied on
Gemini 2.0+, the caller's ``responseJsonSchema`` must win — the

View file

@ -1,6 +1,9 @@
import json
import pytest
import litellm
from litellm.llms.vertex_ai.gemini import transformation
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
@ -338,3 +341,93 @@ def test_map_function_enterprise_web_search_snake_case():
assert len(result) == 1
assert "enterpriseWebSearch" in result[0]
RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA = {
"type": "object",
"additionalProperties": False,
"properties": {
"total": {"type": "number"},
"barcode": {"anyOf": [{"type": "string", "maxLength": 10}, {"type": "null"}]},
},
"required": ["total", "barcode"],
}
def _gemini_request_body(model: str, litellm_params: dict, **completion_kwargs) -> RequestBody:
optional_params = litellm.utils.get_optional_params(
model=model,
custom_llm_provider="vertex_ai",
response_format={
"type": "json_schema",
"json_schema": {
"name": "invoice",
"schema": json.loads(json.dumps(RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA)),
},
},
**completion_kwargs,
)
return transformation._transform_request_body(
messages=[{"role": "user", "content": "extract it"}],
model=model,
optional_params=optional_params,
custom_llm_provider="vertex_ai",
litellm_params=litellm_params,
cached_content=None,
)
def test__transform_request_body_per_request_response_json_schema_opt_out():
"""
vertex_ai_use_response_json_schema=False on the request puts the schema on Vertex's native
responseSchema channel, and the knob itself never reaches the provider body
"""
body = _gemini_request_body(
"gemini-2.5-flash", {}, vertex_ai_use_response_json_schema=False
)
generation_config = body["generationConfig"]
assert "response_json_schema" not in generation_config
assert generation_config["response_schema"]["propertyOrdering"] == ["total", "barcode"]
assert generation_config["response_schema"]["properties"]["barcode"]["anyOf"] == [
{"type": "string", "maxLength": 10, "nullable": True}
]
assert "vertex_ai_use_response_json_schema" not in json.dumps(body)
assert "litellm_param" not in json.dumps(body)
def test__transform_request_body_deployment_response_json_schema_opt_out():
"""A deployment's litellm_params opts every request routed to it out of responseJsonSchema"""
body = _gemini_request_body(
"gemini-2.5-flash", {"vertex_ai_use_response_json_schema": False}
)
generation_config = body["generationConfig"]
assert "response_json_schema" not in generation_config
assert generation_config["response_schema"]["propertyOrdering"] == ["total", "barcode"]
def test__transform_request_body_per_request_opt_in_beats_global_opt_out(monkeypatch):
"""
With the global opted out, vertex_ai_use_response_json_schema=True on the request sends the
client schema verbatim again, additionalProperties included
"""
monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False)
body = _gemini_request_body(
"gemini-2.5-flash", {}, vertex_ai_use_response_json_schema=True
)
generation_config = body["generationConfig"]
assert "response_schema" not in generation_config
assert generation_config["response_json_schema"] == RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA
assert "litellm_param" not in json.dumps(body)
def test__transform_request_body_keeps_response_json_schema_by_default():
"""Without any override, Gemini 2.x keeps sending the verbatim responseJsonSchema"""
body = _gemini_request_body("gemini-2.5-flash", {})
generation_config = body["generationConfig"]
assert "response_schema" not in generation_config
assert generation_config["response_json_schema"] == RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA

View file

@ -11,7 +11,10 @@ from pydantic import BaseModel
import litellm
from litellm import ModelResponse, completion
from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig
from litellm.llms.vertex_ai.common_utils import VertexAIError
from litellm.llms.vertex_ai.common_utils import (
VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM,
VertexAIError,
)
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
@ -337,6 +340,44 @@ def test_vertex_ai_response_json_schema_for_gemini_2():
)
def test_vertex_ai_response_json_schema_opt_out_uses_native_schema(monkeypatch):
"""
litellm.vertex_ai_use_response_json_schema=False sends Gemini 2.x the natively converted
responseSchema, so nullable unions are flattened, constraints survive the flattening and
propertyOrdering is set. The client schema is kept verbatim for a per request opt in.
"""
monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False)
client_schema = {
"type": "object",
"additionalProperties": False,
"properties": {
"total": {"type": "number"},
"barcode": {"anyOf": [{"type": "string", "maxLength": 10}, {"type": "null"}]},
},
"required": ["total", "barcode"],
}
transformed_request = VertexGeminiConfig().map_openai_params(
non_default_params={
"response_format": {
"type": "json_schema",
"json_schema": {"name": "invoice", "schema": deepcopy(client_schema)},
},
},
optional_params={},
model="gemini-2.5-flash",
drop_params=False,
)
assert "response_json_schema" not in transformed_request
assert transformed_request["response_schema"]["propertyOrdering"] == ["total", "barcode"]
assert transformed_request["response_schema"]["properties"]["barcode"]["anyOf"] == [
{"type": "string", "maxLength": 10, "nullable": True}
]
assert "additionalProperties" not in transformed_request["response_schema"]
assert transformed_request[VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM] == client_schema
def test_vertex_ai_response_schema_for_old_models():
"""
Test that older models (Gemini 1.5) automatically use responseSchema.

View file

@ -1,3 +1,4 @@
from copy import deepcopy
from unittest.mock import patch
import pytest
@ -5,13 +6,17 @@ import pytest
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
import litellm
from litellm.llms.vertex_ai.common_utils import (
VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM,
_get_vertex_url,
convert_anyof_null_to_nullable,
get_vertex_location_from_url,
get_vertex_project_id_from_url,
pop_vertex_request_labels,
resolve_response_schema_channel,
set_schema_property_ordering,
should_use_response_json_schema,
supports_response_json_schema,
validate_vertex_location,
vertex_request_labels_from_litellm_params,
@ -178,6 +183,98 @@ def test_supports_response_json_schema(model: str, expected: bool):
assert supports_response_json_schema(model) == expected
CLIENT_SCHEMA = {
"type": "object",
"additionalProperties": False,
"properties": {
"total": {"type": "number"},
"barcode": {"anyOf": [{"type": "string", "maxLength": 10}, {"type": "null"}]},
},
"required": ["total", "barcode"],
}
@pytest.mark.parametrize(
"global_setting, request_override, model, expected",
[
(None, None, "gemini-2.5-flash", True),
(None, None, "gemini-1.5-pro", False),
(False, None, "gemini-2.5-flash", False),
(True, None, "gemini-1.5-pro", True),
(False, True, "gemini-2.5-flash", True),
(True, False, "gemini-2.5-flash", False),
],
)
def test_should_use_response_json_schema_precedence(
monkeypatch, global_setting, request_override, model, expected
):
"""Per request override beats litellm.vertex_ai_use_response_json_schema, which beats the model heuristic"""
monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", global_setting)
assert should_use_response_json_schema(model, request_override) is expected
def test_resolve_response_schema_channel_opt_out_converts_to_native_schema():
"""Opting out per request moves the verbatim JSON Schema onto the native responseSchema channel"""
resolved = resolve_response_schema_channel(
optional_params={
"response_mime_type": "application/json",
"response_json_schema": deepcopy(CLIENT_SCHEMA),
"vertex_ai_use_response_json_schema": False,
},
litellm_params={},
model="gemini-2.5-flash",
)
assert "response_json_schema" not in resolved
assert resolved["response_mime_type"] == "application/json"
assert resolved["response_schema"]["propertyOrdering"] == ["total", "barcode"]
assert resolved["response_schema"]["properties"]["barcode"]["anyOf"] == [
{"type": "string", "maxLength": 10, "nullable": True}
]
def test_resolve_response_schema_channel_reads_deployment_litellm_params():
"""A deployment's litellm_params can opt out for every request routed to it"""
resolved = resolve_response_schema_channel(
optional_params={"response_json_schema": deepcopy(CLIENT_SCHEMA)},
litellm_params={"vertex_ai_use_response_json_schema": False},
model="gemini-2.5-flash",
)
assert "response_json_schema" not in resolved
assert resolved["response_schema"]["propertyOrdering"] == ["total", "barcode"]
def test_resolve_response_schema_channel_opt_in_restores_verbatim_schema(monkeypatch):
"""With the global opted out, a per request opt in sends the client schema verbatim again"""
monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False)
resolved = resolve_response_schema_channel(
optional_params={
"response_schema": {"type": "object", "propertyOrdering": ["total", "barcode"]},
VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM: deepcopy(CLIENT_SCHEMA),
"vertex_ai_use_response_json_schema": True,
},
litellm_params={},
model="gemini-2.5-flash",
)
assert resolved["response_json_schema"] == CLIENT_SCHEMA
assert "response_schema" not in resolved
def test_resolve_response_schema_channel_without_override_keeps_channel():
"""No override leaves the mapped channel untouched"""
optional_params = {"response_json_schema": deepcopy(CLIENT_SCHEMA)}
resolved = resolve_response_schema_channel(
optional_params=optional_params, litellm_params={}, model="gemini-2.5-flash"
)
assert resolved is optional_params
def test_set_schema_property_ordering_with_excessive_nesting():
"""Test set_schema_property_ordering with excessive nesting > max levels +1 deep."""
# generate a schema with excessive nesting