This commit is contained in:
Arthur Maia 2026-08-31 14:34:33 -07:00 committed by GitHub
commit e558b6b2be
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 412 additions and 26 deletions

View file

@ -571,6 +571,7 @@ organization = None
project = None
config_path = None
vertex_ai_safety_settings: Optional[dict] = None
vertex_ai_use_response_json_schema: Optional[bool] = None
####### COMPLETION MODELS ###################
from typing import Set

View file

@ -14,7 +14,7 @@ from litellm.llms.base_llm.google_genai.transformation import (
)
from litellm.llms.vertex_ai.common_utils import (
_build_vertex_schema,
supports_response_json_schema,
should_use_response_json_schema,
)
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.types.router import GenericLiteLLMParams
@ -310,26 +310,34 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
None,
)
if schema_key is None:
source_key: Final = schema_key if schema_key is not None else json_schema_key
if source_key is None:
return
value: Final = generate_content_config_dict[schema_key]
value: Final = generate_content_config_dict[source_key]
if not isinstance(value, dict):
return
if supports_response_json_schema(model):
if should_use_response_json_schema(model):
if json_schema_key is not None:
generate_content_config_dict.pop(schema_key)
if schema_key is not None:
generate_content_config_dict.pop(schema_key)
return
generate_content_config_dict.pop(schema_key)
new_json_schema_key = "response_json_schema" if schema_key == "response_schema" else "responseJsonSchema"
generate_content_config_dict[new_json_schema_key] = value
else:
if json_schema_key is not None:
generate_content_config_dict.pop(json_schema_key)
generate_content_config_dict[schema_key] = _build_vertex_schema(
parameters=deepcopy(value), add_property_ordering=True
)
generate_content_config_dict.pop(source_key)
json_target_key: Final = "response_json_schema" if source_key == "response_schema" else "responseJsonSchema"
generate_content_config_dict[json_target_key] = value
return
if json_schema_key is not None:
generate_content_config_dict.pop(json_schema_key)
native_target_key: Final = (
source_key
if schema_key is not None
else ("response_schema" if source_key == "response_json_schema" else "responseSchema")
)
generate_content_config_dict[native_target_key] = _build_vertex_schema(
parameters=deepcopy(value), add_property_ordering=True
)
def transform_generate_content_request(
self,

View file

@ -1,6 +1,8 @@
import re
from collections.abc import Mapping
from copy import deepcopy
from enum import Enum
from types import MappingProxyType
from typing import Any, Final, Literal, get_type_hints
import httpx
@ -269,6 +271,34 @@ def supports_response_json_schema(model: str) -> bool:
return bool(gemini_2_plus_pattern.search(model_lower))
VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM: Final = "vertex_ai_use_response_json_schema"
VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM: Final = "litellm_param_vertex_ai_verbatim_response_schema"
def should_use_response_json_schema(model: str, request_override: bool | None = None) -> bool:
"""
Resolve which structured output channel a json_schema response_format goes to.
True sends the client schema verbatim as ``responseJsonSchema``, False sends the
natively converted ``responseSchema`` (nullable unions flattened, constraints
hoisted, ``propertyOrdering`` added). Precedence: per request
``vertex_ai_use_response_json_schema``, then
``litellm.vertex_ai_use_response_json_schema``, then the model heuristic. An
override picks between the channels the model has, it never adds one
"""
override: Final = request_override if request_override is not None else litellm.vertex_ai_use_response_json_schema
if override is None:
return supports_response_json_schema(model)
if override and not supports_response_json_schema(model):
verbose_logger.warning(
"vertex_ai_use_response_json_schema=True ignored for model=%s: it is not known to accept "
"responseJsonSchema, so the schema stays on responseSchema",
model,
)
return False
return override
from typing import Literal
all_gemini_url_modes = Literal["chat", "embedding", "batch_embedding", "image_generation", "count_tokens"]
@ -651,6 +681,66 @@ def _build_json_schema(parameters: dict) -> dict:
return parameters
def _response_json_schema_override(
optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> bool | None:
candidates: Final = (
optional_params.get(VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM),
litellm_params.get(VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM),
)
return next((candidate for candidate in candidates if isinstance(candidate, bool)), None)
def _swap_response_schema_key(
optional_params: Mapping[str, object], dropped_key: str, added_key: str, schema: Mapping[str, object]
) -> Mapping[str, object]:
surviving: Final = MappingProxyType({k: v for k, v in optional_params.items() if k != dropped_key})
return MappingProxyType({**surviving, added_key: schema})
def resolve_response_schema_channel(
optional_params: Mapping[str, object], litellm_params: Mapping[str, object], model: str
) -> Mapping[str, object]:
"""
Move an already mapped response schema onto the channel this request asks for.
``vertex_ai_use_response_json_schema`` on the request or on the deployment's
``litellm_params`` beats ``litellm.vertex_ai_use_response_json_schema`` and the model
heuristic, and only the request build sees both, so the mapped channel is settled here
"""
override: Final = _response_json_schema_override(optional_params, litellm_params)
if override is None:
return optional_params
if should_use_response_json_schema(model, override):
if "response_json_schema" in optional_params:
return optional_params
verbatim_schema: Final = optional_params.get(VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM) or litellm_params.get(
VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM
)
if isinstance(verbatim_schema, dict):
return _swap_response_schema_key(
optional_params, "response_schema", "response_json_schema", verbatim_schema
)
if "response_schema" in optional_params:
verbose_logger.warning(
"vertex_ai_use_response_json_schema=True is ignored for model=%s, whose schema was already "
"converted for responseSchema. Set litellm.vertex_ai_use_response_json_schema instead",
model,
)
return optional_params
json_schema: Final = optional_params.get("response_json_schema")
if not isinstance(json_schema, dict):
return optional_params
return _swap_response_schema_key(
optional_params,
"response_json_schema",
"response_schema",
_build_vertex_schema(parameters=deepcopy(json_schema), add_property_ordering=True),
)
def _filter_anyof_fields(schema_dict: dict[str, Any]) -> dict[str, Any]:
"""
When anyof is present, only keep the anyof field and its contents - otherwise VertexAI will throw an error - https://github.com/BerriAI/litellm/issues/11164

View file

@ -28,7 +28,11 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
response_schema_prompt,
)
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels
from litellm.llms.vertex_ai.common_utils import (
VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM,
pop_vertex_request_labels,
resolve_response_schema_channel,
)
from litellm.types.files import (
get_file_mime_type_for_file_type,
get_file_type_from_extension,
@ -1178,7 +1182,14 @@ def _transform_request_body(
litellm_params.update({k: v})
remove_keys.append(k)
optional_params = {k: v for k, v in optional_params.items() if k not in remove_keys}
resolved_params: Final = resolve_response_schema_channel(
optional_params=optional_params, litellm_params=litellm_params, model=model
)
optional_params = {
k: v
for k, v in resolved_params.items()
if k not in remove_keys and k != VERTEX_AI_USE_RESPONSE_JSON_SCHEMA_PARAM
}
try:
if custom_llm_provider == "gemini":

View file

@ -83,9 +83,11 @@ from litellm.utils import (
from ....utils import _remove_additional_properties, _remove_strict_from_schema
from ..common_utils import (
VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM,
VertexAIError,
_build_json_schema,
_build_vertex_schema,
should_use_response_json_schema,
supports_response_json_schema,
)
from ..vertex_llm_base import VertexBase
@ -756,14 +758,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# remove 'strict' from json schema (not supported by Gemini)
new_value = _remove_strict_from_schema(new_value)
# Automatically use responseJsonSchema for Gemini 2.0+ models
# responseJsonSchema uses standard JSON Schema format and supports additionalProperties
# For older models (Gemini 1.5), fall back to responseSchema (OpenAPI format)
use_json_schema: Final = supports_response_json_schema(model)
if not use_json_schema:
# For responseSchema, remove 'additionalProperties' (not supported)
new_value = _remove_additional_properties(new_value)
use_json_schema: Final = should_use_response_json_schema(model)
# Handle response type
if new_value.get("type") == "json_object":
@ -793,7 +788,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# - OpenAPI-style format (uppercase types)
# - No additionalProperties support
# - Requires propertyOrdering
optional_params["response_schema"] = self._map_response_schema(value=schema)
if supports_response_json_schema(model):
optional_params[VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM] = deepcopy(schema)
optional_params["response_schema"] = self._map_response_schema(
value=_remove_additional_properties(schema)
)
@staticmethod
def _map_reasoning_effort_to_thinking_budget(

View file

@ -6,6 +6,7 @@ Test to verify the Google GenAI transformation logic for generateContent paramet
import pytest
import litellm
from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
@ -445,6 +446,42 @@ def test_transform_generate_content_request_passes_through_response_json_schema(
assert "responseSchema" not in gen_config
@pytest.mark.parametrize(
"json_schema_key, expected_schema_key",
[("responseJsonSchema", "responseSchema"), ("response_json_schema", "response_schema")],
)
def test_transform_generate_content_request_opt_out_converts_response_json_schema(
monkeypatch, json_schema_key, expected_schema_key
):
"""
litellm.vertex_ai_use_response_json_schema=False also reaches generateContent requests that
only carry a JSON Schema, which the caller's key style decides where to land
"""
monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False)
config = GoogleGenAIConfig()
schema = {
"type": "object",
"properties": {"barcode": {"anyOf": [{"type": "string"}, {"type": "null"}]}},
"required": ["barcode"],
}
result = config.transform_generate_content_request(
model="gemini-2.5-flash",
contents=[{"role": "user", "parts": [{"text": "hi"}]}],
tools=None,
generate_content_config_dict={json_schema_key: schema},
system_instruction=None,
)
gen_config = result["generationConfig"]
assert json_schema_key not in gen_config
assert gen_config[expected_schema_key]["propertyOrdering"] == ["barcode"]
assert gen_config[expected_schema_key]["properties"]["barcode"]["anyOf"] == [
{"type": "string", "nullable": True}
]
def test_transform_generate_content_request_preserves_response_json_schema_when_response_schema_co_present():
"""When both ``responseJsonSchema`` and ``responseSchema`` are supplied on
Gemini 2.0+, the caller's ``responseJsonSchema`` must win — the

View file

@ -1,6 +1,12 @@
import json
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final, Optional
import pytest
import litellm
from litellm.llms.vertex_ai.gemini import transformation
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
@ -338,3 +344,94 @@ def test_map_function_enterprise_web_search_snake_case():
assert len(result) == 1
assert "enterpriseWebSearch" in result[0]
RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA = {
"type": "object",
"additionalProperties": False,
"properties": {
"total": {"type": "number"},
"barcode": {"anyOf": [{"type": "string", "maxLength": 10}, {"type": "null"}]},
},
"required": ["total", "barcode"],
}
def _gemini_request_body(
model: str,
litellm_params: Mapping[str, bool],
request_override: Optional[bool] = None,
) -> RequestBody:
override_kwargs: Final[Mapping[str, bool]] = MappingProxyType(
{} if request_override is None else {"vertex_ai_use_response_json_schema": request_override}
)
optional_params = litellm.utils.get_optional_params(
model=model,
custom_llm_provider="vertex_ai",
response_format={
"type": "json_schema",
"json_schema": {
"name": "invoice",
"schema": json.loads(json.dumps(RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA)),
},
},
**override_kwargs,
)
return transformation._transform_request_body(
messages=[{"role": "user", "content": "extract it"}],
model=model,
optional_params=optional_params,
custom_llm_provider="vertex_ai",
litellm_params=dict(litellm_params),
cached_content=None,
)
def test__transform_request_body_per_request_response_json_schema_opt_out():
"""
vertex_ai_use_response_json_schema=False on the request puts the schema on Vertex's native
responseSchema channel, and the knob itself never reaches the provider body
"""
body = _gemini_request_body("gemini-2.5-flash", {}, request_override=False)
generation_config = body["generationConfig"]
assert "response_json_schema" not in generation_config
assert generation_config["response_schema"]["propertyOrdering"] == ["total", "barcode"]
assert generation_config["response_schema"]["properties"]["barcode"]["anyOf"] == [
{"type": "string", "maxLength": 10, "nullable": True}
]
assert "vertex_ai_use_response_json_schema" not in json.dumps(body)
assert "litellm_param" not in json.dumps(body)
def test__transform_request_body_deployment_response_json_schema_opt_out():
"""A deployment's litellm_params opts every request routed to it out of responseJsonSchema"""
body = _gemini_request_body("gemini-2.5-flash", {"vertex_ai_use_response_json_schema": False})
generation_config = body["generationConfig"]
assert "response_json_schema" not in generation_config
assert generation_config["response_schema"]["propertyOrdering"] == ["total", "barcode"]
def test__transform_request_body_per_request_opt_in_beats_global_opt_out(monkeypatch):
"""
With the global opted out, vertex_ai_use_response_json_schema=True on the request sends the
client schema verbatim again, additionalProperties included
"""
monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False)
body = _gemini_request_body("gemini-2.5-flash", {}, request_override=True)
generation_config = body["generationConfig"]
assert "response_schema" not in generation_config
assert generation_config["response_json_schema"] == RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA
assert "litellm_param" not in json.dumps(body)
def test__transform_request_body_keeps_response_json_schema_by_default():
"""Without any override, Gemini 2.x keeps sending the verbatim responseJsonSchema"""
body = _gemini_request_body("gemini-2.5-flash", {})
generation_config = body["generationConfig"]
assert "response_schema" not in generation_config
assert generation_config["response_json_schema"] == RESPONSE_SCHEMA_CHANNEL_CLIENT_SCHEMA

View file

@ -11,7 +11,10 @@ from pydantic import BaseModel
import litellm
from litellm import ModelResponse, completion
from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig
from litellm.llms.vertex_ai.common_utils import VertexAIError
from litellm.llms.vertex_ai.common_utils import (
VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM,
VertexAIError,
)
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
@ -337,6 +340,44 @@ def test_vertex_ai_response_json_schema_for_gemini_2():
)
def test_vertex_ai_response_json_schema_opt_out_uses_native_schema(monkeypatch):
"""
litellm.vertex_ai_use_response_json_schema=False sends Gemini 2.x the natively converted
responseSchema, so nullable unions are flattened, constraints survive the flattening and
propertyOrdering is set. The client schema is kept verbatim for a per request opt in.
"""
monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False)
client_schema = {
"type": "object",
"additionalProperties": False,
"properties": {
"total": {"type": "number"},
"barcode": {"anyOf": [{"type": "string", "maxLength": 10}, {"type": "null"}]},
},
"required": ["total", "barcode"],
}
transformed_request = VertexGeminiConfig().map_openai_params(
non_default_params={
"response_format": {
"type": "json_schema",
"json_schema": {"name": "invoice", "schema": deepcopy(client_schema)},
},
},
optional_params={},
model="gemini-2.5-flash",
drop_params=False,
)
assert "response_json_schema" not in transformed_request
assert transformed_request["response_schema"]["propertyOrdering"] == ["total", "barcode"]
assert transformed_request["response_schema"]["properties"]["barcode"]["anyOf"] == [
{"type": "string", "maxLength": 10, "nullable": True}
]
assert "additionalProperties" not in transformed_request["response_schema"]
assert transformed_request[VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM] == client_schema
def test_vertex_ai_response_schema_for_old_models():
"""
Test that older models (Gemini 1.5) automatically use responseSchema.

View file

@ -1,3 +1,4 @@
from copy import deepcopy
from unittest.mock import patch
import pytest
@ -5,13 +6,17 @@ import pytest
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
import litellm
from litellm.llms.vertex_ai.common_utils import (
VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM,
_get_vertex_url,
convert_anyof_null_to_nullable,
get_vertex_location_from_url,
get_vertex_project_id_from_url,
pop_vertex_request_labels,
resolve_response_schema_channel,
set_schema_property_ordering,
should_use_response_json_schema,
supports_response_json_schema,
validate_vertex_location,
vertex_request_labels_from_litellm_params,
@ -178,6 +183,103 @@ def test_supports_response_json_schema(model: str, expected: bool):
assert supports_response_json_schema(model) == expected
CLIENT_SCHEMA = {
"type": "object",
"additionalProperties": False,
"properties": {
"total": {"type": "number"},
"barcode": {"anyOf": [{"type": "string", "maxLength": 10}, {"type": "null"}]},
},
"required": ["total", "barcode"],
}
@pytest.mark.parametrize(
"global_setting, request_override, model, expected",
[
(None, None, "gemini-2.5-flash", True),
(None, None, "gemini-1.5-pro", False),
(False, None, "gemini-2.5-flash", False),
(True, None, "gemini-flash-latest", False),
(True, None, "gemini-1.5-pro", False),
(None, True, "gemini-1.5-pro", False),
(False, True, "gemini-2.5-flash", True),
(True, False, "gemini-2.5-flash", False),
],
)
def test_should_use_response_json_schema_precedence(
monkeypatch, global_setting, request_override, model, expected
):
"""
Per request override beats litellm.vertex_ai_use_response_json_schema, which beats the model
heuristic, and neither can select responseJsonSchema for a model not known to accept it
"""
monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", global_setting)
assert should_use_response_json_schema(model, request_override) is expected
def test_resolve_response_schema_channel_opt_out_converts_to_native_schema():
"""Opting out per request moves the verbatim JSON Schema onto the native responseSchema channel"""
resolved = resolve_response_schema_channel(
optional_params={
"response_mime_type": "application/json",
"response_json_schema": deepcopy(CLIENT_SCHEMA),
"vertex_ai_use_response_json_schema": False,
},
litellm_params={},
model="gemini-2.5-flash",
)
assert "response_json_schema" not in resolved
assert resolved["response_mime_type"] == "application/json"
assert resolved["response_schema"]["propertyOrdering"] == ["total", "barcode"]
assert resolved["response_schema"]["properties"]["barcode"]["anyOf"] == [
{"type": "string", "maxLength": 10, "nullable": True}
]
def test_resolve_response_schema_channel_reads_deployment_litellm_params():
"""A deployment's litellm_params can opt out for every request routed to it"""
resolved = resolve_response_schema_channel(
optional_params={"response_json_schema": deepcopy(CLIENT_SCHEMA)},
litellm_params={"vertex_ai_use_response_json_schema": False},
model="gemini-2.5-flash",
)
assert "response_json_schema" not in resolved
assert resolved["response_schema"]["propertyOrdering"] == ["total", "barcode"]
def test_resolve_response_schema_channel_opt_in_restores_verbatim_schema(monkeypatch):
"""With the global opted out, a per request opt in sends the client schema verbatim again"""
monkeypatch.setattr(litellm, "vertex_ai_use_response_json_schema", False)
resolved = resolve_response_schema_channel(
optional_params={
"response_schema": {"type": "object", "propertyOrdering": ["total", "barcode"]},
VERTEX_AI_VERBATIM_RESPONSE_SCHEMA_PARAM: deepcopy(CLIENT_SCHEMA),
"vertex_ai_use_response_json_schema": True,
},
litellm_params={},
model="gemini-2.5-flash",
)
assert resolved["response_json_schema"] == CLIENT_SCHEMA
assert "response_schema" not in resolved
def test_resolve_response_schema_channel_without_override_keeps_channel():
"""No override leaves the mapped channel untouched"""
optional_params = {"response_json_schema": deepcopy(CLIENT_SCHEMA)}
resolved = resolve_response_schema_channel(
optional_params=optional_params, litellm_params={}, model="gemini-2.5-flash"
)
assert resolved is optional_params
def test_set_schema_property_ordering_with_excessive_nesting():
"""Test set_schema_property_ordering with excessive nesting > max levels +1 deep."""
# generate a schema with excessive nesting