From 4e417f9ef1b09490e83530b277b6e60238c1c1cb Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Tue, 13 Jan 2026 19:41:41 -0300 Subject: [PATCH] feat(gemini): add opt-in support for responseJsonSchema (#18147) * feat(gemini): add opt-in support for responseJsonSchema Add support for Gemini's native responseJsonSchema parameter which uses standard JSON Schema format instead of OpenAPI-style responseSchema. Benefits of responseJsonSchema (Gemini 2.0+ only): - Standard JSON Schema format (lowercase types) - Supports additionalProperties for stricter validation - Better compatibility with Pydantic's model_json_schema() - No propertyOrdering required Usage: ```python response_format={ "type": "json_schema", "json_schema": {"schema": {...}}, "use_json_schema": True # opt-in } ``` This is backwards compatible - existing code continues to use responseSchema by default. Closes #16340 * docs: add documentation for use_json_schema parameter Document the new use_json_schema option for Gemini 2.0+ models in the JSON Mode documentation. * refactor(gemini): use responseJsonSchema by default for Gemini 2.0+ Remove opt-in flag `use_json_schema` and automatically detect model version: - Gemini 2.0+: uses responseJsonSchema (standard JSON Schema, supports additionalProperties) - Gemini 1.5: uses responseSchema (OpenAPI format, legacy) This follows LiteLLM's philosophy of abstracting provider differences - users write the same code regardless of model version. --- docs/my-website/docs/completion/json_mode.md | 88 ++++++++++++++++- litellm/llms/vertex_ai/common_utils.py | 66 +++++++++++++ .../vertex_and_google_ai_studio_gemini.py | 66 +++++++++---- litellm/types/llms/vertex_ai.py | 1 + ...test_vertex_and_google_ai_studio_gemini.py | 99 ++++++++++++++++++- 5 files changed, 298 insertions(+), 22 deletions(-) diff --git a/docs/my-website/docs/completion/json_mode.md b/docs/my-website/docs/completion/json_mode.md index 0122e202610..14477f99153 100644 --- a/docs/my-website/docs/completion/json_mode.md +++ b/docs/my-website/docs/completion/json_mode.md @@ -341,4 +341,90 @@ curl http://0.0.0.0:4000/v1/chat/completions \ ``` - \ No newline at end of file + + +## Gemini - Native JSON Schema Format (Gemini 2.0+) + +Gemini 2.0+ models automatically use the native `responseJsonSchema` parameter, which provides better compatibility with standard JSON Schema format. + +### Benefits (Gemini 2.0+): +- Standard JSON Schema format (lowercase types like `string`, `object`) +- Supports `additionalProperties: false` for stricter validation +- Better compatibility with Pydantic's `model_json_schema()` +- No `propertyOrdering` required + +### Usage + + + + +```python +from litellm import completion +from pydantic import BaseModel + +class UserInfo(BaseModel): + name: str + age: int + +response = completion( + model="gemini/gemini-2.0-flash", + messages=[{"role": "user", "content": "Extract: John is 25 years old"}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "user_info", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + }, + "required": ["name", "age"], + "additionalProperties": False # Supported on Gemini 2.0+ + } + } + } +) +``` + + + + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -d '{ + "model": "gemini-2.0-flash", + "messages": [ + {"role": "user", "content": "Extract: John is 25 years old"} + ], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "user_info", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + }, + "required": ["name", "age"], + "additionalProperties": false + } + } + } + }' +``` + + + + +### Model Behavior + +| Model | Format Used | `additionalProperties` Support | +|-------|-------------|-------------------------------| +| Gemini 2.0+ | `responseJsonSchema` (JSON Schema) | ✅ Yes | +| Gemini 1.5 | `responseSchema` (OpenAPI) | ❌ No | + +LiteLLM automatically selects the appropriate format based on the model version. \ No newline at end of file diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 2aa6a00c72b..d8ddc2579d2 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -150,6 +150,34 @@ def get_supports_response_schema( return _supports_response_schema +def supports_response_json_schema(model: str) -> bool: + """ + Check if the model supports responseJsonSchema (JSON Schema format). + + responseJsonSchema is supported by Gemini 2.0+ models and uses standard + JSON Schema format with lowercase types (string, object, etc.) instead of + the OpenAPI-style responseSchema with uppercase types (STRING, OBJECT, etc.). + + Benefits of responseJsonSchema: + - Supports additionalProperties for stricter schema validation + - Uses standard JSON Schema format (no type conversion needed) + - Better compatibility with Pydantic's model_json_schema() + + Args: + model: The model name (e.g., "gemini-2.0-flash", "gemini-2.5-pro") + + Returns: + True if the model supports responseJsonSchema, False otherwise + """ + model_lower = model.lower() + + # Gemini 2.0+ and 2.5+ models support responseJsonSchema + # Pattern matches: gemini-2.0-*, gemini-2.5-*, gemini-3-*, etc. + gemini_2_plus_pattern = re.compile(r"gemini-([2-9]|[1-9]\d+)\.") + + return bool(gemini_2_plus_pattern.search(model_lower)) + + from typing import Literal, Optional all_gemini_url_modes = Literal[ @@ -486,6 +514,44 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False): return parameters +def _build_json_schema(parameters: dict) -> dict: + """ + Build a JSON Schema for use with Gemini's responseJsonSchema parameter. + + Unlike _build_vertex_schema (used for responseSchema), this function: + - Does NOT convert types to uppercase (keeps standard JSON Schema format) + - Does NOT add propertyOrdering + - Does NOT filter fields (allows additionalProperties) + - Still unpacks $defs/$ref (Gemini doesn't support JSON Schema references) + + Parameters: + parameters: dict - the JSON schema to process + + Returns: + dict - the processed schema in standard JSON Schema format + """ + # Unpack $defs references (Gemini doesn't support $ref) + defs = parameters.pop("$defs", {}) + for name, value in defs.items(): + unpack_defs(value, defs) + unpack_defs(parameters, defs) + + # Convert anyOf with null to nullable + convert_anyof_null_to_nullable(parameters) + + # Handle empty strings in enum values - Gemini doesn't accept empty strings in enums + _fix_enum_empty_strings(parameters) + + # Remove enums for non-string typed fields (Gemini requires enum only on strings) + _fix_enum_types(parameters) + + # Handle empty items objects + process_items(parameters) + add_object_type(parameters) + + return parameters + + def _filter_anyof_fields(schema_dict: Dict[str, Any]) -> Dict[str, Any]: """ When anyof is present, only keep the anyof field and its contents - otherwise VertexAI will throw an error - https://github.com/BerriAI/litellm/issues/11164 diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 7a37812bcff..692bd5f1993 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -92,7 +92,12 @@ from litellm.utils import ( ) from ....utils import _remove_additional_properties, _remove_strict_from_schema -from ..common_utils import VertexAIError, _build_vertex_schema +from ..common_utils import ( + VertexAIError, + _build_json_schema, + _build_vertex_schema, + supports_response_json_schema, +) from ..vertex_llm_base import VertexBase from .transformation import ( _gemini_convert_messages_with_history, @@ -624,30 +629,55 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) return old_schema - def apply_response_schema_transformation(self, value: dict, optional_params: dict): + def apply_response_schema_transformation( + self, value: dict, optional_params: dict, model: str + ): new_value = deepcopy(value) - # remove 'additionalProperties' from json schema - new_value = _remove_additional_properties(new_value) - # remove 'strict' from json schema + # remove 'strict' from json schema (not supported by Gemini) new_value = _remove_strict_from_schema(new_value) - if new_value["type"] == "json_object": + + # Automatically use responseJsonSchema for Gemini 2.0+ models + # responseJsonSchema uses standard JSON Schema format and supports additionalProperties + # For older models (Gemini 1.5), fall back to responseSchema (OpenAPI format) + use_json_schema = supports_response_json_schema(model) + + if not use_json_schema: + # For responseSchema, remove 'additionalProperties' (not supported) + new_value = _remove_additional_properties(new_value) + + # Handle response type + if new_value.get("type") == "json_object": optional_params["response_mime_type"] = "application/json" - elif new_value["type"] == "text": + elif new_value.get("type") == "text": optional_params["response_mime_type"] = "text/plain" + + # Extract schema from response_format + schema = None if "response_schema" in new_value: optional_params["response_mime_type"] = "application/json" - optional_params["response_schema"] = new_value["response_schema"] - elif new_value["type"] == "json_schema": # type: ignore - if "json_schema" in new_value and "schema" in new_value["json_schema"]: # type: ignore + schema = new_value["response_schema"] + elif new_value.get("type") == "json_schema": + if "json_schema" in new_value and "schema" in new_value["json_schema"]: optional_params["response_mime_type"] = "application/json" - optional_params["response_schema"] = new_value["json_schema"]["schema"] # type: ignore + schema = new_value["json_schema"]["schema"] - if "response_schema" in optional_params and isinstance( - optional_params["response_schema"], dict - ): - optional_params["response_schema"] = self._map_response_schema( - value=optional_params["response_schema"] - ) + if schema and isinstance(schema, dict): + if use_json_schema: + # Use responseJsonSchema (Gemini 2.0+ only, opt-in) + # - Standard JSON Schema format (lowercase types) + # - Supports additionalProperties + # - No propertyOrdering needed + optional_params["response_json_schema"] = _build_json_schema( + deepcopy(schema) + ) + else: + # Use responseSchema (default, backwards compatible) + # - OpenAPI-style format (uppercase types) + # - No additionalProperties support + # - Requires propertyOrdering + optional_params["response_schema"] = self._map_response_schema( + value=schema + ) @staticmethod def _map_reasoning_effort_to_thinking_budget( @@ -947,7 +977,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): optional_params["max_output_tokens"] = value elif param == "response_format" and isinstance(value, dict): # type: ignore self.apply_response_schema_transformation( - value=value, optional_params=optional_params + value=value, optional_params=optional_params, model=model ) elif param == "frequency_penalty": if self._supports_penalty_parameters(model): diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 74d6339887b..0a4a2d0f14c 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -207,6 +207,7 @@ class GenerationConfig(TypedDict, total=False): frequency_penalty: float response_mime_type: Literal["text/plain", "application/json"] response_schema: dict + response_json_schema: dict seed: int responseLogprobs: bool logprobs: int diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index c0b80ae7716..3c77c5cd509 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -74,6 +74,10 @@ def test_get_model_name_from_gemini_spec_model(): def test_vertex_ai_response_schema_dict(): + """ + Test that older Gemini models (1.5) use responseSchema (OpenAPI format). + responseSchema requires propertyOrdering and doesn't support additionalProperties. + """ v = VertexGeminiConfig() non_default_params = { "messages": [{"role": "user", "content": "Hello, world!"}], @@ -109,7 +113,7 @@ def test_vertex_ai_response_schema_dict(): transformed_request = v.map_openai_params( non_default_params=non_default_params, optional_params={}, - model="gemini-2.0-flash-lite", + model="gemini-1.5-flash", # Old model uses responseSchema (OpenAPI format) drop_params=False, ) @@ -160,6 +164,9 @@ class Step(BaseModel): def test_vertex_ai_response_schema_defs(): + """ + Test that $defs are unpacked for older Gemini models using responseSchema. + """ v = VertexGeminiConfig() schema = cast(dict, v.get_json_schema_from_pydantic_object(MathReasoning)) @@ -173,7 +180,7 @@ def test_vertex_ai_response_schema_defs(): "response_format": schema, }, optional_params={}, - model="gemini-2.0-flash-lite", + model="gemini-1.5-flash", # Old model uses responseSchema (OpenAPI format) drop_params=False, ) @@ -203,7 +210,93 @@ def test_vertex_ai_response_schema_defs(): } +def test_vertex_ai_response_json_schema_for_gemini_2(): + """ + Test that Gemini 2.0+ models automatically use responseJsonSchema. + + responseJsonSchema uses standard JSON Schema format: + - lowercase types (string, object, etc.) + - no propertyOrdering required + - supports additionalProperties + """ + v = VertexGeminiConfig() + + transformed_request = v.map_openai_params( + non_default_params={ + "messages": [{"role": "user", "content": "Hello, world!"}], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + }, + "required": ["name"], + "additionalProperties": False, + }, + }, + }, + }, + optional_params={}, + model="gemini-2.0-flash", # Gemini 2.0+ automatically uses responseJsonSchema + drop_params=False, + ) + + # Should use response_json_schema, not response_schema + assert "response_json_schema" in transformed_request + assert "response_schema" not in transformed_request + + # Types should be lowercase (standard JSON Schema format) + assert transformed_request["response_json_schema"]["type"] == "object" + assert transformed_request["response_json_schema"]["properties"]["name"]["type"] == "string" + assert transformed_request["response_json_schema"]["properties"]["age"]["type"] == "integer" + + # Should NOT have propertyOrdering (not needed for responseJsonSchema) + assert "propertyOrdering" not in transformed_request["response_json_schema"] + + # additionalProperties should be preserved (supported by responseJsonSchema) + assert transformed_request["response_json_schema"].get("additionalProperties") == False + + +def test_vertex_ai_response_schema_for_old_models(): + """ + Test that older models (Gemini 1.5) automatically use responseSchema. + """ + v = VertexGeminiConfig() + + transformed_request = v.map_openai_params( + non_default_params={ + "messages": [{"role": "user", "content": "Hello, world!"}], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + }, + }, + }, + }, + }, + optional_params={}, + model="gemini-1.5-flash", # Old model automatically uses responseSchema + drop_params=False, + ) + + # Should use response_schema for older models + assert "response_schema" in transformed_request + assert "response_json_schema" not in transformed_request + + def test_vertex_ai_retain_property_ordering(): + """ + Test that existing propertyOrdering is preserved for older models using responseSchema. + """ v = VertexGeminiConfig() transformed_request = v.map_openai_params( non_default_params={ @@ -224,7 +317,7 @@ def test_vertex_ai_retain_property_ordering(): }, }, optional_params={}, - model="gemini-2.0-flash-lite", + model="gemini-1.5-flash", # Old model uses responseSchema which needs propertyOrdering drop_params=False, )