scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 114

[Bug]: Gemini 2.5 Pro – schema validation fails with OpenAI-style type arrays in tools (#14154)

Browse source 
* fix: _convert_schema_types

* fix recursive detector

* test_convert_schema_types_type_array_conversion

* fix: DEFAULT_NUM_WORKERS_LITELLM_PROXY
This commit is contained in:
Ishaan Jaff 2025-09-01 16:53:20 -07:00
parent dea3b53578
commit 5802dfb93d
6 changed files with 292 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs/my-website/docs/proxy/config_settings.md
View file

@ -431,6 +431,7 @@ router_settings:
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy. Default is 4. **We strongly recommend setting NUM Workers to Number of vCPUs available** |
| DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7
| DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03
| DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET | Default reasoning effort disable thinking budget. Default is 0

43
litellm/llms/vertex_ai/common_utils.py
View file

@ -215,6 +215,8 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
# * https://github.com/pydantic/pydantic/discussions/4872
convert_anyof_null_to_nullable(parameters)
_convert_schema_types(parameters)
# Handle empty items objects
process_items(parameters)
add_object_type(parameters)
@ -439,6 +441,47 @@ def _convert_vertex_datetime_to_openai_datetime(vertex_datetime: str) -> int:
return int(dt.timestamp())
def _convert_schema_types(schema, depth=0):
"""
Convert type arrays and lowercase types for Vertex AI compatibility.
Transforms OpenAI-style schemas to Vertex AI format by converting type arrays
like ["string", "number"] to anyOf format and converting all types to uppercase.
"""
if depth > DEFAULT_MAX_RECURSE_DEPTH:
raise ValueError(
f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema. Please check the schema for excessive nesting."
)
if not isinstance(schema, dict):
return
# Handle type field
if "type" in schema:
type_val = schema["type"]
if isinstance(type_val, list) and len(type_val) > 1:
# Convert ["string", "number"] -> {"anyOf": [{"type": "STRING"}, {"type": "NUMBER"}]}
schema["anyOf"] = [{"type": t} for t in type_val if isinstance(t, str)]
schema.pop("type")
elif isinstance(type_val, list) and len(type_val) == 1:
schema["type"] = type_val[0]
elif isinstance(type_val, str):
schema["type"] = type_val
# Recursively process nested properties, items, and anyOf
for key in ["properties", "items", "anyOf"]:
if key in schema:
value = schema[key]
if key == "properties" and isinstance(value, dict):
for prop_schema in value.values():
_convert_schema_types(prop_schema, depth + 1)
elif key == "items":
_convert_schema_types(value, depth + 1)
elif key == "anyOf" and isinstance(value, list):
for anyof_schema in value:
_convert_schema_types(anyof_schema, depth + 1)
def get_vertex_project_id_from_url(url: str) -> Optional[str]:
"""
Get the vertex project id from the url

156
litellm/model_prices_and_context_window_backup.json
View file

@ -11991,6 +11991,108 @@
"mode": "chat",
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 2e-06,
"output_cost_per_token": 8e-06,
"cache_read_input_token_cost": 5e-07,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-2025-04-14": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 2e-06,
"output_cost_per_token": 8e-06,
"cache_read_input_token_cost": 5e-07,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-mini": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 4e-07,
"output_cost_per_token": 1.6e-06,
"cache_read_input_token_cost": 1e-07,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-mini-2025-04-14": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 4e-07,
"output_cost_per_token": 1.6e-06,
"cache_read_input_token_cost": 1e-07,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-nano": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 4e-07,
"cache_read_input_token_cost": 2.5e-08,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-4.1-nano-2025-04-14": {
"max_tokens": 32768,
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 4e-07,
"cache_read_input_token_cost": 2.5e-08,
"litellm_provider": "openrouter",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-5-mini": {
"max_tokens": 128000,
"max_input_tokens": 400000,
@ -14970,10 +15072,10 @@
"output_cost_per_token": 6e-06,
"max_input_tokens": 262000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"supports_tool_choice": false,
"supports_tool_choice": true,
"source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8"
},
"together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
@ -14981,10 +15083,10 @@
"output_cost_per_token": 2e-06,
"max_input_tokens": 256000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"supports_tool_choice": false,
"supports_tool_choice": true,
"source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct"
},
"together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": {
@ -14992,10 +15094,10 @@
"output_cost_per_token": 3e-06,
"max_input_tokens": 256000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"supports_tool_choice": false,
"supports_tool_choice": true,
"source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507"
},
"together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": {
@ -15038,10 +15140,10 @@
"output_cost_per_token": 2.19e-06,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"supports_tool_choice": false,
"supports_tool_choice": true,
"source": "https://www.together.ai/models/deepseek-r1-0528-throughput"
},
"together_ai/mistralai/Mistral-Small-24B-Instruct-2501": {
@ -15066,9 +15168,9 @@
"output_cost_per_token": 6e-07,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"source": "https://www.together.ai/models/gpt-oss-120b"
},
@ -15077,9 +15179,9 @@
"output_cost_per_token": 2e-07,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"source": "https://www.together.ai/models/gpt-oss-20b"
},
@ -15088,12 +15190,24 @@
"output_cost_per_token": 1.1e-06,
"max_input_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": false,
"supports_tool_choice": false,
"supports_parallel_function_calling": false,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_parallel_function_calling": true,
"mode": "chat",
"source": "https://www.together.ai/models/glm-4-5-air"
},
"together_ai/deepseek-ai/DeepSeek-V3.1": {
"input_cost_per_token": 0.6e-06,
"output_cost_per_token": 1.7e-06,
"max_tokens": 128000,
"litellm_provider": "together_ai",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_reasoning": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://www.together.ai/models/deepseek-v3-1"
},
"ollama/codegemma": {
"max_tokens": 8192,
"max_input_tokens": 8192,

3
tests/code_coverage_tests/recursive_detector.py
View file

@ -25,7 +25,8 @@ IGNORE_FUNCTIONS = [
"filter_value_from_dict", # max depth set.
"normalize_json_schema_types", # max depth set.
"_extract_fields_recursive", # max depth set.
"_remove_json_schema_refs", # max depth set.
"_remove_json_schema_refs", # max depth set.,
"_convert_schema_types", # max depth set.,
]

46
tests/llm_translation/base_llm_unit_tests.py
View file

@ -141,6 +141,52 @@ class BaseLLMChatTest(ABC):
# for OpenAI the content contains the JSON schema, so we need to assert that the content is not None
assert response.choices[0].message.content is not None
def test_tool_call_with_property_type_array(self):
litellm._turn_on_debug()
from litellm.utils import supports_function_calling
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
base_completion_call_args = self.get_base_completion_call_args()
if not supports_function_calling(base_completion_call_args["model"], None):
print("Model does not support function calling")
pytest.skip("Model does not support function calling")
base_completion_call_args = self.get_base_completion_call_args()
response = self.completion_function(
**base_completion_call_args,
messages = [
{
"role": "user",
"content": "Tell me if the shoe brand Air Jordan has more models than the shoe brand Nike."
}
],
tools = [
{
"type": "function",
"function": {
"name": "shoe_get_id",
"description": "Get information about a show by its ID or name",
"parameters": {
"type": "object",
"properties": {
"shoe_id": {
"type": ["string", "number"],
"description": "The shoe ID or name"
}
},
"required": ["shoe_id"],
"additionalProperties": False,
"$schema": "http://json-schema.org/draft-07/schema#"
}
}
},
]
)
print(response)
print(json.dumps(response, indent=4, default=str))
def test_streaming(self):
"""Check if litellm handles streaming correctly"""
from litellm.types.utils import ModelResponseStream

65
tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
View file

@ -677,3 +677,68 @@ def test_vertex_filter_format_uri():
)
assert "uri" not in json.dumps(new_parameters)
def test_convert_schema_types_type_array_conversion():
"""
Test _convert_schema_types function handles type arrays and case conversion.
This test verifies the fix for the issue where type arrays like ["string", "number"]
would raise an exception in Vertex AI schema validation.
Relevant issue: https://github.com/BerriAI/litellm/issues/14091
"""
from litellm.llms.vertex_ai.common_utils import _convert_schema_types
# Input: OpenAI-style schema with type array (the problematic case)
input_schema = {
"type": "object",
"properties": {
"studio": {
"type": ["string", "number"],
"description": "The studio ID or name"
}
},
"required": ["studio"],
"additionalProperties": False,
"$schema": "http://json-schema.org/draft-07/schema#"
}
# Expected output: Vertex AI compatible schema with anyOf and uppercase types
expected_output = {
"type": "object",
"properties": {
"studio": {
"anyOf": [
{"type": "string"},
{"type": "number"}
],
"description": "The studio ID or name"
}
},
"required": ["studio"],
"additionalProperties": False,
"$schema": "http://json-schema.org/draft-07/schema#"
}
# Apply the transformation
_convert_schema_types(input_schema)
# Verify the transformation
assert input_schema == expected_output
# Verify specific transformations:
# 1. Root level type converted to uppercase
assert input_schema["type"] == "object"
# 2. Type array converted to anyOf format
assert "anyOf" in input_schema["properties"]["studio"]
assert "type" not in input_schema["properties"]["studio"]
# 3. Individual types in anyOf are uppercase
anyof_types = input_schema["properties"]["studio"]["anyOf"]
assert anyof_types[0]["type"] == "string"
assert anyof_types[1]["type"] == "number"
# 4. Other properties preserved
assert input_schema["properties"]["studio"]["description"] == "The studio ID or name"
assert input_schema["required"] == ["studio"]