scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-30 06:43:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 249
litellm/tests/unit/test_utils_get_optional_params.py
Mateo Wang 0cf2bb7296
fix(openai): warn when the GPT-5 reasoning effort gate drops a level under drop_params (#47761) 
* fix(openai): warn when the GPT-5 reasoning effort gate drops a level under drop_params

A GPT-5 request whose reasoning_effort the model map does not enable for
that model lost the effort with no log line under drop_params. An Azure
deployment name missing from the cost map lost reasoning_effort none this
way and ran at the model's default effort. The gate now logs a warning
naming the model, the level, the map entry it checked, and the
model_info.base_model and model_info.reasoning_effort_levels fixes.

* fix(openai): warn when the Responses route or a deployment's declared levels drop a reasoning effort

The /v1/responses gate dropped a row-disabled GPT-5 effort, and reasoning for an OpenAI model without
supports_reasoning, with no log line, and the router logged a declared-level drop only at INFO. Both
Responses drops now log a WARNING and the router logs its drop at WARNING. The warning now leads with
model_info.reasoning_effort_levels, since base_model only helps a deployment named differently from
its underlying model.

* fix: warn only when a reasoning level is actually dropped, and on generic reasoning_effort drops

* fix: warn when thinking or a Bedrock Converse reasoning level is dropped, and name the router drop a reasoning level

* fix: skip the drop warning when a non-reasoning model is asked for no reasoning, and honor declared levels on Bedrock Responses

* fix: stay silent when reasoning is turned off on /v1/responses and warn on the Bedrock Responses fallback drop

* fix(reasoning): warn when any reasoning-off request is dropped and give each route only the advice it honors

A request to turn reasoning off on a model name the catalog lacks now warns on every route, since it may be a reasoning model. Chat suggests model_info.reasoning_effort_levels and, for a name with no catalog entry, model_info.base_model. OpenAI Responses suggests declared levels only where the config honors them, and Bedrock suggests neither. The Bedrock and Mantle Responses forwarding of declared levels is reverted until it can be proven live.

* fix(reasoning): suggest reasoning_effort_levels and base_model only where the route reads them

The chat gate suggested model_info.reasoning_effort_levels on routes that never pass declared levels
(github_copilot, chatgpt and other OpenAIConfig delegates) and model_info.base_model on OpenAI and
Azure AI chat, whose gate looks up the deployment name rather than base_model. /v1/responses/compact
suggested levels it never reads either.

Declared levels are now None when a caller passes none, so the advice appears only when a caller fed
them, and the base_model advice is limited to the Azure chat gate, the one that looks up base_model.

The openai row of the chat warning test now expects no base_model advice, the Responses warning test
goes through get_optional_params_responses_api the way /v1/responses and compact call it, and the Azure
direct-config test passes declared levels the way get_optional_params does. Long test lines are wrapped.

* fix: read only the built-in catalog when choosing the reasoning drop warning
2026-10-29 04:46:29 -07:00

2243 lines
82 KiB
Python
Raw Blame History

import json
import logging
from typing import Final
from unittest.mock import MagicMock, patch
import httpx
import litellm
import pytest
import respx
from litellm.utils import (
get_optional_params,
get_optional_params_embeddings,
get_optional_params_image_gen,
get_requester_metadata,
validate_openai_optional_params,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.prompt_templates.factory import map_system_message_pt
from litellm.llms.custom_httpx.http_handler import HTTPHandler
COHERE_CHAT_RESPONSE: Final = {
"id": "response-1",
"finish_reason": "COMPLETE",
"message": {"content": [{"type": "text", "text": "ok"}]},
"usage": {"billed_units": {"input_tokens": 1, "output_tokens": 1}},
}
def _check_additional_properties(schema):
if isinstance(schema, dict):
if "additionalProperties" in schema or "strict" in schema:
raise ValueError("additionalProperties and strict should not be in the schema")
for key, value in schema.items():
_check_additional_properties(value)
elif isinstance(schema, list):
for item in schema:
_check_additional_properties(item)
return schema
@pytest.mark.parametrize("stop_sequence, expected_count", [("\n", 0), (["\n"], 0), (["finish_reason"], 1)])
def test_anthropic_optional_params(stop_sequence, expected_count):
"""
Test if whitespace character optional param is dropped by anthropic
"""
litellm.drop_params = True
optional_params = get_optional_params(model="claude-3", custom_llm_provider="anthropic", stop=stop_sequence)
assert len(optional_params) == expected_count
def test_get_requester_metadata_returns_none_for_empty():
metadata = {"requester_metadata": {}}
assert get_requester_metadata(metadata) is None
@patch("litellm.main.openai_chat_completions.completion")
def test_requester_metadata_forwarded_to_openai(mock_completion):
mock_completion.return_value = MagicMock()
metadata = {
"requester_metadata": {
"custom_meta_key": "value",
"hidden_params": "secret",
"int_value": 123,
}
}
original_api_key = litellm.api_key
litellm.api_key = "sk-test"
original_preview_flag = litellm.enable_preview_features
litellm.enable_preview_features = True
try:
litellm.completion(
model="gpt-4o",
messages=[{"role": "user", "content": "hi"}],
metadata=metadata,
)
finally:
litellm.api_key = original_api_key
litellm.enable_preview_features = original_preview_flag
sent_metadata = mock_completion.call_args.kwargs["optional_params"]["metadata"]
assert sent_metadata == {"custom_meta_key": "value"}
def test_get_optional_params_with_allowed_openai_params():
"""
Test if use can dynamically pass in allowed_openai_params to override default behavior
"""
litellm.drop_params = True
tools = [
{
"type": "function",
"function": {
"name": "get_current_time",
"description": "Get the current time in a given location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. San Francisco",
}
},
"required": ["location"],
},
},
}
]
response_format = {"type": "json"}
reasoning_effort = "low"
optional_params = get_optional_params(
model="cf/llama-3.1-70b-instruct",
custom_llm_provider="cloudflare",
allowed_openai_params=["tools", "reasoning_effort", "response_format"],
tools=tools,
response_format=response_format,
reasoning_effort=reasoning_effort,
)
print(f"optional_params: {optional_params}")
assert optional_params["tools"] == tools
assert optional_params["response_format"] == response_format
assert optional_params["reasoning_effort"] == reasoning_effort
def test_allowed_openai_params_does_not_forward_unset_params():
"""
Regression test for https://github.com/BerriAI/litellm/issues/25697
When a user lists a param in ``allowed_openai_params`` but does not
actually send that param in the request, litellm must not forward it
to the provider SDK as ``None``. The openai SDK rejects unknown
top-level kwargs with
``AsyncCompletions.create() got an unexpected keyword argument 'enable_thinking'``.
Reproduces the reported config where the user listed both
``chat_template_kwargs`` and ``enable_thinking`` in
``allowed_openai_params`` and only sent ``chat_template_kwargs``
(with ``enable_thinking`` nested inside it). Previously the loop
added ``optional_params["enable_thinking"] = None`` which then
crashed the openai client.
"""
from litellm.utils import apply_openai_param_overrides
chat_template_kwargs = {"enable_thinking": False}
optional_params: dict = {}
non_default_params = {"chat_template_kwargs": chat_template_kwargs}
result = apply_openai_param_overrides(
optional_params=optional_params,
non_default_params=non_default_params,
allowed_openai_params=["chat_template_kwargs", "enable_thinking"],
)
assert result["chat_template_kwargs"] == chat_template_kwargs
assert "enable_thinking" not in result
assert "chat_template_kwargs" not in non_default_params
def test_bedrock_optional_params_embeddings():
litellm.drop_params = True
optional_params = get_optional_params_embeddings(
model="", user="John", encoding_format=None, custom_llm_provider="bedrock"
)
assert len(optional_params) == 0
@pytest.mark.parametrize(
"model",
[
"us.anthropic.claude-3-haiku-20240307-v1:0",
"us.meta.llama3-2-11b-instruct-v1:0",
"anthropic.claude-3-haiku-20240307-v1:0",
],
)
def test_bedrock_optional_params_completions(model):
tools = [
{
"type": "function",
"function": {
"name": "structure_output",
"description": "Send structured output back to the user",
"strict": True,
"parameters": {
"type": "object",
"properties": {
"reasoning": {"type": "string"},
"sentiment": {"type": "string"},
},
"required": ["reasoning", "sentiment"],
"additionalProperties": False,
},
"additionalProperties": False,
},
}
]
optional_params = get_optional_params(
model=model,
max_tokens=10,
temperature=0.1,
tools=tools,
custom_llm_provider="bedrock",
)
print(f"optional_params: {optional_params}")
assert len(optional_params) == 4
assert optional_params == {
"maxTokens": 10,
"stream": False,
"temperature": 0.1,
"tools": tools,
}
@pytest.mark.parametrize(
"model",
[
"bedrock/amazon.titan-large",
"bedrock/meta.llama3-2-11b-instruct-v1:0",
"bedrock/ai21.j2-ultra-v1",
"bedrock/cohere.command-nightly",
"bedrock/mistral.mistral-7b",
],
)
def test_bedrock_optional_params_simple(model):
litellm.drop_params = True
get_optional_params(
model=model,
max_tokens=10,
temperature=0.1,
custom_llm_provider="bedrock",
)
assert (
get_optional_params(model=model, max_tokens=10, temperature=0.1, custom_llm_provider="bedrock")["temperature"]
== 0.1
)
@pytest.mark.parametrize(
"model, expected_dimensions, dimensions_kwarg",
[
("bedrock/amazon.titan-embed-text-v1", False, None),
("bedrock/amazon.titan-embed-image-v1", True, "embeddingConfig"),
("bedrock/amazon.titan-embed-text-v2:0", True, "dimensions"),
("bedrock/cohere.embed-multilingual-v3", True, None),
],
)
def test_bedrock_optional_params_embeddings_dimension(model, expected_dimensions, dimensions_kwarg):
litellm.drop_params = True
optional_params = get_optional_params_embeddings(
model=model,
user="John",
encoding_format=None,
dimensions=20,
custom_llm_provider="bedrock",
)
if expected_dimensions:
assert len(optional_params) == 1
else:
assert len(optional_params) == 0
if dimensions_kwarg is not None:
assert dimensions_kwarg in optional_params
def test_google_ai_studio_optional_params_embeddings():
optional_params = get_optional_params_embeddings(
model="",
user="John",
encoding_format=None,
custom_llm_provider="gemini",
drop_params=True,
)
assert len(optional_params) == 0
def test_openai_optional_params_embeddings():
litellm.drop_params = True
optional_params = get_optional_params_embeddings(
model="", user="John", encoding_format=None, custom_llm_provider="openai"
)
assert len(optional_params) == 1
assert optional_params["user"] == "John"
def test_azure_optional_params_embeddings():
litellm.drop_params = True
optional_params = get_optional_params_embeddings(
model="chatgpt-v-3",
user="John",
encoding_format=None,
custom_llm_provider="azure",
)
assert len(optional_params) == 1
assert optional_params["user"] == "John"
def test_databricks_optional_params():
litellm.drop_params = True
optional_params = get_optional_params(
model="",
user="John",
custom_llm_provider="databricks",
max_tokens=10,
temperature=0.2,
stream=True,
)
print(f"optional_params: {optional_params}")
assert len(optional_params) == 3
assert "user" not in optional_params
def test_azure_ai_mistral_optional_params():
litellm.drop_params = True
optional_params = get_optional_params(
model="mistral-large-latest",
user="John",
custom_llm_provider="openai",
max_tokens=10,
temperature=0.2,
)
assert "user" not in optional_params
def test_vertex_ai_llama_3_optional_params(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "vertex_llama3_models", {"meta/llama3-405b-instruct-maas"})
litellm.drop_params = True
optional_params = get_optional_params(
model="meta/llama3-405b-instruct-maas",
user="John",
custom_llm_provider="vertex_ai",
max_tokens=10,
temperature=0.2,
)
assert "user" not in optional_params
def test_vertex_ai_mistral_optional_params(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "vertex_mistral_models", {"mistral-large@2407"})
litellm.drop_params = True
optional_params = get_optional_params(
model="mistral-large@2407",
user="John",
custom_llm_provider="vertex_ai",
max_tokens=10,
temperature=0.2,
)
assert "user" not in optional_params
assert "max_tokens" in optional_params
assert "temperature" in optional_params
def test_azure_gpt_optional_params_gpt_vision():
optional_params = litellm.utils.get_optional_params(
model="",
user="John",
custom_llm_provider="azure",
max_tokens=10,
temperature=0.2,
enhancements={"ocr": {"enabled": True}, "grounding": {"enabled": True}},
dataSources=[
{
"type": "AzureComputerVision",
"parameters": {
"endpoint": "<your_computer_vision_endpoint>",
"key": "<your_computer_vision_key>",
},
}
],
)
print(optional_params)
assert optional_params["max_tokens"] == 10
assert optional_params["temperature"] == 0.2
assert optional_params["extra_body"] == {
"enhancements": {"ocr": {"enabled": True}, "grounding": {"enabled": True}},
"dataSources": [
{
"type": "AzureComputerVision",
"parameters": {
"endpoint": "<your_computer_vision_endpoint>",
"key": "<your_computer_vision_key>",
},
}
],
}
def test_azure_gpt_optional_params_gpt_vision_with_extra_body():
optional_params = litellm.utils.get_optional_params(
model="",
user="John",
custom_llm_provider="azure",
max_tokens=10,
temperature=0.2,
extra_body={
"meta": "hi",
},
enhancements={"ocr": {"enabled": True}, "grounding": {"enabled": True}},
dataSources=[
{
"type": "AzureComputerVision",
"parameters": {
"endpoint": "<your_computer_vision_endpoint>",
"key": "<your_computer_vision_key>",
},
}
],
)
print(optional_params)
assert optional_params["max_tokens"] == 10
assert optional_params["temperature"] == 0.2
assert optional_params["extra_body"] == {
"enhancements": {"ocr": {"enabled": True}, "grounding": {"enabled": True}},
"dataSources": [
{
"type": "AzureComputerVision",
"parameters": {
"endpoint": "<your_computer_vision_endpoint>",
"key": "<your_computer_vision_key>",
},
}
],
"meta": "hi",
}
def test_openai_extra_headers():
optional_params = litellm.utils.get_optional_params(
model="",
user="John",
custom_llm_provider="openai",
max_tokens=10,
temperature=0.2,
extra_headers={"AI-Resource Group": "ishaan-resource"},
)
print(optional_params)
assert optional_params["max_tokens"] == 10
assert optional_params["temperature"] == 0.2
assert optional_params["extra_headers"] == {"AI-Resource Group": "ishaan-resource"}
@pytest.mark.parametrize(
"api_version",
[
"2024-02-01",
"2024-07-01",
"2023-07-01-preview",
"2024-03-01-preview",
],
)
def test_azure_tool_choice(api_version):
"""
Test azure tool choice on older + new version
"""
litellm.drop_params = True
optional_params = litellm.utils.get_optional_params(
model="chatgpt-v-3",
user="John",
custom_llm_provider="azure",
max_tokens=10,
temperature=0.2,
extra_headers={"AI-Resource Group": "ishaan-resource"},
tool_choice="required",
api_version=api_version,
)
print(f"{optional_params}")
if api_version == "2024-07-01":
assert optional_params["tool_choice"] == "required"
else:
assert "tool_choice" not in optional_params, (
"tool choice should not be present. Got - tool_choice={} for api version={}".format(
optional_params["tool_choice"], api_version
)
)
@pytest.mark.parametrize(
"model, provider, should_drop",
[("command-r", "cohere", True), ("gpt-3.5-turbo", "openai", False)],
)
def test_drop_params_parallel_tool_calls(model, provider, should_drop):
"""
https://github.com/BerriAI/litellm/issues/4584
"""
response = litellm.utils.get_optional_params(
model=model,
custom_llm_provider=provider,
response_format={"type": "json"},
parallel_tool_calls=True,
drop_params=True,
)
print(response)
if should_drop:
assert "response_format" not in response
assert "parallel_tool_calls" not in response
else:
assert "response_format" in response
assert "parallel_tool_calls" in response
def test_dynamic_drop_additional_params_stream_options():
"""
Make a call to vertex ai, dropping 'stream_options' specifically
"""
optional_params = litellm.utils.get_optional_params(
model="mistral-large-2411@001",
custom_llm_provider="vertex_ai",
stream_options={"include_usage": True},
additional_drop_params=["stream_options"],
)
assert "stream_options" not in optional_params
def test_get_optional_params_image_gen():
response = litellm.utils.get_optional_params_image_gen(aws_region_name="us-east-1", custom_llm_provider="openai")
print(response)
assert "aws_region_name" not in response
response = litellm.utils.get_optional_params_image_gen(aws_region_name="us-east-1", custom_llm_provider="bedrock")
print(response)
assert "aws_region_name" in response
def test_bedrock_optional_params_embeddings_provider_specific_params():
optional_params = get_optional_params_embeddings(
model="my-custom-model",
custom_llm_provider="huggingface",
wait_for_model=True,
)
assert len(optional_params) == 1
@pytest.mark.parametrize(
"provider",
[
"vertex_ai",
"vertex_ai_beta",
],
)
def test_vertex_safety_settings(provider, monkeypatch):
monkeypatch.setattr(litellm, "vertex_ai_safety_settings", [
{
"category": "HARM_CATEGORY_HARASSMENT",
"threshold": "BLOCK_NONE",
},
{
"category": "HARM_CATEGORY_HATE_SPEECH",
"threshold": "BLOCK_NONE",
},
{
"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
"threshold": "BLOCK_NONE",
},
{
"category": "HARM_CATEGORY_DANGEROUS_CONTENT",
"threshold": "BLOCK_NONE",
},
])
optional_params = get_optional_params(
model="gemini-1.5-pro", custom_llm_provider=provider
)
assert len(optional_params) == 1
@pytest.mark.parametrize(
"model, provider, expectedAddProp",
[("gemini-1.5-pro", "vertex_ai_beta", False), ("gpt-3.5-turbo", "openai", True)],
)
def test_parse_additional_properties_json_schema(model, provider, expectedAddProp):
optional_params = get_optional_params(
model=model,
custom_llm_provider=provider,
response_format={
"type": "json_schema",
"json_schema": {
"name": "math_reasoning",
"schema": {
"type": "object",
"properties": {
"steps": {
"type": "array",
"items": {
"type": "object",
"properties": {
"explanation": {"type": "string"},
"output": {"type": "string"},
},
"required": ["explanation", "output"],
"additionalProperties": False,
},
},
"final_answer": {"type": "string"},
},
"required": ["steps", "final_answer"],
"additionalProperties": False,
},
"strict": True,
},
},
)
print(optional_params)
if provider == "vertex_ai_beta":
schema = optional_params["response_schema"]
elif provider == "openai":
schema = optional_params["response_format"]["json_schema"]["schema"]
assert ("additionalProperties" in schema) == expectedAddProp
def test_o1_model_params():
optional_params = get_optional_params(
model="o1-2024-12-17",
custom_llm_provider="openai",
seed=10,
user="John",
)
assert optional_params["seed"] == 10
assert optional_params["user"] == "John"
def test_azure_o1_model_params():
optional_params = get_optional_params(
model="o1",
custom_llm_provider="azure",
seed=10,
user="John",
)
assert optional_params["seed"] == 10
assert optional_params["user"] == "John"
@pytest.mark.parametrize(
"temperature, expected_error",
[(0.2, True), (1, False), (0, True)],
)
@pytest.mark.parametrize("provider", ["openai", "azure"])
def test_o1_model_temperature_params(provider, temperature, expected_error):
if expected_error:
with pytest.raises(litellm.UnsupportedParamsError):
get_optional_params(
model="o1",
custom_llm_provider=provider,
temperature=temperature,
)
else:
get_optional_params(
model="o1-2024-12-17",
custom_llm_provider="openai",
temperature=temperature,
)
def test_unmapped_gemini_model_params():
"""
Test if unmapped gemini model optional params are translated correctly
"""
optional_params = get_optional_params(
model="gemini-new-model",
custom_llm_provider="vertex_ai",
stop="stop_word",
)
assert optional_params["stop_sequences"] == ["stop_word"]
@pytest.mark.parametrize(
"provider, model",
[
("hosted_vllm", "my-vllm-model"),
("gemini", "gemini-1.5-pro"),
("vertex_ai", "gemini-1.5-pro"),
],
)
def test_drop_nested_params_add_prop_and_strict(provider, model):
"""
Relevant issue - https://github.com/BerriAI/litellm/issues/5288
Relevant issue - https://github.com/BerriAI/litellm/issues/6136
"""
tools = [
{
"type": "function",
"function": {
"name": "structure_output",
"description": "Send structured output back to the user",
"strict": True,
"parameters": {
"type": "object",
"properties": {
"reasoning": {"type": "string"},
"sentiment": {"type": "string"},
},
"required": ["reasoning", "sentiment"],
"additionalProperties": False,
},
"additionalProperties": False,
},
}
]
tool_choice = {"type": "function", "function": {"name": "structure_output"}}
optional_params = get_optional_params(
model=model,
custom_llm_provider=provider,
temperature=0.2,
tools=tools,
tool_choice=tool_choice,
additional_drop_params=[
["tools", "function", "strict"],
["tools", "function", "additionalProperties"],
],
)
_check_additional_properties(optional_params["tools"])
def test_hosted_vllm_tool_param():
"""
Relevant issue - https://github.com/BerriAI/litellm/issues/6228
"""
optional_params = get_optional_params(
model="my-vllm-model",
custom_llm_provider="hosted_vllm",
temperature=0.2,
tools=None,
tool_choice=None,
)
assert "tools" not in optional_params
assert "tool_choice" not in optional_params
def test_unmapped_vertex_anthropic_model():
optional_params = get_optional_params(
model="claude-3-5-sonnet-v250@20241022",
custom_llm_provider="vertex_ai",
max_retries=10,
)
assert "max_retries" not in optional_params
@pytest.mark.parametrize("provider", ["anthropic", "vertex_ai"])
def test_anthropic_parallel_tool_calls(provider):
optional_params = get_optional_params(
model="claude-3-5-sonnet-v250@20241022",
custom_llm_provider=provider,
parallel_tool_calls=True,
)
print(f"optional_params: {optional_params}")
assert optional_params["tool_choice"]["disable_parallel_tool_use"] is False
def test_anthropic_computer_tool_use():
tools = [
{
"type": "computer_20241022",
"function": {
"name": "computer",
"parameters": {
"display_height_px": 100,
"display_width_px": 100,
"display_number": 1,
},
},
}
]
optional_params = get_optional_params(
model="claude-3-5-sonnet-v250@20241022",
custom_llm_provider="anthropic",
tools=tools,
)
assert optional_params["tools"][0]["type"] == "computer_20241022"
assert optional_params["tools"][0]["display_height_px"] == 100
assert optional_params["tools"][0]["display_width_px"] == 100
assert optional_params["tools"][0]["display_number"] == 1
def test_vertex_schema_field():
tools = [
{
"type": "function",
"function": {
"name": "json",
"description": "Respond with a JSON object.",
"parameters": {
"type": "object",
"properties": {
"thinking": {
"type": "string",
"description": "Your internal thoughts on different problem details given the guidance.",
},
"problems": {
"type": "array",
"items": {
"type": "object",
"properties": {
"icon": {
"type": "string",
"enum": [
"BarChart2",
"Bell",
],
"description": "The name of a Lucide icon to display",
},
"color": {
"type": "string",
"description": "A Tailwind color class for the icon, e.g., 'text-red-500'",
},
"problem": {
"type": "string",
"description": "The title of the problem being addressed, approximately 3-5 words.",
},
"description": {
"type": "string",
"description": "A brief explanation of the problem, approximately 20 words.",
},
"impacts": {
"type": "array",
"items": {"type": "string"},
"description": "A list of potential impacts or consequences of the problem, approximately 3 words each.",
},
"automations": {
"type": "array",
"items": {"type": "string"},
"description": "A list of potential automations to address the problem, approximately 3-5 words each.",
},
},
"required": [
"icon",
"color",
"problem",
"description",
"impacts",
"automations",
],
"additionalProperties": False,
},
"description": "Please generate problem cards that match this guidance.",
},
},
"required": ["thinking", "problems"],
"additionalProperties": False,
"$schema": "http://json-schema.org/draft-07/schema#",
},
},
}
]
optional_params = get_optional_params(
model="gemini-1.5-flash",
custom_llm_provider="vertex_ai",
tools=tools,
)
print(optional_params)
print(optional_params["tools"][0]["function_declarations"][0])
assert "$schema" not in optional_params["tools"][0]["function_declarations"][0]["parameters"]
def test_watsonx_tool_choice():
optional_params = get_optional_params(model="gemini-1.5-pro", custom_llm_provider="watsonx", tool_choice="auto")
print(optional_params)
assert optional_params["tool_choice_option"] == "auto"
def test_watsonx_text_top_k():
optional_params = get_optional_params(model="gemini-1.5-pro", custom_llm_provider="watsonx_text", top_k=10)
print(optional_params)
assert optional_params["top_k"] == 10
def test_together_ai_model_params():
optional_params = get_optional_params(model="together_ai", custom_llm_provider="together_ai", logprobs=1)
print(optional_params)
assert optional_params["logprobs"] == 1
def test_forward_user_param():
from litellm.utils import get_supported_openai_params, get_optional_params
model = "claude-3-5-sonnet-20240620"
optional_params = get_optional_params(
model=model,
user="test_user",
custom_llm_provider="anthropic",
)
assert optional_params["metadata"]["user_id"] == "test_user"
def test_lm_studio_embedding_params():
optional_params = get_optional_params_embeddings(
model="lm_studio/gemma2-9b-it",
custom_llm_provider="lm_studio",
dimensions=1024,
drop_params=True,
)
assert len(optional_params) == 0
def test_ollama_pydantic_obj():
from pydantic import BaseModel
class ResponseFormat(BaseModel):
x: str
y: str
get_optional_params(
model="qwen2:0.5b",
custom_llm_provider="ollama",
response_format=ResponseFormat,
)
assert get_optional_params(model="qwen2:0.5b", custom_llm_provider="ollama", response_format=ResponseFormat)[
"format"
]["required"] == ["x", "y"]
def test_gemini_frequency_penalty_listed_in_vertex_ai_supported_params():
from litellm.utils import get_supported_openai_params
optional_params = get_supported_openai_params(
model="gemini-1.5-flash",
custom_llm_provider="vertex_ai",
request_type="chat_completion",
)
assert optional_params is not None
assert "frequency_penalty" in optional_params
def test_litellm_proxy_claude_3_5_sonnet():
tools = [
{
"type": "function",
"function": {
"name": "get_current_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA",
},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
},
"required": ["location"],
},
},
}
]
tool_choice = "auto"
optional_params = get_optional_params(
model="claude-3-5-sonnet",
custom_llm_provider="litellm_proxy",
tools=tools,
tool_choice=tool_choice,
)
assert optional_params["tools"] == tools
assert optional_params["tool_choice"] == tool_choice
def test_is_vertex_anthropic_model():
assert (
litellm.VertexAIAnthropicConfig().is_supported_model(
model="claude-3-5-sonnet", custom_llm_provider="litellm_proxy"
)
is False
)
def test_groq_response_format_json_schema():
optional_params = get_optional_params(
model="llama-3.1-70b-versatile",
custom_llm_provider="groq",
response_format={"type": "json_object"},
)
assert optional_params is not None
assert "response_format" in optional_params
assert optional_params["response_format"]["type"] == "json_object"
def test_gemini_frequency_penalty():
optional_params = get_optional_params(model="gemini-1.5-flash", custom_llm_provider="gemini", frequency_penalty=0.5)
assert optional_params["frequency_penalty"] == 0.5
def test_azure_prediction_param():
optional_params = get_optional_params(
model="chatgpt-v2",
custom_llm_provider="azure",
prediction={
"type": "content",
"content": "LiteLLM is a very useful way to connect to a variety of LLMs.",
},
)
assert optional_params["prediction"] == {
"type": "content",
"content": "LiteLLM is a very useful way to connect to a variety of LLMs.",
}
def test_vertex_ai_ft_llama():
optional_params = get_optional_params(
model="1984786713414729728",
custom_llm_provider="vertex_ai",
frequency_penalty=0.5,
max_retries=10,
)
assert optional_params["frequency_penalty"] == 0.5
assert "max_retries" not in optional_params
@pytest.mark.parametrize(
"model, expected_thinking",
[
("claude-3-5-sonnet", False),
("claude-3-7-sonnet", True),
("gpt-3.5-turbo", False),
],
)
def test_anthropic_thinking_param(model, expected_thinking):
optional_params = get_optional_params(
model=model,
custom_llm_provider="anthropic",
thinking={"type": "enabled", "budget_tokens": 1024},
drop_params=True,
)
if expected_thinking:
assert "thinking" in optional_params
else:
assert "thinking" not in optional_params
def test_bedrock_invoke_anthropic_max_tokens():
passed_params = {
"model": "invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"functions": None,
"function_call": None,
"temperature": 0.8,
"top_p": None,
"n": 1,
"stream": False,
"stream_options": None,
"stop": None,
"max_tokens": None,
"max_completion_tokens": 1024,
"modalities": None,
"prediction": None,
"audio": None,
"presence_penalty": None,
"frequency_penalty": None,
"logit_bias": None,
"user": None,
"custom_llm_provider": "bedrock",
"response_format": {"type": "text"},
"seed": None,
"tools": [
{
"type": "function",
"function": {
"name": "generate_plan",
"description": "Generate a plan to execute the task using only the tools outlined in your context.",
"input_schema": {
"type": "object",
"properties": {
"steps": {
"type": "array",
"items": {
"type": "object",
"properties": {
"type": {
"type": "string",
"description": "The type of step to execute",
},
"tool_name": {
"type": "string",
"description": "The name of the tool to use for this step",
},
"tool_input": {
"type": "object",
"description": "The input to pass to the tool. Make sure this complies with the schema for the tool.",
},
"tool_output": {
"type": "object",
"description": "(Optional) The output from the tool if needed for future steps. Make sure this complies with the schema for the tool.",
},
},
"required": ["type"],
},
}
},
},
},
},
{
"type": "function",
"function": {
"name": "generate_wire_tool",
"description": "Create a wire transfer with complete wire instructions",
"input_schema": {
"type": "object",
"properties": {
"company_id": {
"type": "integer",
"description": "The ID of the company receiving the investment",
},
"investment_id": {
"type": "integer",
"description": "The ID of the investment memo",
},
"dollar_amount": {
"type": "number",
"description": "The amount to wire in USD",
},
"wiring_instructions": {
"type": "object",
"description": "Complete bank account and routing information for the wire",
"properties": {
"account_name": {
"type": "string",
"description": "Name on the bank account",
},
"address_1": {
"type": "string",
"description": "Primary address line",
},
"address_2": {
"type": "string",
"description": "Secondary address line (optional)",
},
"city": {"type": "string"},
"state": {"type": "string"},
"zip": {"type": "string"},
"country": {"type": "string", "default": "US"},
"bank_name": {"type": "string"},
"account_number": {"type": "string"},
"routing_number": {"type": "string"},
"account_type": {
"type": "string",
"enum": ["checking", "savings"],
"default": "checking",
},
"swift_code": {
"type": "string",
"description": "Required for international wires",
},
"iban": {
"type": "string",
"description": "Required for some international wires",
},
"bank_city": {"type": "string"},
"bank_state": {"type": "string"},
"bank_country": {"type": "string", "default": "US"},
"bank_to_bank_instructions": {
"type": "string",
"description": "Additional instructions for the bank (optional)",
},
"intermediary_bank_name": {
"type": "string",
"description": "Name of intermediary bank if required (optional)",
},
},
"required": [
"account_name",
"address_1",
"country",
"bank_name",
"account_number",
"routing_number",
"account_type",
"bank_country",
],
},
},
"required": [
"company_id",
"investment_id",
"dollar_amount",
"wiring_instructions",
],
},
},
},
{
"type": "function",
"function": {
"name": "search_companies",
"description": "Search for companies by name or other criteria to get their IDs",
"input_schema": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Name or part of name to search for",
},
"batch": {
"type": "string",
"description": 'Optional batch filter (e.g., "W21", "S22")',
},
"status": {
"type": "string",
"enum": [
"live",
"dead",
"adrift",
"exited",
"went_public",
"all",
],
"description": "Filter by company status",
"default": "live",
},
"limit": {
"type": "integer",
"description": "Maximum number of results to return",
"default": 10,
},
},
"required": ["query"],
},
"output_schema": {
"type": "object",
"properties": {
"status": {
"type": "string",
"description": "Success or error status",
},
"results": {
"type": "array",
"description": "List of companies matching the search criteria",
"items": {
"type": "object",
"properties": {
"id": {
"type": "integer",
"description": "Company ID to use in other API calls",
},
"name": {"type": "string"},
"batch": {"type": "string"},
"status": {"type": "string"},
"valuation": {"type": "string"},
"url": {"type": "string"},
"description": {"type": "string"},
"founders": {"type": "string"},
},
},
},
"results_count": {
"type": "integer",
"description": "Number of companies returned",
},
"total_matches": {
"type": "integer",
"description": "Total number of matches found",
},
},
},
},
},
],
"tool_choice": None,
"max_retries": 0,
"logprobs": None,
"top_logprobs": None,
"extra_headers": None,
"api_version": None,
"parallel_tool_calls": None,
"drop_params": True,
"reasoning_effort": None,
"additional_drop_params": None,
"messages": [
{
"role": "system",
"content": "You are an AI assistant that helps prepare a wire for a pro rata investment.",
},
{"role": "user", "content": [{"type": "text", "text": "hi"}]},
],
"thinking": None,
"kwargs": {},
}
optional_params = get_optional_params(**passed_params)
print(f"optional_params: {optional_params}")
assert "max_tokens_to_sample" not in optional_params
assert optional_params["max_tokens"] == 1024
def test_bedrock_invoke_claude_4_anthropic_max_tokens():
passed_params = {
"model": "invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
"functions": None,
"function_call": None,
"temperature": 0.8,
"top_p": None,
"n": 1,
"stream": False,
"stream_options": None,
"stop": None,
"max_tokens": None,
"max_completion_tokens": 1024,
"modalities": None,
"prediction": None,
"audio": None,
"presence_penalty": None,
"frequency_penalty": None,
"logit_bias": None,
"user": None,
"custom_llm_provider": "bedrock",
"response_format": {"type": "text"},
"seed": None,
"tools": [
{
"type": "function",
"function": {
"name": "generate_plan",
"description": "Generate a plan to execute the task using only the tools outlined in your context.",
"input_schema": {
"type": "object",
"properties": {
"steps": {
"type": "array",
"items": {
"type": "object",
"properties": {
"type": {
"type": "string",
"description": "The type of step to execute",
},
"tool_name": {
"type": "string",
"description": "The name of the tool to use for this step",
},
"tool_input": {
"type": "object",
"description": "The input to pass to the tool. Make sure this complies with the schema for the tool.",
},
"tool_output": {
"type": "object",
"description": "(Optional) The output from the tool if needed for future steps. Make sure this complies with the schema for the tool.",
},
},
"required": ["type"],
},
}
},
},
},
},
{
"type": "function",
"function": {
"name": "generate_wire_tool",
"description": "Create a wire transfer with complete wire instructions",
"input_schema": {
"type": "object",
"properties": {
"company_id": {
"type": "integer",
"description": "The ID of the company receiving the investment",
},
"investment_id": {
"type": "integer",
"description": "The ID of the investment memo",
},
"dollar_amount": {
"type": "number",
"description": "The amount to wire in USD",
},
"wiring_instructions": {
"type": "object",
"description": "Complete bank account and routing information for the wire",
"properties": {
"account_name": {
"type": "string",
"description": "Name on the bank account",
},
"address_1": {
"type": "string",
"description": "Primary address line",
},
"address_2": {
"type": "string",
"description": "Secondary address line (optional)",
},
"city": {"type": "string"},
"state": {"type": "string"},
"zip": {"type": "string"},
"country": {"type": "string", "default": "US"},
"bank_name": {"type": "string"},
"account_number": {"type": "string"},
"routing_number": {"type": "string"},
"account_type": {
"type": "string",
"enum": ["checking", "savings"],
"default": "checking",
},
"swift_code": {
"type": "string",
"description": "Required for international wires",
},
"iban": {
"type": "string",
"description": "Required for some international wires",
},
"bank_city": {"type": "string"},
"bank_state": {"type": "string"},
"bank_country": {"type": "string", "default": "US"},
"bank_to_bank_instructions": {
"type": "string",
"description": "Additional instructions for the bank (optional)",
},
"intermediary_bank_name": {
"type": "string",
"description": "Name of intermediary bank if required (optional)",
},
},
"required": [
"account_name",
"address_1",
"country",
"bank_name",
"account_number",
"routing_number",
"account_type",
"bank_country",
],
},
},
"required": [
"company_id",
"investment_id",
"dollar_amount",
"wiring_instructions",
],
},
},
},
{
"type": "function",
"function": {
"name": "search_companies",
"description": "Search for companies by name or other criteria to get their IDs",
"input_schema": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Name or part of name to search for",
},
"batch": {
"type": "string",
"description": 'Optional batch filter (e.g., "W21", "S22")',
},
"status": {
"type": "string",
"enum": [
"live",
"dead",
"adrift",
"exited",
"went_public",
"all",
],
"description": "Filter by company status",
"default": "live",
},
"limit": {
"type": "integer",
"description": "Maximum number of results to return",
"default": 10,
},
},
"required": ["query"],
},
"output_schema": {
"type": "object",
"properties": {
"status": {
"type": "string",
"description": "Success or error status",
},
"results": {
"type": "array",
"description": "List of companies matching the search criteria",
"items": {
"type": "object",
"properties": {
"id": {
"type": "integer",
"description": "Company ID to use in other API calls",
},
"name": {"type": "string"},
"batch": {"type": "string"},
"status": {"type": "string"},
"valuation": {"type": "string"},
"url": {"type": "string"},
"description": {"type": "string"},
"founders": {"type": "string"},
},
},
},
"results_count": {
"type": "integer",
"description": "Number of companies returned",
},
"total_matches": {
"type": "integer",
"description": "Total number of matches found",
},
},
},
},
},
],
"tool_choice": None,
"max_retries": 0,
"logprobs": None,
"top_logprobs": None,
"extra_headers": None,
"api_version": None,
"parallel_tool_calls": None,
"drop_params": True,
"reasoning_effort": None,
"additional_drop_params": None,
"messages": [
{
"role": "system",
"content": "You are an AI assistant that helps prepare a wire for a pro rata investment.",
},
{"role": "user", "content": [{"type": "text", "text": "hi"}]},
],
"thinking": None,
"kwargs": {},
}
optional_params = get_optional_params(**passed_params)
print(f"optional_params: {optional_params}")
assert "max_tokens_to_sample" not in optional_params
assert optional_params["max_tokens"] == 1024
def test_azure_modalities_param():
optional_params = get_optional_params(
model="chatgpt-v2",
custom_llm_provider="azure",
modalities=["text", "audio"],
audio={"type": "audio_input", "input": "test.wav"},
)
assert optional_params["modalities"] == ["text", "audio"]
assert optional_params["audio"] == {"type": "audio_input", "input": "test.wav"}
def test_litellm_proxy_thinking_param():
optional_params = get_optional_params(
model="gpt-4o",
custom_llm_provider="litellm_proxy",
thinking={"type": "enabled", "budget_tokens": 1024},
)
assert optional_params["extra_body"]["thinking"] == {
"type": "enabled",
"budget_tokens": 1024,
}
def test_gemini_modalities_param():
optional_params = get_optional_params(
model="gemini-1.5-pro",
custom_llm_provider="gemini",
modalities=["text", "image"],
)
assert optional_params["responseModalities"] == ["TEXT", "IMAGE"]
def test_azure_response_format_param():
optional_params = litellm.get_optional_params(
model="azure/o_series/test-o3-mini",
custom_llm_provider="azure/o_series",
tools=[
{
"type": "function",
"function": {
"name": "get_current_time",
"description": "Get the current time in a given location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name, e.g. San Francisco",
}
},
"required": ["location"],
},
},
}
],
)
assert optional_params["tools"][0]["function"]["name"] == "get_current_time"
@pytest.mark.parametrize(
"model, provider",
[
("claude-3-7-sonnet-20240620-v1:0", "anthropic"),
("anthropic.claude-sonnet-4-5-20250929-v1:0", "bedrock"),
("invoke/anthropic.claude-3-7-sonnet-20240620-v1:0", "bedrock"),
("claude-3-7-sonnet@20250219", "vertex_ai"),
],
)
def test_anthropic_unified_reasoning_content(model, provider):
from litellm.constants import DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET
optional_params = get_optional_params(
model=model,
custom_llm_provider=provider,
reasoning_effort="high",
)
assert optional_params["thinking"] == {
"type": "enabled",
"budget_tokens": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
}
def test_azure_response_format(monkeypatch):
monkeypatch.setenv("AZURE_API_VERSION", "2025-02-01")
optional_params = get_optional_params(
model="azure/gpt-4o-mini",
custom_llm_provider="azure",
response_format={"type": "json_object"},
)
assert optional_params["response_format"] == {"type": "json_object"}
def test_cohere_embed_dimensions_param():
optional_params = get_optional_params_embeddings(
model="embed-multilingual-v3.0",
custom_llm_provider="cohere",
encoding_format="float",
)
assert optional_params["embedding_types"] == ["float"]
def test_optional_params_with_additional_drop_params():
optional_params = get_optional_params(
model="gpt-4o",
custom_llm_provider="openai",
additional_drop_params=["red"],
drop_params=True,
red="blue",
)
print(f"optional_params: {optional_params}")
assert "red" not in optional_params
assert "red" not in optional_params["extra_body"]
def test_azure_ai_cohere_embed_input_type_param():
optional_params = get_optional_params_embeddings(
model="embed-v-4-0",
custom_llm_provider="azure_ai",
input_type="text",
dimensions=1536,
)
assert optional_params["dimensions"] == 1536
assert optional_params["extra_body"]["input_type"] == "text"
def test_optional_params_image_gen_with_aspect_ratio():
optional_params = get_optional_params_image_gen(
model="imagen-4.0-ultra-generate-001",
custom_llm_provider="vertex_ai",
aspect_ratio="16:9",
)
assert optional_params["aspect_ratio"] == "16:9"
def test_validate_openai_optional_params_stop_truncation():
"""
Test that validate_openai_optional_params truncates stop sequences to 4 elements
when more than 4 are provided, as OpenAI only supports up to 4 stop sequences.
"""
stop_sequences = ["stop1", "stop2", "stop3", "stop4", "stop5", "stop6"]
result = validate_openai_optional_params(stop=stop_sequences)
assert result == ["stop1", "stop2", "stop3", "stop4"]
assert len(result) == 4
stop_sequences_4 = ["stop1", "stop2", "stop3", "stop4"]
result = validate_openai_optional_params(stop=stop_sequences_4)
assert result == ["stop1", "stop2", "stop3", "stop4"]
assert len(result) == 4
stop_sequences_2 = ["stop1", "stop2"]
result = validate_openai_optional_params(stop=stop_sequences_2)
assert result == ["stop1", "stop2"]
assert len(result) == 2
stop_string = "stop1"
result = validate_openai_optional_params(stop=stop_string)
assert result == "stop1"
result = validate_openai_optional_params(stop=None)
assert result is None
result = validate_openai_optional_params(stop=[])
assert result == []
def test_validate_openai_optional_params_disable_stop_sequence_limit():
"""
Test that validate_openai_optional_params respects the disable_stop_sequence_limit flag.
When litellm.disable_stop_sequence_limit is True, stop sequences should not be truncated.
"""
original_value = litellm.disable_stop_sequence_limit
try:
litellm.disable_stop_sequence_limit = True
stop_sequences = ["stop1", "stop2", "stop3", "stop4", "stop5", "stop6"]
result = validate_openai_optional_params(stop=stop_sequences)
assert result == ["stop1", "stop2", "stop3", "stop4", "stop5", "stop6"]
assert len(result) == 6
litellm.disable_stop_sequence_limit = False
stop_sequences = ["stop1", "stop2", "stop3", "stop4", "stop5", "stop6"]
result = validate_openai_optional_params(stop=stop_sequences)
assert result == ["stop1", "stop2", "stop3", "stop4"]
assert len(result) == 4
finally:
litellm.disable_stop_sequence_limit = original_value
def test_validate_openai_optional_params_integration():
"""
Test that validate_openai_optional_params is properly integrated in the completion flow.
"""
try:
with patch("litellm.llms.openai.openai.OpenAI") as mock_client:
mock_response = MagicMock()
mock_response.choices = [MagicMock()]
mock_response.choices[0].message.content = "Test response"
mock_response.model = "gpt-3.5-turbo"
mock_response.id = "test-id"
mock_response.created = 1234567890
mock_response.usage = MagicMock()
mock_response.usage.prompt_tokens = 10
mock_response.usage.completion_tokens = 5
mock_response.usage.total_tokens = 15
mock_client.return_value.chat.completions.create.return_value = mock_response
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello"}],
stop=["stop1", "stop2", "stop3", "stop4", "stop5", "stop6"],
mock_response="Test response",
)
assert response is not None
except Exception as e:
pytest.fail(f"validate_openai_optional_params integration failed: {e}")
def test_drop_store_param_for_anthropic():
"""
Test that the OpenAI-specific `store` parameter is correctly dropped
when calling Anthropic with drop_params=True.
`store` is an OpenAI Chat Completion parameter (for storing completions
for distillation/evals) that Anthropic does not support. Without proper
handling, it leaks through to the Anthropic API and causes a
"store: Extra inputs are not permitted" error.
Ref: https://github.com/BerriAI/litellm/issues/19700
"""
optional_params = get_optional_params(
model="claude-sonnet-4-5-20250929",
custom_llm_provider="anthropic",
drop_params=True,
store=True,
)
assert "store" not in optional_params
def test_additional_drop_params_store_for_anthropic():
"""
Test that `additional_drop_params=["store"]` correctly strips the `store`
parameter for non-OpenAI providers like Anthropic.
Ref: https://github.com/BerriAI/litellm/issues/19700
"""
optional_params = get_optional_params(
model="claude-sonnet-4-5-20250929",
custom_llm_provider="anthropic",
additional_drop_params=["store"],
store=True,
)
assert "store" not in optional_params
def test_store_in_openai_chat_completion_params():
"""
Test that `store` is recognized as a standard OpenAI Chat Completion
parameter. This ensures it is correctly handled by helper functions
like `get_standard_openai_params()` and provider configs that rely on
`OPENAI_CHAT_COMPLETION_PARAMS`.
Without `store` in this list, functions that filter by known OpenAI
params will silently drop it for OpenAI calls or incorrectly treat
it as a provider-specific param for non-OpenAI providers.
Ref: https://github.com/BerriAI/litellm/issues/19700
"""
from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
assert "store" in OPENAI_CHAT_COMPLETION_PARAMS
from litellm.utils import get_standard_openai_params
result = get_standard_openai_params({"store": True, "temperature": 0.7})
assert "store" in result
assert result["store"] is True
def test_store_param_passed_through_openai_azure():
"""
Test that the `store` parameter is correctly passed through to OpenAI
and Azure OpenAI providers when using get_optional_params().
This verifies the fix for the regression where `store` was being filtered
out by get_non_default_completion_params() due to architectural issues
in parameter processing pipeline.
Ref: https://github.com/BerriAI/litellm/issues/19700
"""
optional_params_openai = get_optional_params(
model="gpt-4o",
custom_llm_provider="openai",
store=True,
)
assert "store" in optional_params_openai
assert optional_params_openai["store"] is True
optional_params_azure = get_optional_params(
model="gpt-4.1-2025-04-14",
custom_llm_provider="azure",
store=True,
)
assert "store" in optional_params_azure
assert optional_params_azure["store"] is True
optional_params_false = get_optional_params(
model="gpt-4o",
custom_llm_provider="openai",
store=False,
)
assert "store" in optional_params_false
assert optional_params_false["store"] is False
def test_cohere_drop_params_removes_unsupported_response_format(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
params: Final = get_optional_params(
model="command-r-plus-08-2024",
custom_llm_provider="cohere",
response_format={"type": "json_object"},
drop_params=True,
)
assert "response_format" not in params
@pytest.mark.parametrize("drop_params", [False, None])
def test_cohere_drop_params_rejects_unsupported_response_format(
drop_params: bool | None, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
with pytest.raises(litellm.UnsupportedParamsError):
get_optional_params(
model="command-r-plus-08-2024",
custom_llm_provider="cohere",
response_format={"type": "json_object"},
drop_params=drop_params,
)
@pytest.mark.respx(assert_all_called=True)
def test_cohere_completion_drops_response_format_from_request(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post("https://api.cohere.com/v2/chat")
route.return_value = httpx.Response(200, json=COHERE_CHAT_RESPONSE)
litellm.completion(
model="cohere/command-r-plus-08-2024",
messages=[{"role": "user", "content": "hello"}],
response_format={"type": "json_object"},
drop_params=True,
api_key="test-api-key",
client=HTTPHandler(),
)
request_body: Final = json.loads(route.calls[0].request.content)
assert request_body == {"model": "command-r-plus-08-2024", "messages": [{"role": "user", "content": "hello"}]}
@pytest.mark.respx(assert_all_called=True)
def test_cohere_completion_drops_parallel_tool_calls_from_request(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post("https://api.cohere.com/v2/chat")
route.return_value = httpx.Response(200, json=COHERE_CHAT_RESPONSE)
litellm.completion(
model="cohere/command-r-plus-08-2024",
messages=[{"role": "user", "content": "hello"}],
parallel_tool_calls=True,
drop_params=True,
api_key="test-api-key",
client=HTTPHandler(),
)
request_body: Final = json.loads(route.calls[0].request.content)
assert request_body == {"model": "command-r-plus-08-2024", "messages": [{"role": "user", "content": "hello"}]}
@pytest.mark.respx(assert_all_called=True)
def test_cohere_completion_passes_extra_params_without_api_key(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post("https://api.cohere.com/v2/chat")
route.return_value = httpx.Response(200, json=COHERE_CHAT_RESPONSE)
litellm.completion(
model="cohere/command-r-plus-08-2024",
messages=[{"role": "user", "content": "hello"}],
custom_param="test",
api_key="test-api-key",
client=HTTPHandler(),
)
request_body: Final = json.loads(route.calls[0].request.content)
assert request_body == {
"model": "command-r-plus-08-2024",
"messages": [{"role": "user", "content": "hello"}],
"custom_param": "test",
}
def test_cohere_additional_drop_params_removes_only_selected_param(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
params: Final = get_optional_params(
model="command-r-plus-08-2024",
custom_llm_provider="cohere",
max_tokens=32,
response_format={"type": "json_object"},
additional_drop_params=["response_format"],
)
assert params["max_tokens"] == 32
assert "response_format" not in params
@pytest.mark.respx(assert_all_called=True)
def test_cohere_completion_applies_additional_drop_params_to_request(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post("https://api.cohere.com/v2/chat")
route.return_value = httpx.Response(200, json=COHERE_CHAT_RESPONSE)
litellm.completion(
model="cohere/command-r-plus-08-2024",
messages=[{"role": "user", "content": "hello"}],
response_format={"type": "json_object"},
additional_drop_params=["response_format"],
api_key="test-api-key",
client=HTTPHandler(),
)
request_body: Final = json.loads(route.calls[0].request.content)
assert request_body == {"model": "command-r-plus-08-2024", "messages": [{"role": "user", "content": "hello"}]}
class _PreCallOptionalParamsCapture(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.max_retries: tuple[object, ...] = ()
def log_pre_api_call(self, model: str, messages: list[object], kwargs: dict[str, dict[str, object]]) -> None:
self.max_retries = (*self.max_retries, kwargs["optional_params"].get("max_retries"))
@pytest.mark.respx(assert_all_called=True)
def test_completion_passes_num_retries_to_optional_params_as_max_retries(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
capture: Final = _PreCallOptionalParamsCapture()
monkeypatch.setattr(litellm, "callbacks", [capture])
respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
return_value=httpx.Response(
200,
json={
"id": "chatcmpl-retries",
"object": "chat.completion",
"created": 0,
"model": "gpt-5.6",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
)
)
response: Final = litellm.completion(
model="gpt-5.6",
messages=[{"role": "user", "content": "Hello world"}],
num_retries=10,
api_key="sk-openai-test",
)
assert capture.max_retries == (10,)
assert response.choices[0].message.content == "ok"
@pytest.mark.respx(assert_all_called=True)
def test_responses_allowed_openai_params_are_sent(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post("https://api.perplexity.ai/v1/responses")
route.return_value = httpx.Response(
200,
json={
"id": "resp_1",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gpt-5.6",
"output": [
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
}
],
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
},
)
with pytest.raises(litellm.UnsupportedParamsError):
litellm.responses(
model="perplexity/sonar",
input="Hello",
store=True,
api_key="test-api-key",
client=HTTPHandler(),
)
litellm.responses(
model="perplexity/sonar",
input="Hello",
store=True,
allowed_openai_params=["store"],
api_key="test-api-key",
client=HTTPHandler(),
)
request_body: Final = json.loads(route.calls[0].request.content)
assert request_body["store"] is True
def test_supports_system_message_merges_system_prompt_into_user_message() -> None:
messages: Final = [
{"role": "system", "content": "Listen here!"},
{"role": "user", "content": "Hello there!"},
]
assert map_system_message_pt(messages=messages) == [{"role": "user", "content": "Listen here! Hello there!"}]
def test_dropping_an_unsupported_reasoning_effort_logs_one_warning(
local_model_cost_map: None, caplog: pytest.LogCaptureFixture
) -> None:
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
optional_params: Final = get_optional_params(
model="terra-ptu", custom_llm_provider="azure", reasoning_effort="none", drop_params=True
)
assert "reasoning_effort" not in optional_params
drop_warnings: Final = [
record.getMessage() for record in caplog.records if "Dropping reasoning_effort" in record.getMessage()
]
assert len(drop_warnings) == 1
assert all(fragment in drop_warnings[0] for fragment in ("'none'", "terra-ptu", "model_info.base_model"))
@pytest.mark.parametrize("provider", ["ollama", "ollama_chat", "lemonade", "huggingface"])
def test_dropping_thinking_for_an_unmapped_local_model_sends_no_request(
local_model_cost_map: None, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, provider: str
) -> None:
outbound: Final = respx.MockRouter(assert_all_mocked=False)
monkeypatch.setattr(
litellm, "module_level_client", HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(outbound.handler)))
)
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
optional_params: Final = get_optional_params(
model="unmapped-local-model", custom_llm_provider=provider, thinking={"type": "disabled"}, drop_params=True
)
assert "thinking" not in optional_params
assert len([record for record in caplog.records if "Dropping thinking" in record.getMessage()]) == 1
assert not outbound.calls
def test_dropping_an_unsupported_thinking_logs_one_warning(
local_model_cost_map: None, caplog: pytest.LogCaptureFixture
) -> None:
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
optional_params: Final = get_optional_params(
model="terra-ptu", custom_llm_provider="azure", thinking={"type": "disabled"}, drop_params=True
)
assert "thinking" not in optional_params
drop_warnings: Final = [
record.getMessage() for record in caplog.records if "Dropping thinking" in record.getMessage()
]
assert len(drop_warnings) == 1
assert all(fragment in drop_warnings[0] for fragment in ("'disabled'", "terra-ptu", "model_info.base_model"))
@pytest.mark.parametrize(
("reasoning_kwargs", "warns"),
[
({"reasoning_effort": "none"}, False),
({"thinking": {"type": "disabled"}}, False),
({"reasoning_effort": "high"}, True),
({"thinking": {"type": "enabled", "budget_tokens": 1024}}, True),
],
)
def test_reasoning_drop_warns_unless_a_mapped_non_reasoning_model_was_asked_for_none(
local_model_cost_map: None,
caplog: pytest.LogCaptureFixture,
reasoning_kwargs: dict[str, object],
warns: bool,
) -> None:
litellm.register_model(
{
"azure/plain-chat-test-deployment": {
"litellm_provider": "azure",
"mode": "chat",
"max_tokens": 1024,
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"supports_reasoning": False,
}
}
)
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
optional_params: Final = get_optional_params(
model="plain-chat-test-deployment", custom_llm_provider="azure", drop_params=True, **reasoning_kwargs
)
assert not set(reasoning_kwargs) & set(optional_params)
assert any("Dropping" in record.getMessage() for record in caplog.records) is warns
@pytest.mark.parametrize("reasoning_kwargs", [{"reasoning_effort": "none"}, {"thinking": {"type": "disabled"}}])
def test_turning_reasoning_off_warns_when_only_custom_pricing_describes_the_model(
local_model_cost_map: None, caplog: pytest.LogCaptureFixture, reasoning_kwargs: dict[str, object]
) -> None:
litellm.register_model(
{
"azure/pricing-only-test-deployment": {
"litellm_provider": "azure",
"mode": "chat",
"input_cost_per_token": 0.000001,
"output_cost_per_token": 0.000002,
}
}
)
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
optional_params: Final = get_optional_params(
model="pricing-only-test-deployment", custom_llm_provider="azure", drop_params=True, **reasoning_kwargs
)
assert not set(reasoning_kwargs) & set(optional_params)
drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()]
assert len(drop_warnings) == 1
assert all(fragment in drop_warnings[0] for fragment in ("pricing-only-test-deployment", "model_info.base_model"))
def test_dropping_reasoning_for_a_catalog_reasoning_model_does_not_suggest_base_model(
local_model_cost_map: None, caplog: pytest.LogCaptureFixture
) -> None:
assert litellm.model_cost["azure/gpt-5.6-sol"].get("supports_reasoning") is True
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
optional_params: Final = get_optional_params(
model="gpt-5.6-sol", custom_llm_provider="azure", thinking={"type": "disabled"}, drop_params=True
)
assert "thinking" not in optional_params
drop_warnings: Final = [
record.getMessage() for record in caplog.records if "Dropping thinking" in record.getMessage()
]
assert len(drop_warnings) == 1
assert "model_info.base_model" not in drop_warnings[0]
def test_turning_reasoning_off_for_a_catalog_non_reasoning_model_drops_it_silently(
local_model_cost_map: None, caplog: pytest.LogCaptureFixture
) -> None:
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
optional_params: Final = get_optional_params(
model="gpt-4o", custom_llm_provider="azure", reasoning_effort="none", drop_params=True
)
assert "reasoning_effort" not in optional_params
assert not [record for record in caplog.records if "Dropping" in record.getMessage()]
Reference in a new issue View git blame Copy permalink