scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 267

test: retarget cost-map tests to the rows mirrored from main

Browse source 
Test-only. Takes main's version of test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag, test_cache_control_injection_tool_config_honors_ttl_for_regional_model_lacking_own_pricing and test_get_model_info_gemini; retargets three tests whose rows main changed or removed (gemini-2.0-flash-001 -> gemini-2.5-flash, gpt-5.1-codex-max -> gpt-5.2 for xhigh, gpt-5.1-chat-latest -> gpt-5.1); registers this line's unsupported providers in PROVIDER_LABELS_WITHOUT_A_MODEL_SET; adds extraction mode, main's enum lists and every key main's map uses to test_utils' inline schema, plus main's seedance cost exceptions.
This commit is contained in:
Mrinal Chanshetty 2026-10-20 18:20:07 -07:00
parent dc314e7f6a
commit f419a596a7
7 changed files with 99 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
View file

@ -970,23 +970,16 @@ def test_shipped_tool_search_rule_version_boundaries(shipped_cost_map, model, pr
def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipped_cost_map):
"""A mapped Claude 4.5+ entry with no supports_tool_search key gets it from the rule
on Anthropic direct, Vertex and Bedrock, a mapped pre-4.5 entry stays without one,
and Azure Foundry and reseller copies of the same model are not touched."""
on Anthropic direct, Azure Foundry, Vertex and Bedrock, a mapped pre-4.5 entry stays
without one, and reseller copies of the same model are not touched."""
for key, model, provider in (
("claude-opus-4-7", "claude-opus-4-7", "anthropic"),
("azure_ai/claude-opus-5", "claude-opus-5", "azure_ai"),
("vertex_ai/claude-opus-5", "claude-opus-5", "vertex_ai"),
):
assert "supports_tool_search" not in litellm.model_cost[key]
assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_tool_search"] is True
assert "supports_tool_search" not in litellm.model_cost["claude-opus-4-1"]
opus_4_1_info = litellm.get_model_info("claude-opus-4-1", custom_llm_provider="anthropic")
assert opus_4_1_info.get("supports_tool_search") is None
assert "supports_tool_search" not in litellm.model_cost["azure_ai/claude-opus-5"]
azure_opus_5_info = litellm.get_model_info("claude-opus-5", custom_llm_provider="azure_ai")
assert azure_opus_5_info.get("supports_tool_search") is None
assert match_fill_missing_generalizations("claude-opus-5", "bedrock")["supports_tool_search"] is True
assert "supports_tool_search" not in match_fill_missing_generalizations("claude-opus-5", "azure_ai")
assert match_fill_missing_generalizations("claude-opus-5", "azure_ai")["supports_tool_search"] is True
assert match_fill_missing_generalizations("claude-opus-5", "perplexity") is None

2
tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py
View file

@ -157,7 +157,7 @@ def test_a_gpt_5_name_without_a_foundry_row_keeps_reading_its_own_entry(
monkeypatch.setenv("AZURE_AI_API_KEY", "placeholder")
optional_params = litellm.utils.get_optional_params(
model="gpt-5.1-chat-latest",
model="gpt-5.1",
custom_llm_provider="azure_ai",
temperature=0.2,
top_p=0.9,

9
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
View file

@ -5417,9 +5417,14 @@ def test_cache_control_injection_tool_config_honors_ttl_for_regional_model_lacki
old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
old_cost = litellm.model_cost
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
cost_map = dict(litellm.get_model_cost_map(url=""))
cost_map["jp.anthropic.claude-opus-4-7"] = {
k: v
for k, v in cost_map["jp.anthropic.claude-opus-4-7"].items()
if k != "cache_creation_input_token_cost_above_1hr"
}
litellm.model_cost = cost_map
try:
assert "cache_creation_input_token_cost_above_1hr" not in litellm.model_cost["jp.anthropic.claude-opus-4-7"]
assert "cache_creation_input_token_cost_above_1hr" in litellm.model_cost["anthropic.claude-opus-4-7"]
config = AmazonConverseConfig()
messages = [

4
tests/test_litellm/llms/openai/test_gpt5_transformation.py
View file

@ -248,11 +248,11 @@ def test_gpt5_1_reasoning_effort_none(config: OpenAIConfig):
assert params["reasoning_effort"] == effort
def test_gpt5_1_codex_max_allows_reasoning_effort_xhigh(config: OpenAIConfig):
def test_gpt5_2_allows_reasoning_effort_xhigh(config: OpenAIConfig):
params = config.map_openai_params(
non_default_params={"reasoning_effort": "xhigh"},
optional_params={},
model="gpt-5.1-codex-max",
model="gpt-5.2",
drop_params=False,
)
assert params["reasoning_effort"] == "xhigh"

6
tests/test_litellm/test_cost_calculator.py
View file

@ -166,7 +166,7 @@ def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch):
image_tokens=20,
),
)
mr = ModelResponse(usage=usage, model="gemini-2.0-flash-001")
mr = ModelResponse(usage=usage, model="gemini-2.5-flash")
result = response_cost_calculator(
response_object=mr,
@ -178,7 +178,7 @@ def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch):
base_model=None,
)
model_info = litellm.model_cost["gemini-2.0-flash-001"]
model_info = litellm.model_cost["gemini-2.5-flash"]
# Step 1: Test a model where input_cost_per_image_token is not set.
# In this case the calculation should use input_cost_per_token as fallback.
@ -203,7 +203,7 @@ def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch):
monkeypatch.setattr(
litellm,
"model_cost",
{"gemini-2.0-flash-001": temp_model_info_object},
{"gemini-2.5-flash": temp_model_info_object},
)
# Invalidate caches after modifying litellm.model_cost

14
tests/test_litellm/test_model_prices_schema.py
View file

@ -408,7 +408,19 @@ def test_deepseek_rows_bill_half_rate_outside_weekday_peak_hours(path: Path):
assert all(name in rows for name in DEEPSEEK_PRICED_ROWS)
PROVIDER_LABELS_WITHOUT_A_MODEL_SET: Final = frozenset({"sagemaker", "bedrock_converse"})
PROVIDER_LABELS_WITHOUT_A_MODEL_SET: Final = frozenset(
{
"sagemaker",
"bedrock_converse",
"coralbricks",
"inferenceapis",
"microsoft_365_copilot",
"openinfer",
"prism",
"sail",
"scaledown",
}
)
MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY: Final = frozenset({"search", "evaluation"})
VERTEX_FAMILIES_A_VERTEX_WILDCARD_GRANT_DOES_NOT_LIST: Final = frozenset(
{

69
tests/test_litellm/test_utils.py
View file

@ -720,6 +720,66 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"type": "object",
"properties": {
"supports_computer_use": {"type": "boolean"},
"output_cost_per_image_0.5K": {"type": "number"},
"cache_creation_input_token_cost_above_272k_tokens_decisions": {"type": "number"},
"cache_creation_input_token_cost_above_272k_tokens_ultrafast": {"type": "number"},
"cache_creation_input_token_cost_decisions": {"type": "number"},
"cache_creation_input_token_cost_ultrafast": {"type": "number"},
"cache_read_input_image_token_cost": {"type": "number"},
"cache_read_input_token_cost_above_200k_tokens_batches": {"type": "number"},
"cache_read_input_token_cost_above_272k_tokens_decisions": {"type": "number"},
"cache_read_input_token_cost_above_272k_tokens_ultrafast": {"type": "number"},
"cache_read_input_token_cost_balanced": {"type": "number"},
"cache_read_input_token_cost_decisions": {"type": "number"},
"cache_read_input_token_cost_ultrafast": {"type": "number"},
"cost_per_second": {"type": "number"},
"explicit_cache_read_input_token_cost": {"type": "number"},
"input_cost_per_text_message": {"type": "number"},
"input_cost_per_token_above_200k_tokens_batches": {"type": "number"},
"input_cost_per_token_above_272k_tokens_decisions": {"type": "number"},
"input_cost_per_token_above_272k_tokens_ultrafast": {"type": "number"},
"input_cost_per_token_balanced": {"type": "number"},
"input_cost_per_token_decisions": {"type": "number"},
"input_cost_per_token_ultrafast": {"type": "number"},
"output_cost_per_image_1024": {"type": "number"},
"output_cost_per_image_1536": {"type": "number"},
"output_cost_per_image_1K": {"type": "number"},
"output_cost_per_image_2K": {"type": "number"},
"output_cost_per_image_4K": {"type": "number"},
"output_cost_per_image_512": {"type": "number"},
"output_cost_per_image_token_batches": {"type": "number"},
"output_cost_per_second_2k": {"type": "number"},
"output_cost_per_second_768p": {"type": "number"},
"output_cost_per_token_above_200k_tokens_batches": {"type": "number"},
"output_cost_per_token_above_272k_tokens_decisions": {"type": "number"},
"output_cost_per_token_above_272k_tokens_ultrafast": {"type": "number"},
"output_cost_per_token_balanced": {"type": "number"},
"output_cost_per_token_decisions": {"type": "number"},
"output_cost_per_token_ultrafast": {"type": "number"},
"search_free_requests": {
"type": "object",
"properties": {
"count": {"type": "integer", "minimum": 1},
"period": {"type": "string", "enum": ["day", "month"]},
"shared_group": {"type": "string"},
},
"required": ["count", "period", "shared_group"],
"additionalProperties": False,
},
"supports_bedrock_converse_tool_result_images": {"type": "boolean"},
"supports_bedrock_native_structured_output": {"type": "boolean"},
"supports_bedrock_runtime_chat_completions_inline_reasoning": {"type": "boolean"},
"supports_bedrock_runtime_chat_completions_response_format": {"type": "boolean"},
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": {"type": "boolean"},
"supports_box_questions": {"type": "boolean"},
"supports_regex_lookaround": {"type": "boolean"},
"token_count_framing": {"type": "integer"},
"token_count_model": {
"type": "object",
"properties": {"model": {"type": "string"}, "region": {"type": "string"}},
"required": ["model"],
},
"token_count_tokenizer": {"type": "string", "enum": ["kimi_k3"]},
"cache_creation_input_audio_token_cost": {"type": "number"},
"cache_creation_input_token_cost": {"type": "number"},
"cache_creation_input_token_cost_above_1hr": {"type": "number"},
@ -827,6 +887,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"image_edit",
"embedding",
"evaluation",
"extraction",
"guardrail",
"image_generation",
"video_generation",
@ -957,8 +1018,11 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"/v1/audio/transcriptions",
"/v1/audio/speech",
"/v1/ocr",
"/v1/videos",
"/vertex_ai/live",
"/v1/listen",
"/v1/systemone",
"/v1/decisions",
"/v1beta/interactions",
],
},
@ -1086,6 +1150,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
# Add any model IDs that should be exempt from the cost validation
# Example: "expensive-model-id",
"runwayml/seedance2", # 4K output is 150 credits/second = $1.50/second
"fal_ai/bytedance/seedance-2.0/text-to-video",
"fal_ai/bytedance/seedance-2.0/image-to-video",
"fal_ai/bytedance/seedance-2.0/reference-to-video",
]
is_valid, violations = validate_model_cost_values(actual_json, exceptions)
@ -5776,6 +5843,8 @@ def test_get_model_info_gemini(monkeypatch):
and "veo" not in model
and "lyria" not in model
and "robotics" not in model
and "3.8-flash-tts" not in model
and "3.8-flash-lite-tts" not in model
):
assert info.get("tpm") is not None, f"{model} does not have tpm"
assert info.get("rpm") is not None, f"{model} does not have rpm"