diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index 25a12bebf9a..ffd964be088 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -970,23 +970,16 @@ def test_shipped_tool_search_rule_version_boundaries(shipped_cost_map, model, pr def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipped_cost_map): """A mapped Claude 4.5+ entry with no supports_tool_search key gets it from the rule - on Anthropic direct, Vertex and Bedrock, a mapped pre-4.5 entry stays without one, - and Azure Foundry and reseller copies of the same model are not touched.""" + on Anthropic direct, Azure Foundry, Vertex and Bedrock, a mapped pre-4.5 entry stays + without one, and reseller copies of the same model are not touched.""" for key, model, provider in ( ("claude-opus-4-7", "claude-opus-4-7", "anthropic"), + ("azure_ai/claude-opus-5", "claude-opus-5", "azure_ai"), ("vertex_ai/claude-opus-5", "claude-opus-5", "vertex_ai"), ): assert "supports_tool_search" not in litellm.model_cost[key] assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_tool_search"] is True - assert "supports_tool_search" not in litellm.model_cost["claude-opus-4-1"] - opus_4_1_info = litellm.get_model_info("claude-opus-4-1", custom_llm_provider="anthropic") - assert opus_4_1_info.get("supports_tool_search") is None - - assert "supports_tool_search" not in litellm.model_cost["azure_ai/claude-opus-5"] - azure_opus_5_info = litellm.get_model_info("claude-opus-5", custom_llm_provider="azure_ai") - assert azure_opus_5_info.get("supports_tool_search") is None - assert match_fill_missing_generalizations("claude-opus-5", "bedrock")["supports_tool_search"] is True - assert "supports_tool_search" not in match_fill_missing_generalizations("claude-opus-5", "azure_ai") + assert match_fill_missing_generalizations("claude-opus-5", "azure_ai")["supports_tool_search"] is True assert match_fill_missing_generalizations("claude-opus-5", "perplexity") is None diff --git a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py index 2c39326b0f2..a49fdd2f25a 100644 --- a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -157,7 +157,7 @@ def test_a_gpt_5_name_without_a_foundry_row_keeps_reading_its_own_entry( monkeypatch.setenv("AZURE_AI_API_KEY", "placeholder") optional_params = litellm.utils.get_optional_params( - model="gpt-5.1-chat-latest", + model="gpt-5.1", custom_llm_provider="azure_ai", temperature=0.2, top_p=0.9, diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 1b2a6d1ed2c..79609f1517c 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -5417,9 +5417,14 @@ def test_cache_control_injection_tool_config_honors_ttl_for_regional_model_lacki old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") + cost_map = dict(litellm.get_model_cost_map(url="")) + cost_map["jp.anthropic.claude-opus-4-7"] = { + k: v + for k, v in cost_map["jp.anthropic.claude-opus-4-7"].items() + if k != "cache_creation_input_token_cost_above_1hr" + } + litellm.model_cost = cost_map try: - assert "cache_creation_input_token_cost_above_1hr" not in litellm.model_cost["jp.anthropic.claude-opus-4-7"] assert "cache_creation_input_token_cost_above_1hr" in litellm.model_cost["anthropic.claude-opus-4-7"] config = AmazonConverseConfig() messages = [ diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 89aee40f4fe..24414ecfb98 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -248,11 +248,11 @@ def test_gpt5_1_reasoning_effort_none(config: OpenAIConfig): assert params["reasoning_effort"] == effort -def test_gpt5_1_codex_max_allows_reasoning_effort_xhigh(config: OpenAIConfig): +def test_gpt5_2_allows_reasoning_effort_xhigh(config: OpenAIConfig): params = config.map_openai_params( non_default_params={"reasoning_effort": "xhigh"}, optional_params={}, - model="gpt-5.1-codex-max", + model="gpt-5.2", drop_params=False, ) assert params["reasoning_effort"] == "xhigh" diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index aeca8462b67..0b35d8498e4 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -166,7 +166,7 @@ def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch): image_tokens=20, ), ) - mr = ModelResponse(usage=usage, model="gemini-2.0-flash-001") + mr = ModelResponse(usage=usage, model="gemini-2.5-flash") result = response_cost_calculator( response_object=mr, @@ -178,7 +178,7 @@ def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch): base_model=None, ) - model_info = litellm.model_cost["gemini-2.0-flash-001"] + model_info = litellm.model_cost["gemini-2.5-flash"] # Step 1: Test a model where input_cost_per_image_token is not set. # In this case the calculation should use input_cost_per_token as fallback. @@ -203,7 +203,7 @@ def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch): monkeypatch.setattr( litellm, "model_cost", - {"gemini-2.0-flash-001": temp_model_info_object}, + {"gemini-2.5-flash": temp_model_info_object}, ) # Invalidate caches after modifying litellm.model_cost diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py index 052278631e2..f5af02b4802 100644 --- a/tests/test_litellm/test_model_prices_schema.py +++ b/tests/test_litellm/test_model_prices_schema.py @@ -408,7 +408,19 @@ def test_deepseek_rows_bill_half_rate_outside_weekday_peak_hours(path: Path): assert all(name in rows for name in DEEPSEEK_PRICED_ROWS) -PROVIDER_LABELS_WITHOUT_A_MODEL_SET: Final = frozenset({"sagemaker", "bedrock_converse"}) +PROVIDER_LABELS_WITHOUT_A_MODEL_SET: Final = frozenset( + { + "sagemaker", + "bedrock_converse", + "coralbricks", + "inferenceapis", + "microsoft_365_copilot", + "openinfer", + "prism", + "sail", + "scaledown", + } +) MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY: Final = frozenset({"search", "evaluation"}) VERTEX_FAMILIES_A_VERTEX_WILDCARD_GRANT_DOES_NOT_LIST: Final = frozenset( { diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 56c3256a8a0..92be4d99d4d 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -720,6 +720,66 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "type": "object", "properties": { "supports_computer_use": {"type": "boolean"}, + "output_cost_per_image_0.5K": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_decisions": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": {"type": "number"}, + "cache_creation_input_token_cost_decisions": {"type": "number"}, + "cache_creation_input_token_cost_ultrafast": {"type": "number"}, + "cache_read_input_image_token_cost": {"type": "number"}, + "cache_read_input_token_cost_above_200k_tokens_batches": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_decisions": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": {"type": "number"}, + "cache_read_input_token_cost_balanced": {"type": "number"}, + "cache_read_input_token_cost_decisions": {"type": "number"}, + "cache_read_input_token_cost_ultrafast": {"type": "number"}, + "cost_per_second": {"type": "number"}, + "explicit_cache_read_input_token_cost": {"type": "number"}, + "input_cost_per_text_message": {"type": "number"}, + "input_cost_per_token_above_200k_tokens_batches": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_decisions": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_ultrafast": {"type": "number"}, + "input_cost_per_token_balanced": {"type": "number"}, + "input_cost_per_token_decisions": {"type": "number"}, + "input_cost_per_token_ultrafast": {"type": "number"}, + "output_cost_per_image_1024": {"type": "number"}, + "output_cost_per_image_1536": {"type": "number"}, + "output_cost_per_image_1K": {"type": "number"}, + "output_cost_per_image_2K": {"type": "number"}, + "output_cost_per_image_4K": {"type": "number"}, + "output_cost_per_image_512": {"type": "number"}, + "output_cost_per_image_token_batches": {"type": "number"}, + "output_cost_per_second_2k": {"type": "number"}, + "output_cost_per_second_768p": {"type": "number"}, + "output_cost_per_token_above_200k_tokens_batches": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_decisions": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_ultrafast": {"type": "number"}, + "output_cost_per_token_balanced": {"type": "number"}, + "output_cost_per_token_decisions": {"type": "number"}, + "output_cost_per_token_ultrafast": {"type": "number"}, + "search_free_requests": { + "type": "object", + "properties": { + "count": {"type": "integer", "minimum": 1}, + "period": {"type": "string", "enum": ["day", "month"]}, + "shared_group": {"type": "string"}, + }, + "required": ["count", "period", "shared_group"], + "additionalProperties": False, + }, + "supports_bedrock_converse_tool_result_images": {"type": "boolean"}, + "supports_bedrock_native_structured_output": {"type": "boolean"}, + "supports_bedrock_runtime_chat_completions_inline_reasoning": {"type": "boolean"}, + "supports_bedrock_runtime_chat_completions_response_format": {"type": "boolean"}, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": {"type": "boolean"}, + "supports_box_questions": {"type": "boolean"}, + "supports_regex_lookaround": {"type": "boolean"}, + "token_count_framing": {"type": "integer"}, + "token_count_model": { + "type": "object", + "properties": {"model": {"type": "string"}, "region": {"type": "string"}}, + "required": ["model"], + }, + "token_count_tokenizer": {"type": "string", "enum": ["kimi_k3"]}, "cache_creation_input_audio_token_cost": {"type": "number"}, "cache_creation_input_token_cost": {"type": "number"}, "cache_creation_input_token_cost_above_1hr": {"type": "number"}, @@ -827,6 +887,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "image_edit", "embedding", "evaluation", + "extraction", "guardrail", "image_generation", "video_generation", @@ -957,8 +1018,11 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "/v1/audio/transcriptions", "/v1/audio/speech", "/v1/ocr", + "/v1/videos", "/vertex_ai/live", "/v1/listen", + "/v1/systemone", + "/v1/decisions", "/v1beta/interactions", ], }, @@ -1086,6 +1150,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): # Add any model IDs that should be exempt from the cost validation # Example: "expensive-model-id", "runwayml/seedance2", # 4K output is 150 credits/second = $1.50/second + "fal_ai/bytedance/seedance-2.0/text-to-video", + "fal_ai/bytedance/seedance-2.0/image-to-video", + "fal_ai/bytedance/seedance-2.0/reference-to-video", ] is_valid, violations = validate_model_cost_values(actual_json, exceptions) @@ -5776,6 +5843,8 @@ def test_get_model_info_gemini(monkeypatch): and "veo" not in model and "lyria" not in model and "robotics" not in model + and "3.8-flash-tts" not in model + and "3.8-flash-lite-tts" not in model ): assert info.get("tpm") is not None, f"{model} does not have tpm" assert info.get("rpm") is not None, f"{model} does not have rpm"