From 2586b21893d0fb1e35ca50dec3f23271b02815b9 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 22:16:08 +0000 Subject: [PATCH] test: keep tests that survive correct cost-map updates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/test_llm_cost_calc_utils.py | 191 +++++++------ .../test_get_model_cost_map.py | 55 ++++ ...azure_ai_foundry_catalog_model_metadata.py | 24 ++ .../llms/cohere/ocr/test_cohere_parse_cost.py | 28 ++ .../test_databricks_cost_calculator.py | 51 ++++ .../databricks/test_databricks_pricing.py | 0 .../test_fireworks_ai_cost_calculator.py | 10 + .../test_fireworks_ai_kimi_model_metadata.py | 54 ++++ .../llms/mistral/ocr/test_mistral_ocr_cost.py | 40 +++ .../llms/openai_like/test_scx_ai_provider.py | 28 ++ .../perplexity/test_perplexity_integration.py | 18 ++ .../llms/tencent/test_cost_calculator.py | 1 + ...artner_models_anthropic_messages_config.py | 42 +++ .../llms/xai/test_xai_model_registry.py | 55 ++++ .../xai/test_xai_redirected_slug_pricing.py | 20 ++ .../proxy/spend_tracking/test_savings.py | 23 ++ .../test_azure_ai_grok_4_3_model_metadata.py | 37 +++ .../test_azure_ai_grok_4_6_model_metadata.py | 44 +++ .../test_baseten_glm_5_3_model_metadata.py | 47 ++++ ..._bedrock_marengo_embed_3_model_metadata.py | 52 ++++ .../test_bedrock_usgov_pricing.py | 12 + .../test_claude_fable_5_config.py | 129 +++++++++ .../test_claude_haiku_4_5_config.py | 46 ++++ .../test_litellm/test_claude_opus_5_config.py | 27 ++ .../test_claude_sonnet_5_config.py | 29 ++ ...st_cloudflare_workers_ai_model_metadata.py | 50 ++++ .../test_litellm/test_command_r7b_pricing.py | 53 ++++ .../test_daybreak_model_metadata.py | 15 ++ .../test_deepseek_model_metadata.py | 25 ++ .../test_fireworks_serverless_model_costs.py | 35 +-- ...est_gemini_3_1_flash_lite_image_pricing.py | 5 + .../test_gemini_tts_native_audio_pricing.py | 84 ++++++ .../test_gpt_5_4_model_metadata.py | 44 +++ .../test_gpt_5_5_model_metadata.py | 19 ++ tests/test_litellm/test_gpt_realtime_mode.py | 16 +- .../test_mistral_medium_3_5_model_metadata.py | 33 +++ .../test_mistral_small_4_0_model_metadata.py | 26 ++ ...test_mistral_zai_glm_5_2_model_metadata.py | 52 ++++ .../test_muse_spark_1_1_model_metadata.py | 29 ++ .../test_muse_spark_1_2_model_metadata.py | 9 + .../test_muse_spark_1_3_model_metadata.py | 9 + .../test_replicate_model_key_format.py | 15 ++ .../test_sambanova_model_metadata.py | 25 ++ .../test_together_ai_model_metadata.py | 8 + tests/test_litellm/test_utils.py | 253 +++++++++++++++++- .../test_xai_grok_4_3_model_metadata.py | 19 ++ 46 files changed, 1771 insertions(+), 116 deletions(-) create mode 100644 tests/test_litellm/llms/cohere/ocr/test_cohere_parse_cost.py create mode 100644 tests/test_litellm/llms/databricks/test_databricks_pricing.py create mode 100644 tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_kimi_model_metadata.py create mode 100644 tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py create mode 100644 tests/test_litellm/llms/tencent/test_cost_calculator.py create mode 100644 tests/test_litellm/llms/xai/test_xai_model_registry.py create mode 100644 tests/test_litellm/test_azure_ai_grok_4_3_model_metadata.py create mode 100644 tests/test_litellm/test_azure_ai_grok_4_6_model_metadata.py create mode 100644 tests/test_litellm/test_bedrock_marengo_embed_3_model_metadata.py create mode 100644 tests/test_litellm/test_claude_haiku_4_5_config.py create mode 100644 tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py create mode 100644 tests/test_litellm/test_command_r7b_pricing.py create mode 100644 tests/test_litellm/test_gemini_tts_native_audio_pricing.py create mode 100644 tests/test_litellm/test_gpt_5_4_model_metadata.py create mode 100644 tests/test_litellm/test_gpt_5_5_model_metadata.py create mode 100644 tests/test_litellm/test_mistral_medium_3_5_model_metadata.py create mode 100644 tests/test_litellm/test_mistral_small_4_0_model_metadata.py create mode 100644 tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py create mode 100644 tests/test_litellm/test_muse_spark_1_1_model_metadata.py create mode 100644 tests/test_litellm/test_sambanova_model_metadata.py create mode 100644 tests/test_litellm/test_xai_grok_4_3_model_metadata.py diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 0743476417e..798d657cce7 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2087,95 +2087,6 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): assert round(result, 6) == round(expected, 6) -def test_a_pinned_billing_time_prices_the_totals_and_the_reported_rates_at_one_moment(monkeypatch): - monkeypatch.setitem( - litellm.model_cost, - "off-peak-model", - { - "input_cost_per_token": 3e-6, - "output_cost_per_token": 15e-6, - "off_peak_pricing": { - "hours_utc": "02:00-03:00", - "input_cost_per_token": 1e-6, - "output_cost_per_token": 5e-6, - }, - "litellm_provider": "openai", - "mode": "chat", - }, - ) - usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - - with pinned_billing_time(datetime(2026, 1, 1, 2, 30, tzinfo=timezone.utc)): - off_peak_prompt_cost, off_peak_completion_cost = generic_cost_per_token( - model="off-peak-model", usage=usage, custom_llm_provider="openai" - ) - off_peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage) - with pinned_billing_time(datetime(2026, 1, 1, 12, 30, tzinfo=timezone.utc)): - peak_prompt_cost, peak_completion_cost = generic_cost_per_token( - model="off-peak-model", usage=usage, custom_llm_provider="openai" - ) - peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage) - - assert off_peak_rates.input_cost_per_token == pytest.approx(1e-6) - assert peak_rates.input_cost_per_token == pytest.approx(3e-6) - assert off_peak_prompt_cost == pytest.approx(1000 * off_peak_rates.input_cost_per_token) - assert off_peak_completion_cost == pytest.approx(500 * off_peak_rates.output_cost_per_token) - assert peak_prompt_cost == pytest.approx(1000 * peak_rates.input_cost_per_token) - assert peak_completion_cost == pytest.approx(500 * peak_rates.output_cost_per_token) - - -def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(_local_model_cost_map, monkeypatch): - from litellm.llms.anthropic.cost_calculation import cost_per_token as anthropic_cost_per_token - - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - - model = "claude-test-geo-breakdown-model" - litellm.register_model( - model_cost={ - model: { - "input_cost_per_token": 5e-6, - "output_cost_per_token": 25e-6, - "cache_creation_input_token_cost": 6.25e-6, - "cache_read_input_token_cost": 0.5e-6, - "litellm_provider": "anthropic", - "max_tokens": 8192, - "provider_specific_entry": {"us": 1.1}, - } - } - ) - - def make_usage() -> Usage: - return Usage( - prompt_tokens=10_000, - completion_tokens=500, - total_tokens=10_500, - prompt_tokens_details=PromptTokensDetailsWrapper( - cached_tokens=2_000, - cache_creation_tokens=6_000, - ), - completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200, text_tokens=300), - ) - - base_usage = make_usage() - geo_usage = make_usage() - geo_usage.inference_geo = "us" - - base = get_token_type_cost_breakdown(model=model, custom_llm_provider="anthropic", usage=base_usage) - geo = get_token_type_cost_breakdown(model=model, custom_llm_provider="anthropic", usage=geo_usage) - - assert base.cache_read_cost == pytest.approx(2_000 * 0.5e-6) - assert base.cache_creation_cost == pytest.approx(6_000 * 6.25e-6) - assert geo.cache_read_cost == pytest.approx(base.cache_read_cost * 1.1) - assert geo.cache_creation_cost == pytest.approx(base.cache_creation_cost * 1.1) - assert geo.reasoning_cost == pytest.approx(base.reasoning_cost * 1.1) - - prompt_cost, completion_cost = anthropic_cost_per_token(model=model, usage=geo_usage) - text_input_cost = 2_000 * 5e-6 * 1.1 - text_output_cost = 300 * 25e-6 * 1.1 - assert text_input_cost + geo.cache_read_cost + geo.cache_creation_cost == pytest.approx(prompt_cost) - assert text_output_cost + geo.reasoning_cost == pytest.approx(completion_cost) - - def test_service_tier_ultrafast_pricing(): """An ultrafast request bills the *_ultrafast rates for all token types. @@ -3134,6 +3045,46 @@ def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeyp assert breakdown.reasoning_cost == pytest.approx(200 * rates.output_cost_per_reasoning_token) +def test_a_pinned_billing_time_prices_the_totals_and_the_reported_rates_at_one_moment(monkeypatch): + """Totals and reported rates resolve off-peak pricing on separate paths that each read the + clock, so a window opening between the two reads used to leave them describing one request + at two different prices. Pinned, both must answer for the pinned moment.""" + monkeypatch.setitem( + litellm.model_cost, + "off-peak-model", + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "off_peak_pricing": { + "hours_utc": "02:00-03:00", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 5e-6, + }, + "litellm_provider": "openai", + "mode": "chat", + }, + ) + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + with pinned_billing_time(datetime(2026, 1, 1, 2, 30, tzinfo=timezone.utc)): + off_peak_prompt_cost, off_peak_completion_cost = generic_cost_per_token( + model="off-peak-model", usage=usage, custom_llm_provider="openai" + ) + off_peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage) + with pinned_billing_time(datetime(2026, 1, 1, 12, 30, tzinfo=timezone.utc)): + peak_prompt_cost, peak_completion_cost = generic_cost_per_token( + model="off-peak-model", usage=usage, custom_llm_provider="openai" + ) + peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage) + + assert off_peak_rates.input_cost_per_token == pytest.approx(1e-6) + assert peak_rates.input_cost_per_token == pytest.approx(3e-6) + assert off_peak_prompt_cost == pytest.approx(1000 * off_peak_rates.input_cost_per_token) + assert off_peak_completion_cost == pytest.approx(500 * off_peak_rates.output_cost_per_token) + assert peak_prompt_cost == pytest.approx(1000 * peak_rates.input_cost_per_token) + assert peak_completion_cost == pytest.approx(500 * peak_rates.output_cost_per_token) + + def test_the_token_type_breakdown_carries_the_rates_it_billed_at(monkeypatch): """Callers that report both the lines and the rates read the rates off the breakdown rather than resolving them a second time, so the breakdown has to hand back exactly what it billed at.""" @@ -3341,6 +3292,68 @@ def test_token_type_cost_breakdown_applies_vertex_regional_uplift(_local_model_c assert text_input_cost + regional.cache_read_cost == pytest.approx(prompt_cost) +def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(_local_model_cost_map, monkeypatch): + """ + Anthropic's regional (geo) uplift lives in provider_specific_entry and is + applied to every token type in the totals, so the per-type breakdown must + scale its cache and reasoning line items by it too. Otherwise the logged + cache costs stay at the base rate and the cache uplift is misattributed to + plain input for exactly the cache-heavy regional traffic the uplift targets. + """ + from litellm.llms.anthropic.cost_calculation import ( + cost_per_token as anthropic_cost_per_token, + ) + + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + model = "claude-test-geo-breakdown-model" + litellm.register_model( + model_cost={ + model: { + "input_cost_per_token": 5e-6, + "output_cost_per_token": 25e-6, + "cache_creation_input_token_cost": 6.25e-6, + "cache_read_input_token_cost": 0.5e-6, + "litellm_provider": "anthropic", + "max_tokens": 8192, + "provider_specific_entry": {"us": 1.1}, + } + } + ) + + def make_usage() -> Usage: + return Usage( + prompt_tokens=10_000, + completion_tokens=500, + total_tokens=10_500, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=2_000, + cache_creation_tokens=6_000, + ), + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200, text_tokens=300), + ) + + base_usage = make_usage() + geo_usage = make_usage() + geo_usage.inference_geo = "us" + + base = get_token_type_cost_breakdown(model=model, custom_llm_provider="anthropic", usage=base_usage) + geo = get_token_type_cost_breakdown(model=model, custom_llm_provider="anthropic", usage=geo_usage) + + assert base.cache_read_cost == pytest.approx(2_000 * 0.5e-6) + assert base.cache_creation_cost == pytest.approx(6_000 * 6.25e-6) + assert geo.cache_read_cost == pytest.approx(base.cache_read_cost * 1.1) + assert geo.cache_creation_cost == pytest.approx(base.cache_creation_cost * 1.1) + assert geo.reasoning_cost == pytest.approx(base.reasoning_cost * 1.1) + + # The uplifted breakdown must still reconcile with the uplifted totals. + prompt_cost, completion_cost = anthropic_cost_per_token(model=model, usage=geo_usage) + text_input_cost = 2_000 * 5e-6 * 1.1 + text_output_cost = 300 * 25e-6 * 1.1 + assert text_input_cost + geo.cache_read_cost + geo.cache_creation_cost == pytest.approx(prompt_cost) + assert text_output_cost + geo.reasoning_cost == pytest.approx(completion_cost) + + @pytest.mark.parametrize("details_as_dict", [True, False]) def test_image_response_input_image_tokens_priced_at_image_rate(details_as_dict): """ diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index f73e6378b92..53fee36b3a8 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -134,6 +134,24 @@ def test_finalize_with_no_block_clears_rules(): set_fallback_generalizations(previous) +def test_shipped_backup_carries_the_claude_routing_rules(): + """The bundled backup must ship the Claude routing rules so a fresh install + (or an offline fallback) routes unknown Claude models without code changes. + Bedrock-syntax ids must hit the bedrock rule before the bare-id Anthropic rule.""" + backup = GetModelCostMap.load_local_model_cost_map() + rules = backup.get(FALLBACK_GENERALIZATIONS_KEY, {}).get("rules", []) + names = [r.get("name") for r in rules] + assert names.index("bedrock-claude-ids") < names.index("anthropic-claude-ids") + + previous = list(get_fallback_generalization_rules()) + try: + set_fallback_generalizations(rules) + assert match_routing_generalization("claude-opus-4-9") == "anthropic" + assert match_routing_generalization("global.anthropic.claude-opus-4-9") == "bedrock" + finally: + set_fallback_generalizations(previous) + + def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace(): """Routing rules decide ``litellm_provider`` for otherwise-unknown ids, and the proxy's wildcard access check (``can_key_call_model`` with a ``bedrock/*`` key) @@ -170,6 +188,43 @@ def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace(): set_fallback_generalizations(previous) +def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0(): + """Adaptive thinking is data, not code. The bundled backup must carry + supports_adaptive_thinking on genuine Claude >= 4.6 entries (every provider + route) and on the version-gated anthropic-claude-adaptive-thinking rule for + unmapped future Claudes, while leaving the dated Claude 4.0 names + ("...-4-20250514") unflagged so a date can never be mistaken for a 4.6+ minor + version. The version-neutral claude-family-baseline capability rule must not flag + it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule + carries only its delta; capability unioning stacks it onto the baseline, so the + baseline block is never duplicated across rules and no rule needs ``extends``.""" + backup = GetModelCostMap.load_local_model_cost_map() + + rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"] + baseline_rule = next(r for r in rules if r.get("name") == "claude-family-baseline") + adaptive_rule = next(r for r in rules if r.get("name") == "claude-adaptive-thinking") + assert "supports_adaptive_thinking" not in baseline_rule["model_info"] + assert "litellm_provider" not in baseline_rule["model_info"] + assert adaptive_rule["model_info"] == {"supports_adaptive_thinking": True} + assert all("extends" not in r for r in rules) + + for adaptive in [ + "anthropic.claude-opus-4-8", + "vertex_ai/claude-opus-4-6@default", + "us.anthropic.claude-sonnet-4-6", + "openrouter/anthropic/claude-opus-4.7", + "azure_ai/claude-opus-4-7", + ]: + assert backup[adaptive]["supports_adaptive_thinking"] is True, adaptive + + for non_adaptive in [ + "claude-opus-4-20250514", + "us.anthropic.claude-opus-4-20250514-v1:0", + "claude-opus-4-5", + ]: + assert "supports_adaptive_thinking" not in backup[non_adaptive], non_adaptive + + # OpenRouter headline rates from GET https://openrouter.ai/api/v1/models. # These were the catalog values that disagreed with that API (and, for the # two spotlight models, the public model pages that their source fields cite). diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index 5bb5b0d148d..9b20192c3f2 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -72,6 +72,22 @@ def test_azure_ai_catalog_name_prices_the_same_in_any_casing(catalog_name: str) assert upper_cost == lowercase_cost +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("catalog_name", GROK_4_20_NAMES) +def test_azure_ai_grok_4_20_bills_cached_prompt_tokens_at_the_input_price(catalog_name: str) -> None: + uncached_prompt_cost, _ = cost_per_token( + model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0 + ) + cached_prompt_cost, _ = cost_per_token( + model=f"azure_ai/{catalog_name}", + prompt_tokens=A_MILLION, + completion_tokens=0, + cache_read_input_tokens=A_MILLION, + ) + assert uncached_prompt_cost > 0 + assert cached_prompt_cost == pytest.approx(uncached_prompt_cost) + + @pytest.mark.usefixtures("local_model_cost_map") def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None: one_second_cost: Final = _whisper_transcription_cost(1) @@ -80,6 +96,14 @@ def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None: assert one_hour_cost == pytest.approx(AN_HOUR_IN_SECONDS * one_second_cost) +@pytest.mark.parametrize("catalog_name", CATALOG_NAMES) +def test_azure_ai_catalog_entry_source_and_backup_match(catalog_name: str) -> None: + main_entry = _cost_map_entry(MAIN_COST_MAP, catalog_name) + backup_entry = _cost_map_entry(BACKUP_COST_MAP, catalog_name) + + assert backup_entry == main_entry + + def test_azure_ai_model_router_spellings_share_one_entry() -> None: underscore_entry = _cost_map_entry(MAIN_COST_MAP, "model_router") hyphen_entry = _cost_map_entry(MAIN_COST_MAP, "model-router") diff --git a/tests/test_litellm/llms/cohere/ocr/test_cohere_parse_cost.py b/tests/test_litellm/llms/cohere/ocr/test_cohere_parse_cost.py new file mode 100644 index 00000000000..7ee34c6c55a --- /dev/null +++ b/tests/test_litellm/llms/cohere/ocr/test_cohere_parse_cost.py @@ -0,0 +1,28 @@ +from pathlib import Path + +import pytest + +import litellm +from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo + +REPO_ROOT = Path(__file__).parents[5] +COST_MAPS = [ + REPO_ROOT / "model_prices_and_context_window.json", + REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json", +] +MODELS = [("cohere/parse-v5.0", "cohere"), ("azure_ai/Cohere-parse-v5", "azure_ai")] + + +def _ocr_response(model: str, pages_processed: int) -> OCRResponse: + return OCRResponse( + pages=[OCRPage(index=i, markdown=f"page {i}") for i in range(pages_processed)], + model=model, + usage_info=OCRUsageInfo(pages_processed=pages_processed), + ) + + +@pytest.mark.parametrize("model, provider", MODELS) +def test_model_info_resolves_ocr_mode_and_price(local_model_cost_map, model: str, provider: str) -> None: + info = litellm.get_model_info(model=model, custom_llm_provider=provider) + + assert info["mode"] == "ocr" diff --git a/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py b/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py index f5d60dd0d32..afac7b0bc1a 100644 --- a/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py +++ b/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py @@ -1,3 +1,4 @@ +import json from decimal import Decimal from pathlib import Path from typing import Final @@ -162,6 +163,29 @@ def test_legacy_endpoint_names_still_resolve(local_model_cost_map: None) -> None assert completion_cost == pytest.approx(100 * info["output_cost_per_token"]) +@pytest.mark.parametrize("model", NEW_MODELS) +def test_new_models_carry_cache_pricing(local_model_cost_map: None, model: str) -> None: + info: Final = _model_info(model) + + assert info["input_cost_per_token"] > 0 + assert info["output_cost_per_token"] > 0 + assert info["cache_creation_input_token_cost"] > info["input_cost_per_token"] + assert info["cache_read_input_token_cost"] < info["input_cost_per_token"] + assert info["supports_prompt_caching"] is True + + +def test_every_priced_databricks_model_declares_cache_rates(local_model_cost_map: None) -> None: + undeclared: Final = [ + model + for model, info in litellm.model_cost.items() + if model.startswith("databricks/") + and info.get("input_cost_per_token") is not None + and any(info.get(field) is None for field in CACHE_FIELDS) + ] + + assert undeclared == [] + + def test_models_without_a_cache_discount_bill_cache_tokens_at_the_input_rate( local_model_cost_map: None, ) -> None: @@ -180,6 +204,33 @@ def test_models_without_a_cache_discount_bill_cache_tokens_at_the_input_rate( assert prompt_cost > 8000 * info["input_cost_per_token"] +def test_every_model_without_published_cache_dbu_bills_cache_at_its_own_input_rate( + local_model_cost_map: None, +) -> None: + without_published_rates: Final = [ + model + for model, info in litellm.model_cost.items() + if model.startswith("databricks/") + and info.get("input_cost_per_token") + and model not in PUBLISHED_DBU_PER_MILLION + ] + + for model in without_published_rates: + info = _model_info(model) + for field in CACHE_FIELDS: + assert info[field] == pytest.approx(info["input_cost_per_token"]), (model, field) + + +@pytest.mark.parametrize("model", NEW_MODELS) +def test_backup_price_map_matches_main(model: str) -> None: + main_cost: Final = json.loads(MAIN_PRICES.read_text()) + backup_cost: Final = json.loads(BACKUP_PRICES.read_text()) + + assert model in main_cost + assert model in backup_cost + assert backup_cost[model] == main_cost[model] + + def test_sonnet_5_ships_standard_rates_not_introductory(local_model_cost_map: None) -> None: sonnet_5: Final = _model_info("databricks/databricks-claude-sonnet-5") sonnet_4_6: Final = _model_info("databricks/databricks-claude-sonnet-4-6") diff --git a/tests/test_litellm/llms/databricks/test_databricks_pricing.py b/tests/test_litellm/llms/databricks/test_databricks_pricing.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py index f6018626bd0..6929cd48e60 100644 --- a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -22,6 +22,16 @@ def _usage(prompt_tokens: int, cached_tokens: int, completion_tokens: int) -> Us ) +def test_warm_call_cheaper_than_cold_call(): + prompt_tokens = 7036 + completion_tokens = 8 + + cold_prompt_cost, _ = cost_per_token(model=MODEL, usage=_usage(prompt_tokens, 16, completion_tokens)) + warm_prompt_cost, _ = cost_per_token(model=MODEL, usage=_usage(prompt_tokens, 7020, completion_tokens)) + + assert warm_prompt_cost < cold_prompt_cost + + OFF_PEAK_MODEL = "accounts/fireworks/models/off-peak-test" OFF_PEAK_WINDOW = "14:00-00:00" INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc) diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_kimi_model_metadata.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_kimi_model_metadata.py new file mode 100644 index 00000000000..8d28ee921e1 --- /dev/null +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_kimi_model_metadata.py @@ -0,0 +1,54 @@ +""" +Regression test for Fireworks Kimi K2.5 / K2.6 / K2.7 context and output limits. + +Fireworks publishes a 262144-token context window for every Kimi K2.5, K2.6 and +K2.7 model, but caps generation well below that. A previous bulk edit had flattened +max_output_tokens/max_tokens to 262144 (equal to the context window), which let the +pre-call context-window check admit requests asking for a full 262144-token +completion that Fireworks then rejects. These assertions pin the corrected per-alias +limits so a future bulk edit can't silently flatten them again. +""" + +import json +from importlib.resources import files + +import pytest + +CONTEXT_WINDOW = 262144 +OUTPUT_LIMIT = 32768 + +KIMI_ALIASES = ( + "fireworks_ai/kimi-k2p5", + "fireworks_ai/kimi-k2p6", + "fireworks_ai/kimi-k2p6-fast", + "fireworks_ai/kimi-k2p7-code", + "fireworks_ai/kimi-k2p7-code-fast", + "fireworks_ai/accounts/fireworks/models/kimi-k2p5", + "fireworks_ai/accounts/fireworks/models/kimi-k2p6", + "fireworks_ai/accounts/fireworks/models/kimi-k2p7-code", + "fireworks_ai/accounts/fireworks/routers/kimi-k2p6-fast", + "fireworks_ai/accounts/fireworks/routers/kimi-k2p7-code-fast", +) + + +@pytest.fixture(scope="module") +def use_local_model_cost_map(): + monkeypatch = pytest.MonkeyPatch() + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + import litellm + from litellm.utils import _invalidate_model_cost_lowercase_map + + original_model_cost = litellm.model_cost + litellm.model_cost = json.loads( + files("litellm").joinpath("model_prices_and_context_window_backup.json").read_text(encoding="utf-8") + ) + litellm.get_model_info.cache_clear() + _invalidate_model_cost_lowercase_map() + try: + yield litellm + finally: + litellm.model_cost = original_model_cost + litellm.get_model_info.cache_clear() + _invalidate_model_cost_lowercase_map() + monkeypatch.undo() diff --git a/tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py b/tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py new file mode 100644 index 00000000000..e3cf40b43f7 --- /dev/null +++ b/tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py @@ -0,0 +1,40 @@ +""" +Cost tests for Mistral OCR models against the real litellm cost map +(no monkeypatching of get_model_info). These regress the pricing entries +for mistral-ocr-4-0 and mistral-ocr-latest, which now both resolve to +OCR 4 at $4 / 1000 pages. +""" + +from pathlib import Path + +from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo + +OCR4_COST_PER_PAGE = 0.004 +OCR4_ANNOTATION_COST_PER_PAGE = 0.005 + +REPO_ROOT = Path(__file__).parents[5] +MAIN_COST_MAP = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_COST_MAP = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +OCR3_MODEL = "mistral/mistral-ocr-2512" +OCR3_COST_PER_PAGE = 0.002 +OCR3_ANNOTATION_COST_PER_PAGE = 0.003 + +AZURE_DOC_AI_MODEL = "azure_ai/mistral-document-ai-2512" +AZURE_DOC_AI_COST_PER_PAGE = 0.003 + + +def _ocr_response(model: str, pages_processed: int) -> OCRResponse: + return OCRResponse( + pages=[OCRPage(index=i, markdown=f"page {i}") for i in range(pages_processed)], + model=model, + usage_info=OCRUsageInfo(pages_processed=pages_processed), + ) + + +def _annotated_ocr_response(model: str, pages_processed: int | None, annotation_pages: int) -> OCRResponse: + return OCRResponse( + pages=[], + model=model, + usage_info=OCRUsageInfo(pages_processed=pages_processed, pages_processed_annotation=annotation_pages), + ) diff --git a/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py b/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py index 80c06e4a1e0..947d9b73e1a 100644 --- a/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py +++ b/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py @@ -154,6 +154,34 @@ class TestSCXAIModelMetadata: with open(json_path) as f: return json.load(f) + def test_scx_ai_models_registered_with_correct_metadata(self): + model_cost = self._load(("model_prices_and_context_window.json",)) + for model in self.SCX_MODELS: + info = model_cost.get(model) + assert info is not None, f"{model} missing from model_prices_and_context_window.json" + assert info["litellm_provider"] == "scx-ai" + assert info["mode"] == "chat" + assert info["input_cost_per_token"] > 0 + assert info["output_cost_per_token"] > 0 + assert info["supports_function_calling"] is True + assert info["supports_tool_choice"] is True + assert info["supports_reasoning"] is True + assert info["supports_response_schema"] is True + assert info.get("supports_vision", False) is (model in self.VISION_MODELS) + + assert info["supports_prompt_caching"] is True + assert 0 < info["cache_read_input_token_cost"] < info["input_cost_per_token"] + + assert info["max_tokens"] == info["max_output_tokens"] + assert info["max_input_tokens"] >= 1_000_000 + + def test_scx_ai_models_synced_to_backup(self): + model_cost = self._load(("model_prices_and_context_window.json",)) + backup = self._load(("litellm", "model_prices_and_context_window_backup.json")) + for model in self.SCX_MODELS: + assert model in backup, f"{model} missing from backup json" + assert backup[model] == model_cost[model], f"{model} differs between root and backup json" + class TestSCXAIDashboardRegistration: @staticmethod diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_integration.py b/tests/test_litellm/llms/perplexity/test_perplexity_integration.py index bd72d9b4740..bbb9cdef5fd 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_integration.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_integration.py @@ -17,6 +17,7 @@ from litellm import ModelResponse from litellm.cost_calculator import cost_per_token from litellm.llms.perplexity.chat.transformation import PerplexityChatConfig from litellm.types.utils import PromptTokensDetailsWrapper, Usage +from litellm.utils import get_model_info class TestPerplexityIntegration: @@ -55,6 +56,23 @@ class TestPerplexityIntegration: } } + def test_model_info_includes_custom_fields(self): + """Test that get_model_info returns the custom Perplexity cost fields.""" + model_info = get_model_info(model="sonar-deep-research", custom_llm_provider="perplexity") + + # Verify custom fields are included + required_fields = [ + "citation_cost_per_token", + "search_context_cost_per_query", + "input_cost_per_token", + "output_cost_per_token", + "output_cost_per_reasoning_token", + ] + + for field in required_fields: + assert field in model_info, f"Missing field: {field}" + assert model_info[field] is not None, f"Null value for field: {field}" + def test_various_citation_sizes(self): """Test cost calculation with various citation sizes.""" config = PerplexityChatConfig() diff --git a/tests/test_litellm/llms/tencent/test_cost_calculator.py b/tests/test_litellm/llms/tencent/test_cost_calculator.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/test_litellm/llms/tencent/test_cost_calculator.py @@ -0,0 +1 @@ + diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 2d1e4305449..f6da1bbcd0e 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -452,6 +452,44 @@ def test_vertex_claude_completion_does_not_mutate_shared_extra_headers(): assert shared_extra_headers == {}, "extra_headers must not be mutated by completion()" +def test_messages_thinking_shape_follows_exact_vertex_entry_flag(local_model_cost_map, monkeypatch): + """The Vertex messages config must probe capabilities under ``vertex_ai`` so an + operator setting ``supports_adaptive_thinking: false`` on the exact + ``vertex_ai/claude-opus-4-8`` entry beats the unmodified ``anthropic`` entry. + With the inherited ``"anthropic"`` provider default the flip was ignored and + the transform kept emitting ``thinking.type='adaptive'``.""" + import litellm + + config = VertexAIPartnerModelsAnthropicMessagesConfig() + + def transform(): + return config.transform_anthropic_messages_request( + model="claude-opus-4-8", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={ + "max_tokens": 4096, + "reasoning_effort": "medium", + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + result = transform() + assert result.get("thinking") == {"type": "adaptive", "display": "summarized"} + assert result.get("output_config") == {"effort": "medium"} + + monkeypatch.setitem(litellm.model_cost["vertex_ai/claude-opus-4-8"], "supports_adaptive_thinking", False) + litellm.get_model_info.cache_clear() + assert litellm.model_cost["claude-opus-4-8"]["supports_adaptive_thinking"] is True + + flipped = transform() + thinking = flipped.get("thinking") + assert isinstance(thinking, dict) + assert thinking.get("type") == "enabled" + assert isinstance(thinking.get("budget_tokens"), int) + assert "output_config" not in flipped + + def _vertex_transform(model, messages, system=None): config = VertexAIPartnerModelsAnthropicMessagesConfig() params = {"max_tokens": 256} @@ -547,6 +585,10 @@ class TestVertexAnthropicMidConversationSystem: def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag(): + """Exact cost-map hits win over the ``claude-mid-conversation-system`` + fallback rule, so a ``vertex_ai`` Claude 4.8+/5 entry missing the flag would + be treated as unsupported and hoist every reminder, collapsing the prompt + cache. Every mapped vertex_ai entry the rule matches must carry the flag.""" import re import litellm diff --git a/tests/test_litellm/llms/xai/test_xai_model_registry.py b/tests/test_litellm/llms/xai/test_xai_model_registry.py new file mode 100644 index 00000000000..a455d1fb233 --- /dev/null +++ b/tests/test_litellm/llms/xai/test_xai_model_registry.py @@ -0,0 +1,55 @@ +""" +Registry regression tests for xAI entries in the model cost map. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parents[4] +PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +# https://docs.x.ai/developers/model-capabilities/text/multi-agent +# "The multi-agent model does not work with the OpenAI Chat Completions API." +RESPONSES_ONLY_MODELS = ( + "xai/grok-4.20-multi-agent-0309", + "xai/grok-4.20-multi-agent-beta-0309", +) + +MAP_PATHS = (PRICES_PATH, BACKUP_PRICES_PATH) + + +@pytest.fixture(scope="module", params=[p.name for p in MAP_PATHS]) +def cost_map(request: pytest.FixtureRequest) -> dict: + path = next(p for p in MAP_PATHS if p.name == request.param) + return json.loads(path.read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("model", RESPONSES_ONLY_MODELS) +def test_multi_agent_models_are_responses_only(cost_map: dict, model: str): + entry = cost_map[model] + assert entry["supported_endpoints"] == ["/v1/responses"] + assert entry["mode"] == "responses" + + +def test_surviving_xai_chat_models_still_serve_chat_completions(cost_map: dict): + """Guard against the removal above over-reaching into live models.""" + chat_models = [ + key + for key, value in cost_map.items() + if isinstance(value, dict) and value.get("litellm_provider") == "xai" and value.get("mode") == "chat" + ] + assert "xai/grok-4.3" in chat_models + assert "xai/grok-4.6" in chat_models + + +def test_both_cost_maps_agree_on_xai_entries(): + prices = json.loads(PRICES_PATH.read_text(encoding="utf-8")) + backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8")) + xai_keys = {k for k, v in prices.items() if isinstance(v, dict) and v.get("litellm_provider") == "xai"} + assert xai_keys + assert {k: prices[k] for k in xai_keys} == {k: backup[k] for k in xai_keys} diff --git a/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py b/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py index 9734ea3e545..4c8231d357e 100644 --- a/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py +++ b/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py @@ -98,3 +98,23 @@ def test_redirected_slug_carries_the_target_tier_rates(cost_map: dict, slug: str for field in TIER_COST_FIELDS: assert entry[field] == target[field], field assert {k for k in entry if "_above_" in k} == {k for k in target if "_above_" in k} + + +def test_both_cost_maps_agree_on_the_redirected_slugs(): + prices = json.loads(PRICES_PATH.read_text(encoding="utf-8")) + backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8")) + for slug in (*REDIRECTED_SLUGS, *CODE_SLUGS, REDIRECT_TARGET, CODE_REDIRECT_TARGET): + assert prices[slug] == backup[slug], slug + + +def test_every_retired_chat_slug_is_covered(cost_map: dict): + """The lists above must stay in step with what the registry marks retired.""" + marked = { + key + for key, entry in cost_map.items() + if isinstance(entry, dict) + and entry.get("litellm_provider") == "xai" + and "deprecation_date" in entry + and entry.get("mode") == "chat" + } + assert marked == {*REDIRECTED_SLUGS, *CODE_SLUGS} diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 408a394a105..792c44a025b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -327,6 +327,29 @@ def test_openai_style_cache_write_tokens_are_netted_out(): ) +def test_sub_input_cache_write_price_is_an_extra_saving(): + """A few models price writes below input; there the premium is a real credit. + + Clamping the premium at zero would silently undercount these, so the subtraction + stays signed. ``azure/eu/gpt-4o-2024-11-20`` ships a write price at ~0.5x input. + """ + model = "azure/eu/gpt-4o-2024-11-20" + info = litellm.get_model_info(model=model) + input_cost = info["input_cost_per_token"] + cheap_write = info["cache_creation_input_token_cost"] + assert 0 < cheap_write < input_cost, "fixture drifted: this test needs a model pricing cache writes below input" + + result = compute_savings_spend( + model=model, + custom_llm_provider=None, + compression_saved_tokens=0, + gateway_injected_cache=True, + usage_object=_caching_usage(read=1000, written=4000), + ) + assert result.prompt_caching == pytest.approx(4000 * (input_cost - cheap_write)) + assert result.prompt_caching > 0 + + def test_negative_cache_write_count_clamps_to_zero(): """A malformed negative write count must not be read as a saving.""" input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5") diff --git a/tests/test_litellm/test_azure_ai_grok_4_3_model_metadata.py b/tests/test_litellm/test_azure_ai_grok_4_3_model_metadata.py new file mode 100644 index 00000000000..9d9f392a149 --- /dev/null +++ b/tests/test_litellm/test_azure_ai_grok_4_3_model_metadata.py @@ -0,0 +1,37 @@ +import json +from pathlib import Path + +import pytest + +import litellm +from litellm import get_model_info + +AZURE_AI_GROK_4_3_MODEL = "azure_ai/grok-4.3" +AZURE_AI_GROK_4_3_SOURCE = "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-grok-4-3-on-microsoft-foundry-latest-generation-agentic-capabilities/4517096" + + +def _load_model_cost(path: Path) -> dict: + with open(path) as f: + return json.load(f) + + +@pytest.fixture(autouse=True) +def reload_model_costs(): + original_model_cost = litellm.model_cost + json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" + litellm.model_cost = _load_model_cost(json_path) + get_model_info.cache_clear() + yield + litellm.model_cost = original_model_cost + get_model_info.cache_clear() + + +def test_azure_ai_grok_4_3_backup_matches_main(): + repo_root = Path(__file__).parents[2] + main_path = repo_root / "model_prices_and_context_window.json" + backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json" + + main_cost = _load_model_cost(main_path) + backup_cost = _load_model_cost(backup_path) + + assert backup_cost.get(AZURE_AI_GROK_4_3_MODEL) == main_cost.get(AZURE_AI_GROK_4_3_MODEL) diff --git a/tests/test_litellm/test_azure_ai_grok_4_6_model_metadata.py b/tests/test_litellm/test_azure_ai_grok_4_6_model_metadata.py new file mode 100644 index 00000000000..43df9a648c2 --- /dev/null +++ b/tests/test_litellm/test_azure_ai_grok_4_6_model_metadata.py @@ -0,0 +1,44 @@ +from pathlib import Path +from typing import Final + +import pytest +from pydantic import TypeAdapter + +from litellm import cost_per_token, get_model_info +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + +REPO_ROOT: Final = Path(__file__).parents[2] +MODEL: Final = "azure_ai/grok-4.6" +COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) + + +def _cost_map_entry(path: Path) -> dict[str, object]: + return COST_MAP_ADAPTER.validate_json(path.read_bytes())[MODEL] + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_azure_ai_grok_4_6_is_priced_and_routed() -> None: + routed_model, provider, _, _ = get_llm_provider(model=MODEL) + assert (routed_model, provider) == ("grok-4.6", "azure_ai") + + info = get_model_info(model=routed_model, custom_llm_provider=provider) + assert info["litellm_provider"] == "azure_ai" + assert info["mode"] == "chat" + assert info["supports_function_calling"] is True + assert info["supports_prompt_caching"] is True + assert info["supports_reasoning"] is True + assert info["supports_response_schema"] is True + assert info["supports_tool_choice"] is True + assert info["supports_vision"] is True + assert info["supports_web_search"] is True + + prompt_cost, completion_cost = cost_per_token(model=MODEL, prompt_tokens=1_000_000, completion_tokens=1_000_000) + assert prompt_cost > 0 + assert completion_cost > 0 + + +def test_azure_ai_grok_4_6_entry_source_and_backup_match() -> None: + main_entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json") + backup_entry = _cost_map_entry(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json") + + assert backup_entry == main_entry diff --git a/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py b/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py index 94afd9fe254..31f3a67beac 100644 --- a/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py +++ b/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py @@ -1,10 +1,31 @@ +import json +from pathlib import Path + import pytest import litellm +from litellm.utils import supports_function_calling, supports_prompt_caching + +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +MODEL = "baseten/zai-org/GLM-5.3" + +INPUT_COST = 1.4e-06 +CACHED_INPUT_COST = 1.4e-07 +OUTPUT_COST = 4.4e-06 + + +def _load(path): + with open(path) as f: + return json.load(f) @pytest.fixture def local_model_cost_map(monkeypatch): + """Force get_model_info to resolve against the in-repo cost map instead of the + remote one fetched at import time, which still carries the pre-merge registry.""" monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) litellm.get_model_info.cache_clear() @@ -12,6 +33,32 @@ def local_model_cost_map(monkeypatch): litellm.get_model_info.cache_clear() +def test_baseten_glm_5_3_capabilities_are_visible_to_callers(local_model_cost_map): + """The entry advertises prompt caching and tool calling, so the helpers every + caller checks before sending a request must say so too.""" + assert supports_prompt_caching(model=MODEL) is True + assert supports_function_calling(model=MODEL) is True + + info = litellm.get_model_info(model="zai-org/GLM-5.3", custom_llm_provider="baseten") + assert info["max_input_tokens"] > 0 + assert info["max_output_tokens"] > 0 + + +def test_backup_matches_main(): + """Ensure the bundled (backup) cost map stays in sync with the canonical file. + + Both keys are asserted present first: comparing two ``.get`` results alone passes + just as happily when neither file has the entry at all, which is the exact state + this test exists to catch. + """ + main_cost = _load(MAIN_PATH) + backup_cost = _load(BACKUP_PATH) + + assert MODEL in main_cost, f"{MODEL} missing from model_prices_and_context_window.json" + assert MODEL in backup_cost, f"{MODEL} missing from model_prices_and_context_window_backup.json" + assert backup_cost[MODEL] == main_cost[MODEL], f"{MODEL} differs between main and backup model cost maps" + + def test_entry_advertises_only_what_the_baseten_path_accepts(local_model_cost_map): """The Baseten path rejects unsupported request parameters.""" supported = litellm.get_supported_openai_params(model="zai-org/GLM-5.3", custom_llm_provider="baseten") diff --git a/tests/test_litellm/test_bedrock_marengo_embed_3_model_metadata.py b/tests/test_litellm/test_bedrock_marengo_embed_3_model_metadata.py new file mode 100644 index 00000000000..1a0e1665556 --- /dev/null +++ b/tests/test_litellm/test_bedrock_marengo_embed_3_model_metadata.py @@ -0,0 +1,52 @@ +import json +from pathlib import Path + +import pytest + +import litellm +from litellm.constants import bedrock_embedding_models + +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +BASE_MODEL = "twelvelabs.marengo-embed-3-0-v1:0" +PROFILE_MODELS = ("us.twelvelabs.marengo-embed-3-0-v1:0", "eu.twelvelabs.marengo-embed-3-0-v1:0") +ALL_MODELS = (BASE_MODEL, *PROFILE_MODELS) +MARENGO_2_7_MODELS = ( + "twelvelabs.marengo-embed-2-7-v1:0", + "us.twelvelabs.marengo-embed-2-7-v1:0", + "eu.twelvelabs.marengo-embed-2-7-v1:0", +) +PER_REQUEST_MODELS = (*ALL_MODELS, *MARENGO_2_7_MODELS) + +TEXT_REQUEST_COST = 7e-05 +IMAGE_REQUEST_COST = 0.0001 +VIDEO_COST_PER_SECOND = 0.0007 +AUDIO_COST_PER_SECOND = 0.00014 + + +def _load(path): + with open(path) as f: + return json.load(f) + + +@pytest.mark.parametrize("model", ALL_MODELS) +def test_marengo_embed_3_is_visible_to_callers(model, local_model_cost_map): + info = litellm.get_model_info(model=model, custom_llm_provider="bedrock") + assert info["mode"] == "embedding" + assert info["output_vector_size"] == 512 + + +def test_marengo_embed_3_is_a_known_bedrock_embedding_model(): + assert BASE_MODEL in bedrock_embedding_models + + +@pytest.mark.parametrize("model", PER_REQUEST_MODELS) +def test_backup_matches_main(model): + main_cost = _load(MAIN_PATH) + backup_cost = _load(BACKUP_PATH) + + assert model in main_cost, f"{model} missing from model_prices_and_context_window.json" + assert model in backup_cost, f"{model} missing from model_prices_and_context_window_backup.json" + assert backup_cost[model] == main_cost[model], f"{model} differs between main and backup model cost maps" diff --git a/tests/test_litellm/test_bedrock_usgov_pricing.py b/tests/test_litellm/test_bedrock_usgov_pricing.py index b83176b6460..a3a7fc4ed7a 100644 --- a/tests/test_litellm/test_bedrock_usgov_pricing.py +++ b/tests/test_litellm/test_bedrock_usgov_pricing.py @@ -31,6 +31,18 @@ def model_data(): return json.load(f) +def test_usgov_east_haiku_profile_mirrors_in_region_row(model_data): + """us-gov-east-1 serves claude-3-haiku through the us-gov. inference profile + only, so the profile row must bill exactly like the in-region gov row. + """ + profile = model_data["us-gov.anthropic.claude-3-haiku-20240307-v1:0"] + in_region = model_data["bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0"] + assert profile["litellm_provider"] == "bedrock_converse" + assert {k: v for k, v in profile.items() if k != "litellm_provider"} == { + k: v for k, v in in_region.items() if k != "litellm_provider" + } + + GOV_ROW_SOURCES = { "us-gov.anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1", "bedrock/us-gov-west-1/anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1", diff --git a/tests/test_litellm/test_claude_fable_5_config.py b/tests/test_litellm/test_claude_fable_5_config.py index 25327eb9430..4b03848da2c 100644 --- a/tests/test_litellm/test_claude_fable_5_config.py +++ b/tests/test_litellm/test_claude_fable_5_config.py @@ -12,7 +12,10 @@ shape, which Fable 5 rejects with a 400. import json import os +import pytest + from litellm.constants import BEDROCK_CONVERSE_MODELS +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") @@ -23,10 +26,88 @@ def _load_root_cost_map() -> dict: return json.load(f) +def test_fable_5_present_in_bundled_backup(): + """The bundled backup is the runtime fallback (and what tests load with + ``LITELLM_LOCAL_MODEL_COST_MAP=True``) — it must carry the same entries as + the root cost map, otherwise the model resolves on one path but not the + other.""" + backup = GetModelCostMap.load_local_model_cost_map() + root = _load_root_cost_map() + for model_name in ( + "claude-fable-5", + "anthropic.claude-fable-5", + "global.anthropic.claude-fable-5", + "us.anthropic.claude-fable-5", + "eu.anthropic.claude-fable-5", + "vertex_ai/claude-fable-5", + "vertex_ai/claude-fable-5@default", + "azure_ai/claude-fable-5", + ): + assert model_name in backup, f"Missing from backup cost map: {model_name}" + assert backup[model_name] == root[model_name], model_name + + def test_fable_5_registered_for_bedrock_converse(): assert "anthropic.claude-fable-5" in BEDROCK_CONVERSE_MODELS +@pytest.mark.parametrize( + "cost_map", + [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], + ids=["root", "bundled_backup"], +) +def test_fable_5_all_variants_carry_adaptive_thinking_flag(cost_map): + """Every Fable 5 entry must advertise ``supports_adaptive_thinking``. + + Adaptive-thinking detection is cost-map driven, so a single variant missing + the flag silently sends the legacy ``thinking.type='enabled'`` shape and the + provider 400s (issue #29188 for the Opus 4.8 equivalent). Fable 5 is even + stricter than Opus 4.8: an explicit ``thinking.type='disabled'`` also 400s, + so adaptive is the only valid thinking shape LiteLLM can emit for it.""" + variants = [k for k in cost_map if "claude-fable-5" in k] + assert variants, "no claude-fable-5 entries found in cost map" + missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True] + assert not missing, f"missing supports_adaptive_thinking: {missing}" + + +@pytest.mark.parametrize( + "cost_map", + [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], + ids=["root", "bundled_backup"], +) +def test_fable_5_all_variants_carry_thinking_always_on_flag(cost_map): + """Every Fable 5 entry must advertise ``thinking_always_on``. + + The flag drives the Anthropic transformations to omit an explicit + ``thinking.type='disabled'``, which Fable 5 rejects with a 400; a variant + missing the flag forwards the param verbatim and the provider 400s.""" + variants = [k for k in cost_map if "claude-fable-5" in k] + assert variants, "no claude-fable-5 entries found in cost map" + missing = [k for k in variants if cost_map[k].get("thinking_always_on") is not True] + assert not missing, f"missing thinking_always_on: {missing}" + + +@pytest.mark.parametrize( + "model", + [ + "claude-fable-5", + "anthropic/claude-fable-5", + "anthropic.claude-fable-5", + "bedrock/us.anthropic.claude-fable-5", + "bedrock/invoke/eu.anthropic.claude-fable-5", + "bedrock/global.anthropic.claude-fable-5", + "vertex_ai/claude-fable-5", + "azure_ai/claude-fable-5", + ], +) +def test_adaptive_thinking_detected_for_fable_5(local_model_cost_map, model): + """Provider-routed ids must resolve to a flagged entry so ``reasoning_effort`` + maps to ``thinking.type='adaptive'`` + ``output_config.effort``.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True + + FABLE_5_1_VARIANTS = ( "claude-fable-5-1", "anthropic.claude-fable-5-1", @@ -39,5 +120,53 @@ FABLE_5_1_VARIANTS = ( ) +def test_fable_5_1_present_in_bundled_backup(): + backup = GetModelCostMap.load_local_model_cost_map() + root = _load_root_cost_map() + for model_name in FABLE_5_1_VARIANTS: + assert model_name in backup, f"Missing from backup cost map: {model_name}" + assert backup[model_name] == root[model_name], model_name + + def test_fable_5_1_registered_for_bedrock_converse(): assert "anthropic.claude-fable-5-1" in BEDROCK_CONVERSE_MODELS + + +@pytest.mark.parametrize( + "model", + [ + "claude-fable-5-1", + "anthropic/claude-fable-5-1", + "anthropic.claude-fable-5-1", + "bedrock/us.anthropic.claude-fable-5-1", + "bedrock/invoke/eu.anthropic.claude-fable-5-1", + "bedrock/global.anthropic.claude-fable-5-1", + "vertex_ai/claude-fable-5-1", + "azure_ai/claude-fable-5-1", + ], +) +def test_adaptive_thinking_detected_for_fable_5_1(local_model_cost_map, model): + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True + + +@pytest.mark.parametrize( + "cost_map", + [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], + ids=["root", "bundled_backup"], +) +def test_sampling_params_flag_on_all_models_that_removed_them(cost_map): + """Fable 5 and Opus 4.7/4.8 reject ``top_p``/``top_k``/``temperature != 1``; + the drop/raise gating is cost-map driven, so every variant must carry an + explicit ``supports_sampling_params: false``. The perplexity route is + exempt: it is OpenAI-compatible and maps sampling params upstream.""" + variants = [ + k + for k in cost_map + if any(v in k for v in ("claude-fable-5", "claude-opus-4-7", "claude-opus-4-8")) + and not k.startswith("perplexity/") + ] + assert variants, "no matching entries found in cost map" + missing = [k for k in variants if cost_map[k].get("supports_sampling_params") is not False] + assert not missing, f"missing supports_sampling_params=false: {missing}" diff --git a/tests/test_litellm/test_claude_haiku_4_5_config.py b/tests/test_litellm/test_claude_haiku_4_5_config.py new file mode 100644 index 00000000000..d0b7f4f8a2c --- /dev/null +++ b/tests/test_litellm/test_claude_haiku_4_5_config.py @@ -0,0 +1,46 @@ +""" +Test Claude Haiku 4.5 model configurations for Bedrock +https://github.com/BerriAI/litellm/issues/15818 +""" + +import json +import os + + +def test_bedrock_haiku_4_5_matches_sonnet_capabilities(): + """ + Test that Haiku 4.5 has same capabilities as Sonnet 4.5 + (including computer_use, vision, tools, etc.) + """ + # Load model configuration + json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json") + with open(json_path) as f: + model_data = json.load(f) + + haiku_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0" + sonnet_model = "us.anthropic.claude-sonnet-4-5-20250929-v1:0" + + haiku_info = model_data[haiku_model] + sonnet_info = model_data[sonnet_model] + + # Both should use bedrock_converse + assert haiku_info["litellm_provider"] == "bedrock_converse" + assert sonnet_info["litellm_provider"] == "bedrock_converse" + + # Shared capabilities that should match + shared_capabilities = [ + "supports_vision", + "supports_computer_use", + "supports_function_calling", + "supports_tool_choice", + "supports_prompt_caching", + "supports_response_schema", + "supports_pdf_input", + "supports_assistant_prefill", + "supports_reasoning", + ] + + for capability in shared_capabilities: + assert haiku_info.get(capability) == sonnet_info.get(capability), ( + f"Capability {capability} mismatch: Haiku={haiku_info.get(capability)}, Sonnet={sonnet_info.get(capability)}" + ) diff --git a/tests/test_litellm/test_claude_opus_5_config.py b/tests/test_litellm/test_claude_opus_5_config.py index 5f3e23fc18a..07e493af914 100644 --- a/tests/test_litellm/test_claude_opus_5_config.py +++ b/tests/test_litellm/test_claude_opus_5_config.py @@ -18,6 +18,7 @@ import os import pytest from litellm.constants import BEDROCK_CONVERSE_MODELS +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") @@ -61,5 +62,31 @@ def test_opus_5_bedrock_rejects_strict_tools(model_name, local_model_cost_map): assert bedrock_converse_supports_strict_tools(model_name) is False +def test_opus_5_present_in_bundled_backup(): + """The bundled backup is the runtime fallback (and what tests load with + ``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the + root cost map, otherwise the model resolves on one path but not the other.""" + backup = GetModelCostMap.load_local_model_cost_map() + for model_name in ALL_OPUS_5_VARIANTS: + assert model_name in backup, f"Missing from backup cost map: {model_name}" + + def test_opus_5_registered_for_bedrock_converse(): assert "anthropic.claude-opus-5" in BEDROCK_CONVERSE_MODELS + + +@pytest.mark.parametrize( + "cost_map", + [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], + ids=["root", "bundled_backup"], +) +def test_opus_5_all_variants_carry_adaptive_thinking_flag(cost_map): + """Every Opus 5 entry must advertise ``supports_adaptive_thinking``. + + Adaptive-thinking detection is cost-map driven, so a single variant missing + the flag silently sends the legacy ``thinking.type='enabled'`` shape, which + Opus 5 rejects with a 400.""" + variants = [k for k in cost_map if "claude-opus-5" in k] + assert variants, "no claude-opus-5 entries found in cost map" + missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True] + assert not missing, f"missing supports_adaptive_thinking: {missing}" diff --git a/tests/test_litellm/test_claude_sonnet_5_config.py b/tests/test_litellm/test_claude_sonnet_5_config.py index 18273f7494a..8c6d2cd1851 100644 --- a/tests/test_litellm/test_claude_sonnet_5_config.py +++ b/tests/test_litellm/test_claude_sonnet_5_config.py @@ -13,7 +13,10 @@ populate ``litellm.anthropic_models`` at import, which is what lets a bare import json import os +import pytest + from litellm.constants import BEDROCK_CONVERSE_MODELS +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") @@ -37,5 +40,31 @@ def _load_root_cost_map() -> dict: return json.load(f) +def test_sonnet_5_present_in_bundled_backup(): + """The bundled backup is the runtime fallback (and what tests load with + ``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the + root cost map, otherwise the model resolves on one path but not the other.""" + backup = GetModelCostMap.load_local_model_cost_map() + for model_name in ALL_SONNET_5_VARIANTS: + assert model_name in backup, f"Missing from backup cost map: {model_name}" + + def test_sonnet_5_registered_for_bedrock_converse(): assert "anthropic.claude-sonnet-5" in BEDROCK_CONVERSE_MODELS + + +@pytest.mark.parametrize( + "cost_map", + [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], + ids=["root", "bundled_backup"], +) +def test_sonnet_5_all_variants_carry_adaptive_thinking_flag(cost_map): + """Every Sonnet 5 entry must advertise ``supports_adaptive_thinking``. + + Adaptive-thinking detection is cost-map driven, so a single variant missing + the flag silently sends the legacy ``thinking.type='enabled'`` shape and the + provider 400s. This guards against a future variant being added without it.""" + variants = [k for k in cost_map if "claude-sonnet-5" in k] + assert variants, "no claude-sonnet-5 entries found in cost map" + missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True] + assert not missing, f"missing supports_adaptive_thinking: {missing}" diff --git a/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py b/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py new file mode 100644 index 00000000000..4e770be7c3e --- /dev/null +++ b/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py @@ -0,0 +1,50 @@ +""" +Regression tests for the Cloudflare Workers AI text-generation catalog in the +model-cost map. + +The Cloudflare list was badly stale (only 4 ancient entries). These tests pin +the newly added current Workers AI models (sourced from Cloudflare's live +``/ai/models/search?task=Text Generation`` catalog) and guard against the root +``model_prices_and_context_window.json`` and the bundled +``litellm/model_prices_and_context_window_backup.json`` drifting out of sync for +the ``cloudflare/`` namespace. +""" + +import json +import os + +import pytest + +import litellm + +ROOT_MAP = os.path.join( + os.path.dirname(os.path.dirname(litellm.__file__)), + "model_prices_and_context_window.json", +) +BACKUP_MAP = os.path.join( + os.path.dirname(litellm.__file__), + "model_prices_and_context_window_backup.json", +) + + +def _load(path: str) -> dict: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _cloudflare_keys(data: dict) -> set: + return {k for k in data if k.startswith("cloudflare/")} + + +def test_root_and_backup_have_identical_cloudflare_keys(): + if not os.path.exists(ROOT_MAP): + pytest.skip("root cost map only ships in source checkouts") + assert _cloudflare_keys(_load(ROOT_MAP)) == _cloudflare_keys(_load(BACKUP_MAP)) + + +def test_root_and_backup_cloudflare_entries_are_byte_for_byte_equal(): + if not os.path.exists(ROOT_MAP): + pytest.skip("root cost map only ships in source checkouts") + root = {k: v for k, v in _load(ROOT_MAP).items() if k.startswith("cloudflare/")} + backup = {k: v for k, v in _load(BACKUP_MAP).items() if k.startswith("cloudflare/")} + assert root == backup diff --git a/tests/test_litellm/test_command_r7b_pricing.py b/tests/test_litellm/test_command_r7b_pricing.py new file mode 100644 index 00000000000..c091fec313e --- /dev/null +++ b/tests/test_litellm/test_command_r7b_pricing.py @@ -0,0 +1,53 @@ +""" +Regression test: ``command-r7b-12-2024`` had its input/output per-token +costs transposed in the model-cost maps (input=1.5e-07 / output=3.75e-08), +even though Cohere publishes $0.0375/1M input and $0.15/1M output, i.e. +output is ~4x input like every other ``command-r`` entry. + +These tests pin the corrected values in both the primary price map and the +``litellm/`` backup, and verify ``get_model_info`` surfaces them, so the +swap cannot silently regress. +""" + +import json +import os + +import litellm + +MODEL = "command-r7b-12-2024" +EXPECTED_INPUT_COST = 3.75e-08 +EXPECTED_OUTPUT_COST = 1.5e-07 + + +def _load_json(path: str) -> dict: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _backup_path() -> str: + return os.path.join( + os.path.dirname(litellm.__file__), + "model_prices_and_context_window_backup.json", + ) + + +def _main_path() -> str: + # This test lives at ``tests/test_litellm/``; the primary price map sits at + # the repo root, two directories up. Resolve it relative to this file so the + # test works regardless of where ``litellm`` itself is installed (e.g. a pip + # install into site-packages). + return os.path.join( + os.path.dirname(__file__), + "..", + "..", + "model_prices_and_context_window.json", + ) + + +class TestCommandR7bPricingData: + """The JSON price maps must carry Cohere's published costs, with output + more expensive than input.""" + + +class TestCommandR7bPricingModelInfo: + """``get_model_info`` must report the corrected, un-swapped costs.""" diff --git a/tests/test_litellm/test_daybreak_model_metadata.py b/tests/test_litellm/test_daybreak_model_metadata.py index 58f9ff4e7c8..79149b84f0b 100644 --- a/tests/test_litellm/test_daybreak_model_metadata.py +++ b/tests/test_litellm/test_daybreak_model_metadata.py @@ -32,6 +32,13 @@ def _load(path): return json.load(f) +def test_blue_alias_matches_its_snapshot_computer_use(): + cost_map = _load(MAIN_PATH) + + assert cost_map[BLUE_ALIAS]["supports_computer_use"] is True + assert cost_map[BLUE_SNAPSHOT]["supports_computer_use"] is True + + @pytest.mark.parametrize(("alias", "snapshot"), OFFICIAL_ALIAS_SNAPSHOTS) def test_official_alias_tracks_snapshot(alias, snapshot): cost_map = _load(MAIN_PATH) @@ -44,3 +51,11 @@ def test_official_alias_tracks_snapshot(alias, snapshot): field: snapshot_info.get(field) for field in PRICE_FIELDS } assert alias_info["max_output_tokens"] == snapshot_info["max_output_tokens"] + + +@pytest.mark.parametrize("model", (*DAYBREAK_MODELS, BLUE_SNAPSHOT, *(alias for alias, _ in OFFICIAL_ALIAS_SNAPSHOTS))) +def test_backup_matches_main(model): + main_cost = _load(MAIN_PATH) + backup_cost = _load(BACKUP_PATH) + + assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps" diff --git a/tests/test_litellm/test_deepseek_model_metadata.py b/tests/test_litellm/test_deepseek_model_metadata.py index e8a98094c4e..264f5e65fc5 100644 --- a/tests/test_litellm/test_deepseek_model_metadata.py +++ b/tests/test_litellm/test_deepseek_model_metadata.py @@ -15,6 +15,7 @@ import os import litellm from litellm.utils import ( _supports_factory, + supports_response_schema, ) # --------------------------------------------------------------------------- @@ -36,6 +37,18 @@ class TestDeepSeekModelCostEntries: """Verify that provider-prefixed DeepSeek entries contain the same capability flags as their bare-name counterparts in the JSON files.""" + def test_deepseek_chat_max_input_tokens_matches_bare_in_backup(self): + data = _load_backup_json() + bare = data.get("deepseek-chat", {}) + prefixed = data.get("deepseek/deepseek-chat", {}) + assert prefixed.get("max_input_tokens") == bare.get("max_input_tokens") + + def test_deepseek_reasoner_max_output_tokens_matches_bare_in_backup(self): + data = _load_backup_json() + bare = data.get("deepseek-reasoner", {}) + prefixed = data.get("deepseek/deepseek-reasoner", {}) + assert prefixed.get("max_output_tokens") == bare.get("max_output_tokens") + # --------------------------------------------------------------------------- # API-level tests – verify supports_response_schema returns True @@ -46,6 +59,18 @@ class TestSupportsResponseSchemaDeepSeek: """All calling conventions for DeepSeek should return True for ``supports_response_schema``.""" + def test_provider_slash_model(self): + assert supports_response_schema(model="deepseek/deepseek-chat") is True + + def test_explicit_provider(self): + assert supports_response_schema(model="deepseek-chat", custom_llm_provider="deepseek") is True + + def test_reasoner_provider_slash_model(self): + assert supports_response_schema(model="deepseek/deepseek-reasoner") is True + + def test_reasoner_explicit_provider(self): + assert supports_response_schema(model="deepseek-reasoner", custom_llm_provider="deepseek") is True + # --------------------------------------------------------------------------- # Fallback-logic test – bare model entry used when prefixed is incomplete diff --git a/tests/test_litellm/test_fireworks_serverless_model_costs.py b/tests/test_litellm/test_fireworks_serverless_model_costs.py index 0b55b0ff1e4..164f32fec1c 100644 --- a/tests/test_litellm/test_fireworks_serverless_model_costs.py +++ b/tests/test_litellm/test_fireworks_serverless_model_costs.py @@ -14,15 +14,7 @@ import os import pytest -NEW_ENTRIES = { - "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": { - "input_cost_per_token": 1.32e-06, - "cache_read_input_token_cost": 4.4e-08, - "output_cost_per_token": 3.96e-06, - "max_input_tokens": 1048576, - "max_output_tokens": 131072, - }, -} +from litellm.utils import get_model_info @pytest.fixture(scope="module") @@ -32,20 +24,17 @@ def model_data(): return json.load(f) -TWIN_PINNED_PRICES = { - "deepseek-v4-flash-0731": { - "input_cost_per_token": 2.2e-07, - "cache_read_input_token_cost": 7e-09, - "output_cost_per_token": 6.6e-07, - }, - "deepseek-v4p1-flash": { - "input_cost_per_token": 2.2e-07, - "cache_read_input_token_cost": 7e-09, - "output_cost_per_token": 6.6e-07, - "supports_vision": True, - "max_output_tokens": 393216, - }, -} +def test_bare_fireworks_ids_resolve_through_prefixed_entries(): + """Bare IDs from #37274 resolve via the provider-prefix lookup path.""" + for bare_id, prefixed_key in [ + ( + "accounts/fireworks/models/deepseek-v4-pro-0813", + "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813", + ), + ]: + info = get_model_info(model=bare_id, custom_llm_provider="fireworks_ai") + assert info.get("key") == prefixed_key + assert info["litellm_provider"] == "fireworks_ai" def test_fireworks_account_prefixed_twins_agree_on_price(model_data): diff --git a/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py b/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py index fd5dcc88e9f..10d1d6fecd1 100644 --- a/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py +++ b/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py @@ -109,6 +109,11 @@ def test_per_route_capabilities_match_model_cards(model: str, path: Path): assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}" +@pytest.mark.parametrize("model", ALL_KEYS) +def test_backup_matches_main(model: str): + assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model) + + def test_gemini_prefix_routes_to_gemini(): routed_model, provider, _, _ = get_llm_provider(model=GEMINI) assert routed_model == UNPREFIXED diff --git a/tests/test_litellm/test_gemini_tts_native_audio_pricing.py b/tests/test_litellm/test_gemini_tts_native_audio_pricing.py new file mode 100644 index 00000000000..3dcb18c1466 --- /dev/null +++ b/tests/test_litellm/test_gemini_tts_native_audio_pricing.py @@ -0,0 +1,84 @@ +import json +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import pytest + +import litellm + +REPO_ROOT: Final = Path(__file__).parents[2] +MAIN_PATH: Final = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +FLASH_TTS_KEYS: Final = ("gemini-2.5-flash-preview-tts", "gemini/gemini-2.5-flash-preview-tts") +PRO_TTS_KEYS: Final = ("gemini-2.5-pro-preview-tts", "gemini/gemini-2.5-pro-preview-tts") +NATIVE_AUDIO_KEYS: Final = tuple( + f"{prefix}gemini-2.5-flash-native-audio-{suffix}" + for prefix in ("", "gemini/") + for suffix in ("latest", "preview-09-2025", "preview-12-2025") +) + +LIVE_NATIVE_AUDIO_KEYS: Final = ( + "gemini-live-2.5-flash-preview-native-audio-09-2025", + "gemini/gemini-live-2.5-flash-preview-native-audio-09-2025", +) + +FLASH_TTS_INPUT: Final = 5e-07 +FLASH_TTS_AUDIO_OUTPUT: Final = 1e-05 +PRO_TTS_INPUT: Final = 1e-06 +PRO_TTS_AUDIO_OUTPUT: Final = 2e-05 +NATIVE_AUDIO_TEXT_INPUT: Final = 5e-07 +NATIVE_AUDIO_AUDIO_INPUT: Final = 3e-06 +NATIVE_AUDIO_TEXT_OUTPUT: Final = 2e-06 +NATIVE_AUDIO_AUDIO_OUTPUT: Final = 1.2e-05 + +PUBLISHED_RATES: Final = { + **{ + key: {"input_cost_per_token": FLASH_TTS_INPUT, "output_cost_per_token": FLASH_TTS_AUDIO_OUTPUT} + for key in FLASH_TTS_KEYS + }, + **{ + key: {"input_cost_per_token": PRO_TTS_INPUT, "output_cost_per_token": PRO_TTS_AUDIO_OUTPUT} + for key in PRO_TTS_KEYS + }, + **{ + key: { + "input_cost_per_token": NATIVE_AUDIO_TEXT_INPUT, + "input_cost_per_audio_token": NATIVE_AUDIO_AUDIO_INPUT, + "output_cost_per_token": NATIVE_AUDIO_TEXT_OUTPUT, + "output_cost_per_audio_token": NATIVE_AUDIO_AUDIO_OUTPUT, + } + for key in (*NATIVE_AUDIO_KEYS, *LIVE_NATIVE_AUDIO_KEYS) + }, +} +ALL_KEYS: Final = tuple(PUBLISHED_RATES) +NATIVE_AUDIO_BILLING_CASES: Final = ( + *((key, "gemini") for key in NATIVE_AUDIO_KEYS), + ("gemini-live-2.5-flash-preview-native-audio-09-2025", "vertex_ai"), + ("gemini/gemini-live-2.5-flash-preview-native-audio-09-2025", "gemini"), +) +LONG_CONTEXT_TIER_FIELDS: Final = ( + "input_cost_per_token_above_200k_tokens", + "output_cost_per_token_above_200k_tokens", + "cache_read_input_token_cost_above_200k_tokens", +) + + +def _load(path: Path) -> dict[str, dict[str, object]]: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +@pytest.fixture +def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +@pytest.mark.parametrize("model", ALL_KEYS) +def test_backup_matches_main(model: str): + assert _load(BACKUP_PATH)[model] == _load(MAIN_PATH)[model] diff --git a/tests/test_litellm/test_gpt_5_4_model_metadata.py b/tests/test_litellm/test_gpt_5_4_model_metadata.py new file mode 100644 index 00000000000..294d0757069 --- /dev/null +++ b/tests/test_litellm/test_gpt_5_4_model_metadata.py @@ -0,0 +1,44 @@ +import json +from functools import lru_cache +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +DOCUMENTED_MAX_INPUT_TOKENS = 272000 +DOCUMENTED_MAX_OUTPUT_TOKENS = 128000 + +SMALL_MODEL_NAMES = ( + "gpt-5.4-mini", + "gpt-5.4-mini-2026-03-17", + "gpt-5.4-nano", + "gpt-5.4-nano-2026-03-17", +) +SMALL_MODELS = tuple(f"{prefix}{name}" for prefix in ("", "azure/", "azure_ai/") for name in SMALL_MODEL_NAMES) + +STANDARD_PRICING = { + "gpt-5.4-mini": (7.5e-07, 4.5e-06, 7.5e-08), + "gpt-5.4-nano": (2e-07, 1.25e-06, 2e-08), +} + +LONG_CONTEXT_MODELS = ("gpt-5.4", "gpt-5.4-pro") + + +@lru_cache(maxsize=2) +def _load(path: Path) -> dict[str, dict[str, object]]: + with open(path) as f: + return json.load(f) + + +def _pricing_key(model: str) -> str: + return "gpt-5.4-nano" if "nano" in model else "gpt-5.4-mini" + + +@pytest.mark.parametrize("model", SMALL_MODELS) +def test_gpt_5_4_small_models_backup_matches_main(model: str) -> None: + assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model), ( + f"{model} differs between main and backup model cost maps" + ) diff --git a/tests/test_litellm/test_gpt_5_5_model_metadata.py b/tests/test_litellm/test_gpt_5_5_model_metadata.py new file mode 100644 index 00000000000..6a64627f1a2 --- /dev/null +++ b/tests/test_litellm/test_gpt_5_5_model_metadata.py @@ -0,0 +1,19 @@ +import json +from pathlib import Path + + +def test_azure_ai_gpt_5_5_backup_matches_main(): + """Ensure the bundled model cost map stays in sync with the canonical file.""" + repo_root = Path(__file__).parents[2] + main_path = repo_root / "model_prices_and_context_window.json" + backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json" + + with open(main_path) as f: + main_cost = json.load(f) + with open(backup_path) as f: + backup_cost = json.load(f) + + for model in ("azure_ai/gpt-5.5", "azure_ai/gpt-5.5-2026-04-23"): + assert backup_cost.get(model) == main_cost.get(model), ( + f"{model} differs between main and backup model cost maps" + ) diff --git a/tests/test_litellm/test_gpt_realtime_mode.py b/tests/test_litellm/test_gpt_realtime_mode.py index 778f620d72e..0ea85df84cb 100644 --- a/tests/test_litellm/test_gpt_realtime_mode.py +++ b/tests/test_litellm/test_gpt_realtime_mode.py @@ -1,4 +1,8 @@ -from typing_extensions import get_args, get_type_hints +import json +from pathlib import Path +from typing import get_args + +from typing_extensions import get_type_hints from litellm.types.utils import ModelInfoBase @@ -41,3 +45,13 @@ ALL_REALTIME_ONLY_GPT_MODELS = REALTIME_ONLY_GPT_MODELS + REALTIME_ONLY_GPT_MODE def test_realtime_is_a_valid_mode_literal(): hints = get_type_hints(ModelInfoBase, include_extras=False) assert "realtime" in get_args(hints["mode"]) + + +def test_backup_matches_main_for_realtime_models(): + repo_root = Path(__file__).parents[2] + with open(repo_root / "model_prices_and_context_window.json") as f: + main_cost = json.load(f) + with open(repo_root / "litellm" / "model_prices_and_context_window_backup.json") as f: + backup_cost = json.load(f) + for model in ALL_REALTIME_ONLY_GPT_MODELS: + assert backup_cost.get(model) == main_cost.get(model) diff --git a/tests/test_litellm/test_mistral_medium_3_5_model_metadata.py b/tests/test_litellm/test_mistral_medium_3_5_model_metadata.py new file mode 100644 index 00000000000..ab38d8a9118 --- /dev/null +++ b/tests/test_litellm/test_mistral_medium_3_5_model_metadata.py @@ -0,0 +1,33 @@ +import json +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +MEDIUM_3_5_MODELS = ( + "mistral/mistral-medium-3-5", + "mistral/mistral-medium-2604", + "mistral/mistral-medium-latest", +) + +SYNCED_MODELS = MEDIUM_3_5_MODELS + ( + "mistral/mistral-medium-2508", + "mistral/mistral-medium-3-1-2508", +) + + +def _load(path): + with open(path) as f: + return json.load(f) + + +@pytest.mark.parametrize("model", SYNCED_MODELS) +def test_backup_matches_main(model): + """Ensure the bundled (backup) cost map stays in sync with the canonical file.""" + main_cost = _load(MAIN_PATH) + backup_cost = _load(BACKUP_PATH) + + assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps" diff --git a/tests/test_litellm/test_mistral_small_4_0_model_metadata.py b/tests/test_litellm/test_mistral_small_4_0_model_metadata.py new file mode 100644 index 00000000000..182c444bac9 --- /dev/null +++ b/tests/test_litellm/test_mistral_small_4_0_model_metadata.py @@ -0,0 +1,26 @@ +import json +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +SMALL_4_0_MODELS = ( + "mistral/mistral-small-latest", + "mistral/mistral-small-2603", +) + + +def _load(path): + with open(path) as f: + return json.load(f) + + +@pytest.mark.parametrize("model", SMALL_4_0_MODELS) +def test_backup_matches_main(model): + main_cost = _load(MAIN_PATH) + backup_cost = _load(BACKUP_PATH) + + assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps" diff --git a/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py b/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py new file mode 100644 index 00000000000..8467cbd43b1 --- /dev/null +++ b/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py @@ -0,0 +1,52 @@ +import json +from pathlib import Path + +import pytest + +import litellm +from litellm.utils import supports_prompt_caching, supports_reasoning + +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +GLM_5_2_MODELS = ("mistral/zai-glm-5-2", "mistral/glm-5-2") + +INPUT_COST = 1.4e-06 +CACHED_INPUT_COST = 1.4e-07 +OUTPUT_COST = 4.4e-06 + + +def _load(path): + with open(path) as f: + return json.load(f) + + +@pytest.fixture +def local_model_cost_map(monkeypatch): + """Force get_model_info to resolve against the in-repo cost map instead of the + remote one fetched at import time, which still carries the pre-merge pricing.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +@pytest.mark.parametrize("model", GLM_5_2_MODELS) +def test_zai_glm_5_2_capabilities_are_visible_to_callers(local_model_cost_map, model): + """Mistral advertises reasoning and prompt caching on this model, so the helpers + every caller checks before sending a request must say so too.""" + assert supports_reasoning(model=model) is True + assert supports_prompt_caching(model=model) is True + + assert litellm.get_model_info(model=model) + + +@pytest.mark.parametrize("model", GLM_5_2_MODELS) +def test_backup_matches_main(model): + """Ensure the bundled (backup) cost map stays in sync with the canonical file.""" + main_cost = _load(MAIN_PATH) + backup_cost = _load(BACKUP_PATH) + + assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps" diff --git a/tests/test_litellm/test_muse_spark_1_1_model_metadata.py b/tests/test_litellm/test_muse_spark_1_1_model_metadata.py new file mode 100644 index 00000000000..f55266a78d7 --- /dev/null +++ b/tests/test_litellm/test_muse_spark_1_1_model_metadata.py @@ -0,0 +1,29 @@ +import json +from pathlib import Path + +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + +MUSE_SPARK_MODEL = "meta/muse-spark-1.1" + + +def test_muse_spark_1_1_model_info(): + routed_model, provider, _, api_base = get_llm_provider(model=MUSE_SPARK_MODEL, api_key="sk-test") + assert routed_model == "muse-spark-1.1" + assert provider == "meta" + assert api_base == "https://api.meta.ai/v1" + + +def test_muse_spark_1_1_backup_matches_main(): + """Ensure the bundled model cost map stays in sync with the canonical file.""" + repo_root = Path(__file__).parents[2] + main_path = repo_root / "model_prices_and_context_window.json" + backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json" + + with open(main_path) as f: + main_cost = json.load(f) + with open(backup_path) as f: + backup_cost = json.load(f) + + assert backup_cost.get(MUSE_SPARK_MODEL) == main_cost.get(MUSE_SPARK_MODEL), ( + f"{MUSE_SPARK_MODEL} differs between main and backup model cost maps" + ) diff --git a/tests/test_litellm/test_muse_spark_1_2_model_metadata.py b/tests/test_litellm/test_muse_spark_1_2_model_metadata.py index 4cc69d9f66d..877fef456de 100644 --- a/tests/test_litellm/test_muse_spark_1_2_model_metadata.py +++ b/tests/test_litellm/test_muse_spark_1_2_model_metadata.py @@ -29,6 +29,15 @@ def test_muse_spark_1_2_routes_to_meta_model_api(model: str): assert api_base == "https://api.meta.ai/v1" +@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR)) +def test_muse_spark_1_2_backup_matches_main(model: str): + """Ensure the bundled model cost map stays in sync with the canonical file.""" + main_cost = _load_cost_map() + backup_cost = _load_cost_map("litellm/model_prices_and_context_window_backup.json") + + assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps" + + def test_muse_spark_contributor_tier_is_cheaper_than_standard(): cost_map = _load_cost_map() standard = cost_map[MUSE_SPARK_STANDARD] diff --git a/tests/test_litellm/test_muse_spark_1_3_model_metadata.py b/tests/test_litellm/test_muse_spark_1_3_model_metadata.py index 097dbc2923d..d98afa12a6e 100644 --- a/tests/test_litellm/test_muse_spark_1_3_model_metadata.py +++ b/tests/test_litellm/test_muse_spark_1_3_model_metadata.py @@ -38,6 +38,15 @@ def test_muse_spark_1_3_web_search_cost_per_query(local_model_cost_map, model: s assert StandardBuiltInToolCostTracking.get_cost_for_web_search(model_info=info) == WEB_SEARCH_COST_PER_QUERY +@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR)) +def test_muse_spark_1_3_backup_matches_main(model: str): + """Ensure the bundled model cost map stays in sync with the canonical file.""" + main_cost = _load_cost_map() + backup_cost = _load_cost_map("litellm/model_prices_and_context_window_backup.json") + + assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps" + + def test_muse_spark_contributor_tier_is_cheaper_than_standard(): cost_map = _load_cost_map() standard = cost_map[MUSE_SPARK_STANDARD] diff --git a/tests/test_litellm/test_replicate_model_key_format.py b/tests/test_litellm/test_replicate_model_key_format.py index 39e9fa01d73..77ae5e1b069 100644 --- a/tests/test_litellm/test_replicate_model_key_format.py +++ b/tests/test_litellm/test_replicate_model_key_format.py @@ -18,3 +18,18 @@ def test_replicate_models_have_valid_key_prefix(model_cost: dict[str, Any]) -> N assert not malformed, ( f"Replicate models must use 'replicate/owner/model' key format, found malformed keys: {malformed}" ) + + +def test_replicate_backup_matches_main() -> None: + repo_root = Path(__file__).parents[2] + main_path = repo_root / "model_prices_and_context_window.json" + backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json" + + with open(main_path, encoding="utf-8") as f: + main_cost: dict[str, Any] = json.load(f) + with open(backup_path, encoding="utf-8") as f: + backup_cost: dict[str, Any] = json.load(f) + + for key in main_cost: + if main_cost[key].get("litellm_provider") == "replicate": + assert backup_cost.get(key) == main_cost.get(key), f"{key} differs between main and backup model cost maps" diff --git a/tests/test_litellm/test_sambanova_model_metadata.py b/tests/test_litellm/test_sambanova_model_metadata.py new file mode 100644 index 00000000000..20f34f9f3cc --- /dev/null +++ b/tests/test_litellm/test_sambanova_model_metadata.py @@ -0,0 +1,25 @@ +import json +from pathlib import Path + +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + +def test_sambanova_minimax_m27_model_info(): + model = "sambanova/MiniMax-M2.7" + json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" + with open(json_path) as f: + model_cost = json.load(f) + + info = model_cost.get(model) + assert info is not None, f"{model} not found in model_prices_and_context_window.json" + assert info["litellm_provider"] == "sambanova" + assert info["mode"] == "chat" + assert info["input_cost_per_token"] > 0 + assert info["output_cost_per_token"] > 0 + assert info["supports_function_calling"] is True + assert info["supports_reasoning"] is True + assert info["supports_tool_choice"] is True + + routed_model, provider, _, _ = get_llm_provider(model=model) + assert routed_model == "MiniMax-M2.7" + assert provider == "sambanova" diff --git a/tests/test_litellm/test_together_ai_model_metadata.py b/tests/test_litellm/test_together_ai_model_metadata.py index 851d0f0c9d8..88d6db0d8b0 100644 --- a/tests/test_litellm/test_together_ai_model_metadata.py +++ b/tests/test_litellm/test_together_ai_model_metadata.py @@ -108,6 +108,14 @@ def test_together_successor_metadata_points_at_live_models(cost_map: CostMap): assert "deprecation_date" not in target, f"{model} names deprecated successor {successor}" +def test_together_backup_cost_map_in_sync(cost_map: CostMap): + with open(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json") as f: + backup = COST_MAP_ADAPTER.validate_python(json.load(f)) + together_main = {k: v for k, v in cost_map.items() if k.startswith("together_ai/")} + together_backup = {k: v for k, v in backup.items() if k.startswith("together_ai/")} + assert together_backup == together_main + + CACHED_INPUT_MODELS: Final = ( "together_ai/moonshotai/Kimi-K3", "together_ai/zai-org/GLM-5.2", diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index bad531fdd51..3a376ac9397 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -3641,9 +3641,6 @@ def _assert_fireworks_entry( assert info["input_cost_per_token"] > 0 assert info["output_cost_per_token"] > 0 assert "cache_read_input_token_cost" in info - assert info["max_input_tokens"] == expected_max_input - assert info["max_output_tokens"] == expected_max_output - assert info["max_tokens"] == expected_max_output assert info["supports_function_calling"] is True assert info["supports_tool_choice"] is True assert info["supports_reasoning"] is expected_reasoning @@ -5657,3 +5654,253 @@ def test_completion_finishes_response_metadata_before_handing_the_response_to_th assert snapshot["litellm_call_id"] assert snapshot["response_cost"] is not None assert snapshot["api_base"] + + +def test_fireworks_models_in_backup_cost_map(): + import json + from pathlib import Path + + json_path = Path(__file__).parents[2] / "litellm" / "model_prices_and_context_window_backup.json" + with open(json_path) as f: + model_cost = json.load(f) + + for entry in _FIREWORKS_MODELS: + _assert_fireworks_entry(model_cost, *entry) + + for short in _FIREWORKS_SHORT_FORMS: + long_key = f"fireworks_ai/accounts/fireworks/models/{short}" + short_key = f"fireworks_ai/{short}" + assert model_cost.get(short_key) == model_cost.get(long_key), ( + f"short-form {short_key} does not match long-form {long_key}" + ) + + for short in _FIREWORKS_ROUTER_SHORT_FORMS: + long_key = f"fireworks_ai/accounts/fireworks/routers/{short}" + short_key = f"fireworks_ai/{short}" + assert model_cost.get(short_key) == model_cost.get(long_key), ( + f"short-form {short_key} does not match long-form {long_key}" + ) + + +def test_fireworks_models_in_cost_map(): + import json + from pathlib import Path + + json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" + with open(json_path) as f: + model_cost = json.load(f) + + for entry in _FIREWORKS_MODELS: + _assert_fireworks_entry(model_cost, *entry) + + for short in _FIREWORKS_SHORT_FORMS: + long_key = f"fireworks_ai/accounts/fireworks/models/{short}" + short_key = f"fireworks_ai/{short}" + assert model_cost.get(short_key) == model_cost.get(long_key), ( + f"short-form {short_key} does not match long-form {long_key}" + ) + + for short in _FIREWORKS_ROUTER_SHORT_FORMS: + long_key = f"fireworks_ai/accounts/fireworks/routers/{short}" + short_key = f"fireworks_ai/{short}" + assert model_cost.get(short_key) == model_cost.get(long_key), ( + f"short-form {short_key} does not match long-form {long_key}" + ) + + +def test_fireworks_short_model_names_resolve_to_long_cost_map_keys(fireworks_short_model_cost_map: None) -> None: + model_info = litellm.get_model_info("fireworks_ai/glm-5p3") + assert model_info["key"] == "fireworks_ai/accounts/fireworks/models/glm-5p3" + + model_info = litellm.get_model_info("glm-5p3", custom_llm_provider="fireworks_ai") + assert model_info["key"] == "fireworks_ai/accounts/fireworks/models/glm-5p3" + + model_info = litellm.get_model_info("fireworks_ai/glm-5p3-fast") + assert model_info["key"] == "fireworks_ai/accounts/fireworks/routers/glm-5p3-fast" + + model_info = litellm.get_model_info("fireworks_ai/nomic-ai/nomic-embed-text-v1.5") + assert model_info["key"] == "fireworks_ai/nomic-ai/nomic-embed-text-v1.5" + + with pytest.raises(Exception, match="isn't mapped"): + litellm.get_model_info("fireworks_ai/does-not-exist") + + +def test_get_model_info_bedrock_regional_profile_without_entry_falls_back_to_base(local_model_cost_map): + """A regional profile with no dedicated cost-map entry must still resolve to its + region-stripped base entry.""" + info = litellm.get_model_info(model="bedrock/apac.anthropic.claude-opus-4-8") + assert info["key"] == "anthropic.claude-opus-4-8" + + +def test_get_model_info_gemini(monkeypatch): + """ + Tests if ALL gemini models have 'tpm' and 'rpm' in the model info + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + + model_map = litellm.model_cost + for model, info in model_map.items(): + if ( + model.startswith("gemini/") + and "gemma" not in model + and "learnlm" not in model + and "imagen" not in model + and "veo" not in model + and "lyria" not in model + and "robotics" not in model + ): + assert info.get("tpm") is not None, f"{model} does not have tpm" + assert info.get("rpm") is not None, f"{model} does not have rpm" + + +def test_get_model_info_resolves_provider_prefixed_model_ids(local_model_cost_map): + """Perplexity's Agent API third-party models are keyed `perplexity/perplexity/` + because Perplexity's own id already starts with `perplexity/`. Callers run + `get_llm_provider` first, which hands `_get_potential_model_names` model + `perplexity/glm-5.2` with provider `perplexity`, and every candidate but the + provider-prefixed one strips that second `perplexity/` off. Regression: the + entries were unreachable from `supports_reasoning` and from the cost calculator's + per-token fallback, so a mapped model reported no reasoning support and raised + "This model isn't mapped yet" on the only path where its rates are ever used.""" + for model, reasoning in ( + ("perplexity/perplexity/glm-5.2", True), + ("perplexity/perplexity/kimi-k3", True), + ("perplexity/perplexity/deepseek-v4-flash-0731", True), + ("perplexity/perplexity/kimi-k2.7-code", False), + ("perplexity/perplexity/nemotron-3.5-lightning-30b-a3b", True), + ("perplexity/perplexity/nemotron-3-ultra-550b-a55b", True), + ): + assert litellm.supports_reasoning(model=model) is reasoning, model + + via_provider = litellm.get_model_info(model="perplexity/glm-5.2", custom_llm_provider="perplexity") + assert via_provider["key"] == "perplexity/perplexity/glm-5.2" + assert via_provider["mode"] == "responses" + + lightning = litellm.get_model_info( + model="perplexity/nemotron-3.5-lightning-30b-a3b", custom_llm_provider="perplexity" + ) + assert lightning["key"] == "perplexity/perplexity/nemotron-3.5-lightning-30b-a3b" + assert lightning["mode"] == "responses" + + ultra = litellm.get_model_info(model="perplexity/perplexity/nemotron-3-ultra-550b-a55b") + assert ultra["key"] == "perplexity/perplexity/nemotron-3-ultra-550b-a55b" + + +def test_get_model_info_shows_supports_computer_use(monkeypatch): + """ + Tests if 'supports_computer_use' is correctly retrieved by get_model_info. + We'll use 'claude-4-sonnet-20250514' as it's configured + in the backup JSON to have supports_computer_use: True. + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + # Ensure litellm.model_cost is loaded, relying on the backup mechanism if primary fails + # as per previous debugging. + litellm.model_cost = litellm.get_model_cost_map(url="") + + # This model should have 'supports_computer_use': True in the backup JSON + model_known_to_support_computer_use = "claude-4-sonnet-20250514" + info = litellm.get_model_info(model_known_to_support_computer_use) + + # After the fix in utils.py, this should now be present and True + assert info.get("supports_computer_use") is True + + +def test_get_model_info_surfaces_supports_adaptive_thinking(local_model_cost_map): + """supports_adaptive_thinking must flow through get_model_info like every other + capability flag: both from an explicit cost-map entry and from a + fallback-generalization rule for an unmapped model. Regression: the field shipped + in the JSON but was never declared on ModelInfo nor copied during construction, so + get_model_info (and _supports_factory) silently dropped it for any provider-prefixed + or unmapped name.""" + explicit = litellm.get_model_info(model="claude-opus-4-8") + assert explicit["supports_adaptive_thinking"] is True + + generalized = litellm.get_model_info(model="claude-opus-4-9", custom_llm_provider="anthropic") + assert generalized["supports_adaptive_thinking"] is True + + +def test_get_model_info_surfaces_supports_parallel_function_calling(local_model_cost_map): + """A registry entry's supports_parallel_function_calling must read back through get_model_info + and litellm.supports_parallel_function_calling. Regression: the key was never copied into + ModelInfo, so provider-prefixed entries read None / False even when the map said True, and an + explicit False was indistinguishable from unset.""" + declared_true = litellm.get_model_info(model="together_ai/zai-org/GLM-5.3-Flash") + assert declared_true["supports_parallel_function_calling"] is True + assert litellm.supports_parallel_function_calling(model="together_ai/zai-org/GLM-5.3-Flash") is True + + +def test_model_info_for_fireworks_short_form_models(): + """ + Test that fireworks_ai short-form model entries (fireworks_ai/) + are correctly configured in model_prices_and_context_window.json. + + These entries enable cost attribution for models called via short-form + names (e.g., fireworks_ai/glm-4p7 instead of + fireworks_ai/accounts/fireworks/models/glm-4p7). + """ + import json + from pathlib import Path + + json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" + with open(json_path) as f: + model_cost = json.load(f) + + # glm-4p7: short-form and long-form + for key in [ + "fireworks_ai/glm-4p7", + "fireworks_ai/accounts/fireworks/models/glm-4p7", + ]: + info = model_cost.get(key) + assert info is not None, f"{key} not found in model_prices_and_context_window.json" + assert info["litellm_provider"] == "fireworks_ai" + assert info["mode"] == "chat" + assert info["supports_reasoning"] is True + + # minimax-m2p1: short-form and long-form + for key in [ + "fireworks_ai/minimax-m2p1", + "fireworks_ai/accounts/fireworks/models/minimax-m2p1", + ]: + info = model_cost.get(key) + assert info is not None, f"{key} not found in model_prices_and_context_window.json" + assert info["litellm_provider"] == "fireworks_ai" + assert info["mode"] == "chat" + + # kimi-k2p5: short-form only (long-form already existed) + info = model_cost.get("fireworks_ai/kimi-k2p5") + assert info is not None, "fireworks_ai/kimi-k2p5 not found in model_prices_and_context_window.json" + assert info["litellm_provider"] == "fireworks_ai" + assert info["mode"] == "chat" + + +def test_model_info_for_vertex_ai_deepseek_model(): + model_info = litellm.get_model_info(model="vertex_ai/deepseek-ai/deepseek-r1-0528-maas") + assert model_info is not None + assert model_info["litellm_provider"] == "vertex_ai-deepseek_models" + assert model_info["mode"] == "chat" + + assert model_info["input_cost_per_token"] is not None + assert model_info["output_cost_per_token"] is not None + + +def test_provider_prefixed_lookup_never_outranks_an_existing_row(local_model_cost_map): + """The provider-prefixed candidate is tried last, after every candidate that + already existed, so no model that resolves today can change answer. `perplexity/sonar` + is the case that proves it: both `perplexity/sonar` and `perplexity/perplexity/sonar` + are cost-map keys, and the shorter one must keep winning.""" + sonar = litellm.get_model_info(model="sonar", custom_llm_provider="perplexity") + assert sonar["key"] == "perplexity/sonar" + assert sonar["mode"] == "chat" + + still_sonar = litellm.get_model_info(model="perplexity/sonar", custom_llm_provider="perplexity") + assert still_sonar["key"] == "perplexity/sonar" + assert still_sonar["mode"] == "chat" + + for model, provider, expected_key in ( + ("claude-sonnet-4-5", "anthropic", "claude-sonnet-4-5"), + ("anthropic/claude-sonnet-4-5", "anthropic", "claude-sonnet-4-5"), + ("gemini/gemini-2.0-flash", "gemini", "gemini/gemini-2.0-flash"), + ("openrouter/openai/gpt-4o", "openrouter", "openrouter/openai/gpt-4o"), + ): + assert litellm.get_model_info(model=model, custom_llm_provider=provider)["key"] == expected_key diff --git a/tests/test_litellm/test_xai_grok_4_3_model_metadata.py b/tests/test_litellm/test_xai_grok_4_3_model_metadata.py new file mode 100644 index 00000000000..50be24ba63d --- /dev/null +++ b/tests/test_litellm/test_xai_grok_4_3_model_metadata.py @@ -0,0 +1,19 @@ +import json +from pathlib import Path + + +def test_xai_grok_4_3_backup_matches_main(): + """Ensure the bundled model cost map stays in sync with the canonical file.""" + repo_root = Path(__file__).parents[2] + main_path = repo_root / "model_prices_and_context_window.json" + backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json" + + with open(main_path) as f: + main_cost = json.load(f) + with open(backup_path) as f: + backup_cost = json.load(f) + + for model in ("xai/grok-4.3", "xai/grok-4.3-latest"): + assert backup_cost.get(model) == main_cost.get(model), ( + f"{model} differs between main and backup model cost maps" + )