test: keep tests that survive correct cost-map updates

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-09-15 22:16:08 +00:00
parent a428cc8d84
commit 2586b21893
46 changed files with 1771 additions and 116 deletions

View file

@ -2087,95 +2087,6 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
assert round(result, 6) == round(expected, 6)
def test_a_pinned_billing_time_prices_the_totals_and_the_reported_rates_at_one_moment(monkeypatch):
monkeypatch.setitem(
litellm.model_cost,
"off-peak-model",
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"off_peak_pricing": {
"hours_utc": "02:00-03:00",
"input_cost_per_token": 1e-6,
"output_cost_per_token": 5e-6,
},
"litellm_provider": "openai",
"mode": "chat",
},
)
usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
with pinned_billing_time(datetime(2026, 1, 1, 2, 30, tzinfo=timezone.utc)):
off_peak_prompt_cost, off_peak_completion_cost = generic_cost_per_token(
model="off-peak-model", usage=usage, custom_llm_provider="openai"
)
off_peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage)
with pinned_billing_time(datetime(2026, 1, 1, 12, 30, tzinfo=timezone.utc)):
peak_prompt_cost, peak_completion_cost = generic_cost_per_token(
model="off-peak-model", usage=usage, custom_llm_provider="openai"
)
peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage)
assert off_peak_rates.input_cost_per_token == pytest.approx(1e-6)
assert peak_rates.input_cost_per_token == pytest.approx(3e-6)
assert off_peak_prompt_cost == pytest.approx(1000 * off_peak_rates.input_cost_per_token)
assert off_peak_completion_cost == pytest.approx(500 * off_peak_rates.output_cost_per_token)
assert peak_prompt_cost == pytest.approx(1000 * peak_rates.input_cost_per_token)
assert peak_completion_cost == pytest.approx(500 * peak_rates.output_cost_per_token)
def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(_local_model_cost_map, monkeypatch):
from litellm.llms.anthropic.cost_calculation import cost_per_token as anthropic_cost_per_token
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
model = "claude-test-geo-breakdown-model"
litellm.register_model(
model_cost={
model: {
"input_cost_per_token": 5e-6,
"output_cost_per_token": 25e-6,
"cache_creation_input_token_cost": 6.25e-6,
"cache_read_input_token_cost": 0.5e-6,
"litellm_provider": "anthropic",
"max_tokens": 8192,
"provider_specific_entry": {"us": 1.1},
}
}
)
def make_usage() -> Usage:
return Usage(
prompt_tokens=10_000,
completion_tokens=500,
total_tokens=10_500,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=2_000,
cache_creation_tokens=6_000,
),
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200, text_tokens=300),
)
base_usage = make_usage()
geo_usage = make_usage()
geo_usage.inference_geo = "us"
base = get_token_type_cost_breakdown(model=model, custom_llm_provider="anthropic", usage=base_usage)
geo = get_token_type_cost_breakdown(model=model, custom_llm_provider="anthropic", usage=geo_usage)
assert base.cache_read_cost == pytest.approx(2_000 * 0.5e-6)
assert base.cache_creation_cost == pytest.approx(6_000 * 6.25e-6)
assert geo.cache_read_cost == pytest.approx(base.cache_read_cost * 1.1)
assert geo.cache_creation_cost == pytest.approx(base.cache_creation_cost * 1.1)
assert geo.reasoning_cost == pytest.approx(base.reasoning_cost * 1.1)
prompt_cost, completion_cost = anthropic_cost_per_token(model=model, usage=geo_usage)
text_input_cost = 2_000 * 5e-6 * 1.1
text_output_cost = 300 * 25e-6 * 1.1
assert text_input_cost + geo.cache_read_cost + geo.cache_creation_cost == pytest.approx(prompt_cost)
assert text_output_cost + geo.reasoning_cost == pytest.approx(completion_cost)
def test_service_tier_ultrafast_pricing():
"""An ultrafast request bills the *_ultrafast rates for all token types.
@ -3134,6 +3045,46 @@ def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeyp
assert breakdown.reasoning_cost == pytest.approx(200 * rates.output_cost_per_reasoning_token)
def test_a_pinned_billing_time_prices_the_totals_and_the_reported_rates_at_one_moment(monkeypatch):
"""Totals and reported rates resolve off-peak pricing on separate paths that each read the
clock, so a window opening between the two reads used to leave them describing one request
at two different prices. Pinned, both must answer for the pinned moment."""
monkeypatch.setitem(
litellm.model_cost,
"off-peak-model",
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"off_peak_pricing": {
"hours_utc": "02:00-03:00",
"input_cost_per_token": 1e-6,
"output_cost_per_token": 5e-6,
},
"litellm_provider": "openai",
"mode": "chat",
},
)
usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
with pinned_billing_time(datetime(2026, 1, 1, 2, 30, tzinfo=timezone.utc)):
off_peak_prompt_cost, off_peak_completion_cost = generic_cost_per_token(
model="off-peak-model", usage=usage, custom_llm_provider="openai"
)
off_peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage)
with pinned_billing_time(datetime(2026, 1, 1, 12, 30, tzinfo=timezone.utc)):
peak_prompt_cost, peak_completion_cost = generic_cost_per_token(
model="off-peak-model", usage=usage, custom_llm_provider="openai"
)
peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage)
assert off_peak_rates.input_cost_per_token == pytest.approx(1e-6)
assert peak_rates.input_cost_per_token == pytest.approx(3e-6)
assert off_peak_prompt_cost == pytest.approx(1000 * off_peak_rates.input_cost_per_token)
assert off_peak_completion_cost == pytest.approx(500 * off_peak_rates.output_cost_per_token)
assert peak_prompt_cost == pytest.approx(1000 * peak_rates.input_cost_per_token)
assert peak_completion_cost == pytest.approx(500 * peak_rates.output_cost_per_token)
def test_the_token_type_breakdown_carries_the_rates_it_billed_at(monkeypatch):
"""Callers that report both the lines and the rates read the rates off the breakdown rather than
resolving them a second time, so the breakdown has to hand back exactly what it billed at."""
@ -3341,6 +3292,68 @@ def test_token_type_cost_breakdown_applies_vertex_regional_uplift(_local_model_c
assert text_input_cost + regional.cache_read_cost == pytest.approx(prompt_cost)
def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(_local_model_cost_map, monkeypatch):
"""
Anthropic's regional (geo) uplift lives in provider_specific_entry and is
applied to every token type in the totals, so the per-type breakdown must
scale its cache and reasoning line items by it too. Otherwise the logged
cache costs stay at the base rate and the cache uplift is misattributed to
plain input for exactly the cache-heavy regional traffic the uplift targets.
"""
from litellm.llms.anthropic.cost_calculation import (
cost_per_token as anthropic_cost_per_token,
)
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
model = "claude-test-geo-breakdown-model"
litellm.register_model(
model_cost={
model: {
"input_cost_per_token": 5e-6,
"output_cost_per_token": 25e-6,
"cache_creation_input_token_cost": 6.25e-6,
"cache_read_input_token_cost": 0.5e-6,
"litellm_provider": "anthropic",
"max_tokens": 8192,
"provider_specific_entry": {"us": 1.1},
}
}
)
def make_usage() -> Usage:
return Usage(
prompt_tokens=10_000,
completion_tokens=500,
total_tokens=10_500,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=2_000,
cache_creation_tokens=6_000,
),
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200, text_tokens=300),
)
base_usage = make_usage()
geo_usage = make_usage()
geo_usage.inference_geo = "us"
base = get_token_type_cost_breakdown(model=model, custom_llm_provider="anthropic", usage=base_usage)
geo = get_token_type_cost_breakdown(model=model, custom_llm_provider="anthropic", usage=geo_usage)
assert base.cache_read_cost == pytest.approx(2_000 * 0.5e-6)
assert base.cache_creation_cost == pytest.approx(6_000 * 6.25e-6)
assert geo.cache_read_cost == pytest.approx(base.cache_read_cost * 1.1)
assert geo.cache_creation_cost == pytest.approx(base.cache_creation_cost * 1.1)
assert geo.reasoning_cost == pytest.approx(base.reasoning_cost * 1.1)
# The uplifted breakdown must still reconcile with the uplifted totals.
prompt_cost, completion_cost = anthropic_cost_per_token(model=model, usage=geo_usage)
text_input_cost = 2_000 * 5e-6 * 1.1
text_output_cost = 300 * 25e-6 * 1.1
assert text_input_cost + geo.cache_read_cost + geo.cache_creation_cost == pytest.approx(prompt_cost)
assert text_output_cost + geo.reasoning_cost == pytest.approx(completion_cost)
@pytest.mark.parametrize("details_as_dict", [True, False])
def test_image_response_input_image_tokens_priced_at_image_rate(details_as_dict):
"""

View file

@ -134,6 +134,24 @@ def test_finalize_with_no_block_clears_rules():
set_fallback_generalizations(previous)
def test_shipped_backup_carries_the_claude_routing_rules():
"""The bundled backup must ship the Claude routing rules so a fresh install
(or an offline fallback) routes unknown Claude models without code changes.
Bedrock-syntax ids must hit the bedrock rule before the bare-id Anthropic rule."""
backup = GetModelCostMap.load_local_model_cost_map()
rules = backup.get(FALLBACK_GENERALIZATIONS_KEY, {}).get("rules", [])
names = [r.get("name") for r in rules]
assert names.index("bedrock-claude-ids") < names.index("anthropic-claude-ids")
previous = list(get_fallback_generalization_rules())
try:
set_fallback_generalizations(rules)
assert match_routing_generalization("claude-opus-4-9") == "anthropic"
assert match_routing_generalization("global.anthropic.claude-opus-4-9") == "bedrock"
finally:
set_fallback_generalizations(previous)
def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace():
"""Routing rules decide ``litellm_provider`` for otherwise-unknown ids, and the
proxy's wildcard access check (``can_key_call_model`` with a ``bedrock/*`` key)
@ -170,6 +188,43 @@ def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace():
set_fallback_generalizations(previous)
def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
"""Adaptive thinking is data, not code. The bundled backup must carry
supports_adaptive_thinking on genuine Claude >= 4.6 entries (every provider
route) and on the version-gated anthropic-claude-adaptive-thinking rule for
unmapped future Claudes, while leaving the dated Claude 4.0 names
("...-4-20250514") unflagged so a date can never be mistaken for a 4.6+ minor
version. The version-neutral claude-family-baseline capability rule must not flag
it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule
carries only its delta; capability unioning stacks it onto the baseline, so the
baseline block is never duplicated across rules and no rule needs ``extends``."""
backup = GetModelCostMap.load_local_model_cost_map()
rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"]
baseline_rule = next(r for r in rules if r.get("name") == "claude-family-baseline")
adaptive_rule = next(r for r in rules if r.get("name") == "claude-adaptive-thinking")
assert "supports_adaptive_thinking" not in baseline_rule["model_info"]
assert "litellm_provider" not in baseline_rule["model_info"]
assert adaptive_rule["model_info"] == {"supports_adaptive_thinking": True}
assert all("extends" not in r for r in rules)
for adaptive in [
"anthropic.claude-opus-4-8",
"vertex_ai/claude-opus-4-6@default",
"us.anthropic.claude-sonnet-4-6",
"openrouter/anthropic/claude-opus-4.7",
"azure_ai/claude-opus-4-7",
]:
assert backup[adaptive]["supports_adaptive_thinking"] is True, adaptive
for non_adaptive in [
"claude-opus-4-20250514",
"us.anthropic.claude-opus-4-20250514-v1:0",
"claude-opus-4-5",
]:
assert "supports_adaptive_thinking" not in backup[non_adaptive], non_adaptive
# OpenRouter headline rates from GET https://openrouter.ai/api/v1/models.
# These were the catalog values that disagreed with that API (and, for the
# two spotlight models, the public model pages that their source fields cite).

View file

@ -72,6 +72,22 @@ def test_azure_ai_catalog_name_prices_the_same_in_any_casing(catalog_name: str)
assert upper_cost == lowercase_cost
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize("catalog_name", GROK_4_20_NAMES)
def test_azure_ai_grok_4_20_bills_cached_prompt_tokens_at_the_input_price(catalog_name: str) -> None:
uncached_prompt_cost, _ = cost_per_token(
model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0
)
cached_prompt_cost, _ = cost_per_token(
model=f"azure_ai/{catalog_name}",
prompt_tokens=A_MILLION,
completion_tokens=0,
cache_read_input_tokens=A_MILLION,
)
assert uncached_prompt_cost > 0
assert cached_prompt_cost == pytest.approx(uncached_prompt_cost)
@pytest.mark.usefixtures("local_model_cost_map")
def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None:
one_second_cost: Final = _whisper_transcription_cost(1)
@ -80,6 +96,14 @@ def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None:
assert one_hour_cost == pytest.approx(AN_HOUR_IN_SECONDS * one_second_cost)
@pytest.mark.parametrize("catalog_name", CATALOG_NAMES)
def test_azure_ai_catalog_entry_source_and_backup_match(catalog_name: str) -> None:
main_entry = _cost_map_entry(MAIN_COST_MAP, catalog_name)
backup_entry = _cost_map_entry(BACKUP_COST_MAP, catalog_name)
assert backup_entry == main_entry
def test_azure_ai_model_router_spellings_share_one_entry() -> None:
underscore_entry = _cost_map_entry(MAIN_COST_MAP, "model_router")
hyphen_entry = _cost_map_entry(MAIN_COST_MAP, "model-router")

View file

@ -0,0 +1,28 @@
from pathlib import Path
import pytest
import litellm
from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo
REPO_ROOT = Path(__file__).parents[5]
COST_MAPS = [
REPO_ROOT / "model_prices_and_context_window.json",
REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json",
]
MODELS = [("cohere/parse-v5.0", "cohere"), ("azure_ai/Cohere-parse-v5", "azure_ai")]
def _ocr_response(model: str, pages_processed: int) -> OCRResponse:
return OCRResponse(
pages=[OCRPage(index=i, markdown=f"page {i}") for i in range(pages_processed)],
model=model,
usage_info=OCRUsageInfo(pages_processed=pages_processed),
)
@pytest.mark.parametrize("model, provider", MODELS)
def test_model_info_resolves_ocr_mode_and_price(local_model_cost_map, model: str, provider: str) -> None:
info = litellm.get_model_info(model=model, custom_llm_provider=provider)
assert info["mode"] == "ocr"

View file

@ -1,3 +1,4 @@
import json
from decimal import Decimal
from pathlib import Path
from typing import Final
@ -162,6 +163,29 @@ def test_legacy_endpoint_names_still_resolve(local_model_cost_map: None) -> None
assert completion_cost == pytest.approx(100 * info["output_cost_per_token"])
@pytest.mark.parametrize("model", NEW_MODELS)
def test_new_models_carry_cache_pricing(local_model_cost_map: None, model: str) -> None:
info: Final = _model_info(model)
assert info["input_cost_per_token"] > 0
assert info["output_cost_per_token"] > 0
assert info["cache_creation_input_token_cost"] > info["input_cost_per_token"]
assert info["cache_read_input_token_cost"] < info["input_cost_per_token"]
assert info["supports_prompt_caching"] is True
def test_every_priced_databricks_model_declares_cache_rates(local_model_cost_map: None) -> None:
undeclared: Final = [
model
for model, info in litellm.model_cost.items()
if model.startswith("databricks/")
and info.get("input_cost_per_token") is not None
and any(info.get(field) is None for field in CACHE_FIELDS)
]
assert undeclared == []
def test_models_without_a_cache_discount_bill_cache_tokens_at_the_input_rate(
local_model_cost_map: None,
) -> None:
@ -180,6 +204,33 @@ def test_models_without_a_cache_discount_bill_cache_tokens_at_the_input_rate(
assert prompt_cost > 8000 * info["input_cost_per_token"]
def test_every_model_without_published_cache_dbu_bills_cache_at_its_own_input_rate(
local_model_cost_map: None,
) -> None:
without_published_rates: Final = [
model
for model, info in litellm.model_cost.items()
if model.startswith("databricks/")
and info.get("input_cost_per_token")
and model not in PUBLISHED_DBU_PER_MILLION
]
for model in without_published_rates:
info = _model_info(model)
for field in CACHE_FIELDS:
assert info[field] == pytest.approx(info["input_cost_per_token"]), (model, field)
@pytest.mark.parametrize("model", NEW_MODELS)
def test_backup_price_map_matches_main(model: str) -> None:
main_cost: Final = json.loads(MAIN_PRICES.read_text())
backup_cost: Final = json.loads(BACKUP_PRICES.read_text())
assert model in main_cost
assert model in backup_cost
assert backup_cost[model] == main_cost[model]
def test_sonnet_5_ships_standard_rates_not_introductory(local_model_cost_map: None) -> None:
sonnet_5: Final = _model_info("databricks/databricks-claude-sonnet-5")
sonnet_4_6: Final = _model_info("databricks/databricks-claude-sonnet-4-6")

View file

@ -22,6 +22,16 @@ def _usage(prompt_tokens: int, cached_tokens: int, completion_tokens: int) -> Us
)
def test_warm_call_cheaper_than_cold_call():
prompt_tokens = 7036
completion_tokens = 8
cold_prompt_cost, _ = cost_per_token(model=MODEL, usage=_usage(prompt_tokens, 16, completion_tokens))
warm_prompt_cost, _ = cost_per_token(model=MODEL, usage=_usage(prompt_tokens, 7020, completion_tokens))
assert warm_prompt_cost < cold_prompt_cost
OFF_PEAK_MODEL = "accounts/fireworks/models/off-peak-test"
OFF_PEAK_WINDOW = "14:00-00:00"
INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc)

View file

@ -0,0 +1,54 @@
"""
Regression test for Fireworks Kimi K2.5 / K2.6 / K2.7 context and output limits.
Fireworks publishes a 262144-token context window for every Kimi K2.5, K2.6 and
K2.7 model, but caps generation well below that. A previous bulk edit had flattened
max_output_tokens/max_tokens to 262144 (equal to the context window), which let the
pre-call context-window check admit requests asking for a full 262144-token
completion that Fireworks then rejects. These assertions pin the corrected per-alias
limits so a future bulk edit can't silently flatten them again.
"""
import json
from importlib.resources import files
import pytest
CONTEXT_WINDOW = 262144
OUTPUT_LIMIT = 32768
KIMI_ALIASES = (
"fireworks_ai/kimi-k2p5",
"fireworks_ai/kimi-k2p6",
"fireworks_ai/kimi-k2p6-fast",
"fireworks_ai/kimi-k2p7-code",
"fireworks_ai/kimi-k2p7-code-fast",
"fireworks_ai/accounts/fireworks/models/kimi-k2p5",
"fireworks_ai/accounts/fireworks/models/kimi-k2p6",
"fireworks_ai/accounts/fireworks/models/kimi-k2p7-code",
"fireworks_ai/accounts/fireworks/routers/kimi-k2p6-fast",
"fireworks_ai/accounts/fireworks/routers/kimi-k2p7-code-fast",
)
@pytest.fixture(scope="module")
def use_local_model_cost_map():
monkeypatch = pytest.MonkeyPatch()
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
import litellm
from litellm.utils import _invalidate_model_cost_lowercase_map
original_model_cost = litellm.model_cost
litellm.model_cost = json.loads(
files("litellm").joinpath("model_prices_and_context_window_backup.json").read_text(encoding="utf-8")
)
litellm.get_model_info.cache_clear()
_invalidate_model_cost_lowercase_map()
try:
yield litellm
finally:
litellm.model_cost = original_model_cost
litellm.get_model_info.cache_clear()
_invalidate_model_cost_lowercase_map()
monkeypatch.undo()

View file

@ -0,0 +1,40 @@
"""
Cost tests for Mistral OCR models against the real litellm cost map
(no monkeypatching of get_model_info). These regress the pricing entries
for mistral-ocr-4-0 and mistral-ocr-latest, which now both resolve to
OCR 4 at $4 / 1000 pages.
"""
from pathlib import Path
from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo
OCR4_COST_PER_PAGE = 0.004
OCR4_ANNOTATION_COST_PER_PAGE = 0.005
REPO_ROOT = Path(__file__).parents[5]
MAIN_COST_MAP = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_COST_MAP = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
OCR3_MODEL = "mistral/mistral-ocr-2512"
OCR3_COST_PER_PAGE = 0.002
OCR3_ANNOTATION_COST_PER_PAGE = 0.003
AZURE_DOC_AI_MODEL = "azure_ai/mistral-document-ai-2512"
AZURE_DOC_AI_COST_PER_PAGE = 0.003
def _ocr_response(model: str, pages_processed: int) -> OCRResponse:
return OCRResponse(
pages=[OCRPage(index=i, markdown=f"page {i}") for i in range(pages_processed)],
model=model,
usage_info=OCRUsageInfo(pages_processed=pages_processed),
)
def _annotated_ocr_response(model: str, pages_processed: int | None, annotation_pages: int) -> OCRResponse:
return OCRResponse(
pages=[],
model=model,
usage_info=OCRUsageInfo(pages_processed=pages_processed, pages_processed_annotation=annotation_pages),
)

View file

@ -154,6 +154,34 @@ class TestSCXAIModelMetadata:
with open(json_path) as f:
return json.load(f)
def test_scx_ai_models_registered_with_correct_metadata(self):
model_cost = self._load(("model_prices_and_context_window.json",))
for model in self.SCX_MODELS:
info = model_cost.get(model)
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
assert info["litellm_provider"] == "scx-ai"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] > 0
assert info["output_cost_per_token"] > 0
assert info["supports_function_calling"] is True
assert info["supports_tool_choice"] is True
assert info["supports_reasoning"] is True
assert info["supports_response_schema"] is True
assert info.get("supports_vision", False) is (model in self.VISION_MODELS)
assert info["supports_prompt_caching"] is True
assert 0 < info["cache_read_input_token_cost"] < info["input_cost_per_token"]
assert info["max_tokens"] == info["max_output_tokens"]
assert info["max_input_tokens"] >= 1_000_000
def test_scx_ai_models_synced_to_backup(self):
model_cost = self._load(("model_prices_and_context_window.json",))
backup = self._load(("litellm", "model_prices_and_context_window_backup.json"))
for model in self.SCX_MODELS:
assert model in backup, f"{model} missing from backup json"
assert backup[model] == model_cost[model], f"{model} differs between root and backup json"
class TestSCXAIDashboardRegistration:
@staticmethod

View file

@ -17,6 +17,7 @@ from litellm import ModelResponse
from litellm.cost_calculator import cost_per_token
from litellm.llms.perplexity.chat.transformation import PerplexityChatConfig
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
from litellm.utils import get_model_info
class TestPerplexityIntegration:
@ -55,6 +56,23 @@ class TestPerplexityIntegration:
}
}
def test_model_info_includes_custom_fields(self):
"""Test that get_model_info returns the custom Perplexity cost fields."""
model_info = get_model_info(model="sonar-deep-research", custom_llm_provider="perplexity")
# Verify custom fields are included
required_fields = [
"citation_cost_per_token",
"search_context_cost_per_query",
"input_cost_per_token",
"output_cost_per_token",
"output_cost_per_reasoning_token",
]
for field in required_fields:
assert field in model_info, f"Missing field: {field}"
assert model_info[field] is not None, f"Null value for field: {field}"
def test_various_citation_sizes(self):
"""Test cost calculation with various citation sizes."""
config = PerplexityChatConfig()

View file

@ -0,0 +1 @@

View file

@ -452,6 +452,44 @@ def test_vertex_claude_completion_does_not_mutate_shared_extra_headers():
assert shared_extra_headers == {}, "extra_headers must not be mutated by completion()"
def test_messages_thinking_shape_follows_exact_vertex_entry_flag(local_model_cost_map, monkeypatch):
"""The Vertex messages config must probe capabilities under ``vertex_ai`` so an
operator setting ``supports_adaptive_thinking: false`` on the exact
``vertex_ai/claude-opus-4-8`` entry beats the unmodified ``anthropic`` entry.
With the inherited ``"anthropic"`` provider default the flip was ignored and
the transform kept emitting ``thinking.type='adaptive'``."""
import litellm
config = VertexAIPartnerModelsAnthropicMessagesConfig()
def transform():
return config.transform_anthropic_messages_request(
model="claude-opus-4-8",
messages=[{"role": "user", "content": "Hello"}],
anthropic_messages_optional_request_params={
"max_tokens": 4096,
"reasoning_effort": "medium",
},
litellm_params=GenericLiteLLMParams(),
headers={},
)
result = transform()
assert result.get("thinking") == {"type": "adaptive", "display": "summarized"}
assert result.get("output_config") == {"effort": "medium"}
monkeypatch.setitem(litellm.model_cost["vertex_ai/claude-opus-4-8"], "supports_adaptive_thinking", False)
litellm.get_model_info.cache_clear()
assert litellm.model_cost["claude-opus-4-8"]["supports_adaptive_thinking"] is True
flipped = transform()
thinking = flipped.get("thinking")
assert isinstance(thinking, dict)
assert thinking.get("type") == "enabled"
assert isinstance(thinking.get("budget_tokens"), int)
assert "output_config" not in flipped
def _vertex_transform(model, messages, system=None):
config = VertexAIPartnerModelsAnthropicMessagesConfig()
params = {"max_tokens": 256}
@ -547,6 +585,10 @@ class TestVertexAnthropicMidConversationSystem:
def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag():
"""Exact cost-map hits win over the ``claude-mid-conversation-system``
fallback rule, so a ``vertex_ai`` Claude 4.8+/5 entry missing the flag would
be treated as unsupported and hoist every reminder, collapsing the prompt
cache. Every mapped vertex_ai entry the rule matches must carry the flag."""
import re
import litellm

View file

@ -0,0 +1,55 @@
"""
Registry regression tests for xAI entries in the model cost map.
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).parents[4]
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
# https://docs.x.ai/developers/model-capabilities/text/multi-agent
# "The multi-agent model does not work with the OpenAI Chat Completions API."
RESPONSES_ONLY_MODELS = (
"xai/grok-4.20-multi-agent-0309",
"xai/grok-4.20-multi-agent-beta-0309",
)
MAP_PATHS = (PRICES_PATH, BACKUP_PRICES_PATH)
@pytest.fixture(scope="module", params=[p.name for p in MAP_PATHS])
def cost_map(request: pytest.FixtureRequest) -> dict:
path = next(p for p in MAP_PATHS if p.name == request.param)
return json.loads(path.read_text(encoding="utf-8"))
@pytest.mark.parametrize("model", RESPONSES_ONLY_MODELS)
def test_multi_agent_models_are_responses_only(cost_map: dict, model: str):
entry = cost_map[model]
assert entry["supported_endpoints"] == ["/v1/responses"]
assert entry["mode"] == "responses"
def test_surviving_xai_chat_models_still_serve_chat_completions(cost_map: dict):
"""Guard against the removal above over-reaching into live models."""
chat_models = [
key
for key, value in cost_map.items()
if isinstance(value, dict) and value.get("litellm_provider") == "xai" and value.get("mode") == "chat"
]
assert "xai/grok-4.3" in chat_models
assert "xai/grok-4.6" in chat_models
def test_both_cost_maps_agree_on_xai_entries():
prices = json.loads(PRICES_PATH.read_text(encoding="utf-8"))
backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8"))
xai_keys = {k for k, v in prices.items() if isinstance(v, dict) and v.get("litellm_provider") == "xai"}
assert xai_keys
assert {k: prices[k] for k in xai_keys} == {k: backup[k] for k in xai_keys}

View file

@ -98,3 +98,23 @@ def test_redirected_slug_carries_the_target_tier_rates(cost_map: dict, slug: str
for field in TIER_COST_FIELDS:
assert entry[field] == target[field], field
assert {k for k in entry if "_above_" in k} == {k for k in target if "_above_" in k}
def test_both_cost_maps_agree_on_the_redirected_slugs():
prices = json.loads(PRICES_PATH.read_text(encoding="utf-8"))
backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8"))
for slug in (*REDIRECTED_SLUGS, *CODE_SLUGS, REDIRECT_TARGET, CODE_REDIRECT_TARGET):
assert prices[slug] == backup[slug], slug
def test_every_retired_chat_slug_is_covered(cost_map: dict):
"""The lists above must stay in step with what the registry marks retired."""
marked = {
key
for key, entry in cost_map.items()
if isinstance(entry, dict)
and entry.get("litellm_provider") == "xai"
and "deprecation_date" in entry
and entry.get("mode") == "chat"
}
assert marked == {*REDIRECTED_SLUGS, *CODE_SLUGS}

View file

@ -327,6 +327,29 @@ def test_openai_style_cache_write_tokens_are_netted_out():
)
def test_sub_input_cache_write_price_is_an_extra_saving():
"""A few models price writes below input; there the premium is a real credit.
Clamping the premium at zero would silently undercount these, so the subtraction
stays signed. ``azure/eu/gpt-4o-2024-11-20`` ships a write price at ~0.5x input.
"""
model = "azure/eu/gpt-4o-2024-11-20"
info = litellm.get_model_info(model=model)
input_cost = info["input_cost_per_token"]
cheap_write = info["cache_creation_input_token_cost"]
assert 0 < cheap_write < input_cost, "fixture drifted: this test needs a model pricing cache writes below input"
result = compute_savings_spend(
model=model,
custom_llm_provider=None,
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=1000, written=4000),
)
assert result.prompt_caching == pytest.approx(4000 * (input_cost - cheap_write))
assert result.prompt_caching > 0
def test_negative_cache_write_count_clamps_to_zero():
"""A malformed negative write count must not be read as a saving."""
input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5")

View file

@ -0,0 +1,37 @@
import json
from pathlib import Path
import pytest
import litellm
from litellm import get_model_info
AZURE_AI_GROK_4_3_MODEL = "azure_ai/grok-4.3"
AZURE_AI_GROK_4_3_SOURCE = "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-grok-4-3-on-microsoft-foundry-latest-generation-agentic-capabilities/4517096"
def _load_model_cost(path: Path) -> dict:
with open(path) as f:
return json.load(f)
@pytest.fixture(autouse=True)
def reload_model_costs():
original_model_cost = litellm.model_cost
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
litellm.model_cost = _load_model_cost(json_path)
get_model_info.cache_clear()
yield
litellm.model_cost = original_model_cost
get_model_info.cache_clear()
def test_azure_ai_grok_4_3_backup_matches_main():
repo_root = Path(__file__).parents[2]
main_path = repo_root / "model_prices_and_context_window.json"
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
main_cost = _load_model_cost(main_path)
backup_cost = _load_model_cost(backup_path)
assert backup_cost.get(AZURE_AI_GROK_4_3_MODEL) == main_cost.get(AZURE_AI_GROK_4_3_MODEL)

View file

@ -0,0 +1,44 @@
from pathlib import Path
from typing import Final
import pytest
from pydantic import TypeAdapter
from litellm import cost_per_token, get_model_info
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
REPO_ROOT: Final = Path(__file__).parents[2]
MODEL: Final = "azure_ai/grok-4.6"
COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
def _cost_map_entry(path: Path) -> dict[str, object]:
return COST_MAP_ADAPTER.validate_json(path.read_bytes())[MODEL]
@pytest.mark.usefixtures("local_model_cost_map")
def test_azure_ai_grok_4_6_is_priced_and_routed() -> None:
routed_model, provider, _, _ = get_llm_provider(model=MODEL)
assert (routed_model, provider) == ("grok-4.6", "azure_ai")
info = get_model_info(model=routed_model, custom_llm_provider=provider)
assert info["litellm_provider"] == "azure_ai"
assert info["mode"] == "chat"
assert info["supports_function_calling"] is True
assert info["supports_prompt_caching"] is True
assert info["supports_reasoning"] is True
assert info["supports_response_schema"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
assert info["supports_web_search"] is True
prompt_cost, completion_cost = cost_per_token(model=MODEL, prompt_tokens=1_000_000, completion_tokens=1_000_000)
assert prompt_cost > 0
assert completion_cost > 0
def test_azure_ai_grok_4_6_entry_source_and_backup_match() -> None:
main_entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json")
backup_entry = _cost_map_entry(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json")
assert backup_entry == main_entry

View file

@ -1,10 +1,31 @@
import json
from pathlib import Path
import pytest
import litellm
from litellm.utils import supports_function_calling, supports_prompt_caching
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
MODEL = "baseten/zai-org/GLM-5.3"
INPUT_COST = 1.4e-06
CACHED_INPUT_COST = 1.4e-07
OUTPUT_COST = 4.4e-06
def _load(path):
with open(path) as f:
return json.load(f)
@pytest.fixture
def local_model_cost_map(monkeypatch):
"""Force get_model_info to resolve against the in-repo cost map instead of the
remote one fetched at import time, which still carries the pre-merge registry."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
@ -12,6 +33,32 @@ def local_model_cost_map(monkeypatch):
litellm.get_model_info.cache_clear()
def test_baseten_glm_5_3_capabilities_are_visible_to_callers(local_model_cost_map):
"""The entry advertises prompt caching and tool calling, so the helpers every
caller checks before sending a request must say so too."""
assert supports_prompt_caching(model=MODEL) is True
assert supports_function_calling(model=MODEL) is True
info = litellm.get_model_info(model="zai-org/GLM-5.3", custom_llm_provider="baseten")
assert info["max_input_tokens"] > 0
assert info["max_output_tokens"] > 0
def test_backup_matches_main():
"""Ensure the bundled (backup) cost map stays in sync with the canonical file.
Both keys are asserted present first: comparing two ``.get`` results alone passes
just as happily when neither file has the entry at all, which is the exact state
this test exists to catch.
"""
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert MODEL in main_cost, f"{MODEL} missing from model_prices_and_context_window.json"
assert MODEL in backup_cost, f"{MODEL} missing from model_prices_and_context_window_backup.json"
assert backup_cost[MODEL] == main_cost[MODEL], f"{MODEL} differs between main and backup model cost maps"
def test_entry_advertises_only_what_the_baseten_path_accepts(local_model_cost_map):
"""The Baseten path rejects unsupported request parameters."""
supported = litellm.get_supported_openai_params(model="zai-org/GLM-5.3", custom_llm_provider="baseten")

View file

@ -0,0 +1,52 @@
import json
from pathlib import Path
import pytest
import litellm
from litellm.constants import bedrock_embedding_models
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
BASE_MODEL = "twelvelabs.marengo-embed-3-0-v1:0"
PROFILE_MODELS = ("us.twelvelabs.marengo-embed-3-0-v1:0", "eu.twelvelabs.marengo-embed-3-0-v1:0")
ALL_MODELS = (BASE_MODEL, *PROFILE_MODELS)
MARENGO_2_7_MODELS = (
"twelvelabs.marengo-embed-2-7-v1:0",
"us.twelvelabs.marengo-embed-2-7-v1:0",
"eu.twelvelabs.marengo-embed-2-7-v1:0",
)
PER_REQUEST_MODELS = (*ALL_MODELS, *MARENGO_2_7_MODELS)
TEXT_REQUEST_COST = 7e-05
IMAGE_REQUEST_COST = 0.0001
VIDEO_COST_PER_SECOND = 0.0007
AUDIO_COST_PER_SECOND = 0.00014
def _load(path):
with open(path) as f:
return json.load(f)
@pytest.mark.parametrize("model", ALL_MODELS)
def test_marengo_embed_3_is_visible_to_callers(model, local_model_cost_map):
info = litellm.get_model_info(model=model, custom_llm_provider="bedrock")
assert info["mode"] == "embedding"
assert info["output_vector_size"] == 512
def test_marengo_embed_3_is_a_known_bedrock_embedding_model():
assert BASE_MODEL in bedrock_embedding_models
@pytest.mark.parametrize("model", PER_REQUEST_MODELS)
def test_backup_matches_main(model):
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert model in main_cost, f"{model} missing from model_prices_and_context_window.json"
assert model in backup_cost, f"{model} missing from model_prices_and_context_window_backup.json"
assert backup_cost[model] == main_cost[model], f"{model} differs between main and backup model cost maps"

View file

@ -31,6 +31,18 @@ def model_data():
return json.load(f)
def test_usgov_east_haiku_profile_mirrors_in_region_row(model_data):
"""us-gov-east-1 serves claude-3-haiku through the us-gov. inference profile
only, so the profile row must bill exactly like the in-region gov row.
"""
profile = model_data["us-gov.anthropic.claude-3-haiku-20240307-v1:0"]
in_region = model_data["bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0"]
assert profile["litellm_provider"] == "bedrock_converse"
assert {k: v for k, v in profile.items() if k != "litellm_provider"} == {
k: v for k, v in in_region.items() if k != "litellm_provider"
}
GOV_ROW_SOURCES = {
"us-gov.anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1",
"bedrock/us-gov-west-1/anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1",

View file

@ -12,7 +12,10 @@ shape, which Fable 5 rejects with a 400.
import json
import os
import pytest
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
@ -23,10 +26,88 @@ def _load_root_cost_map() -> dict:
return json.load(f)
def test_fable_5_present_in_bundled_backup():
"""The bundled backup is the runtime fallback (and what tests load with
``LITELLM_LOCAL_MODEL_COST_MAP=True``) it must carry the same entries as
the root cost map, otherwise the model resolves on one path but not the
other."""
backup = GetModelCostMap.load_local_model_cost_map()
root = _load_root_cost_map()
for model_name in (
"claude-fable-5",
"anthropic.claude-fable-5",
"global.anthropic.claude-fable-5",
"us.anthropic.claude-fable-5",
"eu.anthropic.claude-fable-5",
"vertex_ai/claude-fable-5",
"vertex_ai/claude-fable-5@default",
"azure_ai/claude-fable-5",
):
assert model_name in backup, f"Missing from backup cost map: {model_name}"
assert backup[model_name] == root[model_name], model_name
def test_fable_5_registered_for_bedrock_converse():
assert "anthropic.claude-fable-5" in BEDROCK_CONVERSE_MODELS
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_fable_5_all_variants_carry_adaptive_thinking_flag(cost_map):
"""Every Fable 5 entry must advertise ``supports_adaptive_thinking``.
Adaptive-thinking detection is cost-map driven, so a single variant missing
the flag silently sends the legacy ``thinking.type='enabled'`` shape and the
provider 400s (issue #29188 for the Opus 4.8 equivalent). Fable 5 is even
stricter than Opus 4.8: an explicit ``thinking.type='disabled'`` also 400s,
so adaptive is the only valid thinking shape LiteLLM can emit for it."""
variants = [k for k in cost_map if "claude-fable-5" in k]
assert variants, "no claude-fable-5 entries found in cost map"
missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True]
assert not missing, f"missing supports_adaptive_thinking: {missing}"
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_fable_5_all_variants_carry_thinking_always_on_flag(cost_map):
"""Every Fable 5 entry must advertise ``thinking_always_on``.
The flag drives the Anthropic transformations to omit an explicit
``thinking.type='disabled'``, which Fable 5 rejects with a 400; a variant
missing the flag forwards the param verbatim and the provider 400s."""
variants = [k for k in cost_map if "claude-fable-5" in k]
assert variants, "no claude-fable-5 entries found in cost map"
missing = [k for k in variants if cost_map[k].get("thinking_always_on") is not True]
assert not missing, f"missing thinking_always_on: {missing}"
@pytest.mark.parametrize(
"model",
[
"claude-fable-5",
"anthropic/claude-fable-5",
"anthropic.claude-fable-5",
"bedrock/us.anthropic.claude-fable-5",
"bedrock/invoke/eu.anthropic.claude-fable-5",
"bedrock/global.anthropic.claude-fable-5",
"vertex_ai/claude-fable-5",
"azure_ai/claude-fable-5",
],
)
def test_adaptive_thinking_detected_for_fable_5(local_model_cost_map, model):
"""Provider-routed ids must resolve to a flagged entry so ``reasoning_effort``
maps to ``thinking.type='adaptive'`` + ``output_config.effort``."""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True
FABLE_5_1_VARIANTS = (
"claude-fable-5-1",
"anthropic.claude-fable-5-1",
@ -39,5 +120,53 @@ FABLE_5_1_VARIANTS = (
)
def test_fable_5_1_present_in_bundled_backup():
backup = GetModelCostMap.load_local_model_cost_map()
root = _load_root_cost_map()
for model_name in FABLE_5_1_VARIANTS:
assert model_name in backup, f"Missing from backup cost map: {model_name}"
assert backup[model_name] == root[model_name], model_name
def test_fable_5_1_registered_for_bedrock_converse():
assert "anthropic.claude-fable-5-1" in BEDROCK_CONVERSE_MODELS
@pytest.mark.parametrize(
"model",
[
"claude-fable-5-1",
"anthropic/claude-fable-5-1",
"anthropic.claude-fable-5-1",
"bedrock/us.anthropic.claude-fable-5-1",
"bedrock/invoke/eu.anthropic.claude-fable-5-1",
"bedrock/global.anthropic.claude-fable-5-1",
"vertex_ai/claude-fable-5-1",
"azure_ai/claude-fable-5-1",
],
)
def test_adaptive_thinking_detected_for_fable_5_1(local_model_cost_map, model):
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_sampling_params_flag_on_all_models_that_removed_them(cost_map):
"""Fable 5 and Opus 4.7/4.8 reject ``top_p``/``top_k``/``temperature != 1``;
the drop/raise gating is cost-map driven, so every variant must carry an
explicit ``supports_sampling_params: false``. The perplexity route is
exempt: it is OpenAI-compatible and maps sampling params upstream."""
variants = [
k
for k in cost_map
if any(v in k for v in ("claude-fable-5", "claude-opus-4-7", "claude-opus-4-8"))
and not k.startswith("perplexity/")
]
assert variants, "no matching entries found in cost map"
missing = [k for k in variants if cost_map[k].get("supports_sampling_params") is not False]
assert not missing, f"missing supports_sampling_params=false: {missing}"

View file

@ -0,0 +1,46 @@
"""
Test Claude Haiku 4.5 model configurations for Bedrock
https://github.com/BerriAI/litellm/issues/15818
"""
import json
import os
def test_bedrock_haiku_4_5_matches_sonnet_capabilities():
"""
Test that Haiku 4.5 has same capabilities as Sonnet 4.5
(including computer_use, vision, tools, etc.)
"""
# Load model configuration
json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json")
with open(json_path) as f:
model_data = json.load(f)
haiku_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
sonnet_model = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
haiku_info = model_data[haiku_model]
sonnet_info = model_data[sonnet_model]
# Both should use bedrock_converse
assert haiku_info["litellm_provider"] == "bedrock_converse"
assert sonnet_info["litellm_provider"] == "bedrock_converse"
# Shared capabilities that should match
shared_capabilities = [
"supports_vision",
"supports_computer_use",
"supports_function_calling",
"supports_tool_choice",
"supports_prompt_caching",
"supports_response_schema",
"supports_pdf_input",
"supports_assistant_prefill",
"supports_reasoning",
]
for capability in shared_capabilities:
assert haiku_info.get(capability) == sonnet_info.get(capability), (
f"Capability {capability} mismatch: Haiku={haiku_info.get(capability)}, Sonnet={sonnet_info.get(capability)}"
)

View file

@ -18,6 +18,7 @@ import os
import pytest
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
@ -61,5 +62,31 @@ def test_opus_5_bedrock_rejects_strict_tools(model_name, local_model_cost_map):
assert bedrock_converse_supports_strict_tools(model_name) is False
def test_opus_5_present_in_bundled_backup():
"""The bundled backup is the runtime fallback (and what tests load with
``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the
root cost map, otherwise the model resolves on one path but not the other."""
backup = GetModelCostMap.load_local_model_cost_map()
for model_name in ALL_OPUS_5_VARIANTS:
assert model_name in backup, f"Missing from backup cost map: {model_name}"
def test_opus_5_registered_for_bedrock_converse():
assert "anthropic.claude-opus-5" in BEDROCK_CONVERSE_MODELS
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_opus_5_all_variants_carry_adaptive_thinking_flag(cost_map):
"""Every Opus 5 entry must advertise ``supports_adaptive_thinking``.
Adaptive-thinking detection is cost-map driven, so a single variant missing
the flag silently sends the legacy ``thinking.type='enabled'`` shape, which
Opus 5 rejects with a 400."""
variants = [k for k in cost_map if "claude-opus-5" in k]
assert variants, "no claude-opus-5 entries found in cost map"
missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True]
assert not missing, f"missing supports_adaptive_thinking: {missing}"

View file

@ -13,7 +13,10 @@ populate ``litellm.anthropic_models`` at import, which is what lets a bare
import json
import os
import pytest
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
@ -37,5 +40,31 @@ def _load_root_cost_map() -> dict:
return json.load(f)
def test_sonnet_5_present_in_bundled_backup():
"""The bundled backup is the runtime fallback (and what tests load with
``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the
root cost map, otherwise the model resolves on one path but not the other."""
backup = GetModelCostMap.load_local_model_cost_map()
for model_name in ALL_SONNET_5_VARIANTS:
assert model_name in backup, f"Missing from backup cost map: {model_name}"
def test_sonnet_5_registered_for_bedrock_converse():
assert "anthropic.claude-sonnet-5" in BEDROCK_CONVERSE_MODELS
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_sonnet_5_all_variants_carry_adaptive_thinking_flag(cost_map):
"""Every Sonnet 5 entry must advertise ``supports_adaptive_thinking``.
Adaptive-thinking detection is cost-map driven, so a single variant missing
the flag silently sends the legacy ``thinking.type='enabled'`` shape and the
provider 400s. This guards against a future variant being added without it."""
variants = [k for k in cost_map if "claude-sonnet-5" in k]
assert variants, "no claude-sonnet-5 entries found in cost map"
missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True]
assert not missing, f"missing supports_adaptive_thinking: {missing}"

View file

@ -0,0 +1,50 @@
"""
Regression tests for the Cloudflare Workers AI text-generation catalog in the
model-cost map.
The Cloudflare list was badly stale (only 4 ancient entries). These tests pin
the newly added current Workers AI models (sourced from Cloudflare's live
``/ai/models/search?task=Text Generation`` catalog) and guard against the root
``model_prices_and_context_window.json`` and the bundled
``litellm/model_prices_and_context_window_backup.json`` drifting out of sync for
the ``cloudflare/`` namespace.
"""
import json
import os
import pytest
import litellm
ROOT_MAP = os.path.join(
os.path.dirname(os.path.dirname(litellm.__file__)),
"model_prices_and_context_window.json",
)
BACKUP_MAP = os.path.join(
os.path.dirname(litellm.__file__),
"model_prices_and_context_window_backup.json",
)
def _load(path: str) -> dict:
with open(path, encoding="utf-8") as f:
return json.load(f)
def _cloudflare_keys(data: dict) -> set:
return {k for k in data if k.startswith("cloudflare/")}
def test_root_and_backup_have_identical_cloudflare_keys():
if not os.path.exists(ROOT_MAP):
pytest.skip("root cost map only ships in source checkouts")
assert _cloudflare_keys(_load(ROOT_MAP)) == _cloudflare_keys(_load(BACKUP_MAP))
def test_root_and_backup_cloudflare_entries_are_byte_for_byte_equal():
if not os.path.exists(ROOT_MAP):
pytest.skip("root cost map only ships in source checkouts")
root = {k: v for k, v in _load(ROOT_MAP).items() if k.startswith("cloudflare/")}
backup = {k: v for k, v in _load(BACKUP_MAP).items() if k.startswith("cloudflare/")}
assert root == backup

View file

@ -0,0 +1,53 @@
"""
Regression test: ``command-r7b-12-2024`` had its input/output per-token
costs transposed in the model-cost maps (input=1.5e-07 / output=3.75e-08),
even though Cohere publishes $0.0375/1M input and $0.15/1M output, i.e.
output is ~4x input like every other ``command-r`` entry.
These tests pin the corrected values in both the primary price map and the
``litellm/`` backup, and verify ``get_model_info`` surfaces them, so the
swap cannot silently regress.
"""
import json
import os
import litellm
MODEL = "command-r7b-12-2024"
EXPECTED_INPUT_COST = 3.75e-08
EXPECTED_OUTPUT_COST = 1.5e-07
def _load_json(path: str) -> dict:
with open(path, encoding="utf-8") as f:
return json.load(f)
def _backup_path() -> str:
return os.path.join(
os.path.dirname(litellm.__file__),
"model_prices_and_context_window_backup.json",
)
def _main_path() -> str:
# This test lives at ``tests/test_litellm/``; the primary price map sits at
# the repo root, two directories up. Resolve it relative to this file so the
# test works regardless of where ``litellm`` itself is installed (e.g. a pip
# install into site-packages).
return os.path.join(
os.path.dirname(__file__),
"..",
"..",
"model_prices_and_context_window.json",
)
class TestCommandR7bPricingData:
"""The JSON price maps must carry Cohere's published costs, with output
more expensive than input."""
class TestCommandR7bPricingModelInfo:
"""``get_model_info`` must report the corrected, un-swapped costs."""

View file

@ -32,6 +32,13 @@ def _load(path):
return json.load(f)
def test_blue_alias_matches_its_snapshot_computer_use():
cost_map = _load(MAIN_PATH)
assert cost_map[BLUE_ALIAS]["supports_computer_use"] is True
assert cost_map[BLUE_SNAPSHOT]["supports_computer_use"] is True
@pytest.mark.parametrize(("alias", "snapshot"), OFFICIAL_ALIAS_SNAPSHOTS)
def test_official_alias_tracks_snapshot(alias, snapshot):
cost_map = _load(MAIN_PATH)
@ -44,3 +51,11 @@ def test_official_alias_tracks_snapshot(alias, snapshot):
field: snapshot_info.get(field) for field in PRICE_FIELDS
}
assert alias_info["max_output_tokens"] == snapshot_info["max_output_tokens"]
@pytest.mark.parametrize("model", (*DAYBREAK_MODELS, BLUE_SNAPSHOT, *(alias for alias, _ in OFFICIAL_ALIAS_SNAPSHOTS)))
def test_backup_matches_main(model):
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"

View file

@ -15,6 +15,7 @@ import os
import litellm
from litellm.utils import (
_supports_factory,
supports_response_schema,
)
# ---------------------------------------------------------------------------
@ -36,6 +37,18 @@ class TestDeepSeekModelCostEntries:
"""Verify that provider-prefixed DeepSeek entries contain the same
capability flags as their bare-name counterparts in the JSON files."""
def test_deepseek_chat_max_input_tokens_matches_bare_in_backup(self):
data = _load_backup_json()
bare = data.get("deepseek-chat", {})
prefixed = data.get("deepseek/deepseek-chat", {})
assert prefixed.get("max_input_tokens") == bare.get("max_input_tokens")
def test_deepseek_reasoner_max_output_tokens_matches_bare_in_backup(self):
data = _load_backup_json()
bare = data.get("deepseek-reasoner", {})
prefixed = data.get("deepseek/deepseek-reasoner", {})
assert prefixed.get("max_output_tokens") == bare.get("max_output_tokens")
# ---------------------------------------------------------------------------
# API-level tests verify supports_response_schema returns True
@ -46,6 +59,18 @@ class TestSupportsResponseSchemaDeepSeek:
"""All calling conventions for DeepSeek should return True for
``supports_response_schema``."""
def test_provider_slash_model(self):
assert supports_response_schema(model="deepseek/deepseek-chat") is True
def test_explicit_provider(self):
assert supports_response_schema(model="deepseek-chat", custom_llm_provider="deepseek") is True
def test_reasoner_provider_slash_model(self):
assert supports_response_schema(model="deepseek/deepseek-reasoner") is True
def test_reasoner_explicit_provider(self):
assert supports_response_schema(model="deepseek-reasoner", custom_llm_provider="deepseek") is True
# ---------------------------------------------------------------------------
# Fallback-logic test bare model entry used when prefixed is incomplete

View file

@ -14,15 +14,7 @@ import os
import pytest
NEW_ENTRIES = {
"fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": {
"input_cost_per_token": 1.32e-06,
"cache_read_input_token_cost": 4.4e-08,
"output_cost_per_token": 3.96e-06,
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
},
}
from litellm.utils import get_model_info
@pytest.fixture(scope="module")
@ -32,20 +24,17 @@ def model_data():
return json.load(f)
TWIN_PINNED_PRICES = {
"deepseek-v4-flash-0731": {
"input_cost_per_token": 2.2e-07,
"cache_read_input_token_cost": 7e-09,
"output_cost_per_token": 6.6e-07,
},
"deepseek-v4p1-flash": {
"input_cost_per_token": 2.2e-07,
"cache_read_input_token_cost": 7e-09,
"output_cost_per_token": 6.6e-07,
"supports_vision": True,
"max_output_tokens": 393216,
},
}
def test_bare_fireworks_ids_resolve_through_prefixed_entries():
"""Bare IDs from #37274 resolve via the provider-prefix lookup path."""
for bare_id, prefixed_key in [
(
"accounts/fireworks/models/deepseek-v4-pro-0813",
"fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813",
),
]:
info = get_model_info(model=bare_id, custom_llm_provider="fireworks_ai")
assert info.get("key") == prefixed_key
assert info["litellm_provider"] == "fireworks_ai"
def test_fireworks_account_prefixed_twins_agree_on_price(model_data):

View file

@ -109,6 +109,11 @@ def test_per_route_capabilities_match_model_cards(model: str, path: Path):
assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}"
@pytest.mark.parametrize("model", ALL_KEYS)
def test_backup_matches_main(model: str):
assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model)
def test_gemini_prefix_routes_to_gemini():
routed_model, provider, _, _ = get_llm_provider(model=GEMINI)
assert routed_model == UNPREFIXED

View file

@ -0,0 +1,84 @@
import json
from collections.abc import Iterator
from pathlib import Path
from typing import Final
import pytest
import litellm
REPO_ROOT: Final = Path(__file__).parents[2]
MAIN_PATH: Final = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
FLASH_TTS_KEYS: Final = ("gemini-2.5-flash-preview-tts", "gemini/gemini-2.5-flash-preview-tts")
PRO_TTS_KEYS: Final = ("gemini-2.5-pro-preview-tts", "gemini/gemini-2.5-pro-preview-tts")
NATIVE_AUDIO_KEYS: Final = tuple(
f"{prefix}gemini-2.5-flash-native-audio-{suffix}"
for prefix in ("", "gemini/")
for suffix in ("latest", "preview-09-2025", "preview-12-2025")
)
LIVE_NATIVE_AUDIO_KEYS: Final = (
"gemini-live-2.5-flash-preview-native-audio-09-2025",
"gemini/gemini-live-2.5-flash-preview-native-audio-09-2025",
)
FLASH_TTS_INPUT: Final = 5e-07
FLASH_TTS_AUDIO_OUTPUT: Final = 1e-05
PRO_TTS_INPUT: Final = 1e-06
PRO_TTS_AUDIO_OUTPUT: Final = 2e-05
NATIVE_AUDIO_TEXT_INPUT: Final = 5e-07
NATIVE_AUDIO_AUDIO_INPUT: Final = 3e-06
NATIVE_AUDIO_TEXT_OUTPUT: Final = 2e-06
NATIVE_AUDIO_AUDIO_OUTPUT: Final = 1.2e-05
PUBLISHED_RATES: Final = {
**{
key: {"input_cost_per_token": FLASH_TTS_INPUT, "output_cost_per_token": FLASH_TTS_AUDIO_OUTPUT}
for key in FLASH_TTS_KEYS
},
**{
key: {"input_cost_per_token": PRO_TTS_INPUT, "output_cost_per_token": PRO_TTS_AUDIO_OUTPUT}
for key in PRO_TTS_KEYS
},
**{
key: {
"input_cost_per_token": NATIVE_AUDIO_TEXT_INPUT,
"input_cost_per_audio_token": NATIVE_AUDIO_AUDIO_INPUT,
"output_cost_per_token": NATIVE_AUDIO_TEXT_OUTPUT,
"output_cost_per_audio_token": NATIVE_AUDIO_AUDIO_OUTPUT,
}
for key in (*NATIVE_AUDIO_KEYS, *LIVE_NATIVE_AUDIO_KEYS)
},
}
ALL_KEYS: Final = tuple(PUBLISHED_RATES)
NATIVE_AUDIO_BILLING_CASES: Final = (
*((key, "gemini") for key in NATIVE_AUDIO_KEYS),
("gemini-live-2.5-flash-preview-native-audio-09-2025", "vertex_ai"),
("gemini/gemini-live-2.5-flash-preview-native-audio-09-2025", "gemini"),
)
LONG_CONTEXT_TIER_FIELDS: Final = (
"input_cost_per_token_above_200k_tokens",
"output_cost_per_token_above_200k_tokens",
"cache_read_input_token_cost_above_200k_tokens",
)
def _load(path: Path) -> dict[str, dict[str, object]]:
with open(path, encoding="utf-8") as f:
return json.load(f)
@pytest.fixture
def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
@pytest.mark.parametrize("model", ALL_KEYS)
def test_backup_matches_main(model: str):
assert _load(BACKUP_PATH)[model] == _load(MAIN_PATH)[model]

View file

@ -0,0 +1,44 @@
import json
from functools import lru_cache
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
DOCUMENTED_MAX_INPUT_TOKENS = 272000
DOCUMENTED_MAX_OUTPUT_TOKENS = 128000
SMALL_MODEL_NAMES = (
"gpt-5.4-mini",
"gpt-5.4-mini-2026-03-17",
"gpt-5.4-nano",
"gpt-5.4-nano-2026-03-17",
)
SMALL_MODELS = tuple(f"{prefix}{name}" for prefix in ("", "azure/", "azure_ai/") for name in SMALL_MODEL_NAMES)
STANDARD_PRICING = {
"gpt-5.4-mini": (7.5e-07, 4.5e-06, 7.5e-08),
"gpt-5.4-nano": (2e-07, 1.25e-06, 2e-08),
}
LONG_CONTEXT_MODELS = ("gpt-5.4", "gpt-5.4-pro")
@lru_cache(maxsize=2)
def _load(path: Path) -> dict[str, dict[str, object]]:
with open(path) as f:
return json.load(f)
def _pricing_key(model: str) -> str:
return "gpt-5.4-nano" if "nano" in model else "gpt-5.4-mini"
@pytest.mark.parametrize("model", SMALL_MODELS)
def test_gpt_5_4_small_models_backup_matches_main(model: str) -> None:
assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model), (
f"{model} differs between main and backup model cost maps"
)

View file

@ -0,0 +1,19 @@
import json
from pathlib import Path
def test_azure_ai_gpt_5_5_backup_matches_main():
"""Ensure the bundled model cost map stays in sync with the canonical file."""
repo_root = Path(__file__).parents[2]
main_path = repo_root / "model_prices_and_context_window.json"
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
with open(main_path) as f:
main_cost = json.load(f)
with open(backup_path) as f:
backup_cost = json.load(f)
for model in ("azure_ai/gpt-5.5", "azure_ai/gpt-5.5-2026-04-23"):
assert backup_cost.get(model) == main_cost.get(model), (
f"{model} differs between main and backup model cost maps"
)

View file

@ -1,4 +1,8 @@
from typing_extensions import get_args, get_type_hints
import json
from pathlib import Path
from typing import get_args
from typing_extensions import get_type_hints
from litellm.types.utils import ModelInfoBase
@ -41,3 +45,13 @@ ALL_REALTIME_ONLY_GPT_MODELS = REALTIME_ONLY_GPT_MODELS + REALTIME_ONLY_GPT_MODE
def test_realtime_is_a_valid_mode_literal():
hints = get_type_hints(ModelInfoBase, include_extras=False)
assert "realtime" in get_args(hints["mode"])
def test_backup_matches_main_for_realtime_models():
repo_root = Path(__file__).parents[2]
with open(repo_root / "model_prices_and_context_window.json") as f:
main_cost = json.load(f)
with open(repo_root / "litellm" / "model_prices_and_context_window_backup.json") as f:
backup_cost = json.load(f)
for model in ALL_REALTIME_ONLY_GPT_MODELS:
assert backup_cost.get(model) == main_cost.get(model)

View file

@ -0,0 +1,33 @@
import json
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
MEDIUM_3_5_MODELS = (
"mistral/mistral-medium-3-5",
"mistral/mistral-medium-2604",
"mistral/mistral-medium-latest",
)
SYNCED_MODELS = MEDIUM_3_5_MODELS + (
"mistral/mistral-medium-2508",
"mistral/mistral-medium-3-1-2508",
)
def _load(path):
with open(path) as f:
return json.load(f)
@pytest.mark.parametrize("model", SYNCED_MODELS)
def test_backup_matches_main(model):
"""Ensure the bundled (backup) cost map stays in sync with the canonical file."""
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"

View file

@ -0,0 +1,26 @@
import json
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
SMALL_4_0_MODELS = (
"mistral/mistral-small-latest",
"mistral/mistral-small-2603",
)
def _load(path):
with open(path) as f:
return json.load(f)
@pytest.mark.parametrize("model", SMALL_4_0_MODELS)
def test_backup_matches_main(model):
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"

View file

@ -0,0 +1,52 @@
import json
from pathlib import Path
import pytest
import litellm
from litellm.utils import supports_prompt_caching, supports_reasoning
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
GLM_5_2_MODELS = ("mistral/zai-glm-5-2", "mistral/glm-5-2")
INPUT_COST = 1.4e-06
CACHED_INPUT_COST = 1.4e-07
OUTPUT_COST = 4.4e-06
def _load(path):
with open(path) as f:
return json.load(f)
@pytest.fixture
def local_model_cost_map(monkeypatch):
"""Force get_model_info to resolve against the in-repo cost map instead of the
remote one fetched at import time, which still carries the pre-merge pricing."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
def test_zai_glm_5_2_capabilities_are_visible_to_callers(local_model_cost_map, model):
"""Mistral advertises reasoning and prompt caching on this model, so the helpers
every caller checks before sending a request must say so too."""
assert supports_reasoning(model=model) is True
assert supports_prompt_caching(model=model) is True
assert litellm.get_model_info(model=model)
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
def test_backup_matches_main(model):
"""Ensure the bundled (backup) cost map stays in sync with the canonical file."""
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"

View file

@ -0,0 +1,29 @@
import json
from pathlib import Path
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
MUSE_SPARK_MODEL = "meta/muse-spark-1.1"
def test_muse_spark_1_1_model_info():
routed_model, provider, _, api_base = get_llm_provider(model=MUSE_SPARK_MODEL, api_key="sk-test")
assert routed_model == "muse-spark-1.1"
assert provider == "meta"
assert api_base == "https://api.meta.ai/v1"
def test_muse_spark_1_1_backup_matches_main():
"""Ensure the bundled model cost map stays in sync with the canonical file."""
repo_root = Path(__file__).parents[2]
main_path = repo_root / "model_prices_and_context_window.json"
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
with open(main_path) as f:
main_cost = json.load(f)
with open(backup_path) as f:
backup_cost = json.load(f)
assert backup_cost.get(MUSE_SPARK_MODEL) == main_cost.get(MUSE_SPARK_MODEL), (
f"{MUSE_SPARK_MODEL} differs between main and backup model cost maps"
)

View file

@ -29,6 +29,15 @@ def test_muse_spark_1_2_routes_to_meta_model_api(model: str):
assert api_base == "https://api.meta.ai/v1"
@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR))
def test_muse_spark_1_2_backup_matches_main(model: str):
"""Ensure the bundled model cost map stays in sync with the canonical file."""
main_cost = _load_cost_map()
backup_cost = _load_cost_map("litellm/model_prices_and_context_window_backup.json")
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"
def test_muse_spark_contributor_tier_is_cheaper_than_standard():
cost_map = _load_cost_map()
standard = cost_map[MUSE_SPARK_STANDARD]

View file

@ -38,6 +38,15 @@ def test_muse_spark_1_3_web_search_cost_per_query(local_model_cost_map, model: s
assert StandardBuiltInToolCostTracking.get_cost_for_web_search(model_info=info) == WEB_SEARCH_COST_PER_QUERY
@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR))
def test_muse_spark_1_3_backup_matches_main(model: str):
"""Ensure the bundled model cost map stays in sync with the canonical file."""
main_cost = _load_cost_map()
backup_cost = _load_cost_map("litellm/model_prices_and_context_window_backup.json")
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"
def test_muse_spark_contributor_tier_is_cheaper_than_standard():
cost_map = _load_cost_map()
standard = cost_map[MUSE_SPARK_STANDARD]

View file

@ -18,3 +18,18 @@ def test_replicate_models_have_valid_key_prefix(model_cost: dict[str, Any]) -> N
assert not malformed, (
f"Replicate models must use 'replicate/owner/model' key format, found malformed keys: {malformed}"
)
def test_replicate_backup_matches_main() -> None:
repo_root = Path(__file__).parents[2]
main_path = repo_root / "model_prices_and_context_window.json"
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
with open(main_path, encoding="utf-8") as f:
main_cost: dict[str, Any] = json.load(f)
with open(backup_path, encoding="utf-8") as f:
backup_cost: dict[str, Any] = json.load(f)
for key in main_cost:
if main_cost[key].get("litellm_provider") == "replicate":
assert backup_cost.get(key) == main_cost.get(key), f"{key} differs between main and backup model cost maps"

View file

@ -0,0 +1,25 @@
import json
from pathlib import Path
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
def test_sambanova_minimax_m27_model_info():
model = "sambanova/MiniMax-M2.7"
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
with open(json_path) as f:
model_cost = json.load(f)
info = model_cost.get(model)
assert info is not None, f"{model} not found in model_prices_and_context_window.json"
assert info["litellm_provider"] == "sambanova"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] > 0
assert info["output_cost_per_token"] > 0
assert info["supports_function_calling"] is True
assert info["supports_reasoning"] is True
assert info["supports_tool_choice"] is True
routed_model, provider, _, _ = get_llm_provider(model=model)
assert routed_model == "MiniMax-M2.7"
assert provider == "sambanova"

View file

@ -108,6 +108,14 @@ def test_together_successor_metadata_points_at_live_models(cost_map: CostMap):
assert "deprecation_date" not in target, f"{model} names deprecated successor {successor}"
def test_together_backup_cost_map_in_sync(cost_map: CostMap):
with open(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json") as f:
backup = COST_MAP_ADAPTER.validate_python(json.load(f))
together_main = {k: v for k, v in cost_map.items() if k.startswith("together_ai/")}
together_backup = {k: v for k, v in backup.items() if k.startswith("together_ai/")}
assert together_backup == together_main
CACHED_INPUT_MODELS: Final = (
"together_ai/moonshotai/Kimi-K3",
"together_ai/zai-org/GLM-5.2",

View file

@ -3641,9 +3641,6 @@ def _assert_fireworks_entry(
assert info["input_cost_per_token"] > 0
assert info["output_cost_per_token"] > 0
assert "cache_read_input_token_cost" in info
assert info["max_input_tokens"] == expected_max_input
assert info["max_output_tokens"] == expected_max_output
assert info["max_tokens"] == expected_max_output
assert info["supports_function_calling"] is True
assert info["supports_tool_choice"] is True
assert info["supports_reasoning"] is expected_reasoning
@ -5657,3 +5654,253 @@ def test_completion_finishes_response_metadata_before_handing_the_response_to_th
assert snapshot["litellm_call_id"]
assert snapshot["response_cost"] is not None
assert snapshot["api_base"]
def test_fireworks_models_in_backup_cost_map():
import json
from pathlib import Path
json_path = Path(__file__).parents[2] / "litellm" / "model_prices_and_context_window_backup.json"
with open(json_path) as f:
model_cost = json.load(f)
for entry in _FIREWORKS_MODELS:
_assert_fireworks_entry(model_cost, *entry)
for short in _FIREWORKS_SHORT_FORMS:
long_key = f"fireworks_ai/accounts/fireworks/models/{short}"
short_key = f"fireworks_ai/{short}"
assert model_cost.get(short_key) == model_cost.get(long_key), (
f"short-form {short_key} does not match long-form {long_key}"
)
for short in _FIREWORKS_ROUTER_SHORT_FORMS:
long_key = f"fireworks_ai/accounts/fireworks/routers/{short}"
short_key = f"fireworks_ai/{short}"
assert model_cost.get(short_key) == model_cost.get(long_key), (
f"short-form {short_key} does not match long-form {long_key}"
)
def test_fireworks_models_in_cost_map():
import json
from pathlib import Path
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
with open(json_path) as f:
model_cost = json.load(f)
for entry in _FIREWORKS_MODELS:
_assert_fireworks_entry(model_cost, *entry)
for short in _FIREWORKS_SHORT_FORMS:
long_key = f"fireworks_ai/accounts/fireworks/models/{short}"
short_key = f"fireworks_ai/{short}"
assert model_cost.get(short_key) == model_cost.get(long_key), (
f"short-form {short_key} does not match long-form {long_key}"
)
for short in _FIREWORKS_ROUTER_SHORT_FORMS:
long_key = f"fireworks_ai/accounts/fireworks/routers/{short}"
short_key = f"fireworks_ai/{short}"
assert model_cost.get(short_key) == model_cost.get(long_key), (
f"short-form {short_key} does not match long-form {long_key}"
)
def test_fireworks_short_model_names_resolve_to_long_cost_map_keys(fireworks_short_model_cost_map: None) -> None:
model_info = litellm.get_model_info("fireworks_ai/glm-5p3")
assert model_info["key"] == "fireworks_ai/accounts/fireworks/models/glm-5p3"
model_info = litellm.get_model_info("glm-5p3", custom_llm_provider="fireworks_ai")
assert model_info["key"] == "fireworks_ai/accounts/fireworks/models/glm-5p3"
model_info = litellm.get_model_info("fireworks_ai/glm-5p3-fast")
assert model_info["key"] == "fireworks_ai/accounts/fireworks/routers/glm-5p3-fast"
model_info = litellm.get_model_info("fireworks_ai/nomic-ai/nomic-embed-text-v1.5")
assert model_info["key"] == "fireworks_ai/nomic-ai/nomic-embed-text-v1.5"
with pytest.raises(Exception, match="isn't mapped"):
litellm.get_model_info("fireworks_ai/does-not-exist")
def test_get_model_info_bedrock_regional_profile_without_entry_falls_back_to_base(local_model_cost_map):
"""A regional profile with no dedicated cost-map entry must still resolve to its
region-stripped base entry."""
info = litellm.get_model_info(model="bedrock/apac.anthropic.claude-opus-4-8")
assert info["key"] == "anthropic.claude-opus-4-8"
def test_get_model_info_gemini(monkeypatch):
"""
Tests if ALL gemini models have 'tpm' and 'rpm' in the model info
"""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
model_map = litellm.model_cost
for model, info in model_map.items():
if (
model.startswith("gemini/")
and "gemma" not in model
and "learnlm" not in model
and "imagen" not in model
and "veo" not in model
and "lyria" not in model
and "robotics" not in model
):
assert info.get("tpm") is not None, f"{model} does not have tpm"
assert info.get("rpm") is not None, f"{model} does not have rpm"
def test_get_model_info_resolves_provider_prefixed_model_ids(local_model_cost_map):
"""Perplexity's Agent API third-party models are keyed `perplexity/perplexity/<id>`
because Perplexity's own id already starts with `perplexity/`. Callers run
`get_llm_provider` first, which hands `_get_potential_model_names` model
`perplexity/glm-5.2` with provider `perplexity`, and every candidate but the
provider-prefixed one strips that second `perplexity/` off. Regression: the
entries were unreachable from `supports_reasoning` and from the cost calculator's
per-token fallback, so a mapped model reported no reasoning support and raised
"This model isn't mapped yet" on the only path where its rates are ever used."""
for model, reasoning in (
("perplexity/perplexity/glm-5.2", True),
("perplexity/perplexity/kimi-k3", True),
("perplexity/perplexity/deepseek-v4-flash-0731", True),
("perplexity/perplexity/kimi-k2.7-code", False),
("perplexity/perplexity/nemotron-3.5-lightning-30b-a3b", True),
("perplexity/perplexity/nemotron-3-ultra-550b-a55b", True),
):
assert litellm.supports_reasoning(model=model) is reasoning, model
via_provider = litellm.get_model_info(model="perplexity/glm-5.2", custom_llm_provider="perplexity")
assert via_provider["key"] == "perplexity/perplexity/glm-5.2"
assert via_provider["mode"] == "responses"
lightning = litellm.get_model_info(
model="perplexity/nemotron-3.5-lightning-30b-a3b", custom_llm_provider="perplexity"
)
assert lightning["key"] == "perplexity/perplexity/nemotron-3.5-lightning-30b-a3b"
assert lightning["mode"] == "responses"
ultra = litellm.get_model_info(model="perplexity/perplexity/nemotron-3-ultra-550b-a55b")
assert ultra["key"] == "perplexity/perplexity/nemotron-3-ultra-550b-a55b"
def test_get_model_info_shows_supports_computer_use(monkeypatch):
"""
Tests if 'supports_computer_use' is correctly retrieved by get_model_info.
We'll use 'claude-4-sonnet-20250514' as it's configured
in the backup JSON to have supports_computer_use: True.
"""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
# Ensure litellm.model_cost is loaded, relying on the backup mechanism if primary fails
# as per previous debugging.
litellm.model_cost = litellm.get_model_cost_map(url="")
# This model should have 'supports_computer_use': True in the backup JSON
model_known_to_support_computer_use = "claude-4-sonnet-20250514"
info = litellm.get_model_info(model_known_to_support_computer_use)
# After the fix in utils.py, this should now be present and True
assert info.get("supports_computer_use") is True
def test_get_model_info_surfaces_supports_adaptive_thinking(local_model_cost_map):
"""supports_adaptive_thinking must flow through get_model_info like every other
capability flag: both from an explicit cost-map entry and from a
fallback-generalization rule for an unmapped model. Regression: the field shipped
in the JSON but was never declared on ModelInfo nor copied during construction, so
get_model_info (and _supports_factory) silently dropped it for any provider-prefixed
or unmapped name."""
explicit = litellm.get_model_info(model="claude-opus-4-8")
assert explicit["supports_adaptive_thinking"] is True
generalized = litellm.get_model_info(model="claude-opus-4-9", custom_llm_provider="anthropic")
assert generalized["supports_adaptive_thinking"] is True
def test_get_model_info_surfaces_supports_parallel_function_calling(local_model_cost_map):
"""A registry entry's supports_parallel_function_calling must read back through get_model_info
and litellm.supports_parallel_function_calling. Regression: the key was never copied into
ModelInfo, so provider-prefixed entries read None / False even when the map said True, and an
explicit False was indistinguishable from unset."""
declared_true = litellm.get_model_info(model="together_ai/zai-org/GLM-5.3-Flash")
assert declared_true["supports_parallel_function_calling"] is True
assert litellm.supports_parallel_function_calling(model="together_ai/zai-org/GLM-5.3-Flash") is True
def test_model_info_for_fireworks_short_form_models():
"""
Test that fireworks_ai short-form model entries (fireworks_ai/<model>)
are correctly configured in model_prices_and_context_window.json.
These entries enable cost attribution for models called via short-form
names (e.g., fireworks_ai/glm-4p7 instead of
fireworks_ai/accounts/fireworks/models/glm-4p7).
"""
import json
from pathlib import Path
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
with open(json_path) as f:
model_cost = json.load(f)
# glm-4p7: short-form and long-form
for key in [
"fireworks_ai/glm-4p7",
"fireworks_ai/accounts/fireworks/models/glm-4p7",
]:
info = model_cost.get(key)
assert info is not None, f"{key} not found in model_prices_and_context_window.json"
assert info["litellm_provider"] == "fireworks_ai"
assert info["mode"] == "chat"
assert info["supports_reasoning"] is True
# minimax-m2p1: short-form and long-form
for key in [
"fireworks_ai/minimax-m2p1",
"fireworks_ai/accounts/fireworks/models/minimax-m2p1",
]:
info = model_cost.get(key)
assert info is not None, f"{key} not found in model_prices_and_context_window.json"
assert info["litellm_provider"] == "fireworks_ai"
assert info["mode"] == "chat"
# kimi-k2p5: short-form only (long-form already existed)
info = model_cost.get("fireworks_ai/kimi-k2p5")
assert info is not None, "fireworks_ai/kimi-k2p5 not found in model_prices_and_context_window.json"
assert info["litellm_provider"] == "fireworks_ai"
assert info["mode"] == "chat"
def test_model_info_for_vertex_ai_deepseek_model():
model_info = litellm.get_model_info(model="vertex_ai/deepseek-ai/deepseek-r1-0528-maas")
assert model_info is not None
assert model_info["litellm_provider"] == "vertex_ai-deepseek_models"
assert model_info["mode"] == "chat"
assert model_info["input_cost_per_token"] is not None
assert model_info["output_cost_per_token"] is not None
def test_provider_prefixed_lookup_never_outranks_an_existing_row(local_model_cost_map):
"""The provider-prefixed candidate is tried last, after every candidate that
already existed, so no model that resolves today can change answer. `perplexity/sonar`
is the case that proves it: both `perplexity/sonar` and `perplexity/perplexity/sonar`
are cost-map keys, and the shorter one must keep winning."""
sonar = litellm.get_model_info(model="sonar", custom_llm_provider="perplexity")
assert sonar["key"] == "perplexity/sonar"
assert sonar["mode"] == "chat"
still_sonar = litellm.get_model_info(model="perplexity/sonar", custom_llm_provider="perplexity")
assert still_sonar["key"] == "perplexity/sonar"
assert still_sonar["mode"] == "chat"
for model, provider, expected_key in (
("claude-sonnet-4-5", "anthropic", "claude-sonnet-4-5"),
("anthropic/claude-sonnet-4-5", "anthropic", "claude-sonnet-4-5"),
("gemini/gemini-2.0-flash", "gemini", "gemini/gemini-2.0-flash"),
("openrouter/openai/gpt-4o", "openrouter", "openrouter/openai/gpt-4o"),
):
assert litellm.get_model_info(model=model, custom_llm_provider=provider)["key"] == expected_key

View file

@ -0,0 +1,19 @@
import json
from pathlib import Path
def test_xai_grok_4_3_backup_matches_main():
"""Ensure the bundled model cost map stays in sync with the canonical file."""
repo_root = Path(__file__).parents[2]
main_path = repo_root / "model_prices_and_context_window.json"
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
with open(main_path) as f:
main_cost = json.load(f)
with open(backup_path) as f:
backup_cost = json.load(f)
for model in ("xai/grok-4.3", "xai/grok-4.3-latest"):
assert backup_cost.get(model) == main_cost.get(model), (
f"{model} differs between main and backup model cost maps"
)