scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 86

Merge pull request #39388 from BerriAI/litellm_registry_audit_2026_09_02

Browse source 
fix(model_prices): verified registry audit, Databricks Sep-2026 catalog, realtime image pricing, deprecation dates
This commit is contained in:
Mateo Wang 2026-09-04 14:51:40 -07:00 • committed by GitHub
commit 922659fb15
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 9727 additions and 515 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4914
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

4914
model_prices_and_context_window.json
View file

File diff suppressed because it is too large Load diff

20
model_prices_and_context_window.schema.json
View file

@ -79,6 +79,11 @@
"minimum": 0,
"description": "USD per token written to the provider's prompt cache."
},
"cache_creation_input_token_cost_above_128k_tokens": {
"type": "number",
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"cache_creation_input_token_cost_above_1hr": {
"type": "number",
"minimum": 0,
@ -94,6 +99,11 @@
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"cache_creation_input_token_cost_above_256k_tokens": {
"type": "number",
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"cache_creation_input_token_cost_above_272k_tokens": {
"type": "number",
"minimum": 0,
@ -128,6 +138,11 @@
"minimum": 0,
"description": "USD per prompt token served from the provider's prompt cache."
},
"cache_read_input_token_cost_above_128k_tokens": {
"type": "number",
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"cache_read_input_token_cost_above_200k_tokens": {
"type": "number",
"minimum": 0,
@ -138,6 +153,11 @@
"minimum": 0,
"description": "Priority service-tier rate for the same-named base field."
},
"cache_read_input_token_cost_above_256k_tokens": {
"type": "number",
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"cache_read_input_token_cost_above_272k_tokens": {
"type": "number",
"minimum": 0,

22
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -4630,6 +4630,28 @@ def test_generic_cost_per_token_grok_46_long_context(_local_model_cost_map):
assert completion_cost == pytest.approx(1_000 * 1.2e-05)
@pytest.mark.parametrize(
("model", "provider", "image_token_rate"),
[
("gpt-realtime-2.1", "openai", 5e-06),
("gpt-realtime-2.1-mini", "openai", 8e-07),
("azure/gpt-realtime-2.1", "azure", 5e-06),
("azure/gpt-realtime-2.1-mini", "azure", 8e-07),
],
)
def test_realtime_image_tokens_priced_per_token(model, provider, image_token_rate, _local_model_cost_map):
"""Realtime image input is billed per 1M image tokens, not per image."""
usage = Usage(
prompt_tokens=1_100,
completion_tokens=0,
total_tokens=1_100,
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=100, image_tokens=1_000),
)
prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=provider)
text_rate = litellm.model_cost[model]["input_cost_per_token"]
assert prompt_cost == pytest.approx(100 * text_rate + 1_000 * image_token_rate)
@pytest.mark.parametrize(
("response_quality", "requested_quality", "expected_cost"),
[

52
tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py
View file

@ -9,7 +9,6 @@ import os
import pytest
from litellm.litellm_core_utils.fallback_generalizations import (
get_fallback_generalization_rules,
match_capability_generalizations,
@ -248,6 +247,57 @@ def test_azure_ai_claude_1m_context_entries(cost_map: dict):
assert cost_map[model]["max_input_tokens"] == 200000, model
# OpenRouter headline rates from GET https://openrouter.ai/api/v1/models.
# These were the catalog values that disagreed with that API (and, for the
# two spotlight models, the public model pages that their source fields cite).
_OPENROUTER_LIVE_COSTS = {
"openrouter/qwen/qwen3.5-plus-02-15": (2.6e-07, 1.56e-06, None),
"openrouter/openai/gpt-oss-120b": (3.7e-08, 1.7e-07, None),
"openrouter/qwen/qwen3-coder-plus": (6.5e-07, 3.25e-06, None),
"openrouter/qwen/qwen3.5-flash-02-23": (6.5e-08, 2.6e-07, None),
"openrouter/qwen/qwen3.5-27b": (1.95e-07, 1.56e-06, None),
"openrouter/gryphe/mythomax-l2-13b": (6e-08, 6e-08, None),
"openrouter/mancer/weaver": (4e-07, 7.5e-07, None),
"openrouter/xiaomi/mimo-v2.5-pro": (4.35e-07, 8.7e-07, 3.6e-09),
"openrouter/moonshotai/kimi-k2.5": (4.5e-07, 2.25e-06, 7e-08),
"openrouter/z-ai/glm-5": (6e-07, 1.92e-06, None),
}
_OPENROUTER_STALE_COSTS = {
"openrouter/qwen/qwen3.5-plus-02-15": (4e-07, 2.4e-06),
"openrouter/openai/gpt-oss-120b": (1.8e-07, 8e-07),
"openrouter/gryphe/mythomax-l2-13b": (1.875e-06, 1.875e-06),
}
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_openrouter_catalog_costs_match_live_headline_rates(cost_map: dict):
"""openrouter/* spend tracking reads these catalog fields. The values must
stay aligned with OpenRouter's published headline rate, not the stale
figures that over/under-counted by up to 30x. Both maps are checked so
the root file and bundled backup cannot drift apart."""
control = cost_map["openrouter/anthropic/claude-opus-5"]
assert control["input_cost_per_token"] == 5e-06
assert control["output_cost_per_token"] == 2.5e-05
assert control["cache_read_input_token_cost"] == 5e-07
for model, (inp, out, cache) in _OPENROUTER_LIVE_COSTS.items():
entry = cost_map[model]
assert entry["input_cost_per_token"] == inp, model
assert entry["output_cost_per_token"] == out, model
if cache is not None:
assert entry["cache_read_input_token_cost"] == cache, model
for model, (stale_in, stale_out) in _OPENROUTER_STALE_COSTS.items():
entry = cost_map[model]
assert entry["input_cost_per_token"] != stale_in, model
assert entry["output_cost_per_token"] != stale_out, model
def test_get_model_cost_map_stamps_loaded_at(monkeypatch):
"""The load time feeds each pod's reload-due decision; a load that does not stamp it
would make manual reload requests race the proxy's startup"""

29
tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py
View file

@ -499,3 +499,32 @@ class TestAzureAIServiceTierCostCalculation:
assert flex_prompt < standard_prompt
assert flex_completion < standard_completion
def test_codestral_2501_model_info_and_cost(local_model_cost_map):
model_info = get_model_info(model="Codestral-2501", custom_llm_provider="azure_ai")
usage = Usage(prompt_tokens=1_000_000, completion_tokens=1_000_000, total_tokens=2_000_000)
prompt_cost, completion_cost = cost_per_token(model="Codestral-2501", usage=usage)
assert model_info["mode"] == "chat"
assert model_info["max_input_tokens"] == 256000
assert model_info["max_output_tokens"] == 4096
assert prompt_cost == pytest.approx(0.3)
assert completion_cost == pytest.approx(0.9)
def test_mai_thinking_1_model_info_and_cost(local_model_cost_map):
model_info = get_model_info(model="MAI-Thinking-1", custom_llm_provider="azure_ai")
usage = Usage(prompt_tokens=1_000_000, completion_tokens=1_000_000, total_tokens=2_000_000)
prompt_cost, completion_cost = cost_per_token(model="MAI-Thinking-1", usage=usage)
assert model_info["mode"] == "chat"
assert model_info["max_input_tokens"] == 256000
assert model_info["max_output_tokens"] == 64000
assert model_info["cache_read_input_token_cost"] == pytest.approx(2e-07)
assert model_info["supports_reasoning"] is True
assert model_info["supports_function_calling"] is True
assert prompt_cost == pytest.approx(2.0)
assert completion_cost == pytest.approx(8.0)

25
tests/test_litellm/llms/azure_ai/test_azure_ai_fw_models_metadata.py
View file

@ -176,6 +176,7 @@ def test_azure_ai_fw_model_info(use_local_model_cost_map, model_key, expected):
("FW-MiniMax-M2.5", 0.33, 1.32),
("FW-Inkling", 1.0, 4.05),
("FW-Nemotron-3-Ultra-NVFP4", 0.6, 2.4),
("FW-Nemotron-Lightning-3.5-30B-A3B", 0.06, 0.22),
],
)
def test_azure_ai_fw_cost_per_token(
@ -196,6 +197,30 @@ def test_azure_ai_fw_cost_per_token(
assert completion_cost == pytest.approx(expected_completion)
def test_azure_ai_fw_nemotron_lightning_model_info(use_local_model_cost_map):
model_info = use_local_model_cost_map.get_model_info(model="azure_ai/FW-Nemotron-Lightning-3.5-30B-A3B")
assert model_info["litellm_provider"] == "azure_ai"
assert model_info["mode"] == "chat"
assert model_info["input_cost_per_token"] == pytest.approx(6e-08)
assert model_info["output_cost_per_token"] == pytest.approx(2.2e-07)
assert model_info["cache_read_input_token_cost"] == pytest.approx(1e-08)
assert model_info["max_input_tokens"] == 262144
assert model_info["supports_function_calling"] is True
assert model_info["supports_reasoning"] is True
assert model_info["supports_tool_choice"] is True
assert model_info["supports_prompt_caching"] is True
assert model_info["supports_vision"] is False
def test_azure_ai_fw_nemotron_lightning_supports_tool_choice(use_local_model_cost_map):
from litellm.llms.azure_ai.chat.transformation import AzureAIStudioConfig
supported_params = AzureAIStudioConfig().get_supported_openai_params("FW-Nemotron-Lightning-3.5-30B-A3B")
assert "tool_choice" in supported_params
def test_azure_ai_fw_kimi_k26_case_insensitive_lookup(use_local_model_cost_map):
upper = use_local_model_cost_map.get_model_info(model="azure_ai/FW-Kimi-K2.6")
lower = use_local_model_cost_map.get_model_info(model="azure_ai/fw-kimi-k2.6")

24
tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py
View file

@ -18,6 +18,10 @@ NEW_MODELS: Final = (
"databricks/databricks-claude-opus-5",
"databricks/databricks-claude-sonnet-5",
"databricks/databricks-claude-fable-5",
"databricks/databricks-claude-fable-5-1",
"databricks/databricks-gpt-5-6-sol",
"databricks/databricks-gpt-5-6-terra",
"databricks/databricks-gpt-5-6-luna",
)
DOLLARS_PER_DBU: Final = Decimal("0.070")
@ -28,6 +32,7 @@ PRICE_FIELDS: Final = (
"cache_read_input_token_cost",
)
PUBLISHED_DBU_PER_MILLION: Final = {
"databricks/databricks-claude-fable-5-1": ("142.858", "714.286", "178.572", "3.572"),
"databricks/databricks-claude-fable-5": ("142.858", "714.286", "178.572", "14.286"),
"databricks/databricks-claude-opus-5": ("71.429", "357.143", "89.286", "7.143"),
"databricks/databricks-claude-opus-4-8": ("71.429", "357.143", "89.286", "7.143"),
@ -52,9 +57,17 @@ PUBLISHED_DBU_PER_MILLION: Final = {
"databricks/databricks-gpt-5-2": ("25.000", "200.000", "25.000", "2.500"),
"databricks/databricks-gpt-5-2-codex": ("25.000", "200.000", "25.000", "2.500"),
"databricks/databricks-gpt-5-3-codex": ("25.000", "200.000", "25.000", "2.500"),
"databricks/databricks-gpt-5-6-sol": ("57.143", "285.714", "71.429", "5.714"),
"databricks/databricks-gpt-5-6-terra": ("35.714", "214.286", "44.643", "3.571"),
"databricks/databricks-gpt-5-6-luna": ("14.286", "85.714", "17.857", "1.429"),
"databricks/databricks-gpt-5-5": ("71.429", "428.571", "71.429", "7.143"),
"databricks/databricks-gpt-5-5-pro": ("428.571", "2571.429", "428.571", "428.571"),
"databricks/databricks-gpt-5-4": ("35.714", "214.286", "35.714", "3.571"),
"databricks/databricks-gpt-5-4-mini": ("10.714", "64.286", "10.714", "1.071"),
"databricks/databricks-gpt-5-4-nano": ("2.857", "17.857", "2.857", "0.286"),
"databricks/databricks-gemini-3-6-flash": ("26.786", "133.929", "26.786", "2.679"),
"databricks/databricks-gemini-3-5-flash": ("26.786", "160.714", "26.786", "2.679"),
"databricks/databricks-gemini-3-5-flash-lite": ("5.357", "44.643", "5.357", "0.536"),
"databricks/databricks-gemini-3-1-pro": ("35.714", "214.286", "35.714", "3.571"),
"databricks/databricks-gemini-3-pro": ("35.714", "214.286", "35.714", "3.571"),
"databricks/databricks-gemini-3-flash": ("8.929", "53.571", "8.929", "0.893"),
@ -65,6 +78,13 @@ PUBLISHED_DBU_PER_MILLION: Final = {
"databricks/databricks-deepseek-v4-flash-0731": ("2.000", "4.000", "2.000", "0.400"),
"databricks/databricks-deepseek-v4-pro-0813": ("18.857", "56.571", "18.857", "1.886"),
"databricks/databricks-glm-5-2": ("20.000", "62.857", "20.000", "3.714"),
"databricks/databricks-glm-5-3": ("20.000", "62.857", "20.000", "3.714"),
"databricks/databricks-glm-5-3-flash": ("2.143", "7.143", "2.143", "0.429"),
"databricks/databricks-inkling": ("14.286", "57.857", "14.286", "2.429"),
"databricks/databricks-grok-4-6": ("35.714", "107.143", "35.714", "8.929"),
"databricks/databricks-qwen35-122b-a10b": ("3.143", "31.429", "3.143", "3.143"),
"databricks/databricks-qwen3-next-80b-a3b-instruct": ("2.143", "17.143", "2.143", "2.143"),
"databricks/databricks-qwen3-embedding-0-6b": ("0.286", "0", "0.286", "0.286"),
}
PROMOTIONAL_DISCOUNT: Final = 0.80
PROMOTION_EXPIRES: Final = "2027-01-31"
@ -73,6 +93,10 @@ ENTRIES_STORING_PROMOTIONAL_RATE: Final = (
"databricks/databricks-gemini-2-5-flash",
)
ENTRIES_STORING_LIST_RATE_DESPITE_PROMOTION: Final = (
"databricks/databricks-gemini-3-6-flash",
"databricks/databricks-gemini-3-5-flash",
"databricks/databricks-gemini-3-5-flash-lite",
"databricks/databricks-grok-4-6",
"databricks/databricks-gemini-3-1-pro",
"databricks/databricks-gemini-3-pro",
"databricks/databricks-gemini-3-flash",

14
tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py
View file

@ -15,6 +15,7 @@ from litellm.cost_calculator import completion_cost
from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo
OCR4_COST_PER_PAGE = 0.004
OCR4_ANNOTATION_COST_PER_PAGE = 0.005
REPO_ROOT = Path(__file__).parents[5]
MAIN_COST_MAP = REPO_ROOT / "model_prices_and_context_window.json"
@ -133,3 +134,16 @@ def test_azure_doc_ai_annotation_pages_fall_back_to_ocr_rate(local_model_cost_ma
call_type="ocr",
)
assert cost == pytest.approx(AZURE_DOC_AI_COST_PER_PAGE)
def test_azure_ocr4_bills_ocr_and_annotation_pages_at_their_own_rates(local_model_cost_map) -> None:
info = litellm.get_model_info(model="azure_ai/mistral-ocr-4-0", custom_llm_provider="azure_ai")
assert info["ocr_cost_per_page"] == OCR4_COST_PER_PAGE
assert info["annotation_cost_per_page"] == OCR4_ANNOTATION_COST_PER_PAGE
cost = completion_cost(
completion_response=_annotated_ocr_response("mistral-ocr-4-0", 2, 3),
model="azure_ai/mistral-ocr-4-0",
custom_llm_provider="azure_ai",
call_type="ocr",
)
assert cost == pytest.approx(2 * OCR4_COST_PER_PAGE + 3 * OCR4_ANNOTATION_COST_PER_PAGE)

5
tests/test_litellm/proxy/spend_tracking/test_savings.py
View file

@ -852,7 +852,7 @@ def test_the_served_arm_is_read_from_the_record_not_repriced():
@pytest.mark.parametrize(
"basis, expected_multiplier",
[
pytest.param({"service_tier": "priority"}, 2.0, id="priority tier doubles the baseline"),
pytest.param({"service_tier": "priority"}, 2.5, id="priority tier uplifts the baseline"),
pytest.param({"data_residency": "eu"}, 1.1, id="eu residency uplifts the baseline"),
pytest.param({}, 1.0, id="no basis recorded prices at standard"),
pytest.param(None, 1.0, id="row predating the field prices at standard"),
@ -872,7 +872,8 @@ def test_the_baseline_is_priced_on_the_basis_the_request_was_billed_at(basis, ex
"""
gpt = litellm.get_model_info("gpt-5.5", "openai")
haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic")
assert gpt.get("input_cost_per_token_priority") == 2 * gpt["input_cost_per_token"]
assert gpt.get("input_cost_per_token_priority") == pytest.approx(2.5 * gpt["input_cost_per_token"])
assert gpt.get("output_cost_per_token_priority") == pytest.approx(2.5 * gpt["output_cost_per_token"])
assert gpt.get("regional_processing_uplift_multiplier_eu") == 1.1
assert haiku.get("input_cost_per_token_priority") is None, "served model must not move with the basis"
assert haiku.get("regional_processing_uplift_multiplier_eu") is None

155
tests/test_litellm/test_baseten_glm_5_3_model_metadata.py Normal file
View file

@ -0,0 +1,155 @@
import json
from pathlib import Path
import pytest
import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
from litellm.utils import supports_function_calling, supports_prompt_caching
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
MODEL = "baseten/zai-org/GLM-5.3"
INPUT_COST = 1.4e-06
CACHED_INPUT_COST = 1.4e-07
OUTPUT_COST = 4.4e-06
def _load(path):
with open(path) as f:
return json.load(f)
@pytest.fixture
def local_model_cost_map(monkeypatch):
"""Force get_model_info to resolve against the in-repo cost map instead of the
remote one fetched at import time, which still carries the pre-merge registry."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
def test_baseten_glm_5_3_specs():
info = _load(MAIN_PATH).get(MODEL)
assert info is not None, f"{MODEL} missing from model_prices_and_context_window.json"
assert info["litellm_provider"] == "baseten"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] == INPUT_COST
assert info["output_cost_per_token"] == OUTPUT_COST
assert info["cache_read_input_token_cost"] == CACHED_INPUT_COST
assert info["max_input_tokens"] == 1048576
assert info["max_output_tokens"] == 262144
assert info["max_tokens"] == 262144
assert info["supports_function_calling"] is True
assert info["supports_prompt_caching"] is True
assert info["supports_response_schema"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
assert info["supported_modalities"] == ["text", "image"]
assert info["supported_output_modalities"] == ["text"]
routed_model, provider, _, _ = get_llm_provider(model=MODEL)
assert routed_model == "zai-org/GLM-5.3"
assert provider == "baseten"
def test_baseten_glm_5_3_capabilities_are_visible_to_callers(local_model_cost_map):
"""The entry advertises prompt caching and tool calling, so the helpers every
caller checks before sending a request must say so too."""
assert supports_prompt_caching(model=MODEL) is True
assert supports_function_calling(model=MODEL) is True
info = litellm.get_model_info(model="zai-org/GLM-5.3", custom_llm_provider="baseten")
assert info["max_input_tokens"] == 1048576
assert info["max_output_tokens"] == 262144
def test_cached_prompt_tokens_bill_at_the_cached_rate(local_model_cost_map):
"""A cache hit reports its reused tokens under prompt_tokens_details, and those
tokens cost a tenth of the input rate, not the full rate and not nothing."""
usage = Usage(
prompt_tokens=21010,
completion_tokens=100,
total_tokens=21110,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=20992),
)
prompt_cost, completion_cost = litellm.cost_per_token(
model=MODEL, usage_object=usage, custom_llm_provider="baseten"
)
assert prompt_cost == pytest.approx(18 * INPUT_COST + 20992 * CACHED_INPUT_COST)
assert completion_cost == pytest.approx(100 * OUTPUT_COST)
def test_backup_matches_main():
"""Ensure the bundled (backup) cost map stays in sync with the canonical file.
Both keys are asserted present first: comparing two ``.get`` results alone passes
just as happily when neither file has the entry at all, which is the exact state
this test exists to catch.
"""
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert MODEL in main_cost, f"{MODEL} missing from model_prices_and_context_window.json"
assert MODEL in backup_cost, f"{MODEL} missing from model_prices_and_context_window_backup.json"
assert backup_cost[MODEL] == main_cost[MODEL], f"{MODEL} differs between main and backup model cost maps"
def test_entry_advertises_only_what_the_baseten_path_accepts(local_model_cost_map):
"""The entry must not claim a capability whose request parameter BasetenConfig
refuses.
``BasetenConfig.get_supported_openai_params`` returns one hardcoded list for every
Baseten model, and it carries neither ``parallel_tool_calls`` nor
``reasoning_effort``. Baseten's own Model API does take ``reasoning_effort``, but
litellm's Baseten path drops it (``drop_params=True``) or raises
``UnsupportedParamsError`` (``drop_params=False``), so declaring
``supports_parallel_function_calling``, ``supports_reasoning`` or
``reasoning_effort_levels`` here would advertise a level the gateway then refuses to
send. Wiring those params through the Baseten config is separate work; until it
lands, the registry stays honest.
"""
supported = litellm.get_supported_openai_params(model="zai-org/GLM-5.3", custom_llm_provider="baseten")
assert supported is not None
entry = _load(MAIN_PATH)[MODEL]
capability_to_param = {
"supports_function_calling": "tools",
"supports_tool_choice": "tool_choice",
"supports_response_schema": "response_format",
"supports_parallel_function_calling": "parallel_tool_calls",
"supports_reasoning": "reasoning_effort",
}
for capability, param in capability_to_param.items():
if entry.get(capability):
assert param in supported, f"{MODEL} advertises {capability} but baseten drops/rejects {param}"
assert "reasoning_effort_levels" not in entry, (
"reasoning_effort_levels advertises accepted reasoning_effort values, which the Baseten path does not accept"
)
assert "thinking_always_on" not in entry, (
"thinking_always_on is only read by AnthropicModelInfo._is_always_on_thinking_model, "
"which no Baseten route reaches"
)
with pytest.raises(litellm.UnsupportedParamsError):
litellm.utils.get_optional_params(
model="zai-org/GLM-5.3",
custom_llm_provider="baseten",
parallel_tool_calls=True,
reasoning_effort="high",
drop_params=False,
)

5
tests/test_litellm/test_cost_calculator.py
View file

@ -175,6 +175,11 @@ def test_wandb_model_api_pricing_entries(_local_model_cost_map):
expected_pricing = {
"wandb/moonshotai/Kimi-K2.5": (6e-07, 3e-06),
"wandb/MiniMaxAI/MiniMax-M2.5": (3e-07, 1.2e-06),
"wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": (1e-07, 1e-07),
"wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": (1e-07, 1e-07),
"wandb/deepseek-ai/DeepSeek-R1-0528": (1.35e-06, 5.4e-06),
"wandb/deepseek-ai/DeepSeek-V3-0324": (1.14e-06, 2.75e-06),
"wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": (1.7e-07, 6.6e-07),
}
for model_name, (input_cost, output_cost) in expected_pricing.items():

28
tests/test_litellm/test_daybreak_model_metadata.py
View file

@ -14,6 +14,17 @@ DAYBREAK_MODELS = (
)
BLUE_ALIAS = "daybreak-blue-latest"
BLUE_SNAPSHOT = "gpt-5.6-sol"
OFFICIAL_ALIAS_SNAPSHOTS = (
("gpt-daybreak-blue-latest", "gpt-5.6-sol"),
("gpt-daybreak-red-latest", "gpt-5.6-cyber"),
)
PRICE_FIELDS = (
"input_cost_per_token",
"output_cost_per_token",
"cache_read_input_token_cost",
"input_cost_per_token_above_272k_tokens",
"output_cost_per_token_above_272k_tokens",
)
def _load(path):
@ -44,7 +55,22 @@ def test_blue_alias_matches_its_snapshot_computer_use():
assert cost_map[BLUE_SNAPSHOT]["supports_computer_use"] is True
@pytest.mark.parametrize("model", (*DAYBREAK_MODELS, BLUE_SNAPSHOT))
@pytest.mark.parametrize(("alias", "snapshot"), OFFICIAL_ALIAS_SNAPSHOTS)
def test_official_alias_tracks_snapshot(alias, snapshot):
cost_map = _load(MAIN_PATH)
alias_info = cost_map[alias]
snapshot_info = cost_map[snapshot]
assert alias_info["supported_endpoints"] == ["/v1/responses"]
assert alias_info["mode"] == "responses"
assert alias_info["source"] == f"https://developers.openai.com/api/docs/models/{alias}"
assert {field: alias_info.get(field) for field in PRICE_FIELDS} == {
field: snapshot_info.get(field) for field in PRICE_FIELDS
}
assert alias_info["max_output_tokens"] == snapshot_info["max_output_tokens"]
@pytest.mark.parametrize("model", (*DAYBREAK_MODELS, BLUE_SNAPSHOT, *(alias for alias, _ in OFFICIAL_ALIAS_SNAPSHOTS)))
def test_backup_matches_main(model):
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)

19
tests/test_litellm/test_gpt_image_cost_calculator.py
View file

@ -172,8 +172,7 @@ class TestGPTImageCostCalculator:
image_tokens=500,
),
completion_tokens_details=CompletionTokensDetailsWrapper(
text_tokens=1000,
image_tokens=4000,
image_tokens=5000,
),
)
@ -189,12 +188,7 @@ class TestGPTImageCostCalculator:
custom_llm_provider="openai",
)
# GPT Image 2 pricing:
# Text input: 100 * $5/1M = 0.0005
# Image input: 500 * $8/1M = 0.004
# Text output: 1000 * $10/1M = 0.01
# Image output: 4000 * $30/1M = 0.12
expected_cost = 0.0005 + 0.004 + 0.01 + 0.12
expected_cost = 100 * 5e-6 + 500 * 8e-6 + 5000 * 3e-5
assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}"
@ -429,10 +423,7 @@ class TestGPTImage2OutputImageTokensNoBreakdown:
f"are likely being priced at the text output_cost_per_token rate."
)
def test_gpt_image_2_chat_usage_without_breakdown_is_costed_not_zero(self):
"""A chat ``Usage`` with ``completion_tokens_details=None`` must still be
costed via ``generic_cost_per_token`` (output at the text rate) rather than
erroring or silently returning 0.0."""
def test_gpt_image_2_chat_usage_without_breakdown_uses_image_rate(self):
from litellm.llms.openai.image_generation.cost_calculator import (
cost_calculator,
)
@ -460,9 +451,7 @@ class TestGPTImage2OutputImageTokensNoBreakdown:
custom_llm_provider="openai",
)
# No output breakdown -> output priced at the text rate (output_cost_per_token):
# text in 100*$5/1M + image in 500*$8/1M + output 5000*$10/1M
expected_cost = 100 * 5e-6 + 500 * 8e-6 + 5000 * 1e-5
expected_cost = 100 * 5e-6 + 500 * 8e-6 + 5000 * 3e-5
assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}"

16
tests/test_litellm/test_utils.py
View file

@ -440,7 +440,7 @@ def test_gpt_image_2_provider_and_model_info(local_model_cost_map):
assert model_info["mode"] == "image_generation"
assert model_info["input_cost_per_token"] == 5e-06
assert model_info["input_cost_per_image_token"] == 8e-06
assert model_info["output_cost_per_token"] == 1e-05
assert model_info["output_cost_per_token"] == 0
assert model_info["output_cost_per_image_token"] == 3e-05
assert (
"/v1/images/generations"
@ -482,7 +482,7 @@ def test_azure_gpt_image_2_model_info(local_model_cost_map):
assert model_info["mode"] == "image_generation"
assert model_info["input_cost_per_token"] == 5e-06
assert model_info["input_cost_per_image_token"] == 8e-06
assert model_info["output_cost_per_token"] == 1e-05
assert model_info["output_cost_per_token"] == 0
assert model_info["output_cost_per_image_token"] == 3e-05
@ -906,7 +906,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"cache_creation_input_audio_token_cost": {"type": "number"},
"cache_creation_input_token_cost": {"type": "number"},
"cache_creation_input_token_cost_above_1hr": {"type": "number"},
"cache_creation_input_token_cost_above_128k_tokens": {"type": "number"},
"cache_creation_input_token_cost_above_200k_tokens": {"type": "number"},
"cache_creation_input_token_cost_above_256k_tokens": {"type": "number"},
"cache_creation_input_token_cost_above_272k_tokens": {"type": "number"},
"cache_creation_input_token_cost_above_272k_tokens_flex": {
"type": "number"
@ -917,7 +919,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"cache_creation_input_token_cost_flex": {"type": "number"},
"cache_creation_input_token_cost_priority": {"type": "number"},
"cache_read_input_token_cost": {"type": "number"},
"cache_read_input_token_cost_above_128k_tokens": {"type": "number"},
"cache_read_input_token_cost_above_200k_tokens": {"type": "number"},
"cache_read_input_token_cost_above_256k_tokens": {"type": "number"},
"cache_read_input_token_cost_above_272k_tokens": {"type": "number"},
"cache_read_input_token_cost_above_272k_tokens_flex": {
"type": "number"
@ -2790,7 +2794,7 @@ def test_model_info_for_openrouter_kimi_k2_5():
Model properties from OpenRouter API:
- context_length: 262144
- pricing: prompt=$0.0000006, completion=$0.000003, input_cache_read=$0.0000001
- pricing: prompt=$0.00000045, completion=$0.00000225, input_cache_read=$0.00000007
- modality: text+image->text (supports vision)
- supports: tool_choice, tools (function calling)
"""
@ -2815,9 +2819,9 @@ def test_model_info_for_openrouter_kimi_k2_5():
assert model_info["max_tokens"] == 262144
# Verify pricing
assert model_info["input_cost_per_token"] == 6e-07
assert model_info["output_cost_per_token"] == 3e-06
assert model_info["cache_read_input_token_cost"] == 1e-07
assert model_info["input_cost_per_token"] == 4.5e-07
assert model_info["output_cost_per_token"] == 2.25e-06
assert model_info["cache_read_input_token_cost"] == 7e-08
# Verify capabilities
assert model_info["supports_vision"] is True