mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
test: keep tests that survive correct cost-map updates
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
a428cc8d84
commit
2586b21893
46 changed files with 1771 additions and 116 deletions
|
|
@ -2087,95 +2087,6 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
|
|||
assert round(result, 6) == round(expected, 6)
|
||||
|
||||
|
||||
def test_a_pinned_billing_time_prices_the_totals_and_the_reported_rates_at_one_moment(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"off-peak-model",
|
||||
{
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"off_peak_pricing": {
|
||||
"hours_utc": "02:00-03:00",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 5e-6,
|
||||
},
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
},
|
||||
)
|
||||
usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
|
||||
|
||||
with pinned_billing_time(datetime(2026, 1, 1, 2, 30, tzinfo=timezone.utc)):
|
||||
off_peak_prompt_cost, off_peak_completion_cost = generic_cost_per_token(
|
||||
model="off-peak-model", usage=usage, custom_llm_provider="openai"
|
||||
)
|
||||
off_peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage)
|
||||
with pinned_billing_time(datetime(2026, 1, 1, 12, 30, tzinfo=timezone.utc)):
|
||||
peak_prompt_cost, peak_completion_cost = generic_cost_per_token(
|
||||
model="off-peak-model", usage=usage, custom_llm_provider="openai"
|
||||
)
|
||||
peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage)
|
||||
|
||||
assert off_peak_rates.input_cost_per_token == pytest.approx(1e-6)
|
||||
assert peak_rates.input_cost_per_token == pytest.approx(3e-6)
|
||||
assert off_peak_prompt_cost == pytest.approx(1000 * off_peak_rates.input_cost_per_token)
|
||||
assert off_peak_completion_cost == pytest.approx(500 * off_peak_rates.output_cost_per_token)
|
||||
assert peak_prompt_cost == pytest.approx(1000 * peak_rates.input_cost_per_token)
|
||||
assert peak_completion_cost == pytest.approx(500 * peak_rates.output_cost_per_token)
|
||||
|
||||
|
||||
def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(_local_model_cost_map, monkeypatch):
|
||||
from litellm.llms.anthropic.cost_calculation import cost_per_token as anthropic_cost_per_token
|
||||
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
|
||||
model = "claude-test-geo-breakdown-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
model: {
|
||||
"input_cost_per_token": 5e-6,
|
||||
"output_cost_per_token": 25e-6,
|
||||
"cache_creation_input_token_cost": 6.25e-6,
|
||||
"cache_read_input_token_cost": 0.5e-6,
|
||||
"litellm_provider": "anthropic",
|
||||
"max_tokens": 8192,
|
||||
"provider_specific_entry": {"us": 1.1},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def make_usage() -> Usage:
|
||||
return Usage(
|
||||
prompt_tokens=10_000,
|
||||
completion_tokens=500,
|
||||
total_tokens=10_500,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
cached_tokens=2_000,
|
||||
cache_creation_tokens=6_000,
|
||||
),
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200, text_tokens=300),
|
||||
)
|
||||
|
||||
base_usage = make_usage()
|
||||
geo_usage = make_usage()
|
||||
geo_usage.inference_geo = "us"
|
||||
|
||||
base = get_token_type_cost_breakdown(model=model, custom_llm_provider="anthropic", usage=base_usage)
|
||||
geo = get_token_type_cost_breakdown(model=model, custom_llm_provider="anthropic", usage=geo_usage)
|
||||
|
||||
assert base.cache_read_cost == pytest.approx(2_000 * 0.5e-6)
|
||||
assert base.cache_creation_cost == pytest.approx(6_000 * 6.25e-6)
|
||||
assert geo.cache_read_cost == pytest.approx(base.cache_read_cost * 1.1)
|
||||
assert geo.cache_creation_cost == pytest.approx(base.cache_creation_cost * 1.1)
|
||||
assert geo.reasoning_cost == pytest.approx(base.reasoning_cost * 1.1)
|
||||
|
||||
prompt_cost, completion_cost = anthropic_cost_per_token(model=model, usage=geo_usage)
|
||||
text_input_cost = 2_000 * 5e-6 * 1.1
|
||||
text_output_cost = 300 * 25e-6 * 1.1
|
||||
assert text_input_cost + geo.cache_read_cost + geo.cache_creation_cost == pytest.approx(prompt_cost)
|
||||
assert text_output_cost + geo.reasoning_cost == pytest.approx(completion_cost)
|
||||
|
||||
|
||||
def test_service_tier_ultrafast_pricing():
|
||||
"""An ultrafast request bills the *_ultrafast rates for all token types.
|
||||
|
||||
|
|
@ -3134,6 +3045,46 @@ def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeyp
|
|||
assert breakdown.reasoning_cost == pytest.approx(200 * rates.output_cost_per_reasoning_token)
|
||||
|
||||
|
||||
def test_a_pinned_billing_time_prices_the_totals_and_the_reported_rates_at_one_moment(monkeypatch):
|
||||
"""Totals and reported rates resolve off-peak pricing on separate paths that each read the
|
||||
clock, so a window opening between the two reads used to leave them describing one request
|
||||
at two different prices. Pinned, both must answer for the pinned moment."""
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"off-peak-model",
|
||||
{
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"off_peak_pricing": {
|
||||
"hours_utc": "02:00-03:00",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 5e-6,
|
||||
},
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
},
|
||||
)
|
||||
usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
|
||||
|
||||
with pinned_billing_time(datetime(2026, 1, 1, 2, 30, tzinfo=timezone.utc)):
|
||||
off_peak_prompt_cost, off_peak_completion_cost = generic_cost_per_token(
|
||||
model="off-peak-model", usage=usage, custom_llm_provider="openai"
|
||||
)
|
||||
off_peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage)
|
||||
with pinned_billing_time(datetime(2026, 1, 1, 12, 30, tzinfo=timezone.utc)):
|
||||
peak_prompt_cost, peak_completion_cost = generic_cost_per_token(
|
||||
model="off-peak-model", usage=usage, custom_llm_provider="openai"
|
||||
)
|
||||
peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage)
|
||||
|
||||
assert off_peak_rates.input_cost_per_token == pytest.approx(1e-6)
|
||||
assert peak_rates.input_cost_per_token == pytest.approx(3e-6)
|
||||
assert off_peak_prompt_cost == pytest.approx(1000 * off_peak_rates.input_cost_per_token)
|
||||
assert off_peak_completion_cost == pytest.approx(500 * off_peak_rates.output_cost_per_token)
|
||||
assert peak_prompt_cost == pytest.approx(1000 * peak_rates.input_cost_per_token)
|
||||
assert peak_completion_cost == pytest.approx(500 * peak_rates.output_cost_per_token)
|
||||
|
||||
|
||||
def test_the_token_type_breakdown_carries_the_rates_it_billed_at(monkeypatch):
|
||||
"""Callers that report both the lines and the rates read the rates off the breakdown rather than
|
||||
resolving them a second time, so the breakdown has to hand back exactly what it billed at."""
|
||||
|
|
@ -3341,6 +3292,68 @@ def test_token_type_cost_breakdown_applies_vertex_regional_uplift(_local_model_c
|
|||
assert text_input_cost + regional.cache_read_cost == pytest.approx(prompt_cost)
|
||||
|
||||
|
||||
def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(_local_model_cost_map, monkeypatch):
|
||||
"""
|
||||
Anthropic's regional (geo) uplift lives in provider_specific_entry and is
|
||||
applied to every token type in the totals, so the per-type breakdown must
|
||||
scale its cache and reasoning line items by it too. Otherwise the logged
|
||||
cache costs stay at the base rate and the cache uplift is misattributed to
|
||||
plain input for exactly the cache-heavy regional traffic the uplift targets.
|
||||
"""
|
||||
from litellm.llms.anthropic.cost_calculation import (
|
||||
cost_per_token as anthropic_cost_per_token,
|
||||
)
|
||||
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
|
||||
model = "claude-test-geo-breakdown-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
model: {
|
||||
"input_cost_per_token": 5e-6,
|
||||
"output_cost_per_token": 25e-6,
|
||||
"cache_creation_input_token_cost": 6.25e-6,
|
||||
"cache_read_input_token_cost": 0.5e-6,
|
||||
"litellm_provider": "anthropic",
|
||||
"max_tokens": 8192,
|
||||
"provider_specific_entry": {"us": 1.1},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def make_usage() -> Usage:
|
||||
return Usage(
|
||||
prompt_tokens=10_000,
|
||||
completion_tokens=500,
|
||||
total_tokens=10_500,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
cached_tokens=2_000,
|
||||
cache_creation_tokens=6_000,
|
||||
),
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200, text_tokens=300),
|
||||
)
|
||||
|
||||
base_usage = make_usage()
|
||||
geo_usage = make_usage()
|
||||
geo_usage.inference_geo = "us"
|
||||
|
||||
base = get_token_type_cost_breakdown(model=model, custom_llm_provider="anthropic", usage=base_usage)
|
||||
geo = get_token_type_cost_breakdown(model=model, custom_llm_provider="anthropic", usage=geo_usage)
|
||||
|
||||
assert base.cache_read_cost == pytest.approx(2_000 * 0.5e-6)
|
||||
assert base.cache_creation_cost == pytest.approx(6_000 * 6.25e-6)
|
||||
assert geo.cache_read_cost == pytest.approx(base.cache_read_cost * 1.1)
|
||||
assert geo.cache_creation_cost == pytest.approx(base.cache_creation_cost * 1.1)
|
||||
assert geo.reasoning_cost == pytest.approx(base.reasoning_cost * 1.1)
|
||||
|
||||
# The uplifted breakdown must still reconcile with the uplifted totals.
|
||||
prompt_cost, completion_cost = anthropic_cost_per_token(model=model, usage=geo_usage)
|
||||
text_input_cost = 2_000 * 5e-6 * 1.1
|
||||
text_output_cost = 300 * 25e-6 * 1.1
|
||||
assert text_input_cost + geo.cache_read_cost + geo.cache_creation_cost == pytest.approx(prompt_cost)
|
||||
assert text_output_cost + geo.reasoning_cost == pytest.approx(completion_cost)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("details_as_dict", [True, False])
|
||||
def test_image_response_input_image_tokens_priced_at_image_rate(details_as_dict):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -134,6 +134,24 @@ def test_finalize_with_no_block_clears_rules():
|
|||
set_fallback_generalizations(previous)
|
||||
|
||||
|
||||
def test_shipped_backup_carries_the_claude_routing_rules():
|
||||
"""The bundled backup must ship the Claude routing rules so a fresh install
|
||||
(or an offline fallback) routes unknown Claude models without code changes.
|
||||
Bedrock-syntax ids must hit the bedrock rule before the bare-id Anthropic rule."""
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
rules = backup.get(FALLBACK_GENERALIZATIONS_KEY, {}).get("rules", [])
|
||||
names = [r.get("name") for r in rules]
|
||||
assert names.index("bedrock-claude-ids") < names.index("anthropic-claude-ids")
|
||||
|
||||
previous = list(get_fallback_generalization_rules())
|
||||
try:
|
||||
set_fallback_generalizations(rules)
|
||||
assert match_routing_generalization("claude-opus-4-9") == "anthropic"
|
||||
assert match_routing_generalization("global.anthropic.claude-opus-4-9") == "bedrock"
|
||||
finally:
|
||||
set_fallback_generalizations(previous)
|
||||
|
||||
|
||||
def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace():
|
||||
"""Routing rules decide ``litellm_provider`` for otherwise-unknown ids, and the
|
||||
proxy's wildcard access check (``can_key_call_model`` with a ``bedrock/*`` key)
|
||||
|
|
@ -170,6 +188,43 @@ def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace():
|
|||
set_fallback_generalizations(previous)
|
||||
|
||||
|
||||
def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
|
||||
"""Adaptive thinking is data, not code. The bundled backup must carry
|
||||
supports_adaptive_thinking on genuine Claude >= 4.6 entries (every provider
|
||||
route) and on the version-gated anthropic-claude-adaptive-thinking rule for
|
||||
unmapped future Claudes, while leaving the dated Claude 4.0 names
|
||||
("...-4-20250514") unflagged so a date can never be mistaken for a 4.6+ minor
|
||||
version. The version-neutral claude-family-baseline capability rule must not flag
|
||||
it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule
|
||||
carries only its delta; capability unioning stacks it onto the baseline, so the
|
||||
baseline block is never duplicated across rules and no rule needs ``extends``."""
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
|
||||
rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"]
|
||||
baseline_rule = next(r for r in rules if r.get("name") == "claude-family-baseline")
|
||||
adaptive_rule = next(r for r in rules if r.get("name") == "claude-adaptive-thinking")
|
||||
assert "supports_adaptive_thinking" not in baseline_rule["model_info"]
|
||||
assert "litellm_provider" not in baseline_rule["model_info"]
|
||||
assert adaptive_rule["model_info"] == {"supports_adaptive_thinking": True}
|
||||
assert all("extends" not in r for r in rules)
|
||||
|
||||
for adaptive in [
|
||||
"anthropic.claude-opus-4-8",
|
||||
"vertex_ai/claude-opus-4-6@default",
|
||||
"us.anthropic.claude-sonnet-4-6",
|
||||
"openrouter/anthropic/claude-opus-4.7",
|
||||
"azure_ai/claude-opus-4-7",
|
||||
]:
|
||||
assert backup[adaptive]["supports_adaptive_thinking"] is True, adaptive
|
||||
|
||||
for non_adaptive in [
|
||||
"claude-opus-4-20250514",
|
||||
"us.anthropic.claude-opus-4-20250514-v1:0",
|
||||
"claude-opus-4-5",
|
||||
]:
|
||||
assert "supports_adaptive_thinking" not in backup[non_adaptive], non_adaptive
|
||||
|
||||
|
||||
# OpenRouter headline rates from GET https://openrouter.ai/api/v1/models.
|
||||
# These were the catalog values that disagreed with that API (and, for the
|
||||
# two spotlight models, the public model pages that their source fields cite).
|
||||
|
|
|
|||
|
|
@ -72,6 +72,22 @@ def test_azure_ai_catalog_name_prices_the_same_in_any_casing(catalog_name: str)
|
|||
assert upper_cost == lowercase_cost
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize("catalog_name", GROK_4_20_NAMES)
|
||||
def test_azure_ai_grok_4_20_bills_cached_prompt_tokens_at_the_input_price(catalog_name: str) -> None:
|
||||
uncached_prompt_cost, _ = cost_per_token(
|
||||
model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0
|
||||
)
|
||||
cached_prompt_cost, _ = cost_per_token(
|
||||
model=f"azure_ai/{catalog_name}",
|
||||
prompt_tokens=A_MILLION,
|
||||
completion_tokens=0,
|
||||
cache_read_input_tokens=A_MILLION,
|
||||
)
|
||||
assert uncached_prompt_cost > 0
|
||||
assert cached_prompt_cost == pytest.approx(uncached_prompt_cost)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None:
|
||||
one_second_cost: Final = _whisper_transcription_cost(1)
|
||||
|
|
@ -80,6 +96,14 @@ def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None:
|
|||
assert one_hour_cost == pytest.approx(AN_HOUR_IN_SECONDS * one_second_cost)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("catalog_name", CATALOG_NAMES)
|
||||
def test_azure_ai_catalog_entry_source_and_backup_match(catalog_name: str) -> None:
|
||||
main_entry = _cost_map_entry(MAIN_COST_MAP, catalog_name)
|
||||
backup_entry = _cost_map_entry(BACKUP_COST_MAP, catalog_name)
|
||||
|
||||
assert backup_entry == main_entry
|
||||
|
||||
|
||||
def test_azure_ai_model_router_spellings_share_one_entry() -> None:
|
||||
underscore_entry = _cost_map_entry(MAIN_COST_MAP, "model_router")
|
||||
hyphen_entry = _cost_map_entry(MAIN_COST_MAP, "model-router")
|
||||
|
|
|
|||
28
tests/test_litellm/llms/cohere/ocr/test_cohere_parse_cost.py
Normal file
28
tests/test_litellm/llms/cohere/ocr/test_cohere_parse_cost.py
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[5]
|
||||
COST_MAPS = [
|
||||
REPO_ROOT / "model_prices_and_context_window.json",
|
||||
REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json",
|
||||
]
|
||||
MODELS = [("cohere/parse-v5.0", "cohere"), ("azure_ai/Cohere-parse-v5", "azure_ai")]
|
||||
|
||||
|
||||
def _ocr_response(model: str, pages_processed: int) -> OCRResponse:
|
||||
return OCRResponse(
|
||||
pages=[OCRPage(index=i, markdown=f"page {i}") for i in range(pages_processed)],
|
||||
model=model,
|
||||
usage_info=OCRUsageInfo(pages_processed=pages_processed),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model, provider", MODELS)
|
||||
def test_model_info_resolves_ocr_mode_and_price(local_model_cost_map, model: str, provider: str) -> None:
|
||||
info = litellm.get_model_info(model=model, custom_llm_provider=provider)
|
||||
|
||||
assert info["mode"] == "ocr"
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
|
@ -162,6 +163,29 @@ def test_legacy_endpoint_names_still_resolve(local_model_cost_map: None) -> None
|
|||
assert completion_cost == pytest.approx(100 * info["output_cost_per_token"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", NEW_MODELS)
|
||||
def test_new_models_carry_cache_pricing(local_model_cost_map: None, model: str) -> None:
|
||||
info: Final = _model_info(model)
|
||||
|
||||
assert info["input_cost_per_token"] > 0
|
||||
assert info["output_cost_per_token"] > 0
|
||||
assert info["cache_creation_input_token_cost"] > info["input_cost_per_token"]
|
||||
assert info["cache_read_input_token_cost"] < info["input_cost_per_token"]
|
||||
assert info["supports_prompt_caching"] is True
|
||||
|
||||
|
||||
def test_every_priced_databricks_model_declares_cache_rates(local_model_cost_map: None) -> None:
|
||||
undeclared: Final = [
|
||||
model
|
||||
for model, info in litellm.model_cost.items()
|
||||
if model.startswith("databricks/")
|
||||
and info.get("input_cost_per_token") is not None
|
||||
and any(info.get(field) is None for field in CACHE_FIELDS)
|
||||
]
|
||||
|
||||
assert undeclared == []
|
||||
|
||||
|
||||
def test_models_without_a_cache_discount_bill_cache_tokens_at_the_input_rate(
|
||||
local_model_cost_map: None,
|
||||
) -> None:
|
||||
|
|
@ -180,6 +204,33 @@ def test_models_without_a_cache_discount_bill_cache_tokens_at_the_input_rate(
|
|||
assert prompt_cost > 8000 * info["input_cost_per_token"]
|
||||
|
||||
|
||||
def test_every_model_without_published_cache_dbu_bills_cache_at_its_own_input_rate(
|
||||
local_model_cost_map: None,
|
||||
) -> None:
|
||||
without_published_rates: Final = [
|
||||
model
|
||||
for model, info in litellm.model_cost.items()
|
||||
if model.startswith("databricks/")
|
||||
and info.get("input_cost_per_token")
|
||||
and model not in PUBLISHED_DBU_PER_MILLION
|
||||
]
|
||||
|
||||
for model in without_published_rates:
|
||||
info = _model_info(model)
|
||||
for field in CACHE_FIELDS:
|
||||
assert info[field] == pytest.approx(info["input_cost_per_token"]), (model, field)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", NEW_MODELS)
|
||||
def test_backup_price_map_matches_main(model: str) -> None:
|
||||
main_cost: Final = json.loads(MAIN_PRICES.read_text())
|
||||
backup_cost: Final = json.loads(BACKUP_PRICES.read_text())
|
||||
|
||||
assert model in main_cost
|
||||
assert model in backup_cost
|
||||
assert backup_cost[model] == main_cost[model]
|
||||
|
||||
|
||||
def test_sonnet_5_ships_standard_rates_not_introductory(local_model_cost_map: None) -> None:
|
||||
sonnet_5: Final = _model_info("databricks/databricks-claude-sonnet-5")
|
||||
sonnet_4_6: Final = _model_info("databricks/databricks-claude-sonnet-4-6")
|
||||
|
|
|
|||
|
|
@ -22,6 +22,16 @@ def _usage(prompt_tokens: int, cached_tokens: int, completion_tokens: int) -> Us
|
|||
)
|
||||
|
||||
|
||||
def test_warm_call_cheaper_than_cold_call():
|
||||
prompt_tokens = 7036
|
||||
completion_tokens = 8
|
||||
|
||||
cold_prompt_cost, _ = cost_per_token(model=MODEL, usage=_usage(prompt_tokens, 16, completion_tokens))
|
||||
warm_prompt_cost, _ = cost_per_token(model=MODEL, usage=_usage(prompt_tokens, 7020, completion_tokens))
|
||||
|
||||
assert warm_prompt_cost < cold_prompt_cost
|
||||
|
||||
|
||||
OFF_PEAK_MODEL = "accounts/fireworks/models/off-peak-test"
|
||||
OFF_PEAK_WINDOW = "14:00-00:00"
|
||||
INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
"""
|
||||
Regression test for Fireworks Kimi K2.5 / K2.6 / K2.7 context and output limits.
|
||||
|
||||
Fireworks publishes a 262144-token context window for every Kimi K2.5, K2.6 and
|
||||
K2.7 model, but caps generation well below that. A previous bulk edit had flattened
|
||||
max_output_tokens/max_tokens to 262144 (equal to the context window), which let the
|
||||
pre-call context-window check admit requests asking for a full 262144-token
|
||||
completion that Fireworks then rejects. These assertions pin the corrected per-alias
|
||||
limits so a future bulk edit can't silently flatten them again.
|
||||
"""
|
||||
|
||||
import json
|
||||
from importlib.resources import files
|
||||
|
||||
import pytest
|
||||
|
||||
CONTEXT_WINDOW = 262144
|
||||
OUTPUT_LIMIT = 32768
|
||||
|
||||
KIMI_ALIASES = (
|
||||
"fireworks_ai/kimi-k2p5",
|
||||
"fireworks_ai/kimi-k2p6",
|
||||
"fireworks_ai/kimi-k2p6-fast",
|
||||
"fireworks_ai/kimi-k2p7-code",
|
||||
"fireworks_ai/kimi-k2p7-code-fast",
|
||||
"fireworks_ai/accounts/fireworks/models/kimi-k2p5",
|
||||
"fireworks_ai/accounts/fireworks/models/kimi-k2p6",
|
||||
"fireworks_ai/accounts/fireworks/models/kimi-k2p7-code",
|
||||
"fireworks_ai/accounts/fireworks/routers/kimi-k2p6-fast",
|
||||
"fireworks_ai/accounts/fireworks/routers/kimi-k2p7-code-fast",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def use_local_model_cost_map():
|
||||
monkeypatch = pytest.MonkeyPatch()
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
|
||||
import litellm
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
original_model_cost = litellm.model_cost
|
||||
litellm.model_cost = json.loads(
|
||||
files("litellm").joinpath("model_prices_and_context_window_backup.json").read_text(encoding="utf-8")
|
||||
)
|
||||
litellm.get_model_info.cache_clear()
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
try:
|
||||
yield litellm
|
||||
finally:
|
||||
litellm.model_cost = original_model_cost
|
||||
litellm.get_model_info.cache_clear()
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
monkeypatch.undo()
|
||||
40
tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py
Normal file
40
tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
"""
|
||||
Cost tests for Mistral OCR models against the real litellm cost map
|
||||
(no monkeypatching of get_model_info). These regress the pricing entries
|
||||
for mistral-ocr-4-0 and mistral-ocr-latest, which now both resolve to
|
||||
OCR 4 at $4 / 1000 pages.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo
|
||||
|
||||
OCR4_COST_PER_PAGE = 0.004
|
||||
OCR4_ANNOTATION_COST_PER_PAGE = 0.005
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[5]
|
||||
MAIN_COST_MAP = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_COST_MAP = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
OCR3_MODEL = "mistral/mistral-ocr-2512"
|
||||
OCR3_COST_PER_PAGE = 0.002
|
||||
OCR3_ANNOTATION_COST_PER_PAGE = 0.003
|
||||
|
||||
AZURE_DOC_AI_MODEL = "azure_ai/mistral-document-ai-2512"
|
||||
AZURE_DOC_AI_COST_PER_PAGE = 0.003
|
||||
|
||||
|
||||
def _ocr_response(model: str, pages_processed: int) -> OCRResponse:
|
||||
return OCRResponse(
|
||||
pages=[OCRPage(index=i, markdown=f"page {i}") for i in range(pages_processed)],
|
||||
model=model,
|
||||
usage_info=OCRUsageInfo(pages_processed=pages_processed),
|
||||
)
|
||||
|
||||
|
||||
def _annotated_ocr_response(model: str, pages_processed: int | None, annotation_pages: int) -> OCRResponse:
|
||||
return OCRResponse(
|
||||
pages=[],
|
||||
model=model,
|
||||
usage_info=OCRUsageInfo(pages_processed=pages_processed, pages_processed_annotation=annotation_pages),
|
||||
)
|
||||
|
|
@ -154,6 +154,34 @@ class TestSCXAIModelMetadata:
|
|||
with open(json_path) as f:
|
||||
return json.load(f)
|
||||
|
||||
def test_scx_ai_models_registered_with_correct_metadata(self):
|
||||
model_cost = self._load(("model_prices_and_context_window.json",))
|
||||
for model in self.SCX_MODELS:
|
||||
info = model_cost.get(model)
|
||||
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
|
||||
assert info["litellm_provider"] == "scx-ai"
|
||||
assert info["mode"] == "chat"
|
||||
assert info["input_cost_per_token"] > 0
|
||||
assert info["output_cost_per_token"] > 0
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_tool_choice"] is True
|
||||
assert info["supports_reasoning"] is True
|
||||
assert info["supports_response_schema"] is True
|
||||
assert info.get("supports_vision", False) is (model in self.VISION_MODELS)
|
||||
|
||||
assert info["supports_prompt_caching"] is True
|
||||
assert 0 < info["cache_read_input_token_cost"] < info["input_cost_per_token"]
|
||||
|
||||
assert info["max_tokens"] == info["max_output_tokens"]
|
||||
assert info["max_input_tokens"] >= 1_000_000
|
||||
|
||||
def test_scx_ai_models_synced_to_backup(self):
|
||||
model_cost = self._load(("model_prices_and_context_window.json",))
|
||||
backup = self._load(("litellm", "model_prices_and_context_window_backup.json"))
|
||||
for model in self.SCX_MODELS:
|
||||
assert model in backup, f"{model} missing from backup json"
|
||||
assert backup[model] == model_cost[model], f"{model} differs between root and backup json"
|
||||
|
||||
|
||||
class TestSCXAIDashboardRegistration:
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm import ModelResponse
|
|||
from litellm.cost_calculator import cost_per_token
|
||||
from litellm.llms.perplexity.chat.transformation import PerplexityChatConfig
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
||||
class TestPerplexityIntegration:
|
||||
|
|
@ -55,6 +56,23 @@ class TestPerplexityIntegration:
|
|||
}
|
||||
}
|
||||
|
||||
def test_model_info_includes_custom_fields(self):
|
||||
"""Test that get_model_info returns the custom Perplexity cost fields."""
|
||||
model_info = get_model_info(model="sonar-deep-research", custom_llm_provider="perplexity")
|
||||
|
||||
# Verify custom fields are included
|
||||
required_fields = [
|
||||
"citation_cost_per_token",
|
||||
"search_context_cost_per_query",
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"output_cost_per_reasoning_token",
|
||||
]
|
||||
|
||||
for field in required_fields:
|
||||
assert field in model_info, f"Missing field: {field}"
|
||||
assert model_info[field] is not None, f"Null value for field: {field}"
|
||||
|
||||
def test_various_citation_sizes(self):
|
||||
"""Test cost calculation with various citation sizes."""
|
||||
config = PerplexityChatConfig()
|
||||
|
|
|
|||
1
tests/test_litellm/llms/tencent/test_cost_calculator.py
Normal file
1
tests/test_litellm/llms/tencent/test_cost_calculator.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -452,6 +452,44 @@ def test_vertex_claude_completion_does_not_mutate_shared_extra_headers():
|
|||
assert shared_extra_headers == {}, "extra_headers must not be mutated by completion()"
|
||||
|
||||
|
||||
def test_messages_thinking_shape_follows_exact_vertex_entry_flag(local_model_cost_map, monkeypatch):
|
||||
"""The Vertex messages config must probe capabilities under ``vertex_ai`` so an
|
||||
operator setting ``supports_adaptive_thinking: false`` on the exact
|
||||
``vertex_ai/claude-opus-4-8`` entry beats the unmodified ``anthropic`` entry.
|
||||
With the inherited ``"anthropic"`` provider default the flip was ignored and
|
||||
the transform kept emitting ``thinking.type='adaptive'``."""
|
||||
import litellm
|
||||
|
||||
config = VertexAIPartnerModelsAnthropicMessagesConfig()
|
||||
|
||||
def transform():
|
||||
return config.transform_anthropic_messages_request(
|
||||
model="claude-opus-4-8",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
anthropic_messages_optional_request_params={
|
||||
"max_tokens": 4096,
|
||||
"reasoning_effort": "medium",
|
||||
},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
result = transform()
|
||||
assert result.get("thinking") == {"type": "adaptive", "display": "summarized"}
|
||||
assert result.get("output_config") == {"effort": "medium"}
|
||||
|
||||
monkeypatch.setitem(litellm.model_cost["vertex_ai/claude-opus-4-8"], "supports_adaptive_thinking", False)
|
||||
litellm.get_model_info.cache_clear()
|
||||
assert litellm.model_cost["claude-opus-4-8"]["supports_adaptive_thinking"] is True
|
||||
|
||||
flipped = transform()
|
||||
thinking = flipped.get("thinking")
|
||||
assert isinstance(thinking, dict)
|
||||
assert thinking.get("type") == "enabled"
|
||||
assert isinstance(thinking.get("budget_tokens"), int)
|
||||
assert "output_config" not in flipped
|
||||
|
||||
|
||||
def _vertex_transform(model, messages, system=None):
|
||||
config = VertexAIPartnerModelsAnthropicMessagesConfig()
|
||||
params = {"max_tokens": 256}
|
||||
|
|
@ -547,6 +585,10 @@ class TestVertexAnthropicMidConversationSystem:
|
|||
|
||||
|
||||
def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag():
|
||||
"""Exact cost-map hits win over the ``claude-mid-conversation-system``
|
||||
fallback rule, so a ``vertex_ai`` Claude 4.8+/5 entry missing the flag would
|
||||
be treated as unsupported and hoist every reminder, collapsing the prompt
|
||||
cache. Every mapped vertex_ai entry the rule matches must carry the flag."""
|
||||
import re
|
||||
|
||||
import litellm
|
||||
|
|
|
|||
55
tests/test_litellm/llms/xai/test_xai_model_registry.py
Normal file
55
tests/test_litellm/llms/xai/test_xai_model_registry.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
"""
|
||||
Registry regression tests for xAI entries in the model cost map.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[4]
|
||||
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
# https://docs.x.ai/developers/model-capabilities/text/multi-agent
|
||||
# "The multi-agent model does not work with the OpenAI Chat Completions API."
|
||||
RESPONSES_ONLY_MODELS = (
|
||||
"xai/grok-4.20-multi-agent-0309",
|
||||
"xai/grok-4.20-multi-agent-beta-0309",
|
||||
)
|
||||
|
||||
MAP_PATHS = (PRICES_PATH, BACKUP_PRICES_PATH)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=[p.name for p in MAP_PATHS])
|
||||
def cost_map(request: pytest.FixtureRequest) -> dict:
|
||||
path = next(p for p in MAP_PATHS if p.name == request.param)
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", RESPONSES_ONLY_MODELS)
|
||||
def test_multi_agent_models_are_responses_only(cost_map: dict, model: str):
|
||||
entry = cost_map[model]
|
||||
assert entry["supported_endpoints"] == ["/v1/responses"]
|
||||
assert entry["mode"] == "responses"
|
||||
|
||||
|
||||
def test_surviving_xai_chat_models_still_serve_chat_completions(cost_map: dict):
|
||||
"""Guard against the removal above over-reaching into live models."""
|
||||
chat_models = [
|
||||
key
|
||||
for key, value in cost_map.items()
|
||||
if isinstance(value, dict) and value.get("litellm_provider") == "xai" and value.get("mode") == "chat"
|
||||
]
|
||||
assert "xai/grok-4.3" in chat_models
|
||||
assert "xai/grok-4.6" in chat_models
|
||||
|
||||
|
||||
def test_both_cost_maps_agree_on_xai_entries():
|
||||
prices = json.loads(PRICES_PATH.read_text(encoding="utf-8"))
|
||||
backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8"))
|
||||
xai_keys = {k for k, v in prices.items() if isinstance(v, dict) and v.get("litellm_provider") == "xai"}
|
||||
assert xai_keys
|
||||
assert {k: prices[k] for k in xai_keys} == {k: backup[k] for k in xai_keys}
|
||||
|
|
@ -98,3 +98,23 @@ def test_redirected_slug_carries_the_target_tier_rates(cost_map: dict, slug: str
|
|||
for field in TIER_COST_FIELDS:
|
||||
assert entry[field] == target[field], field
|
||||
assert {k for k in entry if "_above_" in k} == {k for k in target if "_above_" in k}
|
||||
|
||||
|
||||
def test_both_cost_maps_agree_on_the_redirected_slugs():
|
||||
prices = json.loads(PRICES_PATH.read_text(encoding="utf-8"))
|
||||
backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8"))
|
||||
for slug in (*REDIRECTED_SLUGS, *CODE_SLUGS, REDIRECT_TARGET, CODE_REDIRECT_TARGET):
|
||||
assert prices[slug] == backup[slug], slug
|
||||
|
||||
|
||||
def test_every_retired_chat_slug_is_covered(cost_map: dict):
|
||||
"""The lists above must stay in step with what the registry marks retired."""
|
||||
marked = {
|
||||
key
|
||||
for key, entry in cost_map.items()
|
||||
if isinstance(entry, dict)
|
||||
and entry.get("litellm_provider") == "xai"
|
||||
and "deprecation_date" in entry
|
||||
and entry.get("mode") == "chat"
|
||||
}
|
||||
assert marked == {*REDIRECTED_SLUGS, *CODE_SLUGS}
|
||||
|
|
|
|||
|
|
@ -327,6 +327,29 @@ def test_openai_style_cache_write_tokens_are_netted_out():
|
|||
)
|
||||
|
||||
|
||||
def test_sub_input_cache_write_price_is_an_extra_saving():
|
||||
"""A few models price writes below input; there the premium is a real credit.
|
||||
|
||||
Clamping the premium at zero would silently undercount these, so the subtraction
|
||||
stays signed. ``azure/eu/gpt-4o-2024-11-20`` ships a write price at ~0.5x input.
|
||||
"""
|
||||
model = "azure/eu/gpt-4o-2024-11-20"
|
||||
info = litellm.get_model_info(model=model)
|
||||
input_cost = info["input_cost_per_token"]
|
||||
cheap_write = info["cache_creation_input_token_cost"]
|
||||
assert 0 < cheap_write < input_cost, "fixture drifted: this test needs a model pricing cache writes below input"
|
||||
|
||||
result = compute_savings_spend(
|
||||
model=model,
|
||||
custom_llm_provider=None,
|
||||
compression_saved_tokens=0,
|
||||
gateway_injected_cache=True,
|
||||
usage_object=_caching_usage(read=1000, written=4000),
|
||||
)
|
||||
assert result.prompt_caching == pytest.approx(4000 * (input_cost - cheap_write))
|
||||
assert result.prompt_caching > 0
|
||||
|
||||
|
||||
def test_negative_cache_write_count_clamps_to_zero():
|
||||
"""A malformed negative write count must not be read as a saving."""
|
||||
input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5")
|
||||
|
|
|
|||
37
tests/test_litellm/test_azure_ai_grok_4_3_model_metadata.py
Normal file
37
tests/test_litellm/test_azure_ai_grok_4_3_model_metadata.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import get_model_info
|
||||
|
||||
AZURE_AI_GROK_4_3_MODEL = "azure_ai/grok-4.3"
|
||||
AZURE_AI_GROK_4_3_SOURCE = "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-grok-4-3-on-microsoft-foundry-latest-generation-agentic-capabilities/4517096"
|
||||
|
||||
|
||||
def _load_model_cost(path: Path) -> dict:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reload_model_costs():
|
||||
original_model_cost = litellm.model_cost
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
litellm.model_cost = _load_model_cost(json_path)
|
||||
get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.model_cost = original_model_cost
|
||||
get_model_info.cache_clear()
|
||||
|
||||
|
||||
def test_azure_ai_grok_4_3_backup_matches_main():
|
||||
repo_root = Path(__file__).parents[2]
|
||||
main_path = repo_root / "model_prices_and_context_window.json"
|
||||
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
main_cost = _load_model_cost(main_path)
|
||||
backup_cost = _load_model_cost(backup_path)
|
||||
|
||||
assert backup_cost.get(AZURE_AI_GROK_4_3_MODEL) == main_cost.get(AZURE_AI_GROK_4_3_MODEL)
|
||||
44
tests/test_litellm/test_azure_ai_grok_4_6_model_metadata.py
Normal file
44
tests/test_litellm/test_azure_ai_grok_4_6_model_metadata.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm import cost_per_token, get_model_info
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
REPO_ROOT: Final = Path(__file__).parents[2]
|
||||
MODEL: Final = "azure_ai/grok-4.6"
|
||||
COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
|
||||
|
||||
def _cost_map_entry(path: Path) -> dict[str, object]:
|
||||
return COST_MAP_ADAPTER.validate_json(path.read_bytes())[MODEL]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_azure_ai_grok_4_6_is_priced_and_routed() -> None:
|
||||
routed_model, provider, _, _ = get_llm_provider(model=MODEL)
|
||||
assert (routed_model, provider) == ("grok-4.6", "azure_ai")
|
||||
|
||||
info = get_model_info(model=routed_model, custom_llm_provider=provider)
|
||||
assert info["litellm_provider"] == "azure_ai"
|
||||
assert info["mode"] == "chat"
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_prompt_caching"] is True
|
||||
assert info["supports_reasoning"] is True
|
||||
assert info["supports_response_schema"] is True
|
||||
assert info["supports_tool_choice"] is True
|
||||
assert info["supports_vision"] is True
|
||||
assert info["supports_web_search"] is True
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model=MODEL, prompt_tokens=1_000_000, completion_tokens=1_000_000)
|
||||
assert prompt_cost > 0
|
||||
assert completion_cost > 0
|
||||
|
||||
|
||||
def test_azure_ai_grok_4_6_entry_source_and_backup_match() -> None:
|
||||
main_entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json")
|
||||
backup_entry = _cost_map_entry(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json")
|
||||
|
||||
assert backup_entry == main_entry
|
||||
|
|
@ -1,10 +1,31 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.utils import supports_function_calling, supports_prompt_caching
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[2]
|
||||
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
MODEL = "baseten/zai-org/GLM-5.3"
|
||||
|
||||
INPUT_COST = 1.4e-06
|
||||
CACHED_INPUT_COST = 1.4e-07
|
||||
OUTPUT_COST = 4.4e-06
|
||||
|
||||
|
||||
def _load(path):
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_model_cost_map(monkeypatch):
|
||||
"""Force get_model_info to resolve against the in-repo cost map instead of the
|
||||
remote one fetched at import time, which still carries the pre-merge registry."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
|
@ -12,6 +33,32 @@ def local_model_cost_map(monkeypatch):
|
|||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
def test_baseten_glm_5_3_capabilities_are_visible_to_callers(local_model_cost_map):
|
||||
"""The entry advertises prompt caching and tool calling, so the helpers every
|
||||
caller checks before sending a request must say so too."""
|
||||
assert supports_prompt_caching(model=MODEL) is True
|
||||
assert supports_function_calling(model=MODEL) is True
|
||||
|
||||
info = litellm.get_model_info(model="zai-org/GLM-5.3", custom_llm_provider="baseten")
|
||||
assert info["max_input_tokens"] > 0
|
||||
assert info["max_output_tokens"] > 0
|
||||
|
||||
|
||||
def test_backup_matches_main():
|
||||
"""Ensure the bundled (backup) cost map stays in sync with the canonical file.
|
||||
|
||||
Both keys are asserted present first: comparing two ``.get`` results alone passes
|
||||
just as happily when neither file has the entry at all, which is the exact state
|
||||
this test exists to catch.
|
||||
"""
|
||||
main_cost = _load(MAIN_PATH)
|
||||
backup_cost = _load(BACKUP_PATH)
|
||||
|
||||
assert MODEL in main_cost, f"{MODEL} missing from model_prices_and_context_window.json"
|
||||
assert MODEL in backup_cost, f"{MODEL} missing from model_prices_and_context_window_backup.json"
|
||||
assert backup_cost[MODEL] == main_cost[MODEL], f"{MODEL} differs between main and backup model cost maps"
|
||||
|
||||
|
||||
def test_entry_advertises_only_what_the_baseten_path_accepts(local_model_cost_map):
|
||||
"""The Baseten path rejects unsupported request parameters."""
|
||||
supported = litellm.get_supported_openai_params(model="zai-org/GLM-5.3", custom_llm_provider="baseten")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.constants import bedrock_embedding_models
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[2]
|
||||
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
BASE_MODEL = "twelvelabs.marengo-embed-3-0-v1:0"
|
||||
PROFILE_MODELS = ("us.twelvelabs.marengo-embed-3-0-v1:0", "eu.twelvelabs.marengo-embed-3-0-v1:0")
|
||||
ALL_MODELS = (BASE_MODEL, *PROFILE_MODELS)
|
||||
MARENGO_2_7_MODELS = (
|
||||
"twelvelabs.marengo-embed-2-7-v1:0",
|
||||
"us.twelvelabs.marengo-embed-2-7-v1:0",
|
||||
"eu.twelvelabs.marengo-embed-2-7-v1:0",
|
||||
)
|
||||
PER_REQUEST_MODELS = (*ALL_MODELS, *MARENGO_2_7_MODELS)
|
||||
|
||||
TEXT_REQUEST_COST = 7e-05
|
||||
IMAGE_REQUEST_COST = 0.0001
|
||||
VIDEO_COST_PER_SECOND = 0.0007
|
||||
AUDIO_COST_PER_SECOND = 0.00014
|
||||
|
||||
|
||||
def _load(path):
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ALL_MODELS)
|
||||
def test_marengo_embed_3_is_visible_to_callers(model, local_model_cost_map):
|
||||
info = litellm.get_model_info(model=model, custom_llm_provider="bedrock")
|
||||
assert info["mode"] == "embedding"
|
||||
assert info["output_vector_size"] == 512
|
||||
|
||||
|
||||
def test_marengo_embed_3_is_a_known_bedrock_embedding_model():
|
||||
assert BASE_MODEL in bedrock_embedding_models
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", PER_REQUEST_MODELS)
|
||||
def test_backup_matches_main(model):
|
||||
main_cost = _load(MAIN_PATH)
|
||||
backup_cost = _load(BACKUP_PATH)
|
||||
|
||||
assert model in main_cost, f"{model} missing from model_prices_and_context_window.json"
|
||||
assert model in backup_cost, f"{model} missing from model_prices_and_context_window_backup.json"
|
||||
assert backup_cost[model] == main_cost[model], f"{model} differs between main and backup model cost maps"
|
||||
|
|
@ -31,6 +31,18 @@ def model_data():
|
|||
return json.load(f)
|
||||
|
||||
|
||||
def test_usgov_east_haiku_profile_mirrors_in_region_row(model_data):
|
||||
"""us-gov-east-1 serves claude-3-haiku through the us-gov. inference profile
|
||||
only, so the profile row must bill exactly like the in-region gov row.
|
||||
"""
|
||||
profile = model_data["us-gov.anthropic.claude-3-haiku-20240307-v1:0"]
|
||||
in_region = model_data["bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0"]
|
||||
assert profile["litellm_provider"] == "bedrock_converse"
|
||||
assert {k: v for k, v in profile.items() if k != "litellm_provider"} == {
|
||||
k: v for k, v in in_region.items() if k != "litellm_provider"
|
||||
}
|
||||
|
||||
|
||||
GOV_ROW_SOURCES = {
|
||||
"us-gov.anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1",
|
||||
"bedrock/us-gov-west-1/anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,10 @@ shape, which Fable 5 rejects with a 400.
|
|||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.constants import BEDROCK_CONVERSE_MODELS
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
|
||||
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
|
||||
|
||||
|
|
@ -23,10 +26,88 @@ def _load_root_cost_map() -> dict:
|
|||
return json.load(f)
|
||||
|
||||
|
||||
def test_fable_5_present_in_bundled_backup():
|
||||
"""The bundled backup is the runtime fallback (and what tests load with
|
||||
``LITELLM_LOCAL_MODEL_COST_MAP=True``) — it must carry the same entries as
|
||||
the root cost map, otherwise the model resolves on one path but not the
|
||||
other."""
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
root = _load_root_cost_map()
|
||||
for model_name in (
|
||||
"claude-fable-5",
|
||||
"anthropic.claude-fable-5",
|
||||
"global.anthropic.claude-fable-5",
|
||||
"us.anthropic.claude-fable-5",
|
||||
"eu.anthropic.claude-fable-5",
|
||||
"vertex_ai/claude-fable-5",
|
||||
"vertex_ai/claude-fable-5@default",
|
||||
"azure_ai/claude-fable-5",
|
||||
):
|
||||
assert model_name in backup, f"Missing from backup cost map: {model_name}"
|
||||
assert backup[model_name] == root[model_name], model_name
|
||||
|
||||
|
||||
def test_fable_5_registered_for_bedrock_converse():
|
||||
assert "anthropic.claude-fable-5" in BEDROCK_CONVERSE_MODELS
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cost_map",
|
||||
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
|
||||
ids=["root", "bundled_backup"],
|
||||
)
|
||||
def test_fable_5_all_variants_carry_adaptive_thinking_flag(cost_map):
|
||||
"""Every Fable 5 entry must advertise ``supports_adaptive_thinking``.
|
||||
|
||||
Adaptive-thinking detection is cost-map driven, so a single variant missing
|
||||
the flag silently sends the legacy ``thinking.type='enabled'`` shape and the
|
||||
provider 400s (issue #29188 for the Opus 4.8 equivalent). Fable 5 is even
|
||||
stricter than Opus 4.8: an explicit ``thinking.type='disabled'`` also 400s,
|
||||
so adaptive is the only valid thinking shape LiteLLM can emit for it."""
|
||||
variants = [k for k in cost_map if "claude-fable-5" in k]
|
||||
assert variants, "no claude-fable-5 entries found in cost map"
|
||||
missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True]
|
||||
assert not missing, f"missing supports_adaptive_thinking: {missing}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cost_map",
|
||||
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
|
||||
ids=["root", "bundled_backup"],
|
||||
)
|
||||
def test_fable_5_all_variants_carry_thinking_always_on_flag(cost_map):
|
||||
"""Every Fable 5 entry must advertise ``thinking_always_on``.
|
||||
|
||||
The flag drives the Anthropic transformations to omit an explicit
|
||||
``thinking.type='disabled'``, which Fable 5 rejects with a 400; a variant
|
||||
missing the flag forwards the param verbatim and the provider 400s."""
|
||||
variants = [k for k in cost_map if "claude-fable-5" in k]
|
||||
assert variants, "no claude-fable-5 entries found in cost map"
|
||||
missing = [k for k in variants if cost_map[k].get("thinking_always_on") is not True]
|
||||
assert not missing, f"missing thinking_always_on: {missing}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"claude-fable-5",
|
||||
"anthropic/claude-fable-5",
|
||||
"anthropic.claude-fable-5",
|
||||
"bedrock/us.anthropic.claude-fable-5",
|
||||
"bedrock/invoke/eu.anthropic.claude-fable-5",
|
||||
"bedrock/global.anthropic.claude-fable-5",
|
||||
"vertex_ai/claude-fable-5",
|
||||
"azure_ai/claude-fable-5",
|
||||
],
|
||||
)
|
||||
def test_adaptive_thinking_detected_for_fable_5(local_model_cost_map, model):
|
||||
"""Provider-routed ids must resolve to a flagged entry so ``reasoning_effort``
|
||||
maps to ``thinking.type='adaptive'`` + ``output_config.effort``."""
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True
|
||||
|
||||
|
||||
FABLE_5_1_VARIANTS = (
|
||||
"claude-fable-5-1",
|
||||
"anthropic.claude-fable-5-1",
|
||||
|
|
@ -39,5 +120,53 @@ FABLE_5_1_VARIANTS = (
|
|||
)
|
||||
|
||||
|
||||
def test_fable_5_1_present_in_bundled_backup():
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
root = _load_root_cost_map()
|
||||
for model_name in FABLE_5_1_VARIANTS:
|
||||
assert model_name in backup, f"Missing from backup cost map: {model_name}"
|
||||
assert backup[model_name] == root[model_name], model_name
|
||||
|
||||
|
||||
def test_fable_5_1_registered_for_bedrock_converse():
|
||||
assert "anthropic.claude-fable-5-1" in BEDROCK_CONVERSE_MODELS
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"claude-fable-5-1",
|
||||
"anthropic/claude-fable-5-1",
|
||||
"anthropic.claude-fable-5-1",
|
||||
"bedrock/us.anthropic.claude-fable-5-1",
|
||||
"bedrock/invoke/eu.anthropic.claude-fable-5-1",
|
||||
"bedrock/global.anthropic.claude-fable-5-1",
|
||||
"vertex_ai/claude-fable-5-1",
|
||||
"azure_ai/claude-fable-5-1",
|
||||
],
|
||||
)
|
||||
def test_adaptive_thinking_detected_for_fable_5_1(local_model_cost_map, model):
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cost_map",
|
||||
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
|
||||
ids=["root", "bundled_backup"],
|
||||
)
|
||||
def test_sampling_params_flag_on_all_models_that_removed_them(cost_map):
|
||||
"""Fable 5 and Opus 4.7/4.8 reject ``top_p``/``top_k``/``temperature != 1``;
|
||||
the drop/raise gating is cost-map driven, so every variant must carry an
|
||||
explicit ``supports_sampling_params: false``. The perplexity route is
|
||||
exempt: it is OpenAI-compatible and maps sampling params upstream."""
|
||||
variants = [
|
||||
k
|
||||
for k in cost_map
|
||||
if any(v in k for v in ("claude-fable-5", "claude-opus-4-7", "claude-opus-4-8"))
|
||||
and not k.startswith("perplexity/")
|
||||
]
|
||||
assert variants, "no matching entries found in cost map"
|
||||
missing = [k for k in variants if cost_map[k].get("supports_sampling_params") is not False]
|
||||
assert not missing, f"missing supports_sampling_params=false: {missing}"
|
||||
|
|
|
|||
46
tests/test_litellm/test_claude_haiku_4_5_config.py
Normal file
46
tests/test_litellm/test_claude_haiku_4_5_config.py
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
"""
|
||||
Test Claude Haiku 4.5 model configurations for Bedrock
|
||||
https://github.com/BerriAI/litellm/issues/15818
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
|
||||
def test_bedrock_haiku_4_5_matches_sonnet_capabilities():
|
||||
"""
|
||||
Test that Haiku 4.5 has same capabilities as Sonnet 4.5
|
||||
(including computer_use, vision, tools, etc.)
|
||||
"""
|
||||
# Load model configuration
|
||||
json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json")
|
||||
with open(json_path) as f:
|
||||
model_data = json.load(f)
|
||||
|
||||
haiku_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
sonnet_model = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
|
||||
haiku_info = model_data[haiku_model]
|
||||
sonnet_info = model_data[sonnet_model]
|
||||
|
||||
# Both should use bedrock_converse
|
||||
assert haiku_info["litellm_provider"] == "bedrock_converse"
|
||||
assert sonnet_info["litellm_provider"] == "bedrock_converse"
|
||||
|
||||
# Shared capabilities that should match
|
||||
shared_capabilities = [
|
||||
"supports_vision",
|
||||
"supports_computer_use",
|
||||
"supports_function_calling",
|
||||
"supports_tool_choice",
|
||||
"supports_prompt_caching",
|
||||
"supports_response_schema",
|
||||
"supports_pdf_input",
|
||||
"supports_assistant_prefill",
|
||||
"supports_reasoning",
|
||||
]
|
||||
|
||||
for capability in shared_capabilities:
|
||||
assert haiku_info.get(capability) == sonnet_info.get(capability), (
|
||||
f"Capability {capability} mismatch: Haiku={haiku_info.get(capability)}, Sonnet={sonnet_info.get(capability)}"
|
||||
)
|
||||
|
|
@ -18,6 +18,7 @@ import os
|
|||
import pytest
|
||||
|
||||
from litellm.constants import BEDROCK_CONVERSE_MODELS
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
|
||||
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
|
||||
|
||||
|
|
@ -61,5 +62,31 @@ def test_opus_5_bedrock_rejects_strict_tools(model_name, local_model_cost_map):
|
|||
assert bedrock_converse_supports_strict_tools(model_name) is False
|
||||
|
||||
|
||||
def test_opus_5_present_in_bundled_backup():
|
||||
"""The bundled backup is the runtime fallback (and what tests load with
|
||||
``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the
|
||||
root cost map, otherwise the model resolves on one path but not the other."""
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
for model_name in ALL_OPUS_5_VARIANTS:
|
||||
assert model_name in backup, f"Missing from backup cost map: {model_name}"
|
||||
|
||||
|
||||
def test_opus_5_registered_for_bedrock_converse():
|
||||
assert "anthropic.claude-opus-5" in BEDROCK_CONVERSE_MODELS
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cost_map",
|
||||
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
|
||||
ids=["root", "bundled_backup"],
|
||||
)
|
||||
def test_opus_5_all_variants_carry_adaptive_thinking_flag(cost_map):
|
||||
"""Every Opus 5 entry must advertise ``supports_adaptive_thinking``.
|
||||
|
||||
Adaptive-thinking detection is cost-map driven, so a single variant missing
|
||||
the flag silently sends the legacy ``thinking.type='enabled'`` shape, which
|
||||
Opus 5 rejects with a 400."""
|
||||
variants = [k for k in cost_map if "claude-opus-5" in k]
|
||||
assert variants, "no claude-opus-5 entries found in cost map"
|
||||
missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True]
|
||||
assert not missing, f"missing supports_adaptive_thinking: {missing}"
|
||||
|
|
|
|||
|
|
@ -13,7 +13,10 @@ populate ``litellm.anthropic_models`` at import, which is what lets a bare
|
|||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.constants import BEDROCK_CONVERSE_MODELS
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
|
||||
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
|
||||
|
||||
|
|
@ -37,5 +40,31 @@ def _load_root_cost_map() -> dict:
|
|||
return json.load(f)
|
||||
|
||||
|
||||
def test_sonnet_5_present_in_bundled_backup():
|
||||
"""The bundled backup is the runtime fallback (and what tests load with
|
||||
``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the
|
||||
root cost map, otherwise the model resolves on one path but not the other."""
|
||||
backup = GetModelCostMap.load_local_model_cost_map()
|
||||
for model_name in ALL_SONNET_5_VARIANTS:
|
||||
assert model_name in backup, f"Missing from backup cost map: {model_name}"
|
||||
|
||||
|
||||
def test_sonnet_5_registered_for_bedrock_converse():
|
||||
assert "anthropic.claude-sonnet-5" in BEDROCK_CONVERSE_MODELS
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cost_map",
|
||||
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
|
||||
ids=["root", "bundled_backup"],
|
||||
)
|
||||
def test_sonnet_5_all_variants_carry_adaptive_thinking_flag(cost_map):
|
||||
"""Every Sonnet 5 entry must advertise ``supports_adaptive_thinking``.
|
||||
|
||||
Adaptive-thinking detection is cost-map driven, so a single variant missing
|
||||
the flag silently sends the legacy ``thinking.type='enabled'`` shape and the
|
||||
provider 400s. This guards against a future variant being added without it."""
|
||||
variants = [k for k in cost_map if "claude-sonnet-5" in k]
|
||||
assert variants, "no claude-sonnet-5 entries found in cost map"
|
||||
missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True]
|
||||
assert not missing, f"missing supports_adaptive_thinking: {missing}"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
"""
|
||||
Regression tests for the Cloudflare Workers AI text-generation catalog in the
|
||||
model-cost map.
|
||||
|
||||
The Cloudflare list was badly stale (only 4 ancient entries). These tests pin
|
||||
the newly added current Workers AI models (sourced from Cloudflare's live
|
||||
``/ai/models/search?task=Text Generation`` catalog) and guard against the root
|
||||
``model_prices_and_context_window.json`` and the bundled
|
||||
``litellm/model_prices_and_context_window_backup.json`` drifting out of sync for
|
||||
the ``cloudflare/`` namespace.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
|
||||
ROOT_MAP = os.path.join(
|
||||
os.path.dirname(os.path.dirname(litellm.__file__)),
|
||||
"model_prices_and_context_window.json",
|
||||
)
|
||||
BACKUP_MAP = os.path.join(
|
||||
os.path.dirname(litellm.__file__),
|
||||
"model_prices_and_context_window_backup.json",
|
||||
)
|
||||
|
||||
|
||||
def _load(path: str) -> dict:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _cloudflare_keys(data: dict) -> set:
|
||||
return {k for k in data if k.startswith("cloudflare/")}
|
||||
|
||||
|
||||
def test_root_and_backup_have_identical_cloudflare_keys():
|
||||
if not os.path.exists(ROOT_MAP):
|
||||
pytest.skip("root cost map only ships in source checkouts")
|
||||
assert _cloudflare_keys(_load(ROOT_MAP)) == _cloudflare_keys(_load(BACKUP_MAP))
|
||||
|
||||
|
||||
def test_root_and_backup_cloudflare_entries_are_byte_for_byte_equal():
|
||||
if not os.path.exists(ROOT_MAP):
|
||||
pytest.skip("root cost map only ships in source checkouts")
|
||||
root = {k: v for k, v in _load(ROOT_MAP).items() if k.startswith("cloudflare/")}
|
||||
backup = {k: v for k, v in _load(BACKUP_MAP).items() if k.startswith("cloudflare/")}
|
||||
assert root == backup
|
||||
53
tests/test_litellm/test_command_r7b_pricing.py
Normal file
53
tests/test_litellm/test_command_r7b_pricing.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
"""
|
||||
Regression test: ``command-r7b-12-2024`` had its input/output per-token
|
||||
costs transposed in the model-cost maps (input=1.5e-07 / output=3.75e-08),
|
||||
even though Cohere publishes $0.0375/1M input and $0.15/1M output, i.e.
|
||||
output is ~4x input like every other ``command-r`` entry.
|
||||
|
||||
These tests pin the corrected values in both the primary price map and the
|
||||
``litellm/`` backup, and verify ``get_model_info`` surfaces them, so the
|
||||
swap cannot silently regress.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import litellm
|
||||
|
||||
MODEL = "command-r7b-12-2024"
|
||||
EXPECTED_INPUT_COST = 3.75e-08
|
||||
EXPECTED_OUTPUT_COST = 1.5e-07
|
||||
|
||||
|
||||
def _load_json(path: str) -> dict:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _backup_path() -> str:
|
||||
return os.path.join(
|
||||
os.path.dirname(litellm.__file__),
|
||||
"model_prices_and_context_window_backup.json",
|
||||
)
|
||||
|
||||
|
||||
def _main_path() -> str:
|
||||
# This test lives at ``tests/test_litellm/``; the primary price map sits at
|
||||
# the repo root, two directories up. Resolve it relative to this file so the
|
||||
# test works regardless of where ``litellm`` itself is installed (e.g. a pip
|
||||
# install into site-packages).
|
||||
return os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"..",
|
||||
"..",
|
||||
"model_prices_and_context_window.json",
|
||||
)
|
||||
|
||||
|
||||
class TestCommandR7bPricingData:
|
||||
"""The JSON price maps must carry Cohere's published costs, with output
|
||||
more expensive than input."""
|
||||
|
||||
|
||||
class TestCommandR7bPricingModelInfo:
|
||||
"""``get_model_info`` must report the corrected, un-swapped costs."""
|
||||
|
|
@ -32,6 +32,13 @@ def _load(path):
|
|||
return json.load(f)
|
||||
|
||||
|
||||
def test_blue_alias_matches_its_snapshot_computer_use():
|
||||
cost_map = _load(MAIN_PATH)
|
||||
|
||||
assert cost_map[BLUE_ALIAS]["supports_computer_use"] is True
|
||||
assert cost_map[BLUE_SNAPSHOT]["supports_computer_use"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("alias", "snapshot"), OFFICIAL_ALIAS_SNAPSHOTS)
|
||||
def test_official_alias_tracks_snapshot(alias, snapshot):
|
||||
cost_map = _load(MAIN_PATH)
|
||||
|
|
@ -44,3 +51,11 @@ def test_official_alias_tracks_snapshot(alias, snapshot):
|
|||
field: snapshot_info.get(field) for field in PRICE_FIELDS
|
||||
}
|
||||
assert alias_info["max_output_tokens"] == snapshot_info["max_output_tokens"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", (*DAYBREAK_MODELS, BLUE_SNAPSHOT, *(alias for alias, _ in OFFICIAL_ALIAS_SNAPSHOTS)))
|
||||
def test_backup_matches_main(model):
|
||||
main_cost = _load(MAIN_PATH)
|
||||
backup_cost = _load(BACKUP_PATH)
|
||||
|
||||
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import os
|
|||
import litellm
|
||||
from litellm.utils import (
|
||||
_supports_factory,
|
||||
supports_response_schema,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -36,6 +37,18 @@ class TestDeepSeekModelCostEntries:
|
|||
"""Verify that provider-prefixed DeepSeek entries contain the same
|
||||
capability flags as their bare-name counterparts in the JSON files."""
|
||||
|
||||
def test_deepseek_chat_max_input_tokens_matches_bare_in_backup(self):
|
||||
data = _load_backup_json()
|
||||
bare = data.get("deepseek-chat", {})
|
||||
prefixed = data.get("deepseek/deepseek-chat", {})
|
||||
assert prefixed.get("max_input_tokens") == bare.get("max_input_tokens")
|
||||
|
||||
def test_deepseek_reasoner_max_output_tokens_matches_bare_in_backup(self):
|
||||
data = _load_backup_json()
|
||||
bare = data.get("deepseek-reasoner", {})
|
||||
prefixed = data.get("deepseek/deepseek-reasoner", {})
|
||||
assert prefixed.get("max_output_tokens") == bare.get("max_output_tokens")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API-level tests – verify supports_response_schema returns True
|
||||
|
|
@ -46,6 +59,18 @@ class TestSupportsResponseSchemaDeepSeek:
|
|||
"""All calling conventions for DeepSeek should return True for
|
||||
``supports_response_schema``."""
|
||||
|
||||
def test_provider_slash_model(self):
|
||||
assert supports_response_schema(model="deepseek/deepseek-chat") is True
|
||||
|
||||
def test_explicit_provider(self):
|
||||
assert supports_response_schema(model="deepseek-chat", custom_llm_provider="deepseek") is True
|
||||
|
||||
def test_reasoner_provider_slash_model(self):
|
||||
assert supports_response_schema(model="deepseek/deepseek-reasoner") is True
|
||||
|
||||
def test_reasoner_explicit_provider(self):
|
||||
assert supports_response_schema(model="deepseek-reasoner", custom_llm_provider="deepseek") is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fallback-logic test – bare model entry used when prefixed is incomplete
|
||||
|
|
|
|||
|
|
@ -14,15 +14,7 @@ import os
|
|||
|
||||
import pytest
|
||||
|
||||
NEW_ENTRIES = {
|
||||
"fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": {
|
||||
"input_cost_per_token": 1.32e-06,
|
||||
"cache_read_input_token_cost": 4.4e-08,
|
||||
"output_cost_per_token": 3.96e-06,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
},
|
||||
}
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
|
@ -32,20 +24,17 @@ def model_data():
|
|||
return json.load(f)
|
||||
|
||||
|
||||
TWIN_PINNED_PRICES = {
|
||||
"deepseek-v4-flash-0731": {
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
"cache_read_input_token_cost": 7e-09,
|
||||
"output_cost_per_token": 6.6e-07,
|
||||
},
|
||||
"deepseek-v4p1-flash": {
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
"cache_read_input_token_cost": 7e-09,
|
||||
"output_cost_per_token": 6.6e-07,
|
||||
"supports_vision": True,
|
||||
"max_output_tokens": 393216,
|
||||
},
|
||||
}
|
||||
def test_bare_fireworks_ids_resolve_through_prefixed_entries():
|
||||
"""Bare IDs from #37274 resolve via the provider-prefix lookup path."""
|
||||
for bare_id, prefixed_key in [
|
||||
(
|
||||
"accounts/fireworks/models/deepseek-v4-pro-0813",
|
||||
"fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813",
|
||||
),
|
||||
]:
|
||||
info = get_model_info(model=bare_id, custom_llm_provider="fireworks_ai")
|
||||
assert info.get("key") == prefixed_key
|
||||
assert info["litellm_provider"] == "fireworks_ai"
|
||||
|
||||
|
||||
def test_fireworks_account_prefixed_twins_agree_on_price(model_data):
|
||||
|
|
|
|||
|
|
@ -109,6 +109,11 @@ def test_per_route_capabilities_match_model_cards(model: str, path: Path):
|
|||
assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ALL_KEYS)
|
||||
def test_backup_matches_main(model: str):
|
||||
assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model)
|
||||
|
||||
|
||||
def test_gemini_prefix_routes_to_gemini():
|
||||
routed_model, provider, _, _ = get_llm_provider(model=GEMINI)
|
||||
assert routed_model == UNPREFIXED
|
||||
|
|
|
|||
84
tests/test_litellm/test_gemini_tts_native_audio_pricing.py
Normal file
84
tests/test_litellm/test_gemini_tts_native_audio_pricing.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import json
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
|
||||
REPO_ROOT: Final = Path(__file__).parents[2]
|
||||
MAIN_PATH: Final = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_PATH: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
FLASH_TTS_KEYS: Final = ("gemini-2.5-flash-preview-tts", "gemini/gemini-2.5-flash-preview-tts")
|
||||
PRO_TTS_KEYS: Final = ("gemini-2.5-pro-preview-tts", "gemini/gemini-2.5-pro-preview-tts")
|
||||
NATIVE_AUDIO_KEYS: Final = tuple(
|
||||
f"{prefix}gemini-2.5-flash-native-audio-{suffix}"
|
||||
for prefix in ("", "gemini/")
|
||||
for suffix in ("latest", "preview-09-2025", "preview-12-2025")
|
||||
)
|
||||
|
||||
LIVE_NATIVE_AUDIO_KEYS: Final = (
|
||||
"gemini-live-2.5-flash-preview-native-audio-09-2025",
|
||||
"gemini/gemini-live-2.5-flash-preview-native-audio-09-2025",
|
||||
)
|
||||
|
||||
FLASH_TTS_INPUT: Final = 5e-07
|
||||
FLASH_TTS_AUDIO_OUTPUT: Final = 1e-05
|
||||
PRO_TTS_INPUT: Final = 1e-06
|
||||
PRO_TTS_AUDIO_OUTPUT: Final = 2e-05
|
||||
NATIVE_AUDIO_TEXT_INPUT: Final = 5e-07
|
||||
NATIVE_AUDIO_AUDIO_INPUT: Final = 3e-06
|
||||
NATIVE_AUDIO_TEXT_OUTPUT: Final = 2e-06
|
||||
NATIVE_AUDIO_AUDIO_OUTPUT: Final = 1.2e-05
|
||||
|
||||
PUBLISHED_RATES: Final = {
|
||||
**{
|
||||
key: {"input_cost_per_token": FLASH_TTS_INPUT, "output_cost_per_token": FLASH_TTS_AUDIO_OUTPUT}
|
||||
for key in FLASH_TTS_KEYS
|
||||
},
|
||||
**{
|
||||
key: {"input_cost_per_token": PRO_TTS_INPUT, "output_cost_per_token": PRO_TTS_AUDIO_OUTPUT}
|
||||
for key in PRO_TTS_KEYS
|
||||
},
|
||||
**{
|
||||
key: {
|
||||
"input_cost_per_token": NATIVE_AUDIO_TEXT_INPUT,
|
||||
"input_cost_per_audio_token": NATIVE_AUDIO_AUDIO_INPUT,
|
||||
"output_cost_per_token": NATIVE_AUDIO_TEXT_OUTPUT,
|
||||
"output_cost_per_audio_token": NATIVE_AUDIO_AUDIO_OUTPUT,
|
||||
}
|
||||
for key in (*NATIVE_AUDIO_KEYS, *LIVE_NATIVE_AUDIO_KEYS)
|
||||
},
|
||||
}
|
||||
ALL_KEYS: Final = tuple(PUBLISHED_RATES)
|
||||
NATIVE_AUDIO_BILLING_CASES: Final = (
|
||||
*((key, "gemini") for key in NATIVE_AUDIO_KEYS),
|
||||
("gemini-live-2.5-flash-preview-native-audio-09-2025", "vertex_ai"),
|
||||
("gemini/gemini-live-2.5-flash-preview-native-audio-09-2025", "gemini"),
|
||||
)
|
||||
LONG_CONTEXT_TIER_FIELDS: Final = (
|
||||
"input_cost_per_token_above_200k_tokens",
|
||||
"output_cost_per_token_above_200k_tokens",
|
||||
"cache_read_input_token_cost_above_200k_tokens",
|
||||
)
|
||||
|
||||
|
||||
def _load(path: Path) -> dict[str, dict[str, object]]:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ALL_KEYS)
|
||||
def test_backup_matches_main(model: str):
|
||||
assert _load(BACKUP_PATH)[model] == _load(MAIN_PATH)[model]
|
||||
44
tests/test_litellm/test_gpt_5_4_model_metadata.py
Normal file
44
tests/test_litellm/test_gpt_5_4_model_metadata.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import json
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[2]
|
||||
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
DOCUMENTED_MAX_INPUT_TOKENS = 272000
|
||||
DOCUMENTED_MAX_OUTPUT_TOKENS = 128000
|
||||
|
||||
SMALL_MODEL_NAMES = (
|
||||
"gpt-5.4-mini",
|
||||
"gpt-5.4-mini-2026-03-17",
|
||||
"gpt-5.4-nano",
|
||||
"gpt-5.4-nano-2026-03-17",
|
||||
)
|
||||
SMALL_MODELS = tuple(f"{prefix}{name}" for prefix in ("", "azure/", "azure_ai/") for name in SMALL_MODEL_NAMES)
|
||||
|
||||
STANDARD_PRICING = {
|
||||
"gpt-5.4-mini": (7.5e-07, 4.5e-06, 7.5e-08),
|
||||
"gpt-5.4-nano": (2e-07, 1.25e-06, 2e-08),
|
||||
}
|
||||
|
||||
LONG_CONTEXT_MODELS = ("gpt-5.4", "gpt-5.4-pro")
|
||||
|
||||
|
||||
@lru_cache(maxsize=2)
|
||||
def _load(path: Path) -> dict[str, dict[str, object]]:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _pricing_key(model: str) -> str:
|
||||
return "gpt-5.4-nano" if "nano" in model else "gpt-5.4-mini"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", SMALL_MODELS)
|
||||
def test_gpt_5_4_small_models_backup_matches_main(model: str) -> None:
|
||||
assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model), (
|
||||
f"{model} differs between main and backup model cost maps"
|
||||
)
|
||||
19
tests/test_litellm/test_gpt_5_5_model_metadata.py
Normal file
19
tests/test_litellm/test_gpt_5_5_model_metadata.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_azure_ai_gpt_5_5_backup_matches_main():
|
||||
"""Ensure the bundled model cost map stays in sync with the canonical file."""
|
||||
repo_root = Path(__file__).parents[2]
|
||||
main_path = repo_root / "model_prices_and_context_window.json"
|
||||
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
with open(main_path) as f:
|
||||
main_cost = json.load(f)
|
||||
with open(backup_path) as f:
|
||||
backup_cost = json.load(f)
|
||||
|
||||
for model in ("azure_ai/gpt-5.5", "azure_ai/gpt-5.5-2026-04-23"):
|
||||
assert backup_cost.get(model) == main_cost.get(model), (
|
||||
f"{model} differs between main and backup model cost maps"
|
||||
)
|
||||
|
|
@ -1,4 +1,8 @@
|
|||
from typing_extensions import get_args, get_type_hints
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import get_args
|
||||
|
||||
from typing_extensions import get_type_hints
|
||||
|
||||
from litellm.types.utils import ModelInfoBase
|
||||
|
||||
|
|
@ -41,3 +45,13 @@ ALL_REALTIME_ONLY_GPT_MODELS = REALTIME_ONLY_GPT_MODELS + REALTIME_ONLY_GPT_MODE
|
|||
def test_realtime_is_a_valid_mode_literal():
|
||||
hints = get_type_hints(ModelInfoBase, include_extras=False)
|
||||
assert "realtime" in get_args(hints["mode"])
|
||||
|
||||
|
||||
def test_backup_matches_main_for_realtime_models():
|
||||
repo_root = Path(__file__).parents[2]
|
||||
with open(repo_root / "model_prices_and_context_window.json") as f:
|
||||
main_cost = json.load(f)
|
||||
with open(repo_root / "litellm" / "model_prices_and_context_window_backup.json") as f:
|
||||
backup_cost = json.load(f)
|
||||
for model in ALL_REALTIME_ONLY_GPT_MODELS:
|
||||
assert backup_cost.get(model) == main_cost.get(model)
|
||||
|
|
|
|||
33
tests/test_litellm/test_mistral_medium_3_5_model_metadata.py
Normal file
33
tests/test_litellm/test_mistral_medium_3_5_model_metadata.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[2]
|
||||
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
MEDIUM_3_5_MODELS = (
|
||||
"mistral/mistral-medium-3-5",
|
||||
"mistral/mistral-medium-2604",
|
||||
"mistral/mistral-medium-latest",
|
||||
)
|
||||
|
||||
SYNCED_MODELS = MEDIUM_3_5_MODELS + (
|
||||
"mistral/mistral-medium-2508",
|
||||
"mistral/mistral-medium-3-1-2508",
|
||||
)
|
||||
|
||||
|
||||
def _load(path):
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", SYNCED_MODELS)
|
||||
def test_backup_matches_main(model):
|
||||
"""Ensure the bundled (backup) cost map stays in sync with the canonical file."""
|
||||
main_cost = _load(MAIN_PATH)
|
||||
backup_cost = _load(BACKUP_PATH)
|
||||
|
||||
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"
|
||||
26
tests/test_litellm/test_mistral_small_4_0_model_metadata.py
Normal file
26
tests/test_litellm/test_mistral_small_4_0_model_metadata.py
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[2]
|
||||
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
SMALL_4_0_MODELS = (
|
||||
"mistral/mistral-small-latest",
|
||||
"mistral/mistral-small-2603",
|
||||
)
|
||||
|
||||
|
||||
def _load(path):
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", SMALL_4_0_MODELS)
|
||||
def test_backup_matches_main(model):
|
||||
main_cost = _load(MAIN_PATH)
|
||||
backup_cost = _load(BACKUP_PATH)
|
||||
|
||||
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.utils import supports_prompt_caching, supports_reasoning
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[2]
|
||||
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
GLM_5_2_MODELS = ("mistral/zai-glm-5-2", "mistral/glm-5-2")
|
||||
|
||||
INPUT_COST = 1.4e-06
|
||||
CACHED_INPUT_COST = 1.4e-07
|
||||
OUTPUT_COST = 4.4e-06
|
||||
|
||||
|
||||
def _load(path):
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_model_cost_map(monkeypatch):
|
||||
"""Force get_model_info to resolve against the in-repo cost map instead of the
|
||||
remote one fetched at import time, which still carries the pre-merge pricing."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
|
||||
def test_zai_glm_5_2_capabilities_are_visible_to_callers(local_model_cost_map, model):
|
||||
"""Mistral advertises reasoning and prompt caching on this model, so the helpers
|
||||
every caller checks before sending a request must say so too."""
|
||||
assert supports_reasoning(model=model) is True
|
||||
assert supports_prompt_caching(model=model) is True
|
||||
|
||||
assert litellm.get_model_info(model=model)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
|
||||
def test_backup_matches_main(model):
|
||||
"""Ensure the bundled (backup) cost map stays in sync with the canonical file."""
|
||||
main_cost = _load(MAIN_PATH)
|
||||
backup_cost = _load(BACKUP_PATH)
|
||||
|
||||
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"
|
||||
29
tests/test_litellm/test_muse_spark_1_1_model_metadata.py
Normal file
29
tests/test_litellm/test_muse_spark_1_1_model_metadata.py
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
MUSE_SPARK_MODEL = "meta/muse-spark-1.1"
|
||||
|
||||
|
||||
def test_muse_spark_1_1_model_info():
|
||||
routed_model, provider, _, api_base = get_llm_provider(model=MUSE_SPARK_MODEL, api_key="sk-test")
|
||||
assert routed_model == "muse-spark-1.1"
|
||||
assert provider == "meta"
|
||||
assert api_base == "https://api.meta.ai/v1"
|
||||
|
||||
|
||||
def test_muse_spark_1_1_backup_matches_main():
|
||||
"""Ensure the bundled model cost map stays in sync with the canonical file."""
|
||||
repo_root = Path(__file__).parents[2]
|
||||
main_path = repo_root / "model_prices_and_context_window.json"
|
||||
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
with open(main_path) as f:
|
||||
main_cost = json.load(f)
|
||||
with open(backup_path) as f:
|
||||
backup_cost = json.load(f)
|
||||
|
||||
assert backup_cost.get(MUSE_SPARK_MODEL) == main_cost.get(MUSE_SPARK_MODEL), (
|
||||
f"{MUSE_SPARK_MODEL} differs between main and backup model cost maps"
|
||||
)
|
||||
|
|
@ -29,6 +29,15 @@ def test_muse_spark_1_2_routes_to_meta_model_api(model: str):
|
|||
assert api_base == "https://api.meta.ai/v1"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR))
|
||||
def test_muse_spark_1_2_backup_matches_main(model: str):
|
||||
"""Ensure the bundled model cost map stays in sync with the canonical file."""
|
||||
main_cost = _load_cost_map()
|
||||
backup_cost = _load_cost_map("litellm/model_prices_and_context_window_backup.json")
|
||||
|
||||
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"
|
||||
|
||||
|
||||
def test_muse_spark_contributor_tier_is_cheaper_than_standard():
|
||||
cost_map = _load_cost_map()
|
||||
standard = cost_map[MUSE_SPARK_STANDARD]
|
||||
|
|
|
|||
|
|
@ -38,6 +38,15 @@ def test_muse_spark_1_3_web_search_cost_per_query(local_model_cost_map, model: s
|
|||
assert StandardBuiltInToolCostTracking.get_cost_for_web_search(model_info=info) == WEB_SEARCH_COST_PER_QUERY
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR))
|
||||
def test_muse_spark_1_3_backup_matches_main(model: str):
|
||||
"""Ensure the bundled model cost map stays in sync with the canonical file."""
|
||||
main_cost = _load_cost_map()
|
||||
backup_cost = _load_cost_map("litellm/model_prices_and_context_window_backup.json")
|
||||
|
||||
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"
|
||||
|
||||
|
||||
def test_muse_spark_contributor_tier_is_cheaper_than_standard():
|
||||
cost_map = _load_cost_map()
|
||||
standard = cost_map[MUSE_SPARK_STANDARD]
|
||||
|
|
|
|||
|
|
@ -18,3 +18,18 @@ def test_replicate_models_have_valid_key_prefix(model_cost: dict[str, Any]) -> N
|
|||
assert not malformed, (
|
||||
f"Replicate models must use 'replicate/owner/model' key format, found malformed keys: {malformed}"
|
||||
)
|
||||
|
||||
|
||||
def test_replicate_backup_matches_main() -> None:
|
||||
repo_root = Path(__file__).parents[2]
|
||||
main_path = repo_root / "model_prices_and_context_window.json"
|
||||
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
with open(main_path, encoding="utf-8") as f:
|
||||
main_cost: dict[str, Any] = json.load(f)
|
||||
with open(backup_path, encoding="utf-8") as f:
|
||||
backup_cost: dict[str, Any] = json.load(f)
|
||||
|
||||
for key in main_cost:
|
||||
if main_cost[key].get("litellm_provider") == "replicate":
|
||||
assert backup_cost.get(key) == main_cost.get(key), f"{key} differs between main and backup model cost maps"
|
||||
|
|
|
|||
25
tests/test_litellm/test_sambanova_model_metadata.py
Normal file
25
tests/test_litellm/test_sambanova_model_metadata.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
|
||||
def test_sambanova_minimax_m27_model_info():
|
||||
model = "sambanova/MiniMax-M2.7"
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
info = model_cost.get(model)
|
||||
assert info is not None, f"{model} not found in model_prices_and_context_window.json"
|
||||
assert info["litellm_provider"] == "sambanova"
|
||||
assert info["mode"] == "chat"
|
||||
assert info["input_cost_per_token"] > 0
|
||||
assert info["output_cost_per_token"] > 0
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_reasoning"] is True
|
||||
assert info["supports_tool_choice"] is True
|
||||
|
||||
routed_model, provider, _, _ = get_llm_provider(model=model)
|
||||
assert routed_model == "MiniMax-M2.7"
|
||||
assert provider == "sambanova"
|
||||
|
|
@ -108,6 +108,14 @@ def test_together_successor_metadata_points_at_live_models(cost_map: CostMap):
|
|||
assert "deprecation_date" not in target, f"{model} names deprecated successor {successor}"
|
||||
|
||||
|
||||
def test_together_backup_cost_map_in_sync(cost_map: CostMap):
|
||||
with open(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json") as f:
|
||||
backup = COST_MAP_ADAPTER.validate_python(json.load(f))
|
||||
together_main = {k: v for k, v in cost_map.items() if k.startswith("together_ai/")}
|
||||
together_backup = {k: v for k, v in backup.items() if k.startswith("together_ai/")}
|
||||
assert together_backup == together_main
|
||||
|
||||
|
||||
CACHED_INPUT_MODELS: Final = (
|
||||
"together_ai/moonshotai/Kimi-K3",
|
||||
"together_ai/zai-org/GLM-5.2",
|
||||
|
|
|
|||
|
|
@ -3641,9 +3641,6 @@ def _assert_fireworks_entry(
|
|||
assert info["input_cost_per_token"] > 0
|
||||
assert info["output_cost_per_token"] > 0
|
||||
assert "cache_read_input_token_cost" in info
|
||||
assert info["max_input_tokens"] == expected_max_input
|
||||
assert info["max_output_tokens"] == expected_max_output
|
||||
assert info["max_tokens"] == expected_max_output
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_tool_choice"] is True
|
||||
assert info["supports_reasoning"] is expected_reasoning
|
||||
|
|
@ -5657,3 +5654,253 @@ def test_completion_finishes_response_metadata_before_handing_the_response_to_th
|
|||
assert snapshot["litellm_call_id"]
|
||||
assert snapshot["response_cost"] is not None
|
||||
assert snapshot["api_base"]
|
||||
|
||||
|
||||
def test_fireworks_models_in_backup_cost_map():
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
json_path = Path(__file__).parents[2] / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
for entry in _FIREWORKS_MODELS:
|
||||
_assert_fireworks_entry(model_cost, *entry)
|
||||
|
||||
for short in _FIREWORKS_SHORT_FORMS:
|
||||
long_key = f"fireworks_ai/accounts/fireworks/models/{short}"
|
||||
short_key = f"fireworks_ai/{short}"
|
||||
assert model_cost.get(short_key) == model_cost.get(long_key), (
|
||||
f"short-form {short_key} does not match long-form {long_key}"
|
||||
)
|
||||
|
||||
for short in _FIREWORKS_ROUTER_SHORT_FORMS:
|
||||
long_key = f"fireworks_ai/accounts/fireworks/routers/{short}"
|
||||
short_key = f"fireworks_ai/{short}"
|
||||
assert model_cost.get(short_key) == model_cost.get(long_key), (
|
||||
f"short-form {short_key} does not match long-form {long_key}"
|
||||
)
|
||||
|
||||
|
||||
def test_fireworks_models_in_cost_map():
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
for entry in _FIREWORKS_MODELS:
|
||||
_assert_fireworks_entry(model_cost, *entry)
|
||||
|
||||
for short in _FIREWORKS_SHORT_FORMS:
|
||||
long_key = f"fireworks_ai/accounts/fireworks/models/{short}"
|
||||
short_key = f"fireworks_ai/{short}"
|
||||
assert model_cost.get(short_key) == model_cost.get(long_key), (
|
||||
f"short-form {short_key} does not match long-form {long_key}"
|
||||
)
|
||||
|
||||
for short in _FIREWORKS_ROUTER_SHORT_FORMS:
|
||||
long_key = f"fireworks_ai/accounts/fireworks/routers/{short}"
|
||||
short_key = f"fireworks_ai/{short}"
|
||||
assert model_cost.get(short_key) == model_cost.get(long_key), (
|
||||
f"short-form {short_key} does not match long-form {long_key}"
|
||||
)
|
||||
|
||||
|
||||
def test_fireworks_short_model_names_resolve_to_long_cost_map_keys(fireworks_short_model_cost_map: None) -> None:
|
||||
model_info = litellm.get_model_info("fireworks_ai/glm-5p3")
|
||||
assert model_info["key"] == "fireworks_ai/accounts/fireworks/models/glm-5p3"
|
||||
|
||||
model_info = litellm.get_model_info("glm-5p3", custom_llm_provider="fireworks_ai")
|
||||
assert model_info["key"] == "fireworks_ai/accounts/fireworks/models/glm-5p3"
|
||||
|
||||
model_info = litellm.get_model_info("fireworks_ai/glm-5p3-fast")
|
||||
assert model_info["key"] == "fireworks_ai/accounts/fireworks/routers/glm-5p3-fast"
|
||||
|
||||
model_info = litellm.get_model_info("fireworks_ai/nomic-ai/nomic-embed-text-v1.5")
|
||||
assert model_info["key"] == "fireworks_ai/nomic-ai/nomic-embed-text-v1.5"
|
||||
|
||||
with pytest.raises(Exception, match="isn't mapped"):
|
||||
litellm.get_model_info("fireworks_ai/does-not-exist")
|
||||
|
||||
|
||||
def test_get_model_info_bedrock_regional_profile_without_entry_falls_back_to_base(local_model_cost_map):
|
||||
"""A regional profile with no dedicated cost-map entry must still resolve to its
|
||||
region-stripped base entry."""
|
||||
info = litellm.get_model_info(model="bedrock/apac.anthropic.claude-opus-4-8")
|
||||
assert info["key"] == "anthropic.claude-opus-4-8"
|
||||
|
||||
|
||||
def test_get_model_info_gemini(monkeypatch):
|
||||
"""
|
||||
Tests if ALL gemini models have 'tpm' and 'rpm' in the model info
|
||||
"""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
model_map = litellm.model_cost
|
||||
for model, info in model_map.items():
|
||||
if (
|
||||
model.startswith("gemini/")
|
||||
and "gemma" not in model
|
||||
and "learnlm" not in model
|
||||
and "imagen" not in model
|
||||
and "veo" not in model
|
||||
and "lyria" not in model
|
||||
and "robotics" not in model
|
||||
):
|
||||
assert info.get("tpm") is not None, f"{model} does not have tpm"
|
||||
assert info.get("rpm") is not None, f"{model} does not have rpm"
|
||||
|
||||
|
||||
def test_get_model_info_resolves_provider_prefixed_model_ids(local_model_cost_map):
|
||||
"""Perplexity's Agent API third-party models are keyed `perplexity/perplexity/<id>`
|
||||
because Perplexity's own id already starts with `perplexity/`. Callers run
|
||||
`get_llm_provider` first, which hands `_get_potential_model_names` model
|
||||
`perplexity/glm-5.2` with provider `perplexity`, and every candidate but the
|
||||
provider-prefixed one strips that second `perplexity/` off. Regression: the
|
||||
entries were unreachable from `supports_reasoning` and from the cost calculator's
|
||||
per-token fallback, so a mapped model reported no reasoning support and raised
|
||||
"This model isn't mapped yet" on the only path where its rates are ever used."""
|
||||
for model, reasoning in (
|
||||
("perplexity/perplexity/glm-5.2", True),
|
||||
("perplexity/perplexity/kimi-k3", True),
|
||||
("perplexity/perplexity/deepseek-v4-flash-0731", True),
|
||||
("perplexity/perplexity/kimi-k2.7-code", False),
|
||||
("perplexity/perplexity/nemotron-3.5-lightning-30b-a3b", True),
|
||||
("perplexity/perplexity/nemotron-3-ultra-550b-a55b", True),
|
||||
):
|
||||
assert litellm.supports_reasoning(model=model) is reasoning, model
|
||||
|
||||
via_provider = litellm.get_model_info(model="perplexity/glm-5.2", custom_llm_provider="perplexity")
|
||||
assert via_provider["key"] == "perplexity/perplexity/glm-5.2"
|
||||
assert via_provider["mode"] == "responses"
|
||||
|
||||
lightning = litellm.get_model_info(
|
||||
model="perplexity/nemotron-3.5-lightning-30b-a3b", custom_llm_provider="perplexity"
|
||||
)
|
||||
assert lightning["key"] == "perplexity/perplexity/nemotron-3.5-lightning-30b-a3b"
|
||||
assert lightning["mode"] == "responses"
|
||||
|
||||
ultra = litellm.get_model_info(model="perplexity/perplexity/nemotron-3-ultra-550b-a55b")
|
||||
assert ultra["key"] == "perplexity/perplexity/nemotron-3-ultra-550b-a55b"
|
||||
|
||||
|
||||
def test_get_model_info_shows_supports_computer_use(monkeypatch):
|
||||
"""
|
||||
Tests if 'supports_computer_use' is correctly retrieved by get_model_info.
|
||||
We'll use 'claude-4-sonnet-20250514' as it's configured
|
||||
in the backup JSON to have supports_computer_use: True.
|
||||
"""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
# Ensure litellm.model_cost is loaded, relying on the backup mechanism if primary fails
|
||||
# as per previous debugging.
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
# This model should have 'supports_computer_use': True in the backup JSON
|
||||
model_known_to_support_computer_use = "claude-4-sonnet-20250514"
|
||||
info = litellm.get_model_info(model_known_to_support_computer_use)
|
||||
|
||||
# After the fix in utils.py, this should now be present and True
|
||||
assert info.get("supports_computer_use") is True
|
||||
|
||||
|
||||
def test_get_model_info_surfaces_supports_adaptive_thinking(local_model_cost_map):
|
||||
"""supports_adaptive_thinking must flow through get_model_info like every other
|
||||
capability flag: both from an explicit cost-map entry and from a
|
||||
fallback-generalization rule for an unmapped model. Regression: the field shipped
|
||||
in the JSON but was never declared on ModelInfo nor copied during construction, so
|
||||
get_model_info (and _supports_factory) silently dropped it for any provider-prefixed
|
||||
or unmapped name."""
|
||||
explicit = litellm.get_model_info(model="claude-opus-4-8")
|
||||
assert explicit["supports_adaptive_thinking"] is True
|
||||
|
||||
generalized = litellm.get_model_info(model="claude-opus-4-9", custom_llm_provider="anthropic")
|
||||
assert generalized["supports_adaptive_thinking"] is True
|
||||
|
||||
|
||||
def test_get_model_info_surfaces_supports_parallel_function_calling(local_model_cost_map):
|
||||
"""A registry entry's supports_parallel_function_calling must read back through get_model_info
|
||||
and litellm.supports_parallel_function_calling. Regression: the key was never copied into
|
||||
ModelInfo, so provider-prefixed entries read None / False even when the map said True, and an
|
||||
explicit False was indistinguishable from unset."""
|
||||
declared_true = litellm.get_model_info(model="together_ai/zai-org/GLM-5.3-Flash")
|
||||
assert declared_true["supports_parallel_function_calling"] is True
|
||||
assert litellm.supports_parallel_function_calling(model="together_ai/zai-org/GLM-5.3-Flash") is True
|
||||
|
||||
|
||||
def test_model_info_for_fireworks_short_form_models():
|
||||
"""
|
||||
Test that fireworks_ai short-form model entries (fireworks_ai/<model>)
|
||||
are correctly configured in model_prices_and_context_window.json.
|
||||
|
||||
These entries enable cost attribution for models called via short-form
|
||||
names (e.g., fireworks_ai/glm-4p7 instead of
|
||||
fireworks_ai/accounts/fireworks/models/glm-4p7).
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
# glm-4p7: short-form and long-form
|
||||
for key in [
|
||||
"fireworks_ai/glm-4p7",
|
||||
"fireworks_ai/accounts/fireworks/models/glm-4p7",
|
||||
]:
|
||||
info = model_cost.get(key)
|
||||
assert info is not None, f"{key} not found in model_prices_and_context_window.json"
|
||||
assert info["litellm_provider"] == "fireworks_ai"
|
||||
assert info["mode"] == "chat"
|
||||
assert info["supports_reasoning"] is True
|
||||
|
||||
# minimax-m2p1: short-form and long-form
|
||||
for key in [
|
||||
"fireworks_ai/minimax-m2p1",
|
||||
"fireworks_ai/accounts/fireworks/models/minimax-m2p1",
|
||||
]:
|
||||
info = model_cost.get(key)
|
||||
assert info is not None, f"{key} not found in model_prices_and_context_window.json"
|
||||
assert info["litellm_provider"] == "fireworks_ai"
|
||||
assert info["mode"] == "chat"
|
||||
|
||||
# kimi-k2p5: short-form only (long-form already existed)
|
||||
info = model_cost.get("fireworks_ai/kimi-k2p5")
|
||||
assert info is not None, "fireworks_ai/kimi-k2p5 not found in model_prices_and_context_window.json"
|
||||
assert info["litellm_provider"] == "fireworks_ai"
|
||||
assert info["mode"] == "chat"
|
||||
|
||||
|
||||
def test_model_info_for_vertex_ai_deepseek_model():
|
||||
model_info = litellm.get_model_info(model="vertex_ai/deepseek-ai/deepseek-r1-0528-maas")
|
||||
assert model_info is not None
|
||||
assert model_info["litellm_provider"] == "vertex_ai-deepseek_models"
|
||||
assert model_info["mode"] == "chat"
|
||||
|
||||
assert model_info["input_cost_per_token"] is not None
|
||||
assert model_info["output_cost_per_token"] is not None
|
||||
|
||||
|
||||
def test_provider_prefixed_lookup_never_outranks_an_existing_row(local_model_cost_map):
|
||||
"""The provider-prefixed candidate is tried last, after every candidate that
|
||||
already existed, so no model that resolves today can change answer. `perplexity/sonar`
|
||||
is the case that proves it: both `perplexity/sonar` and `perplexity/perplexity/sonar`
|
||||
are cost-map keys, and the shorter one must keep winning."""
|
||||
sonar = litellm.get_model_info(model="sonar", custom_llm_provider="perplexity")
|
||||
assert sonar["key"] == "perplexity/sonar"
|
||||
assert sonar["mode"] == "chat"
|
||||
|
||||
still_sonar = litellm.get_model_info(model="perplexity/sonar", custom_llm_provider="perplexity")
|
||||
assert still_sonar["key"] == "perplexity/sonar"
|
||||
assert still_sonar["mode"] == "chat"
|
||||
|
||||
for model, provider, expected_key in (
|
||||
("claude-sonnet-4-5", "anthropic", "claude-sonnet-4-5"),
|
||||
("anthropic/claude-sonnet-4-5", "anthropic", "claude-sonnet-4-5"),
|
||||
("gemini/gemini-2.0-flash", "gemini", "gemini/gemini-2.0-flash"),
|
||||
("openrouter/openai/gpt-4o", "openrouter", "openrouter/openai/gpt-4o"),
|
||||
):
|
||||
assert litellm.get_model_info(model=model, custom_llm_provider=provider)["key"] == expected_key
|
||||
|
|
|
|||
19
tests/test_litellm/test_xai_grok_4_3_model_metadata.py
Normal file
19
tests/test_litellm/test_xai_grok_4_3_model_metadata.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_xai_grok_4_3_backup_matches_main():
|
||||
"""Ensure the bundled model cost map stays in sync with the canonical file."""
|
||||
repo_root = Path(__file__).parents[2]
|
||||
main_path = repo_root / "model_prices_and_context_window.json"
|
||||
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
with open(main_path) as f:
|
||||
main_cost = json.load(f)
|
||||
with open(backup_path) as f:
|
||||
backup_cost = json.load(f)
|
||||
|
||||
for model in ("xai/grok-4.3", "xai/grok-4.3-latest"):
|
||||
assert backup_cost.get(model) == main_cost.get(model), (
|
||||
f"{model} differs between main and backup model cost maps"
|
||||
)
|
||||
Loading…
Add table
Reference in a new issue