mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-17 23:51:30 +00:00
Merge pull request #41269 from BerriAI/litellm_ban_vendor_fact_pinning_tests
test: drop tests that pin vendor facts and add the CLAUDE.md rule
This commit is contained in:
kerry-berri
GitHub
commit
7d3917dbf4
38 changed files with 8 additions and 1751 deletions
|
|
@ -25,6 +25,8 @@ Same thing for bug fixes. The tests should make it so that this specific bug can
|
|||
|
||||
Never test structure of code only function of it
|
||||
|
||||
A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent"; assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken
|
||||
|
||||
`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
|
||||
|
||||
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`
|
||||
|
|
|
|||
|
|
@ -2321,36 +2321,6 @@ def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities(_local_mo
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,expected_mode,expected_input,expected_output,expected_cache_read",
|
||||
[
|
||||
("azure/gpt-5.5", "chat", 5e-6, 3e-5, 5e-7),
|
||||
("azure/gpt-5.5-2026-04-23", "chat", 5e-6, 3e-5, 5e-7),
|
||||
("azure/gpt-5.5-pro", "responses", 3e-5, 1.8e-4, 3e-6),
|
||||
("azure/gpt-5.5-pro-2026-04-23", "responses", 3e-5, 1.8e-4, 3e-6),
|
||||
],
|
||||
)
|
||||
def test_azure_gpt55_entries_present_with_correct_pricing(_local_model_cost_map,
|
||||
model, expected_mode, expected_input, expected_output, expected_cache_read
|
||||
):
|
||||
"""Day-0 Azure entries for GPT-5.5 mirror the OpenAI pricing structure.
|
||||
|
||||
Pricing parity with openai/gpt-5.5* (verified against OpenAI's pricing page
|
||||
on 2026-04-24): $5/$30 input/output per 1M for chat, $30/$180 for pro.
|
||||
Cache discount is 10% of input.
|
||||
"""
|
||||
|
||||
m = litellm.model_cost[model]
|
||||
assert m["litellm_provider"] == "azure"
|
||||
assert m["mode"] == expected_mode
|
||||
assert m["input_cost_per_token"] == expected_input
|
||||
assert m["output_cost_per_token"] == expected_output
|
||||
assert m["cache_read_input_token_cost"] == expected_cache_read
|
||||
# Long-context window inherited from gpt-5.4 / openai gpt-5.5.
|
||||
assert m["max_input_tokens"] == 1050000
|
||||
assert m["max_output_tokens"] == 128000
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,expected_none,expected_minimal,expected_xhigh",
|
||||
[
|
||||
|
|
@ -3414,8 +3384,6 @@ def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map):
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-5.4", "gpt-realtime-2.1", "gpt-realtime-2.1-mini"])
|
||||
@pytest.mark.parametrize("data_residency", ["eu", "us"])
|
||||
def test_data_residency_applies_uplift(data_residency, model, _local_model_cost_map):
|
||||
|
|
@ -4556,20 +4524,6 @@ GEMINI_DAY0_LAUNCH_PRICING = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_DAY0_LAUNCH_PRICING)
|
||||
def test_gemini_36_flash_and_35_flash_lite_launch_pricing(_local_model_cost_map, model, input_cost, output_cost, cache_read_cost):
|
||||
|
||||
model_cost_map = litellm.model_cost[model]
|
||||
assert model_cost_map["input_cost_per_token"] == input_cost
|
||||
assert model_cost_map["output_cost_per_token"] == output_cost
|
||||
assert model_cost_map["output_cost_per_reasoning_token"] == output_cost
|
||||
assert model_cost_map["cache_read_input_token_cost"] == cache_read_cost
|
||||
assert model_cost_map["mode"] == "chat"
|
||||
assert model_cost_map["supports_reasoning"] is True
|
||||
assert model_cost_map["supports_function_calling"] is True
|
||||
assert model_cost_map["max_input_tokens"] == 1048576
|
||||
|
||||
|
||||
def test_generic_cost_per_token_gemini_36_flash(_local_model_cost_map):
|
||||
|
||||
usage = Usage(
|
||||
|
|
@ -4598,44 +4552,6 @@ GEMINI_36_FLASH_SERVICE_TIER_PRICING = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"service_tier,input_rate,output_rate,cache_read_rate", GEMINI_36_FLASH_SERVICE_TIER_PRICING
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["gemini-3.6-flash", "gemini/gemini-3.6-flash", "vertex_ai/gemini-3.6-flash"]
|
||||
)
|
||||
def test_gemini_36_flash_service_tier_introductory_pricing(
|
||||
model, service_tier, input_rate, output_rate, cache_read_rate, _local_model_cost_map
|
||||
):
|
||||
"""Regression: every 3.6 Flash tier is on Google's introductory rates through 2026-12-31,
|
||||
so flex and priority requests must not be billed at the post-introductory rates."""
|
||||
usage = Usage(
|
||||
prompt_tokens=1_000,
|
||||
completion_tokens=500,
|
||||
total_tokens=1_500,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=200, text_tokens=800),
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
model=model.split("/")[-1],
|
||||
usage=usage,
|
||||
custom_llm_provider=model.split("/")[0] if "/" in model else "gemini",
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(800 * input_rate + 200 * cache_read_rate, rel=1e-9)
|
||||
assert completion_cost == pytest.approx(500 * output_rate, rel=1e-9)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["gemini-3.6-flash", "gemini/gemini-3.6-flash", "vertex_ai/gemini-3.6-flash"]
|
||||
)
|
||||
def test_gemini_36_flash_batch_introductory_pricing(model, _local_model_cost_map):
|
||||
model_cost_map = litellm.model_cost[model]
|
||||
assert model_cost_map["input_cost_per_token_batches"] == 3.75e-07
|
||||
assert model_cost_map["output_cost_per_token_batches"] == 1.875e-06
|
||||
|
||||
|
||||
def test_generic_cost_per_token_gemini_35_flash_lite(_local_model_cost_map):
|
||||
|
||||
usage = Usage(
|
||||
|
|
@ -4667,43 +4583,6 @@ GEMINI_35_FLASH_LITE_TIER_RATES_BY_SURFACE = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"custom_llm_provider,service_tier,input_rate,output_rate,cache_read_rate",
|
||||
GEMINI_35_FLASH_LITE_TIER_RATES_BY_SURFACE,
|
||||
)
|
||||
def test_gemini_35_flash_lite_service_tier_pricing(
|
||||
custom_llm_provider, service_tier, input_rate, output_rate, cache_read_rate, _local_model_cost_map
|
||||
):
|
||||
"""Regression: Vertex publishes flash-lite flex context caching at $0.015/M while the
|
||||
Gemini API publishes $0.02/M, so vertex_ai flex cache reads must bill 1.5e-08/token
|
||||
instead of the 2e-08 the map used to carry, without disturbing the Gemini API rate."""
|
||||
usage = Usage(
|
||||
prompt_tokens=1_000,
|
||||
completion_tokens=500,
|
||||
total_tokens=1_500,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=200, text_tokens=800),
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
model="gemini-3.5-flash-lite",
|
||||
usage=usage,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(800 * input_rate + 200 * cache_read_rate, rel=1e-9)
|
||||
assert completion_cost == pytest.approx(500 * output_rate, rel=1e-9)
|
||||
|
||||
|
||||
def test_gemini_35_flash_lite_flex_cache_read_map_entries(_local_model_cost_map):
|
||||
"""Each map entry carries its own surface's published flex cache-read rate: the bare
|
||||
and vertex_ai keys are the Vertex surface at $0.015/M, the gemini key is the Gemini
|
||||
API surface at $0.02/M."""
|
||||
assert litellm.model_cost["gemini-3.5-flash-lite"]["cache_read_input_token_cost_flex"] == 1.5e-08
|
||||
assert litellm.model_cost["vertex_ai/gemini-3.5-flash-lite"]["cache_read_input_token_cost_flex"] == 1.5e-08
|
||||
assert litellm.model_cost["gemini/gemini-3.5-flash-lite"]["cache_read_input_token_cost_flex"] == 2e-08
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"service_tier,input_rate,cache_read_rate,cache_write_rate,output_rate",
|
||||
[
|
||||
|
|
@ -4932,19 +4811,6 @@ GEMINI_37_FLASH_LAUNCH_PRICING = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_37_FLASH_LAUNCH_PRICING)
|
||||
def test_gemini_37_flash_launch_pricing(model, input_cost, output_cost, cache_read_cost, _local_model_cost_map):
|
||||
model_cost_map = litellm.model_cost[model]
|
||||
assert model_cost_map["input_cost_per_token"] == input_cost
|
||||
assert model_cost_map["output_cost_per_token"] == output_cost
|
||||
assert model_cost_map["output_cost_per_reasoning_token"] == output_cost
|
||||
assert model_cost_map["cache_read_input_token_cost"] == cache_read_cost
|
||||
assert model_cost_map["mode"] == "chat"
|
||||
assert model_cost_map["supports_reasoning"] is True
|
||||
assert model_cost_map["supports_function_calling"] is True
|
||||
assert model_cost_map["max_input_tokens"] == 1048576
|
||||
|
||||
|
||||
def test_generic_cost_per_token_gemini_37_flash(_local_model_cost_map):
|
||||
usage = Usage(
|
||||
prompt_tokens=1000,
|
||||
|
|
@ -4972,19 +4838,6 @@ GEMINI_38_FLASH_LAUNCH_PRICING = [
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_38_FLASH_LAUNCH_PRICING)
|
||||
def test_gemini_38_flash_launch_pricing(model, input_cost, output_cost, cache_read_cost, _local_model_cost_map):
|
||||
model_cost_map = litellm.model_cost[model]
|
||||
assert model_cost_map["input_cost_per_token"] == input_cost
|
||||
assert model_cost_map["output_cost_per_token"] == output_cost
|
||||
assert model_cost_map["output_cost_per_reasoning_token"] == output_cost
|
||||
assert model_cost_map["cache_read_input_token_cost"] == cache_read_cost
|
||||
assert model_cost_map["mode"] == "chat"
|
||||
assert model_cost_map["supports_reasoning"] is True
|
||||
assert model_cost_map["supports_function_calling"] is True
|
||||
assert model_cost_map["max_input_tokens"] == 1048576
|
||||
|
||||
|
||||
GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH = (
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
|
|
@ -5045,20 +4898,6 @@ def test_generic_cost_per_token_gemini_38_flash(_local_model_cost_map):
|
|||
assert completion_cost == pytest.approx(0.001875)
|
||||
|
||||
|
||||
def test_grok_46_launch_pricing(_local_model_cost_map):
|
||||
model_cost_map = litellm.model_cost["xai/grok-4.6"]
|
||||
assert model_cost_map["input_cost_per_token"] == 2e-06
|
||||
assert model_cost_map["output_cost_per_token"] == 6e-06
|
||||
assert model_cost_map["cache_read_input_token_cost"] == 5e-07
|
||||
assert model_cost_map["input_cost_per_token_above_200k_tokens"] == 4e-06
|
||||
assert model_cost_map["output_cost_per_token_above_200k_tokens"] == 1.2e-05
|
||||
assert model_cost_map["cache_read_input_token_cost_above_200k_tokens"] == 1e-06
|
||||
assert model_cost_map["mode"] == "chat"
|
||||
assert model_cost_map["supports_reasoning"] is True
|
||||
assert model_cost_map["supports_function_calling"] is True
|
||||
assert model_cost_map["max_input_tokens"] == 500000
|
||||
|
||||
|
||||
def test_generic_cost_per_token_grok_46(_local_model_cost_map):
|
||||
usage = Usage(
|
||||
prompt_tokens=1_000,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -892,29 +890,6 @@ def test_gpt_4o_mini_snapshot_bills_web_search_like_its_alias(
|
|||
assert snapshot_cost == alias_cost == 0.025
|
||||
|
||||
|
||||
def test_gpt_4o_mini_web_search_price_matches_in_both_cost_maps():
|
||||
repo_root = Path(__file__).parents[4]
|
||||
cost_maps = tuple(
|
||||
json.loads((repo_root / path).read_text(encoding="utf-8"))
|
||||
for path in (
|
||||
"model_prices_and_context_window.json",
|
||||
"litellm/model_prices_and_context_window_backup.json",
|
||||
)
|
||||
)
|
||||
canonical, backup = cost_maps
|
||||
expected_search_price = {
|
||||
"search_context_size_low": 0.025,
|
||||
"search_context_size_medium": 0.025,
|
||||
"search_context_size_high": 0.025,
|
||||
}
|
||||
for model_name in ("gpt-4o-mini", "gpt-4o-mini-2024-07-18"):
|
||||
canonical_entry = canonical[model_name]
|
||||
backup_entry = backup[model_name]
|
||||
assert canonical_entry["search_context_cost_per_query"] == expected_search_price
|
||||
assert backup_entry["search_context_cost_per_query"] == expected_search_price
|
||||
assert canonical_entry == backup_entry
|
||||
|
||||
|
||||
# Note: File search integration test removed due to complex annotation detection logic
|
||||
# The unit tests in test_azure_assistant_cost_tracking.py provide comprehensive coverage
|
||||
|
||||
|
|
|
|||
|
|
@ -627,17 +627,6 @@ def test_shipped_adaptive_rule_requires_claude_prefix(shipped_cost_map):
|
|||
litellm.get_model_info(model)
|
||||
|
||||
|
||||
def test_shipped_exact_entry_beats_rules(shipped_cost_map):
|
||||
model = "us.anthropic.claude-sonnet-4-6"
|
||||
assert model in litellm.model_cost
|
||||
info = litellm.get_model_info(model, custom_llm_provider="bedrock")
|
||||
assert info["litellm_provider"] == "bedrock_converse"
|
||||
assert info["input_cost_per_token"] == 3.3e-06
|
||||
assert info["max_input_tokens"] == 1000000
|
||||
assert info["supports_adaptive_thinking"] is True
|
||||
assert info.get("supports_mid_conversation_system") is None
|
||||
|
||||
|
||||
def test_shipped_rules_lose_to_exact_entries_across_cost_ladder_variants(shipped_cost_map):
|
||||
"""A route-mangled variant of an exactly-mapped model must never resolve from
|
||||
rules. The cost calculator tries model-name variants in order; a rule-derived
|
||||
|
|
|
|||
|
|
@ -225,36 +225,6 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
|
|||
assert "supports_adaptive_thinking" not in backup[non_adaptive], non_adaptive
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cost_map",
|
||||
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
|
||||
ids=["root", "bundled_backup"],
|
||||
)
|
||||
def test_azure_ai_claude_1m_context_entries(cost_map: dict):
|
||||
"""Microsoft Foundry serves a 1M-token context window for Opus 4.6+ and Sonnet
|
||||
4.6+, so the ``azure_ai`` entries must not advertise the 200k cap that made
|
||||
context-aware clients compact prompts early (LIT-4406). Both the root map (used
|
||||
by default network loading) and the bundled fallback are checked so the two can
|
||||
never drift apart."""
|
||||
for model in [
|
||||
"azure_ai/claude-opus-4-6",
|
||||
"azure_ai/claude-opus-4-7",
|
||||
"azure_ai/claude-opus-4-8",
|
||||
"azure_ai/claude-opus-5",
|
||||
"azure_ai/claude-sonnet-5",
|
||||
"azure_ai/claude-sonnet-4-6",
|
||||
]:
|
||||
assert cost_map[model]["max_input_tokens"] == 1000000, model
|
||||
|
||||
for model in [
|
||||
"azure_ai/claude-opus-4-1",
|
||||
"azure_ai/claude-opus-4-5",
|
||||
"azure_ai/claude-sonnet-4-5",
|
||||
"azure_ai/claude-haiku-4-5",
|
||||
]:
|
||||
assert cost_map[model]["max_input_tokens"] == 200000, model
|
||||
|
||||
|
||||
# OpenRouter headline rates from GET https://openrouter.ai/api/v1/models.
|
||||
# These were the catalog values that disagreed with that API (and, for the
|
||||
# two spotlight models, the public model pages that their source fields cite).
|
||||
|
|
@ -278,34 +248,6 @@ _OPENROUTER_STALE_COSTS = {
|
|||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cost_map",
|
||||
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
|
||||
ids=["root", "bundled_backup"],
|
||||
)
|
||||
def test_openrouter_catalog_costs_match_live_headline_rates(cost_map: dict):
|
||||
"""openrouter/* spend tracking reads these catalog fields. The values must
|
||||
stay aligned with OpenRouter's published headline rate, not the stale
|
||||
figures that over/under-counted by up to 30x. Both maps are checked so
|
||||
the root file and bundled backup cannot drift apart."""
|
||||
control = cost_map["openrouter/anthropic/claude-opus-5"]
|
||||
assert control["input_cost_per_token"] == 5e-06
|
||||
assert control["output_cost_per_token"] == 2.5e-05
|
||||
assert control["cache_read_input_token_cost"] == 5e-07
|
||||
|
||||
for model, (inp, out, cache) in _OPENROUTER_LIVE_COSTS.items():
|
||||
entry = cost_map[model]
|
||||
assert entry["input_cost_per_token"] == inp, model
|
||||
assert entry["output_cost_per_token"] == out, model
|
||||
if cache is not None:
|
||||
assert entry["cache_read_input_token_cost"] == cache, model
|
||||
|
||||
for model, (stale_in, stale_out) in _OPENROUTER_STALE_COSTS.items():
|
||||
entry = cost_map[model]
|
||||
assert entry["input_cost_per_token"] != stale_in, model
|
||||
assert entry["output_cost_per_token"] != stale_out, model
|
||||
|
||||
|
||||
def test_get_model_cost_map_stamps_loaded_at():
|
||||
"""The load time feeds each pod's reload-due decision; a load that does not stamp it
|
||||
would make manual reload requests race the proxy's startup"""
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ REPO_ROOT: Final = Path(__file__).parents[4]
|
|||
MAIN_COST_MAP: Final = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_COST_MAP: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
AZURE_PRICING_PREFIX: Final = "https://azure.microsoft.com/en-us/pricing/details/"
|
||||
A_MILLION: Final = 1_000_000
|
||||
AN_HOUR_IN_SECONDS: Final = 3600
|
||||
|
||||
|
|
@ -76,7 +75,9 @@ def test_azure_ai_catalog_name_prices_the_same_in_any_casing(catalog_name: str)
|
|||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize("catalog_name", GROK_4_20_NAMES)
|
||||
def test_azure_ai_grok_4_20_bills_cached_prompt_tokens_at_the_input_price(catalog_name: str) -> None:
|
||||
uncached_prompt_cost, _ = cost_per_token(model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0)
|
||||
uncached_prompt_cost, _ = cost_per_token(
|
||||
model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0
|
||||
)
|
||||
cached_prompt_cost, _ = cost_per_token(
|
||||
model=f"azure_ai/{catalog_name}",
|
||||
prompt_tokens=A_MILLION,
|
||||
|
|
@ -100,7 +101,6 @@ def test_azure_ai_catalog_entry_source_and_backup_match(catalog_name: str) -> No
|
|||
main_entry = _cost_map_entry(MAIN_COST_MAP, catalog_name)
|
||||
backup_entry = _cost_map_entry(BACKUP_COST_MAP, catalog_name)
|
||||
|
||||
assert str(main_entry["source"]).startswith(AZURE_PRICING_PREFIX)
|
||||
assert backup_entry == main_entry
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
|
@ -24,19 +23,6 @@ def _ocr_response(model: str, pages_processed: int) -> OCRResponse:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cost_map_path", COST_MAPS, ids=lambda path: path.name)
|
||||
@pytest.mark.parametrize("model, provider", MODELS)
|
||||
def test_pricing_entry(cost_map_path: Path, model: str, provider: str) -> None:
|
||||
with open(cost_map_path) as f:
|
||||
info = json.load(f).get(model)
|
||||
|
||||
assert info is not None, f"{model} missing from {cost_map_path.name}"
|
||||
assert info["litellm_provider"] == provider
|
||||
assert info["mode"] == "ocr"
|
||||
assert info["supported_endpoints"] == ["/v1/ocr"]
|
||||
assert info["ocr_cost_per_page"] == COST_PER_PAGE
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model, provider", MODELS)
|
||||
def test_model_info_resolves_ocr_mode_and_price(local_model_cost_map, model: str, provider: str) -> None:
|
||||
info = litellm.get_model_info(model=model, custom_llm_provider=provider)
|
||||
|
|
|
|||
|
|
@ -163,23 +163,6 @@ def test_legacy_endpoint_names_still_resolve(local_model_cost_map: None) -> None
|
|||
assert completion_cost == pytest.approx(100 * info["output_cost_per_token"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", NEW_MODELS)
|
||||
def test_new_models_price_at_published_dbu_rates(local_model_cost_map: None, model: str) -> None:
|
||||
info: Final = _model_info(model)
|
||||
|
||||
for field, dbu_per_million in zip(PRICE_FIELDS, PUBLISHED_DBU_PER_MILLION[model]):
|
||||
assert info[field] == _dollars_per_token(dbu_per_million), field
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", sorted(set(PUBLISHED_DBU_PER_MILLION) - set(ENTRIES_STORING_PROMOTIONAL_RATE)))
|
||||
def test_cache_rates_derive_from_published_cache_dbu(local_model_cost_map: None, model: str) -> None:
|
||||
info: Final = _model_info(model)
|
||||
cache_dbu_per_million: Final = PUBLISHED_DBU_PER_MILLION[model][2:]
|
||||
|
||||
for field, dbu_per_million in zip(CACHE_FIELDS, cache_dbu_per_million):
|
||||
assert info[field] == _dollars_per_token(dbu_per_million), field
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", NEW_MODELS)
|
||||
def test_new_models_carry_cache_pricing(local_model_cost_map: None, model: str) -> None:
|
||||
info: Final = _model_info(model)
|
||||
|
|
@ -255,38 +238,3 @@ def test_sonnet_5_ships_standard_rates_not_introductory(local_model_cost_map: No
|
|||
|
||||
for field in PRICE_FIELDS:
|
||||
assert sonnet_5[field] == pytest.approx(sonnet_4_6[field]), field
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ENTRIES_STORING_PROMOTIONAL_RATE)
|
||||
def test_entries_storing_the_promotional_rate_price_below_the_published_table(
|
||||
local_model_cost_map: None,
|
||||
model: str,
|
||||
) -> None:
|
||||
info: Final = _model_info(model)
|
||||
input_dbu, output_dbu, _, _ = PUBLISHED_DBU_PER_MILLION[model]
|
||||
expiry_hint: Final = f"the gemini promotion expires {PROMOTION_EXPIRES}, after which the list rate applies"
|
||||
|
||||
assert info["input_cost_per_token"] == pytest.approx(
|
||||
_dollars_per_token(input_dbu) * PROMOTIONAL_DISCOUNT, rel=2e-4
|
||||
), expiry_hint
|
||||
assert info["output_cost_per_token"] == pytest.approx(
|
||||
_dollars_per_token(output_dbu) * PROMOTIONAL_DISCOUNT, rel=2e-4
|
||||
), expiry_hint
|
||||
assert info["cache_creation_input_token_cost"] == pytest.approx(info["input_cost_per_token"])
|
||||
assert info["cache_read_input_token_cost"] == pytest.approx(0.1 * info["input_cost_per_token"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ENTRIES_STORING_LIST_RATE_DESPITE_PROMOTION)
|
||||
def test_entries_storing_the_list_rate_bill_above_the_promotional_price(
|
||||
local_model_cost_map: None,
|
||||
model: str,
|
||||
) -> None:
|
||||
info: Final = _model_info(model)
|
||||
input_dbu, _, _, _ = PUBLISHED_DBU_PER_MILLION[model]
|
||||
list_rate: Final = _dollars_per_token(input_dbu)
|
||||
|
||||
assert info["input_cost_per_token"] == pytest.approx(list_rate, rel=2e-4), (
|
||||
f"{model} moved off the list rate; if it now stores the discount that runs to "
|
||||
f"{PROMOTION_EXPIRES}, move it into ENTRIES_STORING_PROMOTIONAL_RATE"
|
||||
)
|
||||
assert info["cache_creation_input_token_cost"] == pytest.approx(info["input_cost_per_token"])
|
||||
|
|
|
|||
|
|
@ -452,33 +452,6 @@ def test_map_traffic_type_to_service_tier(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,custom_llm_provider,expected_cache_read_cost",
|
||||
[
|
||||
("gemini/gemini-flash-latest", "gemini", 3e-08),
|
||||
("gemini/gemini-flash-lite-latest", "gemini", 1e-08),
|
||||
("gemini/gemini-2.5-flash-preview-09-2025", "gemini", 3e-08),
|
||||
("gemini/gemini-2.5-flash-lite-preview-06-17", "gemini", 1e-08),
|
||||
("vertex_ai/gemini-2.5-flash-preview-09-2025", "vertex_ai", 3e-08),
|
||||
("vertex_ai/gemini-2.5-flash-lite-preview-06-17", "vertex_ai", 1e-08),
|
||||
],
|
||||
)
|
||||
def test_flash_alias_cache_read_is_ten_percent_of_input(
|
||||
monkeypatch, model, custom_llm_provider, expected_cache_read_cost
|
||||
):
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
|
||||
model_info = litellm.get_model_info(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
assert model_info["cache_read_input_token_cost"] == expected_cache_read_cost
|
||||
assert model_info["cache_read_input_token_cost"] == pytest.approx(
|
||||
0.10 * model_info["input_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prefixed,bare",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ for mistral-ocr-4-0 and mistral-ocr-latest, which now both resolve to
|
|||
OCR 4 at $4 / 1000 pages.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
|
@ -45,12 +44,6 @@ def _annotated_ocr_response(model: str, pages_processed: int | None, annotation_
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["mistral-ocr-4-0", "mistral-ocr-latest"])
|
||||
def test_model_info_ocr4_price(model: str) -> None:
|
||||
info = litellm.get_model_info(model=f"mistral/{model}", custom_llm_provider="mistral")
|
||||
assert info["ocr_cost_per_page"] == OCR4_COST_PER_PAGE
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["mistral-ocr-4-0", "mistral-ocr-latest"])
|
||||
@pytest.mark.parametrize("pages_processed", [1, 3, 10])
|
||||
def test_ocr4_cost_scales_with_pages(model: str, pages_processed: int) -> None:
|
||||
|
|
@ -63,20 +56,6 @@ def test_ocr4_cost_scales_with_pages(model: str, pages_processed: int) -> None:
|
|||
assert cost == pytest.approx(OCR4_COST_PER_PAGE * pages_processed)
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cost_map_path", [MAIN_COST_MAP, BACKUP_COST_MAP])
|
||||
def test_ocr3_pricing_entry(cost_map_path: Path) -> None:
|
||||
with open(cost_map_path) as f:
|
||||
info = json.load(f).get(OCR3_MODEL)
|
||||
|
||||
assert info is not None, f"{OCR3_MODEL} missing from {cost_map_path.name}"
|
||||
assert info["litellm_provider"] == "mistral"
|
||||
assert info["mode"] == "ocr"
|
||||
assert info["supported_endpoints"] == ["/v1/ocr"]
|
||||
assert info["ocr_cost_per_page"] == OCR3_COST_PER_PAGE
|
||||
assert info["annotation_cost_per_page"] == OCR3_ANNOTATION_COST_PER_PAGE
|
||||
|
||||
|
||||
def test_ocr3_model_info_price(local_model_cost_map) -> None:
|
||||
info = litellm.get_model_info(model=OCR3_MODEL, custom_llm_provider="mistral")
|
||||
assert info["ocr_cost_per_page"] == OCR3_COST_PER_PAGE
|
||||
|
|
|
|||
|
|
@ -54,9 +54,6 @@ CODE_SLUGS = (
|
|||
"xai/grok-code-fast-1",
|
||||
"xai/grok-code-fast-1-0825",
|
||||
)
|
||||
RETIREMENT_DATE = "2026-05-15"
|
||||
GROK_3_MINI_RETIREMENT_DATE = "2026-02-28"
|
||||
|
||||
BASE_COST_FIELDS = ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost")
|
||||
TIER_COST_FIELDS = (
|
||||
"input_cost_per_token_above_200k_tokens",
|
||||
|
|
@ -65,10 +62,6 @@ TIER_COST_FIELDS = (
|
|||
)
|
||||
|
||||
|
||||
def expected_retirement_date(slug: str) -> str:
|
||||
return GROK_3_MINI_RETIREMENT_DATE if slug in GROK_3_MINI_SLUGS else RETIREMENT_DATE
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=[p.name for p in MAP_PATHS])
|
||||
def cost_map(request: pytest.FixtureRequest) -> dict:
|
||||
path = next(p for p in MAP_PATHS if p.name == request.param)
|
||||
|
|
@ -92,15 +85,9 @@ def test_code_slug_bills_at_grok_build_rate(cost_map: dict, slug: str):
|
|||
assert entry[field] == target[field], field
|
||||
|
||||
|
||||
@pytest.mark.parametrize("slug", (*REDIRECTED_SLUGS, *CODE_SLUGS))
|
||||
def test_redirected_slug_keeps_its_retirement_date(cost_map: dict, slug: str):
|
||||
assert cost_map[slug]["deprecation_date"] == expected_retirement_date(slug)
|
||||
|
||||
|
||||
def test_a_live_xai_model_is_untouched(cost_map: dict):
|
||||
"""Guard against the repricing leaking onto models xAI still serves directly."""
|
||||
assert cost_map["xai/grok-4.6"]["input_cost_per_token"] != cost_map[REDIRECT_TARGET]["input_cost_per_token"]
|
||||
assert "deprecation_date" not in cost_map["xai/grok-4.6"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("slug", REDIRECTED_SLUGS)
|
||||
|
|
|
|||
|
|
@ -1,142 +0,0 @@
|
|||
"""
|
||||
Validate that the native (first-party) Anthropic Claude Sonnet 4.5 / 4.6 entries
|
||||
carry the 1-hour prompt-cache write tier (`cache_creation_input_token_cost_above_1hr`)
|
||||
in `model_prices_and_context_window.json`.
|
||||
|
||||
Anthropic's first-party API charges a separate 1-hour cache write rate (2x base
|
||||
input) alongside the 5-minute write (1.25x base input) and cache read (0.1x base
|
||||
input). The 1h/5m ratio is therefore 1.6. Without the 1-hour field, cost tracking
|
||||
on 1-hour-TTL prompt caching falls back to the 5-minute rate and undercounts spend.
|
||||
|
||||
The native (non-bedrock) `claude-sonnet-4-5*` / `claude-sonnet-4-6` entries were
|
||||
missing this field, while every sibling (`vertex_ai/`, `azure_ai/`, the
|
||||
`*.anthropic.*` Bedrock profiles) and the older `claude-sonnet-4-20250514` already
|
||||
carried it. This test guards against regression.
|
||||
|
||||
Values (per token):
|
||||
Sonnet base input 3e-06 -> 5m 3.75e-06, 1h 6e-06
|
||||
Sonnet 4.5 long-context (>200K) base 6e-06 -> 5m 7.5e-06, 1h 1.2e-05
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def model_data():
|
||||
json_path = os.path.join(
|
||||
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
|
||||
)
|
||||
with open(json_path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
# (model_key, expected 1hr write per token, expected 1hr long-context tier or None)
|
||||
EXPECTED = [
|
||||
("claude-sonnet-4-5", 6e-06, 1.2e-05),
|
||||
("claude-sonnet-4-5-20250929", 6e-06, 1.2e-05),
|
||||
("claude-sonnet-4-5-20250929-v1:0", 6e-06, 1.2e-05),
|
||||
("claude-sonnet-4-6", 6e-06, None),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_key, expected_1hr, expected_1hr_lc", EXPECTED)
|
||||
def test_anthropic_sonnet_1hr_cache_write_pricing(
|
||||
model_data, model_key, expected_1hr, expected_1hr_lc
|
||||
):
|
||||
assert model_key in model_data, f"Missing model entry: {model_key}"
|
||||
info = model_data[model_key]
|
||||
|
||||
# Regular 1hr cache write rate must be present and exact.
|
||||
assert "cache_creation_input_token_cost_above_1hr" in info, (
|
||||
f"{model_key}: missing cache_creation_input_token_cost_above_1hr - "
|
||||
"Anthropic charges a separate 1-hour cache write rate for this model"
|
||||
)
|
||||
assert info["cache_creation_input_token_cost_above_1hr"] == expected_1hr, (
|
||||
f"{model_key}: 1hr cache write rate "
|
||||
f"{info['cache_creation_input_token_cost_above_1hr']} does not match "
|
||||
f"expected {expected_1hr}"
|
||||
)
|
||||
|
||||
# 1hr write must be 1.6x the 5-minute write (Anthropic 2x-base / 1.25x-base).
|
||||
ratio = (
|
||||
info["cache_creation_input_token_cost_above_1hr"]
|
||||
/ info["cache_creation_input_token_cost"]
|
||||
)
|
||||
assert (
|
||||
abs(ratio - 1.6) < 1e-9
|
||||
), f"{model_key}: 1hr/5min ratio is {ratio}, expected 1.6"
|
||||
|
||||
# Long-context (>200K) 1hr tier, where the model publishes a >200K tier.
|
||||
if expected_1hr_lc is not None:
|
||||
assert (
|
||||
"cache_creation_input_token_cost_above_1hr_above_200k_tokens" in info
|
||||
), f"{model_key}: missing 1hr cache write tier for >200K context"
|
||||
assert (
|
||||
info["cache_creation_input_token_cost_above_1hr_above_200k_tokens"]
|
||||
== expected_1hr_lc
|
||||
)
|
||||
ratio_lc = (
|
||||
info["cache_creation_input_token_cost_above_1hr_above_200k_tokens"]
|
||||
/ info["cache_creation_input_token_cost_above_200k_tokens"]
|
||||
)
|
||||
assert (
|
||||
abs(ratio_lc - 1.6) < 1e-9
|
||||
), f"{model_key}: long-context 1hr/5min ratio is {ratio_lc}, expected 1.6"
|
||||
else:
|
||||
assert "cache_creation_input_token_cost_above_1hr_above_200k_tokens" not in info
|
||||
|
||||
|
||||
CLAUDE_3_EXPECTED = [
|
||||
("claude-3-haiku-20240307", 5e-07),
|
||||
("claude-3-opus-20240229", 3e-05),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_key, expected_1hr", CLAUDE_3_EXPECTED)
|
||||
def test_claude_3_1hr_cache_write_pricing(model_data, model_key, expected_1hr):
|
||||
"""Haiku 3 and Opus 3 both carried Sonnet's 6e-06 1hr rate, overbilling Haiku 3
|
||||
1-hour cache writes 12x and underbilling Opus 3 5x."""
|
||||
info = model_data[model_key]
|
||||
|
||||
assert info["cache_creation_input_token_cost_above_1hr"] == expected_1hr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_key, expected_1hr", CLAUDE_3_EXPECTED)
|
||||
def test_backup_matches_main_for_claude_3_1hr_cache_write(model_key, expected_1hr):
|
||||
json_path = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"../../litellm/model_prices_and_context_window_backup.json",
|
||||
)
|
||||
with open(json_path) as f:
|
||||
backup = json.load(f)
|
||||
|
||||
assert (
|
||||
backup[model_key]["cache_creation_input_token_cost_above_1hr"] == expected_1hr
|
||||
)
|
||||
|
||||
|
||||
def test_first_party_anthropic_1hr_cache_writes_are_2x_base_input(model_data):
|
||||
"""Anthropic charges 1-hour cache writes at 2x base input for every first-party
|
||||
model, so any entry that drifts off that multiple is a copy-paste error."""
|
||||
offenders = tuple(
|
||||
(
|
||||
model_key,
|
||||
info["input_cost_per_token"],
|
||||
info["cache_creation_input_token_cost_above_1hr"],
|
||||
)
|
||||
for model_key, info in model_data.items()
|
||||
if isinstance(info, dict)
|
||||
and info.get("litellm_provider") == "anthropic"
|
||||
and info.get("input_cost_per_token")
|
||||
and info.get("cache_creation_input_token_cost_above_1hr")
|
||||
and abs(
|
||||
info["cache_creation_input_token_cost_above_1hr"]
|
||||
- 2 * info["input_cost_per_token"]
|
||||
)
|
||||
> 1e-12
|
||||
)
|
||||
|
||||
assert offenders == (), f"1hr cache write is not 2x base input for: {offenders}"
|
||||
|
|
@ -5,7 +5,6 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm import get_model_info
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
AZURE_AI_GROK_4_3_MODEL = "azure_ai/grok-4.3"
|
||||
AZURE_AI_GROK_4_3_SOURCE = "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-grok-4-3-on-microsoft-foundry-latest-generation-agentic-capabilities/4517096"
|
||||
|
|
@ -27,49 +26,6 @@ def reload_model_costs():
|
|||
get_model_info.cache_clear()
|
||||
|
||||
|
||||
def test_azure_ai_grok_4_3_model_info():
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
model_cost = _load_model_cost(json_path)
|
||||
|
||||
info = model_cost.get(AZURE_AI_GROK_4_3_MODEL)
|
||||
assert (
|
||||
info is not None
|
||||
), f"{AZURE_AI_GROK_4_3_MODEL} not found in model_prices_and_context_window.json"
|
||||
|
||||
assert info["litellm_provider"] == "azure_ai"
|
||||
assert info["mode"] == "chat"
|
||||
|
||||
assert info["input_cost_per_token"] == 1.25e-06
|
||||
assert info["output_cost_per_token"] == 2.5e-06
|
||||
assert info["cache_read_input_token_cost"] == 2e-07
|
||||
|
||||
assert info["max_input_tokens"] == 200000
|
||||
assert info["max_output_tokens"] == 200000
|
||||
assert info["max_tokens"] == 200000
|
||||
assert info["source"] == AZURE_AI_GROK_4_3_SOURCE
|
||||
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_prompt_caching"] is True
|
||||
assert info["supports_reasoning"] is True
|
||||
assert info["supports_response_schema"] is True
|
||||
assert info["supports_tool_choice"] is True
|
||||
assert info["supports_vision"] is True
|
||||
assert info["supports_web_search"] is True
|
||||
|
||||
routed_model, provider, _, _ = get_llm_provider(model=AZURE_AI_GROK_4_3_MODEL)
|
||||
assert routed_model == "grok-4.3"
|
||||
assert provider == "azure_ai"
|
||||
|
||||
resolved_info = get_model_info(model="grok-4.3", custom_llm_provider="azure_ai")
|
||||
assert resolved_info["litellm_provider"] == "azure_ai"
|
||||
assert resolved_info["input_cost_per_token"] == info["input_cost_per_token"]
|
||||
assert resolved_info["output_cost_per_token"] == info["output_cost_per_token"]
|
||||
assert (
|
||||
resolved_info["cache_read_input_token_cost"]
|
||||
== info["cache_read_input_token_cost"]
|
||||
)
|
||||
|
||||
|
||||
def test_azure_ai_grok_4_3_backup_matches_main():
|
||||
repo_root = Path(__file__).parents[2]
|
||||
main_path = repo_root / "model_prices_and_context_window.json"
|
||||
|
|
|
|||
|
|
@ -9,10 +9,6 @@ from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
|||
|
||||
REPO_ROOT: Final = Path(__file__).parents[2]
|
||||
MODEL: Final = "azure_ai/grok-4.6"
|
||||
SOURCE: Final = (
|
||||
"https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/"
|
||||
"grok-4-6-comes-to-microsoft-foundry-models-built-for-long-horizon-reasoning-and-/4547578"
|
||||
)
|
||||
COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
|
||||
|
||||
|
|
@ -51,5 +47,4 @@ def test_azure_ai_grok_4_6_entry_source_and_backup_match() -> None:
|
|||
main_entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json")
|
||||
backup_entry = _cost_map_entry(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json")
|
||||
|
||||
assert main_entry["source"] == SOURCE
|
||||
assert backup_entry == main_entry
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ from pathlib import Path
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
from litellm.utils import supports_function_calling, supports_prompt_caching
|
||||
|
||||
|
|
@ -35,34 +34,6 @@ def local_model_cost_map(monkeypatch):
|
|||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
def test_baseten_glm_5_3_specs():
|
||||
info = _load(MAIN_PATH).get(MODEL)
|
||||
assert info is not None, f"{MODEL} missing from model_prices_and_context_window.json"
|
||||
|
||||
assert info["litellm_provider"] == "baseten"
|
||||
assert info["mode"] == "chat"
|
||||
|
||||
assert info["input_cost_per_token"] == INPUT_COST
|
||||
assert info["output_cost_per_token"] == OUTPUT_COST
|
||||
assert info["cache_read_input_token_cost"] == CACHED_INPUT_COST
|
||||
|
||||
assert info["max_input_tokens"] == 1048576
|
||||
assert info["max_output_tokens"] == 262144
|
||||
assert info["max_tokens"] == 262144
|
||||
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_prompt_caching"] is True
|
||||
assert info["supports_response_schema"] is True
|
||||
assert info["supports_tool_choice"] is True
|
||||
assert info["supports_vision"] is True
|
||||
assert info["supported_modalities"] == ["text", "image"]
|
||||
assert info["supported_output_modalities"] == ["text"]
|
||||
|
||||
routed_model, provider, _, _ = get_llm_provider(model=MODEL)
|
||||
assert routed_model == "zai-org/GLM-5.3"
|
||||
assert provider == "baseten"
|
||||
|
||||
|
||||
def test_baseten_glm_5_3_capabilities_are_visible_to_callers(local_model_cost_map):
|
||||
"""The entry advertises prompt caching and tool calling, so the helpers every
|
||||
caller checks before sending a request must say so too."""
|
||||
|
|
@ -108,43 +79,10 @@ def test_backup_matches_main():
|
|||
|
||||
|
||||
def test_entry_advertises_only_what_the_baseten_path_accepts(local_model_cost_map):
|
||||
"""The entry must not claim a capability whose request parameter BasetenConfig
|
||||
refuses.
|
||||
|
||||
``BasetenConfig.get_supported_openai_params`` returns one hardcoded list for every
|
||||
Baseten model, and it carries neither ``parallel_tool_calls`` nor
|
||||
``reasoning_effort``. Baseten's own Model API does take ``reasoning_effort``, but
|
||||
litellm's Baseten path drops it (``drop_params=True``) or raises
|
||||
``UnsupportedParamsError`` (``drop_params=False``), so declaring
|
||||
``supports_parallel_function_calling``, ``supports_reasoning`` or
|
||||
``reasoning_effort_levels`` here would advertise a level the gateway then refuses to
|
||||
send. Wiring those params through the Baseten config is separate work; until it
|
||||
lands, the registry stays honest.
|
||||
"""
|
||||
"""The Baseten path rejects unsupported request parameters."""
|
||||
supported = litellm.get_supported_openai_params(model="zai-org/GLM-5.3", custom_llm_provider="baseten")
|
||||
assert supported is not None
|
||||
|
||||
entry = _load(MAIN_PATH)[MODEL]
|
||||
|
||||
capability_to_param = {
|
||||
"supports_function_calling": "tools",
|
||||
"supports_tool_choice": "tool_choice",
|
||||
"supports_response_schema": "response_format",
|
||||
"supports_parallel_function_calling": "parallel_tool_calls",
|
||||
"supports_reasoning": "reasoning_effort",
|
||||
}
|
||||
for capability, param in capability_to_param.items():
|
||||
if entry.get(capability):
|
||||
assert param in supported, f"{MODEL} advertises {capability} but baseten drops/rejects {param}"
|
||||
|
||||
assert "reasoning_effort_levels" not in entry, (
|
||||
"reasoning_effort_levels advertises accepted reasoning_effort values, which the Baseten path does not accept"
|
||||
)
|
||||
assert "thinking_always_on" not in entry, (
|
||||
"thinking_always_on is only read by AnthropicModelInfo._is_always_on_thinking_model, "
|
||||
"which no Baseten route reaches"
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError):
|
||||
litellm.utils.get_optional_params(
|
||||
model="zai-org/GLM-5.3",
|
||||
|
|
|
|||
|
|
@ -1,154 +0,0 @@
|
|||
"""
|
||||
Validate that Bedrock-hosted Anthropic Claude 4.5/4.6/4.7 entries carry the
|
||||
1-hour prompt-cache write tier (`cache_creation_input_token_cost_above_1hr`)
|
||||
in `model_prices_and_context_window.json`.
|
||||
|
||||
AWS Bedrock pricing (https://aws.amazon.com/bedrock/pricing/) publishes a
|
||||
separate 1-hour cache write column for the Claude 4.5 / 4.6 / 4.7 family.
|
||||
Without these fields, cost tracking on Bedrock 1-hour-TTL prompt caching
|
||||
falls back to the 5-minute write rate and undercounts spend by ~60%.
|
||||
|
||||
Source values (per million tokens) for the 1-hour cache write column,
|
||||
as published on the AWS Bedrock pricing page:
|
||||
|
||||
Global pricing:
|
||||
Opus 4.7 / Opus 4.6 / Opus 4.5 -> $10.00
|
||||
Sonnet 4.6 / Sonnet 4.5 (regular tier) -> $6.00
|
||||
Sonnet 4.5 long-context (>200K tier) -> $12.00
|
||||
Haiku 4.5 -> $2.00
|
||||
|
||||
US pricing (10% premium over Global):
|
||||
Opus 4.7 / Opus 4.6 / Opus 4.5 -> $11.00
|
||||
Sonnet 4.6 / Sonnet 4.5 (regular tier) -> $6.60
|
||||
Sonnet 4.5 long-context (>200K tier) -> $13.20
|
||||
Haiku 4.5 -> $2.20
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def model_data():
|
||||
json_path = os.path.join(
|
||||
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
|
||||
)
|
||||
with open(json_path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
# (model_key, expected 1hr cache write per token, expected 1hr LC tier or None)
|
||||
GLOBAL_EXPECTED = [
|
||||
# Opus 4.7 - $10.00 / MTok
|
||||
("anthropic.claude-opus-4-7", 1e-05, None),
|
||||
("global.anthropic.claude-opus-4-7", 1e-05, None),
|
||||
# Opus 4.6 - $10.00 / MTok
|
||||
("anthropic.claude-opus-4-6-v1", 1e-05, None),
|
||||
("global.anthropic.claude-opus-4-6-v1", 1e-05, None),
|
||||
# Opus 4.5 - $10.00 / MTok
|
||||
("anthropic.claude-opus-4-5-20251101-v1:0", 1e-05, None),
|
||||
("global.anthropic.claude-opus-4-5-20251101-v1:0", 1e-05, None),
|
||||
# Sonnet 4.6 - $6.00 / MTok (no separate LC tier per AWS)
|
||||
("anthropic.claude-sonnet-4-6", 6e-06, None),
|
||||
("global.anthropic.claude-sonnet-4-6", 6e-06, None),
|
||||
# Sonnet 4.5 - $6.00 / MTok regular, $12.00 / MTok long-context (>200K)
|
||||
("anthropic.claude-sonnet-4-5-20250929-v1:0", 6e-06, 1.2e-05),
|
||||
("global.anthropic.claude-sonnet-4-5-20250929-v1:0", 6e-06, 1.2e-05),
|
||||
# Haiku 4.5 - $2.00 / MTok
|
||||
("anthropic.claude-haiku-4-5-20251001-v1:0", 2e-06, None),
|
||||
("anthropic.claude-haiku-4-5@20251001", 2e-06, None),
|
||||
("global.anthropic.claude-haiku-4-5-20251001-v1:0", 2e-06, None),
|
||||
]
|
||||
|
||||
US_EXPECTED = [
|
||||
# US is +10% over Global.
|
||||
("us.anthropic.claude-opus-4-7", 1.1e-05, None),
|
||||
("us.anthropic.claude-opus-4-6-v1", 1.1e-05, None),
|
||||
("us.anthropic.claude-opus-4-5-20251101-v1:0", 1.1e-05, None),
|
||||
("us.anthropic.claude-sonnet-4-6", 6.6e-06, None),
|
||||
("us.anthropic.claude-sonnet-4-5-20250929-v1:0", 6.6e-06, 1.32e-05),
|
||||
("us.anthropic.claude-haiku-4-5-20251001-v1:0", 2.2e-06, None),
|
||||
]
|
||||
|
||||
# EU/AU/JP cross-region inference profiles carry the same +10% regional
|
||||
# premium as US (per AWS Bedrock pricing). Coverage list filters to entries
|
||||
# that actually exist in the pricing JSON - e.g. Opus 4.6 has no JP profile.
|
||||
REGIONAL_EXPECTED = [
|
||||
# Opus 4.6 - $11.00 / MTok (eu/au only; no jp profile)
|
||||
("eu.anthropic.claude-opus-4-6-v1", 1.1e-05, None),
|
||||
("au.anthropic.claude-opus-4-6-v1", 1.1e-05, None),
|
||||
# Opus 4.7 - $11.00 / MTok (eu/au; jp is added in #28567)
|
||||
("eu.anthropic.claude-opus-4-7", 1.1e-05, None),
|
||||
("au.anthropic.claude-opus-4-7", 1.1e-05, None),
|
||||
# Sonnet 4.6 - $6.60 / MTok
|
||||
("eu.anthropic.claude-sonnet-4-6", 6.6e-06, None),
|
||||
("au.anthropic.claude-sonnet-4-6", 6.6e-06, None),
|
||||
("jp.anthropic.claude-sonnet-4-6", 6.6e-06, None),
|
||||
# Sonnet 4.5 - $6.60 / MTok with $13.20 / MTok long-context tier
|
||||
("eu.anthropic.claude-sonnet-4-5-20250929-v1:0", 6.6e-06, 1.32e-05),
|
||||
("au.anthropic.claude-sonnet-4-5-20250929-v1:0", 6.6e-06, 1.32e-05),
|
||||
("jp.anthropic.claude-sonnet-4-5-20250929-v1:0", 6.6e-06, 1.32e-05),
|
||||
# Haiku 4.5 - $2.20 / MTok
|
||||
("eu.anthropic.claude-haiku-4-5-20251001-v1:0", 2.2e-06, None),
|
||||
("au.anthropic.claude-haiku-4-5-20251001-v1:0", 2.2e-06, None),
|
||||
("jp.anthropic.claude-haiku-4-5-20251001-v1:0", 2.2e-06, None),
|
||||
# Note: eu.anthropic.claude-opus-4-5-20251101-v1:0 is intentionally NOT
|
||||
# in this list. The existing entry carries base/global 5m rates
|
||||
# (5e-06 / 6.25e-06) instead of the +10% regional premium (5.5e-06 /
|
||||
# 6.875e-06), which would make the 1.6x 5m-to-1h invariant fail.
|
||||
# Fixing the EU 5m rates first is left to a follow-up so this PR
|
||||
# stays scoped to the 1-hour cache tier addition.
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_key, expected_1hr, expected_1hr_lc",
|
||||
GLOBAL_EXPECTED + US_EXPECTED + REGIONAL_EXPECTED,
|
||||
)
|
||||
def test_bedrock_anthropic_1hr_cache_write_pricing(
|
||||
model_data, model_key, expected_1hr, expected_1hr_lc
|
||||
):
|
||||
assert model_key in model_data, f"Missing model entry: {model_key}"
|
||||
info = model_data[model_key]
|
||||
|
||||
# 1hr cache write rate must be present and exact.
|
||||
assert "cache_creation_input_token_cost_above_1hr" in info, (
|
||||
f"{model_key}: missing cache_creation_input_token_cost_above_1hr - "
|
||||
"AWS Bedrock charges a separate 1-hour cache write rate for this model"
|
||||
)
|
||||
assert info["cache_creation_input_token_cost_above_1hr"] == expected_1hr, (
|
||||
f"{model_key}: 1hr cache write rate "
|
||||
f"{info['cache_creation_input_token_cost_above_1hr']} does not match "
|
||||
f"expected {expected_1hr} from AWS Bedrock pricing"
|
||||
)
|
||||
|
||||
# 1hr cache write rate must be 1.6x the 5-minute rate (AWS standard ratio).
|
||||
five_min = info["cache_creation_input_token_cost"]
|
||||
ratio = info["cache_creation_input_token_cost_above_1hr"] / five_min
|
||||
assert (
|
||||
abs(ratio - 1.6) < 1e-9
|
||||
), f"{model_key}: 1hr/5min ratio is {ratio}, expected 1.6"
|
||||
|
||||
# Long-context (>200K) tier, where AWS publishes one.
|
||||
if expected_1hr_lc is not None:
|
||||
assert (
|
||||
"cache_creation_input_token_cost_above_1hr_above_200k_tokens" in info
|
||||
), f"{model_key}: missing 1hr cache write tier for >200K context"
|
||||
assert (
|
||||
info["cache_creation_input_token_cost_above_1hr_above_200k_tokens"]
|
||||
== expected_1hr_lc
|
||||
), (
|
||||
f"{model_key}: long-context 1hr cache write rate "
|
||||
f"{info['cache_creation_input_token_cost_above_1hr_above_200k_tokens']} "
|
||||
f"does not match expected {expected_1hr_lc}"
|
||||
)
|
||||
five_min_lc = info["cache_creation_input_token_cost_above_200k_tokens"]
|
||||
ratio_lc = (
|
||||
info["cache_creation_input_token_cost_above_1hr_above_200k_tokens"]
|
||||
/ five_min_lc
|
||||
)
|
||||
assert (
|
||||
abs(ratio_lc - 1.6) < 1e-9
|
||||
), f"{model_key}: long-context 1hr/5min ratio is {ratio_lc}, expected 1.6"
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
PRICING_FILES = (
|
||||
"model_prices_and_context_window.json",
|
||||
"litellm/model_prices_and_context_window_backup.json",
|
||||
)
|
||||
|
||||
BEDROCK_BATCH_MODELS = (
|
||||
"qwen.qwen3-235b-a22b-2507-v1:0",
|
||||
"anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"apac.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"au.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"eu.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"global.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"jp.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
"au.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
"claude-sonnet-4-5-20250929-v1:0",
|
||||
"eu.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
"global.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
"jp.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pricing_file", PRICING_FILES)
|
||||
@pytest.mark.parametrize("model", BEDROCK_BATCH_MODELS)
|
||||
def test_bedrock_batch_pricing_is_half_of_on_demand(
|
||||
pricing_file: str, model: str
|
||||
) -> None:
|
||||
model_cost_map = json.loads((Path(__file__).parents[2] / pricing_file).read_text())
|
||||
model_info = model_cost_map[model]
|
||||
|
||||
assert model_info["input_cost_per_token_batches"] == pytest.approx(
|
||||
model_info["input_cost_per_token"] / 2
|
||||
)
|
||||
assert model_info["output_cost_per_token_batches"] == pytest.approx(
|
||||
model_info["output_cost_per_token"] / 2
|
||||
)
|
||||
|
|
@ -5,7 +5,6 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm.constants import bedrock_embedding_models
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[2]
|
||||
|
|
@ -33,37 +32,6 @@ def _load(path):
|
|||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ALL_MODELS)
|
||||
def test_marengo_embed_3_specs(model):
|
||||
info = _load(MAIN_PATH).get(model)
|
||||
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
|
||||
|
||||
assert info["litellm_provider"] == "bedrock"
|
||||
assert info["mode"] == "embedding"
|
||||
assert info["input_cost_per_query"] == TEXT_REQUEST_COST
|
||||
assert info["output_cost_per_token"] == 0.0
|
||||
assert info["max_input_tokens"] == 500
|
||||
assert info["max_tokens"] == 500
|
||||
assert info["output_vector_size"] == 512
|
||||
assert info["supports_embedding_image_input"] is True
|
||||
assert info["supports_image_input"] is True
|
||||
assert "deprecation_date" not in info
|
||||
|
||||
routed_model, provider, _, _ = get_llm_provider(model=f"bedrock/{model}")
|
||||
assert routed_model == model
|
||||
assert provider == "bedrock"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", PER_REQUEST_MODELS)
|
||||
def test_marengo_prices_are_per_request_not_per_token(model):
|
||||
info = _load(MAIN_PATH)[model]
|
||||
assert "input_cost_per_token" not in info
|
||||
assert info["input_cost_per_query"] == TEXT_REQUEST_COST
|
||||
assert info["input_cost_per_image"] == IMAGE_REQUEST_COST
|
||||
assert info["input_cost_per_video_per_second"] == VIDEO_COST_PER_SECOND
|
||||
assert info["input_cost_per_audio_per_second"] == AUDIO_COST_PER_SECOND
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ALL_MODELS)
|
||||
def test_marengo_embed_3_is_visible_to_callers(model, local_model_cost_map):
|
||||
info = litellm.get_model_info(model=model, custom_llm_provider="bedrock")
|
||||
|
|
|
|||
|
|
@ -31,52 +31,6 @@ def model_data():
|
|||
return json.load(f)
|
||||
|
||||
|
||||
def test_usgov_carries_20_percent_premium_over_global(model_data):
|
||||
"""The us-gov rates must equal 1.2x the global anthropic.* rates,
|
||||
matching AWS's documented GovCloud uplift.
|
||||
"""
|
||||
global_key = "anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
usgov_key = "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
global_info = model_data[global_key]
|
||||
usgov_info = model_data[usgov_key]
|
||||
for field in (
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"cache_creation_input_token_cost",
|
||||
"cache_creation_input_token_cost_above_1hr",
|
||||
"cache_read_input_token_cost",
|
||||
):
|
||||
ratio = usgov_info[field] / global_info[field]
|
||||
assert abs(ratio - 1.2) < 1e-9, f"{field}: us-gov / global ratio is {ratio}, expected 1.2"
|
||||
|
||||
|
||||
# The us-gov.anthropic.* cross-region inference profile is the only us-gov
|
||||
# entry that carries the 1M-context `_above_200k_tokens` pricing tier — the
|
||||
# bedrock/us-gov-{east,west}-1/ entries are capped at 200k tokens.
|
||||
USGOV_CROSS_REGION_KEY = "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
|
||||
EXPECTED_USGOV_ABOVE_200K = {
|
||||
"input_cost_per_token_above_200k_tokens": 7.2e-06,
|
||||
"output_cost_per_token_above_200k_tokens": 2.7e-05,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 9.0e-06,
|
||||
"cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.44e-05,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 7.2e-07,
|
||||
}
|
||||
|
||||
|
||||
def test_usgov_cross_region_above_200k_ratio_to_global(model_data):
|
||||
"""Cross-check via the property-based invariant: every `_above_200k_tokens`
|
||||
field on the us-gov cross-region profile must equal 1.2x the global
|
||||
anthropic.* rate, the same GovCloud uplift the base tier carries.
|
||||
"""
|
||||
global_key = "anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
global_info = model_data[global_key]
|
||||
usgov_info = model_data[USGOV_CROSS_REGION_KEY]
|
||||
for field in EXPECTED_USGOV_ABOVE_200K:
|
||||
ratio = usgov_info[field] / global_info[field]
|
||||
assert abs(ratio - 1.2) < 1e-9, f"{field}: us-gov / global ratio is {ratio}, expected 1.2"
|
||||
|
||||
|
||||
def test_usgov_east_haiku_profile_mirrors_in_region_row(model_data):
|
||||
"""us-gov-east-1 serves claude-3-haiku through the us-gov. inference profile
|
||||
only, so the profile row must bill exactly like the in-region gov row.
|
||||
|
|
@ -112,24 +66,12 @@ GOV_ROW_SOURCES = {
|
|||
}
|
||||
|
||||
|
||||
BEDROCK_PRICE_LIST_URL = (
|
||||
"https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
|
||||
)
|
||||
|
||||
|
||||
def _non_pricing_fields(info):
|
||||
return {k: v for k, v in info.items() if "cost" not in k and k not in ("litellm_provider", "source")}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gov_key", GOV_ROW_SOURCES)
|
||||
def test_usgov_rows_keep_commercial_limits_and_capabilities(model_data, gov_key):
|
||||
"""A gov row differs from the commercial row it mirrors only in price and
|
||||
provider: context limits, mode, and capability flags stay identical, so a
|
||||
hand-copied row cannot silently drop tool calling or shrink the context window.
|
||||
The only source a gov row may cite is the AWS price list, which prices the
|
||||
us-gov regions itself; a commercial doc URL copied along with the row is not.
|
||||
"""
|
||||
"""Gov rows preserve the commercial row's non-pricing fields."""
|
||||
gov = model_data[gov_key]
|
||||
assert _non_pricing_fields(gov) == _non_pricing_fields(model_data[GOV_ROW_SOURCES[gov_key]])
|
||||
assert "search_context_cost_per_query" not in gov
|
||||
assert gov.get("source", BEDROCK_PRICE_LIST_URL) == BEDROCK_PRICE_LIST_URL
|
||||
|
|
|
|||
|
|
@ -28,15 +28,6 @@ def _load_root_cost_map() -> dict:
|
|||
return json.load(f)
|
||||
|
||||
|
||||
def test_opus_4_8_fast_mode_multiplier():
|
||||
"""Opus 4.8 dropped fast-mode pricing to 2x base ($10/$50 per MTok);
|
||||
Opus 4.7 was 6x ($30/$150)."""
|
||||
model_data = _load_root_cost_map()
|
||||
entry = model_data["claude-opus-4-8"]["provider_specific_entry"]
|
||||
assert entry["us"] == 1.1
|
||||
assert entry["fast"] == 2.0
|
||||
|
||||
|
||||
def test_opus_4_8_registered_for_bedrock_converse():
|
||||
assert "anthropic.claude-opus-4-8" in BEDROCK_CONVERSE_MODELS
|
||||
|
||||
|
|
|
|||
|
|
@ -51,26 +51,6 @@ def _load_root_cost_map() -> dict:
|
|||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", BEDROCK_OPUS_5_VARIANTS)
|
||||
def test_opus_5_bedrock_entries_declare_no_effort_ceiling(model_name):
|
||||
"""Bedrock accepts every effort level for Opus 5, so no clamp belongs here.
|
||||
|
||||
Opus 4.7/4.8 carry ``bedrock_output_config_effort_ceiling: "xhigh"``, which
|
||||
is what ``normalize_bedrock_opus_output_config_effort`` reads to rewrite a
|
||||
caller's effort down. Verified against Bedrock on 2026-07-24 that
|
||||
``output_config.effort="max"`` returns 200 for the Opus 5 profiles, so the
|
||||
ceiling is deliberately absent; adding one back would silently downgrade
|
||||
requests.
|
||||
|
||||
This asserts the cost-map entry rather than calling the normalizer because
|
||||
``_BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER`` currently ranks ``max`` (3) below
|
||||
``xhigh`` (4), so an ``xhigh`` ceiling never clamps ``max`` and a behavioral
|
||||
assertion would pass either way. Keeping the entry clean means Opus 5 stays
|
||||
correct once that ordering is fixed."""
|
||||
info = _load_root_cost_map()[model_name]
|
||||
assert "bedrock_output_config_effort_ceiling" not in info
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", BEDROCK_OPUS_5_VARIANTS)
|
||||
def test_opus_5_bedrock_rejects_strict_tools(model_name, local_model_cost_map):
|
||||
"""Bedrock Converse routes Opus through a validator that rejects
|
||||
|
|
@ -82,41 +62,6 @@ def test_opus_5_bedrock_rejects_strict_tools(model_name, local_model_cost_map):
|
|||
assert bedrock_converse_supports_strict_tools(model_name) is False
|
||||
|
||||
|
||||
def test_opus_5_prompt_cache_minimum_is_512(local_model_cost_map):
|
||||
"""Opus 5 halves the cacheable-prefix minimum (Opus 4.8 is 1024).
|
||||
|
||||
The router's prompt-caching deployment check reads this value, so a stale
|
||||
1024 would route prompts of 512-1023 tokens away from a warm Opus 5
|
||||
deployment even though they cache fine."""
|
||||
from litellm.utils import get_prompt_cache_min_tokens
|
||||
|
||||
assert get_prompt_cache_min_tokens(model="claude-opus-5") == 512
|
||||
assert get_prompt_cache_min_tokens(model="us.anthropic.claude-opus-5") == 512
|
||||
|
||||
|
||||
def test_opus_5_supports_fast_mode(local_model_cost_map):
|
||||
"""Fast mode is Opus 5 on the first-party API at $10 / $50 per MTok, i.e. 2x
|
||||
base. ``supports_speed`` gates whether ``speed="fast"`` is forwarded at all,
|
||||
and ``provider_specific_entry.fast`` is what prices the response."""
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.cost_calculation import (
|
||||
cost_per_token as anthropic_cost_per_token,
|
||||
)
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
assert (
|
||||
AnthropicConfig._model_supports_speed_param("claude-opus-5", "anthropic") is True
|
||||
)
|
||||
|
||||
usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
|
||||
usage.speed = "fast"
|
||||
prompt_cost, completion_cost = anthropic_cost_per_token(
|
||||
model="claude-opus-5", usage=usage
|
||||
)
|
||||
assert prompt_cost == pytest.approx(1000 * 5e-06 * 2.0)
|
||||
assert completion_cost == pytest.approx(500 * 2.5e-05 * 2.0)
|
||||
|
||||
|
||||
def test_opus_5_present_in_bundled_backup():
|
||||
"""The bundled backup is the runtime fallback (and what tests load with
|
||||
``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the
|
||||
|
|
@ -147,19 +92,3 @@ def test_opus_5_all_variants_carry_adaptive_thinking_flag(cost_map):
|
|||
k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True
|
||||
]
|
||||
assert not missing, f"missing supports_adaptive_thinking: {missing}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cost_map",
|
||||
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
|
||||
ids=["root", "bundled_backup"],
|
||||
)
|
||||
def test_opus_5_all_variants_carry_512_token_cache_minimum(cost_map):
|
||||
variants = [k for k in cost_map if "claude-opus-5" in k]
|
||||
assert variants, "no claude-opus-5 entries found in cost map"
|
||||
wrong = {
|
||||
k: cost_map[k].get("prompt_cache_min_tokens")
|
||||
for k in variants
|
||||
if cost_map[k].get("prompt_cache_min_tokens") != 512
|
||||
}
|
||||
assert not wrong, f"prompt_cache_min_tokens must be 512: {wrong}"
|
||||
|
|
|
|||
|
|
@ -11,47 +11,6 @@ import json
|
|||
import os
|
||||
|
||||
|
||||
def test_bedrock_sonnet_4_6_region_prefixes():
|
||||
"""All documented Bedrock cross-region inference prefixes for
|
||||
claude-sonnet-4-6 must be present in model_prices_and_context_window.json.
|
||||
"""
|
||||
json_path = os.path.join(
|
||||
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
|
||||
)
|
||||
with open(json_path) as f:
|
||||
model_data = json.load(f)
|
||||
|
||||
bedrock_sonnet_4_6_models = [
|
||||
"anthropic.claude-sonnet-4-6",
|
||||
"global.anthropic.claude-sonnet-4-6",
|
||||
"us.anthropic.claude-sonnet-4-6",
|
||||
"eu.anthropic.claude-sonnet-4-6",
|
||||
"au.anthropic.claude-sonnet-4-6",
|
||||
"jp.anthropic.claude-sonnet-4-6",
|
||||
]
|
||||
|
||||
for model in bedrock_sonnet_4_6_models:
|
||||
assert model in model_data, f"Model {model} not found in config"
|
||||
model_info = model_data[model]
|
||||
|
||||
assert (
|
||||
model_info["litellm_provider"] == "bedrock_converse"
|
||||
), f"{model} should use bedrock_converse, got {model_info['litellm_provider']}"
|
||||
assert model_info["mode"] == "chat"
|
||||
assert model_info["max_input_tokens"] == 1000000
|
||||
assert model_info["max_output_tokens"] == 64000
|
||||
assert model_info["max_tokens"] == 64000
|
||||
assert model_info.get("supports_vision") is True
|
||||
assert model_info.get("supports_computer_use") is True
|
||||
assert model_info.get("supports_function_calling") is True
|
||||
assert model_info.get("supports_tool_choice") is True
|
||||
assert model_info.get("supports_prompt_caching") is True
|
||||
assert model_info.get("supports_response_schema") is True
|
||||
assert model_info.get("supports_pdf_input") is True
|
||||
assert model_info.get("supports_assistant_prefill") is True
|
||||
assert model_info.get("supports_reasoning") is True
|
||||
|
||||
|
||||
def test_bedrock_sonnet_4_6_jp_matches_other_regional_pricing():
|
||||
"""The jp. cross-region inference profile shares pricing with the other
|
||||
regional profiles (us./eu./au.), which carry a 10% premium over the
|
||||
|
|
|
|||
|
|
@ -49,18 +49,6 @@ class TestCommandR7bPricingData:
|
|||
"""The JSON price maps must carry Cohere's published costs, with output
|
||||
more expensive than input."""
|
||||
|
||||
def test_backup_costs_not_swapped(self):
|
||||
entry = _load_json(_backup_path())[MODEL]
|
||||
assert entry["input_cost_per_token"] == EXPECTED_INPUT_COST
|
||||
assert entry["output_cost_per_token"] == EXPECTED_OUTPUT_COST
|
||||
assert entry["output_cost_per_token"] > entry["input_cost_per_token"]
|
||||
|
||||
def test_main_costs_not_swapped(self):
|
||||
entry = _load_json(_main_path())[MODEL]
|
||||
assert entry["input_cost_per_token"] == EXPECTED_INPUT_COST
|
||||
assert entry["output_cost_per_token"] == EXPECTED_OUTPUT_COST
|
||||
assert entry["output_cost_per_token"] > entry["input_cost_per_token"]
|
||||
|
||||
|
||||
class TestCommandR7bPricingModelInfo:
|
||||
"""``get_model_info`` must report the corrected, un-swapped costs."""
|
||||
|
|
|
|||
|
|
@ -1,11 +1,8 @@
|
|||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
import litellm
|
||||
|
|
@ -1823,7 +1820,6 @@ def test_azure_ai_cache_cost_calculation(_local_model_cost_map):
|
|||
), f"Output cost mismatch: got {output_cost}, expected {expected_output_cost}"
|
||||
|
||||
|
||||
|
||||
AZURE_GPT_5_6_MAP_KEYS = (
|
||||
"azure/gpt-5.6",
|
||||
"azure/gpt-5.6-sol",
|
||||
|
|
@ -4585,26 +4581,6 @@ def test_claude_3_one_hour_cache_writes_bill_at_double_input(
|
|||
assert prompt_cost == pytest.approx(1000 * expected_1hr_rate, rel=1e-9)
|
||||
|
||||
|
||||
def test_every_one_hour_cache_write_rate_is_double_its_input_rate():
|
||||
"""Guard against pasting one model's 1h cache-write price onto another: every provider
|
||||
LiteLLM tracks (Anthropic, Bedrock, Vertex, Azure) publishes the 1h write at 2x input."""
|
||||
|
||||
cost_map = json.loads(
|
||||
(Path(__file__).parents[2] / "model_prices_and_context_window.json").read_text()
|
||||
)
|
||||
one_hour_prefix = "cache_creation_input_token_cost_above_1hr"
|
||||
deviations = {
|
||||
(name, key): (entry["input_cost_per_token" + key[len(one_hour_prefix) :]], entry[key])
|
||||
for name, entry in cost_map.items()
|
||||
if isinstance(entry, dict)
|
||||
for key in entry
|
||||
if key.startswith(one_hour_prefix)
|
||||
and entry[key] != pytest.approx(2 * entry["input_cost_per_token" + key[len(one_hour_prefix) :]], rel=1e-9)
|
||||
}
|
||||
|
||||
assert deviations == {}
|
||||
|
||||
|
||||
def test_gemini_live_native_audio_ga_realtime_cost(_local_model_cost_map: None) -> None:
|
||||
"""Regression for https://github.com/BerriAI/litellm/issues/31087."""
|
||||
from litellm.types.utils import CompletionTokensDetailsWrapper
|
||||
|
|
|
|||
|
|
@ -47,7 +47,6 @@ def test_official_alias_tracks_snapshot(alias, snapshot):
|
|||
|
||||
assert alias_info["supported_endpoints"] == ["/v1/responses"]
|
||||
assert alias_info["mode"] == "responses"
|
||||
assert alias_info["source"] == f"https://developers.openai.com/api/docs/models/{alias}"
|
||||
assert {field: alias_info.get(field) for field in PRICE_FIELDS} == {
|
||||
field: snapshot_info.get(field) for field in PRICE_FIELDS
|
||||
}
|
||||
|
|
|
|||
|
|
@ -88,18 +88,6 @@ TWIN_PINNED_PRICES = {
|
|||
}
|
||||
|
||||
|
||||
def test_deepseek_v4_flash_twins_pin_published_pricing(model_data):
|
||||
"""Both entries of each Flash twin pair carry the price published at docs.fireworks.ai/serverless/pricing."""
|
||||
for bare_suffix, expected in TWIN_PINNED_PRICES.items():
|
||||
for key in (
|
||||
f"fireworks_ai/{bare_suffix}",
|
||||
f"fireworks_ai/accounts/fireworks/models/{bare_suffix}",
|
||||
):
|
||||
entry = model_data[key]
|
||||
for field, value in expected.items():
|
||||
assert entry[field] == pytest.approx(value), f"{key}.{field}"
|
||||
|
||||
|
||||
def test_fireworks_account_prefixed_twins_agree_on_price(model_data):
|
||||
"""Every accounts/fireworks/models/X entry prices identically to its bare fireworks_ai/X twin."""
|
||||
prefix = "fireworks_ai/accounts/fireworks/models/"
|
||||
|
|
|
|||
|
|
@ -1,35 +0,0 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
|
||||
def test_friendli_glm_5_3_flash_model_info():
|
||||
model = "friendliai/zai-org/GLM-5.3-Flash"
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
info = model_cost.get(model)
|
||||
assert (
|
||||
info is not None
|
||||
), f"{model} not found in model_prices_and_context_window.json"
|
||||
assert info["litellm_provider"] == "friendliai"
|
||||
assert info["mode"] == "chat"
|
||||
assert info["input_cost_per_token"] == 1.5e-07
|
||||
assert info["output_cost_per_token"] == 5e-07
|
||||
assert info["cache_read_input_token_cost"] == 3e-08
|
||||
assert info["max_input_tokens"] == 1048576
|
||||
assert info["max_output_tokens"] == 1048576
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_reasoning"] is True
|
||||
assert info["reasoning_effort_levels"] == ["low", "high", "max"]
|
||||
assert info["supports_tool_choice"] is True
|
||||
assert info["supports_prompt_caching"] is True
|
||||
assert info["supports_vision"] is True
|
||||
assert info["supports_image_input"] is True
|
||||
assert info["supports_video_input"] is True
|
||||
|
||||
routed_model, provider, _, _ = get_llm_provider(model=model)
|
||||
assert routed_model == "zai-org/GLM-5.3-Flash"
|
||||
assert provider == "friendliai"
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
|
||||
def test_friendli_glm_5_3_model_info():
|
||||
model = "friendliai/zai-org/GLM-5.3"
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
info = model_cost.get(model)
|
||||
assert (
|
||||
info is not None
|
||||
), f"{model} not found in model_prices_and_context_window.json"
|
||||
assert info["litellm_provider"] == "friendliai"
|
||||
assert info["mode"] == "chat"
|
||||
assert info["input_cost_per_token"] == 1.26e-06
|
||||
assert info["output_cost_per_token"] == 3.96e-06
|
||||
assert info["cache_read_input_token_cost"] == 2.34e-07
|
||||
assert info["max_input_tokens"] == 1048576
|
||||
assert info["max_output_tokens"] == 1048576
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_reasoning"] is True
|
||||
assert info["reasoning_effort_levels"] == ["low", "high", "max"]
|
||||
assert info["supports_tool_choice"] is True
|
||||
assert info["supports_prompt_caching"] is True
|
||||
assert info["supports_vision"] is False
|
||||
assert info["supports_image_input"] is False
|
||||
|
||||
routed_model, provider, _, _ = get_llm_provider(model=model)
|
||||
assert routed_model == "zai-org/GLM-5.3"
|
||||
assert provider == "friendliai"
|
||||
|
|
@ -114,15 +114,6 @@ def local_model_cost_map(monkeypatch):
|
|||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ALL_KEYS)
|
||||
@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup"))
|
||||
def test_published_prices_are_registered(model: str, path: Path):
|
||||
info = _load(path).get(model)
|
||||
assert info is not None, f"{model} missing from {path.name}"
|
||||
for field, value in SHARED_FIELDS.items():
|
||||
assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ALL_KEYS)
|
||||
@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup"))
|
||||
def test_per_route_capabilities_match_model_cards(model: str, path: Path):
|
||||
|
|
@ -131,19 +122,6 @@ def test_per_route_capabilities_match_model_cards(model: str, path: Path):
|
|||
assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ALL_KEYS)
|
||||
@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup"))
|
||||
def test_grounding_fields_absent(model: str, path: Path):
|
||||
info = _load(path)[model]
|
||||
for field in GROUNDING_FIELDS:
|
||||
assert field not in info, f"{model} should not define {field}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup"))
|
||||
def test_ai_studio_route_has_no_implicit_cache_price(path: Path):
|
||||
assert "cache_read_input_token_cost" not in _load(path)[GEMINI]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ALL_KEYS)
|
||||
def test_backup_matches_main(model: str):
|
||||
assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model)
|
||||
|
|
|
|||
|
|
@ -81,22 +81,6 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
|||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ALL_KEYS)
|
||||
@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup"))
|
||||
def test_published_rates_are_registered(model: str, path: Path):
|
||||
info = _load(path)[model]
|
||||
for field, value in PUBLISHED_RATES[model].items():
|
||||
assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", PRO_TTS_KEYS)
|
||||
@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup"))
|
||||
def test_pro_tts_has_no_long_context_tier(model: str, path: Path):
|
||||
info = _load(path)[model]
|
||||
for field in LONG_CONTEXT_TIER_FIELDS:
|
||||
assert field not in info, f"{model} has {field} but Google publishes one flat TTS rate"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ALL_KEYS)
|
||||
def test_backup_matches_main(model: str):
|
||||
assert _load(BACKUP_PATH)[model] == _load(MAIN_PATH)[model]
|
||||
|
|
|
|||
|
|
@ -37,43 +37,6 @@ def _pricing_key(model: str) -> str:
|
|||
return "gpt-5.4-nano" if "nano" in model else "gpt-5.4-mini"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", SMALL_MODELS)
|
||||
def test_gpt_5_4_small_models_use_documented_token_limits(model: str) -> None:
|
||||
"""gpt-5.4-mini/nano are 400K-window models: 272K in, 128K out, not gpt-5.4's 1.05M window."""
|
||||
info = _load(MAIN_PATH).get(model)
|
||||
assert info is not None, f"{model} not found in model_prices_and_context_window.json"
|
||||
|
||||
assert info["max_input_tokens"] == DOCUMENTED_MAX_INPUT_TOKENS
|
||||
assert info["max_output_tokens"] == DOCUMENTED_MAX_OUTPUT_TOKENS
|
||||
assert info["max_tokens"] == DOCUMENTED_MAX_OUTPUT_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", SMALL_MODELS)
|
||||
def test_gpt_5_4_small_models_have_no_long_context_surcharge(model: str) -> None:
|
||||
"""OpenAI prices prompts above 272K at 2x input / 1.5x output for the 1.05M-window models only."""
|
||||
info = _load(MAIN_PATH)[model]
|
||||
assert [key for key in info if "above_272k" in key] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", SMALL_MODELS)
|
||||
def test_gpt_5_4_small_models_standard_pricing(model: str) -> None:
|
||||
info = _load(MAIN_PATH)[model]
|
||||
input_cost, output_cost, cache_read_cost = STANDARD_PRICING[_pricing_key(model)]
|
||||
|
||||
assert info["input_cost_per_token"] == input_cost
|
||||
assert info["output_cost_per_token"] == output_cost
|
||||
assert info["cache_read_input_token_cost"] == cache_read_cost
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", LONG_CONTEXT_MODELS)
|
||||
def test_gpt_5_4_long_context_models_keep_surcharge(model: str) -> None:
|
||||
"""The mini/nano correction must leave gpt-5.4 and gpt-5.4-pro tiered pricing intact."""
|
||||
info = _load(MAIN_PATH)[model]
|
||||
|
||||
assert info["input_cost_per_token_above_272k_tokens"] == pytest.approx(info["input_cost_per_token"] * 2)
|
||||
assert info["output_cost_per_token_above_272k_tokens"] == pytest.approx(info["output_cost_per_token"] * 1.5)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", SMALL_MODELS)
|
||||
def test_gpt_5_4_small_models_backup_matches_main(model: str) -> None:
|
||||
assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model), (
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ from pathlib import Path
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
from litellm.utils import supports_prompt_caching, supports_reasoning
|
||||
|
||||
|
|
@ -35,34 +34,6 @@ def local_model_cost_map(monkeypatch):
|
|||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
|
||||
def test_zai_glm_5_2_specs(model):
|
||||
info = _load(MAIN_PATH).get(model)
|
||||
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
|
||||
|
||||
assert info["litellm_provider"] == "mistral"
|
||||
assert info["mode"] == "chat"
|
||||
|
||||
assert info["input_cost_per_token"] == INPUT_COST
|
||||
assert info["output_cost_per_token"] == OUTPUT_COST
|
||||
assert info["cache_read_input_token_cost"] == CACHED_INPUT_COST
|
||||
|
||||
assert info["max_input_tokens"] == 1048576
|
||||
assert info["max_output_tokens"] == 131072
|
||||
assert info["max_tokens"] == 131072
|
||||
|
||||
assert info["supports_assistant_prefill"] is True
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_prompt_caching"] is True
|
||||
assert info["supports_reasoning"] is True
|
||||
assert info["supports_response_schema"] is True
|
||||
assert info["supports_tool_choice"] is True
|
||||
|
||||
routed_model, provider, _, _ = get_llm_provider(model=model)
|
||||
assert routed_model == model.split("/", 1)[1]
|
||||
assert provider == "mistral"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
|
||||
def test_zai_glm_5_2_capabilities_are_visible_to_callers(local_model_cost_map, model):
|
||||
"""Mistral advertises reasoning and prompt caching on this model, so the helpers
|
||||
|
|
|
|||
|
|
@ -7,40 +7,6 @@ MUSE_SPARK_MODEL = "meta/muse-spark-1.1"
|
|||
|
||||
|
||||
def test_muse_spark_1_1_model_info():
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
info = model_cost.get(MUSE_SPARK_MODEL)
|
||||
assert info is not None, f"{MUSE_SPARK_MODEL} not found in model_prices_and_context_window.json"
|
||||
|
||||
assert info["litellm_provider"] == "meta"
|
||||
assert info["mode"] == "chat"
|
||||
|
||||
assert info["input_cost_per_token"] == 1.25e-06
|
||||
assert info["output_cost_per_token"] == 4.25e-06
|
||||
assert info["cache_read_input_token_cost"] == 1.5e-07
|
||||
|
||||
assert info["max_input_tokens"] == 1048576
|
||||
assert info["max_output_tokens"] == 131072
|
||||
assert info["max_tokens"] == 131072
|
||||
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_parallel_function_calling"] is True
|
||||
assert info["supports_prompt_caching"] is True
|
||||
assert info["supports_reasoning"] is True
|
||||
assert info["supports_response_schema"] is True
|
||||
assert info["supports_tool_choice"] is True
|
||||
assert info["supports_vision"] is True
|
||||
assert info["supports_pdf_input"] is True
|
||||
assert info["supports_web_search"] is True
|
||||
assert info["supports_minimal_reasoning_effort"] is True
|
||||
assert info["supports_xhigh_reasoning_effort"] is True
|
||||
|
||||
assert info["supported_endpoints"] == ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
|
||||
assert info["supported_modalities"] == ["text", "image", "video"]
|
||||
assert info["supported_output_modalities"] == ["text"]
|
||||
|
||||
routed_model, provider, _, api_base = get_llm_provider(model=MUSE_SPARK_MODEL, api_key="sk-test")
|
||||
assert routed_model == "muse-spark-1.1"
|
||||
assert provider == "meta"
|
||||
|
|
|
|||
|
|
@ -78,38 +78,6 @@ def _load(path: Path) -> dict[str, dict[str, object]]:
|
|||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", [MAIN_PATH, BACKUP_PATH], ids=["main", "backup"])
|
||||
@pytest.mark.parametrize("model", sorted(EXPECTED))
|
||||
def test_service_tier_long_context_rates_are_published(model: str, path: Path) -> None:
|
||||
"""Each tier must carry its own above-272K rates, in both price files."""
|
||||
info = _load(path).get(model)
|
||||
assert info is not None, f"{model} not found in {path.name}"
|
||||
for key, expected in EXPECTED[model].items():
|
||||
assert info.get(key) == pytest.approx(expected), f"{model}.{key} is {info.get(key)!r}, expected {expected!r}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", sorted(EXPECTED))
|
||||
def test_tier_long_context_rate_is_half_or_double_the_standard(model: str) -> None:
|
||||
"""Flex is half the standard long-context rate; priority is double it."""
|
||||
info = _load(MAIN_PATH)[model]
|
||||
tier = "flex" if model in FLEX_LONG_CONTEXT else "priority"
|
||||
ratio = 0.5 if tier == "flex" else 2.0
|
||||
for base in ("input_cost_per_token", "output_cost_per_token"):
|
||||
standard = info[f"{base}_above_272k_tokens"]
|
||||
tiered = info[f"{base}_above_272k_tokens_{tier}"]
|
||||
assert tiered == pytest.approx(standard * ratio), (
|
||||
f"{model}.{base}_above_272k_tokens_{tier} is {tiered!r}, "
|
||||
f"expected {ratio}x the standard long-context rate {standard!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", NO_PUBLISHED_PRIORITY_LONG_CONTEXT)
|
||||
def test_no_priority_long_context_rates_where_openai_publishes_none(model: str) -> None:
|
||||
"""Guard against back-filling a rate OpenAI does not publish."""
|
||||
info = _load(MAIN_PATH)[model]
|
||||
assert "input_cost_per_token_above_272k_tokens_priority" not in info
|
||||
|
||||
|
||||
LONG_CONTEXT_PROMPT_TOKENS = 300_000
|
||||
COMPLETION_TOKENS = 1_000
|
||||
|
||||
|
|
|
|||
|
|
@ -11,15 +11,11 @@ def test_sambanova_minimax_m27_model_info():
|
|||
model_cost = json.load(f)
|
||||
|
||||
info = model_cost.get(model)
|
||||
assert (
|
||||
info is not None
|
||||
), f"{model} not found in model_prices_and_context_window.json"
|
||||
assert info is not None, f"{model} not found in model_prices_and_context_window.json"
|
||||
assert info["litellm_provider"] == "sambanova"
|
||||
assert info["mode"] == "chat"
|
||||
assert info["input_cost_per_token"] > 0
|
||||
assert info["output_cost_per_token"] > 0
|
||||
assert info["max_input_tokens"] == 196608
|
||||
assert info["max_output_tokens"] == 131072
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_reasoning"] is True
|
||||
assert info["supports_tool_choice"] is True
|
||||
|
|
|
|||
|
|
@ -88,13 +88,6 @@ def test_together_chat_entries_never_carry_context_length_as_output_ceiling(cost
|
|||
assert inflated == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", sorted(DEPRECATED_MODELS))
|
||||
def test_together_deprecated_model_carries_deprecation_date(cost_map: CostMap, model: str):
|
||||
info = cost_map.get(model)
|
||||
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
|
||||
assert info.get("deprecation_date") == DEPRECATED_MODELS[model]
|
||||
|
||||
|
||||
def _successor(info: dict[str, object]) -> str | None:
|
||||
metadata = info.get("metadata")
|
||||
if not isinstance(metadata, dict):
|
||||
|
|
|
|||
|
|
@ -94,12 +94,6 @@ def test_non_ocr_wrapper_preserves_logging_executor_and_context(monkeypatch: pyt
|
|||
marker.reset(token)
|
||||
|
||||
|
||||
def test_cloudflare_model_info_includes_rpm(local_model_cost_map: None) -> None:
|
||||
assert litellm.get_model_info("cloudflare/@cf/meta/llama-3.1-8b-instruct-fp8")["rpm"] == 300
|
||||
assert litellm.get_model_info("cloudflare/@cf/moonshotai/kimi-k2.6")["rpm"] == 20
|
||||
assert litellm.get_model_info("cloudflare/@cf/openai/whisper-large-v3-turbo")["rpm"] == 720
|
||||
|
||||
|
||||
def test_get_utc_datetime_returns_current_aware_utc_time() -> None:
|
||||
before: Final = datetime.now(timezone.utc)
|
||||
result: Final = litellm.utils.get_utc_datetime()
|
||||
|
|
@ -160,7 +154,6 @@ def test_prompt_tokens_details_cache_write_creation_stay_in_sync_on_assignment()
|
|||
assert details.cache_write_tokens == details.cache_creation_tokens == 375
|
||||
|
||||
|
||||
|
||||
def test_get_model_info_surfaces_supports_adaptive_thinking(local_model_cost_map):
|
||||
"""supports_adaptive_thinking must flow through get_model_info like every other
|
||||
capability flag: both from an explicit cost-map entry and from a
|
||||
|
|
@ -177,7 +170,6 @@ def test_get_model_info_surfaces_supports_adaptive_thinking(local_model_cost_map
|
|||
assert generalized["supports_adaptive_thinking"] is True
|
||||
|
||||
|
||||
|
||||
def test_get_model_info_surfaces_supports_parallel_function_calling(local_model_cost_map):
|
||||
"""A registry entry's supports_parallel_function_calling must read back through get_model_info
|
||||
and litellm.supports_parallel_function_calling. Regression: the key was never copied into
|
||||
|
|
@ -493,64 +485,6 @@ def test_gpt_image_provider_detection_covers_existing_family():
|
|||
assert custom_llm_provider == "openai"
|
||||
|
||||
|
||||
def test_gpt_image_2_provider_and_model_info(local_model_cost_map):
|
||||
|
||||
model, custom_llm_provider, _, _ = litellm.get_llm_provider(model="gpt-image-2")
|
||||
|
||||
assert model == "gpt-image-2"
|
||||
assert custom_llm_provider == "openai"
|
||||
|
||||
model_info = litellm.get_model_info(model="gpt-image-2")
|
||||
assert model_info["litellm_provider"] == "openai"
|
||||
assert model_info["mode"] == "image_generation"
|
||||
assert model_info["input_cost_per_token"] == 5e-06
|
||||
assert model_info["input_cost_per_image_token"] == 8e-06
|
||||
assert model_info["output_cost_per_token"] == 0
|
||||
assert model_info["output_cost_per_image_token"] == 3e-05
|
||||
assert (
|
||||
"/v1/images/generations"
|
||||
in litellm.model_cost["gpt-image-2"]["supported_endpoints"]
|
||||
)
|
||||
assert (
|
||||
"/v1/images/edits" in litellm.model_cost["gpt-image-2"]["supported_endpoints"]
|
||||
)
|
||||
assert model_info["supports_vision"] is True
|
||||
assert model_info["supports_pdf_input"] is True
|
||||
|
||||
|
||||
def test_gpt_image_2_snapshot_model_info(local_model_cost_map):
|
||||
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
model="gpt-image-2-2026-04-21"
|
||||
)
|
||||
|
||||
assert model == "gpt-image-2-2026-04-21"
|
||||
assert custom_llm_provider == "openai"
|
||||
|
||||
model_info = litellm.get_model_info(model="gpt-image-2-2026-04-21")
|
||||
assert model_info["litellm_provider"] == "openai"
|
||||
assert model_info["mode"] == "image_generation"
|
||||
assert model_info["output_cost_per_image_token"] == 3e-05
|
||||
|
||||
|
||||
def test_azure_gpt_image_2_model_info(local_model_cost_map):
|
||||
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
model="azure/gpt-image-2"
|
||||
)
|
||||
|
||||
assert model == "gpt-image-2"
|
||||
assert custom_llm_provider == "azure"
|
||||
|
||||
model_info = litellm.get_model_info(
|
||||
model="gpt-image-2", custom_llm_provider="azure"
|
||||
)
|
||||
assert model_info["litellm_provider"] == "azure"
|
||||
assert model_info["mode"] == "image_generation"
|
||||
assert model_info["input_cost_per_token"] == 5e-06
|
||||
assert model_info["input_cost_per_image_token"] == 8e-06
|
||||
assert model_info["output_cost_per_token"] == 0
|
||||
assert model_info["output_cost_per_image_token"] == 3e-05
|
||||
|
||||
|
||||
def test_all_model_configs():
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import (
|
||||
VertexAIAi21Config,
|
||||
|
|
@ -2907,158 +2841,6 @@ def test_model_info_for_vertex_ai_deepseek_model():
|
|||
print("vertex deepseek model info", model_info)
|
||||
|
||||
|
||||
def test_model_info_for_openrouter_kimi_k2_5():
|
||||
"""
|
||||
Test that openrouter/moonshotai/kimi-k2.5 model info is correctly configured
|
||||
in model_prices_and_context_window.json.
|
||||
|
||||
Model properties from OpenRouter API:
|
||||
- context_length: 262144
|
||||
- pricing: prompt=$0.00000045, completion=$0.00000225, input_cache_read=$0.00000007
|
||||
- modality: text+image->text (supports vision)
|
||||
- supports: tool_choice, tools (function calling)
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# Load directly from the local JSON file
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
model_info = model_cost.get("openrouter/moonshotai/kimi-k2.5")
|
||||
assert (
|
||||
model_info is not None
|
||||
), "Model not found in model_prices_and_context_window.json"
|
||||
assert model_info["litellm_provider"] == "openrouter"
|
||||
assert model_info["mode"] == "chat"
|
||||
|
||||
# Verify context window
|
||||
assert model_info["max_input_tokens"] == 262144
|
||||
assert model_info["max_output_tokens"] == 262144
|
||||
assert model_info["max_tokens"] == 262144
|
||||
|
||||
# Verify pricing
|
||||
assert model_info["input_cost_per_token"] == 4.5e-07
|
||||
assert model_info["output_cost_per_token"] == 2.25e-06
|
||||
assert model_info["cache_read_input_token_cost"] == 7e-08
|
||||
|
||||
# Verify capabilities
|
||||
assert model_info["supports_vision"] is True
|
||||
assert model_info["supports_function_calling"] is True
|
||||
assert model_info["supports_tool_choice"] is True
|
||||
|
||||
print("openrouter kimi-k2.5 model info", model_info)
|
||||
|
||||
|
||||
def test_gemini_embedding_2_ga_in_cost_map():
|
||||
"""GA and Vertex preview gemini-embedding-2 entries align with multimodal token pricing."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
for key, provider in (
|
||||
("gemini/gemini-embedding-2", "gemini"),
|
||||
("vertex_ai/gemini-embedding-2", "vertex_ai"),
|
||||
("vertex_ai/gemini-embedding-2-preview", "vertex_ai"),
|
||||
("gemini-embedding-2", "vertex_ai-embedding-models"),
|
||||
):
|
||||
info = model_cost.get(key)
|
||||
assert (
|
||||
info is not None
|
||||
), f"{key} missing from model_prices_and_context_window.json"
|
||||
assert info["litellm_provider"] == provider
|
||||
assert info.get("mode") == "embedding"
|
||||
assert info.get("supports_multimodal") is True
|
||||
assert info.get("input_cost_per_token") == 2e-07
|
||||
assert info.get("input_cost_per_audio_token") == 6.5e-06
|
||||
assert info.get("input_cost_per_image_token") == 4.5e-07
|
||||
assert info.get("input_cost_per_video_token") == 1.2e-05
|
||||
assert info.get("input_cost_per_audio_token_batches") == 3.25e-06
|
||||
assert info.get("input_cost_per_image_token_batches") == 2.25e-07
|
||||
assert info.get("input_cost_per_video_token_batches") == 6e-06
|
||||
assert "input_cost_per_image" not in info
|
||||
assert "input_cost_per_audio_per_second" not in info
|
||||
assert "input_cost_per_video_per_second" not in info
|
||||
if provider in ("vertex_ai-embedding-models", "vertex_ai"):
|
||||
assert (
|
||||
info.get("uses_embed_content") is True
|
||||
), f"{key} must have uses_embed_content=true for correct Vertex AI routing"
|
||||
|
||||
|
||||
def test_gemini_lyria_3_preview_models_in_cost_map():
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
clip = model_cost.get("gemini/lyria-3-clip-preview")
|
||||
pro = model_cost.get("gemini/lyria-3-pro-preview")
|
||||
assert clip is not None and pro is not None
|
||||
assert clip["litellm_provider"] == "gemini" and pro["litellm_provider"] == "gemini"
|
||||
assert clip["max_input_tokens"] == 131072 == pro["max_input_tokens"]
|
||||
assert clip["output_cost_per_image"] == 0.04
|
||||
|
||||
|
||||
def test_vertex_ai_lyria_models_in_cost_map():
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
lyria_2 = model_cost.get("vertex_ai/lyria-002")
|
||||
clip = model_cost.get("vertex_ai/lyria-3-clip-preview")
|
||||
pro = model_cost.get("vertex_ai/lyria-3-pro-preview")
|
||||
|
||||
assert lyria_2 is not None
|
||||
assert clip is not None
|
||||
assert pro is not None
|
||||
assert lyria_2["litellm_provider"] == "vertex_ai"
|
||||
assert clip["litellm_provider"] == "vertex_ai"
|
||||
assert pro["litellm_provider"] == "vertex_ai"
|
||||
assert lyria_2["mode"] == "audio_speech"
|
||||
assert clip["mode"] == "audio_speech"
|
||||
assert pro["mode"] == "audio_speech"
|
||||
assert lyria_2["output_cost_per_image"] == 0.06
|
||||
assert lyria_2["supported_modalities"] == ["text"]
|
||||
assert lyria_2["supported_output_modalities"] == ["audio"]
|
||||
assert lyria_2["supports_audio_output"] is True
|
||||
assert lyria_2["supported_audio_formats"] == ["wav"]
|
||||
assert lyria_2["vertex_ai_audio_api"] == "lyria_predict"
|
||||
assert lyria_2["supported_endpoints"] == ["/v1/audio/speech"]
|
||||
assert clip["output_cost_per_image"] == 0.04
|
||||
assert pro["output_cost_per_image"] == 0.08
|
||||
assert clip["supported_audio_formats"] == ["mp3"]
|
||||
assert pro["supported_audio_formats"] == ["mp3", "wav"]
|
||||
assert clip["vertex_ai_audio_api"] == "lyria_interactions"
|
||||
assert pro["vertex_ai_audio_api"] == "lyria_interactions"
|
||||
assert clip["supported_endpoints"] == [
|
||||
"/v1beta/interactions",
|
||||
"/v1/audio/speech",
|
||||
]
|
||||
assert pro["supported_endpoints"] == [
|
||||
"/v1beta/interactions",
|
||||
"/v1/audio/speech",
|
||||
]
|
||||
assert clip["supported_modalities"] == ["text"]
|
||||
assert pro["supported_modalities"] == ["text"]
|
||||
assert clip["supports_vision"] is False
|
||||
assert pro["supports_vision"] is False
|
||||
assert "supports_image_input" not in clip
|
||||
assert "supports_image_input" not in pro
|
||||
assert clip["supported_regions"] == ["global"]
|
||||
assert pro["supported_regions"] == ["global"]
|
||||
assert clip["supports_audio_output"] is True
|
||||
assert pro["supports_audio_output"] is True
|
||||
|
||||
|
||||
def test_model_info_for_fireworks_short_form_models():
|
||||
"""
|
||||
Test that fireworks_ai short-form model entries (fireworks_ai/<model>)
|
||||
|
|
@ -4180,114 +3962,6 @@ class TestValidateAndFixThinkingParam:
|
|||
assert validate_and_fix_thinking_param(thinking=False) is None
|
||||
|
||||
|
||||
def test_deepseek_v4_models_in_cost_map():
|
||||
"""
|
||||
Test that deepseek-v4-flash and deepseek-v4-pro entries are correctly
|
||||
configured in model_prices_and_context_window.json.
|
||||
|
||||
Prices sourced from https://api-docs.deepseek.com/quick_start/pricing:
|
||||
- deepseek-v4-flash: $0.30/M input, $1.20/M output
|
||||
- deepseek-v4-pro: $1.32/M input, $3.96/M output
|
||||
|
||||
Closes https://github.com/BerriAI/litellm/issues/26709
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
# --- bare model names ---
|
||||
for key, expected_input, expected_output, expected_cache, expected_vision in [
|
||||
("deepseek-v4-flash", 3e-07, 1.2e-06, 6e-09, True),
|
||||
("deepseek-v4-pro", 1.32e-06, 3.96e-06, 4.4e-08, False),
|
||||
]:
|
||||
info = model_cost.get(key)
|
||||
assert info is not None, f"{key} missing from model_prices_and_context_window.json"
|
||||
assert info["litellm_provider"] == "deepseek"
|
||||
assert info["mode"] == "chat"
|
||||
assert info["input_cost_per_token"] == expected_input
|
||||
assert info["output_cost_per_token"] == expected_output
|
||||
assert info["cache_read_input_token_cost"] == expected_cache
|
||||
assert info["max_input_tokens"] == 1_000_000
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_tool_choice"] is True
|
||||
assert info.get("supports_vision", False) is expected_vision
|
||||
|
||||
# --- provider-prefixed names ---
|
||||
for key, expected_input, expected_output, expected_cache, expected_vision in [
|
||||
("deepseek/deepseek-v4-flash", 3e-07, 1.2e-06, 6e-09, True),
|
||||
("deepseek/deepseek-v4-pro", 1.32e-06, 3.96e-06, 4.4e-08, False),
|
||||
]:
|
||||
info = model_cost.get(key)
|
||||
assert info is not None, f"{key} missing from model_prices_and_context_window.json"
|
||||
assert info["litellm_provider"] == "deepseek"
|
||||
assert info["mode"] == "chat"
|
||||
assert info["input_cost_per_token"] == expected_input
|
||||
assert info["output_cost_per_token"] == expected_output
|
||||
assert info["cache_read_input_token_cost"] == expected_cache
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_tool_choice"] is True
|
||||
assert info.get("supports_vision", False) is expected_vision
|
||||
|
||||
|
||||
def test_deepseek_v4_models_in_backup_cost_map():
|
||||
"""
|
||||
Test that deepseek-v4-flash and deepseek-v4-pro entries are correctly
|
||||
configured in litellm/model_prices_and_context_window_backup.json.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
json_path = Path(__file__).parents[2] / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
# --- bare model names ---
|
||||
for key, expected_input, expected_output, expected_cache, expected_vision in [
|
||||
("deepseek-v4-flash", 3e-07, 1.2e-06, 6e-09, True),
|
||||
("deepseek-v4-pro", 1.32e-06, 3.96e-06, 4.4e-08, False),
|
||||
]:
|
||||
info = model_cost.get(key)
|
||||
assert info is not None, f"{key} missing from backup JSON"
|
||||
assert info["litellm_provider"] == "deepseek"
|
||||
assert info["mode"] == "chat"
|
||||
assert info["input_cost_per_token"] == expected_input
|
||||
assert info["output_cost_per_token"] == expected_output
|
||||
assert info["cache_read_input_token_cost"] == expected_cache
|
||||
assert info["max_input_tokens"] == 1_000_000
|
||||
assert info.get("supports_vision", False) is expected_vision
|
||||
|
||||
# --- provider-prefixed names ---
|
||||
for key, expected_input, expected_output, expected_cache, expected_vision in [
|
||||
("deepseek/deepseek-v4-flash", 3e-07, 1.2e-06, 6e-09, True),
|
||||
("deepseek/deepseek-v4-pro", 1.32e-06, 3.96e-06, 4.4e-08, False),
|
||||
]:
|
||||
info = model_cost.get(key)
|
||||
assert info is not None, f"{key} missing from backup JSON"
|
||||
assert info["litellm_provider"] == "deepseek"
|
||||
assert info["mode"] == "chat"
|
||||
assert info["input_cost_per_token"] == expected_input
|
||||
assert info["output_cost_per_token"] == expected_output
|
||||
assert info["cache_read_input_token_cost"] == expected_cache
|
||||
assert info.get("supports_vision", False) is expected_vision
|
||||
|
||||
|
||||
def test_deprecation_dates_for_retired_xai_and_groq_models():
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
assert model_cost["xai/grok-imagine-image-quality"]["deprecation_date"] == "2026-11-02"
|
||||
assert model_cost["xai/grok-imagine-image-quality-latest"]["deprecation_date"] == "2026-11-02"
|
||||
assert model_cost["xai/grok-imagine-image-quality-20260403"]["deprecation_date"] == "2026-11-02"
|
||||
assert model_cost["groq/gemma-7b-it"]["deprecation_date"] == "2024-12-18"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_deepseek_flash_completion_cost():
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
|
@ -4979,25 +4653,6 @@ def test_anthropic_reexport_entries_carry_explicit_prompt_cache_min_tokens(local
|
|||
assert not wrong, f"(cost-map value, resolved value) diverge from Anthropic's published minimums: {wrong}"
|
||||
|
||||
|
||||
def test_anthropic_reexport_cache_minimums_present_in_root_cost_map() -> None:
|
||||
"""The root map ships to the CDN independently of the bundled backup, so both must carry the
|
||||
minimum or proxies reading one of them regress to the 1024 default."""
|
||||
root_map_path: Final = os.path.join(os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json")
|
||||
with open(root_map_path) as f:
|
||||
root_map: Final = json.load(f)
|
||||
wrong: Final = {
|
||||
model: root_map[model].get("prompt_cache_min_tokens")
|
||||
for model, expected in ANTHROPIC_REEXPORT_CACHE_MIN.items()
|
||||
if root_map[model].get("prompt_cache_min_tokens") != expected
|
||||
}
|
||||
fable_5_wrong: Final = {
|
||||
model: info.get("prompt_cache_min_tokens")
|
||||
for model, info in root_map.items()
|
||||
if "fable-5" in model and info.get("supports_prompt_caching") and info.get("prompt_cache_min_tokens") != 512
|
||||
}
|
||||
assert not wrong and not fable_5_wrong, f"root cost map diverges: {wrong | fable_5_wrong}"
|
||||
|
||||
|
||||
GEMINI_4096_CACHE_MIN_MODELS: Final = tuple(
|
||||
prefix + base
|
||||
for base in (
|
||||
|
|
@ -5024,20 +4679,6 @@ def test_gemini_3_flash_and_31_pro_preview_resolve_4096_cache_minimum(local_mode
|
|||
assert not wrong, f"prompt_cache_min_tokens must be 4096: {wrong}"
|
||||
|
||||
|
||||
def test_gemini_4096_cache_minimum_present_in_root_cost_map() -> None:
|
||||
"""The root map ships to the CDN independently of the bundled backup, so both must carry the
|
||||
minimum or proxies reading one of them regress to the 1024 default."""
|
||||
root_map_path: Final = os.path.join(os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json")
|
||||
with open(root_map_path) as f:
|
||||
root_map: Final = json.load(f)
|
||||
wrong: Final = {
|
||||
model: root_map[model].get("prompt_cache_min_tokens")
|
||||
for model in GEMINI_4096_CACHE_MIN_MODELS
|
||||
if root_map[model].get("prompt_cache_min_tokens") != 4096
|
||||
}
|
||||
assert not wrong, f"prompt_cache_min_tokens must be 4096: {wrong}"
|
||||
|
||||
|
||||
def test_get_prompt_cache_min_tokens_unmapped_model_falls_back_to_default(local_model_cost_map: None) -> None:
|
||||
"""get_model_info raises for a model it has no entry for. The resolver must swallow that and
|
||||
fall back to the default, otherwise the raise reaches callers that would read it as
|
||||
|
|
@ -6508,7 +6149,6 @@ async def test_async_mock_completion_streaming_obj_raises_mock_exception_before_
|
|||
await _async_mock_stream_snapshots(mock_exception, 51234)
|
||||
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _recording_hidden_params_at_submit(submit_target: str) -> "Iterator[queue.SimpleQueue[dict[str, object]]]":
|
||||
seen: Final = queue.SimpleQueue()
|
||||
|
|
|
|||
|
|
@ -1,49 +1,6 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["xai/grok-4.3", "xai/grok-4.3-latest"])
|
||||
def test_xai_grok_4_3_model_info(model):
|
||||
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
|
||||
with open(json_path) as f:
|
||||
model_cost = json.load(f)
|
||||
|
||||
info = model_cost.get(model)
|
||||
assert (
|
||||
info is not None
|
||||
), f"{model} not found in model_prices_and_context_window.json"
|
||||
|
||||
assert info["litellm_provider"] == "xai"
|
||||
assert info["mode"] == "chat"
|
||||
|
||||
assert info["input_cost_per_token"] == 1.25e-06
|
||||
assert info["output_cost_per_token"] == 2.5e-06
|
||||
assert info["cache_read_input_token_cost"] == 2e-07
|
||||
|
||||
assert info["input_cost_per_token_above_200k_tokens"] == 2.5e-06
|
||||
assert info["output_cost_per_token_above_200k_tokens"] == 5e-06
|
||||
assert info["cache_read_input_token_cost_above_200k_tokens"] == 4e-07
|
||||
|
||||
assert info["max_input_tokens"] == 1000000
|
||||
assert info["max_output_tokens"] == 1000000
|
||||
assert info["max_tokens"] == 1000000
|
||||
|
||||
assert info["supports_function_calling"] is True
|
||||
assert info["supports_prompt_caching"] is True
|
||||
assert info["supports_reasoning"] is True
|
||||
assert info["supports_response_schema"] is True
|
||||
assert info["supports_tool_choice"] is True
|
||||
assert info["supports_vision"] is True
|
||||
assert info["supports_web_search"] is True
|
||||
|
||||
routed_model, provider, _, _ = get_llm_provider(model=model)
|
||||
assert routed_model == model.split("/", 1)[1]
|
||||
assert provider == "xai"
|
||||
|
||||
|
||||
def test_xai_grok_4_3_backup_matches_main():
|
||||
"""Ensure the bundled model cost map stays in sync with the canonical file."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue