scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-19 00:01:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 88

test: drop remaining tests that pin cost-map vendor facts

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-09-15 21:34:00 +00:00
parent 7d3917dbf4
commit 62aa21e810
56 changed files with 512 additions and 6997 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1820
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

File diff suppressed because it is too large Load diff

105
tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py
View file

@ -86,21 +86,6 @@ def test_validation_accepts_healthy_file_with_meta_keys():
)
def test_validation_rejects_significant_shrink_vs_backup():
# 600 real models vs a 2000-model backup is below the 50% shrink threshold.
shrunk = _make_models(600)
shrunk[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []}
assert (
GetModelCostMap.validate_model_cost_map(
fetched_map=shrunk,
backup_model_count=2000,
min_model_count=50,
max_shrink_ratio=0.5,
)
is False
)
def test_finalize_pops_key_and_installs_rules():
previous = list(get_fallback_generalization_rules())
try:
@ -134,24 +119,6 @@ def test_finalize_with_no_block_clears_rules():
set_fallback_generalizations(previous)
def test_shipped_backup_carries_the_claude_routing_rules():
"""The bundled backup must ship the Claude routing rules so a fresh install
(or an offline fallback) routes unknown Claude models without code changes.
Bedrock-syntax ids must hit the bedrock rule before the bare-id Anthropic rule."""
backup = GetModelCostMap.load_local_model_cost_map()
rules = backup.get(FALLBACK_GENERALIZATIONS_KEY, {}).get("rules", [])
names = [r.get("name") for r in rules]
assert names.index("bedrock-claude-ids") < names.index("anthropic-claude-ids")
previous = list(get_fallback_generalization_rules())
try:
set_fallback_generalizations(rules)
assert match_routing_generalization("claude-opus-4-9") == "anthropic"
assert match_routing_generalization("global.anthropic.claude-opus-4-9") == "bedrock"
finally:
set_fallback_generalizations(previous)
def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace():
"""Routing rules decide ``litellm_provider`` for otherwise-unknown ids, and the
proxy's wildcard access check (``can_key_call_model`` with a ``bedrock/*`` key)
@ -188,43 +155,6 @@ def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace():
set_fallback_generalizations(previous)
def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
"""Adaptive thinking is data, not code. The bundled backup must carry
supports_adaptive_thinking on genuine Claude >= 4.6 entries (every provider
route) and on the version-gated anthropic-claude-adaptive-thinking rule for
unmapped future Claudes, while leaving the dated Claude 4.0 names
("...-4-20250514") unflagged so a date can never be mistaken for a 4.6+ minor
version. The version-neutral claude-family-baseline capability rule must not flag
it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule
carries only its delta; capability unioning stacks it onto the baseline, so the
baseline block is never duplicated across rules and no rule needs ``extends``."""
backup = GetModelCostMap.load_local_model_cost_map()
rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"]
baseline_rule = next(r for r in rules if r.get("name") == "claude-family-baseline")
adaptive_rule = next(r for r in rules if r.get("name") == "claude-adaptive-thinking")
assert "supports_adaptive_thinking" not in baseline_rule["model_info"]
assert "litellm_provider" not in baseline_rule["model_info"]
assert adaptive_rule["model_info"] == {"supports_adaptive_thinking": True}
assert all("extends" not in r for r in rules)
for adaptive in [
"anthropic.claude-opus-4-8",
"vertex_ai/claude-opus-4-6@default",
"us.anthropic.claude-sonnet-4-6",
"openrouter/anthropic/claude-opus-4.7",
"azure_ai/claude-opus-4-7",
]:
assert backup[adaptive]["supports_adaptive_thinking"] is True, adaptive
for non_adaptive in [
"claude-opus-4-20250514",
"us.anthropic.claude-opus-4-20250514-v1:0",
"claude-opus-4-5",
]:
assert "supports_adaptive_thinking" not in backup[non_adaptive], non_adaptive
# OpenRouter headline rates from GET https://openrouter.ai/api/v1/models.
# These were the catalog values that disagreed with that API (and, for the
# two spotlight models, the public model pages that their source fields cite).
@ -659,41 +589,6 @@ def test_boot_load_records_the_blob_id_of_the_bytes_served_and_the_fetch_etag():
assert source["loaded_at"] is not None
def test_boot_load_fallback_to_the_backup_reports_its_blob_id_and_drops_the_remote_etag():
remote, _ = _mock_client(
[httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_real_map_bytes())], client_cls=httpx.Client
)
get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=remote)
failing, _ = _mock_client([httpx.Response(404)], client_cls=httpx.Client)
get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=failing)
source = get_model_cost_map_source_info()
assert source["source"] == "local"
assert source["etag"] is None
assert source["source_revision"] == _bundled_blob_id()
def test_boot_load_that_fails_the_integrity_check_reports_the_backup_not_the_rejected_fetch():
remote, _ = _mock_client(
[httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_real_map_bytes())], client_cls=httpx.Client
)
get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=remote)
shrunk_body = b'{"gpt-5.4-mini": {"mode": "chat", "input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06}}'
shrunk, _ = _mock_client(
[httpx.Response(200, headers={"ETag": 'W/"shrunk"'}, content=shrunk_body)], client_cls=httpx.Client
)
get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=shrunk)
source = get_model_cost_map_source_info()
assert source["source"] == "local"
assert source["fallback_reason"] == "Remote data failed integrity validation"
assert source["etag"] is None
assert source["source_revision"] == _bundled_blob_id()
assert source["source_revision"] != git_blob_id(shrunk_body)
@pytest.mark.parametrize(
("argv0", "request_count"),
[

54
tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py
View file

@ -4,7 +4,7 @@ from typing import Final
import pytest
from pydantic import TypeAdapter
from litellm import completion_cost, cost_per_token, get_model_info
from litellm import completion_cost
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.types.utils import TranscriptionResponse
@ -52,58 +52,6 @@ def test_azure_ai_catalog_name_routes_to_azure_ai(catalog_name: str) -> None:
assert (routed_model, provider) == (catalog_name, "azure_ai")
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize("catalog_name", TOKEN_PRICED_NAMES)
def test_azure_ai_catalog_name_charges_its_own_entry_per_token(catalog_name: str) -> None:
entry: Final = get_model_info(f"azure_ai/{catalog_name}")
prompt_cost, completion_cost_usd = cost_per_token(
model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=A_MILLION
)
assert prompt_cost > 0
assert prompt_cost == pytest.approx(A_MILLION * entry["input_cost_per_token"])
assert completion_cost_usd == pytest.approx(A_MILLION * entry["output_cost_per_token"])
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize("catalog_name", TOKEN_PRICED_NAMES)
def test_azure_ai_catalog_name_prices_the_same_in_any_casing(catalog_name: str) -> None:
lowercase_cost = cost_per_token(model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0)
upper_cost = cost_per_token(model=f"azure_ai/{catalog_name.upper()}", prompt_tokens=A_MILLION, completion_tokens=0)
assert upper_cost == lowercase_cost
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize("catalog_name", GROK_4_20_NAMES)
def test_azure_ai_grok_4_20_bills_cached_prompt_tokens_at_the_input_price(catalog_name: str) -> None:
uncached_prompt_cost, _ = cost_per_token(
model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0
)
cached_prompt_cost, _ = cost_per_token(
model=f"azure_ai/{catalog_name}",
prompt_tokens=A_MILLION,
completion_tokens=0,
cache_read_input_tokens=A_MILLION,
)
assert uncached_prompt_cost > 0
assert cached_prompt_cost == pytest.approx(uncached_prompt_cost)
@pytest.mark.usefixtures("local_model_cost_map")
def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None:
one_second_cost: Final = _whisper_transcription_cost(1)
one_hour_cost: Final = _whisper_transcription_cost(AN_HOUR_IN_SECONDS)
assert one_second_cost > 0
assert one_hour_cost == pytest.approx(AN_HOUR_IN_SECONDS * one_second_cost)
@pytest.mark.parametrize("catalog_name", CATALOG_NAMES)
def test_azure_ai_catalog_entry_source_and_backup_match(catalog_name: str) -> None:
main_entry = _cost_map_entry(MAIN_COST_MAP, catalog_name)
backup_entry = _cost_map_entry(BACKUP_COST_MAP, catalog_name)
assert backup_entry == main_entry
def test_azure_ai_model_router_spellings_share_one_entry() -> None:
underscore_entry = _cost_map_entry(MAIN_COST_MAP, "model_router")
hyphen_entry = _cost_map_entry(MAIN_COST_MAP, "model-router")

176
tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py
View file

@ -4,18 +4,13 @@ from typing import NamedTuple
import pytest
import litellm
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
from litellm.llms.bedrock.common_utils import BedrockModelInfo
from litellm.utils import _get_model_info_helper
from litellm.cost_calculator import completion_cost
from litellm.types.utils import (
Choices,
Message,
ModelResponse,
PromptTokensDetailsWrapper,
Usage,
)
@ -31,8 +26,7 @@ def local_model_cost_map(monkeypatch):
litellm.bedrock_converse_models.update(
key
for key, value in litellm.model_cost.items()
if isinstance(value, dict)
and value.get("litellm_provider") == "bedrock_converse"
if isinstance(value, dict) and value.get("litellm_provider") == "bedrock_converse"
)
yield
finally:
@ -56,45 +50,69 @@ class GptProfile(NamedTuple):
GPT_5_6_PROFILES = [
GptProfile(
model_id="us.openai.gpt-5.6-sol",
input_cost=4.4e-06, input_cost_above_272k=8.8e-06,
cache_write=5.5e-06, cache_write_above_272k=1.1e-05,
cache_read=4.4e-07, cache_read_above_272k=8.8e-07,
output_cost=2.2e-05, output_cost_above_272k=3.3e-05,
input_cost=4.4e-06,
input_cost_above_272k=8.8e-06,
cache_write=5.5e-06,
cache_write_above_272k=1.1e-05,
cache_read=4.4e-07,
cache_read_above_272k=8.8e-07,
output_cost=2.2e-05,
output_cost_above_272k=3.3e-05,
),
GptProfile(
model_id="global.openai.gpt-5.6-sol",
input_cost=4e-06, input_cost_above_272k=8e-06,
cache_write=5e-06, cache_write_above_272k=1e-05,
cache_read=4e-07, cache_read_above_272k=8e-07,
output_cost=2e-05, output_cost_above_272k=3e-05,
input_cost=4e-06,
input_cost_above_272k=8e-06,
cache_write=5e-06,
cache_write_above_272k=1e-05,
cache_read=4e-07,
cache_read_above_272k=8e-07,
output_cost=2e-05,
output_cost_above_272k=3e-05,
),
GptProfile(
model_id="us.openai.gpt-5.6-terra",
input_cost=2.2e-06, input_cost_above_272k=4.4e-06,
cache_write=2.75e-06, cache_write_above_272k=5.5e-06,
cache_read=2.2e-07, cache_read_above_272k=4.4e-07,
output_cost=1.32e-05, output_cost_above_272k=1.98e-05,
input_cost=2.2e-06,
input_cost_above_272k=4.4e-06,
cache_write=2.75e-06,
cache_write_above_272k=5.5e-06,
cache_read=2.2e-07,
cache_read_above_272k=4.4e-07,
output_cost=1.32e-05,
output_cost_above_272k=1.98e-05,
),
GptProfile(
model_id="global.openai.gpt-5.6-terra",
input_cost=2e-06, input_cost_above_272k=4e-06,
cache_write=2.5e-06, cache_write_above_272k=5e-06,
cache_read=2e-07, cache_read_above_272k=4e-07,
output_cost=1.2e-05, output_cost_above_272k=1.8e-05,
input_cost=2e-06,
input_cost_above_272k=4e-06,
cache_write=2.5e-06,
cache_write_above_272k=5e-06,
cache_read=2e-07,
cache_read_above_272k=4e-07,
output_cost=1.2e-05,
output_cost_above_272k=1.8e-05,
),
GptProfile(
model_id="us.openai.gpt-5.6-luna",
input_cost=2.2e-07, input_cost_above_272k=4.4e-07,
cache_write=2.75e-07, cache_write_above_272k=5.5e-07,
cache_read=2.2e-08, cache_read_above_272k=4.4e-08,
output_cost=1.32e-06, output_cost_above_272k=1.98e-06,
input_cost=2.2e-07,
input_cost_above_272k=4.4e-07,
cache_write=2.75e-07,
cache_write_above_272k=5.5e-07,
cache_read=2.2e-08,
cache_read_above_272k=4.4e-08,
output_cost=1.32e-06,
output_cost_above_272k=1.98e-06,
),
GptProfile(
model_id="global.openai.gpt-5.6-luna",
input_cost=2e-07, input_cost_above_272k=4e-07,
cache_write=2.5e-07, cache_write_above_272k=5e-07,
cache_read=2e-08, cache_read_above_272k=4e-08,
output_cost=1.2e-06, output_cost_above_272k=1.8e-06,
input_cost=2e-07,
input_cost_above_272k=4e-07,
cache_write=2.5e-07,
cache_write_above_272k=5e-07,
cache_read=2e-08,
cache_read_above_272k=4e-08,
output_cost=1.2e-06,
output_cost_above_272k=1.8e-06,
),
]
@ -116,112 +134,18 @@ def _bedrock_response(model, usage):
)
def test_proxy_cost_calculation_scenario():
"""Test exact GitHub issue scenario: proxy cost calculation"""
model = "litellm_proxy/bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0"
# Test model info lookup works
model_info = _get_model_info_helper(
model=model, custom_llm_provider="litellm_proxy"
)
assert model_info is not None
# Test cost calculation works
response = ModelResponse(
id="test",
created=1234567890,
model=model,
object="chat.completion",
choices=[
Choices(
finish_reason="stop",
index=0,
message=Message(content="Test", role="assistant"),
)
],
usage=Usage(total_tokens=150, prompt_tokens=100, completion_tokens=50),
)
cost = completion_cost(
completion_response=response, model=model, custom_llm_provider="litellm_proxy"
)
expected_cost = (100 * 8e-07) + (50 * 4e-06)
assert cost == expected_cost
@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)
def test_bedrock_gpt_5_6_profiles_route_to_converse(profile, local_model_cost_map):
"""GPT-5.6 is served by Converse on bedrock-runtime, never by Invoke."""
assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}") == "converse"
def test_bedrock_gpt_5_6_above_272k_tier_applies_to_cost(local_model_cost_map):
"""A prompt over 272K tokens is billed at the long-context rate, not the base rate."""
response = _bedrock_response(
"bedrock/us.openai.gpt-5.6-sol",
Usage(prompt_tokens=300000, completion_tokens=1000, total_tokens=301000),
)
cost = completion_cost(
completion_response=response,
model="bedrock/us.openai.gpt-5.6-sol",
custom_llm_provider="bedrock",
)
assert cost == pytest.approx((300000 * 8.8e-06) + (1000 * 3.3e-05), rel=1e-9)
def test_bedrock_gpt_5_6_bills_cache_read_tokens(local_model_cost_map):
"""Bedrock caches long prefixes implicitly and reports them, so a cache-read turn
must be billed at the cache rate rather than dropped to zero."""
usage = Usage(
prompt_tokens=15611,
completion_tokens=5,
total_tokens=15616,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=15609),
)
response = _bedrock_response("bedrock/us.openai.gpt-5.6-sol", usage)
cost = completion_cost(
completion_response=response,
model="bedrock/us.openai.gpt-5.6-sol",
custom_llm_provider="bedrock",
)
expected = (2 * 4.4e-06) + (15609 * 4.4e-07) + (5 * 2.2e-05)
assert cost == pytest.approx(expected, rel=1e-9)
# Without cache_read_input_token_cost the cached prefix bills at zero.
assert cost > (15611 * 4.4e-06) * 0.1
def test_bedrock_gpt_5_6_bills_cache_write_tokens(local_model_cost_map):
"""The write side of the same cache cycle is billed at the 30m cache-write rate."""
usage = Usage(
prompt_tokens=15611,
completion_tokens=5,
total_tokens=15616,
cache_creation_input_tokens=15609,
)
response = _bedrock_response("bedrock/us.openai.gpt-5.6-sol", usage)
cost = completion_cost(
completion_response=response,
model="bedrock/us.openai.gpt-5.6-sol",
custom_llm_provider="bedrock",
)
expected = (2 * 4.4e-06) + (15609 * 5.5e-06) + (5 * 2.2e-05)
assert cost == pytest.approx(expected, rel=1e-9)
@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)
def test_bedrock_gpt_5_6_offers_tools_and_reasoning_effort_but_not_thinking(profile, local_model_cost_map):
"""GPT-5.x on Converse maps reasoning_effort to reasoning.effort, so reasoning_effort
is offered while the Anthropic-only thinking/output_config are not, alongside the tool
params these models accept."""
supported = AmazonConverseConfig().get_supported_openai_params(
model=f"bedrock/{profile.model_id}"
)
supported = AmazonConverseConfig().get_supported_openai_params(model=f"bedrock/{profile.model_id}")
assert "tools" in supported
assert "tool_choice" in supported

44
tests/test_litellm/llms/cohere/ocr/test_cohere_parse_cost.py
View file

@ -1,44 +0,0 @@
from pathlib import Path
import pytest
import litellm
from litellm.cost_calculator import completion_cost
from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo
COST_PER_PAGE = 0.0015
REPO_ROOT = Path(__file__).parents[5]
COST_MAPS = [
REPO_ROOT / "model_prices_and_context_window.json",
REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json",
]
MODELS = [("cohere/parse-v5.0", "cohere"), ("azure_ai/Cohere-parse-v5", "azure_ai")]
def _ocr_response(model: str, pages_processed: int) -> OCRResponse:
return OCRResponse(
pages=[OCRPage(index=i, markdown=f"page {i}") for i in range(pages_processed)],
model=model,
usage_info=OCRUsageInfo(pages_processed=pages_processed),
)
@pytest.mark.parametrize("model, provider", MODELS)
def test_model_info_resolves_ocr_mode_and_price(local_model_cost_map, model: str, provider: str) -> None:
info = litellm.get_model_info(model=model, custom_llm_provider=provider)
assert info["mode"] == "ocr"
assert info["ocr_cost_per_page"] == COST_PER_PAGE
@pytest.mark.parametrize("model, provider", MODELS)
@pytest.mark.parametrize("pages_processed", [1, 3])
def test_cost_scales_with_billed_pages(local_model_cost_map, model: str, provider: str, pages_processed: int) -> None:
cost = completion_cost(
completion_response=_ocr_response(model.split("/", 1)[1], pages_processed),
model=model,
custom_llm_provider=provider,
call_type="ocr",
)
assert cost == pytest.approx(COST_PER_PAGE * pages_processed)

52
tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py
View file

@ -1,4 +1,3 @@
import json
from decimal import Decimal
from pathlib import Path
from typing import Final
@ -163,29 +162,6 @@ def test_legacy_endpoint_names_still_resolve(local_model_cost_map: None) -> None
assert completion_cost == pytest.approx(100 * info["output_cost_per_token"])
@pytest.mark.parametrize("model", NEW_MODELS)
def test_new_models_carry_cache_pricing(local_model_cost_map: None, model: str) -> None:
info: Final = _model_info(model)
assert info["input_cost_per_token"] > 0
assert info["output_cost_per_token"] > 0
assert info["cache_creation_input_token_cost"] > info["input_cost_per_token"]
assert info["cache_read_input_token_cost"] < info["input_cost_per_token"]
assert info["supports_prompt_caching"] is True
def test_every_priced_databricks_model_declares_cache_rates(local_model_cost_map: None) -> None:
undeclared: Final = [
model
for model, info in litellm.model_cost.items()
if model.startswith("databricks/")
and info.get("input_cost_per_token") is not None
and any(info.get(field) is None for field in CACHE_FIELDS)
]
assert undeclared == []
def test_models_without_a_cache_discount_bill_cache_tokens_at_the_input_rate(
local_model_cost_map: None,
) -> None:
@ -204,34 +180,6 @@ def test_models_without_a_cache_discount_bill_cache_tokens_at_the_input_rate(
assert prompt_cost > 8000 * info["input_cost_per_token"]
def test_every_model_without_published_cache_dbu_bills_cache_at_its_own_input_rate(
local_model_cost_map: None,
) -> None:
without_published_rates: Final = [
model
for model, info in litellm.model_cost.items()
if model.startswith("databricks/")
and info.get("input_cost_per_token")
and model not in PUBLISHED_DBU_PER_MILLION
]
assert len(without_published_rates) == 14
for model in without_published_rates:
info = _model_info(model)
for field in CACHE_FIELDS:
assert info[field] == pytest.approx(info["input_cost_per_token"]), (model, field)
@pytest.mark.parametrize("model", NEW_MODELS)
def test_backup_price_map_matches_main(model: str) -> None:
main_cost: Final = json.loads(MAIN_PRICES.read_text())
backup_cost: Final = json.loads(BACKUP_PRICES.read_text())
assert model in main_cost
assert model in backup_cost
assert backup_cost[model] == main_cost[model]
def test_sonnet_5_ships_standard_rates_not_introductory(local_model_cost_map: None) -> None:
sonnet_5: Final = _model_info("databricks/databricks-claude-sonnet-5")
sonnet_4_6: Final = _model_info("databricks/databricks-claude-sonnet-4-6")

51
tests/test_litellm/llms/databricks/test_databricks_pricing.py
View file

@ -1,51 +0,0 @@
import json
import os
import sys
def test_databricks_pricing_integrity():
"""
Verifies that for all Databricks models in model_prices_and_context_window.json:
USD Price == DBU Price * 0.07
"""
json_path = os.path.join(
os.path.dirname(__file__), "../../../../model_prices_and_context_window.json"
)
# Verify file exists
assert os.path.exists(
json_path
), f"Could not find model_prices_and_context_window.json at {json_path}"
with open(json_path, "r") as f:
data = json.load(f)
conversion_rate = 0.07 # 1 DBU = 0.07 USD
errors = []
for model, info in data.items():
if info.get("litellm_provider") == "databricks":
# Check Input Cost
input_usd = info.get("input_cost_per_token")
input_dbu = info.get("input_dbu_cost_per_token")
if input_usd is not None and input_dbu is not None:
expected = input_dbu * conversion_rate
# Allow small floating point difference
if abs(input_usd - expected) > 1e-9:
errors.append(
f"{model} input mismatch: USD={input_usd}, DBU={input_dbu}, Expected={expected}"
)
# Check Output Cost
output_usd = info.get("output_cost_per_token")
output_dbu = info.get("output_dbu_cost_per_token")
if output_usd is not None and output_dbu is not None:
expected = output_dbu * conversion_rate
if abs(output_usd - expected) > 1e-9:
errors.append(
f"{model} output mismatch: USD={output_usd}, DBU={output_dbu}, Expected={expected}"
)
assert not errors, "\n" + "\n".join(errors)

55
tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py
View file

@ -1,10 +1,6 @@
import math
from datetime import datetime, timezone
import pytest
import litellm
from litellm.llms.fireworks_ai.cost_calculator import cost_per_token
from litellm.types.utils import OffPeakPricing, PromptTokensDetailsWrapper, Usage
@ -26,49 +22,6 @@ def _usage(prompt_tokens: int, cached_tokens: int, completion_tokens: int) -> Us
)
def test_cached_prompt_tokens_billed_at_cache_read_rate():
prompt_tokens = 7036
cached_tokens = 7020
completion_tokens = 8
prompt_cost, completion_cost = cost_per_token(
model=MODEL, usage=_usage(prompt_tokens, cached_tokens, completion_tokens)
)
expected_prompt_cost = (prompt_tokens - cached_tokens) * INPUT_COST + cached_tokens * CACHE_READ_COST
assert prompt_cost == pytest.approx(expected_prompt_cost)
assert completion_cost == pytest.approx(completion_tokens * OUTPUT_COST)
full_rate_cost = prompt_tokens * INPUT_COST
assert prompt_cost < full_rate_cost
def test_warm_call_cheaper_than_cold_call():
prompt_tokens = 7036
completion_tokens = 8
cold_prompt_cost, _ = cost_per_token(
model=MODEL, usage=_usage(prompt_tokens, 16, completion_tokens)
)
warm_prompt_cost, _ = cost_per_token(
model=MODEL, usage=_usage(prompt_tokens, 7020, completion_tokens)
)
assert warm_prompt_cost < cold_prompt_cost
def test_no_cached_tokens_matches_full_input_rate():
prompt_tokens = 100
completion_tokens = 10
prompt_cost, completion_cost = cost_per_token(
model=MODEL, usage=_usage(prompt_tokens, 0, completion_tokens)
)
assert prompt_cost == pytest.approx(prompt_tokens * INPUT_COST)
assert completion_cost == pytest.approx(completion_tokens * OUTPUT_COST)
OFF_PEAK_MODEL = "accounts/fireworks/models/off-peak-test"
OFF_PEAK_WINDOW = "14:00-00:00"
INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc)
@ -78,7 +31,9 @@ STANDARD_OUTPUT_COST = 6e-07
STANDARD_CACHE_READ_COST = 1.5e-08
def _register_off_peak_model(off_peak_pricing: OffPeakPricing, cache_read_cost: float | None = STANDARD_CACHE_READ_COST) -> None:
def _register_off_peak_model(
off_peak_pricing: OffPeakPricing, cache_read_cost: float | None = STANDARD_CACHE_READ_COST
) -> None:
litellm.model_cost[f"fireworks_ai/{OFF_PEAK_MODEL}"] = {
"litellm_provider": "fireworks_ai",
"mode": "chat",
@ -151,7 +106,9 @@ def test_off_peak_window_bills_cached_tokens_at_the_off_peak_input_rate_without_
def test_off_peak_defaults_to_the_current_time():
"""The proxy's cost dispatch passes no clock, so an all-day window has to apply on the
default current time."""
_register_off_peak_model({"hours_utc": "00:00-00:00", "input_cost_per_token": 1e-08, "output_cost_per_token": 2e-08})
_register_off_peak_model(
{"hours_utc": "00:00-00:00", "input_cost_per_token": 1e-08, "output_cost_per_token": 2e-08}
)
usage = _usage(prompt_tokens=1000, cached_tokens=0, completion_tokens=200)
prompt_cost, completion_cost = cost_per_token(model=OFF_PEAK_MODEL, usage=usage)

65
tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_kimi_model_metadata.py
View file

@ -1,65 +0,0 @@
"""
Regression test for Fireworks Kimi K2.5 / K2.6 / K2.7 context and output limits.
Fireworks publishes a 262144-token context window for every Kimi K2.5, K2.6 and
K2.7 model, but caps generation well below that. A previous bulk edit had flattened
max_output_tokens/max_tokens to 262144 (equal to the context window), which let the
pre-call context-window check admit requests asking for a full 262144-token
completion that Fireworks then rejects. These assertions pin the corrected per-alias
limits so a future bulk edit can't silently flatten them again.
"""
import json
from importlib.resources import files
import pytest
CONTEXT_WINDOW = 262144
OUTPUT_LIMIT = 32768
KIMI_ALIASES = (
"fireworks_ai/kimi-k2p5",
"fireworks_ai/kimi-k2p6",
"fireworks_ai/kimi-k2p6-fast",
"fireworks_ai/kimi-k2p7-code",
"fireworks_ai/kimi-k2p7-code-fast",
"fireworks_ai/accounts/fireworks/models/kimi-k2p5",
"fireworks_ai/accounts/fireworks/models/kimi-k2p6",
"fireworks_ai/accounts/fireworks/models/kimi-k2p7-code",
"fireworks_ai/accounts/fireworks/routers/kimi-k2p6-fast",
"fireworks_ai/accounts/fireworks/routers/kimi-k2p7-code-fast",
)
@pytest.fixture(scope="module")
def use_local_model_cost_map():
monkeypatch = pytest.MonkeyPatch()
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
import litellm
from litellm.utils import _invalidate_model_cost_lowercase_map
original_model_cost = litellm.model_cost
litellm.model_cost = json.loads(
files("litellm")
.joinpath("model_prices_and_context_window_backup.json")
.read_text(encoding="utf-8")
)
litellm.get_model_info.cache_clear()
_invalidate_model_cost_lowercase_map()
try:
yield litellm
finally:
litellm.model_cost = original_model_cost
litellm.get_model_info.cache_clear()
_invalidate_model_cost_lowercase_map()
monkeypatch.undo()
@pytest.mark.parametrize("alias", KIMI_ALIASES)
def test_fireworks_kimi_get_model_info_limits(use_local_model_cost_map, alias):
model_info = use_local_model_cost_map.get_model_info(model=alias)
assert model_info["max_input_tokens"] == CONTEXT_WINDOW
assert model_info["max_output_tokens"] == OUTPUT_LIMIT
assert model_info["max_tokens"] == OUTPUT_LIMIT

13
tests/test_litellm/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py
View file

@ -4,7 +4,6 @@ import json
import httpx
import pytest
import litellm
from litellm.llms.gemini.audio_transcription.transformation import (
GeminiAudioTranscriptionConfig,
@ -318,15 +317,3 @@ class TestCostRegression:
assert live_entry["input_cost_per_token"] == 3.5e-06
assert live_entry["output_cost_per_token"] == 2.1e-05
assert live_entry["supported_endpoints"] == ["/v1/realtime"]
def test_completion_cost_bills_provider_reported_tokens(self, config, local_cost_map):
payload = json.loads(json.dumps(COMPLETED_RESPONSE))
payload["usage"]["total_output_tokens"] = 10
payload["usage"]["total_tokens"] = 210
response = config.transform_audio_transcription_response(make_response(payload))
cost = litellm.completion_cost(
completion_response=response,
model="gemini/gemini-3.5-transcribe",
call_type="transcription",
)
assert cost == pytest.approx(199 * 2e-06 + 1 * 2e-06 + 10 * 1.2e-05)

128
tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py
View file

@ -1,128 +0,0 @@
"""
Cost tests for Mistral OCR models against the real litellm cost map
(no monkeypatching of get_model_info). These regress the pricing entries
for mistral-ocr-4-0 and mistral-ocr-latest, which now both resolve to
OCR 4 at $4 / 1000 pages.
"""
from pathlib import Path
import pytest
import litellm
from litellm.cost_calculator import completion_cost
from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo
OCR4_COST_PER_PAGE = 0.004
OCR4_ANNOTATION_COST_PER_PAGE = 0.005
REPO_ROOT = Path(__file__).parents[5]
MAIN_COST_MAP = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_COST_MAP = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
OCR3_MODEL = "mistral/mistral-ocr-2512"
OCR3_COST_PER_PAGE = 0.002
OCR3_ANNOTATION_COST_PER_PAGE = 0.003
AZURE_DOC_AI_MODEL = "azure_ai/mistral-document-ai-2512"
AZURE_DOC_AI_COST_PER_PAGE = 0.003
def _ocr_response(model: str, pages_processed: int) -> OCRResponse:
return OCRResponse(
pages=[OCRPage(index=i, markdown=f"page {i}") for i in range(pages_processed)],
model=model,
usage_info=OCRUsageInfo(pages_processed=pages_processed),
)
def _annotated_ocr_response(model: str, pages_processed: int | None, annotation_pages: int) -> OCRResponse:
return OCRResponse(
pages=[],
model=model,
usage_info=OCRUsageInfo(pages_processed=pages_processed, pages_processed_annotation=annotation_pages),
)
@pytest.mark.parametrize("model", ["mistral-ocr-4-0", "mistral-ocr-latest"])
@pytest.mark.parametrize("pages_processed", [1, 3, 10])
def test_ocr4_cost_scales_with_pages(model: str, pages_processed: int) -> None:
cost = completion_cost(
completion_response=_ocr_response(model, pages_processed),
model=f"mistral/{model}",
custom_llm_provider="mistral",
call_type="ocr",
)
assert cost == pytest.approx(OCR4_COST_PER_PAGE * pages_processed)
def test_ocr3_model_info_price(local_model_cost_map) -> None:
info = litellm.get_model_info(model=OCR3_MODEL, custom_llm_provider="mistral")
assert info["ocr_cost_per_page"] == OCR3_COST_PER_PAGE
@pytest.mark.parametrize("pages_processed", [1, 3, 10])
def test_ocr3_cost_scales_with_pages(local_model_cost_map, pages_processed: int) -> None:
cost = completion_cost(
completion_response=_ocr_response("mistral-ocr-2512", pages_processed),
model=OCR3_MODEL,
custom_llm_provider="mistral",
call_type="ocr",
)
assert cost == pytest.approx(OCR3_COST_PER_PAGE * pages_processed)
def test_ocr3_bills_ocr_and_annotation_pages_at_their_own_rates(local_model_cost_map) -> None:
cost = completion_cost(
completion_response=_annotated_ocr_response("mistral-ocr-2512", 2, 3),
model=OCR3_MODEL,
custom_llm_provider="mistral",
call_type="ocr",
)
assert cost == pytest.approx(2 * OCR3_COST_PER_PAGE + 3 * OCR3_ANNOTATION_COST_PER_PAGE)
def test_ocr3_bills_annotation_only_response(local_model_cost_map) -> None:
cost = completion_cost(
completion_response=_annotated_ocr_response("mistral-ocr-2512", 0, 3),
model=OCR3_MODEL,
custom_llm_provider="mistral",
call_type="ocr",
)
assert cost == pytest.approx(3 * OCR3_ANNOTATION_COST_PER_PAGE)
def test_ocr3_bills_annotation_pages_when_pages_processed_missing(local_model_cost_map) -> None:
cost = completion_cost(
completion_response=_annotated_ocr_response("mistral-ocr-2512", None, 4),
model=OCR3_MODEL,
custom_llm_provider="mistral",
call_type="ocr",
)
assert cost == pytest.approx(4 * OCR3_ANNOTATION_COST_PER_PAGE)
def test_azure_doc_ai_annotation_pages_fall_back_to_ocr_rate(local_model_cost_map) -> None:
info = litellm.get_model_info(model=AZURE_DOC_AI_MODEL, custom_llm_provider="azure_ai")
assert info.get("annotation_cost_per_page") is None
assert info["ocr_cost_per_page"] == AZURE_DOC_AI_COST_PER_PAGE
cost = completion_cost(
completion_response=_annotated_ocr_response("mistral-document-ai-2512", 0, 1),
model=AZURE_DOC_AI_MODEL,
custom_llm_provider="azure_ai",
call_type="ocr",
)
assert cost == pytest.approx(AZURE_DOC_AI_COST_PER_PAGE)
def test_azure_ocr4_bills_ocr_and_annotation_pages_at_their_own_rates(local_model_cost_map) -> None:
info = litellm.get_model_info(model="azure_ai/mistral-ocr-4-0", custom_llm_provider="azure_ai")
assert info["ocr_cost_per_page"] == OCR4_COST_PER_PAGE
assert info["annotation_cost_per_page"] == OCR4_ANNOTATION_COST_PER_PAGE
cost = completion_cost(
completion_response=_annotated_ocr_response("mistral-ocr-4-0", 2, 3),
model="azure_ai/mistral-ocr-4-0",
custom_llm_provider="azure_ai",
call_type="ocr",
)
assert cost == pytest.approx(2 * OCR4_COST_PER_PAGE + 3 * OCR4_ANNOTATION_COST_PER_PAGE)

6
tests/test_litellm/llms/openai/test_cost_calculation.py
View file

@ -75,9 +75,3 @@ def test_shipped_per_second_models_bill_a_non_zero_cost(model, provider):
prompt_cost, completion_cost = cost_per_second(model=model, custom_llm_provider=provider, duration=60.0)
assert prompt_cost + completion_cost > 0.0
def test_whisper_bills_its_documented_rate_once():
prompt_cost, completion_cost = cost_per_second(model="whisper-1", custom_llm_provider="openai", duration=30.0)
assert prompt_cost + completion_cost == pytest.approx(0.003)

29
tests/test_litellm/llms/openai_like/test_scx_ai_provider.py
View file

@ -154,35 +154,6 @@ class TestSCXAIModelMetadata:
with open(json_path) as f:
return json.load(f)
def test_scx_ai_models_registered_with_correct_metadata(self):
model_cost = self._load(("model_prices_and_context_window.json",))
for model in self.SCX_MODELS:
info = model_cost.get(model)
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
assert info["litellm_provider"] == "scx-ai"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] > 0
assert info["output_cost_per_token"] > 0
assert info["supports_function_calling"] is True
assert info["supports_tool_choice"] is True
assert info["supports_reasoning"] is True
assert info["supports_response_schema"] is True
assert info.get("supports_vision", False) is (model in self.VISION_MODELS)
assert info["supports_prompt_caching"] is True
assert 0 < info["cache_read_input_token_cost"] < info["input_cost_per_token"]
assert info["max_output_tokens"] == 131072
assert info["max_tokens"] == info["max_output_tokens"]
assert info["max_input_tokens"] >= 1_000_000
def test_scx_ai_models_synced_to_backup(self):
model_cost = self._load(("model_prices_and_context_window.json",))
backup = self._load(("litellm", "model_prices_and_context_window_backup.json"))
for model in self.SCX_MODELS:
assert model in backup, f"{model} missing from backup json"
assert backup[model] == model_cost[model], f"{model} differs between root and backup json"
class TestSCXAIDashboardRegistration:
@staticmethod

334
tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py
View file

@ -14,17 +14,15 @@ from unittest.mock import patch
import pytest
# Add the project root to Python path
import litellm
from litellm.cost_calculator import completion_cost, cost_per_token
from litellm.llms.perplexity.cost_calculator import (
cost_per_token as perplexity_cost_per_token,
)
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
OffPeakPricing,
Usage,
PromptTokensDetailsWrapper,
Usage,
)
@ -64,167 +62,6 @@ class TestPerplexityCostCalculator:
}
}
def test_basic_cost_calculation(self):
"""Test basic cost calculation without additional fields."""
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
prompt_cost, completion_cost = perplexity_cost_per_token(
model="sonar-deep-research", usage=usage
)
# Expected costs:
# Input: 100 tokens * $2e-6 = $0.0002
# Output: 50 tokens * $8e-6 = $0.0004
expected_prompt_cost = 100 * 2e-6
expected_completion_cost = 50 * 8e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6)
def test_citation_tokens_cost_calculation(self):
"""Test cost calculation with citation tokens."""
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
# Add citation tokens
usage.citation_tokens = 25
prompt_cost, completion_cost = perplexity_cost_per_token(
model="sonar-deep-research", usage=usage
)
# Expected costs:
# Input: 100 tokens * $2e-6 = $0.0002
# Citation: 25 tokens * $2e-6 = $0.00005
# Total prompt cost: $0.00025
# Output: 50 tokens * $8e-6 = $0.0004
expected_prompt_cost = (100 * 2e-6) + (25 * 2e-6)
expected_completion_cost = 50 * 8e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6)
def test_search_queries_cost_calculation(self):
"""Test cost calculation with search queries."""
usage = Usage(
prompt_tokens=100,
completion_tokens=50,
total_tokens=150,
prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=3),
)
prompt_cost, completion_cost = perplexity_cost_per_token(
model="sonar-deep-research", usage=usage
)
# Expected costs:
# Input: 100 tokens * $2e-6 = $0.0002
# Output: 50 tokens * $8e-6 = $0.0004
# Search: 3 queries * $0.005 per request = $0.015
# Total completion cost: $0.0154
expected_prompt_cost = 100 * 2e-6
expected_completion_cost = (50 * 8e-6) + (3 * 0.005)
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6)
def test_reasoning_tokens_from_direct_attribute(self):
"""Test reasoning tokens cost calculation from direct attribute."""
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
# Set reasoning tokens directly
usage.reasoning_tokens = 20
prompt_cost, completion_cost = perplexity_cost_per_token(
model="sonar-deep-research", usage=usage
)
# `completion_tokens` includes `reasoning_tokens` per the OpenAI/Perplexity
# convention codified in PR #18607. Non-reasoning portion = 50 - 20 = 30.
# Input: 100 tokens * $2e-6 = $0.0002
# Output (text): 30 tokens * $8e-6 = $0.00024
# Reasoning: 20 tokens * $3e-6 = $0.00006
# Total completion cost = $0.0003
expected_prompt_cost = 100 * 2e-6
expected_completion_cost = ((50 - 20) * 8e-6) + (20 * 3e-6)
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6)
def test_reasoning_tokens_from_completion_tokens_details(self):
"""Test reasoning tokens cost calculation from completion_tokens_details."""
usage = Usage(
prompt_tokens=100,
completion_tokens=50,
total_tokens=150,
reasoning_tokens=20, # This should be stored in completion_tokens_details
)
prompt_cost, completion_cost = perplexity_cost_per_token(
model="sonar-deep-research", usage=usage
)
# Same convention as the direct-attribute case above; reasoning is a subset of
# completion_tokens, so non-reasoning portion = 50 - 20 = 30.
expected_prompt_cost = 100 * 2e-6
expected_completion_cost = ((50 - 20) * 8e-6) + (20 * 3e-6)
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6)
def test_comprehensive_cost_calculation(self):
"""Test cost calculation with all fields combined."""
usage = Usage(
prompt_tokens=100,
completion_tokens=50,
total_tokens=150,
reasoning_tokens=15,
prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=2),
)
# Add custom fields
usage.citation_tokens = 30
prompt_cost, completion_cost = perplexity_cost_per_token(
model="sonar-deep-research", usage=usage
)
# Expected costs (reasoning is a subset of completion_tokens):
# Input: 100 tokens * $2e-6 = $0.0002
# Citation: 30 tokens * $2e-6 = $0.00006
# Total prompt cost = $0.00026
# Output (text): (50 - 15) tokens * $8e-6 = $0.00028
# Reasoning: 15 tokens * $3e-6 = $0.000045
# Search: 2 queries * $0.005 per request = $0.01
# Total completion cost = $0.010325
expected_prompt_cost = (100 * 2e-6) + (30 * 2e-6)
expected_completion_cost = ((50 - 15) * 8e-6) + (15 * 3e-6) + (2 * 0.005)
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6)
def test_zero_values_handling(self):
"""Test that zero or missing values are handled correctly."""
usage = Usage(
prompt_tokens=100,
completion_tokens=50,
total_tokens=150,
prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=0),
)
# These should not raise errors and should not affect cost
usage.citation_tokens = 0
prompt_cost, completion_cost = perplexity_cost_per_token(
model="sonar-deep-research", usage=usage
)
# Should be same as basic calculation
expected_prompt_cost = 100 * 2e-6
expected_completion_cost = 50 * 8e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6)
def test_missing_model_info_fields(self):
"""Test behavior when model info is missing some fields."""
usage = Usage(
@ -237,18 +74,14 @@ class TestPerplexityCostCalculator:
usage.citation_tokens = 25
# Mock get_model_info to return incomplete model info
with patch(
"litellm.llms.perplexity.cost_calculator.get_model_info"
) as mock_get_model_info:
with patch("litellm.llms.perplexity.cost_calculator.get_model_info") as mock_get_model_info:
mock_get_model_info.return_value = {
"input_cost_per_token": 2e-6,
"output_cost_per_token": 8e-6,
# Missing search_queries_cost_per_query
}
prompt_cost, completion_cost = perplexity_cost_per_token(
model="sonar-deep-research", usage=usage
)
prompt_cost, completion_cost = perplexity_cost_per_token(model="sonar-deep-research", usage=usage)
# Should only calculate basic costs when fields are missing
expected_prompt_cost = 100 * 2e-6
@ -257,104 +90,6 @@ class TestPerplexityCostCalculator:
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6)
def test_integration_with_main_cost_calculator(self):
"""Test integration with the main LiteLLM cost calculator."""
usage = Usage(
prompt_tokens=100,
completion_tokens=50,
total_tokens=150,
reasoning_tokens=10,
prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1),
)
usage.citation_tokens = 20
# Test main cost calculator
prompt_cost, completion_cost_val = cost_per_token(
model="sonar-deep-research",
custom_llm_provider="perplexity",
usage_object=usage,
)
# Should match direct call to perplexity cost calculator
expected_prompt, expected_completion = perplexity_cost_per_token(
model="sonar-deep-research", usage=usage
)
assert math.isclose(prompt_cost, expected_prompt, rel_tol=1e-6)
assert math.isclose(completion_cost_val, expected_completion, rel_tol=1e-6)
def test_integration_with_completion_cost_function(self):
"""Test integration with the completion_cost function."""
from litellm import ModelResponse
# Create a mock ModelResponse
usage = Usage(
prompt_tokens=100,
completion_tokens=50,
total_tokens=150,
reasoning_tokens=10,
prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1),
)
usage.citation_tokens = 15
response = ModelResponse()
response.usage = usage
response.model = "sonar-deep-research"
# Test completion_cost function
total_cost = completion_cost(
completion_response=response, custom_llm_provider="perplexity"
)
# Calculate expected total cost (reasoning is a subset of completion_tokens)
expected_prompt_cost = (100 * 2e-6) + (15 * 2e-6) # Input + citation
expected_completion_cost = (
((50 - 10) * 8e-6) + (10 * 3e-6) + (1 * 0.005)
) # Output (text) + reasoning + search
expected_total = expected_prompt_cost + expected_completion_cost
assert math.isclose(total_cost, expected_total, rel_tol=1e-6)
@pytest.mark.parametrize("citation_tokens", [0, 10, 25, 100])
@pytest.mark.parametrize("search_queries", [0, 1, 5, 10])
@pytest.mark.parametrize("reasoning_tokens", [0, 15, 30])
def test_cost_calculation_combinations(
self, citation_tokens, search_queries, reasoning_tokens
):
"""Test various combinations of citation tokens, search queries, and reasoning tokens."""
usage = Usage(
prompt_tokens=100,
completion_tokens=50,
total_tokens=150,
reasoning_tokens=reasoning_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
web_search_requests=search_queries
),
)
usage.citation_tokens = citation_tokens
prompt_cost, completion_cost = perplexity_cost_per_token(
model="sonar-deep-research", usage=usage
)
# Calculate expected costs. `completion_tokens` includes `reasoning_tokens`,
# so non-reasoning portion = 50 - reasoning_tokens.
expected_prompt_cost = (100 * 2e-6) + (citation_tokens * 2e-6)
expected_completion_cost = (
((50 - reasoning_tokens) * 8e-6)
+ (reasoning_tokens * 3e-6)
+ (search_queries * 0.005)
)
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6)
# Ensure costs are non-negative
assert prompt_cost >= 0
assert completion_cost >= 0
def test_uses_perplexity_provided_cost_when_available(self):
"""
Test that when Perplexity provides pre-calculated cost in usage.cost.total_cost,
@ -374,9 +109,7 @@ class TestPerplexityCostCalculator:
"total_cost": 0.008,
}
prompt_cost, completion_cost = perplexity_cost_per_token(
model="sonar-pro", usage=usage
)
prompt_cost, completion_cost = perplexity_cost_per_token(model="sonar-pro", usage=usage)
# When Perplexity provides total_cost, we use it directly
# prompt_cost should be 0, completion_cost should be total_cost
@ -402,9 +135,7 @@ class TestPerplexityCostCalculator:
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
usage.cost = 0.008
prompt_cost, completion_cost = perplexity_cost_per_token(
model="sonar-pro", usage=usage
)
prompt_cost, completion_cost = perplexity_cost_per_token(model="sonar-pro", usage=usage)
assert prompt_cost == 0.0
assert completion_cost == 0.008
@ -417,9 +148,7 @@ class TestPerplexityCostCalculator:
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
# No cost object - should use manual calculation
prompt_cost, completion_cost = perplexity_cost_per_token(
model="sonar-deep-research", usage=usage
)
prompt_cost, completion_cost = perplexity_cost_per_token(model="sonar-deep-research", usage=usage)
# Should calculate manually: 100 * 2e-6 + 50 * 8e-6
expected_prompt = 100 * 2e-6
@ -428,57 +157,6 @@ class TestPerplexityCostCalculator:
assert math.isclose(prompt_cost, expected_prompt, rel_tol=1e-6)
assert math.isclose(completion_cost, expected_completion, rel_tol=1e-6)
def test_reasoning_tokens_not_double_billed(self):
"""
Regression: `completion_tokens` includes `reasoning_tokens` per the
OpenAI/Perplexity usage convention (codified for the central path in PR #18607).
When `output_cost_per_reasoning_token` is configured the manual fallback must
subtract reasoning from completion before applying the output rate so the
reasoning tokens are not billed at BOTH the output rate and the reasoning rate.
Uses the exact usage shape produced by the live response fixture in
`tests/llm_translation/test_perplexity_reasoning.py`.
"""
usage = Usage(
prompt_tokens=9,
completion_tokens=20,
total_tokens=29,
completion_tokens_details=CompletionTokensDetailsWrapper(
reasoning_tokens=15
),
)
prompt_cost, completion_cost = perplexity_cost_per_token(
model="sonar-deep-research", usage=usage
)
# sonar-deep-research rates: input 2e-6, output 8e-6, reasoning 3e-6.
# Non-reasoning portion of the 20 completion tokens = 20 - 15 = 5.
# Pre-fix this asserted 20 * 8e-6 + 15 * 3e-6 = 2.05e-4 (a 2.16x overcharge).
expected_prompt = 9 * 2e-6
expected_completion = (20 - 15) * 8e-6 + 15 * 3e-6
assert math.isclose(prompt_cost, expected_prompt, rel_tol=1e-9)
assert math.isclose(completion_cost, expected_completion, rel_tol=1e-9)
def test_agent_api_fallback_rates_price_a_response_without_metered_cost(self):
"""Perplexity meters cost on the response, but when `usage.cost` is absent the
calculator falls back to the mapped per-token rates. Regression: that fallback
raised "This model isn't mapped yet" for every Agent API third-party model,
because the doubled cost-map key was unreachable from the resolution ladder.
"""
from litellm import ModelResponse
response = ModelResponse()
response.usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
response.model = "perplexity/perplexity/glm-5.2"
total_cost = completion_cost(
completion_response=response, custom_llm_provider="perplexity"
)
assert math.isclose(total_cost, 1000 * 1.4e-06 + 500 * 4.4e-06, rel_tol=1e-9)
OFF_PEAK_MODEL = "sonar-off-peak-test"
OFF_PEAK_WINDOW = "14:00-00:00"
INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc)

184
tests/test_litellm/llms/perplexity/test_perplexity_integration.py
View file

@ -1,7 +1,7 @@
"""
Integration tests for Perplexity cost calculation and transformation.
Tests the end-to-end functionality of Perplexity cost calculation
Tests the end-to-end functionality of Perplexity cost calculation
including integration with the main LiteLLM cost calculator.
"""
@ -12,13 +12,11 @@ import os
import pytest
# Add the project root to Python path
import litellm
from litellm import ModelResponse
from litellm.cost_calculator import completion_cost, cost_per_token
from litellm.cost_calculator import cost_per_token
from litellm.llms.perplexity.chat.transformation import PerplexityChatConfig
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
from litellm.utils import get_model_info
class TestPerplexityIntegration:
@ -57,123 +55,6 @@ class TestPerplexityIntegration:
}
}
def test_end_to_end_cost_calculation_with_transformation(self):
"""Test end-to-end cost calculation with response transformation."""
# Create a Perplexity API response that includes citations and search queries
config = PerplexityChatConfig()
# Create a ModelResponse with basic usage (before transformation)
model_response = ModelResponse()
model_response.model = "sonar-deep-research"
model_response.usage = Usage(
prompt_tokens=100,
completion_tokens=50,
total_tokens=150,
reasoning_tokens=10,
)
# Simulate raw response from Perplexity API
raw_response_dict = {
"choices": [{"message": {"content": "Test response with citations"}}],
"usage": {
"prompt_tokens": 100,
"completion_tokens": 50,
"total_tokens": 150,
"num_search_queries": 2,
},
"citations": [
"This is the first citation with important information about the topic",
"Another citation providing additional context for the response",
],
}
# Apply transformation to extract Perplexity-specific fields
config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict)
# Now calculate the cost with the enhanced usage
total_cost = completion_cost(
completion_response=model_response, custom_llm_provider="perplexity"
)
# Calculate expected cost
citation_chars = sum(
len(citation) for citation in raw_response_dict["citations"]
)
citation_tokens = citation_chars // 4
expected_prompt_cost = (100 * 2e-6) + (citation_tokens * 2e-6)
expected_completion_cost = (
((50 - 10) * 8e-6) + (10 * 3e-6) + (2 * 0.005)
) # Output (text) + reasoning + search
expected_total = expected_prompt_cost + expected_completion_cost
assert math.isclose(total_cost, expected_total, rel_tol=1e-6)
def test_cost_calculation_without_custom_fields(self):
"""Test that cost calculation works normally when custom fields are absent."""
# Create a standard response without Perplexity-specific fields
model_response = ModelResponse()
model_response.model = "sonar-deep-research"
model_response.usage = Usage(
prompt_tokens=100, completion_tokens=50, total_tokens=150
)
# Calculate cost without custom fields
total_cost = completion_cost(
completion_response=model_response, custom_llm_provider="perplexity"
)
# Should only include basic input/output costs
expected_cost = (100 * 2e-6) + (50 * 8e-6)
assert math.isclose(total_cost, expected_cost, rel_tol=1e-6)
def test_main_cost_calculator_integration(self):
"""Test integration with the main LiteLLM cost calculator."""
# Create usage with all Perplexity fields
usage = Usage(
prompt_tokens=200,
completion_tokens=100,
total_tokens=300,
reasoning_tokens=25,
prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=3),
)
usage.citation_tokens = 40
# Test main cost calculator
prompt_cost, completion_cost_val = cost_per_token(
model="sonar-deep-research",
custom_llm_provider="perplexity",
usage_object=usage,
)
expected_prompt_cost = (200 * 2e-6) + (40 * 2e-6)
expected_completion_cost = (
((100 - 25) * 8e-6) + (25 * 3e-6) + (3 * 0.005)
) # Output (text) + reasoning + search
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
assert math.isclose(completion_cost_val, expected_completion_cost, rel_tol=1e-6)
def test_model_info_includes_custom_fields(self):
"""Test that get_model_info returns the custom Perplexity cost fields."""
model_info = get_model_info(
model="sonar-deep-research", custom_llm_provider="perplexity"
)
# Verify custom fields are included
required_fields = [
"citation_cost_per_token",
"search_context_cost_per_query",
"input_cost_per_token",
"output_cost_per_token",
"output_cost_per_reasoning_token",
]
for field in required_fields:
assert field in model_info, f"Missing field: {field}"
assert model_info[field] is not None, f"Null value for field: {field}"
def test_various_citation_sizes(self):
"""Test cost calculation with various citation sizes."""
config = PerplexityChatConfig()
@ -192,9 +73,7 @@ class TestPerplexityIntegration:
for citations, expected_approx_tokens in test_cases:
model_response = ModelResponse()
model_response.model = "sonar-deep-research"
model_response.usage = Usage(
prompt_tokens=100, completion_tokens=50, total_tokens=150
)
model_response.usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
raw_response_dict = {
"usage": {
@ -205,9 +84,7 @@ class TestPerplexityIntegration:
"citations": citations,
}
config._enhance_usage_with_perplexity_fields(
model_response, raw_response_dict
)
config._enhance_usage_with_perplexity_fields(model_response, raw_response_dict)
citation_tokens = getattr(model_response.usage, "citation_tokens", 0)
@ -217,55 +94,6 @@ class TestPerplexityIntegration:
else:
assert abs(citation_tokens - expected_approx_tokens) <= 5
def test_cost_calculation_with_zero_values(self):
"""Test cost calculation handles zero values for custom fields correctly."""
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
# Set custom fields to zero
usage.citation_tokens = 0
usage.prompt_tokens_details = PromptTokensDetailsWrapper(web_search_requests=0)
# Should not add any extra cost
prompt_cost, completion_cost_val = cost_per_token(
model="sonar-deep-research",
custom_llm_provider="perplexity",
usage_object=usage,
)
expected_prompt_cost = 100 * 2e-6
expected_completion_cost = 50 * 8e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6)
assert math.isclose(completion_cost_val, expected_completion_cost, rel_tol=1e-6)
def test_high_volume_cost_calculation(self):
"""Test cost calculation with high token and query counts."""
usage = Usage(
prompt_tokens=50000,
completion_tokens=25000,
total_tokens=75000,
reasoning_tokens=10000,
)
usage.citation_tokens = 5000
usage.prompt_tokens_details = PromptTokensDetailsWrapper(
web_search_requests=100
)
total_cost = completion_cost(
completion_response=ModelResponse(usage=usage, model="sonar-deep-research"),
custom_llm_provider="perplexity",
)
expected_prompt_cost = (50000 * 2e-6) + (5000 * 2e-6)
expected_completion_cost = (
((25000 - 10000) * 8e-6) + (10000 * 3e-6) + (100 * 0.005)
) # $0.65
expected_total = expected_prompt_cost + expected_completion_cost # $0.76
assert math.isclose(total_cost, expected_total, rel_tol=1e-6)
assert total_cost > 0.25
def test_transformation_preserves_existing_usage_fields(self):
"""Test that transformation doesn't overwrite existing standard usage fields."""
config = PerplexityChatConfig()
@ -305,9 +133,7 @@ class TestPerplexityIntegration:
assert hasattr(model_response.usage, "citation_tokens")
assert model_response.usage.prompt_tokens_details.web_search_requests == 3
@pytest.mark.parametrize(
"provider_name", ["perplexity", "PERPLEXITY", "Perplexity"]
)
@pytest.mark.parametrize("provider_name", ["perplexity", "PERPLEXITY", "Perplexity"])
def test_case_insensitive_provider_matching(self, provider_name):
"""Test that cost calculation works with different case variations of provider name."""
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)

29
tests/test_litellm/llms/tencent/test_cost_calculator.py
View file

@ -1,29 +0,0 @@
import pytest
import litellm
from litellm.llms.tencent.cost_calculator import cost_per_token
from litellm.types.utils import Usage
def test_cost_per_token_uses_tencent_model_pricing(local_model_cost_map):
usage = Usage(prompt_tokens=1000, completion_tokens=2000, total_tokens=3000)
prompt_cost, completion_cost = cost_per_token(model="tencent/deepseek-v4-pro", usage=usage)
assert prompt_cost == pytest.approx(1000 * 4.35e-07)
assert completion_cost == pytest.approx(2000 * 8.7e-07)
def test_top_level_dispatcher_routes_tencent_to_wrapper(local_model_cost_map):
from litellm.cost_calculator import cost_per_token as dispatch_cost_per_token
prompt_cost, completion_cost = dispatch_cost_per_token(
model="tencent/deepseek-v4-pro",
prompt_tokens=1000,
completion_tokens=1000,
custom_llm_provider="tencent",
)
assert prompt_cost == pytest.approx(1000 * 4.35e-07)
assert completion_cost == pytest.approx(1000 * 8.7e-07)

236
tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py
View file

@ -1,10 +1,6 @@
import copy
import json
import os
from unittest.mock import MagicMock, patch
import pytest
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import (
VertexAIPartnerModelsAnthropicMessagesConfig,
)
@ -23,12 +19,8 @@ def test_validate_environment_uses_vertex_ai_location():
optional_params = {}
with (
patch.object(
config, "_ensure_access_token", return_value=("token", "test-project")
),
patch.object(
config, "get_complete_vertex_url", return_value="https://mock-url"
) as mock_get_url,
patch.object(config, "_ensure_access_token", return_value=("token", "test-project")),
patch.object(config, "get_complete_vertex_url", return_value="https://mock-url") as mock_get_url,
):
config.validate_anthropic_messages_environment(
headers=headers,
@ -51,17 +43,11 @@ def test_web_search_header_added_for_messages_endpoint():
"vertex_credentials": "{}",
}
# Include web search tool in optional_params
optional_params = {
"tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}]
}
optional_params = {"tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}]}
with (
patch.object(
config, "_ensure_access_token", return_value=("token", "test-project")
),
patch.object(
config, "get_complete_vertex_url", return_value="https://mock-url"
),
patch.object(config, "_ensure_access_token", return_value=("token", "test-project")),
patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"),
):
updated_headers, api_base = config.validate_anthropic_messages_environment(
headers=headers,
@ -73,12 +59,10 @@ def test_web_search_header_added_for_messages_endpoint():
)
# Assert that the anthropic-beta header with web-search is present
assert (
"anthropic-beta" in updated_headers
), "anthropic-beta header should be present"
assert (
updated_headers["anthropic-beta"] == "web-search-2025-03-05"
), f"anthropic-beta should be 'web-search-2025-03-05', got: {updated_headers['anthropic-beta']}"
assert "anthropic-beta" in updated_headers, "anthropic-beta header should be present"
assert updated_headers["anthropic-beta"] == "web-search-2025-03-05", (
f"anthropic-beta should be 'web-search-2025-03-05', got: {updated_headers['anthropic-beta']}"
)
def test_web_search_header_not_added_without_tool():
@ -94,12 +78,8 @@ def test_web_search_header_not_added_without_tool():
optional_params = {}
with (
patch.object(
config, "_ensure_access_token", return_value=("token", "test-project")
),
patch.object(
config, "get_complete_vertex_url", return_value="https://mock-url"
),
patch.object(config, "_ensure_access_token", return_value=("token", "test-project")),
patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"),
):
updated_headers, api_base = config.validate_anthropic_messages_environment(
headers=headers,
@ -111,9 +91,9 @@ def test_web_search_header_not_added_without_tool():
)
# Assert that the anthropic-beta header is NOT present when no web search tool
assert (
"anthropic-beta" not in updated_headers
), "anthropic-beta header should not be present without web search tool"
assert "anthropic-beta" not in updated_headers, (
"anthropic-beta header should not be present without web search tool"
)
def test_compact_context_management_header_added():
@ -129,12 +109,8 @@ def test_compact_context_management_header_added():
optional_params = {"context_management": {"edits": [{"type": "compact_20260112"}]}}
with (
patch.object(
config, "_ensure_access_token", return_value=("token", "test-project")
),
patch.object(
config, "get_complete_vertex_url", return_value="https://mock-url"
),
patch.object(config, "_ensure_access_token", return_value=("token", "test-project")),
patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"),
):
updated_headers, api_base = config.validate_anthropic_messages_environment(
headers=headers,
@ -146,12 +122,10 @@ def test_compact_context_management_header_added():
)
# Assert that the anthropic-beta header with compact-2026-01-12 is present
assert (
"anthropic-beta" in updated_headers
), "anthropic-beta header should be present"
assert (
"compact-2026-01-12" in updated_headers["anthropic-beta"]
), f"anthropic-beta should contain 'compact-2026-01-12', got: {updated_headers['anthropic-beta']}"
assert "anthropic-beta" in updated_headers, "anthropic-beta header should be present"
assert "compact-2026-01-12" in updated_headers["anthropic-beta"], (
f"anthropic-beta should contain 'compact-2026-01-12', got: {updated_headers['anthropic-beta']}"
)
def test_context_management_header_added_for_other_edits():
@ -167,12 +141,8 @@ def test_context_management_header_added_for_other_edits():
optional_params = {"context_management": {"edits": [{"type": "some_other_type"}]}}
with (
patch.object(
config, "_ensure_access_token", return_value=("token", "test-project")
),
patch.object(
config, "get_complete_vertex_url", return_value="https://mock-url"
),
patch.object(config, "_ensure_access_token", return_value=("token", "test-project")),
patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"),
):
updated_headers, api_base = config.validate_anthropic_messages_environment(
headers=headers,
@ -184,12 +154,10 @@ def test_context_management_header_added_for_other_edits():
)
# Assert that the anthropic-beta header with context-management-2025-06-27 is present
assert (
"anthropic-beta" in updated_headers
), "anthropic-beta header should be present"
assert (
"context-management-2025-06-27" in updated_headers["anthropic-beta"]
), f"anthropic-beta should contain 'context-management-2025-06-27', got: {updated_headers['anthropic-beta']}"
assert "anthropic-beta" in updated_headers, "anthropic-beta header should be present"
assert "context-management-2025-06-27" in updated_headers["anthropic-beta"], (
f"anthropic-beta should contain 'context-management-2025-06-27', got: {updated_headers['anthropic-beta']}"
)
def test_both_compact_and_context_management_headers_added():
@ -202,19 +170,11 @@ def test_both_compact_and_context_management_headers_added():
"vertex_credentials": "{}",
}
# Include context_management with both compact and other edit types
optional_params = {
"context_management": {
"edits": [{"type": "compact_20260112"}, {"type": "some_other_type"}]
}
}
optional_params = {"context_management": {"edits": [{"type": "compact_20260112"}, {"type": "some_other_type"}]}}
with (
patch.object(
config, "_ensure_access_token", return_value=("token", "test-project")
),
patch.object(
config, "get_complete_vertex_url", return_value="https://mock-url"
),
patch.object(config, "_ensure_access_token", return_value=("token", "test-project")),
patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"),
):
updated_headers, api_base = config.validate_anthropic_messages_environment(
headers=headers,
@ -226,15 +186,13 @@ def test_both_compact_and_context_management_headers_added():
)
# Assert that both beta headers are present
assert (
"anthropic-beta" in updated_headers
), "anthropic-beta header should be present"
assert (
"compact-2026-01-12" in updated_headers["anthropic-beta"]
), f"anthropic-beta should contain 'compact-2026-01-12', got: {updated_headers['anthropic-beta']}"
assert (
"context-management-2025-06-27" in updated_headers["anthropic-beta"]
), f"anthropic-beta should contain 'context-management-2025-06-27', got: {updated_headers['anthropic-beta']}"
assert "anthropic-beta" in updated_headers, "anthropic-beta header should be present"
assert "compact-2026-01-12" in updated_headers["anthropic-beta"], (
f"anthropic-beta should contain 'compact-2026-01-12', got: {updated_headers['anthropic-beta']}"
)
assert "context-management-2025-06-27" in updated_headers["anthropic-beta"], (
f"anthropic-beta should contain 'context-management-2025-06-27', got: {updated_headers['anthropic-beta']}"
)
def test_validate_environment_always_refreshes_token_ignoring_stale_bearer():
@ -248,12 +206,8 @@ def test_validate_environment_always_refreshes_token_ignoring_stale_bearer():
}
with (
patch.object(
config, "_ensure_access_token", return_value=("fresh-token", "test-project")
) as mock_ensure,
patch.object(
config, "get_complete_vertex_url", return_value="https://mock-vertex-url"
),
patch.object(config, "_ensure_access_token", return_value=("fresh-token", "test-project")) as mock_ensure,
patch.object(config, "get_complete_vertex_url", return_value="https://mock-vertex-url"),
):
updated_headers, api_base = config.validate_anthropic_messages_environment(
headers=headers,
@ -286,9 +240,7 @@ def test_validate_environment_appends_stream_raw_predict_with_custom_api_base():
"get_complete_vertex_url",
wraps=config.get_complete_vertex_url,
) as spy_get_url,
patch.object(
config, "_ensure_access_token", return_value=("token", "test-project")
),
patch.object(config, "_ensure_access_token", return_value=("token", "test-project")),
):
_, api_base = config.validate_anthropic_messages_environment(
headers={},
@ -318,9 +270,7 @@ def test_validate_environment_appends_raw_predict_with_custom_api_base():
"get_complete_vertex_url",
wraps=config.get_complete_vertex_url,
) as spy_get_url,
patch.object(
config, "_ensure_access_token", return_value=("token", "test-project")
),
patch.object(config, "_ensure_access_token", return_value=("token", "test-project")),
):
_, api_base = config.validate_anthropic_messages_environment(
headers={},
@ -447,20 +397,14 @@ def test_validate_environment_does_not_mutate_caller_headers():
caller_headers: dict = {}
with (
patch.object(
config, "_ensure_access_token", return_value=("token", "test-project")
),
patch.object(
config, "get_complete_vertex_url", return_value="https://mock-url"
),
patch.object(config, "_ensure_access_token", return_value=("token", "test-project")),
patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"),
):
config.validate_anthropic_messages_environment(
headers=caller_headers,
model="claude-sonnet-4",
messages=[],
optional_params={
"tools": [{"type": "web_search_20250305", "name": "web_search"}]
},
optional_params={"tools": [{"type": "web_search_20250305", "name": "web_search"}]},
litellm_params={
"vertex_ai_project": "p",
"vertex_ai_location": "us-central1",
@ -468,9 +412,7 @@ def test_validate_environment_does_not_mutate_caller_headers():
api_base=None,
)
assert (
caller_headers == {}
), "validate_anthropic_messages_environment must not mutate the caller's headers dict"
assert caller_headers == {}, "validate_anthropic_messages_environment must not mutate the caller's headers dict"
def test_vertex_claude_completion_does_not_mutate_shared_extra_headers():
@ -483,12 +425,8 @@ def test_vertex_claude_completion_does_not_mutate_shared_extra_headers():
mock_response = MagicMock()
with (
patch.object(
handler, "_ensure_access_token", return_value=("ya29.fresh", "proj")
),
patch.object(
handler, "get_complete_vertex_url", return_value="https://mock-url"
),
patch.object(handler, "_ensure_access_token", return_value=("ya29.fresh", "proj")),
patch.object(handler, "get_complete_vertex_url", return_value="https://mock-url"),
patch(
"litellm.llms.anthropic.chat.AnthropicChatCompletion.completion",
return_value=mock_response,
@ -509,50 +447,7 @@ def test_vertex_claude_completion_does_not_mutate_shared_extra_headers():
litellm_params={},
)
assert (
shared_extra_headers == {}
), "extra_headers must not be mutated by completion()"
def test_messages_thinking_shape_follows_exact_vertex_entry_flag(local_model_cost_map, monkeypatch):
"""The Vertex messages config must probe capabilities under ``vertex_ai`` so an
operator setting ``supports_adaptive_thinking: false`` on the exact
``vertex_ai/claude-opus-4-8`` entry beats the unmodified ``anthropic`` entry.
With the inherited ``"anthropic"`` provider default the flip was ignored and
the transform kept emitting ``thinking.type='adaptive'``."""
import litellm
config = VertexAIPartnerModelsAnthropicMessagesConfig()
def transform():
return config.transform_anthropic_messages_request(
model="claude-opus-4-8",
messages=[{"role": "user", "content": "Hello"}],
anthropic_messages_optional_request_params={
"max_tokens": 4096,
"reasoning_effort": "medium",
},
litellm_params=GenericLiteLLMParams(),
headers={},
)
result = transform()
assert result.get("thinking") == {"type": "adaptive", "display": "summarized"}
assert result.get("output_config") == {"effort": "medium"}
monkeypatch.setitem(
litellm.model_cost["vertex_ai/claude-opus-4-8"], "supports_adaptive_thinking", False
)
litellm.get_model_info.cache_clear()
assert litellm.model_cost["claude-opus-4-8"]["supports_adaptive_thinking"] is True
flipped = transform()
thinking = flipped.get("thinking")
assert isinstance(thinking, dict)
assert thinking.get("type") == "enabled"
assert isinstance(thinking.get("budget_tokens"), int)
assert "output_config" not in flipped
assert shared_extra_headers == {}, "extra_headers must not be mutated by completion()"
def _vertex_transform(model, messages, system=None):
@ -614,9 +509,7 @@ class TestVertexAnthropicMidConversationSystem:
{"role": "assistant", "content": "reading"},
{"role": "user", "content": "continue"},
]
result = _vertex_transform(
"claude-sonnet-4-6", messages, system=[{"type": "text", "text": "Base."}]
)
result = _vertex_transform("claude-sonnet-4-6", messages, system=[{"type": "text", "text": "Base."}])
assert result["messages"] == [
{"role": "user", "content": "read the file"},
{
@ -649,36 +542,3 @@ class TestVertexAnthropicMidConversationSystem:
{"type": "text", "text": "You are terse."},
{"type": "text", "text": "Cite sources."},
]
def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag():
"""Exact cost-map hits win over the ``claude-mid-conversation-system``
fallback rule, so a ``vertex_ai`` Claude 4.8+/5 entry missing the flag would
be treated as unsupported and hoist every reminder, collapsing the prompt
cache. Every mapped vertex_ai entry the rule matches must carry the flag."""
import re
import litellm
cost_map_path = os.path.join(
os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json"
)
with open(cost_map_path) as f:
cost_map = json.load(f)
rules = cost_map["fallback_generalizations"]["rules"]
rule_pattern = next(
(r["pattern"] for r in rules if r["name"] == "claude-mid-conversation-system"),
None,
)
assert rule_pattern is not None, "claude-mid-conversation-system rule not found in fallback_generalizations"
pattern = re.compile(rule_pattern, re.IGNORECASE)
missing = [
key
for key, info in cost_map.items()
if isinstance(info, dict)
and str(info.get("litellm_provider", "")).startswith("vertex_ai")
and "claude" in key
and pattern.search(key)
and info.get("supports_mid_conversation_system") is not True
]
assert missing == []

74
tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py
View file

@ -10,10 +10,7 @@ Source: litellm/llms/xai/responses/transformation.py
from unittest.mock import MagicMock, Mock
import httpx
import pytest
import litellm
from litellm.llms.xai.cost_calculator import cost_per_token
from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.llms.openai import (
@ -305,12 +302,16 @@ class TestXAIResponsesWebSearchBilling:
def _raw_response_json(self, include_web_search: bool) -> dict:
web_search_output = (
[{
"type": "web_search_call",
"id": "ws_1",
"status": "completed",
"action": {"type": "search", "query": "grok"},
}] if include_web_search else []
[
{
"type": "web_search_call",
"id": "ws_1",
"status": "completed",
"action": {"type": "search", "query": "grok"},
}
]
if include_web_search
else []
)
tool_usage = {"server_side_tool_usage_details": self._TOOL_DETAILS} if include_web_search else {}
return {
@ -370,20 +371,6 @@ class TestXAIResponsesWebSearchBilling:
assert bridged.completion_tokens == 20
assert getattr(bridged, "server_side_tool_usage_details") == self._TOOL_DETAILS
def test_completion_cost_bills_web_search_calls(self):
with_search = litellm.completion_cost(
completion_response=self._transform(include_web_search=True),
model="xai/grok-4",
custom_llm_provider="xai",
)
without_search = litellm.completion_cost(
completion_response=self._transform(include_web_search=False),
model="xai/grok-4",
custom_llm_provider="xai",
)
assert with_search - without_search == pytest.approx(2 * 5.0 / 1000.0)
def test_streaming_terminal_event_keeps_schema_and_details(self):
parsed_chunk = {
"type": "response.completed",
@ -436,47 +423,8 @@ class TestXAIResponsesReportedCost:
)
return response.usage
def test_reported_cost_reaches_the_cost_calculator(self):
usage = self._transformed_usage(
{
"input_tokens": 100,
"output_tokens": 200,
"total_tokens": 300,
"cost_in_usd_ticks": 37756000,
}
)
assert usage.cost == 0.0037756
chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
assert cost_per_token(model="grok-4-latest", usage=chat_usage) == (0.0, 0.0037756)
def test_streamed_reported_cost_reaches_the_cost_calculator(self):
event = XAIResponsesAPIConfig().transform_streaming_response(
model="grok-4-latest",
parsed_chunk={
"type": "response.completed",
"sequence_number": 7,
"response": self._response_body(
{
"input_tokens": 100,
"output_tokens": 200,
"total_tokens": 300,
"cost_in_usd_ticks": 37756000,
}
),
},
logging_obj=Mock(),
)
assert isinstance(event, ResponseCompletedEvent)
chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(event.response.usage)
assert cost_per_token(model="grok-4-latest", usage=chat_usage) == (0.0, 0.0037756)
def test_usage_without_a_reported_cost_is_left_alone(self):
usage = self._transformed_usage(
{"input_tokens": 100, "output_tokens": 200, "total_tokens": 300}
)
usage = self._transformed_usage({"input_tokens": 100, "output_tokens": 200, "total_tokens": 300})
assert usage.cost is None

84
tests/test_litellm/llms/xai/test_xai_chat_transformation.py
View file

@ -1,14 +1,10 @@
from unittest.mock import Mock
import httpx
import pytest
import litellm
from litellm.llms.xai.chat.transformation import (
XAIChatCompletionStreamingHandler,
XAIChatConfig,
)
from litellm.llms.xai.cost_calculator import cost_per_token
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
ModelResponse,
@ -26,11 +22,7 @@ class TestXAIReasoningTokenFolding:
total_tokens: int,
reasoning_tokens: int = 0,
) -> ModelResponse:
details = (
CompletionTokensDetailsWrapper(reasoning_tokens=reasoning_tokens)
if reasoning_tokens
else None
)
details = CompletionTokensDetailsWrapper(reasoning_tokens=reasoning_tokens) if reasoning_tokens else None
usage = Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
@ -176,31 +168,11 @@ class TestXAIChatWebSearchBilling:
def test_enhance_noop_without_details(self):
response = self._response_with_usage()
XAIChatConfig()._enhance_usage_with_xai_web_search_fields(
response, {"usage": {"prompt_tokens": 100}}
)
XAIChatConfig()._enhance_usage_with_xai_web_search_fields(response, {"usage": {"prompt_tokens": 100}})
assert response.usage.prompt_tokens_details is None
assert getattr(response.usage, "server_side_tool_usage_details", None) is None
def test_completion_cost_bills_chat_web_search_calls(self):
billed = self._response_with_usage()
XAIChatConfig()._enhance_usage_with_xai_web_search_fields(
billed,
{"usage": {"server_side_tool_usage_details": self._TOOL_DETAILS}},
)
with_search = litellm.completion_cost(
completion_response=billed, model="xai/grok-4", custom_llm_provider="xai"
)
without_search = litellm.completion_cost(
completion_response=self._response_with_usage(),
model="xai/grok-4",
custom_llm_provider="xai",
)
assert with_search - without_search == pytest.approx(3 * 5.0 / 1000.0)
class TestXAIReportedCost:
"""xAI reports what it charged; the transformation moves it to where litellm bills from.
@ -243,23 +215,8 @@ class TestXAIReportedCost:
)
return response.usage
def test_reported_cost_reaches_the_cost_calculator(self):
usage = self._transformed_usage(
{
"prompt_tokens": 100,
"completion_tokens": 200,
"total_tokens": 300,
"cost_in_usd_ticks": 37756000,
}
)
assert usage.cost == 0.0037756
assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0037756)
def test_usage_without_a_reported_cost_is_left_alone(self):
usage = self._transformed_usage(
{"prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300}
)
usage = self._transformed_usage({"prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300})
assert getattr(usage, "cost", None) is None
@ -275,38 +232,3 @@ class TestXAIReportedCost:
)
assert getattr(usage, "cost", None) is None
def test_streamed_reported_cost_survives_chunk_aggregation(self):
"""Streamed spend only matches if the conversion happens on the chunk.
Chunk aggregation rebuilds usage from the fields it models plus ``cost``, so a
chunk still carrying only ``cost_in_usd_ticks`` loses the reported amount.
"""
handler = XAIChatCompletionStreamingHandler(
streaming_response=iter([]), sync_stream=True
)
parsed = handler.chunk_parser(
{
"id": "chatcmpl-xai",
"object": "chat.completion.chunk",
"created": 0,
"model": "grok-4-latest",
"choices": [],
"usage": {
"prompt_tokens": 100,
"completion_tokens": 200,
"total_tokens": 300,
"cost_in_usd_ticks": 37756000,
},
}
)
assert parsed.usage.cost == 0.0037756
assembled = litellm.stream_chunk_builder(chunks=[parsed])
assert assembled.usage.cost == 0.0037756
assert cost_per_token(model="grok-4-latest", usage=assembled.usage) == (
0.0,
0.0037756,
)

373
tests/test_litellm/llms/xai/test_xai_cost_calculator.py
View file

@ -6,16 +6,6 @@ import math
import os
import litellm
from litellm.types.utils import (
Choices,
CompletionTokensDetailsWrapper,
Message,
ModelResponse,
PromptTokensDetailsWrapper,
Usage,
)
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
@ -26,6 +16,13 @@ from litellm.llms.xai.cost_calculator import (
cost_per_token,
cost_per_web_search_request,
)
from litellm.types.utils import (
Choices,
Message,
ModelResponse,
PromptTokensDetailsWrapper,
Usage,
)
class TestXAICostCalculator:
@ -45,241 +42,6 @@ class TestXAICostCalculator:
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
def test_basic_cost_calculation(self):
"""Test basic cost calculation without reasoning tokens."""
usage = Usage(prompt_tokens=12, completion_tokens=125, total_tokens=137)
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
# Expected costs for grok-3-mini:
# Input: 12 tokens * $3e-7 = $0.0000036
# Output: 125 tokens * $5e-7 = $0.0000625
expected_prompt_cost = 12 * 1.25e-6
expected_completion_cost = 125 * 2.5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_reasoning_tokens_cost_calculation(self):
"""Test cost calculation with reasoning tokens from completion_tokens_details."""
usage = Usage(
prompt_tokens=12,
completion_tokens=125,
total_tokens=1086,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=949,
rejected_prediction_tokens=0,
text_tokens=None, # Not set, but doesn't matter for XAI billing
),
)
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
# Expected costs for grok-3-mini:
# Input: 12 tokens * $3e-7 = $0.0000036
# Completion: (125 + 949) tokens * $5e-7 = $0.000537
expected_prompt_cost = 12 * 1.25e-6
expected_completion_cost = (125 + 949) * 2.5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_reasoning_and_text_tokens_cost_calculation(self):
"""Test cost calculation with both reasoning and text tokens."""
usage = Usage(
prompt_tokens=12,
completion_tokens=125,
total_tokens=1086,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=949,
rejected_prediction_tokens=0,
text_tokens=76, # Explicitly set (but ignored in XAI billing)
),
)
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
# Expected costs for grok-3-mini:
# Input: 12 tokens * $3e-7 = $0.0000036
# Completion: (125 + 949) tokens * $5e-7 = $0.000537
# Note: text_tokens field is ignored, only completion_tokens + reasoning_tokens matters
expected_prompt_cost = 12 * 1.25e-6
expected_completion_cost = (125 + 949) * 2.5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_grok_4_cost_calculation(self):
"""Test cost calculation for grok-4 model."""
usage = Usage(
prompt_tokens=10,
completion_tokens=200,
total_tokens=360,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=150,
rejected_prediction_tokens=0,
text_tokens=50, # Ignored in XAI billing
),
)
prompt_cost, completion_cost = cost_per_token(model="grok-4", usage=usage)
# grok-4 was retired on 2026-05-15 and now redirects to grok-4.3, so it bills
# at grok-4.3's rates:
# Input: 10 tokens * $1.25e-6
# Completion: (200 + 150) tokens * $2.5e-6
expected_prompt_cost = 10 * 1.25e-6
expected_completion_cost = (200 + 150) * 2.5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_grok_3_fast_beta_cost_calculation(self):
"""Test cost calculation for grok-3-fast-beta model."""
usage = Usage(
prompt_tokens=20,
completion_tokens=300,
total_tokens=520,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=200,
rejected_prediction_tokens=0,
text_tokens=100, # Ignored in XAI billing
),
)
prompt_cost, completion_cost = cost_per_token(
model="grok-3-fast-beta", usage=usage
)
# Expected costs for grok-3-fast-beta:
# Input: 20 tokens * $5e-6 = $0.0001
# Completion: (300 + 200) tokens * $2.5e-5 = $0.0125
expected_prompt_cost = 20 * 1.25e-6
expected_completion_cost = (300 + 200) * 2.5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_edge_case_large_reasoning_tokens(self):
"""Test cost calculation when reasoning_tokens is larger than completion_tokens."""
usage = Usage(
prompt_tokens=12,
completion_tokens=50, # Less than reasoning_tokens
total_tokens=162,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=100, # More than completion_tokens
rejected_prediction_tokens=0,
text_tokens=None,
),
)
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
# Expected costs:
# Input: 12 tokens * $3e-7 = $0.0000036
# Completion: (50 + 100) tokens * $5e-7 = $0.000075
expected_prompt_cost = 12 * 1.25e-6
expected_completion_cost = (50 + 100) * 2.5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_tiered_pricing_above_200k_tokens(self):
usage = Usage(
prompt_tokens=250000,
completion_tokens=100000,
total_tokens=400000,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=50000,
rejected_prediction_tokens=0,
text_tokens=None,
),
)
prompt_cost, completion_cost = cost_per_token(model="xai/grok-4.3", usage=usage)
expected_prompt_cost = 250000 * 2.5e-6
expected_completion_cost = (100000 + 50000) * 5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_tiered_pricing_below_200k_tokens(self):
usage = Usage(
prompt_tokens=100000,
completion_tokens=50000,
total_tokens=160000,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=10000,
rejected_prediction_tokens=0,
text_tokens=None,
),
)
prompt_cost, completion_cost = cost_per_token(model="xai/grok-4.3", usage=usage)
expected_prompt_cost = 100000 * 1.25e-6
expected_completion_cost = (50000 + 10000) * 2.5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_tiered_pricing_grok_4_latest(self):
"""Test tiered pricing for grok-4-latest model."""
usage = Usage(
prompt_tokens=250000, # Above the 200k threshold
completion_tokens=100000,
total_tokens=400000,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=50000,
rejected_prediction_tokens=0,
text_tokens=None,
),
)
prompt_cost, completion_cost = cost_per_token(
model="xai/grok-4-latest", usage=usage
)
# grok-4-latest redirects to grok-4.3, which tiers at 200k rather than 128k:
# Input: 250000 tokens * $2.5e-6 (ALL tokens at tiered rate since input > 200k)
# Completion: (100000 + 50000) tokens * $5e-6 (tiered rate since input > 200k)
expected_prompt_cost = 250000 * 2.5e-6
expected_completion_cost = (100000 + 50000) * 5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_tiered_pricing_output_tokens_below_200k(self):
usage = Usage(
prompt_tokens=250000,
completion_tokens=50000,
total_tokens=310000,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=10000,
rejected_prediction_tokens=0,
text_tokens=None,
),
)
prompt_cost, completion_cost = cost_per_token(model="xai/grok-4.3", usage=usage)
expected_prompt_cost = 250000 * 2.5e-6
expected_completion_cost = (50000 + 10000) * 5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_tiered_pricing_model_without_tiered_pricing(self):
litellm.model_cost["xai/flat-rate-fixture"] = {
"input_cost_per_token": 3e-7,
@ -294,29 +56,6 @@ class TestXAICostCalculator:
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_already_normalised_usage_does_not_double_count_reasoning(self):
"""Cost calc must not double-bill when Usage is already OpenAI-normalised."""
usage = Usage(
prompt_tokens=12,
completion_tokens=200,
total_tokens=212,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=0,
audio_tokens=0,
reasoning_tokens=100,
rejected_prediction_tokens=0,
text_tokens=None,
),
)
prompt_cost, completion_cost = cost_per_token(model="grok-3-mini", usage=usage)
expected_prompt_cost = 12 * 1.25e-6
expected_completion_cost = 200 * 2.5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_web_search_cost_via_server_side_tool_usage_details(self):
"""usage.server_side_tool_usage_details.web_search_calls at default $5/1k."""
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
@ -344,9 +83,7 @@ class TestXAICostCalculator:
"search_context_size_medium": 0.01,
}
}
web_search_cost = cost_per_web_search_request(
usage=usage, model_info=model_info
)
web_search_cost = cost_per_web_search_request(usage=usage, model_info=model_info)
assert math.isclose(web_search_cost, 0.02, rel_tol=1e-10)
def test_web_search_cost_zero_without_details(self):
@ -355,9 +92,7 @@ class TestXAICostCalculator:
def test_apply_details_sets_web_search_requests_for_cost_gate(self):
usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)
apply_server_side_tool_usage_details_to_usage(
usage, {"web_search_calls": 2, "x_search_calls": 0}
)
apply_server_side_tool_usage_details_to_usage(usage, {"web_search_calls": 2, "x_search_calls": 0})
assert usage.prompt_tokens_details is not None
assert usage.prompt_tokens_details.web_search_requests == 2
assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call(
@ -413,9 +148,7 @@ class TestXAICostCalculator:
assert get_cost_for_web_search_request("xai", usage, {}) > 0.0
reported = Usage(
prompt_tokens=100, completion_tokens=50, total_tokens=150, cost=0.0037756
)
reported = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150, cost=0.0037756)
setattr(reported, "server_side_tool_usage_details", {"web_search_calls": 3})
assert get_cost_for_web_search_request("xai", reported, {}) == 0.0
@ -503,82 +236,6 @@ class TestXAICostCalculator:
assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0)
def test_grok_4_20_beta_reasoning_cost_calculation(self):
"""Test cost calculation for grok-4.20-beta-0309-reasoning model."""
usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300)
prompt_cost, completion_cost = cost_per_token(
model="grok-4.20-beta-0309-reasoning", usage=usage
)
# Input: 100 tokens * $1.25e-6 = $0.000125
# Output: 200 tokens * $2.5e-6 = $0.0005
expected_prompt_cost = 100 * 1.25e-6
expected_completion_cost = 200 * 2.5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_grok_4_20_beta_non_reasoning_cost_calculation(self):
"""Test cost calculation for grok-4.20-beta-0309-non-reasoning model."""
usage = Usage(prompt_tokens=50, completion_tokens=100, total_tokens=150)
prompt_cost, completion_cost = cost_per_token(
model="grok-4.20-beta-0309-non-reasoning", usage=usage
)
# Input: 50 tokens * $1.25e-6 = $0.0000625
# Output: 100 tokens * $2.5e-6 = $0.00025
expected_prompt_cost = 50 * 1.25e-6
expected_completion_cost = 100 * 2.5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_grok_4_20_at_exactly_200k_prompt_tokens_uses_higher_tier(self):
"""xAI bills the >=200k tier once the prompt reaches 200k, so the boundary is inclusive."""
usage = Usage(prompt_tokens=200_000, completion_tokens=1_000, total_tokens=201_000)
prompt_cost, completion_cost = cost_per_token(
model="grok-4.20-0309-reasoning", usage=usage
)
expected_prompt_cost = 200_000 * 2.5e-6
expected_completion_cost = 1_000 * 5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_grok_4_20_just_below_200k_prompt_tokens_uses_base_tier(self):
"""One token under the boundary still bills at the base rates."""
usage = Usage(prompt_tokens=199_999, completion_tokens=1_000, total_tokens=200_999)
prompt_cost, completion_cost = cost_per_token(
model="grok-4.20-0309-reasoning", usage=usage
)
expected_prompt_cost = 199_999 * 1.25e-6
expected_completion_cost = 1_000 * 2.5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_grok_4_20_multi_agent_cost_calculation(self):
"""Test cost calculation for grok-4.20-multi-agent-beta-0309 model."""
usage = Usage(prompt_tokens=200, completion_tokens=300, total_tokens=500)
prompt_cost, completion_cost = cost_per_token(
model="grok-4.20-multi-agent-beta-0309", usage=usage
)
# Input: 200 tokens * $1.25e-6 = $0.00025
# Output: 300 tokens * $2.5e-6 = $0.00075
expected_prompt_cost = 200 * 1.25e-6
expected_completion_cost = 300 * 2.5e-6
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
def test_custom_pricing_beats_the_reported_cost(self):
response = ModelResponse(
id="chatcmpl-xai",
@ -635,10 +292,7 @@ class TestXAIWebSearchCostHelpers:
details = {"web_search_calls": 0, "x_search_calls": 3}
apply_server_side_tool_usage_details_to_usage(usage, details)
assert getattr(usage, "server_side_tool_usage_details") == details
assert (
usage.prompt_tokens_details is None
or usage.prompt_tokens_details.web_search_requests is None
)
assert usage.prompt_tokens_details is None or usage.prompt_tokens_details.web_search_requests is None
def test_apply_details_skips_mirror_when_web_search_calls_invalid(self):
usage = Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2)
@ -660,10 +314,7 @@ class TestXAIWebSearchCostHelpers:
assert usage.prompt_tokens_details.web_search_requests == 4
def test_web_search_cost_per_call_default_when_model_info_empty(self):
assert (
_web_search_cost_per_call_from_model_info({})
== _DEFAULT_WEB_SEARCH_COST_PER_CALL
)
assert _web_search_cost_per_call_from_model_info({}) == _DEFAULT_WEB_SEARCH_COST_PER_CALL
def test_web_search_cost_per_call_prefers_medium_over_low(self):
model_info = {

75
tests/test_litellm/llms/xai/test_xai_model_registry.py
View file

@ -1,75 +0,0 @@
"""
Registry regression tests for xAI entries in the model cost map.
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).parents[4]
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
# Retired by xAI and no longer served: requests to these slugs 404 rather than
# redirecting, and they are absent from https://docs.x.ai/docs/models
RETIRED_MODELS = (
"xai/grok-2",
"xai/grok-2-1212",
"xai/grok-2-latest",
"xai/grok-2-vision",
"xai/grok-2-vision-1212",
"xai/grok-2-vision-latest",
"xai/grok-beta",
"xai/grok-vision-beta",
)
# https://docs.x.ai/developers/model-capabilities/text/multi-agent
# "The multi-agent model does not work with the OpenAI Chat Completions API."
RESPONSES_ONLY_MODELS = (
"xai/grok-4.20-multi-agent-0309",
"xai/grok-4.20-multi-agent-beta-0309",
)
MAP_PATHS = (PRICES_PATH, BACKUP_PRICES_PATH)
@pytest.fixture(scope="module", params=[p.name for p in MAP_PATHS])
def cost_map(request: pytest.FixtureRequest) -> dict:
path = next(p for p in MAP_PATHS if p.name == request.param)
return json.loads(path.read_text(encoding="utf-8"))
@pytest.mark.parametrize("model", RETIRED_MODELS)
def test_retired_xai_models_are_not_advertised(cost_map: dict, model: str):
assert model not in cost_map
@pytest.mark.parametrize("model", RESPONSES_ONLY_MODELS)
def test_multi_agent_models_are_responses_only(cost_map: dict, model: str):
entry = cost_map[model]
assert entry["supported_endpoints"] == ["/v1/responses"]
assert entry["mode"] == "responses"
assert "/v1/chat/completions" not in entry["supported_endpoints"]
def test_surviving_xai_chat_models_still_serve_chat_completions(cost_map: dict):
"""Guard against the removal above over-reaching into live models."""
chat_models = [
key
for key, value in cost_map.items()
if isinstance(value, dict) and value.get("litellm_provider") == "xai" and value.get("mode") == "chat"
]
assert "xai/grok-4.3" in chat_models
assert "xai/grok-4.6" in chat_models
assert not any(key.startswith("xai/grok-2") for key in chat_models)
def test_both_cost_maps_agree_on_xai_entries():
prices = json.loads(PRICES_PATH.read_text(encoding="utf-8"))
backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8"))
xai_keys = {k for k, v in prices.items() if isinstance(v, dict) and v.get("litellm_provider") == "xai"}
assert xai_keys
assert {k: prices[k] for k in xai_keys} == {k: backup[k] for k in xai_keys}

20
tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py
View file

@ -98,23 +98,3 @@ def test_redirected_slug_carries_the_target_tier_rates(cost_map: dict, slug: str
for field in TIER_COST_FIELDS:
assert entry[field] == target[field], field
assert {k for k in entry if "_above_" in k} == {k for k in target if "_above_" in k}
def test_both_cost_maps_agree_on_the_redirected_slugs():
prices = json.loads(PRICES_PATH.read_text(encoding="utf-8"))
backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8"))
for slug in (*REDIRECTED_SLUGS, *CODE_SLUGS, REDIRECT_TARGET, CODE_REDIRECT_TARGET):
assert prices[slug] == backup[slug], slug
def test_every_retired_chat_slug_is_covered(cost_map: dict):
"""The lists above must stay in step with what the registry marks retired."""
marked = {
key
for key, entry in cost_map.items()
if isinstance(entry, dict)
and entry.get("litellm_provider") == "xai"
and "deprecation_date" in entry
and entry.get("mode") == "chat"
}
assert marked == {*REDIRECTED_SLUGS, *CODE_SLUGS}

144
tests/test_litellm/proxy/spend_tracking/test_savings.py
View file

@ -235,33 +235,6 @@ def test_negative_ttl_counts_do_not_become_cache_write_credits() -> None:
assert results[0].prompt_caching < 0
def test_unpublished_one_hour_price_uses_the_ordinary_write_price() -> None:
model: Final = "claude-4-opus-20250514"
pricing: Final = litellm.get_model_info(model=model, custom_llm_provider="anthropic")
assert pricing.get("cache_creation_input_token_cost_above_1hr") is None
assert pricing["cache_creation_input_token_cost"] > pricing["input_cost_per_token"]
results: Final = tuple(
compute_savings_spend(
model=model,
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object={
"prompt_tokens": 6000,
"completion_tokens": 100,
"prompt_tokens_details": {
"text_tokens": 1000,
"cache_creation_tokens": 5000,
"cache_creation_token_details": ttl,
},
},
)
for ttl in (None, {"ephemeral_1h_input_tokens": 5000})
)
assert results[0] == results[1]
assert results[0].prompt_caching < 0
def test_prompt_caching_savings_nets_out_the_cache_write_premium():
"""A cache-writing request is only credited the read discount minus the write premium."""
input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5")
@ -354,108 +327,6 @@ def test_openai_style_cache_write_tokens_are_netted_out():
)
def test_model_without_a_cache_write_price_takes_no_premium():
"""An absent write price must mean zero premium, never a bonus.
``_get_cost_per_unit`` in the cost calculator defaults a missing price to 0.0. Were
that default copied here the premium would be ``0 - input_cost``, and a model with no
write pricing would report cache writes as free money. This is the common case: most
of the pricing map publishes a cache-read price and no cache-write price.
"""
model = "amazon.nova-2-lite-v1:0"
info = litellm.get_model_info(model=model)
input_cost = info["input_cost_per_token"]
cache_read_cost = info["cache_read_input_token_cost"]
assert info.get("cache_creation_input_token_cost") is None, (
"fixture drifted: this test needs a model that publishes no cache-write price"
)
result = compute_savings_spend(
model=model,
custom_llm_provider=None,
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=5000, written=5000),
)
assert result.prompt_caching == pytest.approx(5000 * (input_cost - cache_read_cost))
assert result.prompt_caching > 0
def test_zero_cache_write_price_is_read_as_unpublished():
"""A ``0.0`` write price means "no separate price", not "writes are free".
``deepseek-chat`` carries an explicit zero in the pricing map. Taken literally the
premium would be ``0 - input_cost``, paying out a saving of ``writes * input_cost``
on traffic that cached nothing. No provider gives cache writes away, so a falsy
price falls open to the input cost like an absent one does.
"""
info = litellm.get_model_info(model="deepseek-chat", custom_llm_provider="deepseek")
assert info.get("cache_creation_input_token_cost") == 0.0, (
"fixture drifted: this test exists because deepseek-chat publishes a literal 0.0 write price"
)
result = compute_savings_spend(
model="deepseek-chat",
custom_llm_provider="deepseek",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=0, written=10000),
)
assert result.prompt_caching == pytest.approx(0.0)
def test_zero_cache_read_price_stays_literal():
"""The read leg must NOT copy the write leg's falsy fall-open.
The two zeros mean opposite things. A free cache *write* is unpublished pricing, so
it falls open to input. A free cache *read* is real and is the largest discount
available -- 15 models charge for input and serve reads for nothing. Falling that
open to the input cost would zero out their savings entirely.
"""
model = "gemini-robotics-er-1.5-preview"
info = litellm.get_model_info(model=model)
input_cost = info["input_cost_per_token"]
assert info.get("cache_read_input_token_cost") == 0.0 and input_cost > 0, (
"fixture drifted: this test needs a model with paid input and free cache reads"
)
result = compute_savings_spend(
model=model,
custom_llm_provider=None,
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=10000, written=0),
)
# free reads => the whole input rate is saved, not zero
assert result.prompt_caching == pytest.approx(10000 * input_cost)
def test_sub_input_cache_write_price_is_an_extra_saving():
"""A few models price writes below input; there the premium is a real credit.
Clamping the premium at zero would silently undercount these, so the subtraction
stays signed. ``azure/eu/gpt-4o-2024-11-20`` ships a write price at ~0.5x input.
"""
model = "azure/eu/gpt-4o-2024-11-20"
info = litellm.get_model_info(model=model)
input_cost = info["input_cost_per_token"]
cheap_write = info["cache_creation_input_token_cost"]
assert 0 < cheap_write < input_cost, "fixture drifted: this test needs a model pricing cache writes below input"
# no published read price, so the read leg mirrors input and contributes nothing;
# the whole result is the negative premium, i.e. a credit.
assert info.get("cache_read_input_token_cost") is None
result = compute_savings_spend(
model=model,
custom_llm_provider=None,
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=1000, written=4000),
)
assert result.prompt_caching == pytest.approx(4000 * (input_cost - cheap_write))
assert result.prompt_caching > 0
def test_negative_cache_write_count_clamps_to_zero():
"""A malformed negative write count must not be read as a saving."""
input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5")
@ -728,21 +599,6 @@ def test_malformed_usage_object_does_not_fail_the_spend_write():
assert result.compression > 0
def test_model_without_cache_read_pricing_yields_no_caching_savings():
"""A model with no discounted cache-read rate cannot have saved anything by
reading from cache, so the driver must report zero rather than the full input rate."""
model = "azure/gpt-3.5-turbo"
assert litellm.get_model_info(model=model).get("cache_read_input_token_cost") is None
result = compute_savings_spend(
model=model,
custom_llm_provider="azure",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": 5000},
)
assert result.prompt_caching == 0.0
def test_the_same_deployment_spelled_two_ways_is_not_a_switch():
"""The spend log records a normalized model name while the baseline arrives as the
operator wrote it in config. Comparing the raw strings makes a request that never

39
tests/test_litellm/test_azure_ai_grok_4_3_model_metadata.py
View file

@ -1,39 +0,0 @@
import json
from pathlib import Path
import pytest
import litellm
from litellm import get_model_info
AZURE_AI_GROK_4_3_MODEL = "azure_ai/grok-4.3"
AZURE_AI_GROK_4_3_SOURCE = "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-grok-4-3-on-microsoft-foundry-latest-generation-agentic-capabilities/4517096"
def _load_model_cost(path: Path) -> dict:
with open(path) as f:
return json.load(f)
@pytest.fixture(autouse=True)
def reload_model_costs():
original_model_cost = litellm.model_cost
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
litellm.model_cost = _load_model_cost(json_path)
get_model_info.cache_clear()
yield
litellm.model_cost = original_model_cost
get_model_info.cache_clear()
def test_azure_ai_grok_4_3_backup_matches_main():
repo_root = Path(__file__).parents[2]
main_path = repo_root / "model_prices_and_context_window.json"
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
main_cost = _load_model_cost(main_path)
backup_cost = _load_model_cost(backup_path)
assert backup_cost.get(AZURE_AI_GROK_4_3_MODEL) == main_cost.get(
AZURE_AI_GROK_4_3_MODEL
)

50
tests/test_litellm/test_azure_ai_grok_4_6_model_metadata.py
View file

@ -1,50 +0,0 @@
from pathlib import Path
from typing import Final
import pytest
from pydantic import TypeAdapter
from litellm import cost_per_token, get_model_info
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
REPO_ROOT: Final = Path(__file__).parents[2]
MODEL: Final = "azure_ai/grok-4.6"
COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
def _cost_map_entry(path: Path) -> dict[str, object]:
return COST_MAP_ADAPTER.validate_json(path.read_bytes())[MODEL]
@pytest.mark.usefixtures("local_model_cost_map")
def test_azure_ai_grok_4_6_is_priced_and_routed() -> None:
routed_model, provider, _, _ = get_llm_provider(model=MODEL)
assert (routed_model, provider) == ("grok-4.6", "azure_ai")
info = get_model_info(model=routed_model, custom_llm_provider=provider)
assert info["litellm_provider"] == "azure_ai"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] == 2e-06
assert info["output_cost_per_token"] == 6e-06
assert info["cache_read_input_token_cost"] == 5e-07
assert info["max_input_tokens"] == 200000
assert info["max_output_tokens"] == 128000
assert info["max_tokens"] == 128000
assert info["supports_function_calling"] is True
assert info["supports_prompt_caching"] is True
assert info["supports_reasoning"] is True
assert info["supports_response_schema"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
assert info["supports_web_search"] is True
prompt_cost, completion_cost = cost_per_token(model=MODEL, prompt_tokens=1_000_000, completion_tokens=1_000_000)
assert prompt_cost == pytest.approx(2.0)
assert completion_cost == pytest.approx(6.0)
def test_azure_ai_grok_4_6_entry_source_and_backup_match() -> None:
main_entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json")
backup_entry = _cost_map_entry(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json")
assert backup_entry == main_entry

93
tests/test_litellm/test_baseten_glm_5_3_model_metadata.py
View file

@ -1,93 +0,0 @@
import json
from pathlib import Path
import pytest
import litellm
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
from litellm.utils import supports_function_calling, supports_prompt_caching
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
MODEL = "baseten/zai-org/GLM-5.3"
INPUT_COST = 1.4e-06
CACHED_INPUT_COST = 1.4e-07
OUTPUT_COST = 4.4e-06
def _load(path):
with open(path) as f:
return json.load(f)
@pytest.fixture
def local_model_cost_map(monkeypatch):
"""Force get_model_info to resolve against the in-repo cost map instead of the
remote one fetched at import time, which still carries the pre-merge registry."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
def test_baseten_glm_5_3_capabilities_are_visible_to_callers(local_model_cost_map):
"""The entry advertises prompt caching and tool calling, so the helpers every
caller checks before sending a request must say so too."""
assert supports_prompt_caching(model=MODEL) is True
assert supports_function_calling(model=MODEL) is True
info = litellm.get_model_info(model="zai-org/GLM-5.3", custom_llm_provider="baseten")
assert info["max_input_tokens"] == 1048576
assert info["max_output_tokens"] == 262144
def test_cached_prompt_tokens_bill_at_the_cached_rate(local_model_cost_map):
"""A cache hit reports its reused tokens under prompt_tokens_details, and those
tokens cost a tenth of the input rate, not the full rate and not nothing."""
usage = Usage(
prompt_tokens=21010,
completion_tokens=100,
total_tokens=21110,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=20992),
)
prompt_cost, completion_cost = litellm.cost_per_token(
model=MODEL, usage_object=usage, custom_llm_provider="baseten"
)
assert prompt_cost == pytest.approx(18 * INPUT_COST + 20992 * CACHED_INPUT_COST)
assert completion_cost == pytest.approx(100 * OUTPUT_COST)
def test_backup_matches_main():
"""Ensure the bundled (backup) cost map stays in sync with the canonical file.
Both keys are asserted present first: comparing two ``.get`` results alone passes
just as happily when neither file has the entry at all, which is the exact state
this test exists to catch.
"""
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert MODEL in main_cost, f"{MODEL} missing from model_prices_and_context_window.json"
assert MODEL in backup_cost, f"{MODEL} missing from model_prices_and_context_window_backup.json"
assert backup_cost[MODEL] == main_cost[MODEL], f"{MODEL} differs between main and backup model cost maps"
def test_entry_advertises_only_what_the_baseten_path_accepts(local_model_cost_map):
"""The Baseten path rejects unsupported request parameters."""
supported = litellm.get_supported_openai_params(model="zai-org/GLM-5.3", custom_llm_provider="baseten")
assert supported is not None
with pytest.raises(litellm.UnsupportedParamsError):
litellm.utils.get_optional_params(
model="zai-org/GLM-5.3",
custom_llm_provider="baseten",
parallel_tool_calls=True,
reasoning_effort="high",
drop_params=False,
)

85
tests/test_litellm/test_bedrock_marengo_embed_3_model_metadata.py
View file

@ -1,85 +0,0 @@
import json
from pathlib import Path
import pytest
import litellm
from litellm.constants import bedrock_embedding_models
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
BASE_MODEL = "twelvelabs.marengo-embed-3-0-v1:0"
PROFILE_MODELS = ("us.twelvelabs.marengo-embed-3-0-v1:0", "eu.twelvelabs.marengo-embed-3-0-v1:0")
ALL_MODELS = (BASE_MODEL, *PROFILE_MODELS)
MARENGO_2_7_MODELS = (
"twelvelabs.marengo-embed-2-7-v1:0",
"us.twelvelabs.marengo-embed-2-7-v1:0",
"eu.twelvelabs.marengo-embed-2-7-v1:0",
)
PER_REQUEST_MODELS = (*ALL_MODELS, *MARENGO_2_7_MODELS)
TEXT_REQUEST_COST = 7e-05
IMAGE_REQUEST_COST = 0.0001
VIDEO_COST_PER_SECOND = 0.0007
AUDIO_COST_PER_SECOND = 0.00014
def _load(path):
with open(path) as f:
return json.load(f)
@pytest.mark.parametrize("model", ALL_MODELS)
def test_marengo_embed_3_is_visible_to_callers(model, local_model_cost_map):
info = litellm.get_model_info(model=model, custom_llm_provider="bedrock")
assert info["mode"] == "embedding"
assert info["output_vector_size"] == 512
assert info["max_input_tokens"] == 500
@pytest.mark.parametrize("model", PER_REQUEST_MODELS)
@pytest.mark.parametrize(
"details,expected_cost",
[
(PromptTokensDetailsWrapper(query_count=1), TEXT_REQUEST_COST),
(PromptTokensDetailsWrapper(image_count=1), IMAGE_REQUEST_COST),
(PromptTokensDetailsWrapper(query_count=1, image_count=1), TEXT_REQUEST_COST + IMAGE_REQUEST_COST),
(PromptTokensDetailsWrapper(query_count=1, image_count=2), TEXT_REQUEST_COST + 2 * IMAGE_REQUEST_COST),
(PromptTokensDetailsWrapper(video_length_seconds=10), 10 * VIDEO_COST_PER_SECOND),
(PromptTokensDetailsWrapper(audio_length_seconds=10), 10 * AUDIO_COST_PER_SECOND),
],
)
def test_marengo_requests_are_billed_per_request(model, details, expected_cost, local_model_cost_map):
usage = Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0, prompt_tokens_details=details)
prompt_cost, completion_cost = litellm.cost_per_token(
model=model, usage_object=usage, custom_llm_provider="bedrock"
)
assert prompt_cost == pytest.approx(expected_cost)
assert completion_cost == 0.0
@pytest.mark.parametrize("model", PER_REQUEST_MODELS)
def test_marengo_token_counts_bill_nothing(model, local_model_cost_map):
usage = Usage(prompt_tokens=128, completion_tokens=0, total_tokens=128)
prompt_cost, completion_cost = litellm.cost_per_token(
model=model, usage_object=usage, custom_llm_provider="bedrock"
)
assert prompt_cost == 0.0
assert completion_cost == 0.0
def test_marengo_embed_3_is_a_known_bedrock_embedding_model():
assert BASE_MODEL in bedrock_embedding_models
@pytest.mark.parametrize("model", PER_REQUEST_MODELS)
def test_backup_matches_main(model):
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert model in main_cost, f"{model} missing from model_prices_and_context_window.json"
assert model in backup_cost, f"{model} missing from model_prices_and_context_window_backup.json"
assert backup_cost[model] == main_cost[model], f"{model} differs between main and backup model cost maps"

12
tests/test_litellm/test_bedrock_usgov_pricing.py
View file

@ -31,18 +31,6 @@ def model_data():
return json.load(f)
def test_usgov_east_haiku_profile_mirrors_in_region_row(model_data):
"""us-gov-east-1 serves claude-3-haiku through the us-gov. inference profile
only, so the profile row must bill exactly like the in-region gov row.
"""
profile = model_data["us-gov.anthropic.claude-3-haiku-20240307-v1:0"]
in_region = model_data["bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0"]
assert profile["litellm_provider"] == "bedrock_converse"
assert {k: v for k, v in profile.items() if k != "litellm_provider"} == {
k: v for k, v in in_region.items() if k != "litellm_provider"
}
GOV_ROW_SOURCES = {
"us-gov.anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1",
"bedrock/us-gov-west-1/anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1",

160
tests/test_litellm/test_claude_fable_5_config.py
View file

@ -12,10 +12,7 @@ shape, which Fable 5 rejects with a 400.
import json
import os
import pytest
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
@ -26,99 +23,10 @@ def _load_root_cost_map() -> dict:
return json.load(f)
def test_fable_5_geo_multiplier_without_fast_mode():
"""First-party ``inference_geo='us'`` carries the 1.1x premium, but unlike
the Opus line there is no fast-mode variant for Fable 5; a ``fast`` key
here would silently misprice ``speed='fast'`` requests."""
model_data = _load_root_cost_map()
entry = model_data["claude-fable-5"]["provider_specific_entry"]
assert entry == {"us": 1.1}
def test_fable_5_present_in_bundled_backup():
"""The bundled backup is the runtime fallback (and what tests load with
``LITELLM_LOCAL_MODEL_COST_MAP=True``) — it must carry the same entries as
the root cost map, otherwise the model resolves on one path but not the
other."""
backup = GetModelCostMap.load_local_model_cost_map()
root = _load_root_cost_map()
for model_name in (
"claude-fable-5",
"anthropic.claude-fable-5",
"global.anthropic.claude-fable-5",
"us.anthropic.claude-fable-5",
"eu.anthropic.claude-fable-5",
"vertex_ai/claude-fable-5",
"vertex_ai/claude-fable-5@default",
"azure_ai/claude-fable-5",
):
assert model_name in backup, f"Missing from backup cost map: {model_name}"
assert backup[model_name] == root[model_name], model_name
def test_fable_5_registered_for_bedrock_converse():
assert "anthropic.claude-fable-5" in BEDROCK_CONVERSE_MODELS
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_fable_5_all_variants_carry_adaptive_thinking_flag(cost_map):
"""Every Fable 5 entry must advertise ``supports_adaptive_thinking``.
Adaptive-thinking detection is cost-map driven, so a single variant missing
the flag silently sends the legacy ``thinking.type='enabled'`` shape and the
provider 400s (issue #29188 for the Opus 4.8 equivalent). Fable 5 is even
stricter than Opus 4.8: an explicit ``thinking.type='disabled'`` also 400s,
so adaptive is the only valid thinking shape LiteLLM can emit for it."""
variants = [k for k in cost_map if "claude-fable-5" in k]
assert variants, "no claude-fable-5 entries found in cost map"
missing = [
k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True
]
assert not missing, f"missing supports_adaptive_thinking: {missing}"
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_fable_5_all_variants_carry_thinking_always_on_flag(cost_map):
"""Every Fable 5 entry must advertise ``thinking_always_on``.
The flag drives the Anthropic transformations to omit an explicit
``thinking.type='disabled'``, which Fable 5 rejects with a 400; a variant
missing the flag forwards the param verbatim and the provider 400s."""
variants = [k for k in cost_map if "claude-fable-5" in k]
assert variants, "no claude-fable-5 entries found in cost map"
missing = [k for k in variants if cost_map[k].get("thinking_always_on") is not True]
assert not missing, f"missing thinking_always_on: {missing}"
@pytest.mark.parametrize(
"model",
[
"claude-fable-5",
"anthropic/claude-fable-5",
"anthropic.claude-fable-5",
"bedrock/us.anthropic.claude-fable-5",
"bedrock/invoke/eu.anthropic.claude-fable-5",
"bedrock/global.anthropic.claude-fable-5",
"vertex_ai/claude-fable-5",
"azure_ai/claude-fable-5",
],
)
def test_adaptive_thinking_detected_for_fable_5(local_model_cost_map, model):
"""Provider-routed ids must resolve to a flagged entry so ``reasoning_effort``
maps to ``thinking.type='adaptive'`` + ``output_config.effort``."""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True
FABLE_5_1_VARIANTS = (
"claude-fable-5-1",
"anthropic.claude-fable-5-1",
@ -131,73 +39,5 @@ FABLE_5_1_VARIANTS = (
)
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_fable_5_1_cache_reads_cost_a_quarter_of_fable_5(cost_map):
"""Fable 5.1 prices cache hits at 0.025x base input instead of the usual
0.1x, so copying Fable 5's cache-read price overcharges every cache hit 4x."""
for model_name in FABLE_5_1_VARIANTS:
info = cost_map[model_name]
geo_premium = model_name.startswith(("us.", "eu."))
expected = 2.75e-07 if geo_premium else 2.5e-07
assert info["cache_read_input_token_cost"] == expected, model_name
assert info["cache_read_input_token_cost"] == pytest.approx(
info["input_cost_per_token"] * 0.025
), model_name
def test_fable_5_1_present_in_bundled_backup():
backup = GetModelCostMap.load_local_model_cost_map()
root = _load_root_cost_map()
for model_name in FABLE_5_1_VARIANTS:
assert model_name in backup, f"Missing from backup cost map: {model_name}"
assert backup[model_name] == root[model_name], model_name
def test_fable_5_1_registered_for_bedrock_converse():
assert "anthropic.claude-fable-5-1" in BEDROCK_CONVERSE_MODELS
@pytest.mark.parametrize(
"model",
[
"claude-fable-5-1",
"anthropic/claude-fable-5-1",
"anthropic.claude-fable-5-1",
"bedrock/us.anthropic.claude-fable-5-1",
"bedrock/invoke/eu.anthropic.claude-fable-5-1",
"bedrock/global.anthropic.claude-fable-5-1",
"vertex_ai/claude-fable-5-1",
"azure_ai/claude-fable-5-1",
],
)
def test_adaptive_thinking_detected_for_fable_5_1(local_model_cost_map, model):
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_sampling_params_flag_on_all_models_that_removed_them(cost_map):
"""Fable 5 and Opus 4.7/4.8 reject ``top_p``/``top_k``/``temperature != 1``;
the drop/raise gating is cost-map driven, so every variant must carry an
explicit ``supports_sampling_params: false``. The perplexity route is
exempt: it is OpenAI-compatible and maps sampling params upstream."""
variants = [
k
for k in cost_map
if any(v in k for v in ("claude-fable-5", "claude-opus-4-7", "claude-opus-4-8"))
and not k.startswith("perplexity/")
]
assert variants, "no matching entries found in cost map"
missing = [
k for k in variants if cost_map[k].get("supports_sampling_params") is not False
]
assert not missing, f"missing supports_sampling_params=false: {missing}"

48
tests/test_litellm/test_claude_haiku_4_5_config.py
View file

@ -1,48 +0,0 @@
"""
Test Claude Haiku 4.5 model configurations for Bedrock
https://github.com/BerriAI/litellm/issues/15818
"""
import json
import os
def test_bedrock_haiku_4_5_matches_sonnet_capabilities():
"""
Test that Haiku 4.5 has same capabilities as Sonnet 4.5
(including computer_use, vision, tools, etc.)
"""
# Load model configuration
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path) as f:
model_data = json.load(f)
haiku_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
sonnet_model = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
haiku_info = model_data[haiku_model]
sonnet_info = model_data[sonnet_model]
# Both should use bedrock_converse
assert haiku_info["litellm_provider"] == "bedrock_converse"
assert sonnet_info["litellm_provider"] == "bedrock_converse"
# Shared capabilities that should match
shared_capabilities = [
"supports_vision",
"supports_computer_use",
"supports_function_calling",
"supports_tool_choice",
"supports_prompt_caching",
"supports_response_schema",
"supports_pdf_input",
"supports_assistant_prefill",
"supports_reasoning",
]
for capability in shared_capabilities:
assert haiku_info.get(capability) == sonnet_info.get(
capability
), f"Capability {capability} mismatch: Haiku={haiku_info.get(capability)}, Sonnet={sonnet_info.get(capability)}"

29
tests/test_litellm/test_claude_opus_5_config.py
View file

@ -18,7 +18,6 @@ import os
import pytest
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
@ -62,33 +61,5 @@ def test_opus_5_bedrock_rejects_strict_tools(model_name, local_model_cost_map):
assert bedrock_converse_supports_strict_tools(model_name) is False
def test_opus_5_present_in_bundled_backup():
"""The bundled backup is the runtime fallback (and what tests load with
``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the
root cost map, otherwise the model resolves on one path but not the other."""
backup = GetModelCostMap.load_local_model_cost_map()
for model_name in ALL_OPUS_5_VARIANTS:
assert model_name in backup, f"Missing from backup cost map: {model_name}"
def test_opus_5_registered_for_bedrock_converse():
assert "anthropic.claude-opus-5" in BEDROCK_CONVERSE_MODELS
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_opus_5_all_variants_carry_adaptive_thinking_flag(cost_map):
"""Every Opus 5 entry must advertise ``supports_adaptive_thinking``.
Adaptive-thinking detection is cost-map driven, so a single variant missing
the flag silently sends the legacy ``thinking.type='enabled'`` shape, which
Opus 5 rejects with a 400."""
variants = [k for k in cost_map if "claude-opus-5" in k]
assert variants, "no claude-opus-5 entries found in cost map"
missing = [
k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True
]
assert not missing, f"missing supports_adaptive_thinking: {missing}"

29
tests/test_litellm/test_claude_sonnet_5_config.py
View file

@ -13,10 +13,7 @@ populate ``litellm.anthropic_models`` at import, which is what lets a bare
import json
import os
import pytest
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
@ -40,31 +37,5 @@ def _load_root_cost_map() -> dict:
return json.load(f)
def test_sonnet_5_present_in_bundled_backup():
"""The bundled backup is the runtime fallback (and what tests load with
``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the
root cost map, otherwise the model resolves on one path but not the other."""
backup = GetModelCostMap.load_local_model_cost_map()
for model_name in ALL_SONNET_5_VARIANTS:
assert model_name in backup, f"Missing from backup cost map: {model_name}"
def test_sonnet_5_registered_for_bedrock_converse():
assert "anthropic.claude-sonnet-5" in BEDROCK_CONVERSE_MODELS
@pytest.mark.parametrize(
"cost_map",
[_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()],
ids=["root", "bundled_backup"],
)
def test_sonnet_5_all_variants_carry_adaptive_thinking_flag(cost_map):
"""Every Sonnet 5 entry must advertise ``supports_adaptive_thinking``.
Adaptive-thinking detection is cost-map driven, so a single variant missing
the flag silently sends the legacy ``thinking.type='enabled'`` shape and the
provider 400s. This guards against a future variant being added without it."""
variants = [k for k in cost_map if "claude-sonnet-5" in k]
assert variants, "no claude-sonnet-5 entries found in cost map"
missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True]
assert not missing, f"missing supports_adaptive_thinking: {missing}"

50
tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py
View file

@ -1,50 +0,0 @@
"""
Regression tests for the Cloudflare Workers AI text-generation catalog in the
model-cost map.
The Cloudflare list was badly stale (only 4 ancient entries). These tests pin
the newly added current Workers AI models (sourced from Cloudflare's live
``/ai/models/search?task=Text Generation`` catalog) and guard against the root
``model_prices_and_context_window.json`` and the bundled
``litellm/model_prices_and_context_window_backup.json`` drifting out of sync for
the ``cloudflare/`` namespace.
"""
import json
import os
import pytest
import litellm
ROOT_MAP = os.path.join(
os.path.dirname(os.path.dirname(litellm.__file__)),
"model_prices_and_context_window.json",
)
BACKUP_MAP = os.path.join(
os.path.dirname(litellm.__file__),
"model_prices_and_context_window_backup.json",
)
def _load(path: str) -> dict:
with open(path, encoding="utf-8") as f:
return json.load(f)
def _cloudflare_keys(data: dict) -> set:
return {k for k in data if k.startswith("cloudflare/")}
def test_root_and_backup_have_identical_cloudflare_keys():
if not os.path.exists(ROOT_MAP):
pytest.skip("root cost map only ships in source checkouts")
assert _cloudflare_keys(_load(ROOT_MAP)) == _cloudflare_keys(_load(BACKUP_MAP))
def test_root_and_backup_cloudflare_entries_are_byte_for_byte_equal():
if not os.path.exists(ROOT_MAP):
pytest.skip("root cost map only ships in source checkouts")
root = {k: v for k, v in _load(ROOT_MAP).items() if k.startswith("cloudflare/")}
backup = {k: v for k, v in _load(BACKUP_MAP).items() if k.startswith("cloudflare/")}
assert root == backup

67
tests/test_litellm/test_command_r7b_pricing.py
View file

@ -1,67 +0,0 @@
"""
Regression test: ``command-r7b-12-2024`` had its input/output per-token
costs transposed in the model-cost maps (input=1.5e-07 / output=3.75e-08),
even though Cohere publishes $0.0375/1M input and $0.15/1M output, i.e.
output is ~4x input like every other ``command-r`` entry.
These tests pin the corrected values in both the primary price map and the
``litellm/`` backup, and verify ``get_model_info`` surfaces them, so the
swap cannot silently regress.
"""
import json
import os
import litellm
MODEL = "command-r7b-12-2024"
EXPECTED_INPUT_COST = 3.75e-08
EXPECTED_OUTPUT_COST = 1.5e-07
def _load_json(path: str) -> dict:
with open(path, encoding="utf-8") as f:
return json.load(f)
def _backup_path() -> str:
return os.path.join(
os.path.dirname(litellm.__file__),
"model_prices_and_context_window_backup.json",
)
def _main_path() -> str:
# This test lives at ``tests/test_litellm/``; the primary price map sits at
# the repo root, two directories up. Resolve it relative to this file so the
# test works regardless of where ``litellm`` itself is installed (e.g. a pip
# install into site-packages).
return os.path.join(
os.path.dirname(__file__),
"..",
"..",
"model_prices_and_context_window.json",
)
class TestCommandR7bPricingData:
"""The JSON price maps must carry Cohere's published costs, with output
more expensive than input."""
class TestCommandR7bPricingModelInfo:
"""``get_model_info`` must report the corrected, un-swapped costs."""
def test_get_model_info_costs(self):
# Patch litellm.model_cost with the local backup so the test is not
# dependent on the remote fetch hitting a not-yet-merged main branch.
original = litellm.model_cost
try:
litellm.model_cost = _load_json(_backup_path())
info = litellm.get_model_info(MODEL)
assert info["input_cost_per_token"] == EXPECTED_INPUT_COST
assert info["output_cost_per_token"] == EXPECTED_OUTPUT_COST
assert info["output_cost_per_token"] > info["input_cost_per_token"]
finally:
litellm.model_cost = original

700
tests/test_litellm/test_cost_calculator.py
View file

File diff suppressed because it is too large Load diff

15
tests/test_litellm/test_daybreak_model_metadata.py
View file

@ -32,13 +32,6 @@ def _load(path):
return json.load(f)
def test_blue_alias_matches_its_snapshot_computer_use():
cost_map = _load(MAIN_PATH)
assert cost_map[BLUE_ALIAS]["supports_computer_use"] is True
assert cost_map[BLUE_SNAPSHOT]["supports_computer_use"] is True
@pytest.mark.parametrize(("alias", "snapshot"), OFFICIAL_ALIAS_SNAPSHOTS)
def test_official_alias_tracks_snapshot(alias, snapshot):
cost_map = _load(MAIN_PATH)
@ -51,11 +44,3 @@ def test_official_alias_tracks_snapshot(alias, snapshot):
field: snapshot_info.get(field) for field in PRICE_FIELDS
}
assert alias_info["max_output_tokens"] == snapshot_info["max_output_tokens"]
@pytest.mark.parametrize("model", (*DAYBREAK_MODELS, BLUE_SNAPSHOT, *(alias for alias, _ in OFFICIAL_ALIAS_SNAPSHOTS)))
def test_backup_matches_main(model):
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"

37
tests/test_litellm/test_deepseek_model_metadata.py
View file

@ -12,14 +12,11 @@ field set to ``True``.
import json
import os
import litellm
from litellm.utils import (
_supports_factory,
supports_response_schema,
)
# ---------------------------------------------------------------------------
# Data-level tests – verify the JSON files are in sync
# ---------------------------------------------------------------------------
@ -39,18 +36,6 @@ class TestDeepSeekModelCostEntries:
"""Verify that provider-prefixed DeepSeek entries contain the same
capability flags as their bare-name counterparts in the JSON files."""
def test_deepseek_chat_max_input_tokens_matches_bare_in_backup(self):
data = _load_backup_json()
bare = data.get("deepseek-chat", {})
prefixed = data.get("deepseek/deepseek-chat", {})
assert prefixed.get("max_input_tokens") == bare.get("max_input_tokens")
def test_deepseek_reasoner_max_output_tokens_matches_bare_in_backup(self):
data = _load_backup_json()
bare = data.get("deepseek-reasoner", {})
prefixed = data.get("deepseek/deepseek-reasoner", {})
assert prefixed.get("max_output_tokens") == bare.get("max_output_tokens")
# ---------------------------------------------------------------------------
# API-level tests – verify supports_response_schema returns True
@ -61,28 +46,6 @@ class TestSupportsResponseSchemaDeepSeek:
"""All calling conventions for DeepSeek should return True for
``supports_response_schema``."""
def test_provider_slash_model(self):
assert supports_response_schema(model="deepseek/deepseek-chat") is True
def test_explicit_provider(self):
assert (
supports_response_schema(
model="deepseek-chat", custom_llm_provider="deepseek"
)
is True
)
def test_reasoner_provider_slash_model(self):
assert supports_response_schema(model="deepseek/deepseek-reasoner") is True
def test_reasoner_explicit_provider(self):
assert (
supports_response_schema(
model="deepseek-reasoner", custom_llm_provider="deepseek"
)
is True
)
# ---------------------------------------------------------------------------
# Fallback-logic test – bare model entry used when prefixed is incomplete

44
tests/test_litellm/test_fireworks_serverless_model_costs.py
View file

@ -14,11 +14,6 @@ import os
import pytest
from litellm import completion_cost
from litellm.types.utils import Choices, Message, ModelResponse, Usage
from litellm.utils import get_model_info
NEW_ENTRIES = {
"fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": {
"input_cost_per_token": 1.32e-06,
@ -32,46 +27,11 @@ NEW_ENTRIES = {
@pytest.fixture(scope="module")
def model_data():
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json")
with open(json_path) as f:
return json.load(f)
def test_bare_fireworks_ids_resolve_through_prefixed_entries():
"""Bare IDs from #37274 resolve via the provider-prefix lookup path."""
for bare_id, prefixed_key in [
(
"accounts/fireworks/models/deepseek-v4-pro-0813",
"fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813",
),
]:
info = get_model_info(model=bare_id, custom_llm_provider="fireworks_ai")
expected = NEW_ENTRIES[prefixed_key]
assert info.get("key") == prefixed_key
assert info["litellm_provider"] == "fireworks_ai"
assert info["input_cost_per_token"] == pytest.approx(expected["input_cost_per_token"])
assert info["cache_read_input_token_cost"] == pytest.approx(expected["cache_read_input_token_cost"])
assert info["output_cost_per_token"] == pytest.approx(expected["output_cost_per_token"])
assert info["max_input_tokens"] == expected["max_input_tokens"]
assert info["max_output_tokens"] == expected["max_output_tokens"]
def test_deepseek_v4p1_flash_twin_costs(local_model_cost_map):
for model in (
"fireworks_ai/deepseek-v4p1-flash",
"fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash",
):
response = ModelResponse(
model=model,
choices=[Choices(index=0, message=Message(role="assistant", content="ok"))],
usage=Usage(prompt_tokens=1000, completion_tokens=1000, total_tokens=2000),
)
cost = completion_cost(completion_response=response, model=model)
assert cost == pytest.approx(8.8e-04)
TWIN_PINNED_PRICES = {
"deepseek-v4-flash-0731": {
"input_cost_per_token": 2.2e-07,
@ -95,7 +55,7 @@ def test_fireworks_account_prefixed_twins_agree_on_price(model_data):
for key, entry in model_data.items():
if not key.startswith(prefix):
continue
bare_key = f"fireworks_ai/{key[len(prefix):]}"
bare_key = f"fireworks_ai/{key[len(prefix) :]}"
bare_entry = model_data.get(bare_key)
if bare_entry is None:
continue

126
tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py
View file

@ -4,25 +4,12 @@ from pathlib import Path
import pytest
import litellm
from litellm import completion_cost
from litellm.cost_calculator import cost_per_token
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.llms.gemini.image_generation.cost_calculator import (
cost_calculator as gemini_image_generation_cost_calculator,
)
from litellm.llms.vertex_ai.image_generation.cost_calculator import (
cost_calculator as vertex_image_generation_cost_calculator,
)
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
ImageObject,
ImageResponse,
ImageUsage,
ImageUsageInputTokensDetails,
ModelResponse,
PromptTokensDetailsWrapper,
Usage,
)
REPO_ROOT = Path(__file__).parents[2]
@ -122,16 +109,6 @@ def test_per_route_capabilities_match_model_cards(model: str, path: Path):
assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}"
@pytest.mark.parametrize("model", ALL_KEYS)
def test_backup_matches_main(model: str):
assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model)
def test_one_k_image_price_matches_official_token_math():
assert TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST == pytest.approx(OUTPUT_COST_PER_1K_IMAGE)
assert TOKENS_PER_1K_IMAGE * INPUT_COST == pytest.approx(INPUT_COST_PER_IMAGE)
def test_gemini_prefix_routes_to_gemini():
routed_model, provider, _, _ = get_llm_provider(model=GEMINI)
assert routed_model == UNPREFIXED
@ -144,78 +121,6 @@ def test_vertex_prefix_routes_to_vertex():
assert provider == "vertex_ai"
def test_get_model_info_reports_published_costs(local_model_cost_map):
info = litellm.get_model_info(UNPREFIXED)
assert info["input_cost_per_token"] == INPUT_COST
assert info["output_cost_per_token"] == OUTPUT_TEXT_COST
assert info["cache_read_input_token_cost"] == CACHE_READ_COST
@pytest.mark.parametrize("model", ALL_KEYS)
def test_reasoning_params_are_not_offered_on_an_image_endpoint(model: str, local_model_cost_map):
assert litellm.supports_reasoning(model) is False
def test_text_token_cost(local_model_cost_map):
prompt_cost, text_completion_cost = cost_per_token(
model=GEMINI, prompt_tokens=1000, completion_tokens=500
)
assert prompt_cost == pytest.approx(1000 * INPUT_COST)
assert text_completion_cost == pytest.approx(500 * OUTPUT_TEXT_COST)
def test_completion_cost_bills_one_k_image(local_model_cost_map):
response = ModelResponse()
response.model = UNPREFIXED
response.usage = Usage(
prompt_tokens=7,
completion_tokens=TOKENS_PER_1K_IMAGE,
total_tokens=7 + TOKENS_PER_1K_IMAGE,
completion_tokens_details=CompletionTokensDetailsWrapper(
image_tokens=TOKENS_PER_1K_IMAGE, text_tokens=0
),
)
billed = completion_cost(
completion_response=response,
model=UNPREFIXED,
custom_llm_provider="vertex_ai",
)
expected = TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST + 7 * INPUT_COST
assert billed == pytest.approx(expected)
def test_image_tokens_are_not_billed_as_text(local_model_cost_map):
usage = Usage(
completion_tokens=1345,
prompt_tokens=10,
total_tokens=1355,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=None,
audio_tokens=None,
reasoning_tokens=225,
rejected_prediction_tokens=None,
text_tokens=0,
image_tokens=TOKENS_PER_1K_IMAGE,
),
prompt_tokens_details=PromptTokensDetailsWrapper(
audio_tokens=None, cached_tokens=None, text_tokens=10, image_tokens=None
),
)
_, image_completion_cost = generic_cost_per_token(
model=UNPREFIXED,
usage=usage,
custom_llm_provider="vertex_ai",
)
expected_completion_cost = (
TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST + 225 * OUTPUT_TEXT_COST
)
bugged_text_only_cost = 1345 * OUTPUT_TEXT_COST
assert image_completion_cost > bugged_text_only_cost * 2
assert image_completion_cost == pytest.approx(expected_completion_cost)
def _one_k_image_response() -> ImageResponse:
return ImageResponse(
data=[ImageObject(b64_json="img1")],
@ -229,34 +134,3 @@ def _one_k_image_response() -> ImageResponse:
total_tokens=50 + TOKENS_PER_1K_IMAGE + TOKENS_PER_1K_IMAGE,
),
)
def test_gemini_image_generation_uses_token_pricing(local_model_cost_map):
cost = gemini_image_generation_cost_calculator(
model=GEMINI, image_response=_one_k_image_response()
)
expected = (
50 + TOKENS_PER_1K_IMAGE
) * INPUT_COST + TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST
assert cost == pytest.approx(expected)
assert cost != OUTPUT_COST_PER_1K_IMAGE
def test_vertex_image_generation_uses_token_pricing(local_model_cost_map):
cost = vertex_image_generation_cost_calculator(
model=UNPREFIXED, image_response=_one_k_image_response()
)
expected = (
50 + TOKENS_PER_1K_IMAGE
) * INPUT_COST + TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST
assert cost == pytest.approx(expected)
def test_vertex_image_generation_falls_back_to_flat_image_price(local_model_cost_map):
image_response = ImageResponse(
data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")]
)
cost = vertex_image_generation_cost_calculator(
model=UNPREFIXED, image_response=image_response
)
assert cost == pytest.approx(2 * OUTPUT_COST_PER_1K_IMAGE)

135
tests/test_litellm/test_gemini_tts_native_audio_pricing.py
View file

@ -1,135 +0,0 @@
import json
from collections.abc import Iterator
from pathlib import Path
from typing import Final
import pytest
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.types.utils import CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, Usage
REPO_ROOT: Final = Path(__file__).parents[2]
MAIN_PATH: Final = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
FLASH_TTS_KEYS: Final = ("gemini-2.5-flash-preview-tts", "gemini/gemini-2.5-flash-preview-tts")
PRO_TTS_KEYS: Final = ("gemini-2.5-pro-preview-tts", "gemini/gemini-2.5-pro-preview-tts")
NATIVE_AUDIO_KEYS: Final = tuple(
f"{prefix}gemini-2.5-flash-native-audio-{suffix}"
for prefix in ("", "gemini/")
for suffix in ("latest", "preview-09-2025", "preview-12-2025")
)
LIVE_NATIVE_AUDIO_KEYS: Final = (
"gemini-live-2.5-flash-preview-native-audio-09-2025",
"gemini/gemini-live-2.5-flash-preview-native-audio-09-2025",
)
FLASH_TTS_INPUT: Final = 5e-07
FLASH_TTS_AUDIO_OUTPUT: Final = 1e-05
PRO_TTS_INPUT: Final = 1e-06
PRO_TTS_AUDIO_OUTPUT: Final = 2e-05
NATIVE_AUDIO_TEXT_INPUT: Final = 5e-07
NATIVE_AUDIO_AUDIO_INPUT: Final = 3e-06
NATIVE_AUDIO_TEXT_OUTPUT: Final = 2e-06
NATIVE_AUDIO_AUDIO_OUTPUT: Final = 1.2e-05
PUBLISHED_RATES: Final = {
**{
key: {"input_cost_per_token": FLASH_TTS_INPUT, "output_cost_per_token": FLASH_TTS_AUDIO_OUTPUT}
for key in FLASH_TTS_KEYS
},
**{
key: {"input_cost_per_token": PRO_TTS_INPUT, "output_cost_per_token": PRO_TTS_AUDIO_OUTPUT}
for key in PRO_TTS_KEYS
},
**{
key: {
"input_cost_per_token": NATIVE_AUDIO_TEXT_INPUT,
"input_cost_per_audio_token": NATIVE_AUDIO_AUDIO_INPUT,
"output_cost_per_token": NATIVE_AUDIO_TEXT_OUTPUT,
"output_cost_per_audio_token": NATIVE_AUDIO_AUDIO_OUTPUT,
}
for key in (*NATIVE_AUDIO_KEYS, *LIVE_NATIVE_AUDIO_KEYS)
},
}
ALL_KEYS: Final = tuple(PUBLISHED_RATES)
NATIVE_AUDIO_BILLING_CASES: Final = (
*((key, "gemini") for key in NATIVE_AUDIO_KEYS),
("gemini-live-2.5-flash-preview-native-audio-09-2025", "vertex_ai"),
("gemini/gemini-live-2.5-flash-preview-native-audio-09-2025", "gemini"),
)
LONG_CONTEXT_TIER_FIELDS: Final = (
"input_cost_per_token_above_200k_tokens",
"output_cost_per_token_above_200k_tokens",
"cache_read_input_token_cost_above_200k_tokens",
)
def _load(path: Path) -> dict[str, dict[str, object]]:
with open(path, encoding="utf-8") as f:
return json.load(f)
@pytest.fixture
def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
@pytest.mark.parametrize("model", ALL_KEYS)
def test_backup_matches_main(model: str):
assert _load(BACKUP_PATH)[model] == _load(MAIN_PATH)[model]
@pytest.mark.parametrize(
("model", "provider", "input_rate", "audio_output_rate"),
(
("gemini-2.5-flash-preview-tts", "gemini", FLASH_TTS_INPUT, FLASH_TTS_AUDIO_OUTPUT),
("gemini-2.5-pro-preview-tts", "gemini", PRO_TTS_INPUT, PRO_TTS_AUDIO_OUTPUT),
("gemini-2.5-pro-preview-tts", "vertex_ai", PRO_TTS_INPUT, PRO_TTS_AUDIO_OUTPUT),
),
)
def test_tts_audio_output_is_billed_at_the_audio_rate(
model: str, provider: str, input_rate: float, audio_output_rate: float, local_model_cost_map
):
usage: Final = Usage(
prompt_tokens=9,
completion_tokens=49,
total_tokens=58,
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=9),
completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=49, text_tokens=0),
)
prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=provider)
assert prompt_cost == pytest.approx(9 * input_rate)
assert completion_cost == pytest.approx(49 * audio_output_rate)
@pytest.mark.parametrize("model, provider", NATIVE_AUDIO_BILLING_CASES)
def test_native_audio_output_is_billed_at_the_audio_rate(model: str, provider: str, local_model_cost_map):
usage: Final = Usage(
prompt_tokens=377,
completion_tokens=84,
total_tokens=461,
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=377),
completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=48, reasoning_tokens=36, text_tokens=0),
)
prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=provider)
assert prompt_cost == pytest.approx(377 * NATIVE_AUDIO_TEXT_INPUT)
assert completion_cost == pytest.approx(48 * NATIVE_AUDIO_AUDIO_OUTPUT + 36 * NATIVE_AUDIO_TEXT_OUTPUT)
@pytest.mark.parametrize("model, provider", NATIVE_AUDIO_BILLING_CASES)
def test_native_audio_input_is_billed_at_the_audio_rate(model: str, provider: str, local_model_cost_map):
usage: Final = Usage(
prompt_tokens=1000,
completion_tokens=0,
total_tokens=1000,
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=100, audio_tokens=900),
)
prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=provider)
assert prompt_cost == pytest.approx(100 * NATIVE_AUDIO_TEXT_INPUT + 900 * NATIVE_AUDIO_AUDIO_INPUT)

44
tests/test_litellm/test_gpt_5_4_model_metadata.py
View file

@ -1,44 +0,0 @@
import json
from functools import lru_cache
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
DOCUMENTED_MAX_INPUT_TOKENS = 272000
DOCUMENTED_MAX_OUTPUT_TOKENS = 128000
SMALL_MODEL_NAMES = (
"gpt-5.4-mini",
"gpt-5.4-mini-2026-03-17",
"gpt-5.4-nano",
"gpt-5.4-nano-2026-03-17",
)
SMALL_MODELS = tuple(f"{prefix}{name}" for prefix in ("", "azure/", "azure_ai/") for name in SMALL_MODEL_NAMES)
STANDARD_PRICING = {
"gpt-5.4-mini": (7.5e-07, 4.5e-06, 7.5e-08),
"gpt-5.4-nano": (2e-07, 1.25e-06, 2e-08),
}
LONG_CONTEXT_MODELS = ("gpt-5.4", "gpt-5.4-pro")
@lru_cache(maxsize=2)
def _load(path: Path) -> dict[str, dict[str, object]]:
with open(path) as f:
return json.load(f)
def _pricing_key(model: str) -> str:
return "gpt-5.4-nano" if "nano" in model else "gpt-5.4-mini"
@pytest.mark.parametrize("model", SMALL_MODELS)
def test_gpt_5_4_small_models_backup_matches_main(model: str) -> None:
assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model), (
f"{model} differs between main and backup model cost maps"
)

19
tests/test_litellm/test_gpt_5_5_model_metadata.py
View file

@ -1,19 +0,0 @@
import json
from pathlib import Path
def test_azure_ai_gpt_5_5_backup_matches_main():
"""Ensure the bundled model cost map stays in sync with the canonical file."""
repo_root = Path(__file__).parents[2]
main_path = repo_root / "model_prices_and_context_window.json"
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
with open(main_path) as f:
main_cost = json.load(f)
with open(backup_path) as f:
backup_cost = json.load(f)
for model in ("azure_ai/gpt-5.5", "azure_ai/gpt-5.5-2026-04-23"):
assert backup_cost.get(model) == main_cost.get(
model
), f"{model} differs between main and backup model cost maps"

352
tests/test_litellm/test_gpt_image_cost_calculator.py
View file

@ -10,19 +10,12 @@ gpt-image-1 uses token-based pricing:
- Image Output: $40.00/1M tokens
"""
import pytest
import litellm
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
ImageResponse,
ImageObject,
ImageUsage,
ImageUsageInputTokensDetails,
PromptTokensDetailsWrapper,
Usage,
ImageResponse,
)
@ -42,106 +35,6 @@ def _use_local_model_cost_map(monkeypatch):
class TestGPTImageCostCalculator:
"""Test the OpenAI gpt-image cost calculator"""
def test_gpt_image_1_cost_with_text_only(self):
"""Test cost calculation with only text input tokens"""
from litellm.llms.openai.image_generation.cost_calculator import cost_calculator
usage = ImageUsage(
input_tokens=100,
output_tokens=5000,
total_tokens=5100,
input_tokens_details=ImageUsageInputTokensDetails(
text_tokens=100,
image_tokens=0,
),
)
image_response = ImageResponse(
created=1234567890,
data=[ImageObject(url="http://example.com/image.jpg")],
)
image_response.usage = usage
cost = cost_calculator(
model="gpt-image-1",
image_response=image_response,
custom_llm_provider="openai",
)
# Expected cost:
# Text input: 100 * $5/1M = 0.0005
# Image output: 5000 * $40/1M = 0.2
# Total: 0.2005
expected_cost = 0.0005 + 0.2
assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}"
def test_gpt_image_1_cost_with_image_input(self):
"""Test cost calculation with both text and image input tokens (for edits)"""
from litellm.llms.openai.image_generation.cost_calculator import cost_calculator
usage = ImageUsage(
input_tokens=600,
output_tokens=5000,
total_tokens=5600,
input_tokens_details=ImageUsageInputTokensDetails(
text_tokens=100,
image_tokens=500,
),
)
image_response = ImageResponse(
created=1234567890,
data=[ImageObject(url="http://example.com/image.jpg")],
)
image_response.usage = usage
cost = cost_calculator(
model="gpt-image-1",
image_response=image_response,
custom_llm_provider="openai",
)
# Expected cost:
# Text input: 100 * $5/1M = 0.0005
# Image input: 500 * $10/1M = 0.005
# Image output: 5000 * $40/1M = 0.2
# Total: 0.2055
expected_cost = 0.0005 + 0.005 + 0.2
assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}"
def test_gpt_image_1_mini_cost(self):
"""Test cost calculation for gpt-image-1-mini model"""
from litellm.llms.openai.image_generation.cost_calculator import cost_calculator
usage = ImageUsage(
input_tokens=100,
output_tokens=5000,
total_tokens=5100,
input_tokens_details=ImageUsageInputTokensDetails(
text_tokens=100,
image_tokens=0,
),
)
image_response = ImageResponse(
created=1234567890,
data=[ImageObject(url="http://example.com/image.jpg")],
)
image_response.usage = usage
cost = cost_calculator(
model="gpt-image-1-mini",
image_response=image_response,
custom_llm_provider="openai",
)
# Expected cost for gpt-image-1-mini:
# Text input: 100 * $2/1M = 0.0002
# Image output: 5000 * $8/1M = 0.04
# Total: 0.0402
expected_cost = 0.0002 + 0.04
assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}"
def test_gpt_image_1_cost_no_usage(self):
"""Test that cost returns 0 when no usage data is available"""
from litellm.llms.openai.image_generation.cost_calculator import cost_calculator
@ -159,98 +52,10 @@ class TestGPTImageCostCalculator:
assert cost == 0.0
def test_gpt_image_2_cost_with_text_and_image_tokens(self):
"""Test cost calculation for gpt-image-2 token pricing"""
from litellm.llms.openai.image_generation.cost_calculator import cost_calculator
usage = Usage(
prompt_tokens=600,
completion_tokens=5000,
total_tokens=5600,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=100,
image_tokens=500,
),
completion_tokens_details=CompletionTokensDetailsWrapper(
image_tokens=5000,
),
)
image_response = ImageResponse(
created=1234567890,
data=[ImageObject(url="http://example.com/image.jpg")],
)
image_response.usage = usage
cost = cost_calculator(
model="gpt-image-2",
image_response=image_response,
custom_llm_provider="openai",
)
expected_cost = 100 * 5e-6 + 500 * 8e-6 + 5000 * 3e-5
assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}"
class TestGPTImageCostRouting:
"""Test that gpt-image models are properly routed to the token-based calculator"""
def test_openai_gpt_image_routes_to_token_calculator(self):
"""Test that OpenAI gpt-image-1 routes to token-based calculator"""
from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils
usage = ImageUsage(
input_tokens=100,
output_tokens=5000,
total_tokens=5100,
input_tokens_details=ImageUsageInputTokensDetails(
text_tokens=100,
image_tokens=0,
),
)
image_response = ImageResponse(
created=1234567890,
data=[ImageObject(url="http://example.com/image.jpg")],
)
image_response.usage = usage
cost = CostCalculatorUtils.route_image_generation_cost_calculator(
model="gpt-image-1",
completion_response=image_response,
custom_llm_provider="openai",
)
expected_cost = 0.0005 + 0.2
assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}"
def test_openai_gpt_image_2_routes_to_token_calculator(self):
"""Test that OpenAI gpt-image-2 routes to token-based calculator"""
from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils
usage = Usage(
prompt_tokens=100,
completion_tokens=5000,
total_tokens=5100,
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=100),
completion_tokens_details=CompletionTokensDetailsWrapper(image_tokens=5000),
)
image_response = ImageResponse(
created=1234567890,
data=[ImageObject(url="http://example.com/image.jpg")],
)
image_response.usage = usage
cost = CostCalculatorUtils.route_image_generation_cost_calculator(
model="gpt-image-2",
completion_response=image_response,
custom_llm_provider="openai",
)
expected_cost = 0.0005 + 0.15
assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}"
def test_openai_dalle_routes_to_pixel_calculator(self):
"""Test that OpenAI DALL-E still routes to pixel-based calculator"""
from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils
@ -283,94 +88,10 @@ class TestGPTImage15OutputImageTokens:
and these must be correctly included in cost calculation.
"""
def test_gpt_image_15_output_image_tokens_cost(self):
"""
Test that output image tokens are correctly included in cost calculation.
This tests the fix for issue #19508 where output_tokens_details.image_tokens
were not being included in the cost calculation, causing costs to be
underreported (e.g., $0.046 instead of $0.14).
"""
# Simulate gpt-image-1.5 response with output_tokens_details
# This is what the API returns and what convert_to_image_response transforms
usage = Usage(
prompt_tokens=169,
completion_tokens=4599,
total_tokens=4768,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=169,
image_tokens=0,
),
completion_tokens_details=CompletionTokensDetailsWrapper(
text_tokens=439,
image_tokens=4160,
),
)
image_response = ImageResponse(
created=1234567890,
data=[ImageObject(b64_json="test")],
)
image_response.usage = usage
image_response._hidden_params = {"custom_llm_provider": "openai"}
cost = litellm.completion_cost(
completion_response=image_response,
model="gpt-image-1.5",
call_type="image_generation",
custom_llm_provider="openai",
)
# gpt-image-1.5 pricing:
# - input_cost_per_token: 5e-06 ($5/1M for text input)
# - output_cost_per_token: 1e-05 ($10/1M for text output)
# - output_cost_per_image_token: 3.2e-05 ($32/1M for image output)
#
# Expected cost:
# Input text: 169 * $5/1M = $0.000845
# Output text: 439 * $10/1M = $0.00439
# Output image: 4160 * $32/1M = $0.13312
# Total: $0.138355
expected_cost = 169 * 5e-06 + 439 * 1e-05 + 4160 * 3.2e-05
assert abs(cost - expected_cost) < 1e-6, (
f"Expected {expected_cost}, got {cost}. "
f"Image tokens may not be included in cost calculation."
)
class TestCompletionCostIntegration:
"""Test the full completion_cost integration for gpt-image-1"""
def test_completion_cost_gpt_image_1(self):
"""Test completion_cost correctly calculates gpt-image-1 costs"""
usage = ImageUsage(
input_tokens=100,
output_tokens=5000,
total_tokens=5100,
input_tokens_details=ImageUsageInputTokensDetails(
text_tokens=100,
image_tokens=0,
),
)
image_response = ImageResponse(
created=1234567890,
data=[ImageObject(url="http://example.com/image.jpg")],
)
image_response.usage = usage
image_response._hidden_params = {"custom_llm_provider": "openai"}
cost = litellm.completion_cost(
completion_response=image_response,
model="gpt-image-1",
call_type="image_generation",
custom_llm_provider="openai",
)
expected_cost = 0.0005 + 0.2
assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}"
class TestGPTImage2OutputImageTokensNoBreakdown:
"""
@ -383,77 +104,6 @@ class TestGPTImage2OutputImageTokensNoBreakdown:
cost component.
"""
def test_gpt_image_2_output_priced_as_image_when_no_breakdown(self):
from litellm.llms.openai.image_generation.cost_calculator import (
cost_calculator,
)
# Mirrors a real gpt-image-2 /v1/images/edits response: input breakdown is
# present, but there is no usable output token breakdown.
usage = ImageUsage(
input_tokens=3987,
output_tokens=5488,
total_tokens=9475,
input_tokens_details=ImageUsageInputTokensDetails(
text_tokens=943,
image_tokens=3044,
),
)
image_response = ImageResponse(
created=1234567890,
data=[ImageObject(b64_json="test")],
)
image_response.usage = usage
image_response._hidden_params = {"custom_llm_provider": "openai"}
cost = cost_calculator(
model="gpt-image-2",
image_response=image_response,
custom_llm_provider="openai",
)
# gpt-image-2 pricing:
# text input: 943 * $5/1M = 0.004715
# image input: 3044 * $8/1M = 0.024352
# image output: 5488 * $30/1M = 0.164640 (NOT text output $10/1M = 0.054880)
expected_cost = 943 * 5e-6 + 3044 * 8e-6 + 5488 * 3e-5
assert abs(cost - expected_cost) < 1e-6, (
f"Expected {expected_cost}, got {cost}. Generated image output tokens "
f"are likely being priced at the text output_cost_per_token rate."
)
def test_gpt_image_2_chat_usage_without_breakdown_uses_image_rate(self):
from litellm.llms.openai.image_generation.cost_calculator import (
cost_calculator,
)
usage = Usage(
prompt_tokens=600,
completion_tokens=5000,
total_tokens=5600,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=100,
image_tokens=500,
),
)
image_response = ImageResponse(
created=1234567890,
data=[ImageObject(b64_json="test")],
)
image_response.usage = usage
image_response._hidden_params = {"custom_llm_provider": "openai"}
cost = cost_calculator(
model="gpt-image-2",
image_response=image_response,
custom_llm_provider="openai",
)
expected_cost = 100 * 5e-6 + 500 * 8e-6 + 5000 * 3e-5
assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}"
if __name__ == "__main__":
pytest.main([__file__, "-v"])

13
tests/test_litellm/test_gpt_realtime_mode.py
View file

@ -1,6 +1,3 @@
import json
from pathlib import Path
from typing_extensions import get_args, get_type_hints
from litellm.types.utils import ModelInfoBase
@ -44,13 +41,3 @@ ALL_REALTIME_ONLY_GPT_MODELS = REALTIME_ONLY_GPT_MODELS + REALTIME_ONLY_GPT_MODE
def test_realtime_is_a_valid_mode_literal():
hints = get_type_hints(ModelInfoBase, include_extras=False)
assert "realtime" in get_args(hints["mode"])
def test_backup_matches_main_for_realtime_models():
repo_root = Path(__file__).parents[2]
with open(repo_root / "model_prices_and_context_window.json") as f:
main_cost = json.load(f)
with open(repo_root / "litellm" / "model_prices_and_context_window_backup.json") as f:
backup_cost = json.load(f)
for model in ALL_REALTIME_ONLY_GPT_MODELS:
assert backup_cost.get(model) == main_cost.get(model)

34
tests/test_litellm/test_mistral_medium_3_5_model_metadata.py
View file

@ -1,34 +0,0 @@
import json
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
MEDIUM_3_5_MODELS = (
"mistral/mistral-medium-3-5",
"mistral/mistral-medium-2604",
"mistral/mistral-medium-latest",
)
SYNCED_MODELS = MEDIUM_3_5_MODELS + (
"mistral/mistral-medium-2508",
"mistral/mistral-medium-3-1-2508",
)
def _load(path):
with open(path) as f:
return json.load(f)
@pytest.mark.parametrize("model", SYNCED_MODELS)
def test_backup_matches_main(model):
"""Ensure the bundled (backup) cost map stays in sync with the canonical file."""
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"

26
tests/test_litellm/test_mistral_small_4_0_model_metadata.py
View file

@ -1,26 +0,0 @@
import json
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
SMALL_4_0_MODELS = (
"mistral/mistral-small-latest",
"mistral/mistral-small-2603",
)
def _load(path):
with open(path) as f:
return json.load(f)
@pytest.mark.parametrize("model", SMALL_4_0_MODELS)
def test_backup_matches_main(model):
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"

74
tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py
View file

@ -1,74 +0,0 @@
import json
from pathlib import Path
import pytest
import litellm
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
from litellm.utils import supports_prompt_caching, supports_reasoning
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
GLM_5_2_MODELS = ("mistral/zai-glm-5-2", "mistral/glm-5-2")
INPUT_COST = 1.4e-06
CACHED_INPUT_COST = 1.4e-07
OUTPUT_COST = 4.4e-06
def _load(path):
with open(path) as f:
return json.load(f)
@pytest.fixture
def local_model_cost_map(monkeypatch):
"""Force get_model_info to resolve against the in-repo cost map instead of the
remote one fetched at import time, which still carries the pre-merge pricing."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
def test_zai_glm_5_2_capabilities_are_visible_to_callers(local_model_cost_map, model):
"""Mistral advertises reasoning and prompt caching on this model, so the helpers
every caller checks before sending a request must say so too."""
assert supports_reasoning(model=model) is True
assert supports_prompt_caching(model=model) is True
info = litellm.get_model_info(model=model)
assert info["max_input_tokens"] == 1048576
assert info["max_output_tokens"] == 131072
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
def test_cached_prompt_tokens_bill_at_the_cached_rate(local_model_cost_map, model):
"""A cache hit reports its reused tokens under prompt_tokens_details, and those
tokens cost a tenth of the input rate, not the full rate and not nothing."""
usage = Usage(
prompt_tokens=21010,
completion_tokens=100,
total_tokens=21110,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=20992),
)
prompt_cost, completion_cost = litellm.cost_per_token(
model=model, usage_object=usage, custom_llm_provider="mistral"
)
assert prompt_cost == pytest.approx(18 * INPUT_COST + 20992 * CACHED_INPUT_COST)
assert completion_cost == pytest.approx(100 * OUTPUT_COST)
@pytest.mark.parametrize("model", GLM_5_2_MODELS)
def test_backup_matches_main(model):
"""Ensure the bundled (backup) cost map stays in sync with the canonical file."""
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"

29
tests/test_litellm/test_muse_spark_1_1_model_metadata.py
View file

@ -1,29 +0,0 @@
import json
from pathlib import Path
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
MUSE_SPARK_MODEL = "meta/muse-spark-1.1"
def test_muse_spark_1_1_model_info():
routed_model, provider, _, api_base = get_llm_provider(model=MUSE_SPARK_MODEL, api_key="sk-test")
assert routed_model == "muse-spark-1.1"
assert provider == "meta"
assert api_base == "https://api.meta.ai/v1"
def test_muse_spark_1_1_backup_matches_main():
"""Ensure the bundled model cost map stays in sync with the canonical file."""
repo_root = Path(__file__).parents[2]
main_path = repo_root / "model_prices_and_context_window.json"
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
with open(main_path) as f:
main_cost = json.load(f)
with open(backup_path) as f:
backup_cost = json.load(f)
assert backup_cost.get(MUSE_SPARK_MODEL) == main_cost.get(MUSE_SPARK_MODEL), (
f"{MUSE_SPARK_MODEL} differs between main and backup model cost maps"
)

29
tests/test_litellm/test_muse_spark_1_2_model_metadata.py
View file

@ -3,10 +3,7 @@ from pathlib import Path
import pytest
import litellm
from litellm.cost_calculator import cost_per_token
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import StandardBuiltInToolCostTracking
MUSE_SPARK_STANDARD = "meta/muse-spark-1.2"
MUSE_SPARK_CONTRIBUTOR = "meta/muse-spark-1.2-contributor"
@ -23,16 +20,6 @@ def _load_cost_map(filename: str = "model_prices_and_context_window.json") -> di
return json.load(f)
@pytest.mark.parametrize("model, input_cost, cached_cost, output_cost", PRICING)
def test_muse_spark_1_2_cost_per_token(
local_model_cost_map, model: str, input_cost: float, cached_cost: float, output_cost: float
):
prompt_cost, completion_cost = cost_per_token(model=model, prompt_tokens=1000, completion_tokens=500)
assert prompt_cost == pytest.approx(1000 * input_cost)
assert completion_cost == pytest.approx(500 * output_cost)
@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR))
def test_muse_spark_1_2_routes_to_meta_model_api(model: str):
routed_model, provider, _, api_base = get_llm_provider(model=model, api_key="sk-test")
@ -42,22 +29,6 @@ def test_muse_spark_1_2_routes_to_meta_model_api(model: str):
assert api_base == "https://api.meta.ai/v1"
@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR))
def test_muse_spark_1_2_web_search_cost_per_query(local_model_cost_map, model: str):
info = litellm.get_model_info(model=model)
assert StandardBuiltInToolCostTracking.get_cost_for_web_search(model_info=info) == WEB_SEARCH_COST_PER_QUERY
@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR))
def test_muse_spark_1_2_backup_matches_main(model: str):
"""Ensure the bundled model cost map stays in sync with the canonical file."""
main_cost = _load_cost_map()
backup_cost = _load_cost_map("litellm/model_prices_and_context_window_backup.json")
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"
def test_muse_spark_contributor_tier_is_cheaper_than_standard():
cost_map = _load_cost_map()
standard = cost_map[MUSE_SPARK_STANDARD]

20
tests/test_litellm/test_muse_spark_1_3_model_metadata.py
View file

@ -4,7 +4,6 @@ from pathlib import Path
import pytest
import litellm
from litellm.cost_calculator import cost_per_token
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import StandardBuiltInToolCostTracking
@ -23,16 +22,6 @@ def _load_cost_map(filename: str = "model_prices_and_context_window.json") -> di
return json.load(f)
@pytest.mark.parametrize("model, input_cost, cached_cost, output_cost", PRICING)
def test_muse_spark_1_3_cost_per_token(
local_model_cost_map, model: str, input_cost: float, cached_cost: float, output_cost: float
):
prompt_cost, completion_cost = cost_per_token(model=model, prompt_tokens=1000, completion_tokens=500)
assert prompt_cost == pytest.approx(1000 * input_cost)
assert completion_cost == pytest.approx(500 * output_cost)
@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR))
def test_muse_spark_1_3_routes_to_meta_model_api(model: str):
routed_model, provider, _, api_base = get_llm_provider(model=model, api_key="sk-test")
@ -49,15 +38,6 @@ def test_muse_spark_1_3_web_search_cost_per_query(local_model_cost_map, model: s
assert StandardBuiltInToolCostTracking.get_cost_for_web_search(model_info=info) == WEB_SEARCH_COST_PER_QUERY
@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR))
def test_muse_spark_1_3_backup_matches_main(model: str):
"""Ensure the bundled model cost map stays in sync with the canonical file."""
main_cost = _load_cost_map()
backup_cost = _load_cost_map("litellm/model_prices_and_context_window_backup.json")
assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps"
def test_muse_spark_contributor_tier_is_cheaper_than_standard():
cost_map = _load_cost_map()
standard = cost_map[MUSE_SPARK_STANDARD]

24
tests/test_litellm/test_openai_service_tier_long_context_pricing.py
View file

@ -106,27 +106,3 @@ def test_cost_per_token_bills_long_context_at_the_tier_rate(
)
assert input_cost == pytest.approx(LONG_CONTEXT_PROMPT_TOKENS * input_rate)
assert output_cost == pytest.approx(COMPLETION_TOKENS * output_rate)
@pytest.mark.parametrize("model,tier,input_rate,output_rate", TIERED_COST_CASES)
def test_cost_per_token_tier_differs_from_the_standard_long_context_cost(
model: str, tier: str, input_rate: float, output_rate: float
) -> None:
"""Flex halves the standard long-context bill and priority doubles it."""
ratio = 0.5 if tier == "flex" else 2.0
standard = sum(
litellm.cost_per_token(
model=model,
prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS,
completion_tokens=COMPLETION_TOKENS,
)
)
tiered = sum(
litellm.cost_per_token(
model=model,
prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS,
completion_tokens=COMPLETION_TOKENS,
service_tier=tier,
)
)
assert tiered == pytest.approx(standard * ratio)

15
tests/test_litellm/test_replicate_model_key_format.py
View file

@ -18,18 +18,3 @@ def test_replicate_models_have_valid_key_prefix(model_cost: dict[str, Any]) -> N
assert not malformed, (
f"Replicate models must use 'replicate/owner/model' key format, found malformed keys: {malformed}"
)
def test_replicate_backup_matches_main() -> None:
repo_root = Path(__file__).parents[2]
main_path = repo_root / "model_prices_and_context_window.json"
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
with open(main_path, encoding="utf-8") as f:
main_cost: dict[str, Any] = json.load(f)
with open(backup_path, encoding="utf-8") as f:
backup_cost: dict[str, Any] = json.load(f)
for key in main_cost:
if main_cost[key].get("litellm_provider") == "replicate":
assert backup_cost.get(key) == main_cost.get(key), f"{key} differs between main and backup model cost maps"

25
tests/test_litellm/test_sambanova_model_metadata.py
View file

@ -1,25 +0,0 @@
import json
from pathlib import Path
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
def test_sambanova_minimax_m27_model_info():
model = "sambanova/MiniMax-M2.7"
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
with open(json_path) as f:
model_cost = json.load(f)
info = model_cost.get(model)
assert info is not None, f"{model} not found in model_prices_and_context_window.json"
assert info["litellm_provider"] == "sambanova"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] > 0
assert info["output_cost_per_token"] > 0
assert info["supports_function_calling"] is True
assert info["supports_reasoning"] is True
assert info["supports_tool_choice"] is True
routed_model, provider, _, _ = get_llm_provider(model=model)
assert routed_model == "MiniMax-M2.7"
assert provider == "sambanova"

9
tests/test_litellm/test_together_ai_model_metadata.py
View file

@ -5,7 +5,6 @@ from typing import Final
import pytest
from pydantic import TypeAdapter
REPO_ROOT: Final = Path(__file__).parents[2]
CostMap = dict[str, dict[str, object]]
@ -109,14 +108,6 @@ def test_together_successor_metadata_points_at_live_models(cost_map: CostMap):
assert "deprecation_date" not in target, f"{model} names deprecated successor {successor}"
def test_together_backup_cost_map_in_sync(cost_map: CostMap):
with open(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json") as f:
backup = COST_MAP_ADAPTER.validate_python(json.load(f))
together_main = {k: v for k, v in cost_map.items() if k.startswith("together_ai/")}
together_backup = {k: v for k, v in backup.items() if k.startswith("together_ai/")}
assert together_backup == together_main
CACHED_INPUT_MODELS: Final = (
"together_ai/moonshotai/Kimi-K3",
"together_ai/zai-org/GLM-5.2",

907
tests/test_litellm/test_utils.py
View file

File diff suppressed because it is too large Load diff

19
tests/test_litellm/test_xai_grok_4_3_model_metadata.py
View file

@ -1,19 +0,0 @@
import json
from pathlib import Path
def test_xai_grok_4_3_backup_matches_main():
"""Ensure the bundled model cost map stays in sync with the canonical file."""
repo_root = Path(__file__).parents[2]
main_path = repo_root / "model_prices_and_context_window.json"
backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
with open(main_path) as f:
main_cost = json.load(f)
with open(backup_path) as f:
backup_cost = json.load(f)
for model in ("xai/grok-4.3", "xai/grok-4.3-latest"):
assert backup_cost.get(model) == main_cost.get(
model
), f"{model} differs between main and backup model cost maps"