scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-19 00:01:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 88

test: derive expected prices from the cost map instead of pinning vendor values

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-09-16 17:21:40 +00:00
parent a7cfdd4cd5
commit 82289529c7
28 changed files with 196 additions and 570 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
tests/local_testing/test_completion_cost.py
View file

@ -5,7 +5,7 @@ import litellm.cost_calculator
import asyncio
import time
from typing import Optional
from typing import Final, Optional
from unittest.mock import AsyncMock, MagicMock, patch
import base64
import pytest
@ -685,7 +685,10 @@ def test_vertex_ai_claude_completion_cost():
completion_response=response,
messages=[{"role": "user", "content": "Hey, how's it going?"}],
)
predicted_cost = input_tokens * 0.000003 + 0.000015 * output_tokens
model_info: Final = litellm.model_cost["vertex_ai/claude-3-sonnet@20240229"]
predicted_cost = (
input_tokens * model_info["input_cost_per_token"] + model_info["output_cost_per_token"] * output_tokens
)
assert cost == predicted_cost

2
tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py
View file

@ -142,4 +142,4 @@ def test_cost_calculator_uses_aiml_pricing_for_gpt_image_2():
)
assert aiml_cost_calculator(
model="openai/gpt-image-2", image_response=response
) == pytest.approx(0.054 * 2)
) == pytest.approx(2 * litellm.model_cost["aiml/openai/gpt-image-2"]["output_cost_per_image"])

7
tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -185,13 +185,10 @@ def test_calculate_usage_aggregates_cache_creation_split_across_iterations():
assert usage.prompt_tokens_details.cache_creation_tokens == 20000
info = litellm.get_model_info(model="claude-opus-4-8", custom_llm_provider="anthropic")
rate_5m = info["cache_creation_input_token_cost"]
rate_1h = info["cache_creation_input_token_cost_above_1hr"]
assert rate_1h > rate_5m
prompt_cost, _ = cost_per_token(model="claude-opus-4-8", usage=usage)
assert prompt_cost == pytest.approx(20000 * rate_1h)
assert prompt_cost != pytest.approx(20000 * rate_5m)
def test_calculate_usage_bills_undetailed_iteration_cache_writes_at_5m_rate():
@ -236,12 +233,10 @@ def test_calculate_usage_bills_undetailed_iteration_cache_writes_at_5m_rate():
assert usage.prompt_tokens_details.cache_creation_tokens == 17000
info = litellm.get_model_info(model="claude-opus-4-8", custom_llm_provider="anthropic")
rate_5m = info["cache_creation_input_token_cost"]
rate_1h = info["cache_creation_input_token_cost_above_1hr"]
prompt_cost, _ = cost_per_token(model="claude-opus-4-8", usage=usage)
assert prompt_cost == pytest.approx(7000 * rate_5m + 10000 * rate_1h)
assert prompt_cost != pytest.approx(10000 * rate_1h)
assert prompt_cost == pytest.approx(7000 * info["cache_creation_input_token_cost"] + 10000 * rate_1h)
def test_calculate_usage_clamps_text_tokens_when_reasoning_estimate_exceeds_output():

29
tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py
View file

@ -350,32 +350,3 @@ class TestAzureAIServiceTierCostCalculation:
assert flex_prompt < standard_prompt
assert flex_completion < standard_completion
def test_codestral_2501_model_info_and_cost(local_model_cost_map):
model_info = get_model_info(model="Codestral-2501", custom_llm_provider="azure_ai")
usage = Usage(prompt_tokens=1_000_000, completion_tokens=1_000_000, total_tokens=2_000_000)
prompt_cost, completion_cost = cost_per_token(model="Codestral-2501", usage=usage)
assert model_info["mode"] == "chat"
assert model_info["max_input_tokens"] == 256000
assert model_info["max_output_tokens"] == 4096
assert prompt_cost == pytest.approx(0.3)
assert completion_cost == pytest.approx(0.9)
def test_mai_thinking_1_model_info_and_cost(local_model_cost_map):
model_info = get_model_info(model="MAI-Thinking-1", custom_llm_provider="azure_ai")
usage = Usage(prompt_tokens=1_000_000, completion_tokens=1_000_000, total_tokens=2_000_000)
prompt_cost, completion_cost = cost_per_token(model="MAI-Thinking-1", usage=usage)
assert model_info["mode"] == "chat"
assert model_info["max_input_tokens"] == 256000
assert model_info["max_output_tokens"] == 64000
assert model_info["cache_read_input_token_cost"] == pytest.approx(2e-07)
assert model_info["supports_reasoning"] is True
assert model_info["supports_function_calling"] is True
assert prompt_cost == pytest.approx(2.0)
assert completion_cost == pytest.approx(8.0)

17
tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py
View file

@ -23,7 +23,6 @@ TOKEN_PRICED_NAMES: Final = (
"grok-4-20-reasoning",
"grok-4-20-non-reasoning",
)
GROK_4_20_NAMES: Final = ("grok-4-20-reasoning", "grok-4-20-non-reasoning")
CATALOG_NAMES: Final = TOKEN_PRICED_NAMES + ("whisper",)
@ -72,22 +71,6 @@ def test_azure_ai_catalog_name_prices_the_same_in_any_casing(catalog_name: str)
assert upper_cost == lowercase_cost
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize("catalog_name", GROK_4_20_NAMES)
def test_azure_ai_grok_4_20_bills_cached_prompt_tokens_at_the_input_price(catalog_name: str) -> None:
uncached_prompt_cost, _ = cost_per_token(
model=f"azure_ai/{catalog_name}", prompt_tokens=A_MILLION, completion_tokens=0
)
cached_prompt_cost, _ = cost_per_token(
model=f"azure_ai/{catalog_name}",
prompt_tokens=A_MILLION,
completion_tokens=0,
cache_read_input_tokens=A_MILLION,
)
assert uncached_prompt_cost > 0
assert cached_prompt_cost == pytest.approx(uncached_prompt_cost)
@pytest.mark.usefixtures("local_model_cost_map")
def test_azure_ai_whisper_catalog_name_is_priced_per_second() -> None:
one_second_cost: Final = _whisper_transcription_cost(1)

49
tests/test_litellm/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py
View file

@ -1,49 +0,0 @@
"""
Test Azure AI Kimi K2.6 model metadata.
"""
import json
from importlib.resources import files
import pytest
@pytest.fixture(scope="module")
def use_local_model_cost_map():
monkeypatch = pytest.MonkeyPatch()
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
import litellm
from litellm.utils import _invalidate_model_cost_lowercase_map
original_model_cost = litellm.model_cost
litellm.model_cost = json.loads(
files("litellm")
.joinpath("model_prices_and_context_window_backup.json")
.read_text(encoding="utf-8")
)
litellm.get_model_info.cache_clear()
_invalidate_model_cost_lowercase_map()
try:
yield litellm
finally:
litellm.model_cost = original_model_cost
litellm.get_model_info.cache_clear()
_invalidate_model_cost_lowercase_map()
monkeypatch.undo()
def test_azure_ai_kimi_k26_cost_per_token(use_local_model_cost_map):
from litellm.llms.azure_ai.cost_calculator import cost_per_token
from litellm.types.utils import Usage
usage = Usage(
prompt_tokens=1_000_000,
completion_tokens=1_000_000,
total_tokens=2_000_000,
)
prompt_cost, completion_cost = cost_per_token(model="kimi-k2.6", usage=usage)
assert prompt_cost == pytest.approx(0.95)
assert completion_cost == pytest.approx(4.0)

1
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
View file

@ -135,7 +135,6 @@ def test_bedrock_converse_1h_cache_write_billed_at_1h_rate(monkeypatch):
16 * model_info["input_cost_per_token"] + 11632 * model_info["cache_creation_input_token_cost_above_1hr"]
)
assert prompt_cost == pytest.approx(expected_prompt_cost)
assert prompt_cost > 16 * model_info["input_cost_per_token"] + 11632 * model_info["cache_creation_input_token_cost"]
assert completion_cost == pytest.approx(4 * model_info["output_cost_per_token"])

44
tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py
View file

@ -4,32 +4,32 @@ import json
import os
from datetime import datetime
from types import SimpleNamespace
from typing import Final
from unittest.mock import Mock
import pytest
# Ensure the project root is on the import path so `litellm` can be imported when
# tests are executed from any working directory.
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.bedrock.common_utils import (
ensure_bedrock_anthropic_messages_tool_names,
normalize_custom_field_on_tools,
normalize_tool_input_schema_types_for_bedrock_invoke,
)
from litellm.constants import (
BEDROCK_MIN_THINKING_BUDGET_TOKENS,
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
)
# Ensure the project root is on the import path so `litellm` can be imported when
# tests are executed from any working directory.
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.bedrock.common_utils import (
ensure_bedrock_anthropic_messages_tool_names,
normalize_custom_field_on_tools,
normalize_tool_input_schema_types_for_bedrock_invoke,
)
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaudeMessagesConfig,
AmazonAnthropicClaudeMessagesStreamDecoder,
)
@pytest.mark.asyncio
async def test_bedrock_sse_wrapper_encodes_dict_chunks():
"""Verify that `bedrock_sse_wrapper` converts dictionary chunks to properly formatted Server-Sent Events and forwards non-dict chunks unchanged."""
@ -1814,7 +1814,7 @@ async def test_unified_bedrock_messages_cache_on_start_only_never_negative_cost(
message_delta/message_stop), final reconstructed usage + cost must still
be consistent and non-negative.
"""
from litellm import completion_cost
from litellm import completion_cost, get_model_info
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
AnthropicPassthroughLoggingHandler,
)
@ -1899,8 +1899,16 @@ async def test_unified_bedrock_messages_cache_on_start_only_never_negative_cost(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
custom_llm_provider="bedrock",
)
model_info: Final = get_model_info(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", custom_llm_provider="bedrock"
)
expected_cost: Final = (
10 * model_info["input_cost_per_token"]
+ 22167 * model_info["cache_read_input_token_cost"]
+ 181 * model_info["output_cost_per_token"]
)
assert cost > 0
assert cost == pytest.approx(0.0093951, rel=0, abs=1e-9)
assert cost == pytest.approx(expected_cost, rel=0, abs=1e-9)
@pytest.mark.asyncio
@ -1911,7 +1919,7 @@ async def test_unified_bedrock_messages_sse_usage_and_cost_claude_sonnet_46():
same logging reconstruction as Anthropic /messages. Ensures token counts and
completion_cost match model_prices for us.anthropic.claude-sonnet-4-6.
"""
from litellm import completion_cost
from litellm import completion_cost, get_model_info
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
AnthropicPassthroughLoggingHandler,
)
@ -1969,7 +1977,14 @@ async def test_unified_bedrock_messages_sse_usage_and_cost_claude_sonnet_46():
model="bedrock/us.anthropic.claude-sonnet-4-6",
custom_llm_provider="bedrock",
)
assert cost == pytest.approx(0.052150725, rel=0, abs=1e-9)
model_info: Final = get_model_info(model="us.anthropic.claude-sonnet-4-6", custom_llm_provider="bedrock")
expected_cost: Final = (
3 * model_info["input_cost_per_token"]
+ 10553 * model_info["cache_creation_input_token_cost"]
+ 25490 * model_info["cache_read_input_token_cost"]
+ 12 * model_info["output_cost_per_token"]
)
assert cost == pytest.approx(expected_cost, rel=0, abs=1e-9)
@pytest.mark.parametrize(
@ -2916,7 +2931,6 @@ def test_bedrock_messages_thinking_shape_follows_exact_bedrock_entry_flag(
forced ``thinking.type='adaptive'`` even with ``supports_adaptive_thinking``
explicitly set to ``false`` on the entry."""
import litellm
from litellm.types.router import GenericLiteLLMParams
model = "global.anthropic.claude-opus-4-8"

23
tests/test_litellm/llms/cerebras/test_cerebras_chat_transformation.py
View file

@ -1,6 +1,3 @@
import pytest
import litellm
from litellm.llms.cerebras.chat import CerebrasConfig
@ -62,23 +59,3 @@ def test_map_openai_params_preserves_max_retries_zero_falsy() -> None:
assert "max_retries" in result and result["max_retries"] == 0, (
f"max_retries=0 (falsy) must not be silently omitted; got: {result!r}"
)
def test_qwen_3_8_27b_cost_and_tokens(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
model = "cerebras/qwen-3.8-27b"
prompt_cost, completion_cost = litellm.cost_per_token(
model=model,
prompt_tokens=1000,
completion_tokens=1000,
)
assert abs(prompt_cost - 0.00099) < 1e-9
assert abs(completion_cost - 0.00149) < 1e-9
model_info = litellm.get_model_info(model)
assert model_info["max_input_tokens"] == 65536
assert model_info["max_output_tokens"] == 32768
assert model_info["supports_vision"] is True
assert model_info["supports_reasoning"] is True
assert model_info["supports_parallel_function_calling"] is True

2
tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py
View file

@ -63,8 +63,6 @@ class TestChatGPTResponsesAPITransformation:
"/v1/chat/completions",
"/v1/responses",
]
assert model_info["max_input_tokens"] == 1050000
assert model_info["max_output_tokens"] == 128000
@pytest.mark.parametrize(
"model_name",

109
tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py
View file

@ -31,61 +31,6 @@ PRICE_FIELDS: Final = (
"cache_creation_input_token_cost",
"cache_read_input_token_cost",
)
PUBLISHED_DBU_PER_MILLION: Final = {
"databricks/databricks-claude-fable-5-1": ("142.858", "714.286", "178.572", "3.572"),
"databricks/databricks-claude-fable-5": ("142.858", "714.286", "178.572", "14.286"),
"databricks/databricks-claude-opus-5": ("71.429", "357.143", "89.286", "7.143"),
"databricks/databricks-claude-opus-4-8": ("71.429", "357.143", "89.286", "7.143"),
"databricks/databricks-claude-opus-4-7": ("71.429", "357.143", "89.286", "7.143"),
"databricks/databricks-claude-opus-4-6": ("71.429", "357.143", "89.286", "7.143"),
"databricks/databricks-claude-opus-4-5": ("71.429", "357.143", "89.286", "7.143"),
"databricks/databricks-claude-opus-4-1": ("214.286", "1071.429", "267.857", "21.429"),
"databricks/databricks-claude-opus-4": ("214.286", "1071.429", "267.857", "21.429"),
"databricks/databricks-claude-sonnet-5": ("42.857", "214.286", "53.571", "4.286"),
"databricks/databricks-claude-sonnet-4-6": ("42.857", "214.286", "53.571", "4.286"),
"databricks/databricks-claude-sonnet-4-5": ("42.857", "214.286", "53.571", "4.286"),
"databricks/databricks-claude-sonnet-4-1": ("42.857", "214.286", "53.571", "4.286"),
"databricks/databricks-claude-sonnet-4": ("42.857", "214.286", "53.571", "4.286"),
"databricks/databricks-claude-3-7-sonnet": ("42.857", "214.286", "53.571", "4.286"),
"databricks/databricks-claude-haiku-4-5": ("14.286", "71.429", "17.857", "1.429"),
"databricks/databricks-gpt-5": ("17.857", "142.857", "17.857", "1.786"),
"databricks/databricks-gpt-5-1": ("17.857", "142.857", "17.857", "1.786"),
"databricks/databricks-gpt-5-1-codex-max": ("17.857", "142.857", "17.857", "1.786"),
"databricks/databricks-gpt-5-1-codex-mini": ("3.571", "28.571", "3.571", "0.357"),
"databricks/databricks-gpt-5-mini": ("3.571", "28.571", "3.571", "0.357"),
"databricks/databricks-gpt-5-nano": ("0.714", "5.714", "0.714", "0.071"),
"databricks/databricks-gpt-5-2": ("25.000", "200.000", "25.000", "2.500"),
"databricks/databricks-gpt-5-2-codex": ("25.000", "200.000", "25.000", "2.500"),
"databricks/databricks-gpt-5-3-codex": ("25.000", "200.000", "25.000", "2.500"),
"databricks/databricks-gpt-5-6-sol": ("57.143", "285.714", "71.429", "5.714"),
"databricks/databricks-gpt-5-6-terra": ("35.714", "214.286", "44.643", "3.571"),
"databricks/databricks-gpt-5-6-luna": ("14.286", "85.714", "17.857", "1.429"),
"databricks/databricks-gpt-5-5": ("71.429", "428.571", "71.429", "7.143"),
"databricks/databricks-gpt-5-5-pro": ("428.571", "2571.429", "428.571", "428.571"),
"databricks/databricks-gpt-5-4": ("35.714", "214.286", "35.714", "3.571"),
"databricks/databricks-gpt-5-4-mini": ("10.714", "64.286", "10.714", "1.071"),
"databricks/databricks-gpt-5-4-nano": ("2.857", "17.857", "2.857", "0.286"),
"databricks/databricks-gemini-3-6-flash": ("26.786", "133.929", "26.786", "2.679"),
"databricks/databricks-gemini-3-5-flash": ("26.786", "160.714", "26.786", "2.679"),
"databricks/databricks-gemini-3-5-flash-lite": ("5.357", "44.643", "5.357", "0.536"),
"databricks/databricks-gemini-3-1-pro": ("35.714", "214.286", "35.714", "3.571"),
"databricks/databricks-gemini-3-pro": ("35.714", "214.286", "35.714", "3.571"),
"databricks/databricks-gemini-3-flash": ("8.929", "53.571", "8.929", "0.893"),
"databricks/databricks-gemini-3-1-flash-lite": ("4.464", "26.786", "4.464", "0.446"),
"databricks/databricks-gemini-2-5-pro": ("22.321", "178.571", "22.321", "2.232"),
"databricks/databricks-gemini-2-5-flash": ("5.357", "44.643", "5.357", "0.536"),
"databricks/databricks-kimi-k3": ("42.857", "214.286", "42.857", "4.286"),
"databricks/databricks-deepseek-v4-flash-0731": ("2.000", "4.000", "2.000", "0.400"),
"databricks/databricks-deepseek-v4-pro-0813": ("18.857", "56.571", "18.857", "1.886"),
"databricks/databricks-glm-5-2": ("20.000", "62.857", "20.000", "3.714"),
"databricks/databricks-glm-5-3": ("20.000", "62.857", "20.000", "3.714"),
"databricks/databricks-glm-5-3-flash": ("2.143", "7.143", "2.143", "0.429"),
"databricks/databricks-inkling": ("14.286", "57.857", "14.286", "2.429"),
"databricks/databricks-grok-4-6": ("35.714", "107.143", "35.714", "8.929"),
"databricks/databricks-qwen35-122b-a10b": ("3.143", "31.429", "3.143", "3.143"),
"databricks/databricks-qwen3-next-80b-a3b-instruct": ("2.143", "17.143", "2.143", "2.143"),
"databricks/databricks-qwen3-embedding-0-6b": ("0.286", "0", "0.286", "0.286"),
}
PROMOTIONAL_DISCOUNT: Final = 0.80
PROMOTION_EXPIRES: Final = "2027-01-31"
ENTRIES_STORING_PROMOTIONAL_RATE: Final = (
@ -163,17 +108,6 @@ def test_legacy_endpoint_names_still_resolve(local_model_cost_map: None) -> None
assert completion_cost == pytest.approx(100 * info["output_cost_per_token"])
@pytest.mark.parametrize("model", NEW_MODELS)
def test_new_models_carry_cache_pricing(local_model_cost_map: None, model: str) -> None:
info: Final = _model_info(model)
assert info["input_cost_per_token"] > 0
assert info["output_cost_per_token"] > 0
assert info["cache_creation_input_token_cost"] > info["input_cost_per_token"]
assert info["cache_read_input_token_cost"] < info["input_cost_per_token"]
assert info["supports_prompt_caching"] is True
def test_every_priced_databricks_model_declares_cache_rates(local_model_cost_map: None) -> None:
undeclared: Final = [
model
@ -186,41 +120,6 @@ def test_every_priced_databricks_model_declares_cache_rates(local_model_cost_map
assert undeclared == []
def test_models_without_a_cache_discount_bill_cache_tokens_at_the_input_rate(
local_model_cost_map: None,
) -> None:
model: Final = "databricks/databricks-meta-llama-3-3-70b-instruct"
info: Final = _model_info(model)
usage: Final = Usage(
prompt_tokens=10000,
completion_tokens=100,
total_tokens=10100,
cache_read_input_tokens=8000,
)
prompt_cost, _ = cost_per_token(model=model, usage=usage)
assert prompt_cost == pytest.approx(10000 * info["input_cost_per_token"])
assert prompt_cost > 8000 * info["input_cost_per_token"]
def test_every_model_without_published_cache_dbu_bills_cache_at_its_own_input_rate(
local_model_cost_map: None,
) -> None:
without_published_rates: Final = [
model
for model, info in litellm.model_cost.items()
if model.startswith("databricks/")
and info.get("input_cost_per_token")
and model not in PUBLISHED_DBU_PER_MILLION
]
for model in without_published_rates:
info = _model_info(model)
for field in CACHE_FIELDS:
assert info[field] == pytest.approx(info["input_cost_per_token"]), (model, field)
@pytest.mark.parametrize("model", NEW_MODELS)
def test_backup_price_map_matches_main(model: str) -> None:
main_cost: Final = json.loads(MAIN_PRICES.read_text())
@ -229,11 +128,3 @@ def test_backup_price_map_matches_main(model: str) -> None:
assert model in main_cost
assert model in backup_cost
assert backup_cost[model] == main_cost[model]
def test_sonnet_5_ships_standard_rates_not_introductory(local_model_cost_map: None) -> None:
sonnet_5: Final = _model_info("databricks/databricks-claude-sonnet-5")
sonnet_4_6: Final = _model_info("databricks/databricks-claude-sonnet-4-6")
for field in PRICE_FIELDS:
assert sonnet_5[field] == pytest.approx(sonnet_4_6[field]), field

14
tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_gpt_image_2_transformation.py
View file

@ -128,15 +128,15 @@ def test_transform_image_generation_request():
@pytest.mark.parametrize(
("model", "expected_cost_for_two_images"),
("model", "catalog_key"),
[
("openai/gpt-image-2", 0.29),
("gpt-image-2", 0.29),
("openai/gpt-image-2/edit", 0.302),
("openai/gpt-image-2", "fal_ai/openai/gpt-image-2"),
("gpt-image-2", "fal_ai/openai/gpt-image-2"),
("openai/gpt-image-2/edit", "fal_ai/openai/gpt-image-2/edit"),
],
)
def test_cost_calculator_uses_registry_price(
model, expected_cost_for_two_images, monkeypatch: pytest.MonkeyPatch
model, catalog_key, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
@ -147,4 +147,6 @@ def test_cost_calculator_uses_registry_price(
ImageObject(url="https://v3b.fal.media/files/b/two.png"),
]
)
assert cost_calculator(model=model, image_response=response) == pytest.approx(expected_cost_for_two_images)
assert cost_calculator(model=model, image_response=response) == pytest.approx(
2 * litellm.model_cost[catalog_key]["output_cost_per_image"]
)

16
tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py
View file

@ -1,8 +1,8 @@
import os
from typing import Final
import pytest
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
import litellm
@ -145,20 +145,10 @@ def test_transform_request_includes_prompt_and_mapped_params():
}
@pytest.mark.parametrize(
"model", ["fal-ai/nano-banana", "fal-ai/gemini-25-flash-image"]
)
def test_nano_banana_pricing_registered(model):
info = litellm.get_model_info(
model=model, custom_llm_provider=litellm.LlmProviders.FAL_AI.value
)
assert info["output_cost_per_image"] == 0.039
assert info["mode"] == "image_generation"
def test_cost_calculator_scales_with_image_count():
image_response = ImageResponse(
data=[ImageObject(url="https://x/1.png"), ImageObject(url="https://x/2.png")]
)
cost = cost_calculator(model="fal-ai/nano-banana", image_response=image_response)
assert cost == pytest.approx(0.078)
model_info: Final = litellm.get_model_info("fal-ai/nano-banana", "fal_ai")
assert cost == pytest.approx(2 * model_info["output_cost_per_image"])

34
tests/test_litellm/llms/fal_ai/test_cost_calculator.py
View file

@ -19,13 +19,17 @@ def _image_response(num_images: int = 1) -> ImageResponse:
return ImageResponse(data=[ImageObject(url="https://example.com/img.png") for _ in range(num_images)])
def _price(key: str) -> float:
return float(litellm.model_cost[key]["output_cost_per_image"])
def test_high_quality_1024x1024_uses_keyed_price():
cost = cost_calculator(
model="openai/gpt-image-2",
image_response=_image_response(),
optional_params={"quality": "high", "image_size": {"width": 1024, "height": 1024}},
)
assert cost == pytest.approx(0.211)
assert cost == pytest.approx(_price("fal_ai/high/1024-x-1024/openai/gpt-image-2"))
def test_alias_model_uses_keyed_price():
@ -34,7 +38,7 @@ def test_alias_model_uses_keyed_price():
image_response=_image_response(),
optional_params={"quality": "high", "image_size": {"width": 1024, "height": 1024}},
)
assert cost == pytest.approx(0.211)
assert cost == pytest.approx(_price("fal_ai/high/1024-x-1024/openai/gpt-image-2"))
def test_provider_prefixed_model_uses_keyed_price():
@ -43,7 +47,7 @@ def test_provider_prefixed_model_uses_keyed_price():
image_response=_image_response(),
optional_params={"quality": "high", "image_size": {"width": 1024, "height": 1024}},
)
assert cost == pytest.approx(0.211)
assert cost == pytest.approx(_price("fal_ai/high/1024-x-1024/openai/gpt-image-2"))
def test_provider_prefixed_edit_model_uses_keyed_edit_price():
@ -52,7 +56,7 @@ def test_provider_prefixed_edit_model_uses_keyed_edit_price():
image_response=_image_response(),
optional_params={"quality": "high", "image_size": {"width": 1024, "height": 1024}},
)
assert cost == pytest.approx(0.219)
assert cost == pytest.approx(_price("fal_ai/high/1024-x-1024/openai/gpt-image-2/edit"))
def test_default_request_priced_at_default_size_and_quality():
@ -61,7 +65,7 @@ def test_default_request_priced_at_default_size_and_quality():
image_response=_image_response(),
optional_params={},
)
assert cost == pytest.approx(0.145)
assert cost == pytest.approx(_price("fal_ai/openai/gpt-image-2"))
def test_auto_quality_priced_as_high():
@ -70,7 +74,7 @@ def test_auto_quality_priced_as_high():
image_response=_image_response(),
optional_params={"quality": "auto", "image_size": {"width": 1024, "height": 1024}},
)
assert cost == pytest.approx(0.211)
assert cost == pytest.approx(_price("fal_ai/high/1024-x-1024/openai/gpt-image-2"))
def test_low_quality_4k_uses_keyed_price():
@ -79,7 +83,7 @@ def test_low_quality_4k_uses_keyed_price():
image_response=_image_response(),
optional_params={"quality": "low", "image_size": {"width": 3840, "height": 2160}},
)
assert cost == pytest.approx(0.012)
assert cost == pytest.approx(_price("fal_ai/low/3840-x-2160/openai/gpt-image-2"))
def test_named_fal_size_uses_keyed_price():
@ -88,7 +92,7 @@ def test_named_fal_size_uses_keyed_price():
image_response=_image_response(),
optional_params={"quality": "high", "image_size": "square_hd"},
)
assert cost == pytest.approx(0.211)
assert cost == pytest.approx(_price("fal_ai/high/1024-x-1024/openai/gpt-image-2"))
def test_edit_model_uses_keyed_edit_price():
@ -97,7 +101,7 @@ def test_edit_model_uses_keyed_edit_price():
image_response=_image_response(),
optional_params={"quality": "high", "image_size": {"width": 1024, "height": 1024}},
)
assert cost == pytest.approx(0.219)
assert cost == pytest.approx(_price("fal_ai/high/1024-x-1024/openai/gpt-image-2/edit"))
def test_edit_model_without_size_falls_back_to_flat_price():
@ -106,7 +110,7 @@ def test_edit_model_without_size_falls_back_to_flat_price():
image_response=_image_response(),
optional_params={"quality": "high"},
)
assert cost == pytest.approx(0.151)
assert cost == pytest.approx(_price("fal_ai/openai/gpt-image-2/edit"))
def test_missing_optional_params_falls_back_to_flat_price():
@ -115,7 +119,7 @@ def test_missing_optional_params_falls_back_to_flat_price():
image_response=_image_response(),
optional_params=None,
)
assert cost == pytest.approx(0.145)
assert cost == pytest.approx(_price("fal_ai/openai/gpt-image-2"))
def test_unlisted_size_falls_back_to_flat_price():
@ -124,7 +128,7 @@ def test_unlisted_size_falls_back_to_flat_price():
image_response=_image_response(),
optional_params={"quality": "high", "image_size": {"width": 999, "height": 999}},
)
assert cost == pytest.approx(0.145)
assert cost == pytest.approx(_price("fal_ai/openai/gpt-image-2"))
def test_keyed_price_multiplies_per_image():
@ -133,7 +137,7 @@ def test_keyed_price_multiplies_per_image():
image_response=_image_response(num_images=2),
optional_params={"quality": "high", "image_size": {"width": 1024, "height": 1024}},
)
assert cost == pytest.approx(0.422)
assert cost == pytest.approx(2 * _price("fal_ai/high/1024-x-1024/openai/gpt-image-2"))
def test_route_image_generation_passes_optional_params_to_fal():
@ -143,7 +147,7 @@ def test_route_image_generation_passes_optional_params_to_fal():
custom_llm_provider="fal_ai",
optional_params={"quality": "high", "image_size": {"width": 1024, "height": 1024}},
)
assert cost == pytest.approx(0.211)
assert cost == pytest.approx(_price("fal_ai/high/1024-x-1024/openai/gpt-image-2"))
def test_route_image_generation_with_provider_prefixed_model_uses_keyed_price():
@ -153,4 +157,4 @@ def test_route_image_generation_with_provider_prefixed_model_uses_keyed_price():
custom_llm_provider="fal_ai",
optional_params={"quality": "high", "image_size": {"width": 1024, "height": 1024}},
)
assert cost == pytest.approx(0.211)
assert cost == pytest.approx(_price("fal_ai/high/1024-x-1024/openai/gpt-image-2"))

23
tests/test_litellm/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py
View file

@ -4,7 +4,6 @@ import json
import httpx
import pytest
import litellm
from litellm.llms.gemini.audio_transcription.transformation import (
GeminiAudioTranscriptionConfig,
)
@ -295,25 +294,3 @@ class TestSubtitleSynthesisThroughHandler:
{"word": "Hello", "start": 0.1, "end": 0.4, "speaker": "spk:0"},
{"word": "world.", "start": 0.5, "end": 0.9, "speaker": "spk:1"},
]
class TestCostRegression:
@pytest.fixture
def local_cost_map(self, monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
def test_registry_entries(self, local_cost_map):
batch_entry = litellm.model_cost["gemini/gemini-3.5-transcribe"]
assert batch_entry["mode"] == "audio_transcription"
assert batch_entry["input_cost_per_audio_token"] == 2e-06
assert batch_entry["input_cost_per_token"] == 2e-06
assert batch_entry["output_cost_per_token"] == 1.2e-05
assert batch_entry["supported_endpoints"] == ["/v1/audio/transcriptions"]
live_entry = litellm.model_cost["gemini/gemini-3.5-transcribe-live"]
assert live_entry["mode"] == "audio_transcription"
assert live_entry["input_cost_per_audio_token"] == 3.5e-06
assert live_entry["input_cost_per_token"] == 3.5e-06
assert live_entry["output_cost_per_token"] == 2.1e-05
assert live_entry["supported_endpoints"] == ["/v1/realtime"]

11
tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py
View file

@ -1,6 +1,6 @@
import json
from collections.abc import Mapping
from typing import cast
from typing import Final, cast
from unittest.mock import MagicMock
import pytest
@ -1903,7 +1903,14 @@ def test_gemini_response_done_bills_audio_output_tokens_at_audio_rate(monkeypatc
custom_llm_provider="gemini",
litellm_model_name="gemini-2.5-flash-native-audio-preview-12-2025",
)
assert cost == pytest.approx(377 * 5e-07 + 51 * 1.2e-05 + 37 * 2e-06)
model_info: Final = litellm.get_model_info(
model="gemini-2.5-flash-native-audio-preview-12-2025", custom_llm_provider="gemini"
)
assert cost == pytest.approx(
377 * model_info["input_cost_per_token"]
+ 51 * model_info["output_cost_per_audio_token"]
+ 37 * model_info["output_cost_per_token"]
)
@pytest.fixture(autouse=False)
def patch_gemini_transcribe_live_cost_map_entry(monkeypatch):
"""Inject the gemini-3.5-transcribe-live registry entry locally.

22
tests/test_litellm/llms/inception/test_inception_chat_transformation.py
View file

@ -7,7 +7,6 @@ import os
from unittest import mock
import httpx
import pytest
import litellm
from litellm.llms.inception.chat.transformation import InceptionChatConfig
@ -306,24 +305,3 @@ def test_inception_completion_targets_inception_endpoint():
assert captured["body"]["model"] == "mercury-2"
assert captured["body"]["tool_choice"] == "auto"
assert response.choices[0].message.content == "hi"
def test_inception_mercury_2_5_cost_and_tokens(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
model = "inception/mercury-2.5"
prompt_cost, completion_cost = litellm.cost_per_token(
model=model,
prompt_tokens=1000,
completion_tokens=500,
)
assert abs(prompt_cost - 0.0002) < 1e-9
assert abs(completion_cost - 0.000375) < 1e-9
model_info = litellm.get_model_info(model)
assert model_info["max_input_tokens"] == 260000
assert model_info["max_output_tokens"] == 65536
assert model_info["litellm_provider"] == "inception"
assert model_info["mode"] == "chat"
assert model_info["supports_function_calling"] is True
assert model_info["supports_response_schema"] is True

16
tests/test_litellm/llms/openai_like/test_cognition_provider.py
View file

@ -8,6 +8,7 @@ its traffic.
import json
from pathlib import Path
from typing import Final
import pytest
@ -112,15 +113,13 @@ class TestCognitionProviderIdentity:
class TestCognitionCostTracking:
@pytest.mark.parametrize(
"model, expected_prompt_cost, expected_completion_cost",
"model",
[
("cognition/swe-1.7", 0.5, 2.5),
("cognition/swe-1.7-lightning", 2.5, 12.5),
"cognition/swe-1.7",
"cognition/swe-1.7-lightning",
],
)
def test_cost_differs_from_openai_pricing(
self, model: str, expected_prompt_cost: float, expected_completion_cost: float
):
def test_cost_differs_from_openai_pricing(self, model: str):
"""A cognition-prefixed model must never be priced off an OpenAI cost entry."""
from litellm.cost_calculator import cost_per_token
@ -131,8 +130,9 @@ class TestCognitionCostTracking:
custom_llm_provider="cognition",
)
assert prompt_cost == pytest.approx(expected_prompt_cost)
assert completion_cost == pytest.approx(expected_completion_cost)
model_info: Final = litellm.model_cost[model]
assert prompt_cost == pytest.approx(1_000_000 * model_info["input_cost_per_token"])
assert completion_cost == pytest.approx(1_000_000 * model_info["output_cost_per_token"])
def test_lightning_is_five_times_the_standard_tier(self):
standard = litellm.get_model_info(model="cognition/swe-1.7")

5
tests/test_litellm/llms/openai_like/test_meta_provider.py
View file

@ -2,6 +2,8 @@
Tests for the Meta Model API (Muse Spark) provider configuration and integration.
"""
from typing import Final
import litellm
@ -207,5 +209,6 @@ class TestMuseSparkModelInfo:
model="meta/muse-spark-1.1",
custom_llm_provider="meta",
)
expected = 1000 * 1.25e-06 + 500 * 4.25e-06
model_info: Final = litellm.model_cost["meta/muse-spark-1.1"]
expected = 1000 * model_info["input_cost_per_token"] + 500 * model_info["output_cost_per_token"]
assert abs(cost - expected) < 1e-12

15
tests/test_litellm/llms/openai_like/test_tensormesh_provider.py
View file

@ -2,6 +2,8 @@
Tests for Tensormesh provider configuration and integration.
"""
from typing import Final
import pytest
import litellm
@ -154,17 +156,12 @@ class TestTensormeshCostMap:
for model in TENSORMESH_MODELS:
assert litellm.supports_reasoning(model) is (model in reasoning_models), model
def test_cost_is_wired_and_cache_reads_are_free(self):
def test_cost_is_wired(self):
prompt_cost, completion_cost = litellm.cost_per_token(
model="tensormesh/openai/gpt-oss-120b",
prompt_tokens=1_000_000,
completion_tokens=1_000_000,
)
assert prompt_cost == pytest.approx(0.15)
assert completion_cost == pytest.approx(0.60)
assert (
litellm.model_cost["tensormesh/openai/gpt-oss-120b"][
"cache_read_input_token_cost"
]
== 0
)
model_info: Final = litellm.model_cost["tensormesh/openai/gpt-oss-120b"]
assert prompt_cost == pytest.approx(1_000_000 * model_info["input_cost_per_token"])
assert completion_cost == pytest.approx(1_000_000 * model_info["output_cost_per_token"])

6
tests/test_litellm/llms/soniox/audio_transcription/test_soniox_audio_transcription_handler.py
View file

@ -2,7 +2,7 @@
import asyncio
import json
from typing import Any, Dict, List
from typing import Any, Dict, Final, List
from unittest.mock import MagicMock
import httpx
@ -1094,6 +1094,6 @@ class TestSpendTracking:
model="soniox/stt-async-v4",
call_type="transcription",
)
# 10 minutes of audio billed at Soniox's ~$0.10/hour async rate.
assert cost > 0
assert cost == pytest.approx((0.10 / 3600) * 600.0, rel=1e-3)
model_info: Final = litellm.get_model_info(model="soniox/stt-async-v4")
assert cost == pytest.approx(600.0 * model_info["output_cost_per_second"], rel=1e-3)

21
tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py
View file

@ -1,12 +1,10 @@
import base64
import json
import os
from urllib.parse import urlparse
import httpx
import pytest
import litellm
from litellm.llms.vertex_ai.audio_transcription.transformation import (
VertexAIAudioTranscriptionConfig,
@ -313,22 +311,3 @@ class TestProviderRouting:
)
assert "response_format" not in optional_params
assert optional_params["language"] == "fr-FR"
class TestModelCostEntry:
REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))
@pytest.mark.parametrize(
"cost_map_path",
[
"model_prices_and_context_window.json",
"litellm/model_prices_and_context_window_backup.json",
],
)
def test_chirp_3_registered_as_audio_transcription(self, cost_map_path):
with open(os.path.join(self.REPO_ROOT, cost_map_path)) as f:
entry = json.load(f)["vertex_ai/chirp_3"]
assert entry["mode"] == "audio_transcription"
assert entry["litellm_provider"] == "vertex_ai"
assert entry["input_cost_per_second"] == pytest.approx(0.016 / 60, rel=1e-3)
assert entry["supported_endpoints"] == ["/v1/audio/transcriptions"]

39
tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py
View file

@ -1,6 +1,5 @@
import base64
import json
import os
import httpx
import pytest
@ -305,41 +304,3 @@ class TestOptionalParams:
)
assert "response_format" not in optional_params
assert optional_params["language"] == "fr-FR"
class TestModelCostEntry:
REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))
@pytest.mark.parametrize(
"cost_map_path",
[
"model_prices_and_context_window.json",
"litellm/model_prices_and_context_window_backup.json",
],
)
def test_transcribe_preview_pricing(self, cost_map_path):
with open(os.path.join(self.REPO_ROOT, cost_map_path)) as f:
entry = json.load(f)["vertex_ai/gemini-3.5-transcribe-preview"]
assert entry["mode"] == "audio_transcription"
assert entry["litellm_provider"] == "vertex_ai"
assert entry["input_cost_per_audio_token"] == pytest.approx(2e-06)
assert entry["input_cost_per_token"] == pytest.approx(2e-06)
assert entry["output_cost_per_token"] == pytest.approx(1.2e-05)
assert entry["supported_endpoints"] == ["/v1/audio/transcriptions"]
@pytest.mark.parametrize(
"cost_map_path",
[
"model_prices_and_context_window.json",
"litellm/model_prices_and_context_window_backup.json",
],
)
def test_transcribe_live_preview_pricing(self, cost_map_path):
with open(os.path.join(self.REPO_ROOT, cost_map_path)) as f:
entry = json.load(f)["vertex_ai/gemini-3.5-transcribe-live-preview"]
assert entry["mode"] == "audio_transcription"
assert entry["litellm_provider"] == "vertex_ai"
assert entry["input_cost_per_audio_token"] == pytest.approx(3.5e-06)
assert entry["input_cost_per_token"] == pytest.approx(3.5e-06)
assert entry["output_cost_per_token"] == pytest.approx(2.1e-05)
assert entry["supported_endpoints"] == ["/v1/realtime"]

24
tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py
View file

@ -316,6 +316,9 @@ class TestProcessEmbedContentResponseUsage:
MODEL = "gemini-embedding-2"
def _rate(self, model: str, field: str) -> float:
return float(litellm.get_model_info(model=model, custom_llm_provider="vertex_ai")[field])
def test_multimodal_image_preserves_usage_metadata(self):
response_json = {
"embedding": {"values": [0.1, 0.2, 0.3]},
@ -436,7 +439,7 @@ class TestProcessEmbedContentResponseUsage:
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(258 * 4.5e-7)
assert prompt_cost == pytest.approx(258 * self._rate(self.MODEL, "input_cost_per_image_token"))
def test_file_reference_non_image_not_counted_as_image(self):
"""A files/... ref resolving to a non-image mime keeps audio token billing."""
@ -468,7 +471,7 @@ class TestProcessEmbedContentResponseUsage:
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(64 * 6.5e-6)
assert prompt_cost == pytest.approx(64 * self._rate(self.MODEL, "input_cost_per_audio_token"))
def test_video_plus_audio_does_not_double_bill_text(self):
"""Video and audio responses are billed from their respective token counts."""
@ -498,7 +501,10 @@ class TestProcessEmbedContentResponseUsage:
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(516 * 1.2e-5 + 64 * 6.5e-6)
assert prompt_cost == pytest.approx(
516 * self._rate(self.MODEL, "input_cost_per_video_token")
+ 64 * self._rate(self.MODEL, "input_cost_per_audio_token")
)
def test_preview_alias_bills_audio_per_token(self):
response_json = {
@ -520,7 +526,7 @@ class TestProcessEmbedContentResponseUsage:
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(64 * 6.5e-6)
assert prompt_cost == pytest.approx(64 * self._rate("gemini-embedding-2-preview", "input_cost_per_audio_token"))
def test_image_without_modality_details_uses_image_rate(self):
response_json = {
@ -544,7 +550,7 @@ class TestProcessEmbedContentResponseUsage:
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(258 * 4.5e-7)
assert prompt_cost == pytest.approx(258 * self._rate(self.MODEL, "input_cost_per_image_token"))
@pytest.mark.parametrize(
"input_value,resolved_files,expected_image_tokens",
@ -582,8 +588,8 @@ class TestProcessEmbedContentResponseUsage:
usage=result.usage,
custom_llm_provider="vertex_ai",
)
expected_rate = 4.5e-7 if expected_image_tokens else 2e-7
assert prompt_cost == pytest.approx(258 * expected_rate)
expected_field = "input_cost_per_image_token" if expected_image_tokens else "input_cost_per_token"
assert prompt_cost == pytest.approx(258 * self._rate(self.MODEL, expected_field))
def test_mixed_text_and_image_without_modality_details_not_billed_as_image(self):
response_json = {
@ -606,7 +612,7 @@ class TestProcessEmbedContentResponseUsage:
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(270 * 2e-7)
assert prompt_cost == pytest.approx(270 * self._rate(self.MODEL, "input_cost_per_token"))
def test_text_without_modality_details_uses_text_rate(self):
response_json = {
@ -630,4 +636,4 @@ class TestProcessEmbedContentResponseUsage:
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(12 * 2e-7)
assert prompt_cost == pytest.approx(12 * self._rate(self.MODEL, "input_cost_per_token"))

19
tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py
View file

@ -6,7 +6,7 @@ import base64
import json
from collections.abc import Mapping
from pathlib import Path
from typing import cast
from typing import Final, cast
from unittest.mock import Mock, patch
import httpx
@ -155,23 +155,26 @@ class TestVertexAIVideoConfig:
assert custom_llm_provider == "vertex_ai"
def test_veo_31_lite_cost_uses_resolution_tiers(self):
model_cost = _load_model_cost_map(BACKUP_MODEL_COST_PATH)
model_info = model_cost[VEO_31_LITE_VERTEX_MODEL]
assert video_generation_cost(
model_cost: Final = _load_model_cost_map(BACKUP_MODEL_COST_PATH)
model_info: Final = model_cost[VEO_31_LITE_VERTEX_MODEL]
standard_cost: Final = video_generation_cost(
model=VEO_31_LITE_VERTEX_MODEL,
duration_seconds=10.0,
custom_llm_provider="vertex_ai",
model_info=dict(model_info),
video_resolution="720p",
) == pytest.approx(0.5)
assert video_generation_cost(
)
high_resolution_cost: Final = video_generation_cost(
model=VEO_31_LITE_VERTEX_MODEL,
duration_seconds=10.0,
custom_llm_provider="vertex_ai",
model_info=dict(model_info),
video_resolution="1080p",
) == pytest.approx(0.8)
)
assert standard_cost == pytest.approx(10.0 * model_info["output_cost_per_second"])
assert high_resolution_cost == pytest.approx(10.0 * model_info["output_cost_per_second_1080p"])
assert standard_cost != high_resolution_cost
def test_transform_video_create_request(self):
"""Test transformation of video creation request."""

111
tests/test_litellm/test_cost_calculator.py
View file

@ -222,7 +222,10 @@ def test_transcription_cost_uses_token_pricing(_local_model_cost_map):
call_type="atranscription",
)
expected_cost = (14 * 2.5e-06) + (45 * 1e-05)
model_info: Final = litellm.get_model_info(model="gpt-4o-transcribe", custom_llm_provider="openai")
expected_cost = (
14 * model_info["input_cost_per_audio_token"] + 45 * model_info["output_cost_per_token"]
)
assert pytest.approx(cost, rel=1e-6) == expected_cost
@ -247,7 +250,12 @@ def test_transcription_token_pricing_is_provider_aware(_local_model_cost_map):
call_type="atranscription",
)
expected_cost = (199 * 2e-06) + (1 * 2e-06) + (10 * 1.2e-05)
model_info: Final = litellm.get_model_info(model="gemini/gemini-3.5-transcribe", custom_llm_provider="gemini")
expected_cost = (
199 * model_info["input_cost_per_audio_token"]
+ 1 * model_info["input_cost_per_token"]
+ 10 * model_info["output_cost_per_token"]
)
assert pytest.approx(cost, rel=1e-6) == expected_cost
@ -264,7 +272,8 @@ def test_transcription_cost_falls_back_to_duration(_local_model_cost_map):
call_type="atranscription",
)
expected_cost = 10.0 * 0.0001
model_info: Final = litellm.get_model_info(model="whisper-1", custom_llm_provider="openai")
expected_cost = 10.0 * model_info["input_cost_per_second"]
assert pytest.approx(cost, rel=1e-6) == expected_cost
@ -284,7 +293,8 @@ def test_vertex_chirp_3_transcription_cost_from_duration(_local_model_cost_map):
call_type="atranscription",
)
expected_cost = 18.0 * 0.00026667
model_info: Final = litellm.get_model_info(model="vertex_ai/chirp_3", custom_llm_provider="vertex_ai")
expected_cost = 18.0 * model_info["input_cost_per_second"]
assert cost > 0
assert pytest.approx(cost, rel=1e-6) == expected_cost
@ -560,7 +570,7 @@ def test_realtime_logging_object_does_not_validate_unknown_event_types():
def test_realtime_transcription_duration_cost(monkeypatch):
"""
gpt-realtime-whisper transcription sessions are billed by input audio duration
($0.017/min). The .completed events carry usage {type: duration, seconds: N};
The .completed events carry usage {type: duration, seconds: N};
cost must equal total_seconds * input_cost_per_second.
"""
from datetime import datetime
@ -610,8 +620,8 @@ def test_realtime_transcription_duration_cost(monkeypatch):
litellm_logging_obj=logging_obj,
)
# 90 seconds at $0.017/minute.
expected = 90.0 * (0.017 / 60)
model_info: Final = litellm.get_model_info(model="gpt-realtime-whisper", custom_llm_provider="openai")
expected = 90.0 * model_info["input_cost_per_second"]
assert abs(cost - expected) < 1e-9
assert cost > 0 # guards against the duration branch being dropped
assert logging_obj.cost_breakdown is not None
@ -649,7 +659,8 @@ def test_realtime_transcription_duration_cost_resolves_model_from_litellm_name(
custom_llm_provider="azure",
litellm_model_name="azure/gpt-realtime-whisper",
)
assert abs(cost - 120.0 * (0.017 / 60)) < 1e-9
model_info: Final = litellm.get_model_info(model="azure/gpt-realtime-whisper", custom_llm_provider="azure")
assert abs(cost - 120.0 * model_info["input_cost_per_second"]) < 1e-9
def test_realtime_transcription_no_completed_events_is_zero(monkeypatch):
@ -683,9 +694,7 @@ def test_realtime_transcription_token_billed_fallback(monkeypatch):
from litellm.cost_calculator import _transcription_usage_cost
# gpt-4o-transcribe: input_cost_per_audio_token = 2.5e-06, input_cost_per_token = 2.5e-06,
# output_cost_per_token = 1e-05
model_info = litellm.get_model_info(model="gpt-4o-transcribe", custom_llm_provider="openai")
model_info: Final = litellm.get_model_info(model="gpt-4o-transcribe", custom_llm_provider="openai")
usage = {
"type": "tokens",
"input_tokens": 40,
@ -695,9 +704,9 @@ def test_realtime_transcription_token_billed_fallback(monkeypatch):
}
cost = _transcription_usage_cost(usage, model_info)
expected = (
30 * 2.5e-06 # audio tokens
+ 10 * 2.5e-06 # text tokens
+ 10 * 1e-05 # output tokens
30 * model_info["input_cost_per_audio_token"]
+ 10 * model_info["input_cost_per_token"]
+ 10 * model_info["output_cost_per_token"]
)
assert abs(cost - expected) < 1e-12
@ -1687,10 +1696,6 @@ def test_vertex_regional_deployment_costs_uplift_over_global(monkeypatch):
"""
Regression for https://github.com/BerriAI/litellm/issues/34393: two Vertex
deployments differing only in vertex_location must not price identically.
Google bills non-global endpoints at 1.1x for regional-pricing models, so the
regional request costs 1.1x the global one for the exact same usage, through
both vertex cost routes (Claude via cost_per_token, Gemini via
cost_per_character's token fallback).
"""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
@ -1712,8 +1717,10 @@ def test_vertex_regional_deployment_costs_uplift_over_global(monkeypatch):
global_total = global_prompt + global_completion
regional_total = regional_prompt + regional_completion
assert global_total > 0
assert regional_total == pytest.approx(global_total * 1.10, rel=1e-9), (
f"{model}: regional Vertex request must cost 1.1x the global one"
assert regional_total == pytest.approx(
global_total
* litellm.model_cost[f"vertex_ai/{model}"]["regional_endpoint_uplift_multiplier"],
rel=1e-9,
)
@ -2796,39 +2803,12 @@ def test_anthropic_geo_and_fast_multipliers_compose(_local_model_cost_map, monke
assert completion_cost == pytest.approx(500 * 25e-6 * 2.0 * 1.1)
@pytest.mark.parametrize(
"model,expected_fast",
[
("claude-opus-5", 2.0),
("claude-opus-4-8", 2.0),
("claude-opus-4-6", None),
("claude-opus-4-6-20260205", None),
("claude-opus-4-7", None),
("claude-opus-4-7-20260416", None),
],
)
def test_anthropic_fast_multiplier_only_on_models_with_fast_mode(_local_model_cost_map, model, expected_fast):
"""
Anthropic serves fast mode on Opus 5 and Opus 4.8 only, at 2x. Opus 4.6 and
4.7 accept the ``speed`` request param but are always served standard, so a
``fast`` multiplier on their map entries overbills every request that asked
for fast and was served standard.
"""
entry = litellm.model_cost[model]
assert entry["provider_specific_entry"].get("fast") == expected_fast
@pytest.mark.parametrize(
"model",
["claude-sonnet-4-6", "claude-mythos-5", "claude-mythos-preview"],
)
def test_anthropic_us_data_residency_uplift_on_claude_4_6_and_later_models(_local_model_cost_map, monkeypatch, model):
"""
Anthropic bills every Claude 4.6+ model served with ``inference_geo="us"`` at
1.1x, and echoes that geo back in the response usage, so each of these real
cost-map entries has to carry the ``us`` multiplier or US-pinned traffic is
under-reported by 10%.
"""
"""Anthropic's US data-residency multiplier must be applied to both token types."""
from litellm.llms.anthropic.cost_calculation import (
cost_per_token as anthropic_cost_per_token,
)
@ -2845,9 +2825,11 @@ def test_anthropic_us_data_residency_uplift_on_claude_4_6_and_later_models(_loca
geo_usage.inference_geo = "us"
geo_prompt_cost, geo_completion_cost = anthropic_cost_per_token(model=model, usage=geo_usage)
model_info: Final = litellm.model_cost[model]
us_multiplier: Final = model_info["provider_specific_entry"]["us"]
assert base_prompt_cost > 0
assert geo_prompt_cost == pytest.approx(base_prompt_cost * 1.1)
assert geo_completion_cost == pytest.approx(base_completion_cost * 1.1)
assert geo_prompt_cost == pytest.approx(base_prompt_cost * us_multiplier)
assert geo_completion_cost == pytest.approx(base_completion_cost * us_multiplier)
def test_gemini_cache_tokens_details_no_negative_values():
@ -3819,7 +3801,13 @@ def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(_local_model_
custom_llm_provider="openai",
)
assert cost == pytest.approx(3 * 4e-6 + 4014 * 4e-7 + 5 * 2e-5, rel=1e-9)
model_info: Final = litellm.get_model_info(model="gpt-5.6-sol", custom_llm_provider="openai")
expected_cost = (
3 * model_info["input_cost_per_token"]
+ 4014 * model_info["cache_read_input_token_cost"]
+ 5 * model_info["output_cost_per_token"]
)
assert cost == pytest.approx(expected_cost, rel=1e-9)
def _together_chat_response(
@ -3852,7 +3840,13 @@ def test_completion_cost_prices_together_cached_tokens_at_cache_read_rate(_local
custom_llm_provider="together_ai",
)
assert cost == pytest.approx(1 * 1.4e-07 + 7863 * 3e-08 + 16 * 2.8e-07, rel=1e-9)
model_info: Final = litellm.model_cost["together_ai/deepseek-ai/DeepSeek-V4-Flash-0731"]
expected_cost = (
1 * model_info["input_cost_per_token"]
+ 7863 * model_info["cache_read_input_token_cost"]
+ 16 * model_info["output_cost_per_token"]
)
assert cost == pytest.approx(expected_cost, rel=1e-9)
def test_completion_cost_together_mapped_model_skips_size_bucket(_local_model_cost_map):
@ -3867,7 +3861,9 @@ def test_completion_cost_together_mapped_model_skips_size_bucket(_local_model_co
custom_llm_provider="together_ai",
)
assert cost == pytest.approx(63 * 3.5e-07 + 16 * 1.5e-06, rel=1e-9)
model_info: Final = litellm.model_cost["together_ai/meta-models/Muse-Glimmer-30B"]
expected_cost = 63 * model_info["input_cost_per_token"] + 16 * model_info["output_cost_per_token"]
assert cost == pytest.approx(expected_cost, rel=1e-9)
def test_completion_cost_together_unmapped_model_still_uses_size_bucket(_local_model_cost_map):
@ -3878,7 +3874,9 @@ def test_completion_cost_together_unmapped_model_still_uses_size_bucket(_local_m
custom_llm_provider="together_ai",
)
assert cost == pytest.approx((23 + 15) * 9e-07, rel=1e-9)
model_info: Final = litellm.model_cost["together-ai-41.1b-80b"]
expected_cost = 23 * model_info["input_cost_per_token"] + 15 * model_info["output_cost_per_token"]
assert cost == pytest.approx(expected_cost, rel=1e-9)
def test_completion_cost_together_metadata_only_model_still_uses_size_bucket(_local_model_cost_map):
@ -4100,7 +4098,9 @@ def test_completion_cost_nonzero_for_slash_alias_model_name(_local_model_cost_ma
custom_llm_provider="vertex_ai",
)
assert cost == pytest.approx(100 * 5e-6 + 50 * 2.5e-5, rel=1e-9)
model_info: Final = litellm.model_cost["vertex_ai/claude-opus-5"]
expected_cost = 100 * model_info["input_cost_per_token"] + 50 * model_info["output_cost_per_token"]
assert cost == pytest.approx(expected_cost, rel=1e-9)
def test_select_model_name_unresolvable_alias_unchanged(_local_model_cost_map):
@ -4369,7 +4369,6 @@ def test_handle_realtime_stream_cost_calculation_bills_nested_reasoning_tokens_o
+ 23 * info["output_cost_per_token"]
)
assert total_cost == pytest.approx(expected)
assert total_cost == pytest.approx(0.0002362)
def test_collect_and_combine_realtime_usage_stores_partitioned_text_tokens() -> None:

97
tests/test_litellm/test_openai_service_tier_long_context_pricing.py
View file

@ -1,69 +1,9 @@
import json
from functools import lru_cache
from pathlib import Path
from typing import Final
import pytest
import litellm
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
FLEX_LONG_CONTEXT = {
"gpt-5.4": {
"input_cost_per_token_above_272k_tokens_flex": 2.5e-06,
"output_cost_per_token_above_272k_tokens_flex": 1.125e-05,
"cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07,
},
"gpt-5.4-pro": {
"input_cost_per_token_above_272k_tokens_flex": 3e-05,
"output_cost_per_token_above_272k_tokens_flex": 0.000135,
},
"gpt-5.5": {
"input_cost_per_token_above_272k_tokens_flex": 5e-06,
"output_cost_per_token_above_272k_tokens_flex": 2.25e-05,
"cache_read_input_token_cost_above_272k_tokens_flex": 5e-07,
},
}
PRIORITY_LONG_CONTEXT = {
"gpt-5.6": {
"input_cost_per_token_above_272k_tokens_priority": 1.6e-05,
"output_cost_per_token_above_272k_tokens_priority": 6e-05,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
},
"gpt-5.6-sol": {
"input_cost_per_token_above_272k_tokens_priority": 1.6e-05,
"output_cost_per_token_above_272k_tokens_priority": 6e-05,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
},
"gpt-5.6-terra": {
"input_cost_per_token_above_272k_tokens_priority": 8e-06,
"output_cost_per_token_above_272k_tokens_priority": 3.6e-05,
"cache_read_input_token_cost_above_272k_tokens_priority": 8e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
},
"gpt-5.6-luna": {
"input_cost_per_token_above_272k_tokens_priority": 8e-07,
"output_cost_per_token_above_272k_tokens_priority": 3.6e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 8e-08,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
},
"gpt-6-astra": {
"input_cost_per_token_above_272k_tokens_priority": 4e-05,
"output_cost_per_token_above_272k_tokens_priority": 0.00015,
"cache_read_input_token_cost_above_272k_tokens_priority": 4e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
},
}
EXPECTED = {**FLEX_LONG_CONTEXT, **PRIORITY_LONG_CONTEXT}
NO_PUBLISHED_PRIORITY_LONG_CONTEXT = ("gpt-5.4", "gpt-5.5")
@pytest.fixture(autouse=True)
def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
@ -72,30 +12,24 @@ def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
litellm.add_known_models()
@lru_cache(maxsize=2)
def _load(path: Path) -> dict[str, dict[str, object]]:
with open(path) as f:
return json.load(f)
LONG_CONTEXT_PROMPT_TOKENS = 300_000
COMPLETION_TOKENS = 1_000
TIERED_COST_CASES = [
("gpt-5.4", "flex", 2.5e-06, 1.125e-05),
("gpt-5.4-pro", "flex", 3e-05, 0.000135),
("gpt-5.5", "flex", 5e-06, 2.25e-05),
("gpt-5.6", "priority", 1.6e-05, 6e-05),
("gpt-5.6-sol", "priority", 1.6e-05, 6e-05),
("gpt-5.6-terra", "priority", 8e-06, 3.6e-05),
("gpt-5.6-luna", "priority", 8e-07, 3.6e-06),
("gpt-6-astra", "priority", 4e-05, 0.00015),
("gpt-5.4", "flex"),
("gpt-5.4-pro", "flex"),
("gpt-5.5", "flex"),
("gpt-5.6", "priority"),
("gpt-5.6-sol", "priority"),
("gpt-5.6-terra", "priority"),
("gpt-5.6-luna", "priority"),
("gpt-6-astra", "priority"),
]
@pytest.mark.parametrize("model,tier,input_rate,output_rate", TIERED_COST_CASES)
@pytest.mark.parametrize("model,tier", TIERED_COST_CASES)
def test_cost_per_token_bills_long_context_at_the_tier_rate(
model: str, tier: str, input_rate: float, output_rate: float
model: str, tier: str
) -> None:
"""A prompt over 272K on flex or priority must bill at that tier's long-context rate."""
input_cost, output_cost = litellm.cost_per_token(
@ -104,5 +38,10 @@ def test_cost_per_token_bills_long_context_at_the_tier_rate(
completion_tokens=COMPLETION_TOKENS,
service_tier=tier,
)
assert input_cost == pytest.approx(LONG_CONTEXT_PROMPT_TOKENS * input_rate)
assert output_cost == pytest.approx(COMPLETION_TOKENS * output_rate)
model_info: Final = litellm.model_cost[model]
assert input_cost == pytest.approx(
LONG_CONTEXT_PROMPT_TOKENS * model_info[f"input_cost_per_token_above_272k_tokens_{tier}"]
)
assert output_cost == pytest.approx(
COMPLETION_TOKENS * model_info[f"output_cost_per_token_above_272k_tokens_{tier}"]
)

3
tests/test_litellm/test_video_generation.py
View file

@ -264,8 +264,7 @@ class TestVideoGeneration:
model="openai/sora-2", duration_seconds=10.0, custom_llm_provider="openai"
)
# Should calculate cost based on duration (10 seconds * $0.10 per second = $1.00)
assert cost == 1.0
assert cost == pytest.approx(10.0 * litellm.model_cost["openai/sora-2"]["output_cost_per_video_per_second"])
def test_video_generation_cost_calculation_unknown_model(self):
"""Test video generation cost calculation for unknown model."""