scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 133

test: drop static cost-map value assertions

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-09-08 02:19:32 +00:00
parent 13df85cceb
commit 3023497590
54 changed files with 9 additions and 2664 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
tests/llm_translation/test_bedrock_embedding.py
View file

@ -1,14 +1,12 @@
import json
import os
from datetime import datetime
from unittest.mock import AsyncMock, Mock, patch
from unittest.mock import Mock, patch
import pytest
import base64
import httpx
import litellm
from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler
from litellm.llms.custom_httpx.http_handler import HTTPHandler
titan_embedding_response = {"embedding": [0.1, 0.2, 0.3], "inputTextTokenCount": 10}
@ -394,8 +392,6 @@ def test_bedrock_embedding_uses_correct_region_when_specified():
os.environ["AWS_REGION_NAME"] = original_region_name
else:
os.environ.pop("AWS_REGION_NAME", None)
def test_bedrock_embedding_region_bug_reproduction():
"""
Reproduces the bug where aws_region_name is ignored when passed explicitly.
@ -458,13 +454,3 @@ def test_bedrock_embedding_region_bug_reproduction():
os.environ["AWS_REGION_NAME"] = original_region_name
else:
os.environ.pop("AWS_REGION_NAME", None)
def test_bedrock_titan_g1_text_02_model_info():
"""Test that amazon.titan-embed-g1-text-02 has correct pricing metadata"""
model_info = litellm.get_model_info("amazon.titan-embed-g1-text-02")
assert model_info is not None, "Model info should not be None"
assert model_info["litellm_provider"] == "bedrock"
assert model_info["mode"] == "embedding"
assert model_info["input_cost_per_token"] == 1e-07
assert model_info["max_input_tokens"] == 8192

34
tests/llm_translation/test_bedrock_embedding_pricing.py
View file

@ -1,34 +0,0 @@
"""
Tests for AWS Bedrock embedding model pricing in the model cost map.
Regression test for the Amazon Titan Text Embeddings V2 commercial price,
which was previously set 10x too high (2e-07 instead of 2e-08).
AWS lists Titan Text Embeddings V2 at $0.02 per 1M input tokens
(= $0.00002 per 1K tokens = 2e-08 per token).
"""
import importlib
class TestBedrockEmbeddingPricing:
"""Test suite for Bedrock embedding model pricing in the cost map."""
def test_titan_embed_v2_commercial_input_cost(self, monkeypatch):
"""Titan Text Embeddings V2 should be priced at $0.02 / 1M tokens (2e-08)."""
# Scope the local-cost-map flag to this test only, so it does not leak
# into sibling tests. monkeypatch restores the environment on teardown.
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
import litellm.litellm_core_utils.get_model_cost_map
import litellm
# Reload so the cost map is re-read from the local file with the flag set.
importlib.reload(litellm.litellm_core_utils.get_model_cost_map)
importlib.reload(litellm)
model = litellm.model_cost["amazon.titan-embed-text-v2:0"]
assert model["input_cost_per_token"] == 2e-08
assert model["output_cost_per_token"] == 0.0
assert model["litellm_provider"] == "bedrock"
assert model["mode"] == "embedding"

158
tests/llm_translation/test_bedrock_govcloud.py
View file

@ -40,37 +40,6 @@ class TestBedrockGovCloudSupport:
assert "us-gov-east-1" in all_regions
assert "us-gov-west-1" in all_regions
def test_govcloud_models_in_model_cost(self):
"""Test that GovCloud models are present in model cost configuration"""
from litellm import model_cost
# Test Claude models in GovCloud
assert (
"bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0"
in model_cost
)
assert (
"bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0"
in model_cost
)
assert (
"bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0" in model_cost
)
assert (
"bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0" in model_cost
)
assert "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0" in model_cost
assert "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0" in model_cost
# Test Llama models in GovCloud
assert "bedrock/us-gov-east-1/meta.llama3-8b-instruct-v1:0" in model_cost
assert "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0" in model_cost
assert "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0" in model_cost
assert "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0" in model_cost
# Test Titan models in GovCloud
assert "bedrock/us-gov-east-1/amazon.titan-text-lite-v1" in model_cost
assert "bedrock/us-gov-west-1/amazon.titan-text-lite-v1" in model_cost
def test_govcloud_model_routing(self):
"""Test that GovCloud models are routed correctly"""
@ -148,134 +117,7 @@ class TestBedrockGovCloudSupport:
assert not any("us-gov-east-1" in model for model in litellm.bedrock_models)
assert not any("us-gov-west-1" in model for model in litellm.bedrock_models)
def test_govcloud_model_cost_properties(self):
"""Test that GovCloud models have proper cost configuration"""
from litellm import model_cost
# Check a specific GovCloud model has all required properties
govcloud_model = model_cost[
"bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0"
]
assert "max_tokens" in govcloud_model
assert "max_input_tokens" in govcloud_model
assert "max_output_tokens" in govcloud_model
assert "input_cost_per_token" in govcloud_model
assert "output_cost_per_token" in govcloud_model
assert govcloud_model["litellm_provider"] == "bedrock"
assert govcloud_model["mode"] == "chat"
def test_govcloud_model_pricing_verification(self):
"""Test that GovCloud models have correct pricing that differs from base models"""
from litellm import model_cost
# Claude Haiku 4.5 commercial list pricing is under the us.* inference profile id
base_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
gov_east_model = (
"bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0"
)
gov_west_model = (
"bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0"
)
# Verify base model pricing (us.* inference profile: $1.10/$5.50 per MTok)
base_pricing = model_cost[base_model]
assert base_pricing["input_cost_per_token"] == 1.1e-06
assert base_pricing["output_cost_per_token"] == 5.5e-06
# Verify GovCloud models have different (higher) pricing
gov_east_pricing = model_cost[gov_east_model]
gov_west_pricing = model_cost[gov_west_model]
# GovCloud models should have ~20% higher pricing than base models
assert gov_east_pricing["input_cost_per_token"] == 1.2e-06
assert gov_east_pricing["output_cost_per_token"] == 6e-06
assert gov_west_pricing["input_cost_per_token"] == 1.2e-06
assert gov_west_pricing["output_cost_per_token"] == 6e-06
# Verify the pricing difference is approximately 20%
assert (
abs(
gov_east_pricing["input_cost_per_token"]
/ base_pricing["input_cost_per_token"]
- 1.2
)
< 0.15
)
assert (
abs(
gov_east_pricing["output_cost_per_token"]
/ base_pricing["output_cost_per_token"]
- 1.2
)
< 0.15
)
assert (
abs(
gov_west_pricing["input_cost_per_token"]
/ base_pricing["input_cost_per_token"]
- 1.2
)
< 0.15
)
assert (
abs(
gov_west_pricing["output_cost_per_token"]
/ base_pricing["output_cost_per_token"]
- 1.2
)
< 0.15
)
# Test Claude 3 Haiku pricing
base_haiku_model = "anthropic.claude-3-haiku-20240307-v1:0"
gov_east_haiku_model = (
"bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0"
)
gov_west_haiku_model = (
"bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0"
)
# Verify base Haiku model pricing
base_haiku_pricing = model_cost[base_haiku_model]
assert base_haiku_pricing["input_cost_per_token"] == 2.5e-07 # 0.00000025
assert base_haiku_pricing["output_cost_per_token"] == 1.25e-06 # 0.00000125
# Verify GovCloud Haiku models have different (higher) pricing
gov_east_haiku_pricing = model_cost[gov_east_haiku_model]
gov_west_haiku_pricing = model_cost[gov_west_haiku_model]
# GovCloud Haiku models should have 20% higher pricing than base models
assert (
gov_east_haiku_pricing["input_cost_per_token"] == 3e-07
) # 0.0000003 (20% higher)
assert (
gov_east_haiku_pricing["output_cost_per_token"] == 1.5e-06
) # 0.0000015 (20% higher)
assert (
gov_west_haiku_pricing["input_cost_per_token"] == 3e-07
) # 0.0000003 (20% higher)
assert (
gov_west_haiku_pricing["output_cost_per_token"] == 1.5e-06
) # 0.0000015 (20% higher)
# Verify the pricing difference is exactly 20%
assert (
gov_east_haiku_pricing["input_cost_per_token"]
== base_haiku_pricing["input_cost_per_token"] * 1.2
)
assert (
gov_east_haiku_pricing["output_cost_per_token"]
== base_haiku_pricing["output_cost_per_token"] * 1.2
)
assert (
gov_west_haiku_pricing["input_cost_per_token"]
== base_haiku_pricing["input_cost_per_token"] * 1.2
)
assert (
gov_west_haiku_pricing["output_cost_per_token"]
== base_haiku_pricing["output_cost_per_token"] * 1.2
)
@patch("litellm.completion")
def test_govcloud_completion_cost_calculation(self, mock_completion):

36
tests/llm_translation/test_crusoe.py
View file

@ -4,7 +4,6 @@ Tests for Crusoe provider integration
import os
from unittest import mock
import litellm
CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1"
@ -71,38 +70,3 @@ def test_get_llm_provider_crusoe():
)
assert model == "meta-llama/Llama-3.3-70B-Instruct"
assert provider == "crusoe"
def test_crusoe_models_configuration():
"""Test that Crusoe models are configured correctly"""
from litellm import get_model_info
original_model_cost = litellm.model_cost
original_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
try:
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
crusoe_models = [
"crusoe/meta-llama/Llama-3.3-70B-Instruct",
"crusoe/deepseek-ai/DeepSeek-R1-0528",
"crusoe/deepseek-ai/DeepSeek-V3-0324",
"crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507",
"crusoe/moonshotai/Kimi-K2-Thinking",
"crusoe/openai/gpt-oss-120b",
"crusoe/google/gemma-3-12b-it",
]
for model in crusoe_models:
model_info = get_model_info(model)
assert model_info is not None, f"Model info not found for {model}"
assert model_info.get("litellm_provider") == "crusoe", (
f"{model} should have crusoe as provider"
)
assert model_info.get("mode") == "chat", f"{model} should be in chat mode"
finally:
litellm.model_cost = original_model_cost
if original_env is None:
os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None)
else:
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = original_env

30
tests/llm_translation/test_hyperbolic.py
View file

@ -1,8 +1,4 @@
import os
from datetime import datetime
from unittest.mock import MagicMock
import pytest
import litellm
@ -69,32 +65,6 @@ def test_hyperbolic_in_provider_lists():
assert "https://api.hyperbolic.xyz/v1" in openai_compatible_endpoints
def test_hyperbolic_models_configuration():
"""Test that Hyperbolic models are properly configured"""
import json
# Load model configuration directly from the JSON file
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path, "r") as f:
model_data = json.load(f)
# Test a few key models
test_models = [
"hyperbolic/deepseek-ai/DeepSeek-V3",
"hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct",
"hyperbolic/deepseek-ai/DeepSeek-R1",
]
for model in test_models:
assert model in model_data
model_info = model_data[model]
assert model_info["litellm_provider"] == "hyperbolic"
assert model_info["mode"] == "chat"
assert "max_tokens" in model_info
assert "input_cost_per_token" in model_info
assert "output_cost_per_token" in model_info
def test_hyperbolic_supported_params():

41
tests/llm_translation/test_lambda_ai.py
View file

@ -8,7 +8,6 @@ from unittest import mock
import pytest
import litellm
from litellm import completion
from litellm.llms.lambda_ai.chat.transformation import LambdaAIChatConfig
@ -103,46 +102,6 @@ async def test_lambda_ai_completion_call():
raise
def test_lambda_ai_models_configuration():
"""Test that Lambda AI models are configured correctly"""
from litellm import get_model_info
# Reload model cost map to pick up local changes
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
# Clear and repopulate lambda_ai_models list after reloading model_cost
litellm.lambda_ai_models = set()
litellm.add_known_models()
# Some Lambda AI models to test
lambda_ai_models = [
"lambda_ai/deepseek-llama3.3-70b",
"lambda_ai/hermes3-8b",
"lambda_ai/llama3.1-8b-instruct",
"lambda_ai/llama3.2-11b-vision-instruct",
"lambda_ai/qwen25-coder-32b-instruct",
]
for model in lambda_ai_models:
model_info = get_model_info(model)
assert model_info is not None, f"Model info not found for {model}"
assert (
model_info.get("litellm_provider") == "lambda_ai"
), f"{model} should have lambda_ai as provider"
assert model_info.get("mode") == "chat", f"{model} should be in chat mode"
assert (
model_info.get("supports_function_calling") is True
), f"{model} should support function calling"
assert (
model_info.get("supports_system_messages") is True
), f"{model} should support system messages"
# Check vision support for vision models
if "vision" in model:
assert (
model_info.get("supports_vision") is True
), f"{model} should support vision"
def test_lambda_ai_model_list_populated():

16
tests/llm_translation/test_morph.py
View file

@ -68,22 +68,6 @@ def test_morph_in_provider_lists():
)
def test_morph_model_info():
"""Test that morph models have correct configuration."""
import litellm
model_info = litellm.get_model_info("morph/morph-v3-large")
assert model_info["litellm_provider"] == "morph"
assert model_info["mode"] == "chat"
assert model_info["max_tokens"] == 16000
assert model_info["max_input_tokens"] == 16000
assert model_info["max_output_tokens"] == 16000
assert model_info["input_cost_per_token"] == 9e-07 # $0.9/1M tokens
assert model_info["output_cost_per_token"] == 1.9e-06 # $1.9/1M tokens
assert model_info["supports_function_calling"] is False
assert model_info["supports_vision"] is False
assert model_info["supports_system_messages"] is True
def test_morph_supported_params():

15
tests/llm_translation/test_openai_o1.py
View file

@ -1,15 +1,12 @@
import json
import os
from datetime import datetime
from unittest.mock import AsyncMock, patch, MagicMock
from unittest.mock import patch
import httpx
import pytest
import litellm
from litellm import Choices, Message, ModelResponse
from litellm import ModelResponse
from base_llm_unit_tests import BaseLLMChatTest, BaseOSeriesModelsTest
@ -74,7 +71,6 @@ async def test_o1_handle_tool_calling_optional_params(
- max_tokens is translated to 'max_completion_tokens'
- role 'system' is translated to 'user'
"""
from openai import AsyncOpenAI
from litellm.utils import ProviderConfigManager
from litellm.types.utils import LlmProviders
@ -186,13 +182,6 @@ class TestOpenAIO3(BaseOSeriesModelsTest, BaseLLMChatTest):
pass
def test_o1_supports_vision():
"""Test that o1 supports vision"""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
for k, v in litellm.model_cost.items():
if k.startswith("o1") and v.get("litellm_provider") == "openai":
assert v.get("supports_vision") is True, f"{k} does not support vision"
def test_o3_reasoning_effort():

31
tests/llm_translation/test_v0.py
View file

@ -8,7 +8,6 @@ from unittest import mock
import pytest
import litellm
from litellm import completion
from litellm.llms.v0.chat.transformation import V0ChatConfig
@ -111,33 +110,3 @@ def test_v0_supported_params():
]
assert set(supported_params) == set(expected_params)
def test_v0_models_configuration():
"""Test that v0 models are configured correctly"""
from litellm import get_model_info
# Reload model cost map to pick up local changes
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
# All v0 models
v0_models = ["v0/v0-1.0-md", "v0/v0-1.5-md", "v0/v0-1.5-lg"]
for model in v0_models:
model_info = get_model_info(model)
assert model_info is not None, f"Model info not found for {model}"
# All v0 models support vision (multimodal)
assert (
model_info.get("supports_vision") is True
), f"{model} should support vision"
assert (
model_info.get("litellm_provider") == "v0"
), f"{model} should have v0 as provider"
assert model_info.get("mode") == "chat", f"{model} should be in chat mode"
assert (
model_info.get("supports_function_calling") is True
), f"{model} should support function calling"
assert (
model_info.get("supports_system_messages") is True
), f"{model} should support system messages"

26
tests/local_testing/test_get_model_info.py
View file

@ -1,8 +1,6 @@
# What is this?
## Unit testing for the 'get_model_info()' function
import os
import traceback
import json
from typing import List, Dict, Any
@ -11,7 +9,7 @@ import pytest
import litellm
from litellm import get_model_info
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import MagicMock, patch
def test_get_model_info_simple_model_name():
@ -49,32 +47,12 @@ def test_get_model_info_custom_llm_with_same_name_vllm(monkeypatch):
assert model_info["input_cost_per_token"] == 0.0
def test_get_model_info_shows_correct_supports_vision():
info = litellm.get_model_info("gemini/gemini-2.0-flash")
print("info", info)
assert info["supports_vision"] is True
def test_get_model_info_shows_assistant_prefill():
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
info = litellm.get_model_info("deepseek/deepseek-chat")
print("info", info)
assert info.get("supports_assistant_prefill") is True
def test_get_model_info_shows_supports_prompt_caching():
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
info = litellm.get_model_info("deepseek/deepseek-chat")
print("info", info)
assert info.get("supports_prompt_caching") is True
def test_get_model_info_finetuned_models():
info = litellm.get_model_info("ft:gpt-3.5-turbo:my-org:custom_suffix:id")
print("info", info)
assert info["input_cost_per_token"] == 0.000003
def test_get_model_info_gemini_pro():
@ -219,7 +197,7 @@ def test_model_info_bedrock_converse_enforcement(monkeypatch):
def test_get_model_info_custom_provider():
# Custom provider example copied from https://docs.litellm.ai/docs/providers/custom_llm_server:
import litellm
from litellm import CustomLLM, completion, get_llm_provider
from litellm import CustomLLM, completion
class MyCustomLLM(CustomLLM):
def completion(self, *args, **kwargs) -> litellm.ModelResponse:

7
tests/test_litellm/litellm_core_utils/test_xai_oauth_routing.py
View file

@ -1,6 +1,5 @@
import litellm
from litellm import LlmProviders
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.get_llm_provider_logic import (
@ -46,12 +45,6 @@ def test_xai_openai_compatible_provider_info():
assert dynamic_api_key == "api-key"
def test_xai_get_model_info_uses_xai_pricing_metadata():
model_info = litellm.get_model_info("xai/grok-3-mini")
assert model_info["litellm_provider"] == "xai"
assert model_info["key"] == "xai/grok-3-mini"
assert model_info["mode"] == "chat"
def test_xai_validate_environment_reads_api_key(monkeypatch):

19
tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py
View file

@ -1,4 +1,3 @@
import os
from unittest.mock import MagicMock
import httpx
@ -38,24 +37,6 @@ class TestAzureMAIImageGeneration:
assert not AzureFoundryMAIImageGenerationConfig.is_mai_model("flux.2-pro")
assert not AzureFoundryMAIImageGenerationConfig.is_mai_model("MAI-DS-R1")
def test_mai_flash_and_2e_model_pricing_in_cost_map(self, monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
flash_info = litellm.get_model_info(
model="azure_ai/MAI-Image-2.5-Flash",
custom_llm_provider="azure_ai",
)
assert flash_info["input_cost_per_token"] == 1.75e-06
assert flash_info["input_cost_per_image_token"] == 1.75e-06
assert flash_info["output_cost_per_image_token"] == 3.3e-05
image_2e_info = litellm.get_model_info(
model="azure_ai/MAI-Image-2e",
custom_llm_provider="azure_ai",
)
assert image_2e_info["input_cost_per_token"] == 5e-06
assert image_2e_info["output_cost_per_image_token"] == 1.95e-05
def test_get_mai_image_generation_url(self):
url = AzureFoundryMAIImageGenerationConfig.get_mai_image_generation_url(

139
tests/test_litellm/llms/azure_ai/test_azure_ai_fw_models_metadata.py
View file

@ -12,111 +12,6 @@ from importlib.resources import files
import pytest
FW_MODELS = {
"azure_ai/FW-Kimi-K2.5": {
"input_cost_per_token": 6.6e-07,
"output_cost_per_token": 3.3e-06,
"cache_read_input_token_cost": 1.1e-07,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"supports_vision": True,
},
"azure_ai/FW-Kimi-K2.6": {
"input_cost_per_token": 1.045e-06,
"output_cost_per_token": 4.4e-06,
"cache_read_input_token_cost": 1.76e-07,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"supports_vision": True,
},
"azure_ai/FW-Kimi-K2.7-Code": {
"input_cost_per_token": 1.05e-06,
"output_cost_per_token": 4.4e-06,
"cache_read_input_token_cost": 2.1e-07,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"supports_vision": True,
},
"azure_ai/FW-Kimi-K3": {
"input_cost_per_token": 3.3e-06,
"output_cost_per_token": 1.65e-05,
"cache_read_input_token_cost": 3.3e-07,
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"supports_vision": True,
},
"azure_ai/FW-Inkling": {
"input_cost_per_token": 1e-06,
"output_cost_per_token": 4.05e-06,
"cache_read_input_token_cost": 1.7e-07,
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
},
"azure_ai/FW-DeepSeek-V3.2": {
"input_cost_per_token": 6.2e-07,
"output_cost_per_token": 1.85e-06,
"cache_read_input_token_cost": 3.1e-07,
"max_input_tokens": 163840,
"max_output_tokens": 163840,
},
"azure_ai/FW-DeepSeek-V4-Pro": {
"input_cost_per_token": 1.925e-06,
"output_cost_per_token": 3.828e-06,
"cache_read_input_token_cost": 1.65e-07,
"max_input_tokens": 1000000,
"max_output_tokens": 384000,
},
"azure_ai/FW-MiniMax-M3": {
"input_cost_per_token": 3.3e-07,
"output_cost_per_token": 1.32e-06,
"cache_read_input_token_cost": 6.6e-08,
"max_input_tokens": 512000,
"max_output_tokens": 512000,
"supports_vision": True,
},
"azure_ai/FW-MiniMax-M2.5": {
"input_cost_per_token": 3.3e-07,
"output_cost_per_token": 1.32e-06,
"cache_read_input_token_cost": 3.3e-08,
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
},
"azure_ai/FW-Nemotron-3-Ultra-NVFP4": {
"input_cost_per_token": 6e-07,
"output_cost_per_token": 2.4e-06,
"cache_read_input_token_cost": 1.19e-07,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
},
"azure_ai/FW-GLM-5.2-Fast": {
"input_cost_per_token": 2.1e-06,
"output_cost_per_token": 6.6e-06,
"cache_read_input_token_cost": 2.1e-07,
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
},
"azure_ai/FW-GLM-5.2": {
"input_cost_per_token": 1.54e-06,
"output_cost_per_token": 4.84e-06,
"cache_read_input_token_cost": 1.5e-07,
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
},
"azure_ai/FW-GLM-5.1": {
"input_cost_per_token": 1.54e-06,
"output_cost_per_token": 4.84e-06,
"cache_read_input_token_cost": 2.86e-07,
"max_input_tokens": 202800,
"max_output_tokens": 131072,
},
"azure_ai/FW-GLM-5": {
"input_cost_per_token": 1.1e-06,
"output_cost_per_token": 3.52e-06,
"cache_read_input_token_cost": 2.2e-07,
"max_input_tokens": 200000,
"max_output_tokens": 128000,
},
}
@pytest.fixture(scope="module")
@ -144,26 +39,6 @@ def use_local_model_cost_map():
monkeypatch.undo()
@pytest.mark.parametrize("model_key,expected", list(FW_MODELS.items()))
def test_azure_ai_fw_model_info(use_local_model_cost_map, model_key, expected):
model_info = use_local_model_cost_map.get_model_info(model=model_key)
assert model_info["litellm_provider"] == "azure_ai"
assert model_info["mode"] == "chat"
assert model_info["input_cost_per_token"] == pytest.approx(expected["input_cost_per_token"])
assert model_info["output_cost_per_token"] == pytest.approx(expected["output_cost_per_token"])
assert model_info["cache_read_input_token_cost"] == pytest.approx(
expected["cache_read_input_token_cost"]
)
assert model_info["max_input_tokens"] == expected["max_input_tokens"]
assert model_info["max_output_tokens"] == expected["max_output_tokens"]
assert model_info["max_tokens"] == expected["max_output_tokens"]
assert model_info["supports_function_calling"] is True
assert model_info["supports_reasoning"] is True
assert model_info["supports_tool_choice"] is True
assert model_info["supports_prompt_caching"] is True
if expected.get("supports_vision"):
assert model_info["supports_vision"] is True
@pytest.mark.parametrize(
@ -197,20 +72,6 @@ def test_azure_ai_fw_cost_per_token(
assert completion_cost == pytest.approx(expected_completion)
def test_azure_ai_fw_nemotron_lightning_model_info(use_local_model_cost_map):
model_info = use_local_model_cost_map.get_model_info(model="azure_ai/FW-Nemotron-Lightning-3.5-30B-A3B")
assert model_info["litellm_provider"] == "azure_ai"
assert model_info["mode"] == "chat"
assert model_info["input_cost_per_token"] == pytest.approx(6e-08)
assert model_info["output_cost_per_token"] == pytest.approx(2.2e-07)
assert model_info["cache_read_input_token_cost"] == pytest.approx(1e-08)
assert model_info["max_input_tokens"] == 262144
assert model_info["supports_function_calling"] is True
assert model_info["supports_reasoning"] is True
assert model_info["supports_tool_choice"] is True
assert model_info["supports_prompt_caching"] is True
assert model_info["supports_vision"] is False
def test_azure_ai_fw_nemotron_lightning_supports_tool_choice(use_local_model_cost_map):

23
tests/test_litellm/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py
View file

@ -33,31 +33,8 @@ def use_local_model_cost_map():
monkeypatch.undo()
def test_azure_ai_kimi_k26_model_info(use_local_model_cost_map):
model_info = use_local_model_cost_map.get_model_info(model="azure_ai/kimi-k2.6")
assert model_info["litellm_provider"] == "azure_ai"
assert model_info["mode"] == "chat"
assert model_info["max_input_tokens"] == 262144
assert model_info["max_output_tokens"] == 262144
assert model_info["max_tokens"] == 262144
assert model_info["input_cost_per_token"] == pytest.approx(9.5e-07)
assert model_info["output_cost_per_token"] == pytest.approx(4e-06)
assert model_info["supports_function_calling"] is True
assert model_info["supports_reasoning"] is True
assert model_info["supports_tool_choice"] is True
assert model_info["supports_vision"] is True
def test_azure_ai_kimi_k26_raw_model_cost_entry(use_local_model_cost_map):
model_info = use_local_model_cost_map.model_cost["azure_ai/kimi-k2.6"]
assert model_info["supported_modalities"] == ["text", "image"]
assert model_info["supported_output_modalities"] == ["text"]
assert model_info["supports_function_calling"] is True
assert model_info["supports_reasoning"] is True
assert model_info["supports_tool_choice"] is True
assert model_info["supports_vision"] is True
def test_azure_ai_kimi_k26_cost_per_token(use_local_model_cost_map):

70
tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py
View file

@ -1,8 +1,5 @@
"""Test Bedrock cross-region inference profile model mapping"""
import json
from functools import lru_cache
from pathlib import Path
from typing import NamedTuple
import pytest
@ -102,11 +99,6 @@ GPT_5_6_PROFILES = [
]
@lru_cache(maxsize=1)
def _packaged_cost_map():
"""The map litellm actually resolves against, for fields ModelInfoBase drops."""
path = Path(litellm.__file__).parent / "model_prices_and_context_window_backup.json"
return json.loads(path.read_text())
def _bedrock_response(model, usage):
@ -126,15 +118,6 @@ def _bedrock_response(model, usage):
)
def test_bedrock_cross_region_inference_profile_mapping():
"""Test that bedrock cross-region inference profile model is mapped"""
model = "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0"
model_info = _get_model_info_helper(model=model, custom_llm_provider="bedrock")
assert model_info is not None
assert model_info["litellm_provider"] == "bedrock"
assert model_info["input_cost_per_token"] == 8e-07
def test_proxy_cost_calculation_scenario():
@ -176,36 +159,6 @@ def test_bedrock_gpt_5_6_profiles_route_to_converse(profile, local_model_cost_ma
assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}") == "converse"
@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)
def test_bedrock_gpt_5_6_published_rates(profile, local_model_cost_map):
"""Geo and Global profiles carry their own published rates, per context tier."""
model_info = _get_model_info_helper(
model=f"bedrock/{profile.model_id}", custom_llm_provider="bedrock"
)
assert model_info["litellm_provider"] == "bedrock_converse"
assert model_info["mode"] == "chat"
assert model_info["max_input_tokens"] == 1000000
assert model_info["input_cost_per_token"] == profile.input_cost
assert (
model_info["input_cost_per_token_above_272k_tokens"]
== profile.input_cost_above_272k
)
assert model_info["output_cost_per_token"] == profile.output_cost
assert (
model_info["output_cost_per_token_above_272k_tokens"]
== profile.output_cost_above_272k
)
assert model_info["cache_creation_input_token_cost"] == profile.cache_write
assert (
model_info["cache_creation_input_token_cost_above_272k_tokens"]
== profile.cache_write_above_272k
)
assert model_info["cache_read_input_token_cost"] == profile.cache_read
assert (
model_info["cache_read_input_token_cost_above_272k_tokens"]
== profile.cache_read_above_272k
)
def test_bedrock_gpt_5_6_above_272k_tier_applies_to_cost(local_model_cost_map):
@ -267,29 +220,6 @@ def test_bedrock_gpt_5_6_bills_cache_write_tokens(local_model_cost_map):
assert cost == pytest.approx(expected, rel=1e-9)
@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)
def test_bedrock_gpt_5_6_advertises_only_converse_supported_features(
profile, local_model_cost_map
):
model_info = _get_model_info_helper(
model=f"bedrock/{profile.model_id}", custom_llm_provider="bedrock"
)
assert model_info["supports_function_calling"] is True
assert model_info["supports_tool_choice"] is True
assert model_info["supports_vision"] is True
# Bedrock rejects an explicit cachePoint block for these models, so the flag that
# offers caller-driven caching stays off even though the cache rates are declared.
assert not model_info.get("supports_prompt_caching")
# ModelInfoBase drops these two, so they are read from the map litellm resolves.
raw = _packaged_cost_map()[profile.model_id]
assert raw["supported_modalities"] == ["text", "image"]
assert raw["supported_output_modalities"] == ["text"]
# No bedrock_converse entry declares supported_endpoints; these models are reachable
# on chat completions and on the Responses API without it.
assert "supported_endpoints" not in raw
@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)

101
tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py
View file

@ -8,9 +8,7 @@ gate, the URL construction for both paths, and the shared Bearer auth.
"""
import copy
import json
import logging
from pathlib import Path
import pytest
from botocore.exceptions import (
@ -1777,53 +1775,9 @@ class TestBedrockMantleResponsesSigV4:
class TestBedrockMantleResponsesPricing:
def test_gpt_5_5_pricing_and_mode(self, local_cost_map):
info = litellm.get_model_info("bedrock_mantle/openai.gpt-5.5")
assert info["mode"] == "responses"
assert info["input_cost_per_token"] == pytest.approx(5.5e-06)
assert info["output_cost_per_token"] == pytest.approx(3.3e-05)
assert info["cache_read_input_token_cost"] == pytest.approx(5.5e-07)
assert info["max_input_tokens"] == 1050000
def test_gpt_5_4_pricing_and_mode(self, local_cost_map):
info = litellm.get_model_info("bedrock_mantle/openai.gpt-5.4")
assert info["mode"] == "responses"
assert info["input_cost_per_token"] == pytest.approx(2.75e-06)
assert info["output_cost_per_token"] == pytest.approx(1.65e-05)
assert info["cache_read_input_token_cost"] == pytest.approx(2.75e-07)
assert info["max_input_tokens"] == 1050000
def test_gpt_5_6_cyber_pricing_and_mode(self, local_cost_map):
info = litellm.get_model_info("bedrock_mantle/openai.gpt-5.6-cyber")
assert info["mode"] == "responses"
assert info["input_cost_per_token"] == pytest.approx(1.375e-05)
assert info["cache_creation_input_token_cost"] == pytest.approx(1.71875e-05)
assert info["cache_read_input_token_cost"] == pytest.approx(1.375e-06)
assert info["output_cost_per_token"] == pytest.approx(8.25e-05)
assert info["max_input_tokens"] == 272000
@pytest.mark.parametrize(
"model, input_cost, cache_creation_cost, cache_read_cost, output_cost",
[
("openai.gpt-5.6-sol", 5.5e-06, 6.875e-06, 5.5e-07, 3.3e-05),
("openai.gpt-5.6-terra", 2.2e-06, 2.75e-06, 2.2e-07, 1.32e-05),
("openai.gpt-5.6-luna", 2.2e-07, 2.75e-07, 2.2e-08, 1.32e-06),
],
)
def test_gpt_5_6_pricing_and_mode(
self, local_cost_map, model, input_cost, cache_creation_cost, cache_read_cost, output_cost
):
info = litellm.get_model_info(f"bedrock_mantle/{model}")
assert info["mode"] == "responses"
assert info["input_cost_per_token"] == pytest.approx(input_cost)
assert info["cache_creation_input_token_cost"] == pytest.approx(cache_creation_cost)
assert info["cache_read_input_token_cost"] == pytest.approx(cache_read_cost)
assert info["output_cost_per_token"] == pytest.approx(output_cost)
assert info["max_input_tokens"] == 1050000
assert info["input_cost_per_token_above_272k_tokens"] == pytest.approx(input_cost * 2)
assert info["cache_creation_input_token_cost_above_272k_tokens"] == pytest.approx(cache_creation_cost * 2)
assert info["cache_read_input_token_cost_above_272k_tokens"] == pytest.approx(cache_read_cost * 2)
assert info["output_cost_per_token_above_272k_tokens"] == pytest.approx(output_cost * 1.5)
@pytest.mark.parametrize(
"model, input_cost, output_cost",
@ -1861,58 +1815,3 @@ class TestBedrockMantleResponsesPricing:
def test_models_registered(self, local_cost_map):
assert "bedrock_mantle/openai.gpt-5.5" in litellm.bedrock_mantle_models
assert "bedrock_mantle/openai.gpt-5.4" in litellm.bedrock_mantle_models
def _repo_cost_map(map_name: str) -> dict[str, dict[str, object]]:
repo_root = Path(__file__).resolve().parents[4]
paths = {
"root": repo_root / "model_prices_and_context_window.json",
"bundled_backup": repo_root / "litellm" / "model_prices_and_context_window_backup.json",
}
return json.loads(paths[map_name].read_text())
class TestMantleGptRegistryEntries:
"""Locks the OpenAI GPT entries to Bedrock Mantle's live behavior.
Mantle enforces a 1,050,000-token prompt maximum for gpt-5.6 sol/terra/luna
and for gpt-5.5 and gpt-5.4 (oversize requests 400 with "prompt tokens (N)
exceed model maximum (1050000)", and a 1,030,590-token request completes
on every one of them), while the AWS model cards still quote 272K for
gpt-5.5 and gpt-5.4. mode must stay "responses": Mantle's native
/v1/chat/completions rejects function tools unless reasoning_effort is
"none", so chat traffic has to keep bridging to the Responses API
(see the responses_api_bridge tests above).
"""
@pytest.mark.parametrize("map_name", ("root", "bundled_backup"))
@pytest.mark.parametrize(
"key",
(
"bedrock_mantle/openai.gpt-5.6-sol",
"bedrock_mantle/openai.gpt-5.6-terra",
"bedrock_mantle/openai.gpt-5.6-luna",
),
)
def test_entry_matches_mantle_enforced_limits(self, map_name, key):
entry = _repo_cost_map(map_name)[key]
assert entry["max_input_tokens"] == 1050000
assert entry["max_output_tokens"] == 128000
assert entry["mode"] == "responses"
assert entry["use_openai_responses_path"] is True
assert entry["supported_endpoints"] == ["/v1/chat/completions", "/v1/responses"]
@pytest.mark.parametrize("map_name", ("root", "bundled_backup"))
@pytest.mark.parametrize(
"key",
(
"bedrock_mantle/openai.gpt-5.5",
"bedrock_mantle/openai.gpt-5.4",
),
)
def test_gpt_55_and_54_entries_match_mantle_enforced_limits(self, map_name, key):
entry = _repo_cost_map(map_name)[key]
assert entry["max_input_tokens"] == 1050000
assert entry["max_output_tokens"] == 128000
assert entry["mode"] == "responses"
assert entry["use_openai_responses_path"] is True

70
tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py
View file

@ -684,39 +684,8 @@ class TestBedrockMantleProviderResolution:
class TestBedrockMantlePricing:
"""Tests that verify Bedrock Mantle uses correct AWS Bedrock pricing, not OpenAI pricing."""
def test_gpt_oss_120b_pricing(self, monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true")
litellm.add_known_models()
info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-120b")
# Bedrock pricing: $0.15/M input, $0.60/M output
assert info["input_cost_per_token"] == pytest.approx(1.5e-7)
assert info["output_cost_per_token"] == pytest.approx(6e-7)
def test_gpt_oss_20b_pricing(self, monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true")
litellm.add_known_models()
info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-20b")
# Bedrock pricing: $0.075/M input, $0.30/M output
assert info["input_cost_per_token"] == pytest.approx(7.5e-8)
assert info["output_cost_per_token"] == pytest.approx(3e-7)
def test_pricing_significantly_cheaper_than_openai_native(self, monkeypatch):
"""
Verify Bedrock Mantle pricing is cheaper than OpenAI's direct API pricing.
This is the core issue the provider addition fixes — previously users were being
billed at OpenAI rates instead of the cheaper Bedrock rates.
"""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true")
litellm.add_known_models()
bedrock_info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-120b")
# OpenAI direct pricing for gpt-oss-120b is ~$0.039/M input, $0.190/M output
# Bedrock should be cheaper at $0.15/M input and $0.60/M output... wait
# Actually, Bedrock ADDS value not reduces cost vs OpenAI direct for these models.
# The key fix is that we now use Bedrock-specific prices instead of mapping to
# some unrelated OpenAI model (like gpt-4) pricing.
# Just validate the pricing is as expected from AWS docs.
assert bedrock_info["input_cost_per_token"] == pytest.approx(1.5e-7)
assert bedrock_info["output_cost_per_token"] == pytest.approx(6e-7)
def test_safeguard_models_have_larger_output_tokens(self, monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true")
@ -727,48 +696,9 @@ class TestBedrockMantlePricing:
)
assert info_safeguard["max_output_tokens"] > info_120b["max_output_tokens"]
def test_reasoning_support(self, monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true")
litellm.add_known_models()
info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-120b")
assert info.get("supports_reasoning") is True
def test_context_window(self, monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true")
litellm.add_known_models()
info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-120b")
assert info["max_input_tokens"] == 131072
@pytest.mark.parametrize(
"model_id,input_cost,output_cost,max_tokens",
[
("google.gemma-4-31b", 1.4e-07, 4e-07, 256000),
("google.gemma-4-26b-a4b", 1.3e-07, 4e-07, 256000),
("google.gemma-4-e2b", 4e-08, 8e-08, 128000),
],
)
def test_gemma_4_bedrock_mantle_model_metadata(
local_cost_map, model_id, input_cost, output_cost, max_tokens
):
full_model_name = f"bedrock_mantle/{model_id}"
info = litellm.get_model_info(full_model_name)
assert info["mode"] == "chat"
assert info["input_cost_per_token"] == pytest.approx(input_cost)
assert info["output_cost_per_token"] == pytest.approx(output_cost)
assert info["max_input_tokens"] == max_tokens
assert info["max_output_tokens"] == max_tokens
assert info["supports_function_calling"] is True
assert info["supports_reasoning"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
assert (
litellm.supports_parallel_function_calling(
model=full_model_name, custom_llm_provider="bedrock_mantle"
)
is False
)
@pytest.mark.parametrize(

18
tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py
View file

@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch
import pytest
import litellm
from litellm import get_model_info, supports_reasoning, supports_vision
from litellm import supports_reasoning, supports_vision
from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY
from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig
from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id
@ -16,15 +16,6 @@ from litellm.types.utils import (
)
@pytest.fixture(autouse=True)
def force_local_model_cost(monkeypatch):
"""Force local model cost map usage for all tests in this file."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
# Refresh model_cost from local map
import litellm
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map
litellm.model_cost = get_model_cost_map(url=litellm.model_cost_map_url)
def test_validate_environment_sets_session_affinity_from_litellm_session_id():
@ -404,13 +395,6 @@ def test_get_supported_openai_params_parallel_tool_calls_without_tool_choice(
assert "tool_choice" not in supported_params
def test_get_model_info_respects_explicit_fireworks_capabilities():
"""Test that get_model_info preserves explicit capability flags from the model map."""
model_info = get_model_info("fireworks_ai/accounts/fireworks/models/glm-5p1")
assert model_info["supports_function_calling"] is True
assert model_info["supports_reasoning"] is True
assert model_info["supports_tool_choice"] is True
def test_get_provider_info_omits_false_supports_reasoning(monkeypatch):

9
tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_kimi_model_metadata.py
View file

@ -56,15 +56,6 @@ def use_local_model_cost_map():
monkeypatch.undo()
@pytest.mark.parametrize("alias", KIMI_ALIASES)
def test_fireworks_kimi_raw_cost_entry_limits(use_local_model_cost_map, alias):
entry = use_local_model_cost_map.model_cost[alias]
assert entry["litellm_provider"] == "fireworks_ai"
assert entry["max_input_tokens"] == CONTEXT_WINDOW
assert entry["max_output_tokens"] == OUTPUT_LIMIT
assert entry["max_tokens"] == OUTPUT_LIMIT
assert entry["max_output_tokens"] < entry["max_input_tokens"]
@pytest.mark.parametrize("alias", KIMI_ALIASES)

27
tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py
View file

@ -1,13 +1,11 @@
import json
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import MagicMock
import httpx
import pytest
import litellm
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
def test_gemini_realtime_transformation_session_created():
@ -308,18 +306,6 @@ def test_gemini_realtime_transformation_generation_complete():
assert contains_audio_done_event, "Expected audio done event"
def test_gemini_3_1_flash_live_preview_model_cost_map_entry():
for key in (
"gemini-3.1-flash-live-preview",
"gemini/gemini-3.1-flash-live-preview",
):
assert key in litellm.model_cost
info = litellm.model_cost[key]
assert "/v1/realtime" in info.get("supported_endpoints", [])
assert info.get("max_input_tokens") == 131072
assert info.get("max_output_tokens") == 65536
assert "video" in info.get("supported_modalities", [])
assert info.get("supports_function_calling") is True
def test_gemini_realtime_tool_call_transformation():
@ -1845,17 +1831,6 @@ def test_is_audio_only_live_model_uses_cost_map(model, expected, patch_gemini_au
assert GeminiRealtimeConfig._is_audio_only_live_model(model) == expected
def test_gemini_live_native_audio_entry_is_vertex_only():
import json
from pathlib import Path
from typing import Final
catalog_path: Final = Path(__file__).parents[5] / "model_prices_and_context_window.json"
catalog: Final = json.loads(catalog_path.read_text())
vertex_key: Final = "gemini-live-2.5-flash-native-audio"
assert catalog[vertex_key]["litellm_provider"] == "vertex_ai-language-models"
assert catalog[vertex_key].get("gemini_native_audio") is True
assert "gemini/gemini-live-2.5-flash-native-audio" not in catalog, "the Gemini API does not serve this model"
def test_is_setup_message_and_is_content_message():

18
tests/test_litellm/llms/inception/test_inception_chat_transformation.py
View file

@ -231,24 +231,6 @@ def test_inception_in_provider_lists():
assert "https://api.inceptionlabs.ai/v1" in litellm.openai_compatible_endpoints
def test_inception_model_configuration(monkeypatch):
from litellm import get_model_info
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
litellm.inception_models = set()
litellm.add_known_models()
info = get_model_info("inception/mercury-2")
assert info.get("litellm_provider") == "inception"
assert info.get("mode") == "chat"
assert info.get("max_input_tokens") == 128000
assert info.get("input_cost_per_token") == 2.5e-07
assert info.get("output_cost_per_token") == 7.5e-07
assert info.get("cache_read_input_token_cost") == 2.5e-08
assert info.get("supports_function_calling") is True
assert info.get("supports_tool_choice") is True
assert info.get("supports_response_schema") is True
def test_inception_model_list_populated(monkeypatch):

16
tests/test_litellm/llms/inception/test_inception_completion_transformation.py
View file

@ -143,22 +143,6 @@ async def test_inception_fim_async():
assert r.choices[0].text == "a + b"
def test_inception_fim_model_configuration(monkeypatch):
from litellm import get_model_info
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
litellm.text_completion_inception_models = set()
litellm.add_known_models()
assert (
"text-completion-inception/mercury-edit-2"
in litellm.text_completion_inception_models
)
info = get_model_info("text-completion-inception/mercury-edit-2")
assert info.get("litellm_provider") == "text-completion-inception"
assert info.get("mode") == "completion"
assert info.get("max_input_tokens") == 32000
def test_inception_fim_targets_fim_endpoint():

30
tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py
View file

@ -708,37 +708,10 @@ class TestKimiK26ModelRegistry:
"""Load directly from the bundled backup so tests don't depend on remote fetch."""
return GetModelCostMap.load_local_model_cost_map()
def test_kimi_k26_in_model_cost_map(self, model_cost_map):
"""kimi-k2.6 should be present in the model cost map."""
assert "moonshot/kimi-k2.6" in model_cost_map, "moonshot/kimi-k2.6 not found in model_cost"
def test_kimi_k26_pricing(self, model_cost_map):
"""kimi-k2.6 pricing should match official Kimi API rates."""
model_info = model_cost_map["moonshot/kimi-k2.6"]
assert model_info["input_cost_per_token"] == pytest.approx(9.5e-07)
assert model_info["output_cost_per_token"] == pytest.approx(4e-06)
assert model_info["cache_read_input_token_cost"] == pytest.approx(1.6e-07)
def test_kimi_k26_context_window(self, model_cost_map):
"""kimi-k2.6 should have a 256K (262144 token) context window."""
model_info = model_cost_map["moonshot/kimi-k2.6"]
assert model_info["max_input_tokens"] == 262144
assert model_info["max_output_tokens"] == 262144
assert model_info["max_tokens"] == 262144
def test_kimi_k26_capabilities(self, model_cost_map):
"""kimi-k2.6 should support function calling, vision, video input, tool choice, and reasoning."""
model_info = model_cost_map["moonshot/kimi-k2.6"]
assert model_info.get("supports_function_calling") is True
assert model_info.get("supports_tool_choice") is True
assert model_info.get("supports_vision") is True
assert model_info.get("supports_video_input") is True
assert model_info.get("supports_reasoning") is True
def test_kimi_k26_provider(self, model_cost_map):
"""kimi-k2.6 should be assigned to the moonshot provider."""
model_info = model_cost_map["moonshot/kimi-k2.6"]
assert model_info["litellm_provider"] == "moonshot"
class TestMoonshotResponseSchemaSupport:
@ -762,9 +735,6 @@ class TestMoonshotResponseSchemaSupport:
def model_cost_map(self):
return GetModelCostMap.load_local_model_cost_map()
@pytest.mark.parametrize("model", LIVE_MODELS)
def test_live_model_supports_response_schema(self, model, model_cost_map):
assert model_cost_map[model].get("supports_response_schema") is True
def test_supports_response_schema_utility_reports_true(self, model_cost_map, monkeypatch):
monkeypatch.setattr(litellm, "model_cost", model_cost_map)

16
tests/test_litellm/llms/openai_like/test_cognition_provider.py
View file

@ -110,22 +110,6 @@ class TestCognitionProviderIdentity:
class TestCognitionCostTracking:
@pytest.mark.parametrize(
"model, input_cost, output_cost, cache_read_cost",
[
("cognition/swe-1.6", 5e-07, 2.5e-06, 2e-07),
("cognition/swe-1.7", 5e-07, 2.5e-06, 2e-07),
("cognition/swe-1.7-lightning", 2.5e-06, 1.25e-05, 1e-06),
],
)
def test_cost_map_entries(self, model: str, input_cost: float, output_cost: float, cache_read_cost: float):
info = litellm.get_model_info(model=model)
assert info["litellm_provider"] == "cognition"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] == input_cost
assert info["output_cost_per_token"] == output_cost
assert info["cache_read_input_token_cost"] == cache_read_cost
@pytest.mark.parametrize(
"model, expected_prompt_cost, expected_completion_cost",

21
tests/test_litellm/llms/openai_like/test_json_providers.py
View file

@ -2,10 +2,9 @@
Tests for JSON-based provider configuration system.
"""
import json
import os
import sys
from unittest.mock import MagicMock, patch
from unittest.mock import patch
try:
import pytest
@ -318,24 +317,6 @@ class TestDarkbloom:
assert config is not None
assert config.custom_llm_provider == "darkbloom"
def test_darkbloom_model_cost_map(self):
with open(
os.path.join(workspace_path, "model_prices_and_context_window.json")
) as f:
model_cost = json.load(f)
expected_models = {
"darkbloom/gemma-4-26b": (3e-08, 1.65e-07),
"darkbloom/gpt-oss-20b": (1.45e-08, 7e-08),
}
for model, (input_cost, output_cost) in expected_models.items():
assert model in model_cost
assert model_cost[model]["litellm_provider"] == "darkbloom"
assert model_cost[model]["max_output_tokens"] == 32768
assert model_cost[model]["supports_function_calling"] is True
assert model_cost[model]["supports_tool_choice"] is True
assert model_cost[model]["input_cost_per_token"] == input_cost
assert model_cost[model]["output_cost_per_token"] == output_cost
class TestPublicAIIntegration:

29
tests/test_litellm/llms/openai_like/test_libertai_provider.py
View file

@ -59,22 +59,6 @@ class TestLibertAIProviderConfig:
assert api_base == "https://custom.example.com/v1"
assert api_key == "sk-test"
def test_libertai_model_cost_map(self):
"""Test that libertai models are present in the model cost map"""
model_cost = litellm.model_cost
assert "libertai/qwen3.6-27b" in model_cost
info = model_cost["libertai/qwen3.6-27b"]
assert info["litellm_provider"] == "libertai"
assert info["mode"] == "chat"
assert info["max_input_tokens"] == 262144
assert info["max_output_tokens"] == 262144
# thinking variants are marked as reasoning models
assert (
model_cost["libertai/qwen3.6-27b-thinking"].get("supports_reasoning")
is True
)
def test_libertai_router_config(self):
"""Test that libertai can be used in Router configuration"""
@ -95,19 +79,6 @@ class TestLibertAIProviderConfig:
assert len(router.model_list) == 1
assert router.model_list[0]["model_name"] == "libertai-chat"
def test_libertai_model_modes(self):
"""Chat models carry mode 'chat'; the embedding model carries mode 'embedding'."""
model_cost = litellm.model_cost
# chat model
assert model_cost["libertai/qwen3.6-27b"]["mode"] == "chat"
# embedding model (bge-m3) must be normalized to mode 'embedding' so
# /embeddings routing and the supported-endpoints matrix stay consistent
assert "libertai/bge-m3" in model_cost
bge = model_cost["libertai/bge-m3"]
assert bge["litellm_provider"] == "libertai"
assert bge["mode"] == "embedding"
def test_libertai_supported_endpoints_matrix(self):
"""The runtime-served backup matrix (GET /public/supported_endpoints) lists libertai."""

13
tests/test_litellm/llms/openai_like/test_meta_provider.py
View file

@ -193,19 +193,6 @@ class TestMetaAnthropicMessages:
class TestMuseSparkModelInfo:
def test_muse_spark_pricing_and_capabilities(self):
info = litellm.get_model_info("meta/muse-spark-1.1")
assert info["litellm_provider"] == "meta"
assert info["input_cost_per_token"] == 1.25e-06
assert info["output_cost_per_token"] == 4.25e-06
assert info["cache_read_input_token_cost"] == 1.5e-07
assert info["max_input_tokens"] == 1048576
assert info["supports_reasoning"] is True
assert info["supports_web_search"] is True
assert info["supports_vision"] is True
assert info["supports_function_calling"] is True
assert info["supports_prompt_caching"] is True
def test_muse_spark_cost_calculation(self):
from litellm import completion_cost

23
tests/test_litellm/llms/perplexity/embedding/test_perplexity_embedding_transformation.py
View file

@ -3,7 +3,6 @@ Unit tests for Perplexity embedding transformation logic.
"""
import base64
import json
import struct
from unittest.mock import MagicMock
@ -298,25 +297,3 @@ class TestPerplexityEmbeddingProviderConfig:
)
assert config is not None
assert isinstance(config, PerplexityEmbeddingConfig)
class TestPerplexityEmbeddingModelInfo:
"""Test that Perplexity embedding models are in model_prices_and_context_window."""
def test_model_info_available(self):
import litellm
info = litellm.get_model_info("perplexity/pplx-embed-v1-0.6b")
assert info is not None
assert info["mode"] == "embedding"
assert info["max_input_tokens"] == 32768
assert info["output_vector_size"] == 1024
def test_model_info_4b_available(self):
import litellm
info = litellm.get_model_info("perplexity/pplx-embed-v1-4b")
assert info is not None
assert info["mode"] == "embedding"
assert info["max_input_tokens"] == 32768
assert info["output_vector_size"] == 2560

45
tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py
View file

@ -26,7 +26,6 @@ from litellm.types.utils import (
Usage,
PromptTokensDetailsWrapper,
)
from litellm.utils import get_model_info
class TestPerplexityCostCalculator:
@ -317,20 +316,6 @@ class TestPerplexityCostCalculator:
assert math.isclose(total_cost, expected_total, rel_tol=1e-6)
def test_model_info_access(self):
"""Test that model info correctly returns the new cost fields."""
model_info = get_model_info(
model="sonar-deep-research", custom_llm_provider="perplexity"
)
# Check that the new fields are accessible
assert "citation_cost_per_token" in model_info
assert model_info["citation_cost_per_token"] == 2e-6
assert model_info["search_context_cost_per_query"] == {
"search_context_size_low": 0.005,
"search_context_size_medium": 0.005,
"search_context_size_high": 0.005,
}
@pytest.mark.parametrize("citation_tokens", [0, 10, 25, 100])
@pytest.mark.parametrize("search_queries", [0, 1, 5, 10])
@ -477,36 +462,6 @@ class TestPerplexityCostCalculator:
assert math.isclose(prompt_cost, expected_prompt, rel_tol=1e-9)
assert math.isclose(completion_cost, expected_completion, rel_tol=1e-9)
@pytest.mark.parametrize(
"model_id, usd_per_1m_input, usd_per_1m_output, usd_per_1m_cache_read",
[
("deepseek-v4-flash-0731", 0.13, 0.26, 0.028),
("glm-5.2", 1.4, 4.4, 0.14),
("kimi-k3", 3.0, 15.0, 0.3),
("kimi-k2.7-code", 0.95, 4.0, 0.19),
],
)
def test_agent_api_entries_carry_perplexity_published_rates(
self, model_id, usd_per_1m_input, usd_per_1m_output, usd_per_1m_cache_read
):
"""The Agent API third-party models are priced from Perplexity's own catalog
(GET https://api.perplexity.ai/v1/models, `pricing` in usd_per_1m_tokens).
Perplexity's model id already starts with `perplexity/`, so the cost-map key
doubles the prefix. Regression: glm-5.2 shipped glm-5.3's 0.26 cache-read rate,
copied from the neighbouring catalog row, an 86% overcharge on cached input.
"""
info = get_model_info(
model=f"perplexity/{model_id}", custom_llm_provider="perplexity"
)
assert info["key"] == f"perplexity/perplexity/{model_id}"
assert info["litellm_provider"] == "perplexity"
assert info["mode"] == "responses"
assert math.isclose(info["input_cost_per_token"], usd_per_1m_input / 1e6, rel_tol=1e-9)
assert math.isclose(info["output_cost_per_token"], usd_per_1m_output / 1e6, rel_tol=1e-9)
assert math.isclose(
info["cache_read_input_token_cost"], usd_per_1m_cache_read / 1e6, rel_tol=1e-9
)
def test_agent_api_fallback_rates_price_a_response_without_metered_cost(self):
"""Perplexity meters cost on the response, but when `usage.cost` is absent the

12
tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py
View file

@ -136,18 +136,6 @@ class TestVertexAIVideoConfig:
# Should NOT include endpoint
assert not url.endswith(":predictLongRunning")
def test_veo_31_lite_model_cost_entries_match_pricing(self):
for path in (ROOT_MODEL_COST_PATH, BACKUP_MODEL_COST_PATH):
model_cost = _load_model_cost_map(path)
info = model_cost.get(VEO_31_LITE_VERTEX_MODEL)
assert info is not None, f"{VEO_31_LITE_VERTEX_MODEL} missing from {path}"
assert info["litellm_provider"] == "vertex_ai-video-models"
assert info["mode"] == "video_generation"
assert info["max_input_tokens"] == 1024
assert info["output_cost_per_second"] == 0.05
assert info["output_cost_per_second_1080p"] == 0.08
assert info["supported_modalities"] == ["text", "image"]
def test_veo_31_lite_provider_routing_from_local_model_map(
self, monkeypatch: pytest.MonkeyPatch

14
tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py
View file

@ -63,11 +63,6 @@ TIER_COST_FIELDS = (
"output_cost_per_token_above_200k_tokens",
"cache_read_input_token_cost_above_200k_tokens",
)
STALE_TIER_FIELDS = (
"input_cost_per_token_above_128k_tokens",
"output_cost_per_token_above_128k_tokens",
"cache_read_input_token_cost_above_128k_tokens",
)
def expected_retirement_date(slug: str) -> str:
@ -102,11 +97,6 @@ def test_redirected_slug_keeps_its_retirement_date(cost_map: dict, slug: str):
assert cost_map[slug]["deprecation_date"] == expected_retirement_date(slug)
@pytest.mark.parametrize("slug", REDIRECTED_SLUGS)
def test_no_slug_keeps_the_superseded_128k_tier(cost_map: dict, slug: str):
"""The 128k tier belonged to the retired model; grok-4.3 tiers at 200k."""
for field in STALE_TIER_FIELDS:
assert field not in cost_map[slug], field
@pytest.mark.parametrize("slug", REDIRECTED_SLUGS)
@ -118,10 +108,6 @@ def test_redirected_slug_carries_the_target_tier_rates(cost_map: dict, slug: str
assert entry[field] == target[field], field
def test_a_live_xai_model_is_untouched(cost_map: dict):
"""Guard against the repricing leaking onto models xAI still serves directly."""
assert cost_map["xai/grok-4.6"]["input_cost_per_token"] != cost_map[REDIRECT_TARGET]["input_cost_per_token"]
assert "deprecation_date" not in cost_map["xai/grok-4.6"]
def test_both_cost_maps_agree_on_the_redirected_slugs():

38
tests/test_litellm/llms/zai/test_zai_provider.py
View file

@ -2,11 +2,9 @@
Tests for Z.AI (Zhipu AI) provider - GLM models
"""
import json
import math
import pytest
import respx
import litellm
from litellm import completion
@ -57,31 +55,11 @@ def test_zai_in_provider_lists():
assert "zai" in litellm.provider_list
def test_zai_models_in_model_cost(local_model_cost_map):
"""Test that ZAI models are in the model cost map"""
zai_models = [
"zai/glm-4.7",
"zai/glm-4.6",
"zai/glm-4.5",
"zai/glm-4.5v",
"zai/glm-4.5-x",
"zai/glm-4.5-air",
"zai/glm-4.5-airx",
"zai/glm-4-32b-0414-128k",
"zai/glm-4.5-flash",
]
for model in zai_models:
assert model in litellm.model_cost, f"Model {model} not found in model_cost"
assert litellm.model_cost[model]["litellm_provider"] == "zai"
def test_zai_glm46_cost_calculation(local_model_cost_map):
"""Test the cost calculation for glm-4.6"""
key = "zai/glm-4.6"
info = litellm.model_cost[key]
prompt_cost, completion_cost = cost_per_token(
model="zai/glm-4.6",
@ -94,24 +72,8 @@ def test_zai_glm46_cost_calculation(local_model_cost_map):
assert math.isclose(completion_cost, 2.2, rel_tol=1e-6)
def test_zai_flash_model_is_free(local_model_cost_map):
"""Test that glm-4.5-flash has zero cost"""
key = "zai/glm-4.5-flash"
info = litellm.model_cost[key]
assert info["input_cost_per_token"] == 0
assert info["output_cost_per_token"] == 0
def test_glm47_supports_reasoning(local_model_cost_map):
"""Test that GLM-4.7 supports reasoning"""
key = "zai/glm-4.7"
assert key in litellm.model_cost, f"Model {key} not found in model_cost"
info = litellm.model_cost[key]
assert info["supports_reasoning"] is True
def test_glm47_cost_calculation(local_model_cost_map):

54
tests/test_litellm/test_bedrock_extended_beta_models.py
View file

@ -91,20 +91,6 @@ MODEL_CONFIGS = [
class TestBedrockNewModels:
"""Unified test suite for all new Bedrock models"""
@pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS)
def test_model_info_primary_region(
self, model_name, regions, max_input, max_output
):
"""Test model configuration in primary region (us-east-1)"""
model = f"bedrock/us-east-1/{model_name}"
model_info = get_model_info(model)
assert model_info is not None, f"Model {model_name} not found"
assert model_info["max_input_tokens"] == max_input
assert model_info["max_output_tokens"] == max_output
assert model_info["litellm_provider"] == "bedrock"
assert model_info["mode"] == "chat"
assert model_info["supports_function_calling"] is True
@pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS)
def test_pricing_configured(self, model_name, regions, max_input, max_output):
@ -128,43 +114,3 @@ class TestBedrockNewModels:
assert model_info is not None, f"Model {model_name} not found in {region}"
assert model_info["max_input_tokens"] == max_input
assert model_info["max_output_tokens"] == max_output
@pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS)
def test_sample_regional_variants(self, model_name, regions, max_input, max_output):
"""Test sample regional variants (us-east-1, eu-west-1, ap-northeast-1)"""
for region in ["us-east-1", "ap-northeast-1"]:
if region in regions:
model = f"bedrock/{region}/{model_name}"
model_info = get_model_info(model)
assert (
model_info is not None
), f"Model {model_name} not found in {region}"
assert model_info["max_input_tokens"] == max_input
assert model_info["litellm_provider"] == "bedrock"
class TestModelSpecificFeatures:
"""Model-specific capability tests"""
def test_deepseek_v3_2_context_window(self):
"""DeepSeek V3.2 has 163K context window"""
model_info = get_model_info("bedrock/us-east-1/deepseek.v3.2")
assert model_info["max_input_tokens"] == 163840
def test_minimax_m2_1_context_window(self):
"""Minimax M2.1 has 196K input, 8K output"""
model_info = get_model_info("bedrock/us-east-1/minimax.minimax-m2.1")
assert model_info["max_input_tokens"] == 196000
assert model_info["max_output_tokens"] == 8192
def test_moonshotai_kimi_k2_5_context_window(self):
"""Moonshot AI Kimi K2.5 has 256K context window"""
model_info = get_model_info("bedrock/us-east-1/moonshotai.kimi-k2.5")
assert model_info["max_input_tokens"] == 262144
assert model_info["max_output_tokens"] == 262144
def test_qwen3_coder_next_context_window(self):
"""Qwen3 Coder Next has 256K input, 8K output"""
model_info = get_model_info("bedrock/us-east-1/qwen.qwen3-coder-next")
assert model_info["max_input_tokens"] == 262144
assert model_info["max_output_tokens"] == 8192

51
tests/test_litellm/test_bedrock_nemotron_super.py
View file

@ -1,51 +0,0 @@
"""
Test suite for NVIDIA Nemotron Super 3 120B on AWS Bedrock
Verifies model configuration, pricing, and regional availability.
"""
import os
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "true"
import pytest
from litellm import get_model_info
MODEL_NAME = "nvidia.nemotron-super-3-120b"
class TestNemotronSuper3120B:
"""Test model definition for nvidia.nemotron-super-3-120b"""
def test_model_info_primary_region(self):
"""Test model resolves in us-east-1"""
model_info = get_model_info(f"bedrock/us-east-1/{MODEL_NAME}")
assert model_info is not None, f"Model {MODEL_NAME} not found"
assert model_info["max_input_tokens"] == 256000
assert model_info["max_output_tokens"] == 32768
assert model_info["litellm_provider"] == "bedrock_converse"
assert model_info["mode"] == "chat"
assert model_info["supports_function_calling"] is True
def test_pricing_configured(self):
"""Verify pricing matches AWS Bedrock rates"""
model_info = get_model_info(f"bedrock/us-east-1/{MODEL_NAME}")
assert model_info["input_cost_per_token"] == 1.5e-07
assert model_info["output_cost_per_token"] == 6.5e-07
def test_context_window(self):
"""Nemotron Super 3 120B has 256K input, 32K output on Bedrock"""
model_info = get_model_info(f"bedrock/us-east-1/{MODEL_NAME}")
assert model_info["max_input_tokens"] == 256000
assert model_info["max_output_tokens"] == 32768
def test_resolves_without_region(self):
"""Test model resolves with just bedrock/ prefix"""
model_info = get_model_info(f"bedrock/{MODEL_NAME}")
assert model_info is not None, f"Model {MODEL_NAME} not found without region"
assert model_info["max_input_tokens"] == 256000

47
tests/test_litellm/test_bedrock_usgov_haiku_1hr_cache.py
View file

@ -1,47 +0,0 @@
"""
Validate that AWS GovCloud (Bedrock us-gov-*) Haiku 4.5 entries carry
the 1-hour cache write tier.
AWS Bedrock GovCloud pricing applies a +20% premium over global
Anthropic rates. Global Haiku 4.5 1h cache write is $2.00/MTok; us-gov
is therefore $2.40/MTok — exactly 1.6x the 5-minute rate of $1.50/MTok.
Source: https://aws.amazon.com/bedrock/pricing/
"""
import json
import os
import pytest
@pytest.fixture(scope="module")
def model_data():
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path) as f:
return json.load(f)
HAIKU_USGOV_KEYS = [
"bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0",
"bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0",
]
@pytest.mark.parametrize("model_key", HAIKU_USGOV_KEYS)
def test_usgov_haiku_4_5_1hr_cache_write(model_data, model_key):
assert model_key in model_data, f"Missing model entry: {model_key}"
info = model_data[model_key]
assert (
info["cache_creation_input_token_cost"] == 1.5e-06
), f"{model_key}: 5m cache write should be $1.50/MTok"
assert (
info["cache_creation_input_token_cost_above_1hr"] == 2.4e-06
), f"{model_key}: 1h cache write should be $2.40/MTok"
ratio = (
info["cache_creation_input_token_cost_above_1hr"]
/ info["cache_creation_input_token_cost"]
)
assert abs(ratio - 1.6) < 1e-9, f"{model_key}: 1h/5m ratio is {ratio}, expected 1.6"

271
tests/test_litellm/test_bedrock_usgov_pricing.py
View file

@ -31,32 +31,8 @@ def model_data():
return json.load(f)
SONNET_4_5_USGOV_KEYS = [
"bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0",
"bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0",
"bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0",
"bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0",
"us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0",
]
@pytest.mark.parametrize("model_key", SONNET_4_5_USGOV_KEYS)
def test_usgov_sonnet_4_5_pricing(model_data, model_key):
"""Each us-gov sonnet-4-5 entry must carry the +20%-over-global rates
that AWS publishes on the GovCloud pricing page.
"""
assert model_key in model_data, f"Missing model entry: {model_key}"
info = model_data[model_key]
assert info["input_cost_per_token"] == 3.6e-06, (
f"{model_key}: input_cost_per_token should be $3.60/MTok (got {info['input_cost_per_token']})"
)
assert info["output_cost_per_token"] == 1.8e-05, f"{model_key}: output_cost_per_token should be $18.00/MTok"
assert info["cache_creation_input_token_cost"] == 4.5e-06, f"{model_key}: 5m cache write should be $4.50/MTok"
assert info["cache_creation_input_token_cost_above_1hr"] == 7.2e-06, (
f"{model_key}: 1h cache write should be $7.20/MTok"
)
assert info["cache_read_input_token_cost"] == 3.6e-07, f"{model_key}: cache read should be $0.36/MTok"
def test_usgov_carries_20_percent_premium_over_global(model_data):
@ -117,165 +93,24 @@ def test_usgov_cross_region_above_200k_ratio_to_global(model_data):
assert abs(ratio - 1.2) < 1e-9, f"{field}: us-gov / global ratio is {ratio}, expected 1.2"
CLAUDE_GOV_EXPECTED = {
"anthropic.claude-sonnet-5": {
"input_cost_per_token": 2.4e-06,
"output_cost_per_token": 1.2e-05,
"cache_creation_input_token_cost": 3e-06,
"cache_creation_input_token_cost_above_1hr": 4.8e-06,
"cache_read_input_token_cost": 2.4e-07,
},
"anthropic.claude-opus-4-8": {
"input_cost_per_token": 6e-06,
"output_cost_per_token": 3e-05,
"cache_creation_input_token_cost": 7.5e-06,
"cache_creation_input_token_cost_above_1hr": 1.2e-05,
"cache_read_input_token_cost": 6e-07,
},
"anthropic.claude-opus-5": {
"input_cost_per_token": 6e-06,
"output_cost_per_token": 3e-05,
"cache_creation_input_token_cost": 7.5e-06,
"cache_creation_input_token_cost_above_1hr": 1.2e-05,
"cache_read_input_token_cost": 6e-07,
},
"anthropic.claude-fable-5-1": {
"input_cost_per_token": 1.2e-05,
"output_cost_per_token": 6e-05,
"cache_creation_input_token_cost": 1.5e-05,
"cache_creation_input_token_cost_above_1hr": 2.4e-05,
"cache_read_input_token_cost": 3e-07,
},
}
USGOV_CLAUDE_KEY_TEMPLATES = {
"bedrock/us-gov-east-1/{base_key}": "bedrock",
"bedrock/us-gov-west-1/{base_key}": "bedrock",
"us-gov.{base_key}": "bedrock_converse",
}
@pytest.mark.parametrize("base_key", CLAUDE_GOV_EXPECTED)
@pytest.mark.parametrize("key_template,expected_provider", USGOV_CLAUDE_KEY_TEMPLATES.items())
def test_usgov_claude_pricing(model_data, key_template, expected_provider, base_key):
"""Sonnet 5, Opus 4.8, Opus 5, and Fable 5.1 gov entries, both in-region keys
and the us-gov. geo inference profile the model cards list for GovCloud, must
carry the 1.2x GovCloud premium over the global anthropic.* rates. No public
AWS source (offer files, pricing page) lists Claude GovCloud rows; the premium
is the one AWS quotes for Opus 4.8 in GovCloud ($6/$30 per million).
"""
gov_key = key_template.format(base_key=base_key)
assert gov_key in model_data, f"Missing model entry: {gov_key}"
info = model_data[gov_key]
assert info["litellm_provider"] == expected_provider
assert "search_context_cost_per_query" not in info
for field, expected in CLAUDE_GOV_EXPECTED[base_key].items():
assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})"
ratio = info[field] / model_data[base_key][field]
assert abs(ratio - 1.2) < 1e-9, f"{gov_key}: {field} gov/global ratio is {ratio}, expected 1.2"
CONVERSE_GOV_EXPECTED = {
"nvidia.nemotron-nano-3-30b": (7.2e-08, 2.88e-07),
"nvidia.nemotron-nano-9b-v2": (7.2e-08, 2.76e-07),
"nvidia.nemotron-nano-12b-v2": (2.4e-07, 7.2e-07),
"nvidia.nemotron-super-3-120b": (1.8e-07, 7.8e-07),
"openai.gpt-oss-20b-1:0": (8.4e-08, 3.6e-07),
"openai.gpt-oss-120b-1:0": (1.8e-07, 7.2e-07),
}
@pytest.mark.parametrize("base_key", CONVERSE_GOV_EXPECTED)
@pytest.mark.parametrize("key_template,expected_provider", USGOV_CLAUDE_KEY_TEMPLATES.items())
def test_usgov_converse_model_pricing(model_data, key_template, expected_provider, base_key):
"""Nemotron and gpt-oss gov entries, in-region and the us-gov. geo inference
profile both GovCloud regions list as ACTIVE, must match the AWS Bedrock
offer file, which prices both regions identically at 1.2x commercial.
"""
gov_key = key_template.format(base_key=base_key)
assert gov_key in model_data, f"Missing model entry: {gov_key}"
info = model_data[gov_key]
expected_input, expected_output = CONVERSE_GOV_EXPECTED[base_key]
assert info["input_cost_per_token"] == expected_input
assert info["output_cost_per_token"] == expected_output
assert info["litellm_provider"] == expected_provider
base = model_data[base_key]
assert abs(info["input_cost_per_token"] / base["input_cost_per_token"] - 1.2) < 1e-9
assert abs(info["output_cost_per_token"] / base["output_cost_per_token"] - 1.2) < 1e-9
def test_usgov_west_llama3_8b_output_price_fixed(model_data):
"""The us-gov-west-1 llama3-8b entry carried the 70B output rate ($2.65/MTok);
the AWS Bedrock offer file prices output at $0.60/MTok. AWS lists the model
in us-gov-west-1 only, so there is no east entry to check.
"""
info = model_data["bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0"]
assert info["input_cost_per_token"] == 3e-07
assert info["output_cost_per_token"] == 6e-07
MANTLE_GOV_TIERED_EXPECTED = {
"openai.gpt-5.6-luna": {
"input_cost_per_token": 2.64e-07,
"input_cost_per_token_above_272k_tokens": 5.28e-07,
"cache_creation_input_token_cost": 3.3e-07,
"cache_creation_input_token_cost_above_272k_tokens": 6.6e-07,
"cache_read_input_token_cost": 2.64e-08,
"cache_read_input_token_cost_above_272k_tokens": 5.28e-08,
"output_cost_per_token": 1.584e-06,
"output_cost_per_token_above_272k_tokens": 2.376e-06,
},
"openai.gpt-5.6-terra": {
"input_cost_per_token": 2.64e-06,
"input_cost_per_token_above_272k_tokens": 5.28e-06,
"cache_creation_input_token_cost": 3.3e-06,
"cache_creation_input_token_cost_above_272k_tokens": 6.6e-06,
"cache_read_input_token_cost": 2.64e-07,
"cache_read_input_token_cost_above_272k_tokens": 5.28e-07,
"output_cost_per_token": 1.584e-05,
"output_cost_per_token_above_272k_tokens": 2.376e-05,
},
}
@pytest.mark.parametrize("model", MANTLE_GOV_TIERED_EXPECTED)
def test_usgov_west_mantle_terra_luna_pricing(model_data, model):
"""Terra and Luna carry 1.2x commercial across every tier in the
us-gov-west-1 offer file; the us-gov-east-1 offer file has no SKUs for them.
"""
gov_key = f"bedrock_mantle/us-gov-west-1/{model}"
assert gov_key in model_data, f"Missing model entry: {gov_key}"
info = model_data[gov_key]
for field, expected in MANTLE_GOV_TIERED_EXPECTED[model].items():
assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})"
assert info["litellm_provider"] == "bedrock_mantle"
assert f"bedrock_mantle/us-gov-east-1/{model}" not in model_data
@pytest.mark.parametrize("region", ["us-gov-east-1", "us-gov-west-1"])
def test_usgov_mantle_gpt_5_4_pricing_has_no_long_context_tier(model_data, region):
"""gpt-5.4 gov rates come from the offer file, which publishes only the
standard tier in GovCloud: no long-context SKUs exist there, unlike commercial.
"""
gov_key = f"bedrock_mantle/{region}/openai.gpt-5.4"
assert gov_key in model_data, f"Missing model entry: {gov_key}"
info = model_data[gov_key]
assert info["input_cost_per_token"] == 3.3e-06
assert info["cache_read_input_token_cost"] == 3.3e-07
assert info["output_cost_per_token"] == 1.98e-05
assert not any(field.endswith("_above_272k_tokens") for field in info)
def test_usgov_mantle_grok_4_3_west_only(model_data):
"""grok-4.3 is priced in the us-gov-west-1 offer file only; the east offer
file carries grok-4.6 instead.
"""
info = model_data["bedrock_mantle/us-gov-west-1/xai.grok-4.3"]
assert info["input_cost_per_token"] == 1.5e-06
assert info["output_cost_per_token"] == 3e-06
assert info["cache_read_input_token_cost"] == 2.4e-07
assert "bedrock_mantle/us-gov-east-1/xai.grok-4.3" not in model_data
def test_usgov_east_haiku_profile_mirrors_in_region_row(model_data):
@ -290,94 +125,18 @@ def test_usgov_east_haiku_profile_mirrors_in_region_row(model_data):
}
GROK_4_6_GOV_KEYS = {
"us-gov.xai.grok-4.6": ("us.xai.grok-4.6", "bedrock_converse"),
"bedrock_mantle/us-gov-west-1/xai.grok-4.6": ("bedrock_mantle/xai.grok-4.6", "bedrock_mantle"),
"bedrock_mantle/us-gov-east-1/xai.grok-4.6": ("bedrock_mantle/xai.grok-4.6", "bedrock_mantle"),
}
@pytest.mark.parametrize("gov_key", GROK_4_6_GOV_KEYS)
def test_usgov_grok_4_6_pricing(model_data, gov_key):
"""Both GovCloud regions serve grok-4.6 through the us-gov. profile only, and
both offer files price its standard SKU at 1.2x the commercial US rate.
"""
base_key, expected_provider = GROK_4_6_GOV_KEYS[gov_key]
assert gov_key in model_data, f"Missing model entry: {gov_key}"
info = model_data[gov_key]
assert info["litellm_provider"] == expected_provider
assert info["input_cost_per_token"] == 2.64e-06
assert info["output_cost_per_token"] == 7.92e-06
assert info["cache_read_input_token_cost"] == 6.6e-07
for field in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost"):
assert abs(info[field] / model_data[base_key][field] - 1.2) < 1e-9
NOVA_GOV_WEST_EXPECTED = {
"amazon.nova-lite-v1:0": (7.2e-08, 2.88e-07),
"amazon.nova-micro-v1:0": (4.2e-08, 1.68e-07),
}
@pytest.mark.parametrize("base_key", NOVA_GOV_WEST_EXPECTED)
def test_usgov_west_nova_lite_micro_pricing(model_data, base_key):
"""Nova Lite and Micro are on-demand in us-gov-west-1 only; the offer file
prices them at 1.2x commercial, like the Nova Pro row that was already there.
"""
gov_key = f"bedrock/us-gov-west-1/{base_key}"
assert gov_key in model_data, f"Missing model entry: {gov_key}"
info = model_data[gov_key]
expected_input, expected_output = NOVA_GOV_WEST_EXPECTED[base_key]
assert info["litellm_provider"] == "bedrock"
assert info["input_cost_per_token"] == expected_input
assert info["output_cost_per_token"] == expected_output
assert abs(info["input_cost_per_token"] / model_data[base_key]["input_cost_per_token"] - 1.2) < 1e-9
assert abs(info["output_cost_per_token"] / model_data[base_key]["output_cost_per_token"] - 1.2) < 1e-9
assert f"bedrock/us-gov-east-1/{base_key}" not in model_data
def test_usgov_west_nova_2_multimodal_embeddings_pricing(model_data):
"""Every meter of the multimodal embedding model (tokens, images, audio and
video seconds) carries the 1.2x uplift the us-gov-west-1 offer file lists.
"""
gov_key = "bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0"
assert gov_key in model_data, f"Missing model entry: {gov_key}"
info = model_data[gov_key]
assert info["litellm_provider"] == "bedrock"
assert info["mode"] == "embedding"
assert info["input_cost_per_token"] == 1.62e-07
assert info["input_cost_per_image"] == 7.2e-05
assert info["input_cost_per_audio_per_second"] == 0.000168
assert info["input_cost_per_video_per_second"] == 0.00084
assert "bedrock/us-gov-east-1/amazon.nova-2-multimodal-embeddings-v1:0" not in model_data
MANTLE_GOV_FLAT_EXPECTED = {
"google.gemma-4-e2b": (4.8e-08, 9.6e-08, ("us-gov-west-1",)),
"google.gemma-4-26b-a4b": (1.56e-07, 4.8e-07, ("us-gov-west-1",)),
"google.gemma-4-31b": (1.68e-07, 4.8e-07, ("us-gov-west-1",)),
"openai.gpt-oss-20b": (8.4e-08, 3.6e-07, ("us-gov-west-1", "us-gov-east-1")),
"openai.gpt-oss-120b": (1.8e-07, 7.2e-07, ("us-gov-west-1", "us-gov-east-1")),
}
@pytest.mark.parametrize("model", MANTLE_GOV_FLAT_EXPECTED)
def test_usgov_mantle_gemma_and_gpt_oss_pricing(model_data, model):
"""Gemma 4 is priced in the us-gov-west-1 offer file only and gpt-oss in both;
each Mantle gov row carries the offer file's standard SKU, and no row exists
for a region whose offer file has no SKU.
"""
expected_input, expected_output, regions = MANTLE_GOV_FLAT_EXPECTED[model]
for region in ("us-gov-west-1", "us-gov-east-1"):
gov_key = f"bedrock_mantle/{region}/{model}"
if region not in regions:
assert gov_key not in model_data
continue
assert gov_key in model_data, f"Missing model entry: {gov_key}"
info = model_data[gov_key]
assert info["litellm_provider"] == "bedrock_mantle"
assert info["input_cost_per_token"] == expected_input
assert info["output_cost_per_token"] == expected_output
GOV_ROW_SOURCES = {
@ -417,33 +176,3 @@ def test_usgov_rows_keep_commercial_limits_and_capabilities(model_data, gov_key)
assert _non_pricing_fields(gov) == _non_pricing_fields(model_data[GOV_ROW_SOURCES[gov_key]])
assert "search_context_cost_per_query" not in gov
assert "source" not in gov
AZURE_GOV_EXPECTED = {
"azure/us-gov/gpt-5.1": {
"input_cost_per_token": 1.71875e-06,
"cache_read_input_token_cost": 1.71875e-07,
"output_cost_per_token": 1.375e-05,
},
"azure/us-gov/o3-mini": {
"input_cost_per_token": 1.513e-06,
"cache_read_input_token_cost": 7.57e-07,
"output_cost_per_token": 6.05e-06,
},
"azure/us-gov/text-embedding-3-large": {"input_cost_per_token": 1.63e-07},
"azure/us-gov/text-embedding-3-small": {"input_cost_per_token": 2.5e-08},
}
@pytest.mark.parametrize("gov_key", AZURE_GOV_EXPECTED)
def test_azure_usgov_pricing(model_data, gov_key):
"""Azure Government meters from the Azure retail prices API
(usgovvirginia/usgovarizona, serviceName 'Foundry Models'). No Government
retirement schedule is published, so these entries carry no deprecation_date.
"""
assert gov_key in model_data, f"Missing model entry: {gov_key}"
info = model_data[gov_key]
for field, expected in AZURE_GOV_EXPECTED[gov_key].items():
assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})"
assert info["litellm_provider"] == "azure"
assert "deprecation_date" not in info

165
tests/test_litellm/test_claude_fable_5_config.py
View file

@ -14,7 +14,6 @@ import os
import pytest
import litellm
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
@ -28,86 +27,8 @@ def _load_root_cost_map() -> dict:
def test_fable_5_model_pricing_and_capabilities():
model_data = _load_root_cost_map()
expected_models = [
("claude-fable-5", "anthropic"),
("anthropic.claude-fable-5", "bedrock_converse"),
("vertex_ai/claude-fable-5", "vertex_ai-anthropic_models"),
# Unlike Opus 4.8 (200k on Foundry), Fable 5 has the full 1M context
# window on Microsoft Foundry.
("azure_ai/claude-fable-5", "azure_ai"),
]
for model_name, provider in expected_models:
assert model_name in model_data, f"Missing model entry: {model_name}"
info = model_data[model_name]
assert info["litellm_provider"] == provider
assert info["mode"] == "chat"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
assert info["max_tokens"] == 128000
# $10 / $50 per MTok (2x Opus 4.8), with the standard 1.25x 5m
# cache-write, 2x 1h cache-write, and 0.1x cache-read multipliers.
assert info["input_cost_per_token"] == 1e-05
assert info["output_cost_per_token"] == 5e-05
assert info["cache_creation_input_token_cost"] == 1.25e-05
assert info["cache_creation_input_token_cost_above_1hr"] == 2e-05
assert info["cache_read_input_token_cost"] == 1e-06
# Flat-rate across the full 1M context window.
assert "input_cost_per_token_above_200k_tokens" not in info
assert "output_cost_per_token_above_200k_tokens" not in info
assert info["supports_assistant_prefill"] is False
assert info["supports_function_calling"] is True
assert info["supports_prompt_caching"] is True
assert info["supports_reasoning"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
assert info["supports_xhigh_reasoning_effort"] is True
assert info["supports_max_reasoning_effort"] is True
def test_fable_5_bedrock_regional_model_pricing():
model_data = _load_root_cost_map()
# Fable 5 launched with us/eu geo inference profiles plus a global profile
# (no au/apac/jp). Global uses base pricing; geo profiles carry the
# standard 10% regional premium.
expected_models = {
"global.anthropic.claude-fable-5": {
"input_cost_per_token": 1e-05,
"output_cost_per_token": 5e-05,
"cache_creation_input_token_cost": 1.25e-05,
"cache_read_input_token_cost": 1e-06,
},
"us.anthropic.claude-fable-5": {
"input_cost_per_token": 1.1e-05,
"output_cost_per_token": 5.5e-05,
"cache_creation_input_token_cost": 1.375e-05,
"cache_read_input_token_cost": 1.1e-06,
},
"eu.anthropic.claude-fable-5": {
"input_cost_per_token": 1.1e-05,
"output_cost_per_token": 5.5e-05,
"cache_creation_input_token_cost": 1.375e-05,
"cache_read_input_token_cost": 1.1e-06,
},
}
for model_name, expected in expected_models.items():
assert model_name in model_data, f"Missing model entry: {model_name}"
info = model_data[model_name]
assert info["litellm_provider"] == "bedrock_converse"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
assert info["bedrock_output_config_effort_ceiling"] == "xhigh"
for key, value in expected.items():
assert info[key] == value
def test_fable_5_geo_multiplier_without_fast_mode():
@ -144,11 +65,6 @@ def test_fable_5_registered_for_bedrock_converse():
assert "anthropic.claude-fable-5" in BEDROCK_CONVERSE_MODELS
def test_fable_5_provider_resolves_via_model_info(local_model_cost_map):
info = litellm.get_model_info(model="claude-fable-5")
assert info["litellm_provider"] == "anthropic"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
@pytest.mark.parametrize(
@ -222,44 +138,6 @@ FABLE_5_1_VARIANTS = (
)
def test_fable_5_1_model_pricing_and_capabilities():
model_data = _load_root_cost_map()
expected_models = [
("claude-fable-5-1", "anthropic"),
("anthropic.claude-fable-5-1", "bedrock_converse"),
("vertex_ai/claude-fable-5-1", "vertex_ai-anthropic_models"),
("azure_ai/claude-fable-5-1", "azure_ai"),
]
for model_name, provider in expected_models:
assert model_name in model_data, f"Missing model entry: {model_name}"
info = model_data[model_name]
assert info["litellm_provider"] == provider
assert info["mode"] == "chat"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
assert info["max_tokens"] == 128000
assert info["input_cost_per_token"] == 1e-05
assert info["output_cost_per_token"] == 5e-05
assert info["cache_creation_input_token_cost"] == 1.25e-05
assert info["cache_creation_input_token_cost_above_1hr"] == 2e-05
assert "input_cost_per_token_above_200k_tokens" not in info
assert "output_cost_per_token_above_200k_tokens" not in info
assert info["supports_assistant_prefill"] is False
assert info["supports_forced_tool_use"] is False
assert info["supports_function_calling"] is True
assert info["supports_prompt_caching"] is True
assert info["supports_reasoning"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
assert info["supports_xhigh_reasoning_effort"] is True
assert info["supports_max_reasoning_effort"] is True
assert info["prompt_cache_min_tokens"] == 512
@pytest.mark.parametrize(
@ -280,46 +158,8 @@ def test_fable_5_1_cache_reads_cost_a_quarter_of_fable_5(cost_map):
), model_name
def test_fable_5_1_bedrock_regional_model_pricing():
model_data = _load_root_cost_map()
expected_models = {
"global.anthropic.claude-fable-5-1": {
"input_cost_per_token": 1e-05,
"output_cost_per_token": 5e-05,
"cache_creation_input_token_cost": 1.25e-05,
"cache_read_input_token_cost": 2.5e-07,
},
"us.anthropic.claude-fable-5-1": {
"input_cost_per_token": 1.1e-05,
"output_cost_per_token": 5.5e-05,
"cache_creation_input_token_cost": 1.375e-05,
"cache_read_input_token_cost": 2.75e-07,
},
"eu.anthropic.claude-fable-5-1": {
"input_cost_per_token": 1.1e-05,
"output_cost_per_token": 5.5e-05,
"cache_creation_input_token_cost": 1.375e-05,
"cache_read_input_token_cost": 2.75e-07,
},
}
for model_name, expected in expected_models.items():
assert model_name in model_data, f"Missing model entry: {model_name}"
info = model_data[model_name]
assert info["litellm_provider"] == "bedrock_converse"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
assert info["bedrock_output_config_effort_ceiling"] == "xhigh"
for key, value in expected.items():
assert info[key] == value
def test_fable_5_1_geo_multiplier_without_fast_mode():
"""Fable 5.1 has no fast mode, so a ``fast`` key here would misprice
``speed='fast'`` requests."""
model_data = _load_root_cost_map()
assert model_data["claude-fable-5-1"]["provider_specific_entry"] == {"us": 1.1}
def test_fable_5_1_present_in_bundled_backup():
@ -334,11 +174,6 @@ def test_fable_5_1_registered_for_bedrock_converse():
assert "anthropic.claude-fable-5-1" in BEDROCK_CONVERSE_MODELS
def test_fable_5_1_provider_resolves_via_model_info(local_model_cost_map):
info = litellm.get_model_info(model="claude-fable-5-1")
assert info["litellm_provider"] == "anthropic"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
@pytest.mark.parametrize(

82
tests/test_litellm/test_claude_haiku_4_5_config.py
View file

@ -7,55 +7,6 @@ import json
import os
def test_bedrock_haiku_4_5_configuration():
"""Test that all Bedrock Claude Haiku 4.5 models use bedrock_converse provider"""
# Load model configuration
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path) as f:
model_data = json.load(f)
# All Bedrock Haiku 4.5 variants that should use bedrock_converse
bedrock_haiku_models = [
"anthropic.claude-haiku-4-5-20251001-v1:0",
"anthropic.claude-haiku-4-5@20251001",
"us.anthropic.claude-haiku-4-5-20251001-v1:0",
"eu.anthropic.claude-haiku-4-5-20251001-v1:0",
"apac.anthropic.claude-haiku-4-5-20251001-v1:0",
"jp.anthropic.claude-haiku-4-5-20251001-v1:0",
"global.anthropic.claude-haiku-4-5-20251001-v1:0",
"au.anthropic.claude-haiku-4-5-20251001-v1:0",
]
for model in bedrock_haiku_models:
assert model in model_data, f"Model {model} not found in config"
model_info = model_data[model]
# Verify uses bedrock_converse (not legacy bedrock provider)
assert (
model_info["litellm_provider"] == "bedrock_converse"
), f"{model} should use bedrock_converse provider, got {model_info['litellm_provider']}"
# Verify supports vision (key missing capability)
assert (
model_info.get("supports_vision") is True
), f"{model} should support vision"
# Verify core capabilities
assert model_info.get("supports_computer_use") is True
assert model_info.get("supports_function_calling") is True
assert model_info.get("supports_tool_choice") is True
assert model_info.get("supports_prompt_caching") is True
assert model_info.get("supports_response_schema") is True
assert model_info.get("supports_pdf_input") is True
assert model_info.get("supports_assistant_prefill") is True
assert model_info.get("supports_reasoning") is True
# Verify token limits
assert model_info["max_input_tokens"] == 200000
assert model_info["max_output_tokens"] == 64000
assert model_info["mode"] == "chat"
def test_bedrock_haiku_4_5_matches_sonnet_capabilities():
@ -97,36 +48,3 @@ def test_bedrock_haiku_4_5_matches_sonnet_capabilities():
assert haiku_info.get(capability) == sonnet_info.get(
capability
), f"Capability {capability} mismatch: Haiku={haiku_info.get(capability)}, Sonnet={sonnet_info.get(capability)}"
def test_anthropic_api_haiku_4_5_configuration():
"""Test that Anthropic API Claude Haiku 4.5 has correct configuration"""
# Load model configuration
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path) as f:
model_data = json.load(f)
# Anthropic API models (not Bedrock)
anthropic_models = [
"claude-haiku-4-5-20251001",
"claude-haiku-4-5",
]
for model in anthropic_models:
assert model in model_data, f"Model {model} not found in config"
model_info = model_data[model]
# Should use anthropic provider (not bedrock)
assert (
model_info["litellm_provider"] == "anthropic"
), f"{model} should use anthropic provider"
# Should support vision
assert (
model_info.get("supports_vision") is True
), f"{model} should support vision"
# Should have larger output token limit (64K for Anthropic API)
assert model_info["max_output_tokens"] == 64000

115
tests/test_litellm/test_claude_opus_4_6_config.py
View file

@ -71,123 +71,8 @@ def test_claude_4_6_australia_region_uses_au_prefix_not_apac():
), "apac.anthropic.claude-sonnet-4-6 should not be in bedrock_converse_models"
def test_opus_4_6_model_pricing_and_capabilities():
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path) as f:
model_data = json.load(f)
expected_models = {
"claude-opus-4-6": {
"provider": "anthropic",
"has_long_context_pricing": False,
"max_input_tokens": 1000000,
},
"claude-opus-4-6-20260205": {
"provider": "anthropic",
"has_long_context_pricing": False,
"max_input_tokens": 1000000,
},
"anthropic.claude-opus-4-6-v1": {
"provider": "bedrock_converse",
"has_long_context_pricing": False,
"max_input_tokens": 1000000,
},
"vertex_ai/claude-opus-4-6": {
"provider": "vertex_ai-anthropic_models",
"has_long_context_pricing": False,
"max_input_tokens": 1000000,
},
"azure_ai/claude-opus-4-6": {
"provider": "azure_ai",
"has_long_context_pricing": False,
"max_input_tokens": 1000000,
},
}
for model_name, config in expected_models.items():
assert model_name in model_data, f"Missing model entry: {model_name}"
info = model_data[model_name]
assert info["litellm_provider"] == config["provider"]
assert info["mode"] == "chat"
assert info["max_input_tokens"] == config["max_input_tokens"]
assert info["max_output_tokens"] == 128000
assert info["max_tokens"] == 128000
assert info["input_cost_per_token"] == 5e-06
assert info["output_cost_per_token"] == 2.5e-05
assert info["cache_creation_input_token_cost"] == 6.25e-06
assert info["cache_read_input_token_cost"] == 5e-07
if config["has_long_context_pricing"]:
assert info["input_cost_per_token_above_200k_tokens"] == 1e-05
assert info["output_cost_per_token_above_200k_tokens"] == 3.75e-05
assert info["cache_creation_input_token_cost_above_200k_tokens"] == 1.25e-05
assert info["cache_read_input_token_cost_above_200k_tokens"] == 1e-06
else:
assert "input_cost_per_token_above_200k_tokens" not in info
assert "output_cost_per_token_above_200k_tokens" not in info
assert "cache_creation_input_token_cost_above_200k_tokens" not in info
assert "cache_read_input_token_cost_above_200k_tokens" not in info
assert info["supports_assistant_prefill"] is False
assert info["supports_function_calling"] is True
assert info["supports_prompt_caching"] is True
assert info["supports_reasoning"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
def test_opus_4_6_bedrock_regional_model_pricing():
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path) as f:
model_data = json.load(f)
expected_models = {
"global.anthropic.claude-opus-4-6-v1": {
"input_cost_per_token": 5e-06,
"output_cost_per_token": 2.5e-05,
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
},
"us.anthropic.claude-opus-4-6-v1": {
"input_cost_per_token": 5.5e-06,
"output_cost_per_token": 2.75e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_read_input_token_cost": 5.5e-07,
},
"eu.anthropic.claude-opus-4-6-v1": {
"input_cost_per_token": 5.5e-06,
"output_cost_per_token": 2.75e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_read_input_token_cost": 5.5e-07,
},
"au.anthropic.claude-opus-4-6-v1": {
"input_cost_per_token": 5.5e-06,
"output_cost_per_token": 2.75e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_read_input_token_cost": 5.5e-07,
},
}
for model_name, expected in expected_models.items():
assert model_name in model_data, f"Missing model entry: {model_name}"
info = model_data[model_name]
assert info["litellm_provider"] == "bedrock_converse"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
assert info["max_tokens"] == 128000
assert info["supports_assistant_prefill"] is False
assert "input_cost_per_token_above_200k_tokens" not in info
assert "output_cost_per_token_above_200k_tokens" not in info
assert "cache_creation_input_token_cost_above_200k_tokens" not in info
assert "cache_read_input_token_cost_above_200k_tokens" not in info
for key, value in expected.items():
assert info[key] == value
def test_opus_4_6_alias_and_dated_metadata_match():

122
tests/test_litellm/test_claude_opus_4_8_config.py
View file

@ -16,7 +16,6 @@ import os
import pytest
import litellm
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
@ -30,99 +29,8 @@ def _load_root_cost_map() -> dict:
def test_opus_4_8_model_pricing_and_capabilities():
model_data = _load_root_cost_map()
expected_models = {
"claude-opus-4-8": {
"provider": "anthropic",
"max_input_tokens": 1000000,
},
"anthropic.claude-opus-4-8": {
"provider": "bedrock_converse",
"max_input_tokens": 1000000,
},
"vertex_ai/claude-opus-4-8": {
"provider": "vertex_ai-anthropic_models",
"max_input_tokens": 1000000,
},
"azure_ai/claude-opus-4-8": {
"provider": "azure_ai",
"max_input_tokens": 1000000,
},
}
for model_name, config in expected_models.items():
assert model_name in model_data, f"Missing model entry: {model_name}"
info = model_data[model_name]
assert info["litellm_provider"] == config["provider"]
assert info["mode"] == "chat"
assert info["max_input_tokens"] == config["max_input_tokens"]
assert info["max_output_tokens"] == 128000
assert info["max_tokens"] == 128000
# Base pricing matches Opus 4.7: $5 / $25 per MTok, with the standard
# 1.25x cache-write and 0.1x cache-read multipliers.
assert info["input_cost_per_token"] == 5e-06
assert info["output_cost_per_token"] == 2.5e-05
assert info["cache_creation_input_token_cost"] == 6.25e-06
assert info["cache_read_input_token_cost"] == 5e-07
# Opus 4.x flagships are flat-rate across the full context window.
assert "input_cost_per_token_above_200k_tokens" not in info
assert "output_cost_per_token_above_200k_tokens" not in info
assert info["supports_assistant_prefill"] is False
assert info["supports_function_calling"] is True
assert info["supports_prompt_caching"] is True
assert info["supports_reasoning"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
assert model_data["claude-opus-4-8"]["supports_native_structured_output"] is True
def test_opus_4_8_bedrock_regional_model_pricing():
model_data = _load_root_cost_map()
# Global endpoints use base pricing; regional endpoints carry a 10% premium.
expected_models = {
"global.anthropic.claude-opus-4-8": {
"input_cost_per_token": 5e-06,
"output_cost_per_token": 2.5e-05,
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
},
"us.anthropic.claude-opus-4-8": {
"input_cost_per_token": 5.5e-06,
"output_cost_per_token": 2.75e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_read_input_token_cost": 5.5e-07,
},
"eu.anthropic.claude-opus-4-8": {
"input_cost_per_token": 5.5e-06,
"output_cost_per_token": 2.75e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_read_input_token_cost": 5.5e-07,
},
"au.anthropic.claude-opus-4-8": {
"input_cost_per_token": 5.5e-06,
"output_cost_per_token": 2.75e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_read_input_token_cost": 5.5e-07,
},
}
for model_name, expected in expected_models.items():
assert model_name in model_data, f"Missing model entry: {model_name}"
info = model_data[model_name]
assert info["litellm_provider"] == "bedrock_converse"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
assert info["bedrock_output_config_effort_ceiling"] == "xhigh"
for key, value in expected.items():
assert info[key] == value
def test_opus_4_8_fast_mode_multiplier():
@ -134,42 +42,12 @@ def test_opus_4_8_fast_mode_multiplier():
assert entry["fast"] == 2.0
def test_opus_4_8_present_in_bundled_backup():
"""The bundled backup is the runtime fallback (and what tests load with
``LITELLM_LOCAL_MODEL_COST_MAP=True``) — it must carry the same entries as
the root cost map, otherwise the model resolves on one path but not the
other."""
backup = GetModelCostMap.load_local_model_cost_map()
for model_name in (
"claude-opus-4-8",
"anthropic.claude-opus-4-8",
"global.anthropic.claude-opus-4-8",
"us.anthropic.claude-opus-4-8",
"eu.anthropic.claude-opus-4-8",
"au.anthropic.claude-opus-4-8",
"vertex_ai/claude-opus-4-8",
"vertex_ai/claude-opus-4-8@default",
"azure_ai/claude-opus-4-8",
):
assert model_name in backup, f"Missing from backup cost map: {model_name}"
assert backup["claude-opus-4-8"]["supports_native_structured_output"] is True
def test_opus_4_8_registered_for_bedrock_converse():
assert "anthropic.claude-opus-4-8" in BEDROCK_CONVERSE_MODELS
def test_opus_4_8_provider_resolves_via_model_info(local_model_cost_map):
"""Regression: ``claude-opus-4-8`` must resolve to provider ``anthropic``.
Before the cost-map entry existed, the model was unknown to LiteLLM, so it
could not be tied to the ``anthropic`` provider and an ``anthropic/*``
wildcard deployment would not match it.
"""
info = litellm.get_model_info(model="claude-opus-4-8")
assert info["litellm_provider"] == "anthropic"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
@pytest.mark.parametrize(

91
tests/test_litellm/test_claude_opus_5_config.py
View file

@ -17,7 +17,6 @@ import os
import pytest
import litellm
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
@ -53,88 +52,8 @@ def _load_root_cost_map() -> dict:
def test_opus_5_pricing_and_capabilities():
model_data = _load_root_cost_map()
expected_providers = {
"claude-opus-5": "anthropic",
"anthropic.claude-opus-5": "bedrock_converse",
"vertex_ai/claude-opus-5": "vertex_ai-anthropic_models",
"azure_ai/claude-opus-5": "azure_ai",
}
for model_name, provider in expected_providers.items():
assert model_name in model_data, f"Missing model entry: {model_name}"
info = model_data[model_name]
assert info["litellm_provider"] == provider
assert info["mode"] == "chat"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
assert info["max_tokens"] == 128000
# Opus 5 ships at Opus 4.8's rates: $5 / $25 per MTok, with the standard
# 1.25x cache-write, 2x 1-hour cache-write, and 0.1x cache-read multipliers.
assert info["input_cost_per_token"] == 5e-06
assert info["output_cost_per_token"] == 2.5e-05
assert info["cache_creation_input_token_cost"] == 6.25e-06
assert info["cache_creation_input_token_cost_above_1hr"] == 1e-05
assert info["cache_read_input_token_cost"] == 5e-07
# Flat rate across the full 1M window, no long-context premium.
assert "input_cost_per_token_above_200k_tokens" not in info
assert "output_cost_per_token_above_200k_tokens" not in info
# gen-5 adaptive-thinking profile: effort-driven, no sampling params, no
# assistant prefill.
assert info["supports_adaptive_thinking"] is True
assert info["supports_reasoning"] is True
assert info["supports_sampling_params"] is False
assert info["supports_assistant_prefill"] is False
assert info["supports_xhigh_reasoning_effort"] is True
assert info["supports_max_reasoning_effort"] is True
assert info["supports_function_calling"] is True
assert info["supports_prompt_caching"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
def test_opus_5_bedrock_regional_pricing():
"""Global/base endpoints use base pricing; the us./eu./au./jp. regional
cross-region inference profiles carry a 10% premium."""
model_data = _load_root_cost_map()
base_pricing = {
"input_cost_per_token": 5e-06,
"output_cost_per_token": 2.5e-05,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
}
regional_pricing = {
"input_cost_per_token": 5.5e-06,
"output_cost_per_token": 2.75e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
}
expected = {
"anthropic.claude-opus-5": base_pricing,
"global.anthropic.claude-opus-5": base_pricing,
"us.anthropic.claude-opus-5": regional_pricing,
"eu.anthropic.claude-opus-5": regional_pricing,
"au.anthropic.claude-opus-5": regional_pricing,
"jp.anthropic.claude-opus-5": regional_pricing,
}
for model_name, pricing in expected.items():
assert model_name in model_data, f"Missing model entry: {model_name}"
info = model_data[model_name]
assert info["litellm_provider"] == "bedrock_converse"
for key, value in pricing.items():
assert info[key] == value, f"{model_name}.{key} = {info[key]}, want {value}"
@pytest.mark.parametrize("model_name", BEDROCK_OPUS_5_VARIANTS)
@ -216,16 +135,6 @@ def test_opus_5_registered_for_bedrock_converse():
assert "anthropic.claude-opus-5" in BEDROCK_CONVERSE_MODELS
def test_opus_5_provider_resolves_via_model_info(local_model_cost_map):
"""Regression: ``claude-opus-5`` must resolve to provider ``anthropic``.
Without the cost-map entry the model is unknown to LiteLLM, so it cannot be
tied to the ``anthropic`` provider and an ``anthropic/*`` wildcard deployment
would not match it."""
info = litellm.get_model_info(model="claude-opus-5")
assert info["litellm_provider"] == "anthropic"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
@pytest.mark.parametrize(

96
tests/test_litellm/test_claude_sonnet_5_config.py
View file

@ -15,7 +15,6 @@ import os
import pytest
import litellm
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
@ -42,93 +41,8 @@ def _load_root_cost_map() -> dict:
def test_sonnet_5_pricing_and_capabilities():
model_data = _load_root_cost_map()
expected_providers = {
"claude-sonnet-5": "anthropic",
"anthropic.claude-sonnet-5": "bedrock_converse",
"vertex_ai/claude-sonnet-5": "vertex_ai-anthropic_models",
"azure_ai/claude-sonnet-5": "azure_ai",
}
for model_name, provider in expected_providers.items():
assert model_name in model_data, f"Missing model entry: {model_name}"
info = model_data[model_name]
assert info["litellm_provider"] == provider
assert info["mode"] == "chat"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
assert info["max_tokens"] == 128000
# Introductory Sonnet 5 pricing through 2026-08-31: $2 / $10 per MTok,
# with the 1.25x cache-write and 0.1x cache-read multipliers. On
# 2026-09-01 flip these five fields back to the sticker rate, here and
# in both cost-map JSON files (all ten claude-sonnet-5 entries):
# input_cost_per_token: 3e-06
# output_cost_per_token: 1.5e-05
# cache_creation_input_token_cost: 3.75e-06
# cache_creation_input_token_cost_above_1hr: 6e-06
# cache_read_input_token_cost: 3e-07
# Regional Bedrock profiles (us./eu./au./jp.) stay at 1.1x those values:
# 3.3e-06 / 1.65e-05 / 4.125e-06 / 6.6e-06 / 3.3e-07 (see
# test_sonnet_5_bedrock_regional_pricing below).
assert info["input_cost_per_token"] == 2e-06
assert info["output_cost_per_token"] == 1e-05
assert info["cache_creation_input_token_cost"] == 2.5e-06
assert info["cache_creation_input_token_cost_above_1hr"] == 4e-06
assert info["cache_read_input_token_cost"] == 2e-07
# gen-5 adaptive-thinking profile: effort-driven, no sampling params, no
# assistant prefill.
assert info["supports_adaptive_thinking"] is True
assert info["supports_reasoning"] is True
assert info["supports_sampling_params"] is False
assert info["supports_assistant_prefill"] is False
assert info["supports_function_calling"] is True
assert info["supports_prompt_caching"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
def test_sonnet_5_bedrock_regional_pricing():
"""Global/base endpoints use base pricing; the us./eu./au./jp. regional
cross-region inference profiles carry a 10% premium."""
model_data = _load_root_cost_map()
base_pricing = {
"input_cost_per_token": 2e-06,
"output_cost_per_token": 1e-05,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
}
regional_pricing = {
"input_cost_per_token": 2.2e-06,
"output_cost_per_token": 1.1e-05,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_1hr": 4.4e-06,
"cache_read_input_token_cost": 2.2e-07,
}
expected = {
"anthropic.claude-sonnet-5": base_pricing,
"global.anthropic.claude-sonnet-5": base_pricing,
"us.anthropic.claude-sonnet-5": regional_pricing,
"eu.anthropic.claude-sonnet-5": regional_pricing,
"au.anthropic.claude-sonnet-5": regional_pricing,
"jp.anthropic.claude-sonnet-5": regional_pricing,
}
for model_name, pricing in expected.items():
assert model_name in model_data, f"Missing model entry: {model_name}"
info = model_data[model_name]
assert info["litellm_provider"] == "bedrock_converse"
assert info["bedrock_output_config_effort_ceiling"] == "xhigh"
for key, value in pricing.items():
assert info[key] == value, f"{model_name}.{key} = {info[key]}, want {value}"
def test_sonnet_5_present_in_bundled_backup():
@ -144,16 +58,6 @@ def test_sonnet_5_registered_for_bedrock_converse():
assert "anthropic.claude-sonnet-5" in BEDROCK_CONVERSE_MODELS
def test_sonnet_5_provider_resolves_via_model_info(local_model_cost_map):
"""Regression: ``claude-sonnet-5`` must resolve to provider ``anthropic``.
Before the cost-map entry existed, the model was unknown to LiteLLM, so it
could not be tied to the ``anthropic`` provider and an ``anthropic/*``
wildcard deployment would not match it."""
info = litellm.get_model_info(model="claude-sonnet-5")
assert info["litellm_provider"] == "anthropic"
assert info["max_input_tokens"] == 1000000
assert info["max_output_tokens"] == 128000
@pytest.mark.parametrize(

45
tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py
View file

@ -27,15 +27,6 @@ BACKUP_MAP = os.path.join(
)
@pytest.fixture(autouse=True)
def _use_local_model_cost_map(monkeypatch):
original_model_cost = litellm.model_cost
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
litellm.model_cost = litellm.get_model_cost_map(url="")
try:
yield
finally:
litellm.model_cost = original_model_cost
def _load(path: str) -> dict:
@ -47,48 +38,12 @@ def _cloudflare_keys(data: dict) -> set:
return {k for k in data if k.startswith("cloudflare/")}
def test_glm_5_2_entry_is_present_and_well_formed():
entry = litellm.model_cost["cloudflare/@cf/zai-org/glm-5.2"]
assert entry["litellm_provider"] == "cloudflare"
assert entry["mode"] == "chat"
assert entry["supports_function_calling"] is True
assert entry["input_cost_per_token"] > 0
assert entry["output_cost_per_token"] > 0
def test_vision_model_is_flagged_supports_vision():
entry = litellm.model_cost["cloudflare/@cf/meta/llama-3.2-11b-vision-instruct"]
assert entry["litellm_provider"] == "cloudflare"
assert entry.get("supports_vision") is True
def test_additional_current_models_are_present():
for key in (
"cloudflare/@cf/openai/gpt-oss-120b",
"cloudflare/@cf/meta/llama-3.3-70b-instruct-fp8-fast",
):
entry = litellm.model_cost[key]
assert entry["litellm_provider"] == "cloudflare"
assert entry["mode"] == "chat"
assert entry["supports_function_calling"] is True
assert entry["input_cost_per_token"] > 0
assert entry["output_cost_per_token"] > 0
@pytest.mark.parametrize(
"key, published_price_per_audio_minute",
[
("cloudflare/@cf/openai/whisper", 0.00045),
("cloudflare/@cf/openai/whisper-large-v3-turbo", 0.00051),
],
)
def test_whisper_transcription_pricing_is_stored_per_second(key, published_price_per_audio_minute):
entry = litellm.model_cost[key]
assert entry["litellm_provider"] == "cloudflare"
assert entry["mode"] == "audio_transcription"
assert entry["supported_endpoints"] == ["/v1/audio/transcriptions"]
assert entry["output_cost_per_second"] == 0.0
assert entry["input_cost_per_second"] == pytest.approx(published_price_per_audio_minute / 60)
def test_root_and_backup_have_identical_cloudflare_keys():

14
tests/test_litellm/test_daybreak_model_metadata.py
View file

@ -32,20 +32,6 @@ def _load(path):
return json.load(f)
@pytest.mark.parametrize("model", DAYBREAK_MODELS)
def test_daybreak_capability_contract(model):
info = _load(MAIN_PATH).get(model)
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
assert info["litellm_provider"] == "openai"
assert info["mode"] == "chat"
assert info["supported_endpoints"] == ["/v1/chat/completions", "/v1/responses"]
assert info["supports_computer_use"] is True
assert info["supports_parallel_function_calling"] is True
assert info["supports_function_calling"] is True
assert info["supports_reasoning"] is True
assert info["supports_vision"] is True
def test_blue_alias_matches_its_snapshot_computer_use():

34
tests/test_litellm/test_deepseek_model_metadata.py
View file

@ -39,25 +39,9 @@ class TestDeepSeekModelCostEntries:
"""Verify that provider-prefixed DeepSeek entries contain the same
capability flags as their bare-name counterparts in the JSON files."""
def test_deepseek_chat_supports_response_schema_in_backup(self):
data = _load_backup_json()
entry = data.get("deepseek/deepseek-chat", {})
assert entry.get("supports_response_schema") is True
def test_deepseek_reasoner_supports_response_schema_in_backup(self):
data = _load_backup_json()
entry = data.get("deepseek/deepseek-reasoner", {})
assert entry.get("supports_response_schema") is True
def test_deepseek_chat_supports_system_messages_in_backup(self):
data = _load_backup_json()
entry = data.get("deepseek/deepseek-chat", {})
assert entry.get("supports_system_messages") is True
def test_deepseek_reasoner_supports_system_messages_in_backup(self):
data = _load_backup_json()
entry = data.get("deepseek/deepseek-reasoner", {})
assert entry.get("supports_system_messages") is True
def test_deepseek_chat_max_input_tokens_matches_bare_in_backup(self):
data = _load_backup_json()
@ -71,25 +55,7 @@ class TestDeepSeekModelCostEntries:
prefixed = data.get("deepseek/deepseek-reasoner", {})
assert prefixed.get("max_output_tokens") == bare.get("max_output_tokens")
def test_main_json_deepseek_chat_supports_response_schema(self):
main_path = os.path.join(
os.path.dirname(os.path.dirname(litellm.__file__)),
"model_prices_and_context_window.json",
)
with open(main_path, encoding="utf-8") as f:
data = json.load(f)
entry = data.get("deepseek/deepseek-chat", {})
assert entry.get("supports_response_schema") is True
def test_main_json_deepseek_reasoner_supports_response_schema(self):
main_path = os.path.join(
os.path.dirname(os.path.dirname(litellm.__file__)),
"model_prices_and_context_window.json",
)
with open(main_path, encoding="utf-8") as f:
data = json.load(f)
entry = data.get("deepseek/deepseek-reasoner", {})
assert entry.get("supports_response_schema") is True
# ---------------------------------------------------------------------------

26
tests/test_litellm/test_fireworks_serverless_model_costs.py
View file

@ -14,24 +14,9 @@ import os
import pytest
import litellm
from litellm.utils import get_model_info
@pytest.fixture(scope="module", autouse=True)
def _local_model_cost_map():
"""
Point litellm at the bundled cost map for the duration of this module
only. ``mp.undo()`` restores both the environment variable and
``litellm.model_cost`` so nothing leaks into later tests.
"""
mp = pytest.MonkeyPatch()
mp.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
mp.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
get_model_info.cache_clear()
yield
mp.undo()
get_model_info.cache_clear()
NEW_ENTRIES = {
@ -54,17 +39,6 @@ def model_data():
return json.load(f)
def test_fireworks_serverless_entries_exist(model_data):
"""The new prefixed entry carries the pricing and metadata from #37274."""
for key, expected in NEW_ENTRIES.items():
assert key in model_data, f"{key} is missing from model_prices_and_context_window.json"
entry = model_data[key]
for field, value in expected.items():
assert entry[field] == pytest.approx(value), f"{key}.{field}"
assert entry["litellm_provider"] == "fireworks_ai"
assert entry["mode"] == "chat"
assert entry["supports_function_calling"] is True
assert entry["supports_vision"] is False
def test_bare_fireworks_ids_resolve_through_prefixed_entries():

44
tests/test_litellm/test_gpt_5_5_model_metadata.py
View file

@ -1,53 +1,9 @@
import json
from pathlib import Path
import pytest
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
@pytest.mark.parametrize("model", ["azure_ai/gpt-5.5", "azure_ai/gpt-5.5-2026-04-23"])
def test_azure_ai_gpt_5_5_model_info(model):
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
with open(json_path) as f:
model_cost = json.load(f)
info = model_cost.get(model)
assert (
info is not None
), f"{model} not found in model_prices_and_context_window.json"
assert info["litellm_provider"] == "azure_ai"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] == 5e-06
assert info["output_cost_per_token"] == 3e-05
assert info["cache_read_input_token_cost"] == 5e-07
assert info["input_cost_per_token_above_272k_tokens"] == 1e-05
assert info["output_cost_per_token_above_272k_tokens"] == 4.5e-05
assert info["cache_read_input_token_cost_above_272k_tokens"] == 1e-06
assert info["input_cost_per_token_priority"] == 1e-05
assert info["output_cost_per_token_priority"] == 6e-05
assert info["max_input_tokens"] == 1050000
assert info["max_output_tokens"] == 128000
assert info["max_tokens"] == 128000
assert info["supports_function_calling"] is True
assert info["supports_prompt_caching"] is True
assert info["supports_reasoning"] is True
assert info["supports_response_schema"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
assert info["supports_web_search"] is True
# gpt-5.5 dropped minimal reasoning effort support (true on gpt-5.4)
assert info["supports_minimal_reasoning_effort"] is False
routed_model, provider, _, _ = get_llm_provider(model=model)
assert routed_model == model.split("/", 1)[1]
assert provider == "azure_ai"
def test_azure_ai_gpt_5_5_backup_matches_main():

27
tests/test_litellm/test_gpt_realtime_mode.py
View file

@ -1,10 +1,8 @@
import json
from pathlib import Path
import pytest
from typing_extensions import get_args, get_type_hints
import litellm
from litellm.types.utils import ModelInfoBase
REALTIME_ONLY_GPT_MODELS = (
@ -43,10 +41,6 @@ REALTIME_ONLY_GPT_MODELS_WITHOUT_ENDPOINTS = (
ALL_REALTIME_ONLY_GPT_MODELS = REALTIME_ONLY_GPT_MODELS + REALTIME_ONLY_GPT_MODELS_WITHOUT_ENDPOINTS
def _load_cost_map() -> dict:
json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
with open(json_path) as f:
return json.load(f)
def test_realtime_is_a_valid_mode_literal():
@ -54,31 +48,10 @@ def test_realtime_is_a_valid_mode_literal():
assert "realtime" in get_args(hints["mode"])
@pytest.mark.parametrize("model", REALTIME_ONLY_GPT_MODELS)
def test_realtime_only_gpt_models_are_mode_realtime(model):
"""These models only serve /v1/realtime and are rejected by /v1/chat/completions
("This is not a chat model ..."), so they must not be tagged mode=chat."""
info = _load_cost_map()[model]
assert info["supported_endpoints"] == ["/v1/realtime"]
assert info["mode"] == "realtime"
@pytest.mark.parametrize("model", REALTIME_ONLY_GPT_MODELS_WITHOUT_ENDPOINTS)
def test_realtime_only_gpt_4o_models_are_mode_realtime(model):
"""gpt-4o(-mini)-realtime-preview are realtime-only and must not be mode=chat."""
assert _load_cost_map()[model]["mode"] == "realtime"
def test_get_model_info_reports_realtime_mode(monkeypatch):
"""get_model_info must resolve the retag against the bundled cost map, not the
hosted map fetched from main, which lags this repo until the next promotion."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
try:
assert litellm.get_model_info("gpt-realtime-mini")["mode"] == "realtime"
finally:
litellm.get_model_info.cache_clear()
def test_backup_matches_main_for_realtime_models():

45
tests/test_litellm/test_mistral_medium_3_5_model_metadata.py
View file

@ -3,8 +3,6 @@ from pathlib import Path
import pytest
import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
@ -28,53 +26,10 @@ def _load(path):
@pytest.mark.parametrize("model", MEDIUM_3_5_MODELS)
def test_medium_3_5_specs(model):
info = _load(MAIN_PATH).get(model)
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
assert info["litellm_provider"] == "mistral"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] == 1.5e-06
assert info["output_cost_per_token"] == 7.5e-06
assert info["max_input_tokens"] == 262144
assert info["max_output_tokens"] == 262144
assert info["max_tokens"] == 262144
assert info["supports_reasoning"] is True
assert info["supports_vision"] is True
assert info["supports_function_calling"] is True
assert info["supports_response_schema"] is True
assert info["supports_tool_choice"] is True
assert info["supports_assistant_prefill"] is True
routed_model, provider, _, _ = get_llm_provider(model=model)
assert routed_model == model.split("/", 1)[1]
assert provider == "mistral"
def test_mistral_medium_latest_resolves_to_medium_3_5(local_model_cost_map):
"""LIT-3883: the -latest alias was retargeted to Medium 3.5; get_model_info must
return the 3.5 pricing/context/reasoning, not the stale Medium 3.1 values."""
info = litellm.get_model_info(model="mistral/mistral-medium-latest")
assert info["input_cost_per_token"] == 1.5e-06
assert info["output_cost_per_token"] == 7.5e-06
assert info["max_input_tokens"] == 262144
assert info["supports_reasoning"] is True
def test_mistral_medium_2508_keeps_medium_3_1_specs():
"""The date-pinned 2508 alias is Medium 3.1 and must not inherit 3.5 pricing."""
info = _load(MAIN_PATH).get("mistral/mistral-medium-2508")
assert info is not None, "mistral/mistral-medium-2508 missing from cost map"
assert info["input_cost_per_token"] == 4e-07
assert info["output_cost_per_token"] == 2e-06
assert info["max_input_tokens"] == 131072
assert info.get("supports_reasoning") is not True
@pytest.mark.parametrize("model", SYNCED_MODELS)

21
tests/test_litellm/test_mistral_small_4_0_model_metadata.py
View file

@ -18,27 +18,6 @@ def _load(path):
return json.load(f)
@pytest.mark.parametrize("model", SMALL_4_0_MODELS)
def test_small_4_0_specs(model):
info = _load(MAIN_PATH).get(model)
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
assert info["litellm_provider"] == "mistral"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] == 1.5e-07
assert info["output_cost_per_token"] == 6e-07
assert info["max_input_tokens"] == 262144
assert info["max_output_tokens"] == 262144
assert info["max_tokens"] == 262144
assert info["supports_reasoning"] is True
assert info["supports_vision"] is True
assert info["supports_function_calling"] is True
assert info["supports_response_schema"] is True
assert info["supports_tool_choice"] is True
assert info["supports_assistant_prefill"] is True
@pytest.mark.parametrize("model", SMALL_4_0_MODELS)

37
tests/test_litellm/test_muse_spark_1_2_model_metadata.py
View file

@ -24,43 +24,6 @@ def _load_cost_map(filename: str = "model_prices_and_context_window.json") -> di
@pytest.mark.parametrize("model, input_cost, cached_cost, output_cost", PRICING)
def test_muse_spark_1_2_model_info(model: str, input_cost: float, cached_cost: float, output_cost: float):
info = _load_cost_map().get(model)
assert info is not None, f"{model} not found in model_prices_and_context_window.json"
assert info["litellm_provider"] == "meta"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] == input_cost
assert info["output_cost_per_token"] == output_cost
assert info["cache_read_input_token_cost"] == cached_cost
assert info["max_input_tokens"] == 1048576
assert info["max_output_tokens"] == 131072
assert info["max_tokens"] == 131072
assert info["supports_function_calling"] is True
assert info["supports_parallel_function_calling"] is True
assert info["supports_prompt_caching"] is True
assert info["supports_reasoning"] is True
assert info["supports_response_schema"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
assert info["supports_pdf_input"] is True
assert info["supports_web_search"] is True
assert info["supports_minimal_reasoning_effort"] is True
assert info["supports_xhigh_reasoning_effort"] is True
assert info["supported_endpoints"] == ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
assert info["supported_modalities"] == ["text", "image", "video"]
assert info["supported_output_modalities"] == ["text"]
assert info["search_context_cost_per_query"] == {
"search_context_size_high": WEB_SEARCH_COST_PER_QUERY,
"search_context_size_low": WEB_SEARCH_COST_PER_QUERY,
"search_context_size_medium": WEB_SEARCH_COST_PER_QUERY,
}
@pytest.mark.parametrize("model, input_cost, cached_cost, output_cost", PRICING)

37
tests/test_litellm/test_muse_spark_1_3_model_metadata.py
View file

@ -24,43 +24,6 @@ def _load_cost_map(filename: str = "model_prices_and_context_window.json") -> di
@pytest.mark.parametrize("model, input_cost, cached_cost, output_cost", PRICING)
def test_muse_spark_1_3_model_info(model: str, input_cost: float, cached_cost: float, output_cost: float):
info = _load_cost_map().get(model)
assert info is not None, f"{model} not found in model_prices_and_context_window.json"
assert info["litellm_provider"] == "meta"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] == input_cost
assert info["output_cost_per_token"] == output_cost
assert info["cache_read_input_token_cost"] == cached_cost
assert info["max_input_tokens"] == 1048576
assert info["max_output_tokens"] == 131072
assert info["max_tokens"] == 131072
assert info["supports_function_calling"] is True
assert info["supports_parallel_function_calling"] is True
assert info["supports_prompt_caching"] is True
assert info["supports_reasoning"] is True
assert info["supports_response_schema"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
assert info["supports_pdf_input"] is True
assert info["supports_web_search"] is True
assert info["supports_minimal_reasoning_effort"] is True
assert info["supports_xhigh_reasoning_effort"] is True
assert info["supported_endpoints"] == ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
assert info["supported_modalities"] == ["text", "image", "video"]
assert info["supported_output_modalities"] == ["text"]
assert info["search_context_cost_per_query"] == {
"search_context_size_high": WEB_SEARCH_COST_PER_QUERY,
"search_context_size_low": WEB_SEARCH_COST_PER_QUERY,
"search_context_size_medium": WEB_SEARCH_COST_PER_QUERY,
}
@pytest.mark.parametrize("model, input_cost, cached_cost, output_cost", PRICING)

6
tests/test_litellm/test_replicate_model_key_format.py
View file

@ -20,12 +20,6 @@ def test_replicate_models_have_valid_key_prefix(model_cost: dict[str, Any]) -> N
)
def test_replicate_openai_gpt_oss_20b_key_exists(model_cost: dict[str, Any]) -> None:
assert "replicate/openai/gpt-oss-20b" in model_cost
info = model_cost["replicate/openai/gpt-oss-20b"]
assert info["litellm_provider"] == "replicate"
assert info["mode"] == "chat"
assert info["supports_function_calling"] is True
def test_replicate_backup_matches_main() -> None:

80
tests/test_litellm/test_together_ai_model_metadata.py
View file

@ -5,7 +5,6 @@ from typing import Final
import pytest
from pydantic import TypeAdapter
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
REPO_ROOT: Final = Path(__file__).parents[2]
@ -77,57 +76,12 @@ def cost_map() -> CostMap:
return COST_MAP_ADAPTER.validate_python(json.load(f))
@pytest.mark.parametrize("model", SERVERLESS_CHAT_MODELS)
def test_together_serverless_chat_model_is_mapped(cost_map: CostMap, model: str):
info = cost_map.get(model)
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
assert info["litellm_provider"] == "together_ai"
assert info["mode"] == "chat"
assert info["input_cost_per_token"] >= 0
assert info["output_cost_per_token"] >= info["input_cost_per_token"]
assert "deprecation_date" not in info
routed_model, provider, _, _ = get_llm_provider(model=model)
assert routed_model == model.removeprefix("together_ai/")
assert provider == "together_ai"
def test_together_kimi_k3_pricing_and_capabilities(cost_map: CostMap):
info = cost_map["together_ai/moonshotai/Kimi-K3"]
assert info["input_cost_per_token"] == 3e-06
assert info["output_cost_per_token"] == 1.5e-05
assert info["max_input_tokens"] == 1048576
assert info["supports_function_calling"] is True
assert info["supports_tool_choice"] is True
assert info["supports_response_schema"] is True
assert info["supports_vision"] is True
assert info["supports_reasoning"] is True
def test_together_glm_52_pricing(cost_map: CostMap):
info = cost_map["together_ai/zai-org/GLM-5.2"]
assert info["input_cost_per_token"] == 1.4e-06
assert info["output_cost_per_token"] == 4.4e-06
assert info["max_input_tokens"] == 1048575
assert info["max_output_tokens"] == 128000
assert info["supports_function_calling"] is True
assert info["supports_reasoning"] is True
def test_together_glm_53_flash_pricing_and_capabilities(cost_map: CostMap):
info = cost_map["together_ai/zai-org/GLM-5.3-Flash"]
assert info["input_cost_per_token"] == 1.5e-07
assert info["output_cost_per_token"] == 5e-07
assert info["cache_read_input_token_cost"] == 3e-08
assert info["max_input_tokens"] == 1048575
assert info["max_output_tokens"] == 128000
assert info["supports_function_calling"] is True
assert info["supports_parallel_function_calling"] is True
assert info["supports_prompt_caching"] is True
assert info["supports_tool_choice"] is True
assert info["supports_response_schema"] is True
assert info["supports_vision"] is True
assert info["supports_reasoning"] is True
def test_together_chat_entries_never_carry_context_length_as_output_ceiling(cost_map: CostMap):
@ -142,19 +96,8 @@ def test_together_chat_entries_never_carry_context_length_as_output_ceiling(cost
assert inflated == []
def test_together_multilingual_e5_embedding_entry(cost_map: CostMap):
info = cost_map["together_ai/intfloat/multilingual-e5-large-instruct"]
assert info["mode"] == "embedding"
assert info["input_cost_per_token"] == 2e-08
assert info["max_input_tokens"] == 514
assert info["output_vector_size"] == 1024
def test_together_llama_33_70b_repriced_to_current_together_rate(cost_map: CostMap):
info = cost_map["together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"]
assert info["input_cost_per_token"] == 1.04e-06
assert info["output_cost_per_token"] == 1.04e-06
assert info["max_input_tokens"] == 131072
@pytest.mark.parametrize("model", sorted(DEPRECATED_MODELS))
@ -210,32 +153,9 @@ CACHED_INPUT_MODELS: Final = (
)
@pytest.mark.parametrize("model", CACHED_INPUT_MODELS)
def test_together_cached_input_model_carries_cache_read_pricing(cost_map: CostMap, model: str):
info = cost_map.get(model)
assert info is not None, f"{model} missing from model_prices_and_context_window.json"
assert info.get("supports_prompt_caching") is True
cache_read = info.get("cache_read_input_token_cost")
assert isinstance(cache_read, float)
assert 0 < cache_read < info["input_cost_per_token"]
assert "cache_creation_input_token_cost" not in info
def test_together_prompt_caching_flag_implies_cache_read_rate(cost_map: CostMap):
for model, info in cost_map.items():
if model.startswith("together_ai/") and info.get("supports_prompt_caching"):
assert "cache_read_input_token_cost" in info, f"{model} flags caching without a cache read rate"
def test_together_deepseek_v4_flash_cache_read_rate(cost_map: CostMap):
info = cost_map["together_ai/deepseek-ai/DeepSeek-V4-Flash-0731"]
assert info["input_cost_per_token"] == 1.4e-07
assert info["cache_read_input_token_cost"] == 3e-08
assert info["output_cost_per_token"] == 2.8e-07
def test_together_qwen_37_max_repriced_to_current_together_rate(cost_map: CostMap):
info = cost_map["together_ai/Qwen/Qwen3.7-Max"]
assert info["input_cost_per_token"] == 2.5e-06
assert info["output_cost_per_token"] == 7.5e-06
assert info["cache_read_input_token_cost"] == 5e-07