diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index e356277ea08..ebee2c64140 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -28,10 +28,8 @@ pub struct ModelInfo { /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub cache_creation_input_token_cost_above_32k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub cache_creation_input_token_cost_above_100k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub cache_creation_input_token_cost_above_100k_tokens_batches: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -39,7 +37,6 @@ pub struct ModelInfo { /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub cache_creation_input_token_cost_above_1hr: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub cache_creation_input_token_cost_above_1hr_above_100k_tokens: Option, /// Rate applied once the prompt exceeds the token threshold in the field name. @@ -90,10 +87,8 @@ pub struct ModelInfo { /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub cache_read_input_token_cost_above_32k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub cache_read_input_token_cost_above_100k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub cache_read_input_token_cost_above_100k_tokens_batches: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -214,10 +209,8 @@ pub struct ModelInfo { /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub input_cost_per_token_above_32k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub input_cost_per_token_above_100k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub input_cost_per_token_above_100k_tokens_batches: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -375,10 +368,8 @@ pub struct ModelInfo { /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub output_cost_per_token_above_32k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub output_cost_per_token_above_100k_tokens: Option, - /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub output_cost_per_token_above_100k_tokens_batches: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index f8f8388b6f7..bd620bc7130 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3946,7 +3946,7 @@ def shared_backend_model_info(model_info: dict[str, Any]) -> dict[str, Any]: return {k: v for k, v in model_info.items() if k in SHARED_BACKEND_MODEL_INFO_FIELDS} -ABOVE_THRESHOLD_COST_KEY_PATTERN: Final = re.compile(r"_above_\d+k?_tokens$") +ABOVE_THRESHOLD_COST_KEY_PATTERN: Final = re.compile(r"_above_\d+k?_tokens(?:_batches)?$") _PRICING_FIELD_EXEMPTIONS: Final[frozenset[str]] = frozenset({"output_vector_size"}) diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 9c5714eeb11..0d896294226 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -3832,6 +3832,11 @@ def test_azure_gpt_5_6_alias_matches_sol_pricing(_local_model_cost_map, region_p assert alias[field] == sol[field], field +# Per-token rates read 2026-10-07 from https://platform.claude.com/docs/en/about-claude/pricing (direct and +# azure_ai, which Microsoft bills at Anthropic's rates per +# https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models-billing) and from the +# AmazonBedrockFoundationModels price list at +# https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json (Bedrock) @pytest.mark.parametrize( ("model", "custom_llm_provider", "prompt_tokens", "input_rate", "cache_read_rate", "output_rate"), [ @@ -3850,24 +3855,62 @@ def test_azure_gpt_5_6_alias_matches_sol_pricing(_local_model_cost_map, region_p ], ) def test_generic_cost_per_token_claude_haiku_5_5_prompt_length_tiers( - _local_model_cost_map, model, custom_llm_provider, prompt_tokens, input_rate, cache_read_rate, output_rate -): + _local_model_cost_map: None, + model: str, + custom_llm_provider: str, + prompt_tokens: int, + input_rate: float, + cache_read_rate: float, + output_rate: float, +) -> None: """Claude Haiku 5.5 bills every token at 5x the base rates once the prompt is over 100,000 tokens.""" - cached_tokens = 10_000 - completion_tokens = 1_000 - usage = Usage( + cached_tokens: Final = 10_000 + completion_tokens: Final = 1_000 + usage: Final = Usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), ) + prompt_cost, completion_cost = generic_cost_per_token( model=model, usage=usage, custom_llm_provider=custom_llm_provider, ) - expected_prompt = (prompt_tokens - cached_tokens) * input_rate + cached_tokens * cache_read_rate - assert prompt_cost == pytest.approx(expected_prompt) + + assert prompt_cost == pytest.approx((prompt_tokens - cached_tokens) * input_rate + cached_tokens * cache_read_rate) + assert completion_cost == pytest.approx(completion_tokens * output_rate) + + +# Batch rates read 2026-10-07 from the Batch processing table at +# https://platform.claude.com/docs/en/about-claude/pricing: $0.05 / $0.25 per MTok input and $0.25 / $1.25 output, +# up to and over 100,000 prompt tokens +@pytest.mark.parametrize( + ("prompt_tokens", "input_rate", "output_rate"), + [(100_000, 5e-08, 2.5e-07), (100_001, 2.5e-07, 1.25e-06)], +) +def test_batch_cost_calculator_claude_haiku_5_5_prompt_length_tiers( + _local_model_cost_map: None, + prompt_tokens: int, + input_rate: float, + output_rate: float, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + completion_tokens: Final = 1_000 + + prompt_cost, completion_cost = batch_cost_calculator( + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + model="claude-haiku-5-5", + custom_llm_provider="anthropic", + ) + + assert prompt_cost == pytest.approx(prompt_tokens * input_rate) assert completion_cost == pytest.approx(completion_tokens * output_rate) @@ -3878,11 +3921,16 @@ def test_generic_cost_per_token_claude_haiku_5_5_prompt_length_tiers( (100_001, (2.5e-07, 1.25e-06, 2.5e-08, 3.125e-07)), ], ) -def test_get_batch_cost_rates_claude_haiku_5_5_prompt_length_tiers(_local_model_cost_map, prompt_tokens, expected): +def test_get_batch_cost_rates_claude_haiku_5_5_prompt_length_tiers( + _local_model_cost_map: None, + prompt_tokens: int, + expected: tuple[float, float, float, float], +) -> None: + """Cache write and cache read batch rates are 50% of the standard rates; Anthropic's batch table omits them.""" from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates - rates = get_batch_cost_rates( - litellm.model_cost["claude-haiku-5-5"], + rates: Final = get_batch_cost_rates( + litellm.get_model_info(model="claude-haiku-5-5", custom_llm_provider="anthropic"), Usage(prompt_tokens=prompt_tokens, completion_tokens=1, total_tokens=prompt_tokens + 1), "anthropic", )