diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a0da0f6ddca..8b02f93fca4 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -37,6 +37,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( _SERVICE_TIER_TO_COST_KEY_SUFFIX, BilledTokenRates, CostCalculatorUtils, + calculate_cache_writing_cost, calculate_cost_component, generic_cost_per_character, generic_cost_per_token, @@ -2969,7 +2970,12 @@ def batch_cost_calculator( total_completion_cost = 0.0 if batch_rates.input is not None: total_prompt_cost = _batch_prompt_cost( - usage, model_info, batch_rates.input, batch_rates.cache_read, batch_rates.cache_creation + usage, + model_info, + batch_rates.input, + batch_rates.cache_read, + batch_rates.cache_creation, + batch_rates.cache_creation_rates_by_ttl, ) elif input_cost_per_token: details: Final = parse_prompt_tokens_details(usage) @@ -3014,11 +3020,13 @@ def _batch_prompt_cost( input_rate: float, cache_read_rate: float | None, cache_creation_rate: float | None, + cache_creation_rates_by_ttl: Mapping[str, float], ) -> float: details: Final = parse_prompt_tokens_details(usage) cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0 - written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0 + bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl) + written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0 audio_tokens, image_tokens, video_tokens = ( details["audio_tokens"], details["image_tokens"], @@ -3035,7 +3043,16 @@ def _batch_prompt_cost( + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0) + cached_audio_tokens * _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0) - + written_tokens * (cache_creation_rate or 0.0) + + ( + calculate_cache_writing_cost( + written_tokens, + details["cache_creation_token_details"], + input_rate if cache_creation_rate is None else cache_creation_rate, + cache_creation_rates_by_ttl, + ) + if bills_cache_writes + else 0.0 + ) ) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 5d4b1a82759..3b49f239afe 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -104,7 +104,8 @@ _BATCH_RATE_PREFIXES: Final = ( "cache_read_input_token_cost", "cache_creation_input_token_cost", ) -_BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_RATE_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$") +_BATCH_TIER_PREFIXES: Final = (*_BATCH_RATE_PREFIXES, *_TTL_PRICED_CACHE_WRITE_RATE_KEYS.values()) +_BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_TIER_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$") _NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX) @@ -314,6 +315,7 @@ class BatchCostRates: output: float | None cache_read: float | None cache_creation: float | None + cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({})) def _batch_rate(model_info: ModelInfo, key: str) -> float | None: @@ -369,6 +371,13 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide output=_batch_rate_for_prefix(model_info, "output_cost_per_token", usage, inclusive), cache_read=_batch_rate_for_prefix(model_info, "cache_read_input_token_cost", usage, inclusive), cache_creation=_batch_rate_for_prefix(model_info, "cache_creation_input_token_cost", usage, inclusive), + cache_creation_rates_by_ttl=MappingProxyType( + { + ttl: rate + for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items() + if (rate := _batch_rate_for_prefix(model_info, key, usage, inclusive)) is not None + } + ), ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ee8683f8ae8..cc7ef851111 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -328,6 +328,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_creation_input_token_cost_above_12hr_above_272k_tokens: ReadOnly[float | None] cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: ReadOnly[float | None] cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_batches: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_ultrafast: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast: ReadOnly[float | None] cache_creation_input_token_cost_flex: float | None # OpenAI flex service tier pricing cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing @@ -3973,6 +3977,10 @@ class MirroredPricingParams(LiteLLMBaseModel): cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None + cache_creation_input_token_cost_above_12hr_batches: float | None = None + cache_creation_input_token_cost_above_12hr_ultrafast: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast: float | None = None tiered_pricing: list[dict[str, Any]] | None = None diff --git a/litellm/utils.py b/litellm/utils.py index acf4594c6b9..e2820f9c682 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6842,6 +6842,18 @@ def get_model_info_helper( cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority=_model_info.get( "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority", None ), + cache_creation_input_token_cost_above_12hr_batches=_model_info.get( + "cache_creation_input_token_cost_above_12hr_batches", None + ), + cache_creation_input_token_cost_above_12hr_ultrafast=_model_info.get( + "cache_creation_input_token_cost_above_12hr_ultrafast", None + ), + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches=_model_info.get( + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches", None + ), + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast=_model_info.get( + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast", None + ), off_peak_pricing=_model_info.get("off_peak_pricing", None), input_cost_per_character=_model_info.get("input_cost_per_character", None), input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None), diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index e978a9a316a..95363e92366 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -91,6 +91,10 @@ class CostMapEntry(BaseModel): cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None + cache_creation_input_token_cost_above_12hr_batches: float | None = None + cache_creation_input_token_cost_above_12hr_ultrafast: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast: float | None = None cache_creation_input_token_cost_above_1hr_above_200k_tokens: float | None = None cache_read_input_token_cost_above_200k_tokens: float | None = None cache_creation_input_token_cost_above_200k_tokens: float | None = None diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 1729db94ed8..21b527250c4 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2179,6 +2179,48 @@ def test_generic_cost_per_token_falls_back_to_default_for_missing_12h_rate() -> assert result[0] == pytest.approx(100 * model_info["cache_creation_input_token_cost"]) +@pytest.mark.parametrize(("prompt_tokens", "twelve_hour_rate"), [(10_000, 7e-6), (300_000, 14e-6)]) +def test_generic_cost_per_token_prices_ultrafast_12h_cache_writes_at_the_ultrafast_rate( + prompt_tokens: int, twelve_hour_rate: float +) -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_ultrafast": 5e-6, + "cache_creation_input_token_cost": 2e-6, + "cache_creation_input_token_cost_ultrafast": 3e-6, + "cache_creation_input_token_cost_above_12hr": 4e-6, + "cache_creation_input_token_cost_above_12hr_ultrafast": 7e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 8e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": 14e-6, + } + usage: Final = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=0, + total_tokens=prompt_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + cache_creation_tokens=5_000, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_30m_input_tokens=3_000, ephemeral_12h_input_tokens=2_000 + ), + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="test-ultrafast-12h-cache-write", + usage=usage, + custom_llm_provider="openai", + service_tier="ultrafast", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx( + (prompt_tokens - 5_000) * model_info["input_cost_per_token_ultrafast"] + + 3_000 * model_info["cache_creation_input_token_cost_ultrafast"] + + 2_000 * twelve_hour_rate + ) + + def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): """ Regression test: when cache_creation_tokens is 0 but cache_creation_token_details @@ -4079,6 +4121,44 @@ def test_get_batch_cost_rates_has_no_cache_write_rate_without_a_cache_write_batc assert rates.cache_creation is None +@pytest.mark.parametrize(("prompt_tokens", "expected"), [(1_000, 2.5e-7), (272_000, 2.5e-7), (300_000, 5e-7)]) +def test_get_batch_cost_rates_reads_the_12h_batch_cache_write_rate_for_the_crossed_tier( + prompt_tokens: int, expected: float +) -> None: + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates: Final = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-7, + cache_creation_input_token_cost_batches=1.25e-7, + cache_creation_input_token_cost_above_12hr_batches=2.5e-7, + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches=5e-7, + ), + Usage(prompt_tokens=prompt_tokens, completion_tokens=1, total_tokens=prompt_tokens + 1), + "openai", + ) + + assert rates.cache_creation == 1.25e-7 + assert rates.cache_creation_rates_by_ttl == {"12hr": expected} + + +def test_get_batch_cost_rates_has_no_12h_cache_write_rate_without_a_12h_batch_key() -> None: + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates: Final = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-7, + cache_creation_input_token_cost_batches=1.25e-7, + cache_creation_input_token_cost_above_12hr=2.5e-7, + cache_creation_input_token_cost_above_12hr_ultrafast=5e-7, + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert rates.cache_creation_rates_by_ttl == {} + + @pytest.mark.parametrize("model_base", ["gpt-6-sol", "gpt-6-luna"]) def test_azure_gpt_6_foundry_price_sheet(_local_model_cost_map, model_base): """Azure Foundry hosts gpt-6-sol and gpt-6-luna at OpenAI's Global rates, with the diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 550fe5f0d1a..27b25c1f98e 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -35,7 +35,7 @@ from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.rerank import RerankResponse -from litellm.types.utils import CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage +from litellm.types.utils import CacheCreationTokenDetails, CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage from litellm.types.videos.main import VideoObject from litellm.utils import supports_prompt_caching import os @@ -5947,6 +5947,110 @@ def test_batch_cost_calculator_bills_cache_write_tokens_at_the_batch_input_rate_ assert prompt_cost == pytest.approx(300_048 * 3e-6) +_TWELVE_HOUR_BATCH_MODEL_INFO: Final = cast( + ModelInfo, + { + "input_cost_per_token_batches": 1e-6, + "input_cost_per_token_above_272k_tokens_batches": 3e-6, + "cache_creation_input_token_cost_batches": 1.25e-6, + "cache_creation_input_token_cost_above_272k_tokens_batches": 3.75e-6, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 7.5e-6, + "output_cost_per_token_batches": 4e-6, + }, +) + + +def _twelve_hour_cache_write_usage(prompt_tokens: int) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=4, + total_tokens=prompt_tokens + 4, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, + cache_creation_tokens=500, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_30m_input_tokens=300, ephemeral_12h_input_tokens=200 + ), + ), + ) + + +@pytest.mark.parametrize( + ("prompt_tokens", "input_rate", "write_rate", "twelve_hour_rate"), + [(1_000, 1e-6, 1.25e-6, 2.5e-6), (272_000, 1e-6, 1.25e-6, 2.5e-6), (300_000, 3e-6, 3.75e-6, 7.5e-6)], +) +def test_batch_cost_calculator_bills_12h_cache_writes_at_the_12h_batch_rate_for_the_crossed_tier( + prompt_tokens: int, input_rate: float, write_rate: float, twelve_hour_rate: float +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=_twelve_hour_cache_write_usage(prompt_tokens), + model="lit-12h-batch-model", + custom_llm_provider="openai", + model_info=_TWELVE_HOUR_BATCH_MODEL_INFO, + ) + + assert prompt_cost == pytest.approx((prompt_tokens - 500) * input_rate + 300 * write_rate + 200 * twelve_hour_rate) + assert completion_cost_value == pytest.approx(4 * 4e-6) + + +@pytest.mark.parametrize( + ("prompt_tokens", "input_rate", "twelve_hour_write_rate"), [(1_000, 1e-6, 2.5e-6), (300_000, 3e-6, 7.5e-6)] +) +def test_batch_cost_calculator_bills_12h_cache_writes_once_when_only_the_12h_batch_key_exists( + prompt_tokens: int, input_rate: float, twelve_hour_write_rate: float +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + model_info: Final = cast( + ModelInfo, + { + key: rate + for key, rate in _TWELVE_HOUR_BATCH_MODEL_INFO.items() + if not key.startswith("cache_creation_input_token_cost_") + or key.startswith("cache_creation_input_token_cost_above_12hr") + }, + ) + + prompt_cost, _ = batch_cost_calculator( + usage=_twelve_hour_cache_write_usage(prompt_tokens), + model="lit-12h-batch-model", + custom_llm_provider="openai", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx((prompt_tokens - 200) * input_rate + 200 * twelve_hour_write_rate) + + +@pytest.mark.parametrize(("prompt_tokens", "write_rate"), [(1_000, 1.25e-6), (300_000, 3.75e-6)]) +def test_batch_cost_calculator_bills_12h_cache_writes_at_the_batch_cache_write_rate_without_a_12h_batch_key( + prompt_tokens: int, write_rate: float +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + model_info: Final = cast( + ModelInfo, + { + key: rate + for key, rate in _TWELVE_HOUR_BATCH_MODEL_INFO.items() + if not key.startswith("cache_creation_input_token_cost_above_12hr") + }, + ) + + prompt_cost, _ = batch_cost_calculator( + usage=_twelve_hour_cache_write_usage(prompt_tokens), + model="lit-12h-batch-model", + custom_llm_provider="openai", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx( + (prompt_tokens - 500) * (1e-6 if prompt_tokens <= 272_000 else 3e-6) + 500 * write_rate + ) + + def test_batch_cost_calculator_prices_modalities_and_cached_tokens_together_in_the_crossed_tier() -> None: from litellm.cost_calculator import batch_cost_calculator diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index a0b4b784a4d..55b7bc78a11 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -21,7 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest import respx -from jsonschema import validate +from jsonschema import Draft7Validator, validate import litellm from litellm.exceptions import MidStreamFallbackError @@ -923,6 +923,456 @@ def validate_model_cost_values(model_data, exceptions=None): return len(violations) == 0, violations +_COST_MAP_ROW_SCHEMA: Final = { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "supports_computer_use": {"type": "boolean"}, + "cache_creation_input_audio_token_cost": {"type": "number"}, + "cache_creation_input_token_cost": {"type": "number"}, + "cache_creation_input_token_cost_above_1hr": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_flex": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_priority": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_batches": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_ultrafast": {"type": "number"}, + "cache_creation_input_token_cost_above_32k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_100k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_100k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_above_128k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_200k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_256k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_flex": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_priority": {"type": "number"}, + "cache_creation_input_token_cost_above_200k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_batches": {"type": "number"}, + "cache_creation_input_token_cost_decisions": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_decisions": {"type": "number"}, + "cache_read_input_token_cost_decisions": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_decisions": {"type": "number"}, + "input_cost_per_token_decisions": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_decisions": {"type": "number"}, + "output_cost_per_token_decisions": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_decisions": {"type": "number"}, + "cache_creation_input_token_cost_flex": {"type": "number"}, + "cache_creation_input_token_cost_priority": {"type": "number"}, + "cache_creation_input_token_cost_ultrafast": {"type": "number"}, + "cache_read_input_token_cost": {"type": "number"}, + "explicit_cache_read_input_token_cost": {"type": "number"}, + "cache_read_input_token_cost_above_32k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_100k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_100k_tokens_batches": {"type": "number"}, + "cache_read_input_token_cost_above_128k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_200k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_200k_tokens_batches": {"type": "number"}, + "cache_read_input_token_cost_above_256k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_flex": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": {"type": "number"}, + "cache_read_input_token_cost_above_512k_tokens": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_ultrafast": {"type": "number"}, + "cache_read_input_token_cost_batches": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, + "cache_read_input_audio_token_cost": {"type": "number"}, + "cache_read_input_audio_token_cost_batches": {"type": "number"}, + "cache_read_input_audio_token_cost_flex": {"type": "number"}, + "cache_read_input_audio_token_cost_priority": {"type": "number"}, + "cache_read_input_image_token_cost": {"type": "number"}, + "audio_transcription_config": {"type": "string"}, + "deprecation_date": {"type": "string"}, + "input_cost_per_audio_per_second": {"type": "number"}, + "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, + "google_maps_grounding_cost_per_query": {"type": "number"}, + "input_cost_per_audio_token": {"type": "number"}, + "input_cost_per_audio_token_flex": {"type": "number"}, + "input_cost_per_image_token": {"type": "number"}, + "input_cost_per_character": {"type": "number"}, + "input_cost_per_character_above_128k_tokens": {"type": "number"}, + "input_cost_per_image": {"type": "number"}, + "input_cost_per_image_above_128k_tokens": {"type": "number"}, + "input_cost_per_video_token": {"type": "number"}, + "input_cost_per_token_above_32k_tokens": {"type": "number"}, + "input_cost_per_token_above_100k_tokens": {"type": "number"}, + "input_cost_per_token_above_100k_tokens_batches": {"type": "number"}, + "input_cost_per_token_above_200k_tokens": {"type": "number"}, + "input_cost_per_token_above_200k_tokens_batches": {"type": "number"}, + "input_cost_per_token_above_256k_tokens": {"type": "number"}, + "input_cost_per_token_above_272k_tokens": {"type": "number"}, + "input_cost_per_token_above_512k_tokens": {"type": "number"}, + "cache_read_input_token_cost_flex": {"type": "number"}, + "cache_read_input_token_cost_priority": {"type": "number"}, + "cache_read_input_token_cost_balanced": {"type": "number"}, + "cache_read_input_token_cost_ultrafast": {"type": "number"}, + "cache_read_input_token_cost_above_200k_tokens_priority": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_priority": {"type": "number"}, + "input_cost_per_token_flex": {"type": "number"}, + "input_cost_per_token_priority": {"type": "number"}, + "input_cost_per_token_balanced": {"type": "number"}, + "input_cost_per_token_ultrafast": {"type": "number"}, + "input_cost_per_token_above_200k_tokens_priority": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_priority": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_batches": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_flex": {"type": "number"}, + "input_cost_per_audio_token_priority": {"type": "number"}, + "output_cost_per_token_flex": {"type": "number"}, + "output_cost_per_token_priority": {"type": "number"}, + "output_cost_per_token_balanced": {"type": "number"}, + "output_cost_per_token_ultrafast": {"type": "number"}, + "output_cost_per_token_above_200k_tokens_priority": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_priority": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_ultrafast": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_batches": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_flex": {"type": "number"}, + "regional_endpoint_uplift_multiplier": {"type": "number"}, + "regional_processing_uplift_multiplier_eu": {"type": "number"}, + "regional_processing_uplift_multiplier_us": {"type": "number"}, + "input_cost_per_pixel": {"type": "number"}, + "input_cost_per_query": {"type": "number"}, + "input_cost_per_request": {"type": "number"}, + "cost_per_second": {"type": "number"}, + "input_cost_per_second": {"type": "number"}, + "input_cost_per_text_message": {"type": "number"}, + "input_cost_per_token": {"type": "number"}, + "input_cost_per_token_above_128k_tokens": {"type": "number"}, + "input_cost_per_audio_token_batches": {"type": "number"}, + "input_cost_per_image_token_batches": {"type": "number"}, + "input_cost_per_token_batches": {"type": "number"}, + "input_cost_per_video_token_batches": {"type": "number"}, + "input_cost_per_token_cache_hit": {"type": "number"}, + "input_cost_per_video_per_second": {"type": "number"}, + "input_cost_per_video_per_second_above_8s_interval": {"type": "number"}, + "input_cost_per_video_per_second_above_15s_interval": {"type": "number"}, + "input_cost_per_video_per_second_above_128k_tokens": {"type": "number"}, + "input_dbu_cost_per_token": {"type": "number"}, + "annotation_cost_per_page": {"type": "number"}, + "annotation_cost_per_page_batches": {"type": "number"}, + "ocr_cost_per_page": {"type": "number"}, + "ocr_cost_per_page_batches": {"type": "number"}, + "ocr_cost_per_credit": {"type": "number"}, + "code_interpreter_cost_per_session": {"type": "number"}, + "inference_geo": {"type": "string"}, + "litellm_provider": {"type": "string"}, + "max_input_tokens": {"type": "number"}, + "max_output_tokens": {"type": "number"}, + "max_tokens": {"type": "number"}, + "metadata": {"type": "object"}, + "provider_specific_entry": {"type": "object"}, + "mode": { + "type": "string", + "enum": [ + "audio_speech", + "audio_transcription", + "chat", + "completion", + "container", + "image_edit", + "embedding", + "evaluation", + "extraction", + "guardrail", + "image_generation", + "video_generation", + "moderation", + "rerank", + "realtime", + "responses", + "ocr", + "search", + "vector_store", + ], + }, + "output_cost_per_audio_token": {"type": "number"}, + "output_cost_per_character": {"type": "number"}, + "output_cost_per_character_above_128k_tokens": {"type": "number"}, + "output_cost_per_image": {"type": "number"}, + "output_cost_per_image_512": {"type": "number"}, + "output_cost_per_image_1024": {"type": "number"}, + "output_cost_per_image_1536": {"type": "number"}, + "output_cost_per_image_0.5K": {"type": "number"}, + "output_cost_per_image_1K": {"type": "number"}, + "output_cost_per_image_2K": {"type": "number"}, + "output_cost_per_image_4K": {"type": "number"}, + "output_cost_per_image_token": {"type": "number"}, + "output_cost_per_image_token_batches": {"type": "number"}, + "output_cost_per_video_token": {"type": "number"}, + "output_cost_per_pixel": {"type": "number"}, + "output_cost_per_second": {"type": "number"}, + "output_cost_per_second_480p": {"type": "number"}, + "output_cost_per_second_720p": {"type": "number"}, + "output_cost_per_second_768p": {"type": "number"}, + "output_cost_per_second_2k": {"type": "number"}, + "output_cost_per_second_1080p": {"type": "number"}, + "output_cost_per_second_4k": {"type": "number"}, + "output_cost_per_token": {"type": "number"}, + "output_cost_per_token_above_32k_tokens": {"type": "number"}, + "output_cost_per_token_above_100k_tokens": {"type": "number"}, + "output_cost_per_token_above_100k_tokens_batches": {"type": "number"}, + "output_cost_per_token_above_128k_tokens": {"type": "number"}, + "output_cost_per_token_above_200k_tokens": {"type": "number"}, + "output_cost_per_token_above_200k_tokens_batches": {"type": "number"}, + "output_cost_per_token_above_256k_tokens": {"type": "number"}, + "output_cost_per_token_above_272k_tokens": {"type": "number"}, + "output_cost_per_token_above_512k_tokens": {"type": "number"}, + "output_cost_per_token_batches": {"type": "number"}, + "output_cost_per_reasoning_token": {"type": "number"}, + "output_cost_per_video_per_second": {"type": "number"}, + "output_db_cost_per_token": {"type": "number"}, + "output_dbu_cost_per_token": {"type": "number"}, + "output_vector_size": {"type": "number"}, + "rpd": {"type": "number"}, + "rpm": {"type": "number"}, + "source": {"type": "string"}, + "comment": {"type": "string"}, + "supports_assistant_prefill": {"type": "boolean"}, + "supports_anthropic_compaction": {"type": "boolean"}, + "supports_audio_input": {"type": "boolean"}, + "supports_audio_output": {"type": "boolean"}, + "gemini_native_audio": {"type": "boolean"}, + "gemini_audio_only_live": {"type": "boolean"}, + "supports_embedding_image_input": {"type": "boolean"}, + "supports_forced_tool_use": {"type": "boolean"}, + "supports_function_calling": {"type": "boolean"}, + "supports_image_input": {"type": "boolean"}, + "supports_nova_canvas_image_edit": {"type": "boolean"}, + "supports_parallel_function_calling": {"type": "boolean"}, + "supports_parallel_tool_use_config": {"type": "boolean"}, + "supports_pdf_input": {"type": "boolean"}, + "prompt_cache_min_tokens": {"type": "number"}, + "supports_prompt_cache_breakpoint": {"type": "boolean"}, + "supports_thinking_cache_preservation": {"type": "boolean"}, + "supports_prompt_caching": {"type": "boolean"}, + "supports_response_schema": {"type": "boolean"}, + "supports_system_messages": {"type": "boolean"}, + "supports_tool_choice": {"type": "boolean"}, + "supports_tool_search": {"type": "boolean"}, + "supports_video_input": {"type": "boolean"}, + "supports_box_questions": {"type": "boolean"}, + "supports_vision": {"type": "boolean"}, + "supports_web_search": {"type": "boolean"}, + "token_count_framing": {"type": "integer"}, + "token_count_model": { + "type": "object", + "properties": {"model": {"type": "string"}, "region": {"type": "string"}}, + "required": ["model"], + }, + "token_count_tokenizer": {"type": "string", "enum": ["kimi_k3"]}, + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": {"type": "boolean"}, + "supports_bedrock_runtime_chat_completions_response_format": {"type": "boolean"}, + "supports_bedrock_runtime_chat_completions_inline_reasoning": {"type": "boolean"}, + "supports_bedrock_converse_tool_result_images": {"type": "boolean"}, + "supports_url_context": {"type": "boolean"}, + "supports_multimodal": {"type": "boolean"}, + "uses_embed_content": {"type": "boolean"}, + "supports_reasoning": {"type": "boolean"}, + "supports_minimal_reasoning_effort": {"type": "boolean"}, + "supports_low_reasoning_effort": {"type": "boolean"}, + "supports_none_reasoning_effort": {"type": "boolean"}, + "supports_xhigh_reasoning_effort": {"type": "boolean"}, + "supports_max_reasoning_effort": {"type": "boolean"}, + "reasoning_effort_levels": { + "type": "array", + "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]}, + }, + "default_reasoning_effort": { + "type": "string", + "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], + }, + "supports_adaptive_thinking": {"type": "boolean"}, + "supports_anthropic_thinking_payload": {"type": "boolean"}, + "supports_legacy_thinking": {"type": "boolean"}, + "thinking_always_on": {"type": "boolean"}, + "supports_mid_conversation_system": {"type": "boolean"}, + "supports_sampling_params": {"type": "boolean"}, + "supports_output_config": {"type": "boolean"}, + "supports_speed": {"type": "boolean"}, + "supports_fast_mode": {"type": "boolean"}, + "supported_audio_formats": { + "type": "array", + "items": { + "type": "string", + "enum": ["mp3", "wav"], + }, + }, + "vertex_ai_audio_api": { + "type": "string", + "enum": ["lyria_predict", "lyria_interactions"], + }, + "bedrock_output_config_effort_ceiling": { + "type": "string", + "enum": ["low", "medium", "high", "max", "xhigh"], + }, + "bedrock_converse_supports_strict_tools": {"type": "boolean"}, + "supports_regex_lookaround": {"type": "boolean"}, + "tpm": {"type": "number"}, + "supported_endpoints": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "/v1/responses", + "/v1/embeddings", + "/v1/chat/completions", + "/v1/completions", + "/v1/messages", + "/v1/images/generations", + "/v1/realtime", + "/v1/realtime/transcription_sessions", + "/v1/images/variations", + "/v1/images/edits", + "/v1/batch", + "/v1beta/interactions", + "/v1beta/models", + "/v1/audio/transcriptions", + "/v1/audio/speech", + "/v1/ocr", + "/v1/videos", + "/vertex_ai/live", + "/v1/listen", + "/v1/systemone", + "/v1/decisions", + "/v1beta/interactions", + ], + }, + }, + "supported_regions": { + "type": "array", + "items": { + "type": "string", + }, + }, + "guardrail_cost_per_unit": { + "type": "object", + "additionalProperties": {"type": "number"}, + }, + "search_context_cost_per_query": { + "type": "object", + "properties": { + "search_context_size_low": {"type": "number"}, + "search_context_size_medium": {"type": "number"}, + "search_context_size_high": {"type": "number"}, + }, + "additionalProperties": False, + }, + "web_search_billing_unit": { + "type": "string", + "enum": ["per_prompt", "per_query"], + }, + "search_free_requests": { + "type": "object", + "properties": { + "count": {"type": "integer", "minimum": 1}, + "period": {"type": "string", "enum": ["day", "month"]}, + "shared_group": {"type": "string"}, + }, + "required": ["count", "period", "shared_group"], + "additionalProperties": False, + }, + "citation_cost_per_token": {"type": "number"}, + "supported_modalities": { + "type": "array", + "items": { + "type": "string", + "enum": ["text", "audio", "image", "video"], + }, + }, + "supported_output_modalities": { + "type": "array", + "items": { + "type": "string", + "enum": ["text", "image", "audio", "code", "video"], + }, + }, + "supports_native_streaming": {"type": "boolean"}, + "supports_image_size": {"type": "boolean"}, + "supports_native_structured_output": {"type": "boolean"}, + "supports_bedrock_native_structured_output": {"type": "boolean"}, + "use_openai_responses_path": {"type": "boolean"}, + "off_peak_pricing": { + "type": "object", + "properties": { + "hours_utc": { + "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}], + }, + "windows": { + "type": "array", + "items": { + "type": "object", + "properties": { + "hours_utc": { + "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}], + }, + "weekdays": { + "type": "array", + "items": {"oneOf": [{"type": "integer"}, {"type": "string"}]}, + }, + }, + "required": ["hours_utc"], + "additionalProperties": False, + }, + }, + "weekday_timezone": {"type": "string"}, + "input_cost_per_token": {"type": "number"}, + "output_cost_per_token": {"type": "number"}, + "output_cost_per_reasoning_token": {"type": "number"}, + "cache_read_input_token_cost": {"type": "number"}, + "cache_creation_input_token_cost": {"type": "number"}, + }, + "additionalProperties": False, + }, + "tiered_pricing": { + "type": "array", + "items": { + "type": "object", + "properties": { + "range": { + "type": "array", + "items": {"type": "number"}, + "minItems": 2, + "maxItems": 2, + }, + "input_cost_per_token": {"type": "number"}, + "output_cost_per_token": {"type": "number"}, + "cache_read_input_token_cost": {"type": "number"}, + "cache_creation_input_token_cost": {"type": "number"}, + "output_cost_per_reasoning_token": {"type": "number"}, + "max_results_range": { + "type": "array", + "items": {"type": "number"}, + "minItems": 2, + "maxItems": 2, + }, + "input_cost_per_query": {"type": "number"}, + }, + "additionalProperties": False, + }, + }, + }, + "additionalProperties": False, + }, +} + + def test_aaamodel_prices_and_context_window_json_is_valid(): """ Validates the `model_prices_and_context_window.json` file. @@ -930,447 +1380,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): If this test fails after you update the json, you need to update the schema or correct the change you made. """ - INTENDED_SCHEMA = { - "type": "object", - "additionalProperties": { - "type": "object", - "properties": { - "supports_computer_use": {"type": "boolean"}, - "cache_creation_input_audio_token_cost": {"type": "number"}, - "cache_creation_input_token_cost": {"type": "number"}, - "cache_creation_input_token_cost_above_1hr": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_flex": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr_priority": {"type": "number"}, - "cache_creation_input_token_cost_above_32k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_100k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_100k_tokens_batches": {"type": "number"}, - "cache_creation_input_token_cost_above_128k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_200k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_256k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_272k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_272k_tokens_flex": {"type": "number"}, - "cache_creation_input_token_cost_above_272k_tokens_ultrafast": {"type": "number"}, - "cache_creation_input_token_cost_above_272k_tokens_priority": {"type": "number"}, - "cache_creation_input_token_cost_above_200k_tokens_batches": {"type": "number"}, - "cache_creation_input_token_cost_above_272k_tokens_batches": {"type": "number"}, - "cache_creation_input_token_cost_batches": {"type": "number"}, - "cache_creation_input_token_cost_decisions": {"type": "number"}, - "cache_creation_input_token_cost_above_272k_tokens_decisions": {"type": "number"}, - "cache_read_input_token_cost_decisions": {"type": "number"}, - "cache_read_input_token_cost_above_272k_tokens_decisions": {"type": "number"}, - "input_cost_per_token_decisions": {"type": "number"}, - "input_cost_per_token_above_272k_tokens_decisions": {"type": "number"}, - "output_cost_per_token_decisions": {"type": "number"}, - "output_cost_per_token_above_272k_tokens_decisions": {"type": "number"}, - "cache_creation_input_token_cost_flex": {"type": "number"}, - "cache_creation_input_token_cost_priority": {"type": "number"}, - "cache_creation_input_token_cost_ultrafast": {"type": "number"}, - "cache_read_input_token_cost": {"type": "number"}, - "explicit_cache_read_input_token_cost": {"type": "number"}, - "cache_read_input_token_cost_above_32k_tokens": {"type": "number"}, - "cache_read_input_token_cost_above_100k_tokens": {"type": "number"}, - "cache_read_input_token_cost_above_100k_tokens_batches": {"type": "number"}, - "cache_read_input_token_cost_above_128k_tokens": {"type": "number"}, - "cache_read_input_token_cost_above_200k_tokens": {"type": "number"}, - "cache_read_input_token_cost_above_200k_tokens_batches": {"type": "number"}, - "cache_read_input_token_cost_above_256k_tokens": {"type": "number"}, - "cache_read_input_token_cost_above_272k_tokens": {"type": "number"}, - "cache_read_input_token_cost_above_272k_tokens_flex": {"type": "number"}, - "cache_read_input_token_cost_above_272k_tokens_ultrafast": {"type": "number"}, - "cache_read_input_token_cost_above_512k_tokens": {"type": "number"}, - "input_cost_per_token_above_272k_tokens_ultrafast": {"type": "number"}, - "cache_read_input_token_cost_batches": {"type": "number"}, - "cache_read_input_token_cost_above_272k_tokens_batches": {"type": "number"}, - "cache_creation_input_token_cost_above_1hr_above_100k_tokens": {"type": "number"}, - "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, - "cache_read_input_audio_token_cost": {"type": "number"}, - "cache_read_input_audio_token_cost_batches": {"type": "number"}, - "cache_read_input_audio_token_cost_flex": {"type": "number"}, - "cache_read_input_audio_token_cost_priority": {"type": "number"}, - "cache_read_input_image_token_cost": {"type": "number"}, - "audio_transcription_config": {"type": "string"}, - "deprecation_date": {"type": "string"}, - "input_cost_per_audio_per_second": {"type": "number"}, - "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, - "google_maps_grounding_cost_per_query": {"type": "number"}, - "input_cost_per_audio_token": {"type": "number"}, - "input_cost_per_audio_token_flex": {"type": "number"}, - "input_cost_per_image_token": {"type": "number"}, - "input_cost_per_character": {"type": "number"}, - "input_cost_per_character_above_128k_tokens": {"type": "number"}, - "input_cost_per_image": {"type": "number"}, - "input_cost_per_image_above_128k_tokens": {"type": "number"}, - "input_cost_per_video_token": {"type": "number"}, - "input_cost_per_token_above_32k_tokens": {"type": "number"}, - "input_cost_per_token_above_100k_tokens": {"type": "number"}, - "input_cost_per_token_above_100k_tokens_batches": {"type": "number"}, - "input_cost_per_token_above_200k_tokens": {"type": "number"}, - "input_cost_per_token_above_200k_tokens_batches": {"type": "number"}, - "input_cost_per_token_above_256k_tokens": {"type": "number"}, - "input_cost_per_token_above_272k_tokens": {"type": "number"}, - "input_cost_per_token_above_512k_tokens": {"type": "number"}, - "cache_read_input_token_cost_flex": {"type": "number"}, - "cache_read_input_token_cost_priority": {"type": "number"}, - "cache_read_input_token_cost_balanced": {"type": "number"}, - "cache_read_input_token_cost_ultrafast": {"type": "number"}, - "cache_read_input_token_cost_above_200k_tokens_priority": {"type": "number"}, - "cache_read_input_token_cost_above_272k_tokens_priority": {"type": "number"}, - "input_cost_per_token_flex": {"type": "number"}, - "input_cost_per_token_priority": {"type": "number"}, - "input_cost_per_token_balanced": {"type": "number"}, - "input_cost_per_token_ultrafast": {"type": "number"}, - "input_cost_per_token_above_200k_tokens_priority": {"type": "number"}, - "input_cost_per_token_above_272k_tokens_priority": {"type": "number"}, - "input_cost_per_token_above_272k_tokens_batches": {"type": "number"}, - "input_cost_per_token_above_272k_tokens_flex": {"type": "number"}, - "input_cost_per_audio_token_priority": {"type": "number"}, - "output_cost_per_token_flex": {"type": "number"}, - "output_cost_per_token_priority": {"type": "number"}, - "output_cost_per_token_balanced": {"type": "number"}, - "output_cost_per_token_ultrafast": {"type": "number"}, - "output_cost_per_token_above_200k_tokens_priority": {"type": "number"}, - "output_cost_per_token_above_272k_tokens_priority": {"type": "number"}, - "output_cost_per_token_above_272k_tokens_ultrafast": {"type": "number"}, - "output_cost_per_token_above_272k_tokens_batches": {"type": "number"}, - "output_cost_per_token_above_272k_tokens_flex": {"type": "number"}, - "regional_endpoint_uplift_multiplier": {"type": "number"}, - "regional_processing_uplift_multiplier_eu": {"type": "number"}, - "regional_processing_uplift_multiplier_us": {"type": "number"}, - "input_cost_per_pixel": {"type": "number"}, - "input_cost_per_query": {"type": "number"}, - "input_cost_per_request": {"type": "number"}, - "cost_per_second": {"type": "number"}, - "input_cost_per_second": {"type": "number"}, - "input_cost_per_text_message": {"type": "number"}, - "input_cost_per_token": {"type": "number"}, - "input_cost_per_token_above_128k_tokens": {"type": "number"}, - "input_cost_per_audio_token_batches": {"type": "number"}, - "input_cost_per_image_token_batches": {"type": "number"}, - "input_cost_per_token_batches": {"type": "number"}, - "input_cost_per_video_token_batches": {"type": "number"}, - "input_cost_per_token_cache_hit": {"type": "number"}, - "input_cost_per_video_per_second": {"type": "number"}, - "input_cost_per_video_per_second_above_8s_interval": {"type": "number"}, - "input_cost_per_video_per_second_above_15s_interval": {"type": "number"}, - "input_cost_per_video_per_second_above_128k_tokens": {"type": "number"}, - "input_dbu_cost_per_token": {"type": "number"}, - "annotation_cost_per_page": {"type": "number"}, - "annotation_cost_per_page_batches": {"type": "number"}, - "ocr_cost_per_page": {"type": "number"}, - "ocr_cost_per_page_batches": {"type": "number"}, - "ocr_cost_per_credit": {"type": "number"}, - "code_interpreter_cost_per_session": {"type": "number"}, - "inference_geo": {"type": "string"}, - "litellm_provider": {"type": "string"}, - "max_input_tokens": {"type": "number"}, - "max_output_tokens": {"type": "number"}, - "max_tokens": {"type": "number"}, - "metadata": {"type": "object"}, - "provider_specific_entry": {"type": "object"}, - "mode": { - "type": "string", - "enum": [ - "audio_speech", - "audio_transcription", - "chat", - "completion", - "container", - "image_edit", - "embedding", - "evaluation", - "extraction", - "guardrail", - "image_generation", - "video_generation", - "moderation", - "rerank", - "realtime", - "responses", - "ocr", - "search", - "vector_store", - ], - }, - "output_cost_per_audio_token": {"type": "number"}, - "output_cost_per_character": {"type": "number"}, - "output_cost_per_character_above_128k_tokens": {"type": "number"}, - "output_cost_per_image": {"type": "number"}, - "output_cost_per_image_512": {"type": "number"}, - "output_cost_per_image_1024": {"type": "number"}, - "output_cost_per_image_1536": {"type": "number"}, - "output_cost_per_image_0.5K": {"type": "number"}, - "output_cost_per_image_1K": {"type": "number"}, - "output_cost_per_image_2K": {"type": "number"}, - "output_cost_per_image_4K": {"type": "number"}, - "output_cost_per_image_token": {"type": "number"}, - "output_cost_per_image_token_batches": {"type": "number"}, - "output_cost_per_video_token": {"type": "number"}, - "output_cost_per_pixel": {"type": "number"}, - "output_cost_per_second": {"type": "number"}, - "output_cost_per_second_480p": {"type": "number"}, - "output_cost_per_second_720p": {"type": "number"}, - "output_cost_per_second_768p": {"type": "number"}, - "output_cost_per_second_2k": {"type": "number"}, - "output_cost_per_second_1080p": {"type": "number"}, - "output_cost_per_second_4k": {"type": "number"}, - "output_cost_per_token": {"type": "number"}, - "output_cost_per_token_above_32k_tokens": {"type": "number"}, - "output_cost_per_token_above_100k_tokens": {"type": "number"}, - "output_cost_per_token_above_100k_tokens_batches": {"type": "number"}, - "output_cost_per_token_above_128k_tokens": {"type": "number"}, - "output_cost_per_token_above_200k_tokens": {"type": "number"}, - "output_cost_per_token_above_200k_tokens_batches": {"type": "number"}, - "output_cost_per_token_above_256k_tokens": {"type": "number"}, - "output_cost_per_token_above_272k_tokens": {"type": "number"}, - "output_cost_per_token_above_512k_tokens": {"type": "number"}, - "output_cost_per_token_batches": {"type": "number"}, - "output_cost_per_reasoning_token": {"type": "number"}, - "output_cost_per_video_per_second": {"type": "number"}, - "output_db_cost_per_token": {"type": "number"}, - "output_dbu_cost_per_token": {"type": "number"}, - "output_vector_size": {"type": "number"}, - "rpd": {"type": "number"}, - "rpm": {"type": "number"}, - "source": {"type": "string"}, - "comment": {"type": "string"}, - "supports_assistant_prefill": {"type": "boolean"}, - "supports_anthropic_compaction": {"type": "boolean"}, - "supports_audio_input": {"type": "boolean"}, - "supports_audio_output": {"type": "boolean"}, - "gemini_native_audio": {"type": "boolean"}, - "gemini_audio_only_live": {"type": "boolean"}, - "supports_embedding_image_input": {"type": "boolean"}, - "supports_forced_tool_use": {"type": "boolean"}, - "supports_function_calling": {"type": "boolean"}, - "supports_image_input": {"type": "boolean"}, - "supports_nova_canvas_image_edit": {"type": "boolean"}, - "supports_parallel_function_calling": {"type": "boolean"}, - "supports_parallel_tool_use_config": {"type": "boolean"}, - "supports_pdf_input": {"type": "boolean"}, - "prompt_cache_min_tokens": {"type": "number"}, - "supports_prompt_cache_breakpoint": {"type": "boolean"}, - "supports_thinking_cache_preservation": {"type": "boolean"}, - "supports_prompt_caching": {"type": "boolean"}, - "supports_response_schema": {"type": "boolean"}, - "supports_system_messages": {"type": "boolean"}, - "supports_tool_choice": {"type": "boolean"}, - "supports_tool_search": {"type": "boolean"}, - "supports_video_input": {"type": "boolean"}, - "supports_box_questions": {"type": "boolean"}, - "supports_vision": {"type": "boolean"}, - "supports_web_search": {"type": "boolean"}, - "token_count_framing": {"type": "integer"}, - "token_count_model": { - "type": "object", - "properties": {"model": {"type": "string"}, "region": {"type": "string"}}, - "required": ["model"], - }, - "token_count_tokenizer": {"type": "string", "enum": ["kimi_k3"]}, - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": {"type": "boolean"}, - "supports_bedrock_runtime_chat_completions_response_format": {"type": "boolean"}, - "supports_bedrock_runtime_chat_completions_inline_reasoning": {"type": "boolean"}, - "supports_bedrock_converse_tool_result_images": {"type": "boolean"}, - "supports_url_context": {"type": "boolean"}, - "supports_multimodal": {"type": "boolean"}, - "uses_embed_content": {"type": "boolean"}, - "supports_reasoning": {"type": "boolean"}, - "supports_minimal_reasoning_effort": {"type": "boolean"}, - "supports_low_reasoning_effort": {"type": "boolean"}, - "supports_none_reasoning_effort": {"type": "boolean"}, - "supports_xhigh_reasoning_effort": {"type": "boolean"}, - "supports_max_reasoning_effort": {"type": "boolean"}, - "reasoning_effort_levels": { - "type": "array", - "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]}, - }, - "default_reasoning_effort": { - "type": "string", - "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], - }, - "supports_adaptive_thinking": {"type": "boolean"}, - "supports_anthropic_thinking_payload": {"type": "boolean"}, - "supports_legacy_thinking": {"type": "boolean"}, - "thinking_always_on": {"type": "boolean"}, - "supports_mid_conversation_system": {"type": "boolean"}, - "supports_sampling_params": {"type": "boolean"}, - "supports_output_config": {"type": "boolean"}, - "supports_speed": {"type": "boolean"}, - "supports_fast_mode": {"type": "boolean"}, - "supported_audio_formats": { - "type": "array", - "items": { - "type": "string", - "enum": ["mp3", "wav"], - }, - }, - "vertex_ai_audio_api": { - "type": "string", - "enum": ["lyria_predict", "lyria_interactions"], - }, - "bedrock_output_config_effort_ceiling": { - "type": "string", - "enum": ["low", "medium", "high", "max", "xhigh"], - }, - "bedrock_converse_supports_strict_tools": {"type": "boolean"}, - "supports_regex_lookaround": {"type": "boolean"}, - "tpm": {"type": "number"}, - "supported_endpoints": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "/v1/responses", - "/v1/embeddings", - "/v1/chat/completions", - "/v1/completions", - "/v1/messages", - "/v1/images/generations", - "/v1/realtime", - "/v1/realtime/transcription_sessions", - "/v1/images/variations", - "/v1/images/edits", - "/v1/batch", - "/v1beta/interactions", - "/v1beta/models", - "/v1/audio/transcriptions", - "/v1/audio/speech", - "/v1/ocr", - "/v1/videos", - "/vertex_ai/live", - "/v1/listen", - "/v1/systemone", - "/v1/decisions", - "/v1beta/interactions", - ], - }, - }, - "supported_regions": { - "type": "array", - "items": { - "type": "string", - }, - }, - "guardrail_cost_per_unit": { - "type": "object", - "additionalProperties": {"type": "number"}, - }, - "search_context_cost_per_query": { - "type": "object", - "properties": { - "search_context_size_low": {"type": "number"}, - "search_context_size_medium": {"type": "number"}, - "search_context_size_high": {"type": "number"}, - }, - "additionalProperties": False, - }, - "web_search_billing_unit": { - "type": "string", - "enum": ["per_prompt", "per_query"], - }, - "search_free_requests": { - "type": "object", - "properties": { - "count": {"type": "integer", "minimum": 1}, - "period": {"type": "string", "enum": ["day", "month"]}, - "shared_group": {"type": "string"}, - }, - "required": ["count", "period", "shared_group"], - "additionalProperties": False, - }, - "citation_cost_per_token": {"type": "number"}, - "supported_modalities": { - "type": "array", - "items": { - "type": "string", - "enum": ["text", "audio", "image", "video"], - }, - }, - "supported_output_modalities": { - "type": "array", - "items": { - "type": "string", - "enum": ["text", "image", "audio", "code", "video"], - }, - }, - "supports_native_streaming": {"type": "boolean"}, - "supports_image_size": {"type": "boolean"}, - "supports_native_structured_output": {"type": "boolean"}, - "supports_bedrock_native_structured_output": {"type": "boolean"}, - "use_openai_responses_path": {"type": "boolean"}, - "off_peak_pricing": { - "type": "object", - "properties": { - "hours_utc": { - "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}], - }, - "windows": { - "type": "array", - "items": { - "type": "object", - "properties": { - "hours_utc": { - "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}], - }, - "weekdays": { - "type": "array", - "items": {"oneOf": [{"type": "integer"}, {"type": "string"}]}, - }, - }, - "required": ["hours_utc"], - "additionalProperties": False, - }, - }, - "weekday_timezone": {"type": "string"}, - "input_cost_per_token": {"type": "number"}, - "output_cost_per_token": {"type": "number"}, - "output_cost_per_reasoning_token": {"type": "number"}, - "cache_read_input_token_cost": {"type": "number"}, - "cache_creation_input_token_cost": {"type": "number"}, - }, - "additionalProperties": False, - }, - "tiered_pricing": { - "type": "array", - "items": { - "type": "object", - "properties": { - "range": { - "type": "array", - "items": {"type": "number"}, - "minItems": 2, - "maxItems": 2, - }, - "input_cost_per_token": {"type": "number"}, - "output_cost_per_token": {"type": "number"}, - "cache_read_input_token_cost": {"type": "number"}, - "cache_creation_input_token_cost": {"type": "number"}, - "output_cost_per_reasoning_token": {"type": "number"}, - "max_results_range": { - "type": "array", - "items": {"type": "number"}, - "minItems": 2, - "maxItems": 2, - }, - "input_cost_per_query": {"type": "number"}, - }, - "additionalProperties": False, - }, - }, - }, - "additionalProperties": False, - }, - } - prod_json = os.path.join(os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json") with open(prod_json, "r") as model_prices_file: actual_json = json.load(model_prices_file) @@ -1379,7 +1388,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): actual_json.pop("fallback_generalizations", None) # reserved meta key, not a model entry # Validate schema - validate(actual_json, INTENDED_SCHEMA) + validate(actual_json, _COST_MAP_ROW_SCHEMA) # Validate cost values # Define exceptions for models that are allowed to have costs > 1 @@ -1401,6 +1410,21 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): raise AssertionError(error_message) +def test_cost_map_schema_accepts_12hr_cache_write_rates_for_batch_and_ultrafast_tiers() -> None: + row: Final = { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_creation_input_token_cost_above_12hr_batches": 3e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 4e-6, + "cache_creation_input_token_cost_above_12hr_ultrafast": 5e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": 6e-6, + } + + assert Draft7Validator(_COST_MAP_ROW_SCHEMA).is_valid({"synthetic-12hr-cache-write-model": row}) + + def test_max_tokens_consistency(): """ Test that max_tokens == max_output_tokens for all models. diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e81fa946833..262e8b6deae 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35223,14 +35223,22 @@ export interface components { cache_creation_input_token_cost_above_12hr?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */ cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Batches */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */ cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */ cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Ultrafast */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Batches */ + cache_creation_input_token_cost_above_12hr_batches?: number | null; /** Cache Creation Input Token Cost Above 12Hr Flex */ cache_creation_input_token_cost_above_12hr_flex?: number | null; /** Cache Creation Input Token Cost Above 12Hr Priority */ cache_creation_input_token_cost_above_12hr_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Ultrafast */ + cache_creation_input_token_cost_above_12hr_ultrafast?: number | null; /** Cache Creation Input Token Cost Above 1Hr */ cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Creation Input Token Cost Above 1Hr Above 100K Tokens */ @@ -50253,14 +50261,22 @@ export interface components { cache_creation_input_token_cost_above_12hr?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */ cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Batches */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */ cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */ cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Ultrafast */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Batches */ + cache_creation_input_token_cost_above_12hr_batches?: number | null; /** Cache Creation Input Token Cost Above 12Hr Flex */ cache_creation_input_token_cost_above_12hr_flex?: number | null; /** Cache Creation Input Token Cost Above 12Hr Priority */ cache_creation_input_token_cost_above_12hr_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Ultrafast */ + cache_creation_input_token_cost_above_12hr_ultrafast?: number | null; /** Cache Creation Input Token Cost Above 1Hr */ cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Read Input Token Cost */ @@ -50558,14 +50574,22 @@ export interface components { cache_creation_input_token_cost_above_12hr?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */ cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Batches */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */ cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null; /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */ cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Ultrafast */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Batches */ + cache_creation_input_token_cost_above_12hr_batches?: number | null; /** Cache Creation Input Token Cost Above 12Hr Flex */ cache_creation_input_token_cost_above_12hr_flex?: number | null; /** Cache Creation Input Token Cost Above 12Hr Priority */ cache_creation_input_token_cost_above_12hr_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Ultrafast */ + cache_creation_input_token_cost_above_12hr_ultrafast?: number | null; /** Cache Creation Input Token Cost Above 1Hr */ cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Creation Input Token Cost Above 1Hr Above 100K Tokens */