diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 99944662bf8..98f6f1409f2 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2879,6 +2879,7 @@ def _batch_rate( model_info: ModelInfo, key: Literal[ "input_cost_per_audio_token_batches", + "cache_read_input_audio_token_cost_batches", "input_cost_per_image_token_batches", "input_cost_per_video_token_batches", ], @@ -3012,6 +3013,7 @@ def _batch_prompt_cost( ) -> float: details: Final = parse_prompt_tokens_details(usage) cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 + cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0 written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0 audio_tokens, image_tokens, video_tokens = ( details["audio_tokens"], @@ -3026,7 +3028,9 @@ def _batch_prompt_cost( + audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate) + image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate) + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate) - + cached_tokens * (cache_read_rate or 0.0) + + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0) + + cached_audio_tokens + * _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0) + written_tokens * (cache_creation_rate or 0.0) ) diff --git a/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py b/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py index 9688b511ea8..3ed598b5e97 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py +++ b/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py @@ -5,6 +5,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never +from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.types.utils import ( CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, @@ -16,6 +17,8 @@ ZERO_COST_COUNTER_NAME: Final = "litellm_zero_cost_requests_total" _TEXT_INPUT_RATE: Final = "input_cost_per_token" _AUDIO_INPUT_RATE: Final = "input_cost_per_audio_token" +_CACHE_READ_INPUT_RATE: Final = "cache_read_input_token_cost" +_CACHE_READ_AUDIO_INPUT_RATE: Final = "cache_read_input_audio_token_cost" _TEXT_OUTPUT_RATE: Final = "output_cost_per_token" _AUDIO_OUTPUT_RATE: Final = "output_cost_per_audio_token" _RATE_KEY_MARKERS: Final = ("cost", "pricing") @@ -33,19 +36,38 @@ def _tokens(value: object) -> int: def used_pricing_keys(usage: Usage) -> tuple[str, ...]: - prompt_audio: Final = _audio_tokens(usage.prompt_tokens_details) + prompt_details: Final = parse_prompt_tokens_details(usage) + prompt_audio: Final = prompt_details["audio_tokens"] completion_audio: Final = _audio_tokens(usage.completion_tokens_details) - prompt_text: Final = _tokens(usage.prompt_tokens) - prompt_audio + prompt_text: Final = max( + _tokens(usage.prompt_tokens) + - prompt_details["cache_hit_tokens"] + - prompt_audio + - prompt_details["image_tokens"] + - prompt_details["video_tokens"], + 0, + ) + cached_text: Final = max(prompt_details["cache_hit_tokens"] - prompt_details["cache_hit_audio_tokens"], 0) completion_text: Final = _tokens(usage.completion_tokens) - completion_audio components: Final = ( (_TEXT_INPUT_RATE, prompt_text), (_AUDIO_INPUT_RATE, prompt_audio), + (_CACHE_READ_INPUT_RATE, cached_text), + (_CACHE_READ_AUDIO_INPUT_RATE, prompt_details["cache_hit_audio_tokens"]), (_TEXT_OUTPUT_RATE, completion_text), (_AUDIO_OUTPUT_RATE, completion_audio), ) return tuple(key for key, count in components if count > 0) +def _effective_pricing_key(key: str, pricing_entry: Mapping[str, object]) -> str: + if key == _CACHE_READ_AUDIO_INPUT_RATE and pricing_entry.get(key) is None: + return _CACHE_READ_INPUT_RATE if pricing_entry.get(_CACHE_READ_INPUT_RATE) is not None else _TEXT_INPUT_RATE + if key == _CACHE_READ_INPUT_RATE and pricing_entry.get(key) is None: + return _TEXT_INPUT_RATE + return key + + def _nested_pricing(value: object) -> Mapping[str, object] | tuple[object, ...] | None: try: return _NESTED_PRICING.validate_python(value) @@ -89,7 +111,8 @@ def diagnose_zero_cost( pricing_entry: Mapping[str, object], calculation_failed: bool, ) -> StandardLoggingZeroCostDiagnostic | None: - used_keys: Final = used_pricing_keys(usage) + effective_keys: Final = tuple(_effective_pricing_key(key, pricing_entry) for key in used_pricing_keys(usage)) + used_keys: Final = tuple(key for index, key in enumerate(effective_keys) if key not in effective_keys[:index]) if not used_keys: return None missing_keys: Final = tuple(key for key in used_keys if pricing_entry.get(key) is None) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ce632bc0eef..d788ccb0f1d 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -42,6 +42,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_httpx_client, ) from litellm.types.llms.anthropic import AnthropicThinkingParam +from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.gemini import BidiGenerateContentServerMessage from litellm.types.llms.openai import ( AllMessageValues, @@ -2364,26 +2365,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif modality == "DOCUMENT": cached_text_tokens = (cached_text_tokens or 0) + token_count - ## Calculate non-cached tokens by subtracting cached from total (per modality) - ## This is necessary because promptTokensDetails includes both cached and non-cached tokens - ## See: https://github.com/BerriAI/litellm/issues/18750 - if cached_text_tokens is not None and prompt_text_tokens is not None: - # Explicit caching: subtract cached tokens per modality from cacheTokensDetails - prompt_text_tokens = prompt_text_tokens - cached_text_tokens - elif ( + if ( cached_tokens is not None and prompt_text_tokens is not None and cached_text_tokens is None and "cacheTokensDetails" not in usage_metadata ): - # Implicit caching: only cachedContentTokenCount is provided (no cacheTokensDetails) - # Subtract from text tokens since implicit caching is primarily for text content - # See: https://github.com/BerriAI/litellm/issues/16341 prompt_text_tokens = prompt_text_tokens - cached_tokens - if cached_audio_tokens is not None and prompt_audio_tokens is not None: - prompt_audio_tokens = prompt_audio_tokens - cached_audio_tokens - if cached_image_tokens is not None and prompt_image_tokens is not None: - prompt_image_tokens = prompt_image_tokens - cached_image_tokens if cached_video_tokens is not None and prompt_video_tokens is not None: prompt_video_tokens = prompt_video_tokens - cached_video_tokens @@ -2404,9 +2392,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): billable_text_tokens: Final = ( prompt_text_tokens if prompt_text_tokens is None else prompt_text_tokens + billable_tool_use_prompt_tokens ) + cached_tokens_details_fields: Final = MappingProxyType( + { + key: token_count + for key, token_count in ( + ("text_tokens", cached_text_tokens), + ("audio_tokens", cached_audio_tokens), + ("image_tokens", cached_image_tokens), + ) + if token_count is not None + } + ) + cached_tokens_details: Final = ( + CachedTokensDetails.model_validate(cached_tokens_details_fields) if cached_tokens_details_fields else None + ) prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=cached_tokens, + cached_tokens_details=cached_tokens_details, audio_tokens=prompt_audio_tokens, text_tokens=billable_text_tokens, image_tokens=prompt_image_tokens, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8c0c6dffdc7..15dfd196c6f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -31137,11 +31137,15 @@ }, "gemini/gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_batches": 1e-07, + "cache_read_input_audio_token_cost_flex": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -31466,11 +31470,15 @@ }, "gemini/gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, + "cache_read_input_audio_token_cost_batches": 3e-08, + "cache_read_input_audio_token_cost_flex": 3e-08, + "cache_read_input_audio_token_cost_priority": 5.4e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, + "input_cost_per_audio_token_flex": 1.5e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -31760,12 +31768,16 @@ }, "gemini/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, + "cache_read_input_audio_token_cost_batches": 2.5e-08, + "cache_read_input_audio_token_cost_flex": 2.5e-08, + "cache_read_input_audio_token_cost_priority": 9e-08, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-07", "input_cost_per_audio_token": 5e-07, + "input_cost_per_audio_token_flex": 2.5e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, @@ -31886,10 +31898,14 @@ }, "gemini/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_batches": 1e-07, + "cache_read_input_audio_token_cost_flex": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ed66cbd8cf0..270d9b43dea 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -333,6 +333,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing cache_read_input_token_cost: float | None cache_read_input_audio_token_cost: ReadOnly[float | None] + cache_read_input_audio_token_cost_flex: ReadOnly[float | None] + cache_read_input_audio_token_cost_priority: ReadOnly[float | None] cache_read_input_image_token_cost: ReadOnly[float | None] explicit_cache_read_input_token_cost: ReadOnly[float | None] cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing @@ -348,6 +350,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_read_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None] cache_read_input_token_cost_above_512k_tokens: float | None cache_read_input_token_cost_batches: ReadOnly[float | None] + cache_read_input_audio_token_cost_batches: ReadOnly[float | None] cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None] cache_read_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None] cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] @@ -361,6 +364,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): off_peak_pricing: ReadOnly[OffPeakPricing | None] # time-windowed off-peak rates input_cost_per_character: float | None # only for vertex ai models input_cost_per_audio_token: float | None + input_cost_per_audio_token_flex: ReadOnly[float | None] + input_cost_per_audio_token_priority: ReadOnly[float | None] input_cost_per_token_above_128k_tokens: float | None # only for vertex ai models input_cost_per_token_above_200k_tokens: float | None # only for vertex ai gemini-2.5-pro models input_cost_per_token_above_100k_tokens: ReadOnly[float | None] @@ -4020,14 +4025,19 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): cache_read_input_token_cost_above_100k_tokens_batches: float | None = None cache_read_input_token_cost_above_200k_tokens_batches: float | None = None cache_read_input_token_cost_above_272k_tokens_batches: float | None = None + cache_read_input_audio_token_cost_batches: float | None = None cache_creation_input_token_cost_batches: float | None = None cache_creation_input_token_cost_above_100k_tokens_batches: float | None = None cache_creation_input_token_cost_above_200k_tokens_batches: float | None = None cache_creation_input_token_cost_above_272k_tokens_batches: float | None = None cache_read_input_audio_token_cost: float | None = None + cache_read_input_audio_token_cost_flex: float | None = None + cache_read_input_audio_token_cost_priority: float | None = None cache_read_input_image_token_cost: float | None = None input_cost_per_character_above_128k_tokens: float | None = None input_cost_per_audio_token: float | None = None + input_cost_per_audio_token_flex: float | None = None + input_cost_per_audio_token_priority: float | None = None input_cost_per_token_cache_hit: float | None = None input_cost_per_token_above_128k_tokens: float | None = None input_cost_per_token_above_100k_tokens: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index 58eaa43e687..bb94ca6d40a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6773,6 +6773,13 @@ def get_model_info_helper( ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None), + cache_read_input_audio_token_cost_flex=_model_info.get("cache_read_input_audio_token_cost_flex", None), + cache_read_input_audio_token_cost_priority=_model_info.get( + "cache_read_input_audio_token_cost_priority", None + ), + cache_read_input_audio_token_cost_batches=_model_info.get( + "cache_read_input_audio_token_cost_batches", None + ), explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( @@ -6858,6 +6865,8 @@ def get_model_info_helper( input_cost_per_second=_model_info.get("input_cost_per_second", None), input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None), input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None), + input_cost_per_audio_token_flex=_model_info.get("input_cost_per_audio_token_flex", None), + input_cost_per_audio_token_priority=_model_info.get("input_cost_per_audio_token_priority", None), input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None), input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None), input_cost_per_audio_token_batches=_model_info.get("input_cost_per_audio_token_batches", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8c0c6dffdc7..15dfd196c6f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -31137,11 +31137,15 @@ }, "gemini/gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_batches": 1e-07, + "cache_read_input_audio_token_cost_flex": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -31466,11 +31470,15 @@ }, "gemini/gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, + "cache_read_input_audio_token_cost_batches": 3e-08, + "cache_read_input_audio_token_cost_flex": 3e-08, + "cache_read_input_audio_token_cost_priority": 5.4e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, + "input_cost_per_audio_token_flex": 1.5e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -31760,12 +31768,16 @@ }, "gemini/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, + "cache_read_input_audio_token_cost_batches": 2.5e-08, + "cache_read_input_audio_token_cost_flex": 2.5e-08, + "cache_read_input_audio_token_cost_priority": 9e-08, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-07", "input_cost_per_audio_token": 5e-07, + "input_cost_per_audio_token_flex": 2.5e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, @@ -31886,10 +31898,14 @@ }, "gemini/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_batches": 1e-07, + "cache_read_input_audio_token_cost_flex": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py index 142d49bfff1..b6c18e0da86 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py @@ -11,6 +11,7 @@ import respx import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.llms.base import CachedTokensDetails from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage TIER_MODEL: Final = "tier-priced-test-model" @@ -31,6 +32,20 @@ PROMPT_TOKENS: Final = 1000 CACHED_TOKENS: Final = 200 COMPLETION_TOKENS: Final = 500 TIER_API_BASE: Final = "https://tier-pricing.invalid/v1" +AUDIO_TIER_MODEL: Final = "tier-priced-gemini-audio-test-model" +AUDIO_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType( + { + "input_cost_per_token": 4e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_audio_token": 3e-06, + "cache_read_input_audio_token_cost": 2e-06, + "input_cost_per_audio_token_flex": 1e-06, + "input_cost_per_audio_token_priority": 2e-06, + "cache_read_input_audio_token_cost_flex": 4e-07, + "cache_read_input_audio_token_cost_priority": 8e-07, + } +) def _cost_at(prices: Mapping[str, float], column_suffix: str) -> float: @@ -78,6 +93,53 @@ def test_completion_cost_bills_the_price_columns_of_the_service_tier( assert cost == pytest.approx(_cost_at(TIER_ROW, column_suffix)) +@pytest.mark.parametrize( + ("service_tier", "audio_rate_key", "cache_audio_rate_key"), + ( + pytest.param("flex", "input_cost_per_audio_token_flex", "cache_read_input_audio_token_cost_flex"), + pytest.param("priority", "input_cost_per_audio_token_priority", "cache_read_input_audio_token_cost_priority"), + ), +) +def test_gemini_audio_and_cached_audio_use_service_tier_rates( + local_model_cost_map: None, service_tier: str, audio_rate_key: str, cache_audio_rate_key: str +) -> None: + litellm.register_model( + { + AUDIO_TIER_MODEL: { + "litellm_provider": "gemini", + "mode": "chat", + **AUDIO_TIER_ROW, + } + } + ) + usage: Final = Usage( + prompt_tokens=40, + completion_tokens=0, + total_tokens=40, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=20, + audio_tokens=20, + cached_tokens=10, + cached_tokens_details=CachedTokensDetails(audio_tokens=10), + ), + ) + response: Final = ModelResponse(model=AUDIO_TIER_MODEL, usage=usage) + + cost: Final = litellm.completion_cost( + completion_response=response, + model=AUDIO_TIER_MODEL, + custom_llm_provider="gemini", + service_tier=service_tier, + ) + + expected: Final = ( + 20 * AUDIO_TIER_ROW["input_cost_per_token"] + + 10 * AUDIO_TIER_ROW[audio_rate_key] + + 10 * AUDIO_TIER_ROW[cache_audio_rate_key] + ) + assert cost == pytest.approx(expected) + + LONG_CONTEXT_TIER_MODEL: Final = "long-context-tier-priced-test-model" LONG_CONTEXT_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType( { diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py index 921dde8bf7b..bdc14c4934b 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py @@ -9,6 +9,7 @@ from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import ( used_pricing_keys, zero_cost_warning, ) +from litellm.types.llms.base import CachedTokensDetails from litellm.types.utils import CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, Usage PER_SECOND_ENTRY: Final = {"cost_per_second": 0.00042} @@ -135,6 +136,36 @@ def test_audio_tokens_need_the_audio_rates() -> None: assert diagnostic["missing_pricing_keys"] == ("input_cost_per_audio_token", "output_cost_per_audio_token") +def test_cached_audio_uses_its_cache_rate_in_zero_cost_diagnostic() -> None: + usage: Final = Usage( + prompt_tokens=20, + completion_tokens=0, + total_tokens=20, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=0, + audio_tokens=20, + cached_tokens=20, + cached_tokens_details=CachedTokensDetails(audio_tokens=20), + ), + ) + pricing_entry: Final = { + "input_cost_per_token": 1e-6, + "input_cost_per_audio_token": 2e-6, + "cache_read_input_audio_token_cost": 0, + } + + assert used_pricing_keys(usage) == ("cache_read_input_audio_token_cost",) + assert ( + diagnose_zero_cost( + usage=usage, + pricing_model="gemini-audio", + pricing_entry=pricing_entry, + calculation_failed=False, + ) + is None + ) + + def test_warning_names_the_request_the_entry_the_missing_keys_and_the_counter() -> None: diagnostic = diagnose_zero_cost( usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PER_SECOND_ENTRY, calculation_failed=False diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 4c3ee9e7ca5..3bc864cc047 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -16,8 +16,10 @@ from pydantic import BaseModel import litellm from litellm import ModelResponse, completion, embedding, image_generation from litellm._logging import verbose_logger +from litellm.cost_calculator import batch_cost_calculator from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, parse_prompt_tokens_details from litellm.llms.anthropic.pass_through.messages import handler as anthropic_messages_handler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig @@ -30,8 +32,9 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( normalize_gemini_speech_config, reject_unknown_gemini_tts_voice_keys, ) +from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.vertex_ai import GeminiFinishReason, RequestBody, UsageMetadata -from litellm.types.utils import ChoiceLogprobs, Usage +from litellm.types.utils import ChoiceLogprobs, ModelInfo, Usage from litellm.utils import _invalidate_model_cost_lowercase_map, CustomStreamWrapper from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.integrations.custom_logger import CustomLogger @@ -1515,6 +1518,7 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities(): v = VertexGeminiConfig() usage_metadata = { "promptTokenCount": 210, + "cachedContentTokenCount": 54, "candidatesTokenCount": 50, "totalTokenCount": 260, "promptTokensDetails": [ @@ -1536,9 +1540,10 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities(): usage_metadata = UsageMetadata(**usage_metadata) result = v.calculate_usage(completion_response={"usageMetadata": usage_metadata}) - # prompt details are total - cached per modality - assert result.prompt_tokens_details.text_tokens == 16 # 20 - 4 - assert result.prompt_tokens_details.image_tokens == 140 # (90 + 100) - (40 + 10) + assert result.prompt_tokens_details.text_tokens == 20 + assert result.prompt_tokens_details.image_tokens == 190 + assert result.prompt_tokens_details.cached_tokens == 54 + assert result.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(text_tokens=4, image_tokens=50) # candidates details accumulate duplicate modalities assert result.completion_tokens_details.text_tokens == 20 # 15 + 5 @@ -1546,6 +1551,118 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities(): assert result.completion_tokens == 50 +def _gemini_usage_with_cached_audio(cached_audio_tokens: int) -> Usage: + cache_tokens_details: Final = ( + ( + {"modality": "TEXT", "tokenCount": 5}, + {"modality": "AUDIO", "tokenCount": cached_audio_tokens}, + ) + if cached_audio_tokens > 0 + else ({"modality": "TEXT", "tokenCount": 5},) + ) + completion_response: Final = { + "usageMetadata": { + "promptTokenCount": 40, + "cachedContentTokenCount": 5 + cached_audio_tokens, + "promptTokensDetails": ( + {"modality": "TEXT", "tokenCount": 20}, + {"modality": "AUDIO", "tokenCount": 20}, + ), + "cacheTokensDetails": cache_tokens_details, + "candidatesTokenCount": 7, + "totalTokenCount": 47, + } + } + return VertexGeminiConfig.calculate_usage(completion_response=completion_response) + + +def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_batch_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "input_cost_per_token_batches": 1e-6, + "input_cost_per_audio_token": 6e-6, + "input_cost_per_audio_token_batches": 3e-6, + "cache_read_input_token_cost": 8e-7, + "cache_read_input_token_cost_batches": 4e-7, + "cache_read_input_audio_token_cost": 5e-7, + "cache_read_input_audio_token_cost_batches": 2e-7, + } + usage: Final = _gemini_usage_with_cached_audio(5) + expected_prompt_cost: Final = ( + 15 * model_info["input_cost_per_token_batches"] + + 15 * model_info["input_cost_per_audio_token_batches"] + + 5 * model_info["cache_read_input_token_cost_batches"] + + 5 * model_info["cache_read_input_audio_token_cost_batches"] + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, + model="gemini-audio-batch-test-model", + custom_llm_provider="gemini", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(expected_prompt_cost) + + +def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_standard_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "input_cost_per_audio_token": 6e-6, + "cache_read_input_token_cost": 8e-7, + "cache_read_input_audio_token_cost": 5e-7, + } + usage: Final = _gemini_usage_with_cached_audio(5) + expected_prompt_cost: Final = ( + 15 * model_info["input_cost_per_token"] + + 15 * model_info["input_cost_per_audio_token"] + + 5 * model_info["cache_read_input_token_cost"] + + 5 * model_info["cache_read_input_audio_token_cost"] + ) + + prompt_cost, _ = generic_cost_per_token( + model="gemini-audio-standard-test-model", + usage=usage, + custom_llm_provider="gemini", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(expected_prompt_cost) + + +def test_gemini_usage_totals_are_subtracted_once_and_audio_cache_rate_matches_audio_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "input_cost_per_audio_token": 6e-6, + "cache_read_input_token_cost": 8e-7, + "cache_read_input_audio_token_cost": 6e-6, + } + cached_usage: Final = _gemini_usage_with_cached_audio(5) + uncached_usage: Final = _gemini_usage_with_cached_audio(0) + cached_details: Final = parse_prompt_tokens_details(cached_usage) + cached_prompt_cost, _ = generic_cost_per_token( + model="gemini-audio-standard-test-model", + usage=cached_usage, + custom_llm_provider="gemini", + model_info=model_info, + ) + uncached_prompt_cost, _ = generic_cost_per_token( + model="gemini-audio-standard-test-model", + usage=uncached_usage, + custom_llm_provider="gemini", + model_info=model_info, + ) + + assert cached_usage.prompt_tokens_details.text_tokens == 20 + assert cached_usage.prompt_tokens_details.audio_tokens == 20 + assert cached_usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails( + text_tokens=5, audio_tokens=5 + ) + assert cached_details["text_tokens"] == 15 + assert cached_details["audio_tokens"] == 15 + assert cached_prompt_cost == pytest.approx(uncached_prompt_cost) + + def test_vertex_ai_map_thinking_param_with_budget_tokens_0(): """ If budget_tokens is 0, do not set includeThoughts to True @@ -6150,7 +6267,7 @@ def test_vertex_ai_usage_metadata_with_document_tokens_in_prompt(): def test_vertex_ai_usage_metadata_with_document_tokens_cached(): - """Test that cached DOCUMENT tokens are correctly subtracted from prompt text tokens.""" + """Test cached DOCUMENT token details and unchanged prompt cost.""" v = VertexGeminiConfig() usage_metadata_dict = { @@ -6174,10 +6291,24 @@ def test_vertex_ai_usage_metadata_with_document_tokens_cached(): completion_response = {"usageMetadata": usage_metadata_dict} result = v.calculate_usage(completion_response=completion_response) - # DOCUMENT cached tokens map to cached_text_tokens, so: - # text_tokens = (8 TEXT + 774 DOCUMENT) - 400 cached = 382 - assert result.prompt_tokens_details.text_tokens == 382, "text_tokens should be (8 + 774) - 400 cached = 382" + assert result.prompt_tokens_details.text_tokens == 782 assert result.prompt_tokens_details.cached_tokens == 400 + assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 400 + + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "cache_read_input_token_cost": 5e-7, + } + prompt_cost, _ = generic_cost_per_token( + model="gemini-document-cache-test-model", + usage=result, + custom_llm_provider="gemini", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx( + 382 * model_info["input_cost_per_token"] + 400 * model_info["cache_read_input_token_cost"] + ) def test_async_streaming_uses_custom_client(): diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 5d4eb5fae12..550fe5f0d1a 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -16,6 +16,7 @@ import litellm from litellm import TranscriptionResponse, model_cost from litellm.constants import REALTIME_METERED_USAGE_KEY from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator +from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, @@ -3337,14 +3338,7 @@ def test_anthropic_us_data_residency_uplift_on_claude_4_6_and_later_models(_loca def test_gemini_cache_tokens_details_no_negative_values(): - """ - Test for Issue #18750: Negative text_tokens with Gemini caching - - When using Gemini with explicit caching, the response includes cacheTokensDetails - which breaks down cached tokens by modality. This test ensures that: - 1. text_tokens is never negative - 2. We correctly subtract cached tokens per modality (not total) - """ + """Ensure issue #18750 cache details remain non-negative after cost parsing.""" from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) @@ -3372,27 +3366,17 @@ def test_gemini_cache_tokens_details_no_negative_values(): usage = VertexGeminiConfig.calculate_usage(completion_response) - # Text tokens should be non-cached text only: 9402 - 9393 = 9 - assert usage.prompt_tokens_details.text_tokens == 9, ( - f"Expected text_tokens=9, got {usage.prompt_tokens_details.text_tokens}" - ) + prompt_details: Final = parse_prompt_tokens_details(usage) - # Image tokens should be non-cached image only: 258 - 258 = 0 - assert usage.prompt_tokens_details.image_tokens == 0, ( - f"Expected image_tokens=0, got {usage.prompt_tokens_details.image_tokens}" + assert usage.prompt_tokens_details.text_tokens == 9402 + assert usage.prompt_tokens_details.image_tokens == 258 + assert usage.prompt_tokens_details.cached_tokens == 9651 + assert usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails( + text_tokens=9393, image_tokens=258 ) - - # Total cached should match - assert usage.prompt_tokens_details.cached_tokens == 9651, ( - f"Expected cached_tokens=9651, got {usage.prompt_tokens_details.cached_tokens}" - ) - - # MOST IMPORTANT: text_tokens should NEVER be negative - assert usage.prompt_tokens_details.text_tokens >= 0, ( - f"BUG: text_tokens is negative ({usage.prompt_tokens_details.text_tokens})! This was the issue in #18750" - ) - - print("✅ Issue #18750 fix verified: text_tokens is correctly calculated and non-negative") + assert prompt_details["text_tokens"] == 9 + assert prompt_details["image_tokens"] == 0 + assert prompt_details["text_tokens"] >= 0 def test_gemini_without_cache_tokens_details(): @@ -5990,6 +5974,45 @@ def test_batch_cost_calculator_prices_modalities_and_cached_tokens_together_in_t assert prompt_cost == pytest.approx(298_926 * 3e-6 + 64 * 5e-6 + 10 * 3e-6 + 1_000 * 3e-7) +def test_batch_cost_calculator_prices_cached_audio_at_its_batch_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "input_cost_per_token_batches": 1e-6, + "input_cost_per_audio_token": 6e-6, + "input_cost_per_audio_token_batches": 3e-6, + "cache_read_input_token_cost": 8e-7, + "cache_read_input_token_cost_batches": 4e-7, + "cache_read_input_audio_token_cost": 5e-7, + "cache_read_input_audio_token_cost_batches": 2e-7, + } + usage: Final = Usage( + prompt_tokens=40, + completion_tokens=0, + total_tokens=40, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=20, + audio_tokens=20, + cached_tokens=10, + cached_tokens_details=CachedTokensDetails(text_tokens=5, audio_tokens=5), + ), + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, + model="gemini-audio-batch-test-model", + custom_llm_provider="gemini", + model_info=model_info, + ) + + expected: Final = ( + 15 * model_info["input_cost_per_token_batches"] + + 15 * model_info["input_cost_per_audio_token_batches"] + + 5 * model_info["cache_read_input_token_cost_batches"] + + 5 * model_info["cache_read_input_audio_token_cost_batches"] + ) + assert prompt_cost == pytest.approx(expected) + + QWEN3_NEXT_REGIONS: Final = ("ap-northeast-1", "ap-south-1", "ap-southeast-2", "eu-west-1", "eu-west-2", "sa-east-1") diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index fcd6b8aa402..338faae188f 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -994,6 +994,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_token_cost_above_1hr_above_100k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, "cache_read_input_audio_token_cost": {"type": "number"}, + "cache_read_input_audio_token_cost_batches": {"type": "number"}, + "cache_read_input_audio_token_cost_flex": {"type": "number"}, + "cache_read_input_audio_token_cost_priority": {"type": "number"}, "cache_read_input_image_token_cost": {"type": "number"}, "audio_transcription_config": {"type": "string"}, "deprecation_date": {"type": "string"}, @@ -1001,6 +1004,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, "google_maps_grounding_cost_per_query": {"type": "number"}, "input_cost_per_audio_token": {"type": "number"}, + "input_cost_per_audio_token_flex": {"type": "number"}, "input_cost_per_image_token": {"type": "number"}, "input_cost_per_character": {"type": "number"}, "input_cost_per_character_above_128k_tokens": {"type": "number"}, diff --git a/tests/unit/test_utils_get_model_info.py b/tests/unit/test_utils_get_model_info.py index bfaac44a483..4bd16a2997f 100644 --- a/tests/unit/test_utils_get_model_info.py +++ b/tests/unit/test_utils_get_model_info.py @@ -61,6 +61,30 @@ def test_get_model_info_custom_llm_with_same_name_vllm(monkeypatch): assert model_info["input_cost_per_token"] == 0.0 +def test_get_model_info_passes_through_gemini_tiered_audio_rates() -> None: + model: Final = "gemini-tiered-audio-rates-test" + prices: Final = { + "input_cost_per_audio_token_flex": 1e-6, + "input_cost_per_audio_token_priority": 2e-6, + "cache_read_input_audio_token_cost_flex": 3e-7, + "cache_read_input_audio_token_cost_priority": 4e-7, + "cache_read_input_audio_token_cost_batches": 5e-7, + } + litellm.register_model( + { + model: { + "litellm_provider": "gemini", + "mode": "chat", + **prices, + } + } + ) + + model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="gemini") + + assert {key: model_info[key] for key in prices} == prices + + def test_get_model_info_ollama_chat(): from litellm.llms.ollama.completion.transformation import OllamaConfig diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index da9e77e9b58..b5a86dd2187 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35259,6 +35259,12 @@ export interface components { cache_creation_input_token_cost_ultrafast?: number | null; /** Cache Read Input Audio Token Cost */ cache_read_input_audio_token_cost?: number | null; + /** Cache Read Input Audio Token Cost Batches */ + cache_read_input_audio_token_cost_batches?: number | null; + /** Cache Read Input Audio Token Cost Flex */ + cache_read_input_audio_token_cost_flex?: number | null; + /** Cache Read Input Audio Token Cost Priority */ + cache_read_input_audio_token_cost_priority?: number | null; /** Cache Read Input Image Token Cost */ cache_read_input_image_token_cost?: number | null; /** Cache Read Input Token Cost */ @@ -35338,6 +35344,10 @@ export interface components { input_cost_per_audio_token?: number | null; /** Input Cost Per Audio Token Batches */ input_cost_per_audio_token_batches?: number | null; + /** Input Cost Per Audio Token Flex */ + input_cost_per_audio_token_flex?: number | null; + /** Input Cost Per Audio Token Priority */ + input_cost_per_audio_token_priority?: number | null; /** Input Cost Per Character */ input_cost_per_character?: number | null; /** Input Cost Per Character Above 128K Tokens */ @@ -50568,6 +50578,12 @@ export interface components { cache_creation_input_token_cost_ultrafast?: number | null; /** Cache Read Input Audio Token Cost */ cache_read_input_audio_token_cost?: number | null; + /** Cache Read Input Audio Token Cost Batches */ + cache_read_input_audio_token_cost_batches?: number | null; + /** Cache Read Input Audio Token Cost Flex */ + cache_read_input_audio_token_cost_flex?: number | null; + /** Cache Read Input Audio Token Cost Priority */ + cache_read_input_audio_token_cost_priority?: number | null; /** Cache Read Input Image Token Cost */ cache_read_input_image_token_cost?: number | null; /** Cache Read Input Token Cost */ @@ -50647,6 +50663,10 @@ export interface components { input_cost_per_audio_token?: number | null; /** Input Cost Per Audio Token Batches */ input_cost_per_audio_token_batches?: number | null; + /** Input Cost Per Audio Token Flex */ + input_cost_per_audio_token_flex?: number | null; + /** Input Cost Per Audio Token Priority */ + input_cost_per_audio_token_priority?: number | null; /** Input Cost Per Character */ input_cost_per_character?: number | null; /** Input Cost Per Character Above 128K Tokens */