diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 440d97d13be..f5319776213 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -97,6 +97,7 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse +from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( HttpxBinaryResponseContent, ImageGenerationRequestQuality, @@ -2381,6 +2382,46 @@ def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str] return [attr for attr in field_names if attr != "cache_creation_tokens"] +def _combine_cached_tokens_details( + current: CachedTokensDetails | None, new: CachedTokensDetails +) -> CachedTokensDetails: + def _sum_optional(current_value: int | None, new_value: int | None) -> int | None: + if current_value is None and new_value is None: + return None + return (current_value or 0) + (new_value or 0) + + return CachedTokensDetails( + text_tokens=_sum_optional(current.text_tokens if current is not None else None, new.text_tokens), + audio_tokens=_sum_optional(current.audio_tokens if current is not None else None, new.audio_tokens), + image_tokens=_sum_optional(current.image_tokens if current is not None else None, new.image_tokens), + ) + + +def _combine_prompt_tokens_details( + current: PromptTokensDetailsWrapper | None, new: PromptTokensDetailsWrapper +) -> PromptTokensDetailsWrapper: + base: Final = current if current is not None else PromptTokensDetailsWrapper() + base_values: Final = MappingProxyType( + {attr: getattr(base, attr) for attr in type(base).model_fields if hasattr(base, attr)} + ) + summed: Final = MappingProxyType( + { + attr: (getattr(base, attr, 0) or 0) + (getattr(new, attr) or 0) + for attr in _summable_prompt_token_fields(new) + if hasattr(new, attr) and isinstance(getattr(new, attr) or 0, (int, float)) + } + ) + new_cached_tokens_details: Final = getattr(new, "cached_tokens_details", None) + cached_tokens_details: Final = ( + _combine_cached_tokens_details(getattr(base, "cached_tokens_details", None), new_cached_tokens_details) + if isinstance(new_cached_tokens_details, CachedTokensDetails) + else getattr(base, "cached_tokens_details", None) + ) + return PromptTokensDetailsWrapper( + **MappingProxyType({**base_values, **summed, "cached_tokens_details": cached_tokens_details}) + ) + + class BaseTokenUsageProcessor: @staticmethod def combine_usage_objects(usage_objects: list[Usage]) -> Usage: @@ -2389,7 +2430,6 @@ class BaseTokenUsageProcessor: """ from litellm.types.utils import ( CompletionTokensDetailsWrapper, - PromptTokensDetailsWrapper, Usage, ) @@ -2408,27 +2448,10 @@ class BaseTokenUsageProcessor: and isinstance(current_val, (int, float)) ): setattr(combined, attr, current_val + new_val) - # Handle nested prompt_tokens_details if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details: - if not hasattr(combined, "prompt_tokens_details") or not combined.prompt_tokens_details: - combined.prompt_tokens_details = PromptTokensDetailsWrapper() - - # Check what keys exist in the model's prompt_tokens_details - # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings - for attr in _summable_prompt_token_fields(usage.prompt_tokens_details): - if ( - hasattr(usage.prompt_tokens_details, attr) - and not attr.startswith("_") - and not callable(_attribute_value(usage.prompt_tokens_details, attr)) - ): - current_val = getattr(combined.prompt_tokens_details, attr, 0) or 0 - new_val = getattr(usage.prompt_tokens_details, attr, 0) or 0 - if new_val is not None and isinstance(new_val, (int, float)): - setattr( - combined.prompt_tokens_details, - attr, - current_val + new_val, - ) + combined.prompt_tokens_details = _combine_prompt_tokens_details( + getattr(combined, "prompt_tokens_details", None), usage.prompt_tokens_details + ) # Handle nested completion_tokens_details if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details: diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index e5977ca4156..8fc428b38ae 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -9,6 +9,8 @@ from types import MappingProxyType from typing import Any, Final, Literal, TypedDict, cast from zoneinfo import ZoneInfo, ZoneInfoNotFoundError +from typing_extensions import ReadOnly + import litellm from litellm._internal_context import current_billing_time from litellm._logging import verbose_logger @@ -772,6 +774,7 @@ def calculate_cache_writing_cost( class PromptTokensDetailsResult(TypedDict): cache_hit_tokens: int + cache_hit_audio_tokens: ReadOnly[int] cache_creation_tokens: int cache_creation_token_details: CacheCreationTokenDetails | None text_tokens: int @@ -802,12 +805,34 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: ) or None ) - text_tokens: Final = ( - cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None)) - or 0 # default to prompt tokens, if this field is not set + cached_tokens_details: Final = getattr(usage.prompt_tokens_details, "cached_tokens_details", None) + cached_audio_tokens: Final = min( + _get_token_detail_value(cached_tokens_details, "audio_tokens") or 0, cache_hit_tokens + ) + cached_text_tokens: Final = min( + _get_token_detail_value(cached_tokens_details, "text_tokens") or 0, + cache_hit_tokens - cached_audio_tokens, + ) + cached_image_tokens: Final = min( + _get_token_detail_value(cached_tokens_details, "image_tokens") or 0, + cache_hit_tokens - cached_audio_tokens - cached_text_tokens, + ) + text_tokens: Final = max( + ( + cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None)) + or 0 # default to prompt tokens, if this field is not set + ) + - cached_text_tokens, + 0, + ) + audio_tokens: Final = max( + (cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0) - cached_audio_tokens, + 0, + ) + image_tokens: Final = max( + (cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0) - cached_image_tokens, + 0, ) - audio_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0 - image_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0 video_tokens: Final = _coerce_token_count(getattr(usage.prompt_tokens_details, "video_tokens", 0)) character_count: Final = ( cast( @@ -835,6 +860,7 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: return PromptTokensDetailsResult( cache_hit_tokens=cache_hit_tokens, + cache_hit_audio_tokens=cached_audio_tokens, cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, text_tokens=text_tokens, @@ -918,7 +944,16 @@ def _calculate_input_cost( prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost ### CACHE READ COST - Now uses tiered pricing - prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost + cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"] + audio_cache_read_rate: Final = _get_cost_per_unit( + model_info, + _get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), + None, + ) + prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost + prompt_cost += float(cache_hit_audio_tokens) * ( + audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost + ) ### AUDIO COST if prompt_tokens_details["audio_tokens"]: @@ -1149,6 +1184,7 @@ def generic_cost_per_token( ### PROCESSING COST prompt_tokens_details = PromptTokensDetailsResult( cache_hit_tokens=0, + cache_hit_audio_tokens=0, cache_creation_tokens=0, cache_creation_token_details=None, text_tokens=usage.prompt_tokens, @@ -1319,6 +1355,7 @@ class BilledTokenRates: input_cost_per_token: float output_cost_per_token: float cache_read_input_token_cost: float + cache_read_input_audio_token_cost: float cache_creation_input_token_cost: float cache_creation_input_token_cost_above_1hr: float output_cost_per_reasoning_token: float @@ -1330,6 +1367,7 @@ class BilledTokenRates: input_cost_per_token=self.input_cost_per_token * multiplier, output_cost_per_token=self.output_cost_per_token * multiplier, cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier, + cache_read_input_audio_token_cost=self.cache_read_input_audio_token_cost * multiplier, cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier, cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier, output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier, @@ -1353,15 +1391,16 @@ def _reasoning_token_count(usage: Usage) -> int: return parsed or _coerce_token_count(getattr(usage, "reasoning_tokens", 0)) -def _cache_token_counts(usage: Usage) -> tuple[int, int, CacheCreationTokenDetails | None]: - """(cache read tokens, cache creation tokens, cache creation details): read from prompt_tokens_details - first, then the private top-level counters the Usage constructor mirrors cache tokens onto for - providers/callers that bypass the details.""" +def _cache_token_counts(usage: Usage) -> tuple[int, int, int, CacheCreationTokenDetails | None]: + """(cache read tokens, cached audio tokens, cache creation tokens, cache creation details): read from + prompt_tokens_details first, then the private top-level counters the Usage constructor mirrors cache + tokens onto for providers/callers that bypass the details.""" parsed: Final = parse_prompt_tokens_details(usage) if usage.prompt_tokens_details is not None else None parsed_read: Final = parsed["cache_hit_tokens"] if parsed is not None else 0 parsed_creation: Final = parsed["cache_creation_tokens"] if parsed is not None else 0 return ( parsed_read or _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0)), + parsed["cache_hit_audio_tokens"] if parsed is not None else 0, parsed_creation or _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0)), parsed["cache_creation_token_details"] if parsed is not None else None, ) @@ -1372,11 +1411,13 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat cache rates (else the input rate) and reasoning at the output rate, as _cost_per_token_custom_pricing_helper does.""" input_rate: Final = custom_cost_per_token["input_cost_per_token"] output_rate: Final = custom_cost_per_token["output_cost_per_token"] + cache_read_rate: Final = custom_cost_per_token.get("cache_read_input_token_cost", input_rate) cache_creation_rate: Final = custom_cost_per_token.get("cache_creation_input_token_cost", input_rate) return BilledTokenRates( input_cost_per_token=input_rate, output_cost_per_token=output_rate, - cache_read_input_token_cost=custom_cost_per_token.get("cache_read_input_token_cost", input_rate), + cache_read_input_token_cost=cache_read_rate, + cache_read_input_audio_token_cost=cache_read_rate, cache_creation_input_token_cost=cache_creation_rate, cache_creation_input_token_cost_above_1hr=cache_creation_rate, output_cost_per_reasoning_token=output_rate, @@ -1413,6 +1454,11 @@ def _cost_map_billed_rates( completion_base_cost=completion_base_cost, current_time=billing_time, ) + audio_cache_read_rate: Final = _get_cost_per_unit( + model_info, + _get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), + None, + ) multiplier: Final = ( _get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift(model_info, vertex_location) @@ -1422,6 +1468,9 @@ def _cost_map_billed_rates( input_cost_per_token=prompt_base_cost, output_cost_per_token=completion_base_cost, cache_read_input_token_cost=cache_read_cost_rate, + cache_read_input_audio_token_cost=( + audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost_rate + ), cache_creation_input_token_cost=cache_creation_cost_rate, cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate, output_cost_per_reasoning_token=reasoning_rate, @@ -1494,7 +1543,9 @@ def get_token_type_cost_breakdown( if rates is None: return TokenTypeCostBreakdown(0.0, 0.0, 0.0) - cache_read_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(usage) + cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts( + usage + ) cache_creation_cost: Final = ( float(cache_creation_tokens) * rates.cache_creation_input_token_cost if custom_cost_per_token is not None @@ -1507,7 +1558,10 @@ def get_token_type_cost_breakdown( ) return TokenTypeCostBreakdown( reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token, - cache_read_cost=float(cache_read_tokens) * rates.cache_read_input_token_cost, + cache_read_cost=( + float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost + + float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost + ), cache_creation_cost=cache_creation_cost, rates=rates, ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2220d0e1fe5..6d63fd8e3a9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5513,7 +5513,8 @@ }, "azure/gpt-realtime-2025-08-28": { "cache_creation_input_audio_token_cost": 4e-06, - "cache_read_input_token_cost": 4e-06, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-03-02", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -5546,7 +5547,8 @@ }, "azure/gpt-realtime-1.5-2026-02-23": { "cache_creation_input_audio_token_cost": 4e-06, - "cache_read_input_token_cost": 4e-06, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-08-24", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -5683,6 +5685,7 @@ }, "azure/gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, @@ -5715,6 +5718,7 @@ }, "azure/gpt-realtime-mini-2025-10-06": { "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, @@ -32678,6 +32682,7 @@ }, "gpt-realtime": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 3.2e-05, @@ -32711,6 +32716,7 @@ }, "gpt-realtime-1.5": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -32847,6 +32853,7 @@ "gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, @@ -32878,6 +32885,7 @@ }, "gpt-realtime-2025-08-28": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 3.2e-05, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index e8aacac9e67..64324c6cad8 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -43,6 +43,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.responses.litellm_completion_transformation.session_handler import ( ResponsesSessionHandler, ) +from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, @@ -2816,27 +2817,24 @@ class LiteLLMCompletionResponsesConfig: # Translate prompt_tokens_details to input_tokens_details if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details is not None: prompt_details: Final = usage.prompt_tokens_details - input_details_dict: Final[dict[str, int]] = {} - - if hasattr(prompt_details, "cached_tokens") and prompt_details.cached_tokens is not None: - input_details_dict["cached_tokens"] = prompt_details.cached_tokens - else: - input_details_dict["cached_tokens"] = 0 - - if hasattr(prompt_details, "text_tokens") and prompt_details.text_tokens is not None: - input_details_dict["text_tokens"] = prompt_details.text_tokens - - if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None: - input_details_dict["audio_tokens"] = prompt_details.audio_tokens - - cache_write_tokens = getattr(prompt_details, "cache_write_tokens", None) or getattr( + cached_tokens_details: Final = getattr(prompt_details, "cached_tokens_details", None) + cache_write_tokens: Final = getattr(prompt_details, "cache_write_tokens", None) or getattr( prompt_details, "cache_creation_tokens", None ) - if cache_write_tokens is not None: - input_details_dict["cache_write_tokens"] = cache_write_tokens - - if input_details_dict: - response_usage.input_tokens_details = InputTokensDetails(**input_details_dict) + cache_write_extra: Final[Mapping[str, int]] = ( + MappingProxyType({"cache_write_tokens": cache_write_tokens}) + if cache_write_tokens is not None + else MappingProxyType({}) + ) + response_usage.input_tokens_details = InputTokensDetails( + cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0, + text_tokens=prompt_details.text_tokens, + audio_tokens=prompt_details.audio_tokens, + cached_tokens_details=( + cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None + ), + **cache_write_extra, + ) # Translate completion_tokens_details to output_tokens_details if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details is not None: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 599e978df6a..d63e3ddf0aa 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1179,6 +1179,9 @@ class ResponseAPILoggingUtils: audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None), text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None), image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None), + cached_tokens_details=getattr( + response_api_usage.input_tokens_details, "cached_tokens_details", None + ), cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), ) completion_tokens_details: CompletionTokensDetailsWrapper | None = None diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index f09727ad92b..938aa8064c9 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -75,3 +75,9 @@ class HiddenParams(OpenAIObject): data: Final = super().model_dump(**kwargs) data["_response_ms"] = self._response_ms return data + + +class CachedTokensDetails(BaseModel): + text_tokens: int | None = None + audio_tokens: int | None = None + image_tokens: int | None = None diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index dfafe27e0a1..e3eac9b9205 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -91,6 +91,8 @@ from litellm.types.responses.main import ( OutputImageGenerationCall, ) +from .base import CachedTokensDetails + FileContent = IO[bytes] | bytes | PathLike FileTypes = ( @@ -1288,6 +1290,7 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject): class InputTokensDetails(BaseLiteLLMOpenAIResponseObject): audio_tokens: int | None = None cached_tokens: int = 0 + cached_tokens_details: CachedTokensDetails | None = None text_tokens: int | None = None model_config = {"extra": "allow"} @@ -2254,10 +2257,17 @@ class OpenAIRealtimeInputAudioTranscriptionCompleted(TypedDict): usage: NotRequired[ReadOnly[Mapping[str, object]]] +class OpenAIRealtimeCachedTokensDetails(TypedDict, total=False): + text_tokens: ReadOnly[int] + audio_tokens: ReadOnly[int] + image_tokens: ReadOnly[int] + + class OpenAIRealtimeUsageTokenDetails(TypedDict): audio_tokens: ReadOnly[int] text_tokens: ReadOnly[int] cached_tokens: NotRequired[ReadOnly[int]] + cached_tokens_details: NotRequired[ReadOnly[OpenAIRealtimeCachedTokensDetails]] class OpenAIRealtimeResponseUsage(TypedDict): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 00c55b35182..1d73542c9bb 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -48,6 +48,7 @@ from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.types.llms.base import ( BaseLiteLLMOpenAIResponseObject, + CachedTokensDetails, LiteLLMPydanticObjectBase, ) from litellm.types.mcp import MCPServerCostInfo @@ -252,6 +253,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing cache_read_input_token_cost: float | None + cache_read_input_audio_token_cost: ReadOnly[float | None] cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing cache_read_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing @@ -1710,6 +1712,9 @@ class PromptTokensDetailsWrapper( cache_creation_token_details: CacheCreationTokenDetails | None = None """Details of cache creation tokens sent to the model. Used for tracking 5m/1h cache creation tokens for Anthropic prompt caching.""" + cached_tokens_details: CachedTokensDetails | None = None + """Details of cached (cache-hit) tokens sent to the model. OpenAI realtime naming; carries the per-modality cache-read split.""" + def __setattr__(self, name: str, value: object) -> None: super().__setattr__(name, value) if name == "cache_write_tokens": @@ -1756,6 +1761,8 @@ class PromptTokensDetailsWrapper( del self.cache_creation_tokens if self.cache_creation_token_details is None: del self.cache_creation_token_details + if self.cached_tokens_details is None: + del self.cached_tokens_details class ServerToolUse(BaseModel): diff --git a/litellm/utils.py b/litellm/utils.py index 7732cd88cb5..04139a124b6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5882,6 +5882,7 @@ def _get_model_info_helper( "cache_creation_input_token_cost_ultrafast", None ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), + cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( "cache_read_input_token_cost_above_200k_tokens", None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2220d0e1fe5..6d63fd8e3a9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5513,7 +5513,8 @@ }, "azure/gpt-realtime-2025-08-28": { "cache_creation_input_audio_token_cost": 4e-06, - "cache_read_input_token_cost": 4e-06, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-03-02", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -5546,7 +5547,8 @@ }, "azure/gpt-realtime-1.5-2026-02-23": { "cache_creation_input_audio_token_cost": 4e-06, - "cache_read_input_token_cost": 4e-06, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-08-24", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -5683,6 +5685,7 @@ }, "azure/gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, @@ -5715,6 +5718,7 @@ }, "azure/gpt-realtime-mini-2025-10-06": { "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, @@ -32678,6 +32682,7 @@ }, "gpt-realtime": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 3.2e-05, @@ -32711,6 +32716,7 @@ }, "gpt-realtime-1.5": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -32847,6 +32853,7 @@ "gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, @@ -32878,6 +32885,7 @@ }, "gpt-realtime-2025-08-28": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 3.2e-05, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index cbe6fe198c9..7cd19c65887 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2648,6 +2648,7 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): prompt_tokens_details: PromptTokensDetailsResult = { "cache_hit_tokens": 0, + "cache_hit_audio_tokens": 0, "cache_creation_tokens": 0, "cache_creation_token_details": CacheCreationTokenDetails( ephemeral_5m_input_tokens=100, @@ -4005,6 +4006,7 @@ def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeyp input_cost_per_token=6e-6, output_cost_per_token=3e-5, cache_read_input_token_cost=6e-7, + cache_read_input_audio_token_cost=6e-7, cache_creation_input_token_cost=7.5e-6, cache_creation_input_token_cost_above_1hr=0.0, output_cost_per_reasoning_token=3e-5, @@ -5147,3 +5149,157 @@ def test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output( assert completion_cost == pytest.approx( 30 * info["output_cost_per_token"] + 70 * info["output_cost_per_audio_token"] ) + + +def test_cached_realtime_audio_tokens_billed_at_audio_cache_read_rate( + _local_model_cost_map: None, +) -> None: + usage = Usage( + prompt_tokens=283, + completion_tokens=0, + total_tokens=283, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=116, + audio_tokens=167, + cached_tokens=192, + cached_tokens_details={"text_tokens": 64, "audio_tokens": 128}, + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="gpt-realtime-2", usage=usage, custom_llm_provider="openai" + ) + assert prompt_cost == pytest.approx(0.0015328) + + +def test_prompt_tokens_details_without_cached_tokens_details_unchanged( + _local_model_cost_map: None, +) -> None: + usage = Usage( + prompt_tokens=283, + completion_tokens=0, + total_tokens=283, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=116, audio_tokens=167, cached_tokens=192 + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="gpt-realtime-2", usage=usage, custom_llm_provider="openai" + ) + assert prompt_cost == pytest.approx(0.0029888) + + +def test_cached_audio_tokens_fall_back_to_cache_read_input_token_cost() -> None: + model_info: ModelInfo = { + "input_cost_per_token": 4e-6, + "input_cost_per_audio_token": 32e-6, + "cache_read_input_token_cost": 5e-7, + } + usage = Usage( + prompt_tokens=283, + completion_tokens=0, + total_tokens=283, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=116, + audio_tokens=167, + cached_tokens=192, + cached_tokens_details={"text_tokens": 64, "audio_tokens": 128}, + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="some-realtime-model", + usage=usage, + custom_llm_provider="openai", + model_info=model_info, + ) + expected = 52 * 4e-6 + 64 * 5e-7 + 39 * 32e-6 + 128 * 5e-7 + assert prompt_cost == pytest.approx(expected) + + +def test_cached_audio_tokens_capped_at_cached_tokens(_local_model_cost_map: None) -> None: + """Nested cached_tokens_details exceeding cached_tokens must not over-subtract the audio bucket.""" + usage = Usage( + prompt_tokens=283, + completion_tokens=0, + total_tokens=283, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=116, + audio_tokens=167, + cached_tokens=100, + cached_tokens_details={"audio_tokens": 128}, + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="gpt-realtime-2", usage=usage, custom_llm_provider="openai" + ) + assert prompt_cost == pytest.approx(116 * 4e-6 + (167 - 100) * 32e-6 + 100 * 4e-7) + + +def test_cached_audio_tokens_billed_at_audio_cache_rate_through_model_info_lookup(_local_model_cost_map: None) -> None: + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=400, + audio_tokens=600, + cached_tokens=500, + cached_tokens_details={"text_tokens": 100, "audio_tokens": 400}, + ), + ) + + prompt_cost, _ = generic_cost_per_token(model="gpt-realtime-2.1-mini", usage=usage, custom_llm_provider="openai") + assert prompt_cost == pytest.approx(300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7) + + +def test_cache_read_breakdown_splits_cached_audio_at_the_audio_cache_rate(_local_model_cost_map: None) -> None: + usage = Usage( + prompt_tokens=4863, + completion_tokens=1087, + total_tokens=5950, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=1693, + audio_tokens=3170, + cached_tokens=2816, + cached_tokens_details={"text_tokens": 896, "audio_tokens": 1920}, + ), + ) + + breakdown = get_token_type_cost_breakdown(model="gpt-realtime-2.1-mini", custom_llm_provider="openai", usage=usage) + prompt_cost, _ = generic_cost_per_token(model="gpt-realtime-2.1-mini", usage=usage, custom_llm_provider="openai") + + assert breakdown.cache_read_cost == pytest.approx(896 * 6e-8 + 1920 * 3e-7) + assert breakdown.rates is not None + assert breakdown.rates.cache_read_input_audio_token_cost == pytest.approx(3e-7) + assert prompt_cost == pytest.approx((1693 - 896) * 6e-7 + (3170 - 1920) * 1e-5 + breakdown.cache_read_cost) + + +@pytest.mark.parametrize( + ("model", "custom_llm_provider", "expected_prompt_cost"), + ( + pytest.param("azure/gpt-realtime-2025-08-28", "azure", 300 * 4e-6 + 100 * 4e-7 + 200 * 3.2e-5 + 400 * 4e-7, id="azure-gpt-realtime"), + pytest.param("azure/gpt-realtime-1.5-2026-02-23", "azure", 300 * 4e-6 + 100 * 4e-7 + 200 * 3.2e-5 + 400 * 4e-7, id="azure-gpt-realtime-1.5"), + pytest.param("azure/gpt-realtime-mini", "azure", 300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7, id="azure-gpt-realtime-mini"), + pytest.param("gpt-realtime-mini", "openai", 300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7, id="openai-gpt-realtime-mini"), + ), +) +def test_realtime_models_bill_cached_text_and_audio_at_their_cache_read_rates( + _local_model_cost_map: None, model: str, custom_llm_provider: str, expected_prompt_cost: float +) -> None: + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=400, + audio_tokens=600, + cached_tokens=500, + cached_tokens_details={"text_tokens": 100, "audio_tokens": 400}, + ), + ) + + prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=custom_llm_provider) + assert prompt_cost == pytest.approx(expected_prompt_cost) diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 3c78bbf79d7..d5a21bccad7 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -2789,6 +2789,7 @@ class TestUsageTransformation: assert response_usage.input_tokens_details is not None assert response_usage.input_tokens_details.cached_tokens == 5 assert response_usage.input_tokens_details.text_tokens == 8 + assert "cache_write_tokens" not in response_usage.input_tokens_details.model_dump() def test_transform_usage_with_cached_tokens_gemini(self): """Test that cached_tokens from Gemini are properly transformed to input_tokens_details""" @@ -2851,6 +2852,7 @@ class TestUsageTransformation: assert response_usage.input_tokens_details is not None assert response_usage.input_tokens_details.cached_tokens == 100 assert getattr(response_usage.input_tokens_details, "cache_write_tokens", None) == 800 + assert response_usage.input_tokens_details.model_dump()["cache_write_tokens"] == 800 def test_transform_usage_with_reasoning_tokens_gemini(self): """Test that reasoning_tokens from Gemini are properly transformed to output_tokens_details""" diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 9d9eefdceb3..4d06b5e7bdc 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -577,6 +577,47 @@ class TestResponseAPILoggingUtils: assert result.completion_tokens_details is not None assert result.completion_tokens_details.reasoning_tokens == 4 + def test_transform_realtime_usage_dict_keeps_cached_tokens_details(self): + usage = { + "input_tokens": 283, + "output_tokens": 0, + "total_tokens": 283, + "input_token_details": { + "text_tokens": 116, + "audio_tokens": 167, + "cached_tokens": 192, + "cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128}, + }, + } + + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage) + + assert result.prompt_tokens_details is not None + assert result.prompt_tokens_details.cached_tokens == 192 + assert result.prompt_tokens_details.cached_tokens_details is not None + assert result.prompt_tokens_details.cached_tokens_details.audio_tokens == 128 + assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 64 + + def test_transform_response_api_usage_object_keeps_cached_tokens_details(self): + usage = ResponseAPIUsage( + input_tokens=283, + output_tokens=0, + total_tokens=283, + input_tokens_details={ + "text_tokens": 116, + "audio_tokens": 167, + "cached_tokens": 192, + "cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128}, + }, + ) + + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage) + + assert result.prompt_tokens_details is not None + assert result.prompt_tokens_details.cached_tokens_details is not None + assert result.prompt_tokens_details.cached_tokens_details.audio_tokens == 128 + assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 64 + class TestResponsesAPIProviderSpecificParams: """ diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index f2659cee3fd..68e9b6143a0 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -19,6 +19,7 @@ from litellm.cost_calculator import ( ) from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo +from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import OpenAIRealtimeStreamList from litellm.types.rerank import RerankResponse from litellm.types.utils import ( @@ -4848,6 +4849,109 @@ def test_collect_and_combine_realtime_usage_stores_partitioned_text_tokens() -> assert combined.completion_tokens_details.audio_tokens == 0 +def test_realtime_combine_sums_nested_cached_tokens_details(): + results: OpenAIRealtimeStreamList = [ + { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 283, + "output_tokens": 0, + "total_tokens": 283, + "input_token_details": { + "text_tokens": 116, + "audio_tokens": 167, + "cached_tokens": 192, + "cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128}, + }, + } + }, + }, + { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 150, + "output_tokens": 0, + "total_tokens": 150, + "input_token_details": { + "text_tokens": 50, + "audio_tokens": 100, + "cached_tokens": 100, + "cached_tokens_details": {"audio_tokens": 100}, + }, + } + }, + }, + ] + + combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results( + results=results, + ) + + assert combined.prompt_tokens_details is not None + assert combined.prompt_tokens_details.cached_tokens == 292 + assert combined.prompt_tokens_details.cached_tokens_details is not None + assert combined.prompt_tokens_details.cached_tokens_details.audio_tokens == 228 + assert combined.prompt_tokens_details.cached_tokens_details.text_tokens == 64 + assert combined.prompt_tokens_details.cached_tokens_details.image_tokens is None + + +@pytest.mark.parametrize("details_first", [True, False]) +def test_realtime_combine_keeps_cached_split_when_only_one_usage_has_details(details_first: bool): + with_details: Final = { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 283, + "output_tokens": 0, + "total_tokens": 283, + "input_token_details": { + "text_tokens": 116, + "audio_tokens": 167, + "cached_tokens": 192, + "cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128}, + }, + } + }, + } + without_details: Final = { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 150, + "output_tokens": 0, + "total_tokens": 150, + "input_token_details": {"text_tokens": 50, "audio_tokens": 100, "cached_tokens": 100}, + } + }, + } + results: OpenAIRealtimeStreamList = ( + [with_details, without_details] if details_first else [without_details, with_details] + ) + + combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results( + results=results, + ) + + assert combined.prompt_tokens_details is not None + assert combined.prompt_tokens_details.cached_tokens == 292 + assert combined.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(text_tokens=64, audio_tokens=128) + + +def test_usage_without_cached_tokens_details_omits_key(): + usage = Usage( + prompt_tokens=10, + completion_tokens=5, + total_tokens=15, + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=10), + ) + + dumped = usage.prompt_tokens_details.model_dump() + assert "cached_tokens_details" not in dumped + assert "cached_tokens_details" not in usage.prompt_tokens_details.model_dump_json() + + UNMAPPED_OCR_MODEL: Final = "azure_ai/some-unmapped-ocr-model-for-testing" MAPPED_OCR_MODEL: Final = "mistral/mistral-ocr-4-0" diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index d89496a0fd0..2ace005a8e2 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -6455,3 +6455,11 @@ def test_completion_finishes_response_metadata_before_handing_the_response_to_th assert snapshot["litellm_call_id"] assert snapshot["response_cost"] is not None assert snapshot["api_base"] + + +def test_get_model_info_carries_cache_read_input_audio_token_cost(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + info = litellm.get_model_info("gpt-realtime-2.1-mini", custom_llm_provider="openai") + assert info["cache_read_input_audio_token_cost"] == 3e-07 + assert info["cache_read_input_token_cost"] == 6e-08 diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7eadaa6c991..839aa52fa84 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16781,7 +16781,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) @@ -16887,7 +16886,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)