From bdc4f6e1577e6f035256ec8ca81520f1455a8a6a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 30 Aug 2026 01:18:34 +0000 Subject: [PATCH] fix(cost): allocate uncached realtime tokens per cached_tokens_details modality split Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 24 +++++++ .../litellm_core_utils/llm_cost_calc/utils.py | 49 ++++++++++++-- litellm/responses/utils.py | 1 + litellm/types/utils.py | 9 +++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 67 +++++++++++++++++++ tests/test_litellm/test_cost_calculator.py | 38 +++++++++++ 6 files changed, 184 insertions(+), 4 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 3adc1c25dfd..f7ffd0714be 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2275,6 +2275,28 @@ def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str] return [attr for attr in field_names if attr != "cache_creation_tokens"] +def _accumulate_cached_tokens_details( + combined_details: "PromptTokensDetailsWrapper", + usage_details: "PromptTokensDetailsWrapper", +) -> None: + from litellm.types.utils import CachedTokensDetails + + cached_details: Final = getattr(usage_details, "cached_tokens_details", None) + if not isinstance(cached_details, CachedTokensDetails): + return + if combined_details.cached_tokens_details is None: + combined_details.cached_tokens_details = CachedTokensDetails() + combined_cached_details: Final = combined_details.cached_tokens_details + for attr in CachedTokensDetails.model_fields: + new_cached_val = getattr(cached_details, attr, None) + if isinstance(new_cached_val, int): + setattr( + combined_cached_details, + attr, + (getattr(combined_cached_details, attr, 0) or 0) + new_cached_val, + ) + + class BaseTokenUsageProcessor: @staticmethod def combine_usage_objects(usage_objects: list[Usage]) -> Usage: @@ -2324,6 +2346,8 @@ class BaseTokenUsageProcessor: current_val + new_val, ) + _accumulate_cached_tokens_details(combined.prompt_tokens_details, usage.prompt_tokens_details) + # Handle nested completion_tokens_details if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details: if not hasattr(combined, "completion_tokens_details") or not combined.completion_tokens_details: diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 19e3f624268..c775c5eba5f 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -7,6 +7,8 @@ from dataclasses import dataclass from types import MappingProxyType from typing import Any, Final, Literal, TypedDict, cast +from typing_extensions import ReadOnly + import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import ( @@ -15,6 +17,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import ( ) from litellm.types.utils import ( CacheCreationTokenDetails, + CachedTokensDetails, CallTypes, CompletionTokensDetailsWrapper, DataResidency, @@ -543,10 +546,38 @@ def calculate_cache_writing_cost( return total_cost +def _allocate_uncached_modalities( + uncached_budget: int, + audio_tokens: int, + image_tokens: int, + video_tokens: int, + cached_details: "CachedTokensDetails | None", +) -> tuple[int, int, int, int]: + """Split an uncached prompt-token budget across modalities. When the provider reports + which modalities the cache covered (cached_tokens_details), subtract the cache from each + modality directly; otherwise fall back to filling audio, then image, then video.""" + cached_audio: Final = (cached_details.audio_tokens or 0) if cached_details is not None else 0 + cached_image: Final = (cached_details.image_tokens or 0) if cached_details is not None else 0 + billable_audio: Final = ( + min(max(audio_tokens - cached_audio, 0), uncached_budget) + if cached_details is not None + else min(audio_tokens, uncached_budget) + ) + billable_image: Final = ( + min(max(image_tokens - cached_image, 0), uncached_budget - billable_audio) + if cached_details is not None + else min(image_tokens, uncached_budget - billable_audio) + ) + billable_video: Final = min(video_tokens, uncached_budget - billable_audio - billable_image) + billable_text: Final = uncached_budget - billable_audio - billable_image - billable_video + return billable_audio, billable_image, billable_video, billable_text + + class PromptTokensDetailsResult(TypedDict): cache_hit_tokens: int cache_creation_tokens: int cache_creation_token_details: CacheCreationTokenDetails | None + cached_tokens_details: ReadOnly[CachedTokensDetails | None] text_tokens: int audio_tokens: int image_tokens: int @@ -574,6 +605,10 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: ) or None ) + raw_cached_tokens_details: Final = getattr(usage.prompt_tokens_details, "cached_tokens_details", None) + cached_tokens_details: Final = ( + raw_cached_tokens_details if isinstance(raw_cached_tokens_details, CachedTokensDetails) else None + ) text_tokens: Final = ( cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None)) or 0 # default to prompt tokens, if this field is not set @@ -608,6 +643,7 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: cache_hit_tokens=cache_hit_tokens, cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, + cached_tokens_details=cached_tokens_details, text_tokens=text_tokens, audio_tokens=audio_tokens, image_tokens=image_tokens, @@ -885,6 +921,7 @@ def generic_cost_per_token( cache_hit_tokens=0, cache_creation_tokens=0, cache_creation_token_details=None, + cached_tokens_details=None, text_tokens=usage.prompt_tokens, audio_tokens=0, image_tokens=0, @@ -917,13 +954,17 @@ def generic_cost_per_token( # cached and per-modality counts are both subsets of prompt_tokens and may overlap, so a # modality can only bill what the cache did not already cover or the overlap is billed twice uncached_budget: Final = max(usage.prompt_tokens - cache_hit - cache_creation, 0) - billable_audio: Final = min(audio_tokens, uncached_budget) - billable_image: Final = min(image_tokens, uncached_budget - billable_audio) - billable_video: Final = min(video_tokens, uncached_budget - billable_audio - billable_image) + billable_audio, billable_image, billable_video, billable_text = _allocate_uncached_modalities( + uncached_budget=uncached_budget, + audio_tokens=audio_tokens, + image_tokens=image_tokens, + video_tokens=video_tokens, + cached_details=prompt_tokens_details["cached_tokens_details"], + ) prompt_tokens_details["audio_tokens"] = billable_audio prompt_tokens_details["image_tokens"] = billable_image prompt_tokens_details["video_tokens"] = billable_video - prompt_tokens_details["text_tokens"] = uncached_budget - billable_audio - billable_image - billable_video + prompt_tokens_details["text_tokens"] = billable_text elif text_tokens == 0 and prompt_tokens_details["image_count"] == 0: # Clamp to zero: inconsistent streaming usage prompt_tokens_details["text_tokens"] = max( diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 39675faf735..70e5cde6640 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1128,6 +1128,7 @@ class ResponseAPILoggingUtils: text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None), image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None), cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), + cached_tokens_details=getattr(response_api_usage.input_tokens_details, "cached_tokens_details", None), ) completion_tokens_details: CompletionTokensDetailsWrapper | None = None output_tokens_details: Final[OutputTokensDetails | None] = getattr( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4bf8289d725..b5a9e894a14 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1603,6 +1603,12 @@ class CacheCreationTokenDetails(BaseModel): ephemeral_1h_input_tokens: int | None = None +class CachedTokensDetails(BaseModel): + text_tokens: int | None = None + audio_tokens: int | None = None + image_tokens: int | None = None + + class PromptTokensDetailsWrapper( SafeAttributeModel, PromptTokensDetails ): # extends with image generation fields (text_tokens, image_tokens) @@ -1645,6 +1651,9 @@ class PromptTokensDetailsWrapper( cache_creation_token_details: CacheCreationTokenDetails | None = None """Details of cache creation tokens sent to the model. Used for tracking 5m/1h cache creation tokens for Anthropic prompt caching.""" + cached_tokens_details: CachedTokensDetails | None = None + """Per-modality breakdown of cached input tokens. OpenAI Realtime API naming (input_token_details.cached_tokens_details).""" + def __setattr__(self, name: str, value: object) -> None: super().__setattr__(name, value) if name == "cache_write_tokens": diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 3c4121977de..a1d8c11f792 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -3946,3 +3946,70 @@ def test_route_image_generation_cost_falls_back_to_requested_size(monkeypatch, r ) assert cost == expected_cost + + +def test_generic_cost_per_token_realtime_uses_cached_modality_split(_local_model_cost_map): + """ + Regression for LIT-6513: with a realtime GA cached_tokens_details split, uncached + tokens must be billed per the reported modalities. The previous audio-first + allocation billed uncached text tokens at the audio rate whenever the modality + details overlapped the cached count. + """ + from litellm.types.utils import CachedTokensDetails + + usage = Usage( + prompt_tokens=3400, + completion_tokens=1100, + total_tokens=4500, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=2000, + text_tokens=1400, + audio_tokens=2000, + cached_tokens_details=CachedTokensDetails(text_tokens=500, audio_tokens=1500), + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="gpt-realtime", + usage=usage, + custom_llm_provider="openai", + ) + + model_cost_map = litellm.model_cost["gpt-realtime"] + uncached_audio = 2000 - 1500 + uncached_text = 3400 - 2000 - uncached_audio + expected_prompt_cost = ( + uncached_text * model_cost_map["input_cost_per_token"] + + uncached_audio * model_cost_map["input_cost_per_audio_token"] + + 2000 * model_cost_map["cache_read_input_token_cost"] + ) + assert round(prompt_cost, 10) == round(expected_prompt_cost, 10) + + +def test_generic_cost_per_token_realtime_without_cached_split_clamps_to_uncached_budget(_local_model_cost_map): + """Without a cached_tokens_details split, overlapping modality details must still be + clamped so cached tokens are never also billed at the full modality rate.""" + usage = Usage( + prompt_tokens=3400, + completion_tokens=1100, + total_tokens=4500, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=2000, + text_tokens=1400, + audio_tokens=2000, + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="gpt-realtime", + usage=usage, + custom_llm_provider="openai", + ) + + model_cost_map = litellm.model_cost["gpt-realtime"] + uncached_budget = 3400 - 2000 + expected_prompt_cost = ( + uncached_budget * model_cost_map["input_cost_per_audio_token"] + + 2000 * model_cost_map["cache_read_input_token_cost"] + ) + assert round(prompt_cost, 10) == round(expected_prompt_cost, 10) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 7c2174018e8..8b789df0c0f 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3952,6 +3952,44 @@ def test_combine_usage_objects_sums_mirrored_cache_write_fields_once(): assert combined_pair.prompt_tokens_details.cache_creation_tokens == 100 +def test_combine_usage_objects_sums_cached_tokens_details(): + """ + Realtime GA usage reports a per-modality cached-token split in + input_token_details.cached_tokens_details. Combining per-response usage + across a session must sum that split, or the cost path loses which + modalities the cache covered and misallocates uncached tokens. + """ + from litellm.types.utils import CachedTokensDetails + + first = Usage( + prompt_tokens=1000, + completion_tokens=100, + total_tokens=1100, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=400, + cached_tokens_details=CachedTokensDetails(text_tokens=100, audio_tokens=300), + ), + ) + second = Usage( + prompt_tokens=2000, + completion_tokens=200, + total_tokens=2200, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=900, + cached_tokens_details=CachedTokensDetails(text_tokens=200, audio_tokens=700), + ), + ) + + combined = BaseTokenUsageProcessor.combine_usage_objects([first, second]) + + assert combined.prompt_tokens_details is not None + assert combined.prompt_tokens_details.cached_tokens == 1300 + cached_details = combined.prompt_tokens_details.cached_tokens_details + assert cached_details is not None + assert cached_details.text_tokens == 300 + assert cached_details.audio_tokens == 1000 + + def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(_local_model_cost_map): """Regression: an Anthropic /v1/messages response reports cache reads as top-level cache_read_input_tokens with input_tokens excluding them. Reading that usage as