fix(cost): allocate uncached realtime tokens per cached_tokens_details modality split

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/cost_calculator.py

@@ -2275,6 +2275,28 @@ def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]
     return [attr for attr in field_names if attr != "cache_creation_tokens"]
 
 
+def _accumulate_cached_tokens_details(
+    combined_details: "PromptTokensDetailsWrapper",
+    usage_details: "PromptTokensDetailsWrapper",
+) -> None:
+    from litellm.types.utils import CachedTokensDetails
+
+    cached_details: Final = getattr(usage_details, "cached_tokens_details", None)
+    if not isinstance(cached_details, CachedTokensDetails):
+        return
+    if combined_details.cached_tokens_details is None:
+        combined_details.cached_tokens_details = CachedTokensDetails()
+    combined_cached_details: Final = combined_details.cached_tokens_details
+    for attr in CachedTokensDetails.model_fields:
+        new_cached_val = getattr(cached_details, attr, None)
+        if isinstance(new_cached_val, int):
+            setattr(
+                combined_cached_details,
+                attr,
+                (getattr(combined_cached_details, attr, 0) or 0) + new_cached_val,
+            )
+
+
 class BaseTokenUsageProcessor:
     @staticmethod
     def combine_usage_objects(usage_objects: list[Usage]) -> Usage:
@@ -2324,6 +2346,8 @@ class BaseTokenUsageProcessor:
                                 current_val + new_val,
                             )
 
+                _accumulate_cached_tokens_details(combined.prompt_tokens_details, usage.prompt_tokens_details)
+
             # Handle nested completion_tokens_details
             if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details:
                 if not hasattr(combined, "completion_tokens_details") or not combined.completion_tokens_details:

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -7,6 +7,8 @@ from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Any, Final, Literal, TypedDict, cast
 
+from typing_extensions import ReadOnly
+
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import (
@@ -15,6 +17,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import (
 )
 from litellm.types.utils import (
     CacheCreationTokenDetails,
+    CachedTokensDetails,
     CallTypes,
     CompletionTokensDetailsWrapper,
     DataResidency,
@@ -543,10 +546,38 @@ def calculate_cache_writing_cost(
     return total_cost
 
 
+def _allocate_uncached_modalities(
+    uncached_budget: int,
+    audio_tokens: int,
+    image_tokens: int,
+    video_tokens: int,
+    cached_details: "CachedTokensDetails | None",
+) -> tuple[int, int, int, int]:
+    """Split an uncached prompt-token budget across modalities. When the provider reports
+    which modalities the cache covered (cached_tokens_details), subtract the cache from each
+    modality directly; otherwise fall back to filling audio, then image, then video."""
+    cached_audio: Final = (cached_details.audio_tokens or 0) if cached_details is not None else 0
+    cached_image: Final = (cached_details.image_tokens or 0) if cached_details is not None else 0
+    billable_audio: Final = (
+        min(max(audio_tokens - cached_audio, 0), uncached_budget)
+        if cached_details is not None
+        else min(audio_tokens, uncached_budget)
+    )
+    billable_image: Final = (
+        min(max(image_tokens - cached_image, 0), uncached_budget - billable_audio)
+        if cached_details is not None
+        else min(image_tokens, uncached_budget - billable_audio)
+    )
+    billable_video: Final = min(video_tokens, uncached_budget - billable_audio - billable_image)
+    billable_text: Final = uncached_budget - billable_audio - billable_image - billable_video
+    return billable_audio, billable_image, billable_video, billable_text
+
+
 class PromptTokensDetailsResult(TypedDict):
     cache_hit_tokens: int
     cache_creation_tokens: int
     cache_creation_token_details: CacheCreationTokenDetails | None
+    cached_tokens_details: ReadOnly[CachedTokensDetails | None]
     text_tokens: int
     audio_tokens: int
     image_tokens: int
@@ -574,6 +605,10 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
         )
         or None
     )
+    raw_cached_tokens_details: Final = getattr(usage.prompt_tokens_details, "cached_tokens_details", None)
+    cached_tokens_details: Final = (
+        raw_cached_tokens_details if isinstance(raw_cached_tokens_details, CachedTokensDetails) else None
+    )
     text_tokens: Final = (
         cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None))
         or 0  # default to prompt tokens, if this field is not set
@@ -608,6 +643,7 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
         cache_hit_tokens=cache_hit_tokens,
         cache_creation_tokens=cache_creation_tokens,
         cache_creation_token_details=cache_creation_token_details,
+        cached_tokens_details=cached_tokens_details,
         text_tokens=text_tokens,
         audio_tokens=audio_tokens,
         image_tokens=image_tokens,
@@ -885,6 +921,7 @@ def generic_cost_per_token(
         cache_hit_tokens=0,
         cache_creation_tokens=0,
         cache_creation_token_details=None,
+        cached_tokens_details=None,
         text_tokens=usage.prompt_tokens,
         audio_tokens=0,
         image_tokens=0,
@@ -917,13 +954,17 @@ def generic_cost_per_token(
         # cached and per-modality counts are both subsets of prompt_tokens and may overlap, so a
         # modality can only bill what the cache did not already cover or the overlap is billed twice
         uncached_budget: Final = max(usage.prompt_tokens - cache_hit - cache_creation, 0)
-        billable_audio: Final = min(audio_tokens, uncached_budget)
-        billable_image: Final = min(image_tokens, uncached_budget - billable_audio)
-        billable_video: Final = min(video_tokens, uncached_budget - billable_audio - billable_image)
+        billable_audio, billable_image, billable_video, billable_text = _allocate_uncached_modalities(
+            uncached_budget=uncached_budget,
+            audio_tokens=audio_tokens,
+            image_tokens=image_tokens,
+            video_tokens=video_tokens,
+            cached_details=prompt_tokens_details["cached_tokens_details"],
+        )
         prompt_tokens_details["audio_tokens"] = billable_audio
         prompt_tokens_details["image_tokens"] = billable_image
         prompt_tokens_details["video_tokens"] = billable_video
-        prompt_tokens_details["text_tokens"] = uncached_budget - billable_audio - billable_image - billable_video
+        prompt_tokens_details["text_tokens"] = billable_text
     elif text_tokens == 0 and prompt_tokens_details["image_count"] == 0:
         # Clamp to zero: inconsistent streaming usage
         prompt_tokens_details["text_tokens"] = max(

litellm/responses/utils.py

@@ -1128,6 +1128,7 @@ class ResponseAPILoggingUtils:
                     text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None),
                     image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None),
                     cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None),
+                    cached_tokens_details=getattr(response_api_usage.input_tokens_details, "cached_tokens_details", None),
                 )
         completion_tokens_details: CompletionTokensDetailsWrapper | None = None
         output_tokens_details: Final[OutputTokensDetails | None] = getattr(

litellm/types/utils.py

@@ -1603,6 +1603,12 @@ class CacheCreationTokenDetails(BaseModel):
     ephemeral_1h_input_tokens: int | None = None
 
 
+class CachedTokensDetails(BaseModel):
+    text_tokens: int | None = None
+    audio_tokens: int | None = None
+    image_tokens: int | None = None
+
+
 class PromptTokensDetailsWrapper(
     SafeAttributeModel, PromptTokensDetails
 ):  # extends with image generation fields (text_tokens, image_tokens)
@@ -1645,6 +1651,9 @@ class PromptTokensDetailsWrapper(
     cache_creation_token_details: CacheCreationTokenDetails | None = None
     """Details of cache creation tokens sent to the model. Used for tracking 5m/1h cache creation tokens for Anthropic prompt caching."""
 
+    cached_tokens_details: CachedTokensDetails | None = None
+    """Per-modality breakdown of cached input tokens. OpenAI Realtime API naming (input_token_details.cached_tokens_details)."""
+
     def __setattr__(self, name: str, value: object) -> None:
         super().__setattr__(name, value)
         if name == "cache_write_tokens":

tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py

@@ -3946,3 +3946,70 @@ def test_route_image_generation_cost_falls_back_to_requested_size(monkeypatch, r
     )
 
     assert cost == expected_cost
+
+
+def test_generic_cost_per_token_realtime_uses_cached_modality_split(_local_model_cost_map):
+    """
+    Regression for LIT-6513: with a realtime GA cached_tokens_details split, uncached
+    tokens must be billed per the reported modalities. The previous audio-first
+    allocation billed uncached text tokens at the audio rate whenever the modality
+    details overlapped the cached count.
+    """
+    from litellm.types.utils import CachedTokensDetails
+
+    usage = Usage(
+        prompt_tokens=3400,
+        completion_tokens=1100,
+        total_tokens=4500,
+        prompt_tokens_details=PromptTokensDetailsWrapper(
+            cached_tokens=2000,
+            text_tokens=1400,
+            audio_tokens=2000,
+            cached_tokens_details=CachedTokensDetails(text_tokens=500, audio_tokens=1500),
+        ),
+    )
+
+    prompt_cost, _ = generic_cost_per_token(
+        model="gpt-realtime",
+        usage=usage,
+        custom_llm_provider="openai",
+    )
+
+    model_cost_map = litellm.model_cost["gpt-realtime"]
+    uncached_audio = 2000 - 1500
+    uncached_text = 3400 - 2000 - uncached_audio
+    expected_prompt_cost = (
+        uncached_text * model_cost_map["input_cost_per_token"]
+        + uncached_audio * model_cost_map["input_cost_per_audio_token"]
+        + 2000 * model_cost_map["cache_read_input_token_cost"]
+    )
+    assert round(prompt_cost, 10) == round(expected_prompt_cost, 10)
+
+
+def test_generic_cost_per_token_realtime_without_cached_split_clamps_to_uncached_budget(_local_model_cost_map):
+    """Without a cached_tokens_details split, overlapping modality details must still be
+    clamped so cached tokens are never also billed at the full modality rate."""
+    usage = Usage(
+        prompt_tokens=3400,
+        completion_tokens=1100,
+        total_tokens=4500,
+        prompt_tokens_details=PromptTokensDetailsWrapper(
+            cached_tokens=2000,
+            text_tokens=1400,
+            audio_tokens=2000,
+        ),
+    )
+
+    prompt_cost, _ = generic_cost_per_token(
+        model="gpt-realtime",
+        usage=usage,
+        custom_llm_provider="openai",
+    )
+
+    model_cost_map = litellm.model_cost["gpt-realtime"]
+    uncached_budget = 3400 - 2000
+    expected_prompt_cost = (
+        uncached_budget * model_cost_map["input_cost_per_audio_token"]
+        + 2000 * model_cost_map["cache_read_input_token_cost"]
+    )
+    assert round(prompt_cost, 10) == round(expected_prompt_cost, 10)

tests/test_litellm/test_cost_calculator.py

@@ -3952,6 +3952,44 @@ def test_combine_usage_objects_sums_mirrored_cache_write_fields_once():
     assert combined_pair.prompt_tokens_details.cache_creation_tokens == 100
 
 
+def test_combine_usage_objects_sums_cached_tokens_details():
+    """
+    Realtime GA usage reports a per-modality cached-token split in
+    input_token_details.cached_tokens_details. Combining per-response usage
+    across a session must sum that split, or the cost path loses which
+    modalities the cache covered and misallocates uncached tokens.
+    """
+    from litellm.types.utils import CachedTokensDetails
+
+    first = Usage(
+        prompt_tokens=1000,
+        completion_tokens=100,
+        total_tokens=1100,
+        prompt_tokens_details=PromptTokensDetailsWrapper(
+            cached_tokens=400,
+            cached_tokens_details=CachedTokensDetails(text_tokens=100, audio_tokens=300),
+        ),
+    )
+    second = Usage(
+        prompt_tokens=2000,
+        completion_tokens=200,
+        total_tokens=2200,
+        prompt_tokens_details=PromptTokensDetailsWrapper(
+            cached_tokens=900,
+            cached_tokens_details=CachedTokensDetails(text_tokens=200, audio_tokens=700),
+        ),
+    )
+
+    combined = BaseTokenUsageProcessor.combine_usage_objects([first, second])
+
+    assert combined.prompt_tokens_details is not None
+    assert combined.prompt_tokens_details.cached_tokens == 1300
+    cached_details = combined.prompt_tokens_details.cached_tokens_details
+    assert cached_details is not None
+    assert cached_details.text_tokens == 300
+    assert cached_details.audio_tokens == 1000
+
+
 def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(_local_model_cost_map):
     """Regression: an Anthropic /v1/messages response reports cache reads as top-level
     cache_read_input_tokens with input_tokens excluding them. Reading that usage as