scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-15 23:31:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 213

fix(cost): bill cached realtime audio tokens at the audio cache-read rate

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
shivam 2026-09-10 22:02:58 +00:00
parent 56b51db451
commit 302a8d43da
11 changed files with 314 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
litellm/cost_calculator.py
View file

@ -108,6 +108,7 @@ from litellm.types.llms.openai import (
)
from litellm.types.rerank import RerankBilledUnits, RerankResponse
from litellm.types.utils import (
CachedTokensDetails,
CallTypesLiteral,
LiteLLMRealtimeStreamLoggingObject,
LlmProviders,
@ -2310,6 +2311,60 @@ def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]
return [attr for attr in field_names if attr != "cache_creation_tokens"]
def _combine_cached_tokens_details(
current: CachedTokensDetails | None, new: CachedTokensDetails
) -> CachedTokensDetails:
def _sum_optional(current_value: int | None, new_value: int | None) -> int | None:
if current_value is None and new_value is None:
return None
return (current_value or 0) + (new_value or 0)
return CachedTokensDetails(
text_tokens=_sum_optional(
current.text_tokens if current is not None else None, new.text_tokens
),
audio_tokens=_sum_optional(
current.audio_tokens if current is not None else None, new.audio_tokens
),
image_tokens=_sum_optional(
current.image_tokens if current is not None else None, new.image_tokens
),
)
def _combine_prompt_tokens_details(combined: Usage, usage: Usage) -> None:
if not (hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details):
return
if not hasattr(combined, "prompt_tokens_details") or not combined.prompt_tokens_details:
combined.prompt_tokens_details = PromptTokensDetailsWrapper()
# Check what keys exist in the model's prompt_tokens_details
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
for attr in _summable_prompt_token_fields(usage.prompt_tokens_details):
if (
hasattr(usage.prompt_tokens_details, attr)
and not attr.startswith("_")
and not callable(_attribute_value(usage.prompt_tokens_details, attr))
):
current_val = getattr(combined.prompt_tokens_details, attr, 0) or 0
new_val = getattr(usage.prompt_tokens_details, attr, 0) or 0
if new_val is not None and isinstance(new_val, (int, float)):
setattr(
combined.prompt_tokens_details,
attr,
current_val + new_val,
)
new_cached_tokens_details: Final = getattr(
usage.prompt_tokens_details, "cached_tokens_details", None
)
if isinstance(new_cached_tokens_details, CachedTokensDetails):
combined.prompt_tokens_details.cached_tokens_details = _combine_cached_tokens_details(
getattr(combined.prompt_tokens_details, "cached_tokens_details", None),
new_cached_tokens_details,
)
class BaseTokenUsageProcessor:
@staticmethod
def combine_usage_objects(usage_objects: list[Usage]) -> Usage:
@ -2318,7 +2373,6 @@ class BaseTokenUsageProcessor:
"""
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
Usage,
)
@ -2337,27 +2391,7 @@ class BaseTokenUsageProcessor:
and isinstance(current_val, (int, float))
):
setattr(combined, attr, current_val + new_val)
# Handle nested prompt_tokens_details
if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details:
if not hasattr(combined, "prompt_tokens_details") or not combined.prompt_tokens_details:
combined.prompt_tokens_details = PromptTokensDetailsWrapper()
# Check what keys exist in the model's prompt_tokens_details
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
for attr in _summable_prompt_token_fields(usage.prompt_tokens_details):
if (
hasattr(usage.prompt_tokens_details, attr)
and not attr.startswith("_")
and not callable(_attribute_value(usage.prompt_tokens_details, attr))
):
current_val = getattr(combined.prompt_tokens_details, attr, 0) or 0
new_val = getattr(usage.prompt_tokens_details, attr, 0) or 0
if new_val is not None and isinstance(new_val, (int, float)):
setattr(
combined.prompt_tokens_details,
attr,
current_val + new_val,
)
_combine_prompt_tokens_details(combined, usage)
# Handle nested completion_tokens_details
if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details:

40
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -9,6 +9,8 @@ from types import MappingProxyType
from typing import Any, Final, Literal, TypedDict, cast
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from typing_extensions import ReadOnly
import litellm
from litellm._internal_context import current_billing_time
from litellm._logging import verbose_logger
@ -772,6 +774,7 @@ def calculate_cache_writing_cost(
class PromptTokensDetailsResult(TypedDict):
cache_hit_tokens: int
cache_hit_audio_tokens: ReadOnly[int]
cache_creation_tokens: int
cache_creation_token_details: CacheCreationTokenDetails | None
text_tokens: int
@ -802,12 +805,26 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
)
or None
)
text_tokens: Final = (
cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None))
or 0 # default to prompt tokens, if this field is not set
cached_tokens_details: Final = getattr(usage.prompt_tokens_details, "cached_tokens_details", None)
cached_text_tokens: Final = _get_token_detail_value(cached_tokens_details, "text_tokens") or 0
cached_audio_tokens: Final = _get_token_detail_value(cached_tokens_details, "audio_tokens") or 0
cached_image_tokens: Final = _get_token_detail_value(cached_tokens_details, "image_tokens") or 0
text_tokens: Final = max(
(
cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None))
or 0 # default to prompt tokens, if this field is not set
)
- cached_text_tokens,
0,
)
audio_tokens: Final = max(
(cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0) - cached_audio_tokens,
0,
)
image_tokens: Final = max(
(cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0) - cached_image_tokens,
0,
)
audio_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0
image_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0
video_tokens: Final = _coerce_token_count(getattr(usage.prompt_tokens_details, "video_tokens", 0))
character_count: Final = (
cast(
@ -835,6 +852,7 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
return PromptTokensDetailsResult(
cache_hit_tokens=cache_hit_tokens,
cache_hit_audio_tokens=min(cached_audio_tokens, cache_hit_tokens),
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
text_tokens=text_tokens,
@ -918,7 +936,16 @@ def _calculate_input_cost(
prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
### CACHE READ COST - Now uses tiered pricing
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
audio_cache_read_rate: Final = _get_cost_per_unit(
model_info,
_get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
None,
)
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost
prompt_cost += float(cache_hit_audio_tokens) * (
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
)
### AUDIO COST
if prompt_tokens_details["audio_tokens"]:
@ -1149,6 +1176,7 @@ def generic_cost_per_token(
### PROCESSING COST
prompt_tokens_details = PromptTokensDetailsResult(
cache_hit_tokens=0,
cache_hit_audio_tokens=0,
cache_creation_tokens=0,
cache_creation_token_details=None,
text_tokens=usage.prompt_tokens,

3
litellm/model_prices_and_context_window_backup.json
View file

@ -32502,6 +32502,7 @@
},
"gpt-realtime": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 3.2e-05,
@ -32535,6 +32536,7 @@
},
"gpt-realtime-1.5": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -32702,6 +32704,7 @@
},
"gpt-realtime-2025-08-28": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 3.2e-05,

45
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -45,6 +45,7 @@ from litellm.responses.litellm_completion_transformation.session_handler import
)
from litellm.types.llms.openai import (
AllMessageValues,
CachedTokensDetails,
ChatCompletionImageObject,
ChatCompletionImageUrlObject,
ChatCompletionRedactedThinkingBlock,
@ -2681,27 +2682,31 @@ class LiteLLMCompletionResponsesConfig:
# Translate prompt_tokens_details to input_tokens_details
if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details is not None:
prompt_details: Final = usage.prompt_tokens_details
input_details_dict: Final[dict[str, int]] = {}
if hasattr(prompt_details, "cached_tokens") and prompt_details.cached_tokens is not None:
input_details_dict["cached_tokens"] = prompt_details.cached_tokens
else:
input_details_dict["cached_tokens"] = 0
if hasattr(prompt_details, "text_tokens") and prompt_details.text_tokens is not None:
input_details_dict["text_tokens"] = prompt_details.text_tokens
if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None:
input_details_dict["audio_tokens"] = prompt_details.audio_tokens
cache_write_tokens = getattr(prompt_details, "cache_write_tokens", None) or getattr(
prompt_details, "cache_creation_tokens", None
cached_tokens_details: Final = getattr(prompt_details, "cached_tokens_details", None)
response_usage.input_tokens_details = InputTokensDetails(
cached_tokens=(
prompt_details.cached_tokens
if hasattr(prompt_details, "cached_tokens") and prompt_details.cached_tokens is not None
else 0
),
text_tokens=(
prompt_details.text_tokens
if hasattr(prompt_details, "text_tokens") and prompt_details.text_tokens is not None
else None
),
audio_tokens=(
prompt_details.audio_tokens
if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None
else None
),
cache_write_tokens=(
getattr(prompt_details, "cache_write_tokens", None)
or getattr(prompt_details, "cache_creation_tokens", None)
),
cached_tokens_details=(
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
),
)
if cache_write_tokens is not None:
input_details_dict["cache_write_tokens"] = cache_write_tokens
if input_details_dict:
response_usage.input_tokens_details = InputTokensDetails(**input_details_dict)
# Translate completion_tokens_details to output_tokens_details
if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details is not None:

3
litellm/responses/utils.py
View file

@ -1179,6 +1179,9 @@ class ResponseAPILoggingUtils:
audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None),
text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None),
image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None),
cached_tokens_details=getattr(
response_api_usage.input_tokens_details, "cached_tokens_details", None
),
cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None),
)
completion_tokens_details: CompletionTokensDetailsWrapper | None = None

14
litellm/types/llms/openai.py
View file

@ -1284,9 +1284,16 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject):
model_config = {"extra": "allow"}
class CachedTokensDetails(BaseModel):
text_tokens: int | None = None
audio_tokens: int | None = None
image_tokens: int | None = None
class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
audio_tokens: int | None = None
cached_tokens: int = 0
cached_tokens_details: CachedTokensDetails | None = None
text_tokens: int | None = None
model_config = {"extra": "allow"}
@ -2204,10 +2211,17 @@ class OpenAIRealtimeInputAudioTranscriptionCompleted(TypedDict):
transcript: ReadOnly[str]
class OpenAIRealtimeCachedTokensDetails(TypedDict, total=False):
text_tokens: ReadOnly[int]
audio_tokens: ReadOnly[int]
image_tokens: ReadOnly[int]
class OpenAIRealtimeUsageTokenDetails(TypedDict):
audio_tokens: ReadOnly[int]
text_tokens: ReadOnly[int]
cached_tokens: NotRequired[ReadOnly[int]]
cached_tokens_details: NotRequired[ReadOnly[OpenAIRealtimeCachedTokensDetails]]
class OpenAIRealtimeResponseUsage(TypedDict):

6
litellm/types/utils.py
View file

@ -58,6 +58,7 @@ from .llms.base import HiddenParams
from .llms.openai import (
AllMessageValues,
Batch,
CachedTokensDetails,
ChatCompletionAnnotation,
ChatCompletionReasoningItem,
ChatCompletionRedactedThinkingBlock,
@ -1707,6 +1708,9 @@ class PromptTokensDetailsWrapper(
cache_creation_token_details: CacheCreationTokenDetails | None = None
"""Details of cache creation tokens sent to the model. Used for tracking 5m/1h cache creation tokens for Anthropic prompt caching."""
cached_tokens_details: CachedTokensDetails | None = None
"""Details of cached (cache-hit) tokens sent to the model. OpenAI realtime naming; carries the per-modality cache-read split."""
def __setattr__(self, name: str, value: object) -> None:
super().__setattr__(name, value)
if name == "cache_write_tokens":
@ -1753,6 +1757,8 @@ class PromptTokensDetailsWrapper(
del self.cache_creation_tokens
if self.cache_creation_token_details is None:
del self.cache_creation_token_details
if self.cached_tokens_details is None:
del self.cached_tokens_details
class ServerToolUse(BaseModel):

3
model_prices_and_context_window.json
View file

@ -32502,6 +32502,7 @@
},
"gpt-realtime": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 3.2e-05,
@ -32535,6 +32536,7 @@
},
"gpt-realtime-1.5": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -32702,6 +32704,7 @@
},
"gpt-realtime-2025-08-28": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 3.2e-05,

68
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -2648,6 +2648,7 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
prompt_tokens_details: PromptTokensDetailsResult = {
"cache_hit_tokens": 0,
"cache_hit_audio_tokens": 0,
"cache_creation_tokens": 0,
"cache_creation_token_details": CacheCreationTokenDetails(
ephemeral_5m_input_tokens=100,
@ -5147,3 +5148,70 @@ def test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output(
assert completion_cost == pytest.approx(
30 * info["output_cost_per_token"] + 70 * info["output_cost_per_audio_token"]
)
def test_cached_realtime_audio_tokens_billed_at_audio_cache_read_rate(
_local_model_cost_map: None,
) -> None:
usage = Usage(
prompt_tokens=283,
completion_tokens=0,
total_tokens=283,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=116,
audio_tokens=167,
cached_tokens=192,
cached_tokens_details={"text_tokens": 64, "audio_tokens": 128},
),
)
prompt_cost, _ = generic_cost_per_token(
model="gpt-realtime-2", usage=usage, custom_llm_provider="openai"
)
assert prompt_cost == pytest.approx(0.0015328)
def test_prompt_tokens_details_without_cached_tokens_details_unchanged(
_local_model_cost_map: None,
) -> None:
usage = Usage(
prompt_tokens=283,
completion_tokens=0,
total_tokens=283,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=116, audio_tokens=167, cached_tokens=192
),
)
prompt_cost, _ = generic_cost_per_token(
model="gpt-realtime-2", usage=usage, custom_llm_provider="openai"
)
assert prompt_cost == pytest.approx(0.0029888)
def test_cached_audio_tokens_fall_back_to_cache_read_input_token_cost() -> None:
model_info: ModelInfo = {
"input_cost_per_token": 4e-6,
"input_cost_per_audio_token": 32e-6,
"cache_read_input_token_cost": 5e-7,
}
usage = Usage(
prompt_tokens=283,
completion_tokens=0,
total_tokens=283,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=116,
audio_tokens=167,
cached_tokens=192,
cached_tokens_details={"text_tokens": 64, "audio_tokens": 128},
),
)
prompt_cost, _ = generic_cost_per_token(
model="some-realtime-model",
usage=usage,
custom_llm_provider="openai",
model_info=model_info,
)
expected = 52 * 4e-6 + 64 * 5e-7 + 39 * 32e-6 + 128 * 5e-7
assert prompt_cost == pytest.approx(expected)

41
tests/test_litellm/responses/test_responses_utils.py
View file

@ -577,6 +577,47 @@ class TestResponseAPILoggingUtils:
assert result.completion_tokens_details is not None
assert result.completion_tokens_details.reasoning_tokens == 4
def test_transform_realtime_usage_dict_keeps_cached_tokens_details(self):
usage = {
"input_tokens": 283,
"output_tokens": 0,
"total_tokens": 283,
"input_token_details": {
"text_tokens": 116,
"audio_tokens": 167,
"cached_tokens": 192,
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
},
}
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
assert result.prompt_tokens_details is not None
assert result.prompt_tokens_details.cached_tokens == 192
assert result.prompt_tokens_details.cached_tokens_details is not None
assert result.prompt_tokens_details.cached_tokens_details.audio_tokens == 128
assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 64
def test_transform_response_api_usage_object_keeps_cached_tokens_details(self):
usage = ResponseAPIUsage(
input_tokens=283,
output_tokens=0,
total_tokens=283,
input_tokens_details={
"text_tokens": 116,
"audio_tokens": 167,
"cached_tokens": 192,
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
},
)
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
assert result.prompt_tokens_details is not None
assert result.prompt_tokens_details.cached_tokens_details is not None
assert result.prompt_tokens_details.cached_tokens_details.audio_tokens == 128
assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 64
class TestResponsesAPIProviderSpecificParams:
"""

61
tests/test_litellm/test_cost_calculator.py
View file

@ -4768,3 +4768,64 @@ def test_collect_and_combine_realtime_usage_stores_partitioned_text_tokens() ->
assert combined.completion_tokens_details.reasoning_tokens == 95
assert combined.completion_tokens_details.text_tokens == 38
assert combined.completion_tokens_details.audio_tokens == 0
def test_realtime_combine_sums_nested_cached_tokens_details():
results: OpenAIRealtimeStreamList = [
{
"type": "response.done",
"response": {
"usage": {
"input_tokens": 283,
"output_tokens": 0,
"total_tokens": 283,
"input_token_details": {
"text_tokens": 116,
"audio_tokens": 167,
"cached_tokens": 192,
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
},
}
},
},
{
"type": "response.done",
"response": {
"usage": {
"input_tokens": 150,
"output_tokens": 0,
"total_tokens": 150,
"input_token_details": {
"text_tokens": 50,
"audio_tokens": 100,
"cached_tokens": 100,
"cached_tokens_details": {"audio_tokens": 100},
},
}
},
},
]
combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(
results=results,
)
assert combined.prompt_tokens_details is not None
assert combined.prompt_tokens_details.cached_tokens == 292
assert combined.prompt_tokens_details.cached_tokens_details is not None
assert combined.prompt_tokens_details.cached_tokens_details.audio_tokens == 228
assert combined.prompt_tokens_details.cached_tokens_details.text_tokens == 64
assert combined.prompt_tokens_details.cached_tokens_details.image_tokens is None
def test_usage_without_cached_tokens_details_omits_key():
usage = Usage(
prompt_tokens=10,
completion_tokens=5,
total_tokens=15,
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=10),
)
dumped = usage.prompt_tokens_details.model_dump()
assert "cached_tokens_details" not in dumped
assert "cached_tokens_details" not in usage.prompt_tokens_details.model_dump_json()