scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-04 02:31:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 69

Merge remote-tracking branch 'origin/main' into litellm_lit6314_guardrail_metadata_transfer

Browse source
This commit is contained in:
yucheng 2026-09-14 18:01:01 +00:00
commit 4be84a859d
16 changed files with 486 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

65
litellm/cost_calculator.py
View file

@ -97,6 +97,7 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro
from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.openai import (
HttpxBinaryResponseContent,
ImageGenerationRequestQuality,
@ -2381,6 +2382,46 @@ def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]
return [attr for attr in field_names if attr != "cache_creation_tokens"]
def _combine_cached_tokens_details(
current: CachedTokensDetails | None, new: CachedTokensDetails
) -> CachedTokensDetails:
def _sum_optional(current_value: int | None, new_value: int | None) -> int | None:
if current_value is None and new_value is None:
return None
return (current_value or 0) + (new_value or 0)
return CachedTokensDetails(
text_tokens=_sum_optional(current.text_tokens if current is not None else None, new.text_tokens),
audio_tokens=_sum_optional(current.audio_tokens if current is not None else None, new.audio_tokens),
image_tokens=_sum_optional(current.image_tokens if current is not None else None, new.image_tokens),
)
def _combine_prompt_tokens_details(
current: PromptTokensDetailsWrapper | None, new: PromptTokensDetailsWrapper
) -> PromptTokensDetailsWrapper:
base: Final = current if current is not None else PromptTokensDetailsWrapper()
base_values: Final = MappingProxyType(
{attr: getattr(base, attr) for attr in type(base).model_fields if hasattr(base, attr)}
)
summed: Final = MappingProxyType(
{
attr: (getattr(base, attr, 0) or 0) + (getattr(new, attr) or 0)
for attr in _summable_prompt_token_fields(new)
if hasattr(new, attr) and isinstance(getattr(new, attr) or 0, (int, float))
}
)
new_cached_tokens_details: Final = getattr(new, "cached_tokens_details", None)
cached_tokens_details: Final = (
_combine_cached_tokens_details(getattr(base, "cached_tokens_details", None), new_cached_tokens_details)
if isinstance(new_cached_tokens_details, CachedTokensDetails)
else getattr(base, "cached_tokens_details", None)
)
return PromptTokensDetailsWrapper(
**MappingProxyType({**base_values, **summed, "cached_tokens_details": cached_tokens_details})
)
class BaseTokenUsageProcessor:
@staticmethod
def combine_usage_objects(usage_objects: list[Usage]) -> Usage:
@ -2389,7 +2430,6 @@ class BaseTokenUsageProcessor:
"""
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
Usage,
)
@ -2408,27 +2448,10 @@ class BaseTokenUsageProcessor:
and isinstance(current_val, (int, float))
):
setattr(combined, attr, current_val + new_val)
# Handle nested prompt_tokens_details
if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details:
if not hasattr(combined, "prompt_tokens_details") or not combined.prompt_tokens_details:
combined.prompt_tokens_details = PromptTokensDetailsWrapper()
# Check what keys exist in the model's prompt_tokens_details
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
for attr in _summable_prompt_token_fields(usage.prompt_tokens_details):
if (
hasattr(usage.prompt_tokens_details, attr)
and not attr.startswith("_")
and not callable(_attribute_value(usage.prompt_tokens_details, attr))
):
current_val = getattr(combined.prompt_tokens_details, attr, 0) or 0
new_val = getattr(usage.prompt_tokens_details, attr, 0) or 0
if new_val is not None and isinstance(new_val, (int, float)):
setattr(
combined.prompt_tokens_details,
attr,
current_val + new_val,
)
combined.prompt_tokens_details = _combine_prompt_tokens_details(
getattr(combined, "prompt_tokens_details", None), usage.prompt_tokens_details
)
# Handle nested completion_tokens_details
if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details:

80
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -9,6 +9,8 @@ from types import MappingProxyType
from typing import Any, Final, Literal, TypedDict, cast
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from typing_extensions import ReadOnly
import litellm
from litellm._internal_context import current_billing_time
from litellm._logging import verbose_logger
@ -772,6 +774,7 @@ def calculate_cache_writing_cost(
class PromptTokensDetailsResult(TypedDict):
cache_hit_tokens: int
cache_hit_audio_tokens: ReadOnly[int]
cache_creation_tokens: int
cache_creation_token_details: CacheCreationTokenDetails | None
text_tokens: int
@ -802,12 +805,34 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
)
or None
)
text_tokens: Final = (
cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None))
or 0 # default to prompt tokens, if this field is not set
cached_tokens_details: Final = getattr(usage.prompt_tokens_details, "cached_tokens_details", None)
cached_audio_tokens: Final = min(
_get_token_detail_value(cached_tokens_details, "audio_tokens") or 0, cache_hit_tokens
)
cached_text_tokens: Final = min(
_get_token_detail_value(cached_tokens_details, "text_tokens") or 0,
cache_hit_tokens - cached_audio_tokens,
)
cached_image_tokens: Final = min(
_get_token_detail_value(cached_tokens_details, "image_tokens") or 0,
cache_hit_tokens - cached_audio_tokens - cached_text_tokens,
)
text_tokens: Final = max(
(
cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None))
or 0 # default to prompt tokens, if this field is not set
)
- cached_text_tokens,
0,
)
audio_tokens: Final = max(
(cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0) - cached_audio_tokens,
0,
)
image_tokens: Final = max(
(cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0) - cached_image_tokens,
0,
)
audio_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0
image_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0
video_tokens: Final = _coerce_token_count(getattr(usage.prompt_tokens_details, "video_tokens", 0))
character_count: Final = (
cast(
@ -835,6 +860,7 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
return PromptTokensDetailsResult(
cache_hit_tokens=cache_hit_tokens,
cache_hit_audio_tokens=cached_audio_tokens,
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
text_tokens=text_tokens,
@ -918,7 +944,16 @@ def _calculate_input_cost(
prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
### CACHE READ COST - Now uses tiered pricing
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
audio_cache_read_rate: Final = _get_cost_per_unit(
model_info,
_get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
None,
)
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost
prompt_cost += float(cache_hit_audio_tokens) * (
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
)
### AUDIO COST
if prompt_tokens_details["audio_tokens"]:
@ -1149,6 +1184,7 @@ def generic_cost_per_token(
### PROCESSING COST
prompt_tokens_details = PromptTokensDetailsResult(
cache_hit_tokens=0,
cache_hit_audio_tokens=0,
cache_creation_tokens=0,
cache_creation_token_details=None,
text_tokens=usage.prompt_tokens,
@ -1319,6 +1355,7 @@ class BilledTokenRates:
input_cost_per_token: float
output_cost_per_token: float
cache_read_input_token_cost: float
cache_read_input_audio_token_cost: float
cache_creation_input_token_cost: float
cache_creation_input_token_cost_above_1hr: float
output_cost_per_reasoning_token: float
@ -1330,6 +1367,7 @@ class BilledTokenRates:
input_cost_per_token=self.input_cost_per_token * multiplier,
output_cost_per_token=self.output_cost_per_token * multiplier,
cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier,
cache_read_input_audio_token_cost=self.cache_read_input_audio_token_cost * multiplier,
cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier,
cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier,
output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
@ -1353,15 +1391,16 @@ def _reasoning_token_count(usage: Usage) -> int:
return parsed or _coerce_token_count(getattr(usage, "reasoning_tokens", 0))
def _cache_token_counts(usage: Usage) -> tuple[int, int, CacheCreationTokenDetails | None]:
"""(cache read tokens, cache creation tokens, cache creation details): read from prompt_tokens_details
first, then the private top-level counters the Usage constructor mirrors cache tokens onto for
providers/callers that bypass the details."""
def _cache_token_counts(usage: Usage) -> tuple[int, int, int, CacheCreationTokenDetails | None]:
"""(cache read tokens, cached audio tokens, cache creation tokens, cache creation details): read from
prompt_tokens_details first, then the private top-level counters the Usage constructor mirrors cache
tokens onto for providers/callers that bypass the details."""
parsed: Final = parse_prompt_tokens_details(usage) if usage.prompt_tokens_details is not None else None
parsed_read: Final = parsed["cache_hit_tokens"] if parsed is not None else 0
parsed_creation: Final = parsed["cache_creation_tokens"] if parsed is not None else 0
return (
parsed_read or _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0)),
parsed["cache_hit_audio_tokens"] if parsed is not None else 0,
parsed_creation or _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0)),
parsed["cache_creation_token_details"] if parsed is not None else None,
)
@ -1372,11 +1411,13 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat
cache rates (else the input rate) and reasoning at the output rate, as _cost_per_token_custom_pricing_helper does."""
input_rate: Final = custom_cost_per_token["input_cost_per_token"]
output_rate: Final = custom_cost_per_token["output_cost_per_token"]
cache_read_rate: Final = custom_cost_per_token.get("cache_read_input_token_cost", input_rate)
cache_creation_rate: Final = custom_cost_per_token.get("cache_creation_input_token_cost", input_rate)
return BilledTokenRates(
input_cost_per_token=input_rate,
output_cost_per_token=output_rate,
cache_read_input_token_cost=custom_cost_per_token.get("cache_read_input_token_cost", input_rate),
cache_read_input_token_cost=cache_read_rate,
cache_read_input_audio_token_cost=cache_read_rate,
cache_creation_input_token_cost=cache_creation_rate,
cache_creation_input_token_cost_above_1hr=cache_creation_rate,
output_cost_per_reasoning_token=output_rate,
@ -1413,6 +1454,11 @@ def _cost_map_billed_rates(
completion_base_cost=completion_base_cost,
current_time=billing_time,
)
audio_cache_read_rate: Final = _get_cost_per_unit(
model_info,
_get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
None,
)
multiplier: Final = (
_get_regional_uplift_multiplier(model_info, data_residency)
* get_vertex_regional_endpoint_uplift(model_info, vertex_location)
@ -1422,6 +1468,9 @@ def _cost_map_billed_rates(
input_cost_per_token=prompt_base_cost,
output_cost_per_token=completion_base_cost,
cache_read_input_token_cost=cache_read_cost_rate,
cache_read_input_audio_token_cost=(
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost_rate
),
cache_creation_input_token_cost=cache_creation_cost_rate,
cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate,
output_cost_per_reasoning_token=reasoning_rate,
@ -1494,7 +1543,9 @@ def get_token_type_cost_breakdown(
if rates is None:
return TokenTypeCostBreakdown(0.0, 0.0, 0.0)
cache_read_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(usage)
cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(
usage
)
cache_creation_cost: Final = (
float(cache_creation_tokens) * rates.cache_creation_input_token_cost
if custom_cost_per_token is not None
@ -1507,7 +1558,10 @@ def get_token_type_cost_breakdown(
)
return TokenTypeCostBreakdown(
reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token,
cache_read_cost=float(cache_read_tokens) * rates.cache_read_input_token_cost,
cache_read_cost=(
float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost
+ float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost
),
cache_creation_cost=cache_creation_cost,
rates=rates,
)

12
litellm/model_prices_and_context_window_backup.json
View file

@ -5513,7 +5513,8 @@
},
"azure/gpt-realtime-2025-08-28": {
"cache_creation_input_audio_token_cost": 4e-06,
"cache_read_input_token_cost": 4e-06,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-03-02",
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -5546,7 +5547,8 @@
},
"azure/gpt-realtime-1.5-2026-02-23": {
"cache_creation_input_audio_token_cost": 4e-06,
"cache_read_input_token_cost": 4e-06,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-08-24",
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -5683,6 +5685,7 @@
},
"azure/gpt-realtime-mini": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"input_cost_per_audio_token": 1e-05,
"input_cost_per_image_token": 8e-07,
@ -5715,6 +5718,7 @@
},
"azure/gpt-realtime-mini-2025-10-06": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"input_cost_per_audio_token": 1e-05,
"input_cost_per_image_token": 8e-07,
@ -32678,6 +32682,7 @@
},
"gpt-realtime": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 3.2e-05,
@ -32711,6 +32716,7 @@
},
"gpt-realtime-1.5": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -32847,6 +32853,7 @@
"gpt-realtime-mini": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 1e-05,
"input_cost_per_token": 6e-07,
@ -32878,6 +32885,7 @@
},
"gpt-realtime-2025-08-28": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 3.2e-05,

36
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -43,6 +43,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.responses.litellm_completion_transformation.session_handler import (
ResponsesSessionHandler,
)
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
@ -2816,27 +2817,24 @@ class LiteLLMCompletionResponsesConfig:
# Translate prompt_tokens_details to input_tokens_details
if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details is not None:
prompt_details: Final = usage.prompt_tokens_details
input_details_dict: Final[dict[str, int]] = {}
if hasattr(prompt_details, "cached_tokens") and prompt_details.cached_tokens is not None:
input_details_dict["cached_tokens"] = prompt_details.cached_tokens
else:
input_details_dict["cached_tokens"] = 0
if hasattr(prompt_details, "text_tokens") and prompt_details.text_tokens is not None:
input_details_dict["text_tokens"] = prompt_details.text_tokens
if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None:
input_details_dict["audio_tokens"] = prompt_details.audio_tokens
cache_write_tokens = getattr(prompt_details, "cache_write_tokens", None) or getattr(
cached_tokens_details: Final = getattr(prompt_details, "cached_tokens_details", None)
cache_write_tokens: Final = getattr(prompt_details, "cache_write_tokens", None) or getattr(
prompt_details, "cache_creation_tokens", None
)
if cache_write_tokens is not None:
input_details_dict["cache_write_tokens"] = cache_write_tokens
if input_details_dict:
response_usage.input_tokens_details = InputTokensDetails(**input_details_dict)
cache_write_extra: Final[Mapping[str, int]] = (
MappingProxyType({"cache_write_tokens": cache_write_tokens})
if cache_write_tokens is not None
else MappingProxyType({})
)
response_usage.input_tokens_details = InputTokensDetails(
cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
text_tokens=prompt_details.text_tokens,
audio_tokens=prompt_details.audio_tokens,
cached_tokens_details=(
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
),
**cache_write_extra,
)
# Translate completion_tokens_details to output_tokens_details
if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details is not None:

3
litellm/responses/utils.py
View file

@ -1179,6 +1179,9 @@ class ResponseAPILoggingUtils:
audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None),
text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None),
image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None),
cached_tokens_details=getattr(
response_api_usage.input_tokens_details, "cached_tokens_details", None
),
cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None),
)
completion_tokens_details: CompletionTokensDetailsWrapper | None = None

6
litellm/types/llms/base.py
View file

@ -75,3 +75,9 @@ class HiddenParams(OpenAIObject):
data: Final = super().model_dump(**kwargs)
data["_response_ms"] = self._response_ms
return data
class CachedTokensDetails(BaseModel):
text_tokens: int | None = None
audio_tokens: int | None = None
image_tokens: int | None = None

10
litellm/types/llms/openai.py
View file

@ -91,6 +91,8 @@ from litellm.types.responses.main import (
OutputImageGenerationCall,
)
from .base import CachedTokensDetails
FileContent = IO[bytes] | bytes | PathLike
FileTypes = (
@ -1288,6 +1290,7 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject):
class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
audio_tokens: int | None = None
cached_tokens: int = 0
cached_tokens_details: CachedTokensDetails | None = None
text_tokens: int | None = None
model_config = {"extra": "allow"}
@ -2254,10 +2257,17 @@ class OpenAIRealtimeInputAudioTranscriptionCompleted(TypedDict):
usage: NotRequired[ReadOnly[Mapping[str, object]]]
class OpenAIRealtimeCachedTokensDetails(TypedDict, total=False):
text_tokens: ReadOnly[int]
audio_tokens: ReadOnly[int]
image_tokens: ReadOnly[int]
class OpenAIRealtimeUsageTokenDetails(TypedDict):
audio_tokens: ReadOnly[int]
text_tokens: ReadOnly[int]
cached_tokens: NotRequired[ReadOnly[int]]
cached_tokens_details: NotRequired[ReadOnly[OpenAIRealtimeCachedTokensDetails]]
class OpenAIRealtimeResponseUsage(TypedDict):

7
litellm/types/utils.py
View file

@ -48,6 +48,7 @@ from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.types.llms.base import (
BaseLiteLLMOpenAIResponseObject,
CachedTokensDetails,
LiteLLMPydanticObjectBase,
)
from litellm.types.mcp import MCPServerCostInfo
@ -252,6 +253,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing
cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
cache_read_input_token_cost: float | None
cache_read_input_audio_token_cost: ReadOnly[float | None]
cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing
cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing
cache_read_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
@ -1710,6 +1712,9 @@ class PromptTokensDetailsWrapper(
cache_creation_token_details: CacheCreationTokenDetails | None = None
"""Details of cache creation tokens sent to the model. Used for tracking 5m/1h cache creation tokens for Anthropic prompt caching."""
cached_tokens_details: CachedTokensDetails | None = None
"""Details of cached (cache-hit) tokens sent to the model. OpenAI realtime naming; carries the per-modality cache-read split."""
def __setattr__(self, name: str, value: object) -> None:
super().__setattr__(name, value)
if name == "cache_write_tokens":
@ -1756,6 +1761,8 @@ class PromptTokensDetailsWrapper(
del self.cache_creation_tokens
if self.cache_creation_token_details is None:
del self.cache_creation_token_details
if self.cached_tokens_details is None:
del self.cached_tokens_details
class ServerToolUse(BaseModel):

1
litellm/utils.py
View file

@ -5882,6 +5882,7 @@ def _get_model_info_helper(
"cache_creation_input_token_cost_ultrafast", None
),
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None),
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
"cache_read_input_token_cost_above_200k_tokens", None

12
model_prices_and_context_window.json
View file

@ -5513,7 +5513,8 @@
},
"azure/gpt-realtime-2025-08-28": {
"cache_creation_input_audio_token_cost": 4e-06,
"cache_read_input_token_cost": 4e-06,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-03-02",
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -5546,7 +5547,8 @@
},
"azure/gpt-realtime-1.5-2026-02-23": {
"cache_creation_input_audio_token_cost": 4e-06,
"cache_read_input_token_cost": 4e-06,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-08-24",
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -5683,6 +5685,7 @@
},
"azure/gpt-realtime-mini": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"input_cost_per_audio_token": 1e-05,
"input_cost_per_image_token": 8e-07,
@ -5715,6 +5718,7 @@
},
"azure/gpt-realtime-mini-2025-10-06": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"input_cost_per_audio_token": 1e-05,
"input_cost_per_image_token": 8e-07,
@ -32678,6 +32682,7 @@
},
"gpt-realtime": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 3.2e-05,
@ -32711,6 +32716,7 @@
},
"gpt-realtime-1.5": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -32847,6 +32853,7 @@
"gpt-realtime-mini": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 1e-05,
"input_cost_per_token": 6e-07,
@ -32878,6 +32885,7 @@
},
"gpt-realtime-2025-08-28": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 3.2e-05,

156
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -2648,6 +2648,7 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
prompt_tokens_details: PromptTokensDetailsResult = {
"cache_hit_tokens": 0,
"cache_hit_audio_tokens": 0,
"cache_creation_tokens": 0,
"cache_creation_token_details": CacheCreationTokenDetails(
ephemeral_5m_input_tokens=100,
@ -4005,6 +4006,7 @@ def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeyp
input_cost_per_token=6e-6,
output_cost_per_token=3e-5,
cache_read_input_token_cost=6e-7,
cache_read_input_audio_token_cost=6e-7,
cache_creation_input_token_cost=7.5e-6,
cache_creation_input_token_cost_above_1hr=0.0,
output_cost_per_reasoning_token=3e-5,
@ -5147,3 +5149,157 @@ def test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output(
assert completion_cost == pytest.approx(
30 * info["output_cost_per_token"] + 70 * info["output_cost_per_audio_token"]
)
def test_cached_realtime_audio_tokens_billed_at_audio_cache_read_rate(
_local_model_cost_map: None,
) -> None:
usage = Usage(
prompt_tokens=283,
completion_tokens=0,
total_tokens=283,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=116,
audio_tokens=167,
cached_tokens=192,
cached_tokens_details={"text_tokens": 64, "audio_tokens": 128},
),
)
prompt_cost, _ = generic_cost_per_token(
model="gpt-realtime-2", usage=usage, custom_llm_provider="openai"
)
assert prompt_cost == pytest.approx(0.0015328)
def test_prompt_tokens_details_without_cached_tokens_details_unchanged(
_local_model_cost_map: None,
) -> None:
usage = Usage(
prompt_tokens=283,
completion_tokens=0,
total_tokens=283,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=116, audio_tokens=167, cached_tokens=192
),
)
prompt_cost, _ = generic_cost_per_token(
model="gpt-realtime-2", usage=usage, custom_llm_provider="openai"
)
assert prompt_cost == pytest.approx(0.0029888)
def test_cached_audio_tokens_fall_back_to_cache_read_input_token_cost() -> None:
model_info: ModelInfo = {
"input_cost_per_token": 4e-6,
"input_cost_per_audio_token": 32e-6,
"cache_read_input_token_cost": 5e-7,
}
usage = Usage(
prompt_tokens=283,
completion_tokens=0,
total_tokens=283,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=116,
audio_tokens=167,
cached_tokens=192,
cached_tokens_details={"text_tokens": 64, "audio_tokens": 128},
),
)
prompt_cost, _ = generic_cost_per_token(
model="some-realtime-model",
usage=usage,
custom_llm_provider="openai",
model_info=model_info,
)
expected = 52 * 4e-6 + 64 * 5e-7 + 39 * 32e-6 + 128 * 5e-7
assert prompt_cost == pytest.approx(expected)
def test_cached_audio_tokens_capped_at_cached_tokens(_local_model_cost_map: None) -> None:
"""Nested cached_tokens_details exceeding cached_tokens must not over-subtract the audio bucket."""
usage = Usage(
prompt_tokens=283,
completion_tokens=0,
total_tokens=283,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=116,
audio_tokens=167,
cached_tokens=100,
cached_tokens_details={"audio_tokens": 128},
),
)
prompt_cost, _ = generic_cost_per_token(
model="gpt-realtime-2", usage=usage, custom_llm_provider="openai"
)
assert prompt_cost == pytest.approx(116 * 4e-6 + (167 - 100) * 32e-6 + 100 * 4e-7)
def test_cached_audio_tokens_billed_at_audio_cache_rate_through_model_info_lookup(_local_model_cost_map: None) -> None:
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
total_tokens=1000,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=400,
audio_tokens=600,
cached_tokens=500,
cached_tokens_details={"text_tokens": 100, "audio_tokens": 400},
),
)
prompt_cost, _ = generic_cost_per_token(model="gpt-realtime-2.1-mini", usage=usage, custom_llm_provider="openai")
assert prompt_cost == pytest.approx(300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7)
def test_cache_read_breakdown_splits_cached_audio_at_the_audio_cache_rate(_local_model_cost_map: None) -> None:
usage = Usage(
prompt_tokens=4863,
completion_tokens=1087,
total_tokens=5950,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=1693,
audio_tokens=3170,
cached_tokens=2816,
cached_tokens_details={"text_tokens": 896, "audio_tokens": 1920},
),
)
breakdown = get_token_type_cost_breakdown(model="gpt-realtime-2.1-mini", custom_llm_provider="openai", usage=usage)
prompt_cost, _ = generic_cost_per_token(model="gpt-realtime-2.1-mini", usage=usage, custom_llm_provider="openai")
assert breakdown.cache_read_cost == pytest.approx(896 * 6e-8 + 1920 * 3e-7)
assert breakdown.rates is not None
assert breakdown.rates.cache_read_input_audio_token_cost == pytest.approx(3e-7)
assert prompt_cost == pytest.approx((1693 - 896) * 6e-7 + (3170 - 1920) * 1e-5 + breakdown.cache_read_cost)
@pytest.mark.parametrize(
("model", "custom_llm_provider", "expected_prompt_cost"),
(
pytest.param("azure/gpt-realtime-2025-08-28", "azure", 300 * 4e-6 + 100 * 4e-7 + 200 * 3.2e-5 + 400 * 4e-7, id="azure-gpt-realtime"),
pytest.param("azure/gpt-realtime-1.5-2026-02-23", "azure", 300 * 4e-6 + 100 * 4e-7 + 200 * 3.2e-5 + 400 * 4e-7, id="azure-gpt-realtime-1.5"),
pytest.param("azure/gpt-realtime-mini", "azure", 300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7, id="azure-gpt-realtime-mini"),
pytest.param("gpt-realtime-mini", "openai", 300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7, id="openai-gpt-realtime-mini"),
),
)
def test_realtime_models_bill_cached_text_and_audio_at_their_cache_read_rates(
_local_model_cost_map: None, model: str, custom_llm_provider: str, expected_prompt_cost: float
) -> None:
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
total_tokens=1000,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=400,
audio_tokens=600,
cached_tokens=500,
cached_tokens_details={"text_tokens": 100, "audio_tokens": 400},
),
)
prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=custom_llm_provider)
assert prompt_cost == pytest.approx(expected_prompt_cost)

2
tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -2789,6 +2789,7 @@ class TestUsageTransformation:
assert response_usage.input_tokens_details is not None
assert response_usage.input_tokens_details.cached_tokens == 5
assert response_usage.input_tokens_details.text_tokens == 8
assert "cache_write_tokens" not in response_usage.input_tokens_details.model_dump()
def test_transform_usage_with_cached_tokens_gemini(self):
"""Test that cached_tokens from Gemini are properly transformed to input_tokens_details"""
@ -2851,6 +2852,7 @@ class TestUsageTransformation:
assert response_usage.input_tokens_details is not None
assert response_usage.input_tokens_details.cached_tokens == 100
assert getattr(response_usage.input_tokens_details, "cache_write_tokens", None) == 800
assert response_usage.input_tokens_details.model_dump()["cache_write_tokens"] == 800
def test_transform_usage_with_reasoning_tokens_gemini(self):
"""Test that reasoning_tokens from Gemini are properly transformed to output_tokens_details"""

41
tests/test_litellm/responses/test_responses_utils.py
View file

@ -577,6 +577,47 @@ class TestResponseAPILoggingUtils:
assert result.completion_tokens_details is not None
assert result.completion_tokens_details.reasoning_tokens == 4
def test_transform_realtime_usage_dict_keeps_cached_tokens_details(self):
usage = {
"input_tokens": 283,
"output_tokens": 0,
"total_tokens": 283,
"input_token_details": {
"text_tokens": 116,
"audio_tokens": 167,
"cached_tokens": 192,
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
},
}
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
assert result.prompt_tokens_details is not None
assert result.prompt_tokens_details.cached_tokens == 192
assert result.prompt_tokens_details.cached_tokens_details is not None
assert result.prompt_tokens_details.cached_tokens_details.audio_tokens == 128
assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 64
def test_transform_response_api_usage_object_keeps_cached_tokens_details(self):
usage = ResponseAPIUsage(
input_tokens=283,
output_tokens=0,
total_tokens=283,
input_tokens_details={
"text_tokens": 116,
"audio_tokens": 167,
"cached_tokens": 192,
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
},
)
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
assert result.prompt_tokens_details is not None
assert result.prompt_tokens_details.cached_tokens_details is not None
assert result.prompt_tokens_details.cached_tokens_details.audio_tokens == 128
assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 64
class TestResponsesAPIProviderSpecificParams:
"""

104
tests/test_litellm/test_cost_calculator.py
View file

@ -19,6 +19,7 @@ from litellm.cost_calculator import (
)
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.openai import OpenAIRealtimeStreamList
from litellm.types.rerank import RerankResponse
from litellm.types.utils import (
@ -4848,6 +4849,109 @@ def test_collect_and_combine_realtime_usage_stores_partitioned_text_tokens() ->
assert combined.completion_tokens_details.audio_tokens == 0
def test_realtime_combine_sums_nested_cached_tokens_details():
results: OpenAIRealtimeStreamList = [
{
"type": "response.done",
"response": {
"usage": {
"input_tokens": 283,
"output_tokens": 0,
"total_tokens": 283,
"input_token_details": {
"text_tokens": 116,
"audio_tokens": 167,
"cached_tokens": 192,
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
},
}
},
},
{
"type": "response.done",
"response": {
"usage": {
"input_tokens": 150,
"output_tokens": 0,
"total_tokens": 150,
"input_token_details": {
"text_tokens": 50,
"audio_tokens": 100,
"cached_tokens": 100,
"cached_tokens_details": {"audio_tokens": 100},
},
}
},
},
]
combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(
results=results,
)
assert combined.prompt_tokens_details is not None
assert combined.prompt_tokens_details.cached_tokens == 292
assert combined.prompt_tokens_details.cached_tokens_details is not None
assert combined.prompt_tokens_details.cached_tokens_details.audio_tokens == 228
assert combined.prompt_tokens_details.cached_tokens_details.text_tokens == 64
assert combined.prompt_tokens_details.cached_tokens_details.image_tokens is None
@pytest.mark.parametrize("details_first", [True, False])
def test_realtime_combine_keeps_cached_split_when_only_one_usage_has_details(details_first: bool):
with_details: Final = {
"type": "response.done",
"response": {
"usage": {
"input_tokens": 283,
"output_tokens": 0,
"total_tokens": 283,
"input_token_details": {
"text_tokens": 116,
"audio_tokens": 167,
"cached_tokens": 192,
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
},
}
},
}
without_details: Final = {
"type": "response.done",
"response": {
"usage": {
"input_tokens": 150,
"output_tokens": 0,
"total_tokens": 150,
"input_token_details": {"text_tokens": 50, "audio_tokens": 100, "cached_tokens": 100},
}
},
}
results: OpenAIRealtimeStreamList = (
[with_details, without_details] if details_first else [without_details, with_details]
)
combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(
results=results,
)
assert combined.prompt_tokens_details is not None
assert combined.prompt_tokens_details.cached_tokens == 292
assert combined.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(text_tokens=64, audio_tokens=128)
def test_usage_without_cached_tokens_details_omits_key():
usage = Usage(
prompt_tokens=10,
completion_tokens=5,
total_tokens=15,
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=10),
)
dumped = usage.prompt_tokens_details.model_dump()
assert "cached_tokens_details" not in dumped
assert "cached_tokens_details" not in usage.prompt_tokens_details.model_dump_json()
UNMAPPED_OCR_MODEL: Final = "azure_ai/some-unmapped-ocr-model-for-testing"
MAPPED_OCR_MODEL: Final = "mistral/mistral-ocr-4-0"

8
tests/test_litellm/test_utils.py
View file

@ -6455,3 +6455,11 @@ def test_completion_finishes_response_metadata_before_handing_the_response_to_th
assert snapshot["litellm_call_id"]
assert snapshot["response_cost"] is not None
assert snapshot["api_base"]
def test_get_model_info_carries_cache_read_input_audio_token_cost(monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
info = litellm.get_model_info("gpt-realtime-2.1-mini", custom_llm_provider="openai")
assert info["cache_read_input_audio_token_cost"] == 3e-07
assert info["cache_read_input_token_cost"] == 6e-08

2
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -16781,7 +16781,6 @@ export interface paths {
* - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
* - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
* - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
* - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
* - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
* - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
@ -16887,7 +16886,6 @@ export interface paths {
* - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
* - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
* - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
* - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
* - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
* - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)