mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
Merge 3aeae3c7fe into c134fb7a38
This commit is contained in:
devin-ai-integration[bot]
GitHub
commit
a9d55786ff
14 changed files with 441 additions and 286 deletions
|
|
@ -109,6 +109,7 @@ from litellm.types.llms.openai import (
|
|||
)
|
||||
from litellm.types.rerank import RerankBilledUnits, RerankResponse
|
||||
from litellm.types.utils import (
|
||||
CachedTokensDetails,
|
||||
CallTypesLiteral,
|
||||
LiteLLMRealtimeStreamLoggingObject,
|
||||
LlmProviders,
|
||||
|
|
@ -2373,6 +2374,46 @@ def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]
|
|||
return [attr for attr in field_names if attr != "cache_creation_tokens"]
|
||||
|
||||
|
||||
def _combine_cached_tokens_details(
|
||||
current: CachedTokensDetails | None, new: CachedTokensDetails
|
||||
) -> CachedTokensDetails:
|
||||
def _sum_optional(current_value: int | None, new_value: int | None) -> int | None:
|
||||
if current_value is None and new_value is None:
|
||||
return None
|
||||
return (current_value or 0) + (new_value or 0)
|
||||
|
||||
return CachedTokensDetails(
|
||||
text_tokens=_sum_optional(current.text_tokens if current is not None else None, new.text_tokens),
|
||||
audio_tokens=_sum_optional(current.audio_tokens if current is not None else None, new.audio_tokens),
|
||||
image_tokens=_sum_optional(current.image_tokens if current is not None else None, new.image_tokens),
|
||||
)
|
||||
|
||||
|
||||
def _combine_prompt_tokens_details(
|
||||
current: PromptTokensDetailsWrapper | None, new: PromptTokensDetailsWrapper
|
||||
) -> PromptTokensDetailsWrapper:
|
||||
base: Final = current if current is not None else PromptTokensDetailsWrapper()
|
||||
base_values: Final = MappingProxyType(
|
||||
{attr: getattr(base, attr) for attr in type(base).model_fields if hasattr(base, attr)}
|
||||
)
|
||||
summed: Final = MappingProxyType(
|
||||
{
|
||||
attr: (getattr(base, attr, 0) or 0) + (getattr(new, attr) or 0)
|
||||
for attr in _summable_prompt_token_fields(new)
|
||||
if hasattr(new, attr) and isinstance(getattr(new, attr) or 0, (int, float))
|
||||
}
|
||||
)
|
||||
new_cached_tokens_details: Final = getattr(new, "cached_tokens_details", None)
|
||||
cached_tokens_details: Final = (
|
||||
_combine_cached_tokens_details(getattr(base, "cached_tokens_details", None), new_cached_tokens_details)
|
||||
if isinstance(new_cached_tokens_details, CachedTokensDetails)
|
||||
else getattr(base, "cached_tokens_details", None)
|
||||
)
|
||||
return PromptTokensDetailsWrapper(
|
||||
**MappingProxyType({**base_values, **summed, "cached_tokens_details": cached_tokens_details})
|
||||
)
|
||||
|
||||
|
||||
class BaseTokenUsageProcessor:
|
||||
@staticmethod
|
||||
def combine_usage_objects(usage_objects: list[Usage]) -> Usage:
|
||||
|
|
@ -2381,7 +2422,6 @@ class BaseTokenUsageProcessor:
|
|||
"""
|
||||
from litellm.types.utils import (
|
||||
CompletionTokensDetailsWrapper,
|
||||
PromptTokensDetailsWrapper,
|
||||
Usage,
|
||||
)
|
||||
|
||||
|
|
@ -2400,27 +2440,10 @@ class BaseTokenUsageProcessor:
|
|||
and isinstance(current_val, (int, float))
|
||||
):
|
||||
setattr(combined, attr, current_val + new_val)
|
||||
# Handle nested prompt_tokens_details
|
||||
if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details:
|
||||
if not hasattr(combined, "prompt_tokens_details") or not combined.prompt_tokens_details:
|
||||
combined.prompt_tokens_details = PromptTokensDetailsWrapper()
|
||||
|
||||
# Check what keys exist in the model's prompt_tokens_details
|
||||
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
|
||||
for attr in _summable_prompt_token_fields(usage.prompt_tokens_details):
|
||||
if (
|
||||
hasattr(usage.prompt_tokens_details, attr)
|
||||
and not attr.startswith("_")
|
||||
and not callable(_attribute_value(usage.prompt_tokens_details, attr))
|
||||
):
|
||||
current_val = getattr(combined.prompt_tokens_details, attr, 0) or 0
|
||||
new_val = getattr(usage.prompt_tokens_details, attr, 0) or 0
|
||||
if new_val is not None and isinstance(new_val, (int, float)):
|
||||
setattr(
|
||||
combined.prompt_tokens_details,
|
||||
attr,
|
||||
current_val + new_val,
|
||||
)
|
||||
combined.prompt_tokens_details = _combine_prompt_tokens_details(
|
||||
getattr(combined, "prompt_tokens_details", None), usage.prompt_tokens_details
|
||||
)
|
||||
|
||||
# Handle nested completion_tokens_details
|
||||
if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details:
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ from types import MappingProxyType
|
|||
from typing import Any, Final, Literal, TypedDict, cast
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
from typing_extensions import ReadOnly
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import current_billing_time
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -772,6 +774,7 @@ def calculate_cache_writing_cost(
|
|||
|
||||
class PromptTokensDetailsResult(TypedDict):
|
||||
cache_hit_tokens: int
|
||||
cache_hit_audio_tokens: ReadOnly[int]
|
||||
cache_creation_tokens: int
|
||||
cache_creation_token_details: CacheCreationTokenDetails | None
|
||||
text_tokens: int
|
||||
|
|
@ -802,12 +805,34 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
|
|||
)
|
||||
or None
|
||||
)
|
||||
text_tokens: Final = (
|
||||
cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None))
|
||||
or 0 # default to prompt tokens, if this field is not set
|
||||
cached_tokens_details: Final = getattr(usage.prompt_tokens_details, "cached_tokens_details", None)
|
||||
cached_audio_tokens: Final = min(
|
||||
_get_token_detail_value(cached_tokens_details, "audio_tokens") or 0, cache_hit_tokens
|
||||
)
|
||||
cached_text_tokens: Final = min(
|
||||
_get_token_detail_value(cached_tokens_details, "text_tokens") or 0,
|
||||
cache_hit_tokens - cached_audio_tokens,
|
||||
)
|
||||
cached_image_tokens: Final = min(
|
||||
_get_token_detail_value(cached_tokens_details, "image_tokens") or 0,
|
||||
cache_hit_tokens - cached_audio_tokens - cached_text_tokens,
|
||||
)
|
||||
text_tokens: Final = max(
|
||||
(
|
||||
cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None))
|
||||
or 0 # default to prompt tokens, if this field is not set
|
||||
)
|
||||
- cached_text_tokens,
|
||||
0,
|
||||
)
|
||||
audio_tokens: Final = max(
|
||||
(cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0) - cached_audio_tokens,
|
||||
0,
|
||||
)
|
||||
image_tokens: Final = max(
|
||||
(cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0) - cached_image_tokens,
|
||||
0,
|
||||
)
|
||||
audio_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0
|
||||
image_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0
|
||||
video_tokens: Final = _coerce_token_count(getattr(usage.prompt_tokens_details, "video_tokens", 0))
|
||||
character_count: Final = (
|
||||
cast(
|
||||
|
|
@ -835,6 +860,7 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
|
|||
|
||||
return PromptTokensDetailsResult(
|
||||
cache_hit_tokens=cache_hit_tokens,
|
||||
cache_hit_audio_tokens=cached_audio_tokens,
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
cache_creation_token_details=cache_creation_token_details,
|
||||
text_tokens=text_tokens,
|
||||
|
|
@ -918,7 +944,16 @@ def _calculate_input_cost(
|
|||
prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
|
||||
|
||||
### CACHE READ COST - Now uses tiered pricing
|
||||
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
|
||||
cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
|
||||
audio_cache_read_rate: Final = _get_cost_per_unit(
|
||||
model_info,
|
||||
_get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
|
||||
None,
|
||||
)
|
||||
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost
|
||||
prompt_cost += float(cache_hit_audio_tokens) * (
|
||||
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
|
||||
)
|
||||
|
||||
### AUDIO COST
|
||||
if prompt_tokens_details["audio_tokens"]:
|
||||
|
|
@ -1149,6 +1184,7 @@ def generic_cost_per_token(
|
|||
### PROCESSING COST
|
||||
prompt_tokens_details = PromptTokensDetailsResult(
|
||||
cache_hit_tokens=0,
|
||||
cache_hit_audio_tokens=0,
|
||||
cache_creation_tokens=0,
|
||||
cache_creation_token_details=None,
|
||||
text_tokens=usage.prompt_tokens,
|
||||
|
|
|
|||
|
|
@ -32512,6 +32512,7 @@
|
|||
},
|
||||
"gpt-realtime": {
|
||||
"cache_creation_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"deprecation_date": "2027-01-20",
|
||||
"input_cost_per_audio_token": 3.2e-05,
|
||||
|
|
@ -32545,6 +32546,7 @@
|
|||
},
|
||||
"gpt-realtime-1.5": {
|
||||
"cache_creation_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"input_cost_per_audio_token": 3.2e-05,
|
||||
"input_cost_per_image_token": 5e-06,
|
||||
|
|
@ -32712,6 +32714,7 @@
|
|||
},
|
||||
"gpt-realtime-2025-08-28": {
|
||||
"cache_creation_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"deprecation_date": "2027-01-20",
|
||||
"input_cost_per_audio_token": 3.2e-05,
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ from litellm.responses.litellm_completion_transformation.session_handler import
|
|||
)
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
CachedTokensDetails,
|
||||
ChatCompletionAssistantMessage,
|
||||
ChatCompletionImageObject,
|
||||
ChatCompletionImageUrlObject,
|
||||
|
|
@ -2743,27 +2744,21 @@ class LiteLLMCompletionResponsesConfig:
|
|||
# Translate prompt_tokens_details to input_tokens_details
|
||||
if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details is not None:
|
||||
prompt_details: Final = usage.prompt_tokens_details
|
||||
input_details_dict: Final[dict[str, int]] = {}
|
||||
|
||||
if hasattr(prompt_details, "cached_tokens") and prompt_details.cached_tokens is not None:
|
||||
input_details_dict["cached_tokens"] = prompt_details.cached_tokens
|
||||
else:
|
||||
input_details_dict["cached_tokens"] = 0
|
||||
|
||||
if hasattr(prompt_details, "text_tokens") and prompt_details.text_tokens is not None:
|
||||
input_details_dict["text_tokens"] = prompt_details.text_tokens
|
||||
|
||||
if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None:
|
||||
input_details_dict["audio_tokens"] = prompt_details.audio_tokens
|
||||
|
||||
cache_write_tokens = getattr(prompt_details, "cache_write_tokens", None) or getattr(
|
||||
cached_tokens_details: Final = getattr(prompt_details, "cached_tokens_details", None)
|
||||
cache_write_tokens: Final = getattr(prompt_details, "cache_write_tokens", None) or getattr(
|
||||
prompt_details, "cache_creation_tokens", None
|
||||
)
|
||||
input_tokens_details: Final = InputTokensDetails(
|
||||
cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
|
||||
text_tokens=prompt_details.text_tokens,
|
||||
audio_tokens=prompt_details.audio_tokens,
|
||||
cached_tokens_details=(
|
||||
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
|
||||
),
|
||||
)
|
||||
if cache_write_tokens is not None:
|
||||
input_details_dict["cache_write_tokens"] = cache_write_tokens
|
||||
|
||||
if input_details_dict:
|
||||
response_usage.input_tokens_details = InputTokensDetails(**input_details_dict)
|
||||
setattr(input_tokens_details, "cache_write_tokens", cache_write_tokens)
|
||||
response_usage.input_tokens_details = input_tokens_details
|
||||
|
||||
# Translate completion_tokens_details to output_tokens_details
|
||||
if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details is not None:
|
||||
|
|
|
|||
|
|
@ -1179,6 +1179,9 @@ class ResponseAPILoggingUtils:
|
|||
audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None),
|
||||
text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None),
|
||||
image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None),
|
||||
cached_tokens_details=getattr(
|
||||
response_api_usage.input_tokens_details, "cached_tokens_details", None
|
||||
),
|
||||
cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None),
|
||||
)
|
||||
completion_tokens_details: CompletionTokensDetailsWrapper | None = None
|
||||
|
|
|
|||
|
|
@ -1285,9 +1285,16 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject):
|
|||
model_config = {"extra": "allow"}
|
||||
|
||||
|
||||
class CachedTokensDetails(BaseModel):
|
||||
text_tokens: int | None = None
|
||||
audio_tokens: int | None = None
|
||||
image_tokens: int | None = None
|
||||
|
||||
|
||||
class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
|
||||
audio_tokens: int | None = None
|
||||
cached_tokens: int = 0
|
||||
cached_tokens_details: CachedTokensDetails | None = None
|
||||
text_tokens: int | None = None
|
||||
|
||||
model_config = {"extra": "allow"}
|
||||
|
|
@ -2254,10 +2261,17 @@ class OpenAIRealtimeInputAudioTranscriptionCompleted(TypedDict):
|
|||
usage: NotRequired[ReadOnly[Mapping[str, object]]]
|
||||
|
||||
|
||||
class OpenAIRealtimeCachedTokensDetails(TypedDict, total=False):
|
||||
text_tokens: ReadOnly[int]
|
||||
audio_tokens: ReadOnly[int]
|
||||
image_tokens: ReadOnly[int]
|
||||
|
||||
|
||||
class OpenAIRealtimeUsageTokenDetails(TypedDict):
|
||||
audio_tokens: ReadOnly[int]
|
||||
text_tokens: ReadOnly[int]
|
||||
cached_tokens: NotRequired[ReadOnly[int]]
|
||||
cached_tokens_details: NotRequired[ReadOnly[OpenAIRealtimeCachedTokensDetails]]
|
||||
|
||||
|
||||
class OpenAIRealtimeResponseUsage(TypedDict):
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ from .llms.base import HiddenParams
|
|||
from .llms.openai import (
|
||||
AllMessageValues,
|
||||
Batch,
|
||||
CachedTokensDetails,
|
||||
ChatCompletionAnnotation,
|
||||
ChatCompletionReasoningItem,
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
|
|
@ -250,6 +251,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing
|
||||
cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
|
||||
cache_read_input_token_cost: float | None
|
||||
cache_read_input_audio_token_cost: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing
|
||||
cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing
|
||||
cache_read_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
|
||||
|
|
@ -1708,6 +1710,9 @@ class PromptTokensDetailsWrapper(
|
|||
cache_creation_token_details: CacheCreationTokenDetails | None = None
|
||||
"""Details of cache creation tokens sent to the model. Used for tracking 5m/1h cache creation tokens for Anthropic prompt caching."""
|
||||
|
||||
cached_tokens_details: CachedTokensDetails | None = None
|
||||
"""Details of cached (cache-hit) tokens sent to the model. OpenAI realtime naming; carries the per-modality cache-read split."""
|
||||
|
||||
def __setattr__(self, name: str, value: object) -> None:
|
||||
super().__setattr__(name, value)
|
||||
if name == "cache_write_tokens":
|
||||
|
|
@ -1754,6 +1759,8 @@ class PromptTokensDetailsWrapper(
|
|||
del self.cache_creation_tokens
|
||||
if self.cache_creation_token_details is None:
|
||||
del self.cache_creation_token_details
|
||||
if self.cached_tokens_details is None:
|
||||
del self.cached_tokens_details
|
||||
|
||||
|
||||
class ServerToolUse(BaseModel):
|
||||
|
|
|
|||
|
|
@ -5866,6 +5866,7 @@ def _get_model_info_helper(
|
|||
"cache_creation_input_token_cost_ultrafast", None
|
||||
),
|
||||
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
|
||||
cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None),
|
||||
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
|
||||
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
|
||||
"cache_read_input_token_cost_above_200k_tokens", None
|
||||
|
|
|
|||
|
|
@ -32512,6 +32512,7 @@
|
|||
},
|
||||
"gpt-realtime": {
|
||||
"cache_creation_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"deprecation_date": "2027-01-20",
|
||||
"input_cost_per_audio_token": 3.2e-05,
|
||||
|
|
@ -32545,6 +32546,7 @@
|
|||
},
|
||||
"gpt-realtime-1.5": {
|
||||
"cache_creation_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"input_cost_per_audio_token": 3.2e-05,
|
||||
"input_cost_per_image_token": 5e-06,
|
||||
|
|
@ -32712,6 +32714,7 @@
|
|||
},
|
||||
"gpt-realtime-2025-08-28": {
|
||||
"cache_creation_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"deprecation_date": "2027-01-20",
|
||||
"input_cost_per_audio_token": 3.2e-05,
|
||||
|
|
|
|||
|
|
@ -2648,6 +2648,7 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
|
|||
|
||||
prompt_tokens_details: PromptTokensDetailsResult = {
|
||||
"cache_hit_tokens": 0,
|
||||
"cache_hit_audio_tokens": 0,
|
||||
"cache_creation_tokens": 0,
|
||||
"cache_creation_token_details": CacheCreationTokenDetails(
|
||||
ephemeral_5m_input_tokens=100,
|
||||
|
|
@ -5147,3 +5148,107 @@ def test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output(
|
|||
assert completion_cost == pytest.approx(
|
||||
30 * info["output_cost_per_token"] + 70 * info["output_cost_per_audio_token"]
|
||||
)
|
||||
|
||||
|
||||
def test_cached_realtime_audio_tokens_billed_at_audio_cache_read_rate(
|
||||
_local_model_cost_map: None,
|
||||
) -> None:
|
||||
usage = Usage(
|
||||
prompt_tokens=283,
|
||||
completion_tokens=0,
|
||||
total_tokens=283,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=116,
|
||||
audio_tokens=167,
|
||||
cached_tokens=192,
|
||||
cached_tokens_details={"text_tokens": 64, "audio_tokens": 128},
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
model="gpt-realtime-2", usage=usage, custom_llm_provider="openai"
|
||||
)
|
||||
assert prompt_cost == pytest.approx(0.0015328)
|
||||
|
||||
|
||||
def test_prompt_tokens_details_without_cached_tokens_details_unchanged(
|
||||
_local_model_cost_map: None,
|
||||
) -> None:
|
||||
usage = Usage(
|
||||
prompt_tokens=283,
|
||||
completion_tokens=0,
|
||||
total_tokens=283,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=116, audio_tokens=167, cached_tokens=192
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
model="gpt-realtime-2", usage=usage, custom_llm_provider="openai"
|
||||
)
|
||||
assert prompt_cost == pytest.approx(0.0029888)
|
||||
|
||||
|
||||
def test_cached_audio_tokens_fall_back_to_cache_read_input_token_cost() -> None:
|
||||
model_info: ModelInfo = {
|
||||
"input_cost_per_token": 4e-6,
|
||||
"input_cost_per_audio_token": 32e-6,
|
||||
"cache_read_input_token_cost": 5e-7,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=283,
|
||||
completion_tokens=0,
|
||||
total_tokens=283,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=116,
|
||||
audio_tokens=167,
|
||||
cached_tokens=192,
|
||||
cached_tokens_details={"text_tokens": 64, "audio_tokens": 128},
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
model="some-realtime-model",
|
||||
usage=usage,
|
||||
custom_llm_provider="openai",
|
||||
model_info=model_info,
|
||||
)
|
||||
expected = 52 * 4e-6 + 64 * 5e-7 + 39 * 32e-6 + 128 * 5e-7
|
||||
assert prompt_cost == pytest.approx(expected)
|
||||
|
||||
|
||||
def test_cached_audio_tokens_capped_at_cached_tokens(_local_model_cost_map: None) -> None:
|
||||
"""Nested cached_tokens_details exceeding cached_tokens must not over-subtract the audio bucket."""
|
||||
usage = Usage(
|
||||
prompt_tokens=283,
|
||||
completion_tokens=0,
|
||||
total_tokens=283,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=116,
|
||||
audio_tokens=167,
|
||||
cached_tokens=100,
|
||||
cached_tokens_details={"audio_tokens": 128},
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
model="gpt-realtime-2", usage=usage, custom_llm_provider="openai"
|
||||
)
|
||||
assert prompt_cost == pytest.approx(116 * 4e-6 + (167 - 100) * 32e-6 + 100 * 4e-7)
|
||||
|
||||
|
||||
def test_cached_audio_tokens_billed_at_audio_cache_rate_through_model_info_lookup(_local_model_cost_map: None) -> None:
|
||||
usage = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=0,
|
||||
total_tokens=1000,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=400,
|
||||
audio_tokens=600,
|
||||
cached_tokens=500,
|
||||
cached_tokens_details={"text_tokens": 100, "audio_tokens": 400},
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(model="gpt-realtime-2.1-mini", usage=usage, custom_llm_provider="openai")
|
||||
assert prompt_cost == pytest.approx(300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7)
|
||||
|
|
|
|||
|
|
@ -2615,6 +2615,7 @@ class TestUsageTransformation:
|
|||
assert response_usage.input_tokens_details is not None
|
||||
assert response_usage.input_tokens_details.cached_tokens == 5
|
||||
assert response_usage.input_tokens_details.text_tokens == 8
|
||||
assert "cache_write_tokens" not in response_usage.input_tokens_details.model_dump()
|
||||
|
||||
def test_transform_usage_with_cached_tokens_gemini(self):
|
||||
"""Test that cached_tokens from Gemini are properly transformed to input_tokens_details"""
|
||||
|
|
@ -2677,6 +2678,7 @@ class TestUsageTransformation:
|
|||
assert response_usage.input_tokens_details is not None
|
||||
assert response_usage.input_tokens_details.cached_tokens == 100
|
||||
assert getattr(response_usage.input_tokens_details, "cache_write_tokens", None) == 800
|
||||
assert response_usage.input_tokens_details.model_dump()["cache_write_tokens"] == 800
|
||||
|
||||
def test_transform_usage_with_reasoning_tokens_gemini(self):
|
||||
"""Test that reasoning_tokens from Gemini are properly transformed to output_tokens_details"""
|
||||
|
|
|
|||
|
|
@ -577,6 +577,47 @@ class TestResponseAPILoggingUtils:
|
|||
assert result.completion_tokens_details is not None
|
||||
assert result.completion_tokens_details.reasoning_tokens == 4
|
||||
|
||||
def test_transform_realtime_usage_dict_keeps_cached_tokens_details(self):
|
||||
usage = {
|
||||
"input_tokens": 283,
|
||||
"output_tokens": 0,
|
||||
"total_tokens": 283,
|
||||
"input_token_details": {
|
||||
"text_tokens": 116,
|
||||
"audio_tokens": 167,
|
||||
"cached_tokens": 192,
|
||||
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
|
||||
},
|
||||
}
|
||||
|
||||
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
|
||||
|
||||
assert result.prompt_tokens_details is not None
|
||||
assert result.prompt_tokens_details.cached_tokens == 192
|
||||
assert result.prompt_tokens_details.cached_tokens_details is not None
|
||||
assert result.prompt_tokens_details.cached_tokens_details.audio_tokens == 128
|
||||
assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 64
|
||||
|
||||
def test_transform_response_api_usage_object_keeps_cached_tokens_details(self):
|
||||
usage = ResponseAPIUsage(
|
||||
input_tokens=283,
|
||||
output_tokens=0,
|
||||
total_tokens=283,
|
||||
input_tokens_details={
|
||||
"text_tokens": 116,
|
||||
"audio_tokens": 167,
|
||||
"cached_tokens": 192,
|
||||
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
|
||||
},
|
||||
)
|
||||
|
||||
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
|
||||
|
||||
assert result.prompt_tokens_details is not None
|
||||
assert result.prompt_tokens_details.cached_tokens_details is not None
|
||||
assert result.prompt_tokens_details.cached_tokens_details.audio_tokens == 128
|
||||
assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 64
|
||||
|
||||
|
||||
class TestResponsesAPIProviderSpecificParams:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
|
@ -149,9 +148,7 @@ def test_jina_rerank_bills_total_tokens_at_input_rate_only(_local_model_cost_map
|
|||
|
||||
def test_cost_calculator_with_response_cost_in_additional_headers():
|
||||
class MockResponse(BaseModel):
|
||||
_hidden_params = {
|
||||
"additional_headers": {"llm_provider-x-litellm-response-cost": 1000}
|
||||
}
|
||||
_hidden_params = {"additional_headers": {"llm_provider-x-litellm-response-cost": 1000}}
|
||||
|
||||
result = response_cost_calculator(
|
||||
response_object=MockResponse(),
|
||||
|
|
@ -207,7 +204,9 @@ def test_vertex_lyria_speech_cost(
|
|||
call_type=call_type,
|
||||
)
|
||||
|
||||
expected: Final = 0 if runtime_state == "custom_zero" else expected_cost * (2 if runtime_state == "custom_price" else 1)
|
||||
expected: Final = (
|
||||
0 if runtime_state == "custom_zero" else expected_cost * (2 if runtime_state == "custom_price" else 1)
|
||||
)
|
||||
assert cost == pytest.approx(expected)
|
||||
|
||||
|
||||
|
|
@ -334,13 +333,12 @@ def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch):
|
|||
|
||||
# Step 1: Test a model where input_cost_per_image_token is not set.
|
||||
# In this case the calculation should use input_cost_per_token as fallback.
|
||||
assert (
|
||||
model_info.get("input_cost_per_image_token") is None
|
||||
), "Test case expects that input_cost_per_image_token is not set"
|
||||
assert model_info.get("input_cost_per_image_token") is None, (
|
||||
"Test case expects that input_cost_per_image_token is not set"
|
||||
)
|
||||
|
||||
expected_cost = (
|
||||
usage.prompt_tokens_details.audio_tokens
|
||||
* model_info["input_cost_per_audio_token"]
|
||||
usage.prompt_tokens_details.audio_tokens * model_info["input_cost_per_audio_token"]
|
||||
+ usage.prompt_tokens_details.text_tokens * model_info["input_cost_per_token"]
|
||||
+ usage.prompt_tokens_details.image_tokens * model_info["input_cost_per_token"]
|
||||
+ usage.completion_tokens * model_info["output_cost_per_token"]
|
||||
|
|
@ -375,12 +373,9 @@ def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch):
|
|||
)
|
||||
|
||||
expected_cost = (
|
||||
usage.prompt_tokens_details.audio_tokens
|
||||
* temp_model_info_object["input_cost_per_audio_token"]
|
||||
+ usage.prompt_tokens_details.text_tokens
|
||||
* temp_model_info_object["input_cost_per_token"]
|
||||
+ usage.prompt_tokens_details.image_tokens
|
||||
* temp_model_info_object["input_cost_per_image_token"]
|
||||
usage.prompt_tokens_details.audio_tokens * temp_model_info_object["input_cost_per_audio_token"]
|
||||
+ usage.prompt_tokens_details.text_tokens * temp_model_info_object["input_cost_per_token"]
|
||||
+ usage.prompt_tokens_details.image_tokens * temp_model_info_object["input_cost_per_image_token"]
|
||||
+ usage.completion_tokens * temp_model_info_object["output_cost_per_token"]
|
||||
)
|
||||
|
||||
|
|
@ -390,14 +385,11 @@ def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch):
|
|||
def test_transcription_cost_uses_token_pricing(_local_model_cost_map):
|
||||
from litellm import completion_cost
|
||||
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=14,
|
||||
completion_tokens=45,
|
||||
total_tokens=59,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=0, audio_tokens=14
|
||||
),
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=0, audio_tokens=14),
|
||||
)
|
||||
response = TranscriptionResponse(text="demo text")
|
||||
response.usage = usage
|
||||
|
|
@ -441,7 +433,6 @@ def test_transcription_token_pricing_is_provider_aware(_local_model_cost_map):
|
|||
def test_transcription_cost_falls_back_to_duration(_local_model_cost_map):
|
||||
from litellm import completion_cost
|
||||
|
||||
|
||||
response = TranscriptionResponse(text="demo text")
|
||||
response.duration = 10.0
|
||||
|
||||
|
|
@ -462,7 +453,6 @@ def test_vertex_chirp_3_transcription_cost_from_duration(_local_model_cost_map):
|
|||
every transcription priced to $0.00 instead of using input_cost_per_second."""
|
||||
from litellm import completion_cost
|
||||
|
||||
|
||||
response = TranscriptionResponse(text="demo text")
|
||||
response.duration = 18.0
|
||||
|
||||
|
|
@ -486,9 +476,7 @@ def test_handle_realtime_stream_cost_calculation():
|
|||
{"type": "session.created", "session": {"model": "gpt-3.5-turbo"}},
|
||||
{
|
||||
"type": "response.done",
|
||||
"response": {
|
||||
"usage": {"input_tokens": 100, "output_tokens": 50, "total_tokens": 150}
|
||||
},
|
||||
"response": {"usage": {"input_tokens": 100, "output_tokens": 50, "total_tokens": 150}},
|
||||
},
|
||||
{
|
||||
"type": "response.done",
|
||||
|
|
@ -519,9 +507,7 @@ def test_handle_realtime_stream_cost_calculation():
|
|||
expected_cost = (300 * 0.0015 / 1000) + ( # input tokens (100 + 200)
|
||||
150 * 0.002 / 1000
|
||||
) # output tokens (50 + 100)
|
||||
assert (
|
||||
abs(cost - expected_cost) <= 0.00075
|
||||
) # Allow small floating point differences
|
||||
assert abs(cost - expected_cost) <= 0.00075 # Allow small floating point differences
|
||||
|
||||
# Test with different model name in session
|
||||
results[0]["session"]["model"] = "gpt-4"
|
||||
|
|
@ -601,14 +587,7 @@ def test_handle_realtime_stream_cost_calculation_stores_cost_breakdown():
|
|||
assert logging_obj.cost_breakdown is not None
|
||||
assert logging_obj.cost_breakdown["input_cost"] > 0
|
||||
assert logging_obj.cost_breakdown["output_cost"] > 0
|
||||
assert (
|
||||
abs(
|
||||
logging_obj.cost_breakdown["input_cost"]
|
||||
+ logging_obj.cost_breakdown["output_cost"]
|
||||
- total_cost
|
||||
)
|
||||
< 1e-9
|
||||
)
|
||||
assert abs(logging_obj.cost_breakdown["input_cost"] + logging_obj.cost_breakdown["output_cost"] - total_cost) < 1e-9
|
||||
assert abs(logging_obj.cost_breakdown["total_cost"] - total_cost) < 1e-9
|
||||
|
||||
|
||||
|
|
@ -682,9 +661,7 @@ def test_realtime_logging_object_allows_null_transcript_in_conversation_item_add
|
|||
},
|
||||
]
|
||||
|
||||
usage = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(
|
||||
results=results
|
||||
)
|
||||
usage = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(results=results)
|
||||
logging_result = RealtimeAPITokenUsageProcessor.create_logging_realtime_object(
|
||||
usage=usage,
|
||||
results=results,
|
||||
|
|
@ -734,9 +711,7 @@ def test_realtime_logging_object_does_not_validate_unknown_event_types():
|
|||
},
|
||||
]
|
||||
|
||||
usage = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(
|
||||
results=results
|
||||
)
|
||||
usage = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(results=results)
|
||||
# On unfixed code this raises pydantic ValidationError instead of returning.
|
||||
logging_result = RealtimeAPITokenUsageProcessor.create_logging_realtime_object(
|
||||
usage=usage,
|
||||
|
|
@ -748,8 +723,7 @@ def test_realtime_logging_object_does_not_validate_unknown_event_types():
|
|||
unknown_types = {
|
||||
r["type"]
|
||||
for r in logging_result.results
|
||||
if r["type"]
|
||||
in ("rate_limits.updated", "response.function_call_arguments.delta")
|
||||
if r["type"] in ("rate_limits.updated", "response.function_call_arguments.delta")
|
||||
}
|
||||
assert unknown_types == {
|
||||
"rate_limits.updated",
|
||||
|
|
@ -782,9 +756,7 @@ def test_realtime_transcription_duration_cost(monkeypatch):
|
|||
"type": "session.created",
|
||||
"session": {
|
||||
"type": "transcription",
|
||||
"audio": {
|
||||
"input": {"transcription": {"model": "gpt-realtime-whisper"}}
|
||||
},
|
||||
"audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}},
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -799,9 +771,7 @@ def test_realtime_transcription_duration_cost(monkeypatch):
|
|||
},
|
||||
]
|
||||
|
||||
combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(
|
||||
results=results
|
||||
)
|
||||
combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(results=results)
|
||||
logging_obj = Logging(
|
||||
model="gpt-realtime-whisper",
|
||||
messages=[],
|
||||
|
|
@ -894,9 +864,7 @@ def test_realtime_transcription_token_billed_fallback(monkeypatch):
|
|||
|
||||
# gpt-4o-transcribe: input_cost_per_audio_token = 2.5e-06, input_cost_per_token = 2.5e-06,
|
||||
# output_cost_per_token = 1e-05
|
||||
model_info = litellm.get_model_info(
|
||||
model="gpt-4o-transcribe", custom_llm_provider="openai"
|
||||
)
|
||||
model_info = litellm.get_model_info(model="gpt-4o-transcribe", custom_llm_provider="openai")
|
||||
usage = {
|
||||
"type": "tokens",
|
||||
"input_tokens": 40,
|
||||
|
|
@ -977,10 +945,7 @@ def test_get_transcription_model_falls_back_to_session_model(monkeypatch):
|
|||
mock_response=True,
|
||||
)
|
||||
|
||||
assert (
|
||||
result._hidden_params["response_cost"]
|
||||
> result_2._hidden_params["response_cost"]
|
||||
)
|
||||
assert result._hidden_params["response_cost"] > result_2._hidden_params["response_cost"]
|
||||
|
||||
model_info = router.get_deployment_model_info(
|
||||
model_id="my-unique-model-id", model_name="anthropic/claude-sonnet-4-5-20250929"
|
||||
|
|
@ -1143,9 +1108,7 @@ def test_tiered_pricing_only_deployment_selects_router_model_id():
|
|||
assert entry.get("input_cost_per_token") is None
|
||||
assert entry.get("tiered_pricing") is not None
|
||||
# The stripped shared alias must not carry tiered pricing.
|
||||
assert (
|
||||
litellm.model_cost["dashscope/qwen-tier-only-test"].get("tiered_pricing") is None
|
||||
)
|
||||
assert litellm.model_cost["dashscope/qwen-tier-only-test"].get("tiered_pricing") is None
|
||||
|
||||
selected = _select_model_name_for_cost_calc(
|
||||
model="dashscope/qwen-tier-only-test",
|
||||
|
|
@ -1225,9 +1188,7 @@ def test_azure_realtime_cost_calculator(_local_model_cost_map):
|
|||
combined_usage_object=Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=100,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=10, audio_tokens=90
|
||||
),
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=10, audio_tokens=90),
|
||||
),
|
||||
custom_llm_provider="azure",
|
||||
litellm_model_name="my-custom-azure-deployment",
|
||||
|
|
@ -1246,7 +1207,6 @@ def test_azure_audio_output_cost_calculation(_local_model_cost_map):
|
|||
"""
|
||||
from litellm.types.utils import Choices, CompletionTokensDetailsWrapper, Message
|
||||
|
||||
|
||||
# Scenario from issue #19764:
|
||||
# Input: 17 text tokens, 0 audio tokens
|
||||
# Output: 110 text tokens, 482 audio tokens
|
||||
|
|
@ -1302,14 +1262,10 @@ def test_azure_audio_output_cost_calculation(_local_model_cost_map):
|
|||
wrong_total_cost = expected_input_cost + wrong_output_cost
|
||||
|
||||
# Verify audio tokens are NOT charged at text rate (the bug)
|
||||
assert (
|
||||
abs(cost - wrong_total_cost) > 0.001
|
||||
), "Bug: Audio tokens are being charged at text token rate"
|
||||
assert abs(cost - wrong_total_cost) > 0.001, "Bug: Audio tokens are being charged at text token rate"
|
||||
|
||||
# Verify cost matches
|
||||
assert (
|
||||
abs(cost - expected_total_cost) < 0.0000001
|
||||
), f"Expected cost {expected_total_cost}, got {cost}"
|
||||
assert abs(cost - expected_total_cost) < 0.0000001, f"Expected cost {expected_total_cost}, got {cost}"
|
||||
|
||||
|
||||
def test_default_image_cost_calculator(monkeypatch):
|
||||
|
|
@ -1323,9 +1279,7 @@ def test_default_image_cost_calculator(monkeypatch):
|
|||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"model_cost",
|
||||
{
|
||||
"azure/bf9001cd7209f5734ecb4ab937a5a0e2ba5f119708bd68f184db362930f9dc7b": temp_object
|
||||
},
|
||||
{"azure/bf9001cd7209f5734ecb4ab937a5a0e2ba5f119708bd68f184db362930f9dc7b": temp_object},
|
||||
)
|
||||
|
||||
args = {
|
||||
|
|
@ -1541,9 +1495,7 @@ def test_gemini_25_implicit_caching_cost():
|
|||
expected_cost = 0.00068708
|
||||
|
||||
# Allow for small floating point differences
|
||||
assert (
|
||||
abs(result - expected_cost) < 1e-8
|
||||
), f"Expected cost {expected_cost}, but got {result}"
|
||||
assert abs(result - expected_cost) < 1e-8, f"Expected cost {expected_cost}, but got {result}"
|
||||
|
||||
print(f"✓ Gemini 2.5 implicit caching cost calculation is correct: ${result:.8f}")
|
||||
|
||||
|
|
@ -1614,9 +1566,7 @@ def test_log_context_cost_calculation():
|
|||
# Get model info to understand the pricing
|
||||
from litellm import get_model_info
|
||||
|
||||
model_info = get_model_info(
|
||||
model="claude-4-sonnet-20250514", custom_llm_provider="anthropic"
|
||||
)
|
||||
model_info = get_model_info(model="claude-4-sonnet-20250514", custom_llm_provider="anthropic")
|
||||
|
||||
# Calculate expected cost based on actual model pricing
|
||||
input_cost_per_token = model_info.get("input_cost_per_token", 0)
|
||||
|
|
@ -1624,12 +1574,8 @@ def test_log_context_cost_calculation():
|
|||
cache_creation_cost_per_token = model_info.get("cache_creation_input_token_cost", 0)
|
||||
|
||||
# Check if tiered pricing is applied
|
||||
input_cost_above_200k = model_info.get(
|
||||
"input_cost_per_token_above_200k_tokens", input_cost_per_token
|
||||
)
|
||||
output_cost_above_200k = model_info.get(
|
||||
"output_cost_per_token_above_200k_tokens", output_cost_per_token
|
||||
)
|
||||
input_cost_above_200k = model_info.get("input_cost_per_token_above_200k_tokens", input_cost_per_token)
|
||||
output_cost_above_200k = model_info.get("output_cost_per_token_above_200k_tokens", output_cost_per_token)
|
||||
cache_creation_above_200k = model_info.get(
|
||||
"cache_creation_input_token_cost_above_200k_tokens",
|
||||
cache_creation_cost_per_token,
|
||||
|
|
@ -1637,31 +1583,23 @@ def test_log_context_cost_calculation():
|
|||
|
||||
print(f"DEBUG: Base input cost per token: ${input_cost_per_token:.2e}")
|
||||
print(f"DEBUG: Base output cost per token: ${output_cost_per_token:.2e}")
|
||||
print(
|
||||
f"DEBUG: Base cache creation cost per token: ${cache_creation_cost_per_token:.2e}"
|
||||
)
|
||||
print(f"DEBUG: Base cache creation cost per token: ${cache_creation_cost_per_token:.2e}")
|
||||
|
||||
# Handle tiered pricing - if not available, use base pricing
|
||||
if input_cost_above_200k is not None:
|
||||
print(
|
||||
f"DEBUG: Tiered input cost per token (>200k): ${input_cost_above_200k:.2e}"
|
||||
)
|
||||
print(f"DEBUG: Tiered input cost per token (>200k): ${input_cost_above_200k:.2e}")
|
||||
else:
|
||||
print("DEBUG: No tiered input pricing available, using base pricing")
|
||||
input_cost_above_200k = input_cost_per_token
|
||||
|
||||
if output_cost_above_200k is not None:
|
||||
print(
|
||||
f"DEBUG: Tiered output cost per token (>200k): ${output_cost_above_200k:.2e}"
|
||||
)
|
||||
print(f"DEBUG: Tiered output cost per token (>200k): ${output_cost_above_200k:.2e}")
|
||||
else:
|
||||
print("DEBUG: No tiered output pricing available, using base pricing")
|
||||
output_cost_above_200k = output_cost_per_token
|
||||
|
||||
if cache_creation_above_200k is not None:
|
||||
print(
|
||||
f"DEBUG: Tiered cache creation cost per token (>200k): ${cache_creation_above_200k:.2e}"
|
||||
)
|
||||
print(f"DEBUG: Tiered cache creation cost per token (>200k): ${cache_creation_above_200k:.2e}")
|
||||
else:
|
||||
print("DEBUG: No tiered cache creation pricing available, using base pricing")
|
||||
cache_creation_above_200k = cache_creation_cost_per_token
|
||||
|
|
@ -1675,13 +1613,9 @@ def test_log_context_cost_calculation():
|
|||
print(f"DEBUG: Expected total: ${expected_total:.6f}")
|
||||
|
||||
# Allow for small floating point differences
|
||||
assert (
|
||||
abs(result - expected_total) < 1e-6
|
||||
), f"Expected cost ${expected_total:.6f}, but got ${result:.6f}"
|
||||
assert abs(result - expected_total) < 1e-6, f"Expected cost ${expected_total:.6f}, but got ${result:.6f}"
|
||||
|
||||
print(
|
||||
f"✓ Log context cost calculation with tiered pricing is correct: ${result:.6f}"
|
||||
)
|
||||
print(f"✓ Log context cost calculation with tiered pricing is correct: ${result:.6f}")
|
||||
print(f" - Input tokens (300k): ${expected_input_cost:.6f}")
|
||||
print(f" - Output tokens (50k): ${expected_output_cost:.6f}")
|
||||
print(f" - Cache creation (1k): ${expected_cache_cost:.6f}")
|
||||
|
|
@ -1740,8 +1674,7 @@ def test_gemini_25_explicit_caching_cost_direct_usage():
|
|||
|
||||
expected_actual_cost = (
|
||||
model_info["input_cost_per_token"] * usage.prompt_tokens_details.text_tokens
|
||||
+ model_info["cache_read_input_token_cost"]
|
||||
* usage.prompt_tokens_details.cached_tokens
|
||||
+ model_info["cache_read_input_token_cost"] * usage.prompt_tokens_details.cached_tokens
|
||||
+ model_info["output_cost_per_token"] * usage.completion_tokens
|
||||
)
|
||||
|
||||
|
|
@ -1765,7 +1698,6 @@ def test_azure_ai_cache_cost_calculation(_local_model_cost_map):
|
|||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
|
||||
|
||||
# Register a custom azure_ai model with cache pricing
|
||||
test_model_id = "test-azure-ai-claude-model"
|
||||
litellm.register_model(
|
||||
|
|
@ -1814,13 +1746,12 @@ def test_azure_ai_cache_cost_calculation(_local_model_cost_map):
|
|||
print(f"Output cost: {output_cost}, Expected: {expected_output_cost}")
|
||||
print(f"Total cost: {total_cost}")
|
||||
|
||||
assert (
|
||||
abs(input_cost - expected_input_cost) < 1e-10
|
||||
), f"Input cost mismatch: got {input_cost}, expected {expected_input_cost}"
|
||||
assert (
|
||||
abs(output_cost - expected_output_cost) < 1e-10
|
||||
), f"Output cost mismatch: got {output_cost}, expected {expected_output_cost}"
|
||||
|
||||
assert abs(input_cost - expected_input_cost) < 1e-10, (
|
||||
f"Input cost mismatch: got {input_cost}, expected {expected_input_cost}"
|
||||
)
|
||||
assert abs(output_cost - expected_output_cost) < 1e-10, (
|
||||
f"Output cost mismatch: got {output_cost}, expected {expected_output_cost}"
|
||||
)
|
||||
|
||||
|
||||
AZURE_GPT_5_6_MAP_KEYS = (
|
||||
|
|
@ -1889,6 +1820,7 @@ def test_azure_gpt_5_6_rates_match_azure_price_page(_local_model_cost_map, model
|
|||
for key in token_cost_keys:
|
||||
assert entry[key] == pytest.approx(global_entry[key] * 1.1), key
|
||||
|
||||
|
||||
def test_vertex_regional_deployment_costs_uplift_over_global(monkeypatch):
|
||||
"""
|
||||
Regression for https://github.com/BerriAI/litellm/issues/34393: two Vertex
|
||||
|
|
@ -1971,7 +1903,6 @@ def test_cost_discount_vertex_ai(monkeypatch):
|
|||
from litellm import completion_cost
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
# Create mock response (use a model that exists in model_prices_and_context_window.json)
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
|
|
@ -2000,7 +1931,6 @@ def test_cost_discount_vertex_ai(monkeypatch):
|
|||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
|
||||
|
||||
# Verify discount is applied (5% off means 95% of original cost)
|
||||
expected_cost = cost_without_discount * 0.95
|
||||
assert cost_with_discount == pytest.approx(expected_cost, rel=1e-9)
|
||||
|
|
@ -2018,7 +1948,6 @@ def test_cost_discount_not_applied_to_other_providers(monkeypatch):
|
|||
from litellm import completion_cost
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
# Create mock response for OpenAI
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
|
|
@ -2047,7 +1976,6 @@ def test_cost_discount_not_applied_to_other_providers(monkeypatch):
|
|||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
|
||||
# Costs should be the same (no discount applied to OpenAI)
|
||||
assert cost_with_selective_discount == cost_without_discount
|
||||
|
||||
|
|
@ -2063,7 +1991,6 @@ def test_cost_margin_percentage(monkeypatch):
|
|||
from litellm import completion_cost
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
# Create mock response
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
|
|
@ -2092,7 +2019,6 @@ def test_cost_margin_percentage(monkeypatch):
|
|||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
|
||||
# Verify margin is applied (10% margin means 110% of original cost)
|
||||
expected_cost = cost_without_margin * 1.10
|
||||
assert cost_with_margin == pytest.approx(expected_cost, rel=1e-9)
|
||||
|
|
@ -2110,7 +2036,6 @@ def test_cost_margin_fixed_amount(monkeypatch):
|
|||
from litellm import completion_cost
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
# Create mock response
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
|
|
@ -2139,7 +2064,6 @@ def test_cost_margin_fixed_amount(monkeypatch):
|
|||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
|
||||
# Verify fixed margin is applied
|
||||
expected_cost = cost_without_margin + 0.001
|
||||
assert cost_with_margin == pytest.approx(expected_cost, rel=1e-9)
|
||||
|
|
@ -2157,7 +2081,6 @@ def test_cost_margin_combined(monkeypatch):
|
|||
from litellm import completion_cost
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
# Create mock response
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
|
|
@ -2177,9 +2100,7 @@ def test_cost_margin_combined(monkeypatch):
|
|||
)
|
||||
|
||||
# Set 8% margin + $0.0005 fixed for openai
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {
|
||||
"openai": {"percentage": 0.08, "fixed_amount": 0.0005}
|
||||
})
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"percentage": 0.08, "fixed_amount": 0.0005}})
|
||||
|
||||
# Calculate cost with margin
|
||||
cost_with_margin = completion_cost(
|
||||
|
|
@ -2188,7 +2109,6 @@ def test_cost_margin_combined(monkeypatch):
|
|||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
|
||||
# Verify combined margin is applied
|
||||
expected_cost = cost_without_margin * 1.08 + 0.0005
|
||||
assert cost_with_margin == pytest.approx(expected_cost, rel=1e-9)
|
||||
|
|
@ -2206,7 +2126,6 @@ def test_cost_margin_global(monkeypatch):
|
|||
from litellm import completion_cost
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
# Create mock response
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
|
|
@ -2235,7 +2154,6 @@ def test_cost_margin_global(monkeypatch):
|
|||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
|
||||
# Verify global margin is applied
|
||||
expected_cost = cost_without_margin * 1.05
|
||||
assert cost_with_global_margin == pytest.approx(expected_cost, rel=1e-9)
|
||||
|
|
@ -2253,7 +2171,6 @@ def test_cost_margin_provider_overrides_global(monkeypatch):
|
|||
from litellm import completion_cost
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
# Create mock response
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
|
|
@ -2282,16 +2199,13 @@ def test_cost_margin_provider_overrides_global(monkeypatch):
|
|||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
|
||||
# Verify provider-specific margin is used (not global)
|
||||
expected_cost = cost_without_margin * 1.10 # 10% from provider, not 5% from global
|
||||
assert cost_with_provider_margin == pytest.approx(expected_cost, rel=1e-9)
|
||||
|
||||
print("✓ Cost margin provider override test passed:")
|
||||
print(f" - Original cost: ${cost_without_margin:.6f}")
|
||||
print(
|
||||
f" - Cost with provider margin (10%, overrides 5% global): ${cost_with_provider_margin:.6f}"
|
||||
)
|
||||
print(f" - Cost with provider margin (10%, overrides 5% global): ${cost_with_provider_margin:.6f}")
|
||||
print(f" - Margin added: ${cost_with_provider_margin - cost_without_margin:.6f}")
|
||||
|
||||
|
||||
|
|
@ -2302,7 +2216,6 @@ def test_cost_margin_with_discount(monkeypatch):
|
|||
from litellm import completion_cost
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
# Create mock response
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
|
|
@ -2333,7 +2246,6 @@ def test_cost_margin_with_discount(monkeypatch):
|
|||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
|
||||
# Verify: discount applied first, then margin
|
||||
# Base cost -> discount: base * 0.95 -> margin: (base * 0.95) * 1.10
|
||||
expected_cost = base_cost * 0.95 * 1.10
|
||||
|
|
@ -2371,9 +2283,7 @@ def test_azure_image_generation_cost_calculator():
|
|||
size=None,
|
||||
usage=ImageUsage(
|
||||
input_tokens=0,
|
||||
input_tokens_details=ImageUsageInputTokensDetails(
|
||||
image_tokens=0, text_tokens=0
|
||||
),
|
||||
input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0),
|
||||
output_tokens=0,
|
||||
total_tokens=0,
|
||||
),
|
||||
|
|
@ -2403,7 +2313,6 @@ def test_completion_cost_extracts_service_tier_from_response(_local_model_cost_m
|
|||
"""Test that completion_cost extracts service_tier from completion_response object."""
|
||||
from litellm import completion_cost
|
||||
|
||||
|
||||
# Test with gpt-5-nano which has flex pricing
|
||||
model = "gpt-5-nano"
|
||||
|
||||
|
|
@ -2444,23 +2353,18 @@ def test_completion_cost_extracts_service_tier_from_response(_local_model_cost_m
|
|||
assert flex_cost < standard_cost, "Flex cost should be less than standard cost"
|
||||
|
||||
flex_ratio = flex_cost / standard_cost
|
||||
assert (
|
||||
0.45 <= flex_ratio <= 0.55
|
||||
), f"Flex pricing should be ~50% of standard, got {flex_ratio:.2f}"
|
||||
assert 0.45 <= flex_ratio <= 0.55, f"Flex pricing should be ~50% of standard, got {flex_ratio:.2f}"
|
||||
|
||||
|
||||
def test_completion_cost_extracts_service_tier_from_usage(_local_model_cost_map):
|
||||
"""Test that completion_cost extracts service_tier from usage object."""
|
||||
from litellm import completion_cost
|
||||
|
||||
|
||||
# Test with gpt-5-nano which has flex pricing
|
||||
model = "gpt-5-nano"
|
||||
|
||||
# Create usage object with service_tier
|
||||
usage_with_service_tier = Usage(
|
||||
prompt_tokens=1000, completion_tokens=500, total_tokens=1500
|
||||
)
|
||||
usage_with_service_tier = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
|
||||
# Set service_tier as an attribute on the usage object
|
||||
setattr(usage_with_service_tier, "service_tier", "flex")
|
||||
|
||||
|
|
@ -2478,9 +2382,7 @@ def test_completion_cost_extracts_service_tier_from_usage(_local_model_cost_map)
|
|||
)
|
||||
|
||||
# Create usage object without service_tier
|
||||
usage_without_service_tier = Usage(
|
||||
prompt_tokens=1000, completion_tokens=500, total_tokens=1500
|
||||
)
|
||||
usage_without_service_tier = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
|
||||
|
||||
# Create ModelResponse with usage without service_tier
|
||||
response_standard = ModelResponse(
|
||||
|
|
@ -2501,16 +2403,13 @@ def test_completion_cost_extracts_service_tier_from_usage(_local_model_cost_map)
|
|||
assert flex_cost < standard_cost, "Flex cost should be less than standard cost"
|
||||
|
||||
flex_ratio = flex_cost / standard_cost
|
||||
assert (
|
||||
0.45 <= flex_ratio <= 0.55
|
||||
), f"Flex pricing should be ~50% of standard, got {flex_ratio:.2f}"
|
||||
assert 0.45 <= flex_ratio <= 0.55, f"Flex pricing should be ~50% of standard, got {flex_ratio:.2f}"
|
||||
|
||||
|
||||
def test_completion_cost_service_tier_priority(_local_model_cost_map):
|
||||
"""Test that service_tier extraction follows priority: optional_params > completion_response > usage."""
|
||||
from litellm import completion_cost
|
||||
|
||||
|
||||
# Test with gpt-5-nano which has flex pricing
|
||||
model = "gpt-5-nano"
|
||||
|
||||
|
|
@ -2559,16 +2458,13 @@ def test_completion_cost_service_tier_priority(_local_model_cost_map):
|
|||
assert cost_from_usage > 0, "Cost from usage should be greater than 0"
|
||||
|
||||
# Costs should be similar (all using flex)
|
||||
assert (
|
||||
abs(cost_from_params - cost_from_usage) < 1e-6
|
||||
), "Costs from params and usage should be similar (both flex)"
|
||||
assert abs(cost_from_params - cost_from_usage) < 1e-6, "Costs from params and usage should be similar (both flex)"
|
||||
|
||||
|
||||
def test_completion_cost_service_tier_for_bedrock(_local_model_cost_map):
|
||||
"""Test that Bedrock cost calculation applies service_tier-specific pricing."""
|
||||
from litellm import completion_cost
|
||||
|
||||
|
||||
model = "bedrock/us-east-1/test-bedrock-service-tier-cost-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
|
|
@ -2624,7 +2520,6 @@ def test_completion_cost_service_tier_for_anthropic(_local_model_cost_map):
|
|||
from litellm import completion_cost
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
|
||||
model = "claude-test-service-tier-cost-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
|
|
@ -2677,7 +2572,6 @@ def test_completion_cost_anthropic_auto_tier_uses_served_priority_rate(_local_mo
|
|||
from litellm import completion_cost
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
|
||||
model = "claude-test-auto-tier-cost-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
|
|
@ -2771,7 +2665,6 @@ def test_completion_cost_non_string_service_tier_defers_to_served_tier(_local_mo
|
|||
from litellm import completion_cost
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
|
||||
model = "claude-test-non-string-tier-cost-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
|
|
@ -2821,7 +2714,6 @@ def test_completion_cost_non_string_response_service_tier_defers_to_served_tier(
|
|||
from litellm import completion_cost
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
|
||||
model = "claude-test-response-non-string-tier-cost-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
|
|
@ -2844,9 +2736,7 @@ def test_completion_cost_non_string_response_service_tier_defers_to_served_tier(
|
|||
},
|
||||
reasoning_content=None,
|
||||
)
|
||||
response = ModelResponse(
|
||||
usage=usage, model=model, service_tier={"name": "priority"}
|
||||
)
|
||||
response = ModelResponse(usage=usage, model=model, service_tier={"name": "priority"})
|
||||
|
||||
cost = completion_cost(
|
||||
completion_response=response,
|
||||
|
|
@ -2869,7 +2759,6 @@ def test_completion_cost_non_string_usage_service_tier_prices_standard(_local_mo
|
|||
"""
|
||||
from litellm import completion_cost
|
||||
|
||||
|
||||
model = "claude-test-usage-non-string-tier-cost-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
|
|
@ -2916,7 +2805,6 @@ def test_anthropic_cost_per_token_prices_cache_at_served_tier_with_multiplier(_l
|
|||
)
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
|
||||
|
||||
model = "claude-test-priority-cache-fast-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
|
|
@ -2942,9 +2830,7 @@ def test_anthropic_cost_per_token_prices_cache_at_served_tier_with_multiplier(_l
|
|||
)
|
||||
usage.speed = "fast"
|
||||
|
||||
prompt_cost, completion_cost = anthropic_cost_per_token(
|
||||
model=model, usage=usage, service_tier="priority"
|
||||
)
|
||||
prompt_cost, completion_cost = anthropic_cost_per_token(model=model, usage=usage, service_tier="priority")
|
||||
|
||||
expected_prompt = ((1000 - 200) * 6e-6 + 200 * 0.6e-6) * 2
|
||||
expected_completion = 500 * 30e-6 * 2
|
||||
|
|
@ -3074,9 +2960,7 @@ def test_anthropic_fast_multiplier_only_on_models_with_fast_mode(_local_model_co
|
|||
"model",
|
||||
["claude-sonnet-4-6", "claude-mythos-5", "claude-mythos-preview"],
|
||||
)
|
||||
def test_anthropic_us_data_residency_uplift_on_claude_4_6_and_later_models(
|
||||
_local_model_cost_map, monkeypatch, model
|
||||
):
|
||||
def test_anthropic_us_data_residency_uplift_on_claude_4_6_and_later_models(_local_model_cost_map, monkeypatch, model):
|
||||
"""
|
||||
Anthropic bills every Claude 4.6+ model served with ``inference_geo="us"`` at
|
||||
1.1x, and echoes that geo back in the response usage, so each of these real
|
||||
|
|
@ -3141,29 +3025,27 @@ def test_gemini_cache_tokens_details_no_negative_values():
|
|||
usage = VertexGeminiConfig._calculate_usage(completion_response)
|
||||
|
||||
# Text tokens should be non-cached text only: 9402 - 9393 = 9
|
||||
assert (
|
||||
usage.prompt_tokens_details.text_tokens == 9
|
||||
), f"Expected text_tokens=9, got {usage.prompt_tokens_details.text_tokens}"
|
||||
assert usage.prompt_tokens_details.text_tokens == 9, (
|
||||
f"Expected text_tokens=9, got {usage.prompt_tokens_details.text_tokens}"
|
||||
)
|
||||
|
||||
# Image tokens should be non-cached image only: 258 - 258 = 0
|
||||
assert (
|
||||
usage.prompt_tokens_details.image_tokens == 0
|
||||
), f"Expected image_tokens=0, got {usage.prompt_tokens_details.image_tokens}"
|
||||
assert usage.prompt_tokens_details.image_tokens == 0, (
|
||||
f"Expected image_tokens=0, got {usage.prompt_tokens_details.image_tokens}"
|
||||
)
|
||||
|
||||
# Total cached should match
|
||||
assert (
|
||||
usage.prompt_tokens_details.cached_tokens == 9651
|
||||
), f"Expected cached_tokens=9651, got {usage.prompt_tokens_details.cached_tokens}"
|
||||
assert usage.prompt_tokens_details.cached_tokens == 9651, (
|
||||
f"Expected cached_tokens=9651, got {usage.prompt_tokens_details.cached_tokens}"
|
||||
)
|
||||
|
||||
# MOST IMPORTANT: text_tokens should NEVER be negative
|
||||
assert (
|
||||
usage.prompt_tokens_details.text_tokens >= 0
|
||||
), f"BUG: text_tokens is negative ({usage.prompt_tokens_details.text_tokens})! This was the issue in #18750"
|
||||
|
||||
print(
|
||||
"✅ Issue #18750 fix verified: text_tokens is correctly calculated and non-negative"
|
||||
assert usage.prompt_tokens_details.text_tokens >= 0, (
|
||||
f"BUG: text_tokens is negative ({usage.prompt_tokens_details.text_tokens})! This was the issue in #18750"
|
||||
)
|
||||
|
||||
print("✅ Issue #18750 fix verified: text_tokens is correctly calculated and non-negative")
|
||||
|
||||
|
||||
def test_gemini_without_cache_tokens_details():
|
||||
"""
|
||||
|
|
@ -3230,18 +3112,18 @@ def test_gemini_implicit_caching_cost_calculation():
|
|||
usage = VertexGeminiConfig._calculate_usage(completion_response)
|
||||
|
||||
# Verify parsing
|
||||
assert (
|
||||
usage.cache_read_input_tokens == 8000
|
||||
), f"cache_read_input_tokens should be 8000, got {usage.cache_read_input_tokens}"
|
||||
assert (
|
||||
usage.prompt_tokens_details.cached_tokens == 8000
|
||||
), f"cached_tokens should be 8000, got {usage.prompt_tokens_details.cached_tokens}"
|
||||
assert usage.cache_read_input_tokens == 8000, (
|
||||
f"cache_read_input_tokens should be 8000, got {usage.cache_read_input_tokens}"
|
||||
)
|
||||
assert usage.prompt_tokens_details.cached_tokens == 8000, (
|
||||
f"cached_tokens should be 8000, got {usage.prompt_tokens_details.cached_tokens}"
|
||||
)
|
||||
|
||||
# CRITICAL: text_tokens should be (10000 - 8000) = 2000, NOT 10000
|
||||
# This is the fix for issue #16341
|
||||
assert (
|
||||
usage.prompt_tokens_details.text_tokens == 2000
|
||||
), f"text_tokens should be 2000 (10000 - 8000), got {usage.prompt_tokens_details.text_tokens}"
|
||||
assert usage.prompt_tokens_details.text_tokens == 2000, (
|
||||
f"text_tokens should be 2000 (10000 - 8000), got {usage.prompt_tokens_details.text_tokens}"
|
||||
)
|
||||
|
||||
# Verify cost calculation uses cached token pricing
|
||||
response = ModelResponse(
|
||||
|
|
@ -3279,9 +3161,7 @@ def test_gemini_implicit_caching_cost_calculation():
|
|||
f"Cached tokens may not be using reduced pricing."
|
||||
)
|
||||
|
||||
print(
|
||||
"✅ Issue #16341 fix verified: Gemini implicit caching cost calculated correctly"
|
||||
)
|
||||
print("✅ Issue #16341 fix verified: Gemini implicit caching cost calculated correctly")
|
||||
|
||||
|
||||
def test_additional_costs_only_for_azure_ai(_local_model_cost_map):
|
||||
|
|
@ -3295,7 +3175,6 @@ def test_additional_costs_only_for_azure_ai(_local_model_cost_map):
|
|||
"""
|
||||
from litellm.cost_calculator import _get_additional_costs
|
||||
|
||||
|
||||
# Non-azure_ai providers should return None
|
||||
result = _get_additional_costs(
|
||||
model="gpt-4o",
|
||||
|
|
@ -3438,12 +3317,7 @@ def test_custom_pricing_applies_cache_creation_input_cost_via_prompt_details():
|
|||
},
|
||||
)
|
||||
|
||||
expected = (
|
||||
(4000 - 1000 - 500) * 0.0000025
|
||||
+ 1000 * 0.00000025
|
||||
+ 500 * 0.000003125
|
||||
+ 100 * 0.000015
|
||||
)
|
||||
expected = (4000 - 1000 - 500) * 0.0000025 + 1000 * 0.00000025 + 500 * 0.000003125 + 100 * 0.000015
|
||||
|
||||
assert cost == pytest.approx(expected)
|
||||
|
||||
|
|
@ -3488,9 +3362,7 @@ def test_custom_pricing_applies_cache_creation_input_cost_via_cache_write_tokens
|
|||
},
|
||||
)
|
||||
|
||||
expected_prompt = (
|
||||
(4000 - 1000 - 500) * 0.0000025 + 1000 * 0.00000025 + 500 * 0.000003125
|
||||
)
|
||||
expected_prompt = (4000 - 1000 - 500) * 0.0000025 + 1000 * 0.00000025 + 500 * 0.000003125
|
||||
expected_completion = 100 * 0.000015
|
||||
|
||||
assert prompt_cost == pytest.approx(expected_prompt)
|
||||
|
|
@ -3530,10 +3402,7 @@ def test_extract_cache_read_tokens_zero_when_missing():
|
|||
|
||||
assert _extract_cache_read_tokens({}) == 0
|
||||
assert _extract_cache_read_tokens({"cache_read_input_tokens": None}) == 0
|
||||
assert (
|
||||
_extract_cache_read_tokens({"prompt_tokens_details": {"cached_tokens": None}})
|
||||
== 0
|
||||
)
|
||||
assert _extract_cache_read_tokens({"prompt_tokens_details": {"cached_tokens": None}}) == 0
|
||||
|
||||
|
||||
def test_extract_cache_creation_tokens_anthropic_top_level():
|
||||
|
|
@ -3575,12 +3444,7 @@ def test_extract_cache_creation_tokens_zero_when_missing():
|
|||
|
||||
assert _extract_cache_creation_tokens({}) == 0
|
||||
assert _extract_cache_creation_tokens({"cache_creation_input_tokens": None}) == 0
|
||||
assert (
|
||||
_extract_cache_creation_tokens(
|
||||
{"prompt_tokens_details": {"cache_write_tokens": None}}
|
||||
)
|
||||
== 0
|
||||
)
|
||||
assert _extract_cache_creation_tokens({"prompt_tokens_details": {"cache_write_tokens": None}}) == 0
|
||||
|
||||
|
||||
def test_custom_pricing_anthropic_style_cache_tokens_not_double_counted():
|
||||
|
|
@ -3707,7 +3571,6 @@ def test_completion_cost_logs_reasoning_and_cache_breakdown(_local_model_cost_ma
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.types.utils import Choices, CompletionTokensDetailsWrapper, Message
|
||||
|
||||
|
||||
logging_obj = Logging(
|
||||
model="gemini-2.5-flash",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
|
|
@ -3734,12 +3597,8 @@ def test_completion_cost_logs_reasoning_and_cache_breakdown(_local_model_cost_ma
|
|||
prompt_tokens=209,
|
||||
completion_tokens=3996,
|
||||
total_tokens=4205,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(
|
||||
reasoning_tokens=3114, text_tokens=882
|
||||
),
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
cached_tokens=100, text_tokens=109
|
||||
),
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=3114, text_tokens=882),
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100, text_tokens=109),
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -3805,9 +3664,7 @@ def test_completion_cost_logs_the_rates_it_billed_at(monkeypatch):
|
|||
assert rates is not None
|
||||
assert rates.input_cost_per_token == pytest.approx(6e-6)
|
||||
assert rates.cache_read_input_token_cost == pytest.approx(6e-7)
|
||||
assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(
|
||||
100_000 * rates.cache_read_input_token_cost
|
||||
)
|
||||
assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(100_000 * rates.cache_read_input_token_cost)
|
||||
assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(1_000 * rates.output_cost_per_token)
|
||||
|
||||
|
||||
|
|
@ -3978,11 +3835,7 @@ def test_completion_cost_bills_interactions_api_response():
|
|||
cost = completion_cost(completion_response=response, custom_llm_provider="gemini")
|
||||
|
||||
reasoning_rate = model_info.get("output_cost_per_reasoning_token") or model_info["output_cost_per_token"]
|
||||
expected = (
|
||||
100 * model_info["input_cost_per_token"]
|
||||
+ 50 * model_info["output_cost_per_token"]
|
||||
+ 25 * reasoning_rate
|
||||
)
|
||||
expected = 100 * model_info["input_cost_per_token"] + 50 * model_info["output_cost_per_token"] + 25 * reasoning_rate
|
||||
assert cost == pytest.approx(expected)
|
||||
assert cost > 0
|
||||
|
||||
|
|
@ -4153,7 +4006,9 @@ def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(_local_model_
|
|||
assert cost == pytest.approx(3 * 4e-6 + 4014 * 4e-7 + 5 * 2e-5, rel=1e-9)
|
||||
|
||||
|
||||
def _together_chat_response(model: str, prompt_tokens: int, completion_tokens: int, cached_tokens: int) -> ModelResponse:
|
||||
def _together_chat_response(
|
||||
model: str, prompt_tokens: int, completion_tokens: int, cached_tokens: int
|
||||
) -> ModelResponse:
|
||||
return ModelResponse(
|
||||
id="chatcmpl-together-cache",
|
||||
choices=[{"finish_reason": "stop", "index": 0, "message": {"content": "acknowledged", "role": "assistant"}}],
|
||||
|
|
@ -4221,6 +4076,8 @@ def test_completion_cost_together_metadata_only_model_still_uses_size_bucket(_lo
|
|||
)
|
||||
|
||||
assert cost == pytest.approx((23 + 15) * 8e-07, rel=1e-9)
|
||||
|
||||
|
||||
def test_select_model_name_strips_unregistered_alias_prefix(_local_model_cost_map):
|
||||
"""A router-facing model_name alias containing "/" whose leading segment is NOT a
|
||||
registered provider must not be double-prefixed into a non-existent cost key.
|
||||
|
|
@ -4461,9 +4318,7 @@ def test_every_one_hour_cache_write_rate_is_double_its_input_rate():
|
|||
"""Guard against pasting one model's 1h cache-write price onto another: every provider
|
||||
LiteLLM tracks (Anthropic, Bedrock, Vertex, Azure) publishes the 1h write at 2x input."""
|
||||
|
||||
cost_map = json.loads(
|
||||
(Path(__file__).parents[2] / "model_prices_and_context_window.json").read_text()
|
||||
)
|
||||
cost_map = json.loads((Path(__file__).parents[2] / "model_prices_and_context_window.json").read_text())
|
||||
one_hour_prefix = "cache_creation_input_token_cost_above_1hr"
|
||||
deviations = {
|
||||
(name, key): (entry["input_cost_per_token" + key[len(one_hour_prefix) :]], entry[key])
|
||||
|
|
@ -4669,9 +4524,7 @@ def test_batch_cost_calculator_gpt_6_astra_bills_half_the_standard_rate(_local_m
|
|||
|
||||
usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
|
||||
|
||||
prompt_cost, completion_cost = batch_cost_calculator(
|
||||
usage=usage, model="gpt-6-astra", custom_llm_provider="openai"
|
||||
)
|
||||
prompt_cost, completion_cost = batch_cost_calculator(usage=usage, model="gpt-6-astra", custom_llm_provider="openai")
|
||||
|
||||
assert prompt_cost == pytest.approx(1000 * 5e-6)
|
||||
assert completion_cost == pytest.approx(500 * 2.5e-5)
|
||||
|
|
@ -4772,6 +4625,67 @@ def test_collect_and_combine_realtime_usage_stores_partitioned_text_tokens() ->
|
|||
assert combined.completion_tokens_details.audio_tokens == 0
|
||||
|
||||
|
||||
def test_realtime_combine_sums_nested_cached_tokens_details():
|
||||
results: OpenAIRealtimeStreamList = [
|
||||
{
|
||||
"type": "response.done",
|
||||
"response": {
|
||||
"usage": {
|
||||
"input_tokens": 283,
|
||||
"output_tokens": 0,
|
||||
"total_tokens": 283,
|
||||
"input_token_details": {
|
||||
"text_tokens": 116,
|
||||
"audio_tokens": 167,
|
||||
"cached_tokens": 192,
|
||||
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "response.done",
|
||||
"response": {
|
||||
"usage": {
|
||||
"input_tokens": 150,
|
||||
"output_tokens": 0,
|
||||
"total_tokens": 150,
|
||||
"input_token_details": {
|
||||
"text_tokens": 50,
|
||||
"audio_tokens": 100,
|
||||
"cached_tokens": 100,
|
||||
"cached_tokens_details": {"audio_tokens": 100},
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(
|
||||
results=results,
|
||||
)
|
||||
|
||||
assert combined.prompt_tokens_details is not None
|
||||
assert combined.prompt_tokens_details.cached_tokens == 292
|
||||
assert combined.prompt_tokens_details.cached_tokens_details is not None
|
||||
assert combined.prompt_tokens_details.cached_tokens_details.audio_tokens == 228
|
||||
assert combined.prompt_tokens_details.cached_tokens_details.text_tokens == 64
|
||||
assert combined.prompt_tokens_details.cached_tokens_details.image_tokens is None
|
||||
|
||||
|
||||
def test_usage_without_cached_tokens_details_omits_key():
|
||||
usage = Usage(
|
||||
prompt_tokens=10,
|
||||
completion_tokens=5,
|
||||
total_tokens=15,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=10),
|
||||
)
|
||||
|
||||
dumped = usage.prompt_tokens_details.model_dump()
|
||||
assert "cached_tokens_details" not in dumped
|
||||
assert "cached_tokens_details" not in usage.prompt_tokens_details.model_dump_json()
|
||||
|
||||
|
||||
UNMAPPED_OCR_MODEL: Final = "azure_ai/some-unmapped-ocr-model-for-testing"
|
||||
MAPPED_OCR_MODEL: Final = "mistral/mistral-ocr-4-0"
|
||||
|
||||
|
|
|
|||
|
|
@ -6385,3 +6385,11 @@ def test_completion_finishes_response_metadata_before_handing_the_response_to_th
|
|||
assert snapshot["litellm_call_id"]
|
||||
assert snapshot["response_cost"] is not None
|
||||
assert snapshot["api_base"]
|
||||
|
||||
|
||||
def test_get_model_info_carries_cache_read_input_audio_token_cost(monkeypatch):
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
info = litellm.get_model_info("gpt-realtime-2.1-mini", custom_llm_provider="openai")
|
||||
assert info["cache_read_input_audio_token_cost"] == 3e-07
|
||||
assert info["cache_read_input_token_cost"] == 6e-08
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue