scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-15 23:31:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 165

fix(cost): cap nested cached modality counts at cached_tokens

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
shivam 2026-09-10 22:27:58 +00:00
parent 67fc9e4e3d
commit 5737cab258
3 changed files with 32 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/cost_calculator.py
View file

@ -2332,8 +2332,6 @@ def _combine_prompt_tokens_details(combined: Usage, usage: Usage) -> None:
if not hasattr(combined, "prompt_tokens_details") or not combined.prompt_tokens_details:
combined.prompt_tokens_details = PromptTokensDetailsWrapper()
# Check what keys exist in the model's prompt_tokens_details
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
for attr in _summable_prompt_token_fields(usage.prompt_tokens_details):
if (
hasattr(usage.prompt_tokens_details, attr)

16
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -806,9 +806,17 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
or None
)
cached_tokens_details: Final = getattr(usage.prompt_tokens_details, "cached_tokens_details", None)
cached_text_tokens: Final = _get_token_detail_value(cached_tokens_details, "text_tokens") or 0
cached_audio_tokens: Final = _get_token_detail_value(cached_tokens_details, "audio_tokens") or 0
cached_image_tokens: Final = _get_token_detail_value(cached_tokens_details, "image_tokens") or 0
cached_audio_tokens: Final = min(
_get_token_detail_value(cached_tokens_details, "audio_tokens") or 0, cache_hit_tokens
)
cached_text_tokens: Final = min(
_get_token_detail_value(cached_tokens_details, "text_tokens") or 0,
cache_hit_tokens - cached_audio_tokens,
)
cached_image_tokens: Final = min(
_get_token_detail_value(cached_tokens_details, "image_tokens") or 0,
cache_hit_tokens - cached_audio_tokens - cached_text_tokens,
)
text_tokens: Final = max(
(
cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None))
@ -852,7 +860,7 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
return PromptTokensDetailsResult(
cache_hit_tokens=cache_hit_tokens,
cache_hit_audio_tokens=min(cached_audio_tokens, cache_hit_tokens),
cache_hit_audio_tokens=cached_audio_tokens,
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
text_tokens=text_tokens,

20
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -5215,3 +5215,23 @@ def test_cached_audio_tokens_fall_back_to_cache_read_input_token_cost() -> None:
)
expected = 52 * 4e-6 + 64 * 5e-7 + 39 * 32e-6 + 128 * 5e-7
assert prompt_cost == pytest.approx(expected)
def test_cached_audio_tokens_capped_at_cached_tokens(_local_model_cost_map: None) -> None:
"""Nested cached_tokens_details exceeding cached_tokens must not over-subtract the audio bucket."""
usage = Usage(
prompt_tokens=283,
completion_tokens=0,
total_tokens=283,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=116,
audio_tokens=167,
cached_tokens=100,
cached_tokens_details={"audio_tokens": 128},
),
)
prompt_cost, _ = generic_cost_per_token(
model="gpt-realtime-2", usage=usage, custom_llm_provider="openai"
)
assert prompt_cost == pytest.approx(116 * 4e-6 + (167 - 100) * 32e-6 + 100 * 4e-7)