scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-15 09:12:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 311

fix: keep top-level cache token counts with partial usage details

Browse source 
OpenAI-compatible responses can send cache_read_input_tokens alongside a
partial prompt_tokens_details object. Prefer details only when they
explicitly include cached_tokens so a missing key no longer wipes the
top-level cache count to 0 (#39505).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Alphaxiaoteng 2026-09-03 16:23:39 +08:00
parent ed9d29a9b4
commit a05c6e6f18
2 changed files with 65 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
litellm/cost_calculator.py
View file

@ -1343,7 +1343,26 @@ def completion_cost(
and _usage["prompt_tokens_details"]
):
prompt_tokens_details = _usage.get("prompt_tokens_details") or {}
cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0)
# Only prefer details when they actually carry cached_tokens.
# OpenAI-compatible providers often send both top-level
# cache_read_input_tokens and a partial prompt_tokens_details
# object (e.g. audio_tokens only). Using `.get(..., 0)` would
# wipe the top-level cache count with a default of 0 (#39505).
if (
isinstance(prompt_tokens_details, dict)
and "cached_tokens" in prompt_tokens_details
and prompt_tokens_details["cached_tokens"] is not None
):
cache_read_input_tokens = prompt_tokens_details["cached_tokens"]
if (
isinstance(prompt_tokens_details, dict)
and not cache_creation_input_tokens
):
cache_write_tokens = prompt_tokens_details.get(
"cache_write_tokens"
) or prompt_tokens_details.get("cache_creation_tokens")
if isinstance(cache_write_tokens, int):
cache_creation_input_tokens = cache_write_tokens
total_time = getattr(completion_response, "_response_ms", 0)

45
tests/local_testing/test_completion_cost.py
View file

@ -2928,3 +2928,48 @@ def test_batch_cost_calculator():
cost = completion_cost(**args)
assert cost > 0
def test_openai_compatible_top_level_cache_tokens_survive_partial_details(monkeypatch):
"""Regression for #39505.
OpenAI-compatible providers may return top-level cache_read_input_tokens
together with a partial prompt_tokens_details object that omits
cached_tokens. The details branch must not default-wipe the top-level
cache count to 0.
"""
captured = {}
def _fake_cost_per_token(**kwargs):
captured.update(kwargs)
return 0.01, 0.02
monkeypatch.setattr(litellm.cost_calculator, "cost_per_token", _fake_cost_per_token)
monkeypatch.setattr(litellm, "cost_per_token", _fake_cost_per_token)
response = {
"id": "chatcmpl-test-cache",
"object": "chat.completion",
"created": 1,
"model": "gpt-4o",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": 100,
"completion_tokens": 10,
"total_tokens": 110,
"cache_read_input_tokens": 80,
"cache_creation_input_tokens": 5,
"prompt_tokens_details": {"audio_tokens": 0},
},
}
cost = completion_cost(model="gpt-4o", completion_response=response)
assert cost == 0.03
assert captured.get("cache_read_input_tokens") == 80
assert captured.get("cache_creation_input_tokens") == 5