[Fix] Cached tokens always showing zero in UI for OpenAI models

OpenAI returns cached tokens nested in prompt_tokens_details.cached_tokens,
while Anthropic/DeepSeek return them as top-level cache_read_input_tokens.
The daily spend writer only checked the top-level field, so OpenAI cached
tokens were never persisted to the database.

Added normalization in get_usage_as_dict() to promote nested cached_tokens
to the top-level cache_read_input_tokens field when it's missing.
This commit is contained in:
Yuneng Jiang 2026-04-07 13:50:03 -07:00
parent f3bc20056d
commit 30459e949b
No known key found for this signature in database
2 changed files with 151 additions and 5 deletions

View file

@ -4871,28 +4871,63 @@ class StandardLoggingPayloadSetup:
"""
_empty: dict = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
if combined_usage_object is not None:
return combined_usage_object.model_dump()
result = combined_usage_object.model_dump()
return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(result)
if not response_obj:
return _empty
_raw = response_obj.get("usage", None)
if _raw is None:
return _empty
if isinstance(_raw, ResponseAPIUsage):
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
_raw
).model_dump()
return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(result)
if isinstance(_raw, dict):
if ResponseAPILoggingUtils._is_response_api_usage(_raw):
return (
result = (
ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
_raw
).model_dump()
)
return _raw
return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(result)
return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(_raw)
if isinstance(_raw, Usage):
return _raw.model_dump()
result = _raw.model_dump()
return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(result)
return _empty
@staticmethod
def _normalize_usage_cache_tokens(usage_dict: dict) -> dict:
"""
Promote prompt_tokens_details.cached_tokens cache_read_input_tokens
and prompt_tokens_details.cache_creation_tokens cache_creation_input_tokens
when the top-level fields are missing or zero.
OpenAI returns cached tokens nested in prompt_tokens_details, while
Anthropic/DeepSeek return them as top-level fields. The daily spend
tables expect cache_read_input_tokens at the top level, so this
normalization ensures all providers' cached tokens are tracked.
"""
prompt_details = usage_dict.get("prompt_tokens_details")
if not prompt_details:
return usage_dict
if isinstance(prompt_details, dict):
cached = prompt_details.get("cached_tokens")
creation = prompt_details.get("cache_creation_tokens")
else:
cached = getattr(prompt_details, "cached_tokens", None)
creation = getattr(prompt_details, "cache_creation_tokens", None)
if cached and not usage_dict.get("cache_read_input_tokens"):
usage_dict["cache_read_input_tokens"] = cached
if creation and not usage_dict.get("cache_creation_input_tokens"):
usage_dict["cache_creation_input_tokens"] = creation
return usage_dict
@staticmethod
def get_model_cost_information(
base_model: Optional[str],

View file

@ -1522,6 +1522,117 @@ def test_get_usage_as_dict():
assert result == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
def test_normalize_usage_cache_tokens_openai():
"""
Test that prompt_tokens_details.cached_tokens from OpenAI responses
gets promoted to cache_read_input_tokens at the top level.
"""
from litellm.litellm_core_utils.litellm_logging import \
StandardLoggingPayloadSetup
# OpenAI-style: cached_tokens nested in prompt_tokens_details, no top-level cache_read_input_tokens
usage = {
"prompt_tokens": 100,
"completion_tokens": 50,
"total_tokens": 150,
"prompt_tokens_details": {"cached_tokens": 30, "audio_tokens": 0},
}
result = StandardLoggingPayloadSetup._normalize_usage_cache_tokens(usage)
assert result["cache_read_input_tokens"] == 30
def test_normalize_usage_cache_tokens_anthropic_not_overwritten():
"""
Test that existing top-level cache_read_input_tokens (Anthropic) is NOT
overwritten by prompt_tokens_details.cached_tokens.
"""
from litellm.litellm_core_utils.litellm_logging import \
StandardLoggingPayloadSetup
usage = {
"prompt_tokens": 100,
"completion_tokens": 50,
"total_tokens": 150,
"cache_read_input_tokens": 40,
"prompt_tokens_details": {"cached_tokens": 40},
}
result = StandardLoggingPayloadSetup._normalize_usage_cache_tokens(usage)
assert result["cache_read_input_tokens"] == 40
def test_normalize_usage_cache_tokens_no_details():
"""
Test normalization is a no-op when prompt_tokens_details is absent.
"""
from litellm.litellm_core_utils.litellm_logging import \
StandardLoggingPayloadSetup
usage = {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}
result = StandardLoggingPayloadSetup._normalize_usage_cache_tokens(usage)
assert "cache_read_input_tokens" not in result
def test_normalize_usage_cache_creation_tokens():
"""
Test that prompt_tokens_details.cache_creation_tokens gets promoted
to cache_creation_input_tokens at the top level.
"""
from litellm.litellm_core_utils.litellm_logging import \
StandardLoggingPayloadSetup
usage = {
"prompt_tokens": 100,
"completion_tokens": 50,
"total_tokens": 150,
"prompt_tokens_details": {"cached_tokens": 30, "cache_creation_tokens": 70},
}
result = StandardLoggingPayloadSetup._normalize_usage_cache_tokens(usage)
assert result["cache_read_input_tokens"] == 30
assert result["cache_creation_input_tokens"] == 70
def test_get_usage_as_dict_openai_cached_tokens():
"""
End-to-end: get_usage_as_dict promotes OpenAI cached_tokens to
cache_read_input_tokens for the daily spend writer.
"""
from litellm.litellm_core_utils.litellm_logging import \
StandardLoggingPayloadSetup
response_obj = {
"usage": {
"prompt_tokens": 100,
"completion_tokens": 50,
"total_tokens": 150,
"prompt_tokens_details": {"cached_tokens": 25},
}
}
result = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj=response_obj)
assert result["cache_read_input_tokens"] == 25
def test_get_usage_as_dict_combined_usage_object_cached_tokens():
"""
End-to-end: get_usage_as_dict with combined_usage_object that has
prompt_tokens_details.cached_tokens promotes to cache_read_input_tokens.
"""
from litellm.litellm_core_utils.litellm_logging import \
StandardLoggingPayloadSetup
from litellm.types.utils import Usage
combined = Usage(
prompt_tokens=100,
completion_tokens=50,
total_tokens=150,
prompt_tokens_details={"cached_tokens": 35},
)
result = StandardLoggingPayloadSetup.get_usage_as_dict(
response_obj=None,
combined_usage_object=combined,
)
assert result["cache_read_input_tokens"] == 35
def test_append_system_prompt_messages():
"""
Test append_system_prompt_messages prepends system message from kwargs to messages list.