mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-11 22:51:28 +00:00
[Fix] Cached tokens always showing zero in UI for OpenAI models
OpenAI returns cached tokens nested in prompt_tokens_details.cached_tokens, while Anthropic/DeepSeek return them as top-level cache_read_input_tokens. The daily spend writer only checked the top-level field, so OpenAI cached tokens were never persisted to the database. Added normalization in get_usage_as_dict() to promote nested cached_tokens to the top-level cache_read_input_tokens field when it's missing.
This commit is contained in:
parent
f3bc20056d
commit
30459e949b
2 changed files with 151 additions and 5 deletions
|
|
@ -4871,28 +4871,63 @@ class StandardLoggingPayloadSetup:
|
|||
"""
|
||||
_empty: dict = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
|
||||
if combined_usage_object is not None:
|
||||
return combined_usage_object.model_dump()
|
||||
result = combined_usage_object.model_dump()
|
||||
return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(result)
|
||||
if not response_obj:
|
||||
return _empty
|
||||
_raw = response_obj.get("usage", None)
|
||||
if _raw is None:
|
||||
return _empty
|
||||
if isinstance(_raw, ResponseAPIUsage):
|
||||
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
|
||||
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
|
||||
_raw
|
||||
).model_dump()
|
||||
return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(result)
|
||||
if isinstance(_raw, dict):
|
||||
if ResponseAPILoggingUtils._is_response_api_usage(_raw):
|
||||
return (
|
||||
result = (
|
||||
ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
|
||||
_raw
|
||||
).model_dump()
|
||||
)
|
||||
return _raw
|
||||
return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(result)
|
||||
return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(_raw)
|
||||
if isinstance(_raw, Usage):
|
||||
return _raw.model_dump()
|
||||
result = _raw.model_dump()
|
||||
return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(result)
|
||||
return _empty
|
||||
|
||||
@staticmethod
|
||||
def _normalize_usage_cache_tokens(usage_dict: dict) -> dict:
|
||||
"""
|
||||
Promote prompt_tokens_details.cached_tokens → cache_read_input_tokens
|
||||
and prompt_tokens_details.cache_creation_tokens → cache_creation_input_tokens
|
||||
when the top-level fields are missing or zero.
|
||||
|
||||
OpenAI returns cached tokens nested in prompt_tokens_details, while
|
||||
Anthropic/DeepSeek return them as top-level fields. The daily spend
|
||||
tables expect cache_read_input_tokens at the top level, so this
|
||||
normalization ensures all providers' cached tokens are tracked.
|
||||
"""
|
||||
prompt_details = usage_dict.get("prompt_tokens_details")
|
||||
if not prompt_details:
|
||||
return usage_dict
|
||||
|
||||
if isinstance(prompt_details, dict):
|
||||
cached = prompt_details.get("cached_tokens")
|
||||
creation = prompt_details.get("cache_creation_tokens")
|
||||
else:
|
||||
cached = getattr(prompt_details, "cached_tokens", None)
|
||||
creation = getattr(prompt_details, "cache_creation_tokens", None)
|
||||
|
||||
if cached and not usage_dict.get("cache_read_input_tokens"):
|
||||
usage_dict["cache_read_input_tokens"] = cached
|
||||
|
||||
if creation and not usage_dict.get("cache_creation_input_tokens"):
|
||||
usage_dict["cache_creation_input_tokens"] = creation
|
||||
|
||||
return usage_dict
|
||||
|
||||
@staticmethod
|
||||
def get_model_cost_information(
|
||||
base_model: Optional[str],
|
||||
|
|
|
|||
|
|
@ -1522,6 +1522,117 @@ def test_get_usage_as_dict():
|
|||
assert result == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
|
||||
|
||||
|
||||
def test_normalize_usage_cache_tokens_openai():
|
||||
"""
|
||||
Test that prompt_tokens_details.cached_tokens from OpenAI responses
|
||||
gets promoted to cache_read_input_tokens at the top level.
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import \
|
||||
StandardLoggingPayloadSetup
|
||||
|
||||
# OpenAI-style: cached_tokens nested in prompt_tokens_details, no top-level cache_read_input_tokens
|
||||
usage = {
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 50,
|
||||
"total_tokens": 150,
|
||||
"prompt_tokens_details": {"cached_tokens": 30, "audio_tokens": 0},
|
||||
}
|
||||
result = StandardLoggingPayloadSetup._normalize_usage_cache_tokens(usage)
|
||||
assert result["cache_read_input_tokens"] == 30
|
||||
|
||||
|
||||
def test_normalize_usage_cache_tokens_anthropic_not_overwritten():
|
||||
"""
|
||||
Test that existing top-level cache_read_input_tokens (Anthropic) is NOT
|
||||
overwritten by prompt_tokens_details.cached_tokens.
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import \
|
||||
StandardLoggingPayloadSetup
|
||||
|
||||
usage = {
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 50,
|
||||
"total_tokens": 150,
|
||||
"cache_read_input_tokens": 40,
|
||||
"prompt_tokens_details": {"cached_tokens": 40},
|
||||
}
|
||||
result = StandardLoggingPayloadSetup._normalize_usage_cache_tokens(usage)
|
||||
assert result["cache_read_input_tokens"] == 40
|
||||
|
||||
|
||||
def test_normalize_usage_cache_tokens_no_details():
|
||||
"""
|
||||
Test normalization is a no-op when prompt_tokens_details is absent.
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import \
|
||||
StandardLoggingPayloadSetup
|
||||
|
||||
usage = {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}
|
||||
result = StandardLoggingPayloadSetup._normalize_usage_cache_tokens(usage)
|
||||
assert "cache_read_input_tokens" not in result
|
||||
|
||||
|
||||
def test_normalize_usage_cache_creation_tokens():
|
||||
"""
|
||||
Test that prompt_tokens_details.cache_creation_tokens gets promoted
|
||||
to cache_creation_input_tokens at the top level.
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import \
|
||||
StandardLoggingPayloadSetup
|
||||
|
||||
usage = {
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 50,
|
||||
"total_tokens": 150,
|
||||
"prompt_tokens_details": {"cached_tokens": 30, "cache_creation_tokens": 70},
|
||||
}
|
||||
result = StandardLoggingPayloadSetup._normalize_usage_cache_tokens(usage)
|
||||
assert result["cache_read_input_tokens"] == 30
|
||||
assert result["cache_creation_input_tokens"] == 70
|
||||
|
||||
|
||||
def test_get_usage_as_dict_openai_cached_tokens():
|
||||
"""
|
||||
End-to-end: get_usage_as_dict promotes OpenAI cached_tokens to
|
||||
cache_read_input_tokens for the daily spend writer.
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import \
|
||||
StandardLoggingPayloadSetup
|
||||
|
||||
response_obj = {
|
||||
"usage": {
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 50,
|
||||
"total_tokens": 150,
|
||||
"prompt_tokens_details": {"cached_tokens": 25},
|
||||
}
|
||||
}
|
||||
result = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj=response_obj)
|
||||
assert result["cache_read_input_tokens"] == 25
|
||||
|
||||
|
||||
def test_get_usage_as_dict_combined_usage_object_cached_tokens():
|
||||
"""
|
||||
End-to-end: get_usage_as_dict with combined_usage_object that has
|
||||
prompt_tokens_details.cached_tokens promotes to cache_read_input_tokens.
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import \
|
||||
StandardLoggingPayloadSetup
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
combined = Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=50,
|
||||
total_tokens=150,
|
||||
prompt_tokens_details={"cached_tokens": 35},
|
||||
)
|
||||
result = StandardLoggingPayloadSetup.get_usage_as_dict(
|
||||
response_obj=None,
|
||||
combined_usage_object=combined,
|
||||
)
|
||||
assert result["cache_read_input_tokens"] == 35
|
||||
|
||||
|
||||
def test_append_system_prompt_messages():
|
||||
"""
|
||||
Test append_system_prompt_messages prepends system message from kwargs to messages list.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue