From 30459e949b0fcbdc8314b602ae07342304088599 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 7 Apr 2026 13:50:03 -0700 Subject: [PATCH] [Fix] Cached tokens always showing zero in UI for OpenAI models OpenAI returns cached tokens nested in prompt_tokens_details.cached_tokens, while Anthropic/DeepSeek return them as top-level cache_read_input_tokens. The daily spend writer only checked the top-level field, so OpenAI cached tokens were never persisted to the database. Added normalization in get_usage_as_dict() to promote nested cached_tokens to the top-level cache_read_input_tokens field when it's missing. --- litellm/litellm_core_utils/litellm_logging.py | 45 ++++++- .../test_litellm_logging.py | 111 ++++++++++++++++++ 2 files changed, 151 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7395b65626f..01275922c07 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4871,28 +4871,63 @@ class StandardLoggingPayloadSetup: """ _empty: dict = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} if combined_usage_object is not None: - return combined_usage_object.model_dump() + result = combined_usage_object.model_dump() + return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(result) if not response_obj: return _empty _raw = response_obj.get("usage", None) if _raw is None: return _empty if isinstance(_raw, ResponseAPIUsage): - return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( _raw ).model_dump() + return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(result) if isinstance(_raw, dict): if ResponseAPILoggingUtils._is_response_api_usage(_raw): - return ( + result = ( ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( _raw ).model_dump() ) - return _raw + return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(result) + return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(_raw) if isinstance(_raw, Usage): - return _raw.model_dump() + result = _raw.model_dump() + return StandardLoggingPayloadSetup._normalize_usage_cache_tokens(result) return _empty + @staticmethod + def _normalize_usage_cache_tokens(usage_dict: dict) -> dict: + """ + Promote prompt_tokens_details.cached_tokens → cache_read_input_tokens + and prompt_tokens_details.cache_creation_tokens → cache_creation_input_tokens + when the top-level fields are missing or zero. + + OpenAI returns cached tokens nested in prompt_tokens_details, while + Anthropic/DeepSeek return them as top-level fields. The daily spend + tables expect cache_read_input_tokens at the top level, so this + normalization ensures all providers' cached tokens are tracked. + """ + prompt_details = usage_dict.get("prompt_tokens_details") + if not prompt_details: + return usage_dict + + if isinstance(prompt_details, dict): + cached = prompt_details.get("cached_tokens") + creation = prompt_details.get("cache_creation_tokens") + else: + cached = getattr(prompt_details, "cached_tokens", None) + creation = getattr(prompt_details, "cache_creation_tokens", None) + + if cached and not usage_dict.get("cache_read_input_tokens"): + usage_dict["cache_read_input_tokens"] = cached + + if creation and not usage_dict.get("cache_creation_input_tokens"): + usage_dict["cache_creation_input_tokens"] = creation + + return usage_dict + @staticmethod def get_model_cost_information( base_model: Optional[str], diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 9d584446eb5..a5629d2e97c 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -1522,6 +1522,117 @@ def test_get_usage_as_dict(): assert result == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} +def test_normalize_usage_cache_tokens_openai(): + """ + Test that prompt_tokens_details.cached_tokens from OpenAI responses + gets promoted to cache_read_input_tokens at the top level. + """ + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup + + # OpenAI-style: cached_tokens nested in prompt_tokens_details, no top-level cache_read_input_tokens + usage = { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + "prompt_tokens_details": {"cached_tokens": 30, "audio_tokens": 0}, + } + result = StandardLoggingPayloadSetup._normalize_usage_cache_tokens(usage) + assert result["cache_read_input_tokens"] == 30 + + +def test_normalize_usage_cache_tokens_anthropic_not_overwritten(): + """ + Test that existing top-level cache_read_input_tokens (Anthropic) is NOT + overwritten by prompt_tokens_details.cached_tokens. + """ + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup + + usage = { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + "cache_read_input_tokens": 40, + "prompt_tokens_details": {"cached_tokens": 40}, + } + result = StandardLoggingPayloadSetup._normalize_usage_cache_tokens(usage) + assert result["cache_read_input_tokens"] == 40 + + +def test_normalize_usage_cache_tokens_no_details(): + """ + Test normalization is a no-op when prompt_tokens_details is absent. + """ + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup + + usage = {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150} + result = StandardLoggingPayloadSetup._normalize_usage_cache_tokens(usage) + assert "cache_read_input_tokens" not in result + + +def test_normalize_usage_cache_creation_tokens(): + """ + Test that prompt_tokens_details.cache_creation_tokens gets promoted + to cache_creation_input_tokens at the top level. + """ + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup + + usage = { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + "prompt_tokens_details": {"cached_tokens": 30, "cache_creation_tokens": 70}, + } + result = StandardLoggingPayloadSetup._normalize_usage_cache_tokens(usage) + assert result["cache_read_input_tokens"] == 30 + assert result["cache_creation_input_tokens"] == 70 + + +def test_get_usage_as_dict_openai_cached_tokens(): + """ + End-to-end: get_usage_as_dict promotes OpenAI cached_tokens to + cache_read_input_tokens for the daily spend writer. + """ + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup + + response_obj = { + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + "prompt_tokens_details": {"cached_tokens": 25}, + } + } + result = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj=response_obj) + assert result["cache_read_input_tokens"] == 25 + + +def test_get_usage_as_dict_combined_usage_object_cached_tokens(): + """ + End-to-end: get_usage_as_dict with combined_usage_object that has + prompt_tokens_details.cached_tokens promotes to cache_read_input_tokens. + """ + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup + from litellm.types.utils import Usage + + combined = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + prompt_tokens_details={"cached_tokens": 35}, + ) + result = StandardLoggingPayloadSetup.get_usage_as_dict( + response_obj=None, + combined_usage_object=combined, + ) + assert result["cache_read_input_tokens"] == 35 + + def test_append_system_prompt_messages(): """ Test append_system_prompt_messages prepends system message from kwargs to messages list.