From c429a0e4a769450b91e7378c0e17c615fadc4f20 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:36:26 +0000 Subject: [PATCH 1/8] fix(cost_tracking): keep OpenAI prompt cache token details through usage reassembly --- .../litellm_core_utils/llm_cost_calc/utils.py | 2 +- .../streaming_chunk_builder_utils.py | 9 ++-- .../litellm_core_utils/streaming_handler.py | 29 +++++++++++++ .../transformation.py | 6 +++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 26 +++++++++++ .../test_streaming_chunk_builder_utils.py | 43 +++++++++++++++++++ .../test_streaming_handler.py | 43 +++++++++++++++++++ .../test_litellm_completion_responses.py | 21 +++++++++ 8 files changed, 175 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 85ed0665ebf..6f344d687c8 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -735,7 +735,7 @@ def generic_cost_per_token( # Check for double-counting: sum of details > prompt_tokens means overlap total_details = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens + video_tokens - has_double_counting = cache_hit > 0 and total_details > usage.prompt_tokens + has_double_counting = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens if (text_tokens == 0 and prompt_tokens_details["image_count"] == 0) or has_double_counting: text_tokens = usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index d52d9849310..a8b5c21d5da 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -615,9 +615,12 @@ class ChunkProcessor: "web_search_requests", ) - prompt_tokens_details = cast( - Optional[PromptTokensDetailsWrapper], - usage_chunk_dict["prompt_tokens_details"], + prompt_tokens_details = ( + cast( + PromptTokensDetailsWrapper | None, + usage_chunk_dict["prompt_tokens_details"], + ) + or prompt_tokens_details ) cache_creation_token_details = self._capture_cache_creation_token_details( diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 60dbf7c644a..d5a08035bf4 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -16,6 +16,7 @@ from typing import ( List, NoReturn, Optional, + TypeVar, Union, cast, ) @@ -39,9 +40,11 @@ from litellm.types.utils import ( ) from litellm.types.utils import GenericStreamingChunk as GChunk from litellm.types.utils import ( + CompletionTokensDetailsWrapper, LlmProviders, ModelResponse, ModelResponseStream, + PromptTokensDetailsWrapper, StreamingChoices, Usage, ) @@ -2254,11 +2257,27 @@ class CustomStreamWrapper: return chunk +_TokenDetails = TypeVar("_TokenDetails", PromptTokensDetailsWrapper, CompletionTokensDetailsWrapper) + + +def _coerce_token_details( + usage: Union[dict, BaseModel], field: str, details_type: type[_TokenDetails] +) -> _TokenDetails | None: + raw = usage.get(field) if isinstance(usage, dict) else getattr(usage, field, None) + if raw is None: + return None + if isinstance(raw, details_type): + return raw + return details_type(**(raw if isinstance(raw, dict) else raw.model_dump())) + + def calculate_total_usage(chunks: List[ModelResponse]) -> Usage: """Assume most recent usage chunk has total usage uptil then.""" prompt_tokens: int = 0 completion_tokens: int = 0 latest_usage_chunk = None + prompt_tokens_details: PromptTokensDetailsWrapper | None = None + completion_tokens_details: CompletionTokensDetailsWrapper | None = None for chunk in chunks: if "usage" in chunk and chunk["usage"] is not None: @@ -2268,11 +2287,21 @@ def calculate_total_usage(chunks: List[ModelResponse]) -> Usage: prompt_tokens = usage.get("prompt_tokens", 0) or 0 if "completion_tokens" in usage: completion_tokens = usage.get("completion_tokens", 0) or 0 + prompt_tokens_details = ( + _coerce_token_details(usage, "prompt_tokens_details", PromptTokensDetailsWrapper) + or prompt_tokens_details + ) + completion_tokens_details = ( + _coerce_token_details(usage, "completion_tokens_details", CompletionTokensDetailsWrapper) + or completion_tokens_details + ) returned_usage_chunk = Usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=prompt_tokens_details, + completion_tokens_details=completion_tokens_details, ) if latest_usage_chunk is not None: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 6b1ca3564e3..2b1c7274a28 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2011,6 +2011,12 @@ class LiteLLMCompletionResponsesConfig: if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None: input_details_dict["audio_tokens"] = prompt_details.audio_tokens + cache_write_tokens = getattr(prompt_details, "cache_write_tokens", None) or getattr( + prompt_details, "cache_creation_tokens", None + ) + if cache_write_tokens is not None: + input_details_dict["cache_write_tokens"] = cache_write_tokens + if input_details_dict: response_usage.input_tokens_details = InputTokensDetails(**input_details_dict) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index d282e656ce8..b6abbb753ee 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2142,6 +2142,32 @@ def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(): assert prompt_cost > 1000 * info["input_cost_per_token"] +def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens(): + """ + Regression for #34801: when a provider reports text_tokens covering the whole + prompt alongside cache-write tokens (and no cache reads), the cache-write tokens + must be backed out of the text total instead of being billed twice. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gpt-5.6" + usage = Usage( + prompt_tokens=1000, + completion_tokens=10, + total_tokens=1010, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, cache_write_tokens=800, text_tokens=1000 + ), + ) + + prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + + info = litellm.get_model_info(model=model, custom_llm_provider="openai") + expected_prompt = 200 * info["input_cost_per_token"] + 800 * info["cache_creation_input_token_cost"] + assert prompt_cost == pytest.approx(expected_prompt) + + def test_token_type_cost_breakdown_reconciles_with_generic_total(): """ Both-ways check: the reasoning subset must sum with the remaining (text) output diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index be8c5a05601..e14b00cfeac 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -992,3 +992,46 @@ def test_cost_field_in_usage_chunks(): assert usage.cost == 0.00025 assert usage.prompt_tokens == 10 assert usage.completion_tokens == 5 + + +def test_prompt_tokens_details_survive_later_usage_chunk_without_details(): + """Regression for #34801: a trailing usage chunk that omits + `prompt_tokens_details` must not wipe the OpenAI cache-read/cache-write split, + otherwise those tokens get re-priced at the uncached input rate.""" + from litellm.types.utils import PromptTokensDetailsWrapper + + chunk_with_details = ModelResponseStream( + id="chatcmpl-1", + created=1745513206, + model="openai/gpt-5.6-sol", + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) + ], + usage=Usage( + prompt_tokens=6017, + completion_tokens=4, + total_tokens=6021, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=6004, cache_write_tokens=10 + ), + ), + ) + chunk_without_details = ModelResponseStream( + id="chatcmpl-1", + created=1745513207, + model="openai/gpt-5.6-sol", + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], + usage=Usage(prompt_tokens=6017, completion_tokens=4, total_tokens=6021), + ) + + chunks = [chunk_with_details, chunk_without_details] + usage = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, model="openai/gpt-5.6-sol", completion_output="Hi" + ) + + assert usage.prompt_tokens == 6017 + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cached_tokens == 6004 + assert usage.prompt_tokens_details.cache_write_tokens == 10 diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 514714136fd..e94d8495294 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -1449,6 +1449,49 @@ def test_calculate_total_usage_with_dict_usage_cost(): assert getattr(usage, "cost", None) == 0.00025 +def test_calculate_total_usage_preserves_prompt_cache_token_details(): + """Regression for #34801: dropping `prompt_tokens_details` here re-prices OpenAI + cache-read tokens at the uncached input rate, overstating spend.""" + from litellm.litellm_core_utils.streaming_handler import calculate_total_usage + + usage_with_details = Usage( + prompt_tokens=6017, + completion_tokens=4, + total_tokens=6021, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=6004, cache_write_tokens=10 + ), + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=2), + ) + chunk_with_details = ModelResponseStream( + id="chatcmpl-1", + created=1745513206, + model="openai/gpt-5.6-sol", + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) + ], + usage=usage_with_details, + ) + chunk_without_details = ModelResponseStream( + id="chatcmpl-1", + created=1745513207, + model="openai/gpt-5.6-sol", + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], + usage=Usage(prompt_tokens=6017, completion_tokens=4, total_tokens=6021), + ) + + usage = calculate_total_usage([chunk_with_details, chunk_without_details]) + + assert usage.prompt_tokens == 6017 + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cached_tokens == 6004 + assert usage.prompt_tokens_details.cache_write_tokens == 10 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 2 + + @pytest.mark.asyncio async def test_openrouter_streaming_cost_after_finish_reason(logging_obj: Logging): from litellm.utils import ModelResponseListIterator diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index d8e3f495ced..6b76f8bc638 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1751,6 +1751,27 @@ class TestUsageTransformation: assert response_usage.input_tokens_details.cached_tokens == 3 assert response_usage.input_tokens_details.text_tokens == 6 + def test_transform_usage_preserves_cache_write_tokens(self): + """Regression for #34801: the chat-completions to Responses bridge dropped + cache-write tokens, so cache-creation billing disappeared on that route.""" + usage = Usage( + prompt_tokens=1000, + completion_tokens=10, + total_tokens=1010, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=100, + cache_write_tokens=800, + ), + ) + + response_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( + chat_completion_response=usage + ) + + assert response_usage.input_tokens_details is not None + assert response_usage.input_tokens_details.cached_tokens == 100 + assert getattr(response_usage.input_tokens_details, "cache_write_tokens", None) == 800 + def test_transform_usage_with_reasoning_tokens_gemini(self): """Test that reasoning_tokens from Gemini are properly transformed to output_tokens_details""" # Setup: Simulate Gemini usage with thoughtsTokenCount From ed366aafbe12c126765ee8ee2073cf751918ecb1 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:12:25 +0000 Subject: [PATCH 2/8] fix(anthropic): preserve prompt cache tokens in messages to responses api usage Also map gpt-5.6 flex/priority cache_creation rates into ModelInfo so cache writes are not billed at the standard rate on those service tiers --- .../responses_adapters/streaming_iterator.py | 33 +++++---------- .../responses_adapters/transformation.py | 32 +++++++++------ litellm/types/utils.py | 4 ++ litellm/utils.py | 4 ++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 41 +++++++++++++++++++ ...t_responses_adapters_streaming_iterator.py | 29 +++++++++++++ .../test_responses_adapters_transformation.py | 34 +++++++++++++-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 ++++ 8 files changed, 147 insertions(+), 38 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 4fd49a35417..c3f0c8912ca 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -7,6 +7,9 @@ from typing import Any, AsyncIterator, Dict from litellm import verbose_logger from litellm._uuid import uuid +from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage + +from .transformation import LiteLLMAnthropicToResponsesAPIAdapter class AnthropicResponsesStreamWrapper: @@ -226,24 +229,17 @@ class AnthropicResponsesStreamWrapper: event.get("response") if isinstance(event, dict) else None ) stop_reason = "end_turn" - input_tokens = 0 - output_tokens = 0 - cache_creation_tokens = 0 - cache_read_tokens = 0 + anthropic_usage: AnthropicUsage = AnthropicUsage(input_tokens=0, output_tokens=0) if response_obj is not None: status = getattr(response_obj, "status", None) if status == "incomplete": stop_reason = "max_tokens" - usage = getattr(response_obj, "usage", None) - if usage is not None: - input_tokens = getattr(usage, "input_tokens", 0) or 0 - output_tokens = getattr(usage, "output_tokens", 0) or 0 - cache_creation_tokens = getattr(usage, "input_tokens_details", None) # type: ignore[assignment] - cache_read_tokens = getattr(usage, "output_tokens_details", None) # type: ignore[assignment] - # Prefer direct cache fields if present - cache_creation_tokens = int(getattr(usage, "cache_creation_input_tokens", 0) or 0) - cache_read_tokens = int(getattr(usage, "cache_read_input_tokens", 0) or 0) + anthropic_usage = ( + LiteLLMAnthropicToResponsesAPIAdapter.translate_responses_api_usage_to_anthropic_usage( + getattr(response_obj, "usage", None) + ) + ) # Check if tool_use was in the output to override stop_reason if response_obj is not None: @@ -256,20 +252,11 @@ class AnthropicResponsesStreamWrapper: stop_reason = "tool_use" break - usage_delta: Dict[str, Any] = { - "input_tokens": input_tokens, - "output_tokens": output_tokens, - } - if cache_creation_tokens: - usage_delta["cache_creation_input_tokens"] = cache_creation_tokens - if cache_read_tokens: - usage_delta["cache_read_input_tokens"] = cache_read_tokens - self._chunk_queue.append( { "type": "message_delta", "delta": {"stop_reason": stop_reason, "stop_sequence": None}, - "usage": usage_delta, + "usage": dict(anthropic_usage), } ) self._chunk_queue.append({"type": "message_stop"}) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 172e54de98e..8b75b0447fa 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -29,7 +29,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, AnthropicUsage, ) -from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse class LiteLLMAnthropicToResponsesAPIAdapter: @@ -38,6 +38,24 @@ class LiteLLMAnthropicToResponsesAPIAdapter: converts Responses API responses back to Anthropic format. """ + @staticmethod + def translate_responses_api_usage_to_anthropic_usage( + raw_usage: Optional[ResponseAPIUsage], + ) -> AnthropicUsage: + """Map Responses API usage onto Anthropic usage, where ``input_tokens`` + excludes the cache-read and cache-write tokens reported alongside it. + """ + if raw_usage is None: + return AnthropicUsage(input_tokens=0, output_tokens=0) + + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + from litellm.responses.utils import ResponseAPILoggingUtils + + chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(raw_usage) + return LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage(chat_usage) + # ------------------------------------------------------------------ # # Request translation: Anthropic -> Responses API # # ------------------------------------------------------------------ # @@ -396,8 +414,6 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ResponseReasoningItem, ) - from litellm.types.llms.openai import ResponseAPIUsage - content: List[Dict[str, Any]] = [] stop_reason: AnthropicFinishReason = "end_turn" @@ -463,15 +479,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if response.status == "incomplete": stop_reason = "max_tokens" - # usage - raw_usage: Optional[ResponseAPIUsage] = response.usage - input_tokens = int(getattr(raw_usage, "input_tokens", 0) or 0) - output_tokens = int(getattr(raw_usage, "output_tokens", 0) or 0) - - anthropic_usage = AnthropicUsage( - input_tokens=input_tokens, - output_tokens=output_tokens, - ) + anthropic_usage = self.translate_responses_api_usage_to_anthropic_usage(response.usage) return AnthropicMessagesResponse( id=response.id, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e4dfac48141..04388edaf05 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -198,6 +198,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing input_cost_per_token_priority: Optional[float] # OpenAI priority service tier pricing cache_creation_input_token_cost: Optional[float] + cache_creation_input_token_cost_flex: Optional[float] # OpenAI flex service tier pricing + cache_creation_input_token_cost_priority: Optional[float] # OpenAI priority service tier pricing cache_creation_input_token_cost_above_200k_tokens: Optional[float] cache_creation_input_token_cost_above_1hr: Optional[float] cache_read_input_token_cost: Optional[float] @@ -3087,6 +3089,8 @@ class CustomPricingLiteLLMParams(BaseModel): input_cost_per_token_flex: Optional[float] = None input_cost_per_token_priority: Optional[float] = None cache_creation_input_token_cost: Optional[float] = None + cache_creation_input_token_cost_flex: Optional[float] = None + cache_creation_input_token_cost_priority: Optional[float] = None cache_creation_input_token_cost_above_1hr: Optional[float] = None cache_creation_input_token_cost_above_200k_tokens: Optional[float] = None cache_creation_input_audio_token_cost: Optional[float] = None diff --git a/litellm/utils.py b/litellm/utils.py index 944bb61d5e7..3296b39e708 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5407,6 +5407,10 @@ def _get_model_info_helper( input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None), input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None), cache_creation_input_token_cost=_model_info.get("cache_creation_input_token_cost", None), + cache_creation_input_token_cost_flex=_model_info.get("cache_creation_input_token_cost_flex", None), + cache_creation_input_token_cost_priority=_model_info.get( + "cache_creation_input_token_cost_priority", None + ), cache_creation_input_token_cost_above_200k_tokens=_model_info.get( "cache_creation_input_token_cost_above_200k_tokens", None ), diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index d282e656ce8..d1b488190a0 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2399,3 +2399,44 @@ def test_generic_cost_per_token_gemini_35_flash_lite(): ) assert prompt_cost == pytest.approx(0.0003) assert completion_cost == pytest.approx(0.00125) + + +@pytest.mark.parametrize( + "service_tier,input_rate,cache_read_rate,cache_write_rate,output_rate", + [ + ("flex", 2.5e-6, 2.5e-7, 3.125e-6, 1.5e-5), + ("priority", 1e-5, 1e-6, 1.25e-5, 6e-5), + ], +) +def test_service_tier_cache_creation_rates_for_gpt_5_6( + _local_model_cost_map, + service_tier, + input_rate, + cache_read_rate, + cache_write_rate, + output_rate, +): + """Regression: gpt-5.6 publishes cache_creation_input_token_cost_flex/_priority, so a + flex or priority request must bill cache writes at that tier's rate instead of falling + back to the standard 6.25e-6 rate.""" + usage = Usage( + prompt_tokens=10_000, + completion_tokens=500, + total_tokens=10_500, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=6_000, + cache_write_tokens=3_000, + text_tokens=1_000, + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model="gpt-5.6-sol", + usage=usage, + custom_llm_provider="openai", + service_tier=service_tier, + ) + + expected_prompt = 1_000 * input_rate + 6_000 * cache_read_rate + 3_000 * cache_write_rate + assert prompt_cost == pytest.approx(expected_prompt, rel=1e-9) + assert completion_cost == pytest.approx(500 * output_rate, rel=1e-9) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index 9b5197d9028..73b58e71009 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -6,6 +6,7 @@ Tests for AnthropicResponsesStreamWrapper import asyncio import os import sys +from types import SimpleNamespace sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) @@ -130,3 +131,31 @@ class TestProcessEventTextDeltaWithoutOutputItemAdded: ("content_block_start", 0), ("content_block_delta", 0), ] + + +class TestResponseCompletedUsage: + """The Anthropic ``message_delta`` usage must report cache reads/writes and + exclude them from ``input_tokens``, so spend is not billed at the uncached + input rate.""" + + def test_response_completed_usage_carries_cache_tokens(self): + from litellm.types.llms.openai import ResponseAPIUsage + + response = SimpleNamespace( + status="completed", + output=[], + usage=ResponseAPIUsage( + input_tokens=4017, + input_tokens_details={"cached_tokens": 4004, "cache_write_tokens": 10}, + output_tokens=5, + total_tokens=4022, + ), + ) + chunks = _process_all([{"type": "response.completed", "response": response}]) + message_delta = next(c for c in chunks if c["type"] == "message_delta") + assert message_delta["usage"] == { + "input_tokens": 3, + "output_tokens": 5, + "cache_creation_input_tokens": 10, + "cache_read_input_tokens": 4004, + } diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 606ff39b35e..77b6f902368 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -20,6 +20,7 @@ from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transfo LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.types.llms.anthropic import AnthropicMessagesRequest +from litellm.types.llms.openai import ResponseAPIUsage def _make_request(**overrides) -> AnthropicMessagesRequest: @@ -823,11 +824,19 @@ def _make_mock_response( model: str = "gpt-4o", input_tokens: int = 100, output_tokens: int = 50, + cached_tokens: int = 0, + cache_write_tokens: int = 0, ) -> MagicMock: """Build a minimal mock ResponsesAPIResponse.""" - usage = MagicMock() - usage.input_tokens = input_tokens - usage.output_tokens = output_tokens + usage = ResponseAPIUsage( + input_tokens=input_tokens, + input_tokens_details={ + "cached_tokens": cached_tokens, + "cache_write_tokens": cache_write_tokens, + }, + output_tokens=output_tokens, + total_tokens=input_tokens + output_tokens, + ) resp = MagicMock() resp.id = response_id @@ -961,6 +970,25 @@ class TestTranslateResponse: assert result["usage"]["input_tokens"] == 200 assert result["usage"]["output_tokens"] == 75 + def test_cache_tokens_mapped_to_anthropic_usage(self): + """Cache reads/writes reported by the Responses API must survive the + Anthropic mapping, and input_tokens must exclude them so spend is not + billed at the uncached input rate.""" + response = _make_mock_response( + output=[_make_output_message(["OK"])], + input_tokens=4017, + output_tokens=5, + cached_tokens=4004, + cache_write_tokens=10, + ) + result: Any = _ADAPTER.translate_response(response) + assert result["usage"] == { + "input_tokens": 3, + "output_tokens": 5, + "cache_creation_input_tokens": 10, + "cache_read_input_tokens": 4004, + } + def test_model_and_id_preserved(self): """Model and response ID from the Responses API are forwarded.""" response = _make_mock_response( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 541d7a17ae1..b9559113b84 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25822,6 +25822,10 @@ export interface components { cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Creation Input Token Cost Above 200K Tokens */ cache_creation_input_token_cost_above_200k_tokens?: number | null; + /** Cache Creation Input Token Cost Flex */ + cache_creation_input_token_cost_flex?: number | null; + /** Cache Creation Input Token Cost Priority */ + cache_creation_input_token_cost_priority?: number | null; /** Cache Read Input Audio Token Cost */ cache_read_input_audio_token_cost?: number | null; /** Cache Read Input Token Cost */ @@ -33912,6 +33916,10 @@ export interface components { cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Creation Input Token Cost Above 200K Tokens */ cache_creation_input_token_cost_above_200k_tokens?: number | null; + /** Cache Creation Input Token Cost Flex */ + cache_creation_input_token_cost_flex?: number | null; + /** Cache Creation Input Token Cost Priority */ + cache_creation_input_token_cost_priority?: number | null; /** Cache Read Input Audio Token Cost */ cache_read_input_audio_token_cost?: number | null; /** Cache Read Input Token Cost */ From 37744ca944c909774e4d7848ab4dbfcb0d1ccde5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:37:49 +0000 Subject: [PATCH 3/8] fix(cost): price anthropic messages cache read/write tokens instead of full input rate Anthropic-shaped usage was mapped through the Responses API usage converter, which ignores top-level cache_read_input_tokens/cache_creation_input_tokens, so cache hits on /v1/messages were billed entirely at the uncached input rate --- litellm/cost_calculator.py | 10 ++++++- litellm/llms/anthropic/chat/transformation.py | 10 +++++++ tests/test_litellm/test_cost_calculator.py | 27 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 96aed20529f..884c21bebd3 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -886,6 +886,8 @@ def _get_usage_object( return None if isinstance(usage_obj, Usage): return usage_obj + elif isinstance(usage_obj, dict) and litellm.AnthropicConfig.is_anthropic_usage_object(usage_obj): + return litellm.AnthropicConfig().calculate_usage(usage_object=usage_obj, reasoning_content=None) elif ( usage_obj is not None and (isinstance(usage_obj, dict) or isinstance(usage_obj, ResponseAPIUsage)) @@ -1251,7 +1253,13 @@ def completion_cost( else: _usage = usage_obj - if ResponseAPILoggingUtils._is_response_api_usage(_usage): + if litellm.AnthropicConfig.is_anthropic_usage_object(_usage): + _usage = ( + litellm.AnthropicConfig() + .calculate_usage(usage_object=_usage, reasoning_content=None) + .model_dump() + ) + elif ResponseAPILoggingUtils._is_response_api_usage(_usage): _usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( _usage ).model_dump() diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index e99f356f8f2..50a30aa5f54 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2122,6 +2122,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): compaction_blocks, ) + @staticmethod + def is_anthropic_usage_object(usage_object: dict) -> bool: + """Anthropic reports prompt cache tokens as top-level ``cache_read_input_tokens`` / + ``cache_creation_input_tokens``; no other API surface uses those keys, and the + Responses API mapping would silently drop them. + """ + if "prompt_tokens" in usage_object or "input_tokens" not in usage_object: + return False + return any(key in usage_object for key in ("cache_read_input_tokens", "cache_creation_input_tokens")) + def calculate_usage( self, usage_object: dict, diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 276ee96ed65..4ac67621501 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3509,3 +3509,30 @@ def test_combine_usage_objects_sums_mirrored_cache_write_fields_once(): assert combined_pair.prompt_tokens_details is not None assert combined_pair.prompt_tokens_details.cache_write_tokens == 100 assert combined_pair.prompt_tokens_details.cache_creation_tokens == 100 + + +def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(): + """Regression: an Anthropic /v1/messages response reports cache reads as top-level + cache_read_input_tokens with input_tokens excluding them. Reading that usage as + Responses API usage dropped the cache tokens and billed the whole prompt at the + uncached input rate, overstating spend on cache hits.""" + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "gpt-5.6-sol", + "stop_reason": "end_turn", + "content": [{"type": "text", "text": "1"}], + "usage": {"input_tokens": 3, "output_tokens": 5, "cache_read_input_tokens": 4014}, + } + + cost = litellm.completion_cost( + completion_response=response, + model="gpt-5.6-sol", + custom_llm_provider="openai", + ) + + assert cost == pytest.approx(3 * 5e-6 + 4014 * 5e-7 + 5 * 3e-5, rel=1e-9) From 8136c96284485f7460d2a490608c7efdadc4f6ed Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:46:31 +0000 Subject: [PATCH 4/8] test(anthropic): cover usage-shape detection for cache token pricing --- .../test_anthropic_chat_transformation.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 94a4a3fc945..34fbea95e5b 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -23,7 +23,7 @@ from litellm.llms.anthropic.experimental_pass_through.messages.transformation im AnthropicMessagesConfig, ) from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES -from litellm.types.utils import ServerToolUse +from litellm.types.utils import ServerToolUse, Usage def test_response_format_transformation_unit_test(): @@ -5845,3 +5845,25 @@ def test_top_k_forwarded_at_transform_on_models_that_accept_it(): ) assert result["top_k"] == 40 + + +def test_is_anthropic_usage_object_distinguishes_chat_usage(): + """Chat-shaped Usage mirrors cache_read_input_tokens alongside prompt_tokens that already + include the cache tokens, so treating it as Anthropic usage would re-add them and + double-count the prompt. Only the Anthropic shape, where input_tokens excludes cache + tokens, may take the Anthropic mapping.""" + assert AnthropicConfig.is_anthropic_usage_object( + {"input_tokens": 3, "output_tokens": 5, "cache_read_input_tokens": 4014} + ) + assert AnthropicConfig.is_anthropic_usage_object( + {"input_tokens": 3, "output_tokens": 5, "cache_creation_input_tokens": 10} + ) + assert not AnthropicConfig.is_anthropic_usage_object( + Usage( + prompt_tokens=4017, + completion_tokens=5, + total_tokens=4022, + cache_read_input_tokens=4014, + ).model_dump() + ) + assert not AnthropicConfig.is_anthropic_usage_object({"input_tokens": 3, "output_tokens": 5}) From 74f1b934be228f9bbaab260db2277928c3f48384 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 01:23:54 -0700 Subject: [PATCH 5/8] chore(ui): format CacheLeakageCard with prettier --- .../cost-optimization/_components/CacheLeakageCard.tsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx index bb2cb865877..3bc5443b2ea 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx @@ -114,10 +114,7 @@ const CacheLeakageCard: React.FC = ({ activity }) => { - setDimension(value === "model" ? "model" : "key")} - > + setDimension(value === "model" ? "model" : "key")}> By virtual key By model From e8a80b98837ea308003e287b98579668d9d6fc03 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 02:01:53 -0700 Subject: [PATCH 6/8] docs(anthropic): state why usage-shape detection requires a cache key, pin Responses-shape rejection --- litellm/llms/anthropic/chat/transformation.py | 5 +++++ .../chat/test_anthropic_chat_transformation.py | 16 ++++++++++++++++ .../test_responses_adapters_transformation.py | 7 +++++++ 3 files changed, 28 insertions(+) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 1f405de9817..2194b384a23 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2109,6 +2109,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): """Anthropic reports prompt cache tokens as top-level ``cache_read_input_tokens`` / ``cache_creation_input_tokens``; no other API surface uses those keys, and the Responses API mapping would silently drop them. + + Requiring a cache key is deliberate: Responses API usage also carries top-level + ``input_tokens``, so the cache keys are the only shape discriminator between the + two. A cache-free Anthropic payload falls through to the Responses API mapping, + which is safe because both mappings agree whenever no cache tokens are present. """ if "prompt_tokens" in usage_object or "input_tokens" not in usage_object: return False diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 34fbea95e5b..231d3b48754 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -5867,3 +5867,19 @@ def test_is_anthropic_usage_object_distinguishes_chat_usage(): ).model_dump() ) assert not AnthropicConfig.is_anthropic_usage_object({"input_tokens": 3, "output_tokens": 5}) + + +def test_is_anthropic_usage_object_rejects_responses_api_usage(): + """completion_cost checks the Anthropic shape before the Responses API shape, so a + Responses API usage payload, whose cache reads live in nested input_tokens_details, + must never match; matching would route it past the converter that reads the nested + field and its cache reads would be billed at the full input rate.""" + assert not AnthropicConfig.is_anthropic_usage_object( + { + "input_tokens": 4017, + "output_tokens": 5, + "total_tokens": 4022, + "input_tokens_details": {"cached_tokens": 4014}, + "output_tokens_details": {"reasoning_tokens": 0}, + } + ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 77b6f902368..a268bdb640c 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -989,6 +989,13 @@ class TestTranslateResponse: "cache_read_input_tokens": 4004, } + def test_missing_usage_maps_to_zero_tokens(self): + """A response without a usage object must map to zeroed Anthropic usage.""" + assert LiteLLMAnthropicToResponsesAPIAdapter.translate_responses_api_usage_to_anthropic_usage(None) == { + "input_tokens": 0, + "output_tokens": 0, + } + def test_model_and_id_preserved(self): """Model and response ID from the Responses API are forwarded.""" response = _make_mock_response( From a9902fcdb56323731568e2c42f3a379cb5e1ca2c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 02:06:52 -0700 Subject: [PATCH 7/8] fix(streaming): carry Anthropic cache-creation TTL split through fallback usage reassembly --- .../streaming_chunk_builder_utils.py | 62 +++++++++---------- .../litellm_core_utils/streaming_handler.py | 18 ++++-- .../test_streaming_handler.py | 54 ++++++++++++++++ 3 files changed, 98 insertions(+), 36 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 6b1ada07987..3f967e29002 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -39,6 +39,34 @@ if TYPE_CHECKING: ) +def capture_cache_creation_token_details( + prompt_tokens_details: PromptTokensDetailsWrapper | None, + current: CacheCreationTokenDetails | None, +) -> CacheCreationTokenDetails | None: + incoming: Final = cast( + CacheCreationTokenDetails | None, + getattr(prompt_tokens_details, "cache_creation_token_details", None), + ) + if incoming is not None: + return incoming + return current + + +def attach_cache_creation_token_details( + prompt_tokens_details: PromptTokensDetailsWrapper | None, + cache_creation_token_details: CacheCreationTokenDetails | None, +) -> PromptTokensDetailsWrapper | None: + if prompt_tokens_details is None or cache_creation_token_details is None: + return prompt_tokens_details + existing: Final = cast( + CacheCreationTokenDetails | None, + getattr(prompt_tokens_details, "cache_creation_token_details", None), + ) + if existing is not None: + return prompt_tokens_details + return prompt_tokens_details.model_copy(update={"cache_creation_token_details": cache_creation_token_details}) + + class ChunkProcessor: def __init__(self, chunks: list, messages: list | None = None): self.chunks = self._sort_chunks(chunks) @@ -701,16 +729,14 @@ class ChunkProcessor: or prompt_tokens_details ) - cache_creation_token_details = self._capture_cache_creation_token_details( + cache_creation_token_details = capture_cache_creation_token_details( prompt_tokens_details, cache_creation_token_details ) if usage_chunk_dict["cost"] is not None: cost = usage_chunk_dict["cost"] - prompt_tokens_details = self._attach_cache_creation_token_details( - prompt_tokens_details, cache_creation_token_details - ) + prompt_tokens_details = attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details) completion_tokens = self._reset_anthropic_cursor_completion_tokens( chunks=chunks, @@ -730,34 +756,6 @@ class ChunkProcessor: cost=cost, ) - @staticmethod - def _capture_cache_creation_token_details( - prompt_tokens_details: PromptTokensDetailsWrapper | None, - current: CacheCreationTokenDetails | None, - ) -> CacheCreationTokenDetails | None: - incoming: Final = cast( - CacheCreationTokenDetails | None, - getattr(prompt_tokens_details, "cache_creation_token_details", None), - ) - if incoming is not None: - return incoming - return current - - @staticmethod - def _attach_cache_creation_token_details( - prompt_tokens_details: PromptTokensDetailsWrapper | None, - cache_creation_token_details: CacheCreationTokenDetails | None, - ) -> PromptTokensDetailsWrapper | None: - if prompt_tokens_details is None or cache_creation_token_details is None: - return prompt_tokens_details - existing: Final = cast( - CacheCreationTokenDetails | None, - getattr(prompt_tokens_details, "cache_creation_token_details", None), - ) - if existing is not None: - return prompt_tokens_details - return prompt_tokens_details.model_copy(update={"cache_creation_token_details": cache_creation_token_details}) - @staticmethod def _reset_anthropic_cursor_completion_tokens( chunks: list[dict[str, Any] | ModelResponse], diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 8c67bfd97cd..a8d2da5e3b2 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -25,6 +25,7 @@ from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.types.llms.openai import OpenAIChatCompletionChunk from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( + CacheCreationTokenDetails, CompletionTokensDetailsWrapper, Delta, LlmProviders, @@ -2251,11 +2252,17 @@ def _coerce_token_details( def calculate_total_usage(chunks: list[ModelResponse]) -> Usage: """Assume most recent usage chunk has total usage uptil then.""" + from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( + attach_cache_creation_token_details, + capture_cache_creation_token_details, + ) + prompt_tokens: int = 0 completion_tokens: int = 0 latest_usage_chunk = None prompt_tokens_details: PromptTokensDetailsWrapper | None = None completion_tokens_details: CompletionTokensDetailsWrapper | None = None + cache_creation_token_details: CacheCreationTokenDetails | None = None for chunk in chunks: if "usage" in chunk and chunk["usage"] is not None: @@ -2265,10 +2272,13 @@ def calculate_total_usage(chunks: list[ModelResponse]) -> Usage: prompt_tokens = usage.get("prompt_tokens", 0) or 0 if "completion_tokens" in usage: completion_tokens = usage.get("completion_tokens", 0) or 0 - prompt_tokens_details = ( - _coerce_token_details(usage, "prompt_tokens_details", PromptTokensDetailsWrapper) - or prompt_tokens_details + incoming_prompt_tokens_details = _coerce_token_details( + usage, "prompt_tokens_details", PromptTokensDetailsWrapper ) + cache_creation_token_details = capture_cache_creation_token_details( + incoming_prompt_tokens_details, cache_creation_token_details + ) + prompt_tokens_details = incoming_prompt_tokens_details or prompt_tokens_details completion_tokens_details = ( _coerce_token_details(usage, "completion_tokens_details", CompletionTokensDetailsWrapper) or completion_tokens_details @@ -2278,7 +2288,7 @@ def calculate_total_usage(chunks: list[ModelResponse]) -> Usage: prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, - prompt_tokens_details=prompt_tokens_details, + prompt_tokens_details=attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details), completion_tokens_details=completion_tokens_details, ) diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 36d0eed2a59..5806b37539c 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -1492,6 +1492,60 @@ def test_calculate_total_usage_preserves_prompt_cache_token_details(): assert usage.completion_tokens_details.reasoning_tokens == 2 +def test_calculate_total_usage_preserves_anthropic_cache_creation_ttl_breakdown(): + """Anthropic sends the 5m/1h cache-write split only on `message_start`; the later + `message_delta` repeats the flat count without the split. Losing it here bills 1h + cache writes at the cheaper 5m rate.""" + from litellm.litellm_core_utils.streaming_handler import calculate_total_usage + from litellm.types.utils import CacheCreationTokenDetails + + message_start_chunk = ModelResponseStream( + id="chatcmpl-1", + created=1745513206, + model="claude-sonnet-5", + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) + ], + usage=Usage( + prompt_tokens=120, + completion_tokens=1, + total_tokens=121, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, + cache_creation_tokens=100, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_5m_input_tokens=20, ephemeral_1h_input_tokens=80 + ), + ), + ), + ) + message_delta_chunk = ModelResponseStream( + id="chatcmpl-1", + created=1745513207, + model="claude-sonnet-5", + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], + usage=Usage( + prompt_tokens=120, + completion_tokens=4, + total_tokens=124, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, cache_creation_tokens=100 + ), + ), + ) + + usage = calculate_total_usage([message_start_chunk, message_delta_chunk]) + + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cache_creation_tokens == 100 + ttl_breakdown = usage.prompt_tokens_details.cache_creation_token_details + assert ttl_breakdown is not None + assert ttl_breakdown.ephemeral_5m_input_tokens == 20 + assert ttl_breakdown.ephemeral_1h_input_tokens == 80 + + @pytest.mark.asyncio async def test_openrouter_streaming_cost_after_finish_reason(logging_obj: Logging): from litellm.utils import ModelResponseListIterator From 69a80436673d634fad78e48d811157f6b8a09578 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 02:11:05 -0700 Subject: [PATCH 8/8] chore: keep base field ordering for service tier cache write costs --- litellm/types/utils.py | 8 ++++---- litellm/utils.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 70b7beefcce..77f83c5b6f8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -201,13 +201,13 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing input_cost_per_token_priority: Optional[float] # OpenAI priority service tier pricing cache_creation_input_token_cost: Optional[float] - cache_creation_input_token_cost_flex: Optional[float] # OpenAI flex service tier pricing - cache_creation_input_token_cost_priority: Optional[float] # OpenAI priority service tier pricing cache_creation_input_token_cost_above_200k_tokens: Optional[float] cache_creation_input_token_cost_above_272k_tokens: Optional[float] cache_creation_input_token_cost_above_272k_tokens_priority: Optional[float] cache_creation_input_token_cost_above_272k_tokens_flex: Optional[float] cache_creation_input_token_cost_above_1hr: Optional[float] + cache_creation_input_token_cost_flex: Optional[float] # OpenAI flex service tier pricing + cache_creation_input_token_cost_priority: Optional[float] # OpenAI priority service tier pricing cache_read_input_token_cost: Optional[float] cache_read_input_token_cost_flex: Optional[float] # OpenAI flex service tier pricing cache_read_input_token_cost_priority: Optional[float] # OpenAI priority service tier pricing @@ -3258,13 +3258,13 @@ class CustomPricingLiteLLMParams(BaseModel): input_cost_per_token_flex: Optional[float] = None input_cost_per_token_priority: Optional[float] = None cache_creation_input_token_cost: Optional[float] = None - cache_creation_input_token_cost_flex: Optional[float] = None - cache_creation_input_token_cost_priority: Optional[float] = None cache_creation_input_token_cost_above_1hr: Optional[float] = None cache_creation_input_token_cost_above_200k_tokens: Optional[float] = None cache_creation_input_token_cost_above_272k_tokens: Optional[float] = None cache_creation_input_token_cost_above_272k_tokens_priority: Optional[float] = None cache_creation_input_token_cost_above_272k_tokens_flex: Optional[float] = None + cache_creation_input_token_cost_flex: Optional[float] = None + cache_creation_input_token_cost_priority: Optional[float] = None cache_creation_input_audio_token_cost: Optional[float] = None cache_read_input_token_cost: Optional[float] = None cache_read_input_token_cost_flex: Optional[float] = None diff --git a/litellm/utils.py b/litellm/utils.py index f3e90298306..d24a4dc928f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5405,10 +5405,6 @@ def _get_model_info_helper( input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None), input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None), cache_creation_input_token_cost=_model_info.get("cache_creation_input_token_cost", None), - cache_creation_input_token_cost_flex=_model_info.get("cache_creation_input_token_cost_flex", None), - cache_creation_input_token_cost_priority=_model_info.get( - "cache_creation_input_token_cost_priority", None - ), cache_creation_input_token_cost_above_200k_tokens=_model_info.get( "cache_creation_input_token_cost_above_200k_tokens", None ), @@ -5421,6 +5417,10 @@ def _get_model_info_helper( cache_creation_input_token_cost_above_272k_tokens_flex=_model_info.get( "cache_creation_input_token_cost_above_272k_tokens_flex", None ), + cache_creation_input_token_cost_flex=_model_info.get("cache_creation_input_token_cost_flex", None), + cache_creation_input_token_cost_priority=_model_info.get( + "cache_creation_input_token_cost_priority", None + ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get(