diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index aeef604510b..524190c7950 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -746,7 +746,7 @@ def generic_cost_per_token( # Check for double-counting: sum of details > prompt_tokens means overlap total_details: Final = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens + video_tokens - has_double_counting: Final = cache_hit > 0 and total_details > usage.prompt_tokens + has_double_counting: Final = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens if (text_tokens == 0 and prompt_tokens_details["image_count"] == 0) or has_double_counting: text_tokens = usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index a2f9c80f577..3f967e29002 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -39,6 +39,34 @@ if TYPE_CHECKING: ) +def capture_cache_creation_token_details( + prompt_tokens_details: PromptTokensDetailsWrapper | None, + current: CacheCreationTokenDetails | None, +) -> CacheCreationTokenDetails | None: + incoming: Final = cast( + CacheCreationTokenDetails | None, + getattr(prompt_tokens_details, "cache_creation_token_details", None), + ) + if incoming is not None: + return incoming + return current + + +def attach_cache_creation_token_details( + prompt_tokens_details: PromptTokensDetailsWrapper | None, + cache_creation_token_details: CacheCreationTokenDetails | None, +) -> PromptTokensDetailsWrapper | None: + if prompt_tokens_details is None or cache_creation_token_details is None: + return prompt_tokens_details + existing: Final = cast( + CacheCreationTokenDetails | None, + getattr(prompt_tokens_details, "cache_creation_token_details", None), + ) + if existing is not None: + return prompt_tokens_details + return prompt_tokens_details.model_copy(update={"cache_creation_token_details": cache_creation_token_details}) + + class ChunkProcessor: def __init__(self, chunks: list, messages: list | None = None): self.chunks = self._sort_chunks(chunks) @@ -693,21 +721,22 @@ class ChunkProcessor: "web_search_requests", ) - prompt_tokens_details = cast( - PromptTokensDetailsWrapper | None, - usage_chunk_dict["prompt_tokens_details"], + prompt_tokens_details = ( + cast( + PromptTokensDetailsWrapper | None, + usage_chunk_dict["prompt_tokens_details"], + ) + or prompt_tokens_details ) - cache_creation_token_details = self._capture_cache_creation_token_details( + cache_creation_token_details = capture_cache_creation_token_details( prompt_tokens_details, cache_creation_token_details ) if usage_chunk_dict["cost"] is not None: cost = usage_chunk_dict["cost"] - prompt_tokens_details = self._attach_cache_creation_token_details( - prompt_tokens_details, cache_creation_token_details - ) + prompt_tokens_details = attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details) completion_tokens = self._reset_anthropic_cursor_completion_tokens( chunks=chunks, @@ -727,34 +756,6 @@ class ChunkProcessor: cost=cost, ) - @staticmethod - def _capture_cache_creation_token_details( - prompt_tokens_details: PromptTokensDetailsWrapper | None, - current: CacheCreationTokenDetails | None, - ) -> CacheCreationTokenDetails | None: - incoming: Final = cast( - CacheCreationTokenDetails | None, - getattr(prompt_tokens_details, "cache_creation_token_details", None), - ) - if incoming is not None: - return incoming - return current - - @staticmethod - def _attach_cache_creation_token_details( - prompt_tokens_details: PromptTokensDetailsWrapper | None, - cache_creation_token_details: CacheCreationTokenDetails | None, - ) -> PromptTokensDetailsWrapper | None: - if prompt_tokens_details is None or cache_creation_token_details is None: - return prompt_tokens_details - existing: Final = cast( - CacheCreationTokenDetails | None, - getattr(prompt_tokens_details, "cache_creation_token_details", None), - ) - if existing is not None: - return prompt_tokens_details - return prompt_tokens_details.model_copy(update={"cache_creation_token_details": cache_creation_token_details}) - @staticmethod def _reset_anthropic_cursor_completion_tokens( chunks: list[dict[str, Any] | ModelResponse], diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index c09c767483d..a8d2da5e3b2 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -8,7 +8,7 @@ import time import traceback from collections.abc import AsyncIterator, Callable, Iterator from dataclasses import dataclass -from typing import Any, Final, NoReturn, Union, cast +from typing import Any, Final, NoReturn, TypeVar, Union, cast import anyio import httpx @@ -25,10 +25,13 @@ from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.types.llms.openai import OpenAIChatCompletionChunk from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( + CacheCreationTokenDetails, + CompletionTokensDetailsWrapper, Delta, LlmProviders, ModelResponse, ModelResponseStream, + PromptTokensDetailsWrapper, StreamingChoices, Usage, ) @@ -2233,11 +2236,33 @@ class CustomStreamWrapper: return chunk +_TokenDetails = TypeVar("_TokenDetails", PromptTokensDetailsWrapper, CompletionTokensDetailsWrapper) + + +def _coerce_token_details( + usage: dict | BaseModel, field: str, details_type: type[_TokenDetails] +) -> _TokenDetails | None: + raw = usage.get(field) if isinstance(usage, dict) else getattr(usage, field, None) + if raw is None: + return None + if isinstance(raw, details_type): + return raw + return details_type(**(raw if isinstance(raw, dict) else raw.model_dump())) + + def calculate_total_usage(chunks: list[ModelResponse]) -> Usage: """Assume most recent usage chunk has total usage uptil then.""" + from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( + attach_cache_creation_token_details, + capture_cache_creation_token_details, + ) + prompt_tokens: int = 0 completion_tokens: int = 0 latest_usage_chunk = None + prompt_tokens_details: PromptTokensDetailsWrapper | None = None + completion_tokens_details: CompletionTokensDetailsWrapper | None = None + cache_creation_token_details: CacheCreationTokenDetails | None = None for chunk in chunks: if "usage" in chunk and chunk["usage"] is not None: @@ -2247,11 +2272,24 @@ def calculate_total_usage(chunks: list[ModelResponse]) -> Usage: prompt_tokens = usage.get("prompt_tokens", 0) or 0 if "completion_tokens" in usage: completion_tokens = usage.get("completion_tokens", 0) or 0 + incoming_prompt_tokens_details = _coerce_token_details( + usage, "prompt_tokens_details", PromptTokensDetailsWrapper + ) + cache_creation_token_details = capture_cache_creation_token_details( + incoming_prompt_tokens_details, cache_creation_token_details + ) + prompt_tokens_details = incoming_prompt_tokens_details or prompt_tokens_details + completion_tokens_details = ( + _coerce_token_details(usage, "completion_tokens_details", CompletionTokensDetailsWrapper) + or completion_tokens_details + ) returned_usage_chunk: Final = Usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details), + completion_tokens_details=completion_tokens_details, ) if latest_usage_chunk is not None: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 79e05545358..160b3fe3d2d 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2027,6 +2027,12 @@ class LiteLLMCompletionResponsesConfig: if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None: input_details_dict["audio_tokens"] = prompt_details.audio_tokens + cache_write_tokens = getattr(prompt_details, "cache_write_tokens", None) or getattr( + prompt_details, "cache_creation_tokens", None + ) + if cache_write_tokens is not None: + input_details_dict["cache_write_tokens"] = cache_write_tokens + if input_details_dict: response_usage.input_tokens_details = InputTokensDetails(**input_details_dict) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 9145e5dc76d..cc15f191714 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2233,6 +2233,32 @@ def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(): assert prompt_cost > 1000 * info["input_cost_per_token"] +def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens(): + """ + Regression for #34801: when a provider reports text_tokens covering the whole + prompt alongside cache-write tokens (and no cache reads), the cache-write tokens + must be backed out of the text total instead of being billed twice. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gpt-5.6" + usage = Usage( + prompt_tokens=1000, + completion_tokens=10, + total_tokens=1010, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, cache_write_tokens=800, text_tokens=1000 + ), + ) + + prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + + info = litellm.get_model_info(model=model, custom_llm_provider="openai") + expected_prompt = 200 * info["input_cost_per_token"] + 800 * info["cache_creation_input_token_cost"] + assert prompt_cost == pytest.approx(expected_prompt) + + def test_token_type_cost_breakdown_reconciles_with_generic_total(): """ Both-ways check: the reasoning subset must sum with the remaining (text) output diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 0114db381cf..6edde02769b 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -994,6 +994,49 @@ def test_cost_field_in_usage_chunks(): assert usage.completion_tokens == 5 +def test_prompt_tokens_details_survive_later_usage_chunk_without_details(): + """Regression for #34801: a trailing usage chunk that omits + `prompt_tokens_details` must not wipe the OpenAI cache-read/cache-write split, + otherwise those tokens get re-priced at the uncached input rate.""" + from litellm.types.utils import PromptTokensDetailsWrapper + + chunk_with_details = ModelResponseStream( + id="chatcmpl-1", + created=1745513206, + model="openai/gpt-5.6-sol", + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) + ], + usage=Usage( + prompt_tokens=6017, + completion_tokens=4, + total_tokens=6021, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=6004, cache_write_tokens=10 + ), + ), + ) + chunk_without_details = ModelResponseStream( + id="chatcmpl-1", + created=1745513207, + model="openai/gpt-5.6-sol", + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], + usage=Usage(prompt_tokens=6017, completion_tokens=4, total_tokens=6021), + ) + + chunks = [chunk_with_details, chunk_without_details] + usage = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, model="openai/gpt-5.6-sol", completion_output="Hi" + ) + + assert usage.prompt_tokens == 6017 + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cached_tokens == 6004 + assert usage.prompt_tokens_details.cache_write_tokens == 10 + + def test_get_combined_tool_content_custom_tool_call(): from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor from litellm.types.utils import ChatCompletionMessageCustomToolCall diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 48bc3709517..5806b37539c 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -1449,6 +1449,103 @@ def test_calculate_total_usage_with_dict_usage_cost(): assert getattr(usage, "cost", None) == 0.00025 +def test_calculate_total_usage_preserves_prompt_cache_token_details(): + """Regression for #34801: dropping `prompt_tokens_details` here re-prices OpenAI + cache-read tokens at the uncached input rate, overstating spend.""" + from litellm.litellm_core_utils.streaming_handler import calculate_total_usage + + usage_with_details = Usage( + prompt_tokens=6017, + completion_tokens=4, + total_tokens=6021, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=6004, cache_write_tokens=10 + ), + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=2), + ) + chunk_with_details = ModelResponseStream( + id="chatcmpl-1", + created=1745513206, + model="openai/gpt-5.6-sol", + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) + ], + usage=usage_with_details, + ) + chunk_without_details = ModelResponseStream( + id="chatcmpl-1", + created=1745513207, + model="openai/gpt-5.6-sol", + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], + usage=Usage(prompt_tokens=6017, completion_tokens=4, total_tokens=6021), + ) + + usage = calculate_total_usage([chunk_with_details, chunk_without_details]) + + assert usage.prompt_tokens == 6017 + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cached_tokens == 6004 + assert usage.prompt_tokens_details.cache_write_tokens == 10 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 2 + + +def test_calculate_total_usage_preserves_anthropic_cache_creation_ttl_breakdown(): + """Anthropic sends the 5m/1h cache-write split only on `message_start`; the later + `message_delta` repeats the flat count without the split. Losing it here bills 1h + cache writes at the cheaper 5m rate.""" + from litellm.litellm_core_utils.streaming_handler import calculate_total_usage + from litellm.types.utils import CacheCreationTokenDetails + + message_start_chunk = ModelResponseStream( + id="chatcmpl-1", + created=1745513206, + model="claude-sonnet-5", + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) + ], + usage=Usage( + prompt_tokens=120, + completion_tokens=1, + total_tokens=121, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, + cache_creation_tokens=100, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_5m_input_tokens=20, ephemeral_1h_input_tokens=80 + ), + ), + ), + ) + message_delta_chunk = ModelResponseStream( + id="chatcmpl-1", + created=1745513207, + model="claude-sonnet-5", + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], + usage=Usage( + prompt_tokens=120, + completion_tokens=4, + total_tokens=124, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, cache_creation_tokens=100 + ), + ), + ) + + usage = calculate_total_usage([message_start_chunk, message_delta_chunk]) + + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cache_creation_tokens == 100 + ttl_breakdown = usage.prompt_tokens_details.cache_creation_token_details + assert ttl_breakdown is not None + assert ttl_breakdown.ephemeral_5m_input_tokens == 20 + assert ttl_breakdown.ephemeral_1h_input_tokens == 80 + + @pytest.mark.asyncio async def test_openrouter_streaming_cost_after_finish_reason(logging_obj: Logging): from litellm.utils import ModelResponseListIterator diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 3f3f51f0d3f..51f757c9eaf 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1772,6 +1772,27 @@ class TestUsageTransformation: assert response_usage.input_tokens_details.cached_tokens == 3 assert response_usage.input_tokens_details.text_tokens == 6 + def test_transform_usage_preserves_cache_write_tokens(self): + """Regression for #34801: the chat-completions to Responses bridge dropped + cache-write tokens, so cache-creation billing disappeared on that route.""" + usage = Usage( + prompt_tokens=1000, + completion_tokens=10, + total_tokens=1010, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=100, + cache_write_tokens=800, + ), + ) + + response_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( + chat_completion_response=usage + ) + + assert response_usage.input_tokens_details is not None + assert response_usage.input_tokens_details.cached_tokens == 100 + assert getattr(response_usage.input_tokens_details, "cache_write_tokens", None) == 800 + def test_transform_usage_with_reasoning_tokens_gemini(self): """Test that reasoning_tokens from Gemini are properly transformed to output_tokens_details""" # Setup: Simulate Gemini usage with thoughtsTokenCount diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx index bb2cb865877..3bc5443b2ea 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx @@ -114,10 +114,7 @@ const CacheLeakageCard: React.FC = ({ activity }) => { - setDimension(value === "model" ? "model" : "key")} - > + setDimension(value === "model" ? "model" : "key")}> By virtual key By model