scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge pull request #34812 from BerriAI/litellm_fix_openai_cache_token_details_loss

Browse source 
fix(cost_tracking): keep OpenAI prompt cache token details through usage reassembly
This commit is contained in:
Mateo Wang 2026-08-05 02:21:49 -07:00 • committed by GitHub
commit 71f7fad16a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 270 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -746,7 +746,7 @@ def generic_cost_per_token(
# Check for double-counting: sum of details > prompt_tokens means overlap
total_details: Final = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens + video_tokens
has_double_counting: Final = cache_hit > 0 and total_details > usage.prompt_tokens
has_double_counting: Final = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens
if (text_tokens == 0 and prompt_tokens_details["image_count"] == 0) or has_double_counting:
text_tokens = usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens

71
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -39,6 +39,34 @@ if TYPE_CHECKING:
)
def capture_cache_creation_token_details(
prompt_tokens_details: PromptTokensDetailsWrapper | None,
current: CacheCreationTokenDetails | None,
) -> CacheCreationTokenDetails | None:
incoming: Final = cast(
CacheCreationTokenDetails | None,
getattr(prompt_tokens_details, "cache_creation_token_details", None),
)
if incoming is not None:
return incoming
return current
def attach_cache_creation_token_details(
prompt_tokens_details: PromptTokensDetailsWrapper | None,
cache_creation_token_details: CacheCreationTokenDetails | None,
) -> PromptTokensDetailsWrapper | None:
if prompt_tokens_details is None or cache_creation_token_details is None:
return prompt_tokens_details
existing: Final = cast(
CacheCreationTokenDetails | None,
getattr(prompt_tokens_details, "cache_creation_token_details", None),
)
if existing is not None:
return prompt_tokens_details
return prompt_tokens_details.model_copy(update={"cache_creation_token_details": cache_creation_token_details})
class ChunkProcessor:
def __init__(self, chunks: list, messages: list | None = None):
self.chunks = self._sort_chunks(chunks)
@ -693,21 +721,22 @@ class ChunkProcessor:
"web_search_requests",
)
prompt_tokens_details = cast(
PromptTokensDetailsWrapper | None,
usage_chunk_dict["prompt_tokens_details"],
prompt_tokens_details = (
cast(
PromptTokensDetailsWrapper | None,
usage_chunk_dict["prompt_tokens_details"],
)
or prompt_tokens_details
)
cache_creation_token_details = self._capture_cache_creation_token_details(
cache_creation_token_details = capture_cache_creation_token_details(
prompt_tokens_details, cache_creation_token_details
)
if usage_chunk_dict["cost"] is not None:
cost = usage_chunk_dict["cost"]
prompt_tokens_details = self._attach_cache_creation_token_details(
prompt_tokens_details, cache_creation_token_details
)
prompt_tokens_details = attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details)
completion_tokens = self._reset_anthropic_cursor_completion_tokens(
chunks=chunks,
@ -727,34 +756,6 @@ class ChunkProcessor:
cost=cost,
)
@staticmethod
def _capture_cache_creation_token_details(
prompt_tokens_details: PromptTokensDetailsWrapper | None,
current: CacheCreationTokenDetails | None,
) -> CacheCreationTokenDetails | None:
incoming: Final = cast(
CacheCreationTokenDetails | None,
getattr(prompt_tokens_details, "cache_creation_token_details", None),
)
if incoming is not None:
return incoming
return current
@staticmethod
def _attach_cache_creation_token_details(
prompt_tokens_details: PromptTokensDetailsWrapper | None,
cache_creation_token_details: CacheCreationTokenDetails | None,
) -> PromptTokensDetailsWrapper | None:
if prompt_tokens_details is None or cache_creation_token_details is None:
return prompt_tokens_details
existing: Final = cast(
CacheCreationTokenDetails | None,
getattr(prompt_tokens_details, "cache_creation_token_details", None),
)
if existing is not None:
return prompt_tokens_details
return prompt_tokens_details.model_copy(update={"cache_creation_token_details": cache_creation_token_details})
@staticmethod
def _reset_anthropic_cursor_completion_tokens(
chunks: list[dict[str, Any] | ModelResponse],

40
litellm/litellm_core_utils/streaming_handler.py
View file

@ -8,7 +8,7 @@ import time
import traceback
from collections.abc import AsyncIterator, Callable, Iterator
from dataclasses import dataclass
from typing import Any, Final, NoReturn, Union, cast
from typing import Any, Final, NoReturn, TypeVar, Union, cast
import anyio
import httpx
@ -25,10 +25,13 @@ from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.types.llms.openai import OpenAIChatCompletionChunk
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
CacheCreationTokenDetails,
CompletionTokensDetailsWrapper,
Delta,
LlmProviders,
ModelResponse,
ModelResponseStream,
PromptTokensDetailsWrapper,
StreamingChoices,
Usage,
)
@ -2233,11 +2236,33 @@ class CustomStreamWrapper:
return chunk
_TokenDetails = TypeVar("_TokenDetails", PromptTokensDetailsWrapper, CompletionTokensDetailsWrapper)
def _coerce_token_details(
usage: dict | BaseModel, field: str, details_type: type[_TokenDetails]
) -> _TokenDetails | None:
raw = usage.get(field) if isinstance(usage, dict) else getattr(usage, field, None)
if raw is None:
return None
if isinstance(raw, details_type):
return raw
return details_type(**(raw if isinstance(raw, dict) else raw.model_dump()))
def calculate_total_usage(chunks: list[ModelResponse]) -> Usage:
"""Assume most recent usage chunk has total usage uptil then."""
from litellm.litellm_core_utils.streaming_chunk_builder_utils import (
attach_cache_creation_token_details,
capture_cache_creation_token_details,
)
prompt_tokens: int = 0
completion_tokens: int = 0
latest_usage_chunk = None
prompt_tokens_details: PromptTokensDetailsWrapper | None = None
completion_tokens_details: CompletionTokensDetailsWrapper | None = None
cache_creation_token_details: CacheCreationTokenDetails | None = None
for chunk in chunks:
if "usage" in chunk and chunk["usage"] is not None:
@ -2247,11 +2272,24 @@ def calculate_total_usage(chunks: list[ModelResponse]) -> Usage:
prompt_tokens = usage.get("prompt_tokens", 0) or 0
if "completion_tokens" in usage:
completion_tokens = usage.get("completion_tokens", 0) or 0
incoming_prompt_tokens_details = _coerce_token_details(
usage, "prompt_tokens_details", PromptTokensDetailsWrapper
)
cache_creation_token_details = capture_cache_creation_token_details(
incoming_prompt_tokens_details, cache_creation_token_details
)
prompt_tokens_details = incoming_prompt_tokens_details or prompt_tokens_details
completion_tokens_details = (
_coerce_token_details(usage, "completion_tokens_details", CompletionTokensDetailsWrapper)
or completion_tokens_details
)
returned_usage_chunk: Final = Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
prompt_tokens_details=attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details),
completion_tokens_details=completion_tokens_details,
)
if latest_usage_chunk is not None:

6
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -2027,6 +2027,12 @@ class LiteLLMCompletionResponsesConfig:
if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None:
input_details_dict["audio_tokens"] = prompt_details.audio_tokens
cache_write_tokens = getattr(prompt_details, "cache_write_tokens", None) or getattr(
prompt_details, "cache_creation_tokens", None
)
if cache_write_tokens is not None:
input_details_dict["cache_write_tokens"] = cache_write_tokens
if input_details_dict:
response_usage.input_tokens_details = InputTokensDetails(**input_details_dict)

26
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -2233,6 +2233,32 @@ def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6():
assert prompt_cost > 1000 * info["input_cost_per_token"]
def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens():
"""
Regression for #34801: when a provider reports text_tokens covering the whole
prompt alongside cache-write tokens (and no cache reads), the cache-write tokens
must be backed out of the text total instead of being billed twice.
"""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
model = "gpt-5.6"
usage = Usage(
prompt_tokens=1000,
completion_tokens=10,
total_tokens=1010,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=0, cache_write_tokens=800, text_tokens=1000
),
)
prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai")
info = litellm.get_model_info(model=model, custom_llm_provider="openai")
expected_prompt = 200 * info["input_cost_per_token"] + 800 * info["cache_creation_input_token_cost"]
assert prompt_cost == pytest.approx(expected_prompt)
def test_token_type_cost_breakdown_reconciles_with_generic_total():
"""
Both-ways check: the reasoning subset must sum with the remaining (text) output

43
tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py
View file

@ -994,6 +994,49 @@ def test_cost_field_in_usage_chunks():
assert usage.completion_tokens == 5
def test_prompt_tokens_details_survive_later_usage_chunk_without_details():
"""Regression for #34801: a trailing usage chunk that omits
`prompt_tokens_details` must not wipe the OpenAI cache-read/cache-write split,
otherwise those tokens get re-priced at the uncached input rate."""
from litellm.types.utils import PromptTokensDetailsWrapper
chunk_with_details = ModelResponseStream(
id="chatcmpl-1",
created=1745513206,
model="openai/gpt-5.6-sol",
choices=[
StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))
],
usage=Usage(
prompt_tokens=6017,
completion_tokens=4,
total_tokens=6021,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=6004, cache_write_tokens=10
),
),
)
chunk_without_details = ModelResponseStream(
id="chatcmpl-1",
created=1745513207,
model="openai/gpt-5.6-sol",
choices=[
StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))
],
usage=Usage(prompt_tokens=6017, completion_tokens=4, total_tokens=6021),
)
chunks = [chunk_with_details, chunk_without_details]
usage = ChunkProcessor(chunks=chunks).calculate_usage(
chunks=chunks, model="openai/gpt-5.6-sol", completion_output="Hi"
)
assert usage.prompt_tokens == 6017
assert usage.prompt_tokens_details is not None
assert usage.prompt_tokens_details.cached_tokens == 6004
assert usage.prompt_tokens_details.cache_write_tokens == 10
def test_get_combined_tool_content_custom_tool_call():
from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor
from litellm.types.utils import ChatCompletionMessageCustomToolCall

97
tests/test_litellm/litellm_core_utils/test_streaming_handler.py
View file

@ -1449,6 +1449,103 @@ def test_calculate_total_usage_with_dict_usage_cost():
assert getattr(usage, "cost", None) == 0.00025
def test_calculate_total_usage_preserves_prompt_cache_token_details():
"""Regression for #34801: dropping `prompt_tokens_details` here re-prices OpenAI
cache-read tokens at the uncached input rate, overstating spend."""
from litellm.litellm_core_utils.streaming_handler import calculate_total_usage
usage_with_details = Usage(
prompt_tokens=6017,
completion_tokens=4,
total_tokens=6021,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=6004, cache_write_tokens=10
),
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=2),
)
chunk_with_details = ModelResponseStream(
id="chatcmpl-1",
created=1745513206,
model="openai/gpt-5.6-sol",
choices=[
StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))
],
usage=usage_with_details,
)
chunk_without_details = ModelResponseStream(
id="chatcmpl-1",
created=1745513207,
model="openai/gpt-5.6-sol",
choices=[
StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))
],
usage=Usage(prompt_tokens=6017, completion_tokens=4, total_tokens=6021),
)
usage = calculate_total_usage([chunk_with_details, chunk_without_details])
assert usage.prompt_tokens == 6017
assert usage.prompt_tokens_details is not None
assert usage.prompt_tokens_details.cached_tokens == 6004
assert usage.prompt_tokens_details.cache_write_tokens == 10
assert usage.completion_tokens_details is not None
assert usage.completion_tokens_details.reasoning_tokens == 2
def test_calculate_total_usage_preserves_anthropic_cache_creation_ttl_breakdown():
"""Anthropic sends the 5m/1h cache-write split only on `message_start`; the later
`message_delta` repeats the flat count without the split. Losing it here bills 1h
cache writes at the cheaper 5m rate."""
from litellm.litellm_core_utils.streaming_handler import calculate_total_usage
from litellm.types.utils import CacheCreationTokenDetails
message_start_chunk = ModelResponseStream(
id="chatcmpl-1",
created=1745513206,
model="claude-sonnet-5",
choices=[
StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))
],
usage=Usage(
prompt_tokens=120,
completion_tokens=1,
total_tokens=121,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=0,
cache_creation_tokens=100,
cache_creation_token_details=CacheCreationTokenDetails(
ephemeral_5m_input_tokens=20, ephemeral_1h_input_tokens=80
),
),
),
)
message_delta_chunk = ModelResponseStream(
id="chatcmpl-1",
created=1745513207,
model="claude-sonnet-5",
choices=[
StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))
],
usage=Usage(
prompt_tokens=120,
completion_tokens=4,
total_tokens=124,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=0, cache_creation_tokens=100
),
),
)
usage = calculate_total_usage([message_start_chunk, message_delta_chunk])
assert usage.prompt_tokens_details is not None
assert usage.prompt_tokens_details.cache_creation_tokens == 100
ttl_breakdown = usage.prompt_tokens_details.cache_creation_token_details
assert ttl_breakdown is not None
assert ttl_breakdown.ephemeral_5m_input_tokens == 20
assert ttl_breakdown.ephemeral_1h_input_tokens == 80
@pytest.mark.asyncio
async def test_openrouter_streaming_cost_after_finish_reason(logging_obj: Logging):
from litellm.utils import ModelResponseListIterator

21
tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -1772,6 +1772,27 @@ class TestUsageTransformation:
assert response_usage.input_tokens_details.cached_tokens == 3
assert response_usage.input_tokens_details.text_tokens == 6
def test_transform_usage_preserves_cache_write_tokens(self):
"""Regression for #34801: the chat-completions to Responses bridge dropped
cache-write tokens, so cache-creation billing disappeared on that route."""
usage = Usage(
prompt_tokens=1000,
completion_tokens=10,
total_tokens=1010,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=100,
cache_write_tokens=800,
),
)
response_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(
chat_completion_response=usage
)
assert response_usage.input_tokens_details is not None
assert response_usage.input_tokens_details.cached_tokens == 100
assert getattr(response_usage.input_tokens_details, "cache_write_tokens", None) == 800
def test_transform_usage_with_reasoning_tokens_gemini(self):
"""Test that reasoning_tokens from Gemini are properly transformed to output_tokens_details"""
# Setup: Simulate Gemini usage with thoughtsTokenCount

5
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx
View file

@ -114,10 +114,7 @@ const CacheLeakageCard: React.FC<CacheLeakageCardProps> = ({ activity }) => {
<AdvancedDatePicker value={dateValue} onValueChange={onDateChange} />
</div>
</div>
<Tabs
value={dimension}
onValueChange={(value) => setDimension(value === "model" ? "model" : "key")}
>
<Tabs value={dimension} onValueChange={(value) => setDimension(value === "model" ? "model" : "key")}>
<TabsList>
<TabsTrigger value="key">By virtual key</TabsTrigger>
<TabsTrigger value="model">By model</TabsTrigger>