fix(responses): map cache_write_tokens to cache_creation_input_tokens
Some checks failed
LiteLLM Rust / rustfmt, clippy, test (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

This commit is contained in:
michelligabriele 2026-07-13 14:26:15 +02:00
parent f1f0a0bacd
commit 14537d20ae
No known key found for this signature in database
7 changed files with 197 additions and 2 deletions

View file

@ -225,7 +225,8 @@ def _get_token_base_cost(
output_image_cost = _get_cost_per_unit(model_info, "output_cost_per_image_token", None)
if output_image_cost is not None:
completion_base_cost = cast(float, output_image_cost)
cache_creation_cost = cast(float, _get_cost_per_unit(model_info, cache_creation_cost_key))
cache_creation_cost_from_map = _get_cost_per_unit(model_info, cache_creation_cost_key, None)
cache_creation_cost = cache_creation_cost_from_map if cache_creation_cost_from_map is not None else prompt_base_cost
cache_creation_cost_above_1hr = cast(
float,
_get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"),

View file

@ -2011,6 +2011,9 @@ class LiteLLMCompletionResponsesConfig:
if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None:
input_details_dict["audio_tokens"] = prompt_details.audio_tokens
if hasattr(prompt_details, "cache_creation_tokens") and prompt_details.cache_creation_tokens is not None:
input_details_dict["cache_write_tokens"] = prompt_details.cache_creation_tokens
if input_details_dict:
response_usage.input_tokens_details = InputTokensDetails(**input_details_dict)

View file

@ -987,10 +987,15 @@ class ResponseAPILoggingUtils:
prompt_tokens: int = response_api_usage.input_tokens or 0
completion_tokens: int = response_api_usage.output_tokens or 0
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
cache_creation_input_tokens: int | None = None
if response_api_usage.input_tokens_details:
if isinstance(response_api_usage.input_tokens_details, dict):
cache_creation_input_tokens = response_api_usage.input_tokens_details.get("cache_write_tokens")
prompt_tokens_details = PromptTokensDetailsWrapper(**response_api_usage.input_tokens_details)
else:
cache_creation_input_tokens = getattr(
response_api_usage.input_tokens_details, "cache_write_tokens", None
)
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None),
audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None),
@ -1013,6 +1018,11 @@ class ResponseAPILoggingUtils:
total_tokens=prompt_tokens + completion_tokens,
prompt_tokens_details=prompt_tokens_details,
completion_tokens_details=completion_tokens_details,
**(
{"cache_creation_input_tokens": cache_creation_input_tokens}
if cache_creation_input_tokens is not None
else {}
),
)
# Preserve cost attribute if it exists on ResponseAPIUsage

View file

@ -1196,6 +1196,7 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject):
class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
audio_tokens: Optional[int] = None
cached_tokens: int = 0
cache_write_tokens: Optional[int] = None
text_tokens: Optional[int] = None
model_config = {"extra": "allow"}

View file

@ -1916,6 +1916,41 @@ class TestUsageTransformation:
assert response_usage.input_tokens_details is None
assert response_usage.output_tokens_details is None
def test_transform_chat_usage_maps_cache_creation_to_cache_write_tokens(self):
"""Test that cache_creation_tokens (Anthropic cache writes) map to input_tokens_details.cache_write_tokens"""
# Setup: Usage with cache_creation_input_tokens (Anthropic convention)
usage = Usage(
prompt_tokens=6429,
completion_tokens=100,
total_tokens=6529,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=0),
cache_creation_input_tokens=5429,
)
chat_completion_response = ModelResponse(
id="test-response-id",
created=1234567890,
model="claude-sonnet-4",
object="chat.completion",
usage=usage,
choices=[
Choices(
finish_reason="stop",
index=0,
message=Message(content="Hello!", role="assistant"),
)
],
)
# Execute
response_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(
chat_completion_response=chat_completion_response
)
# Assert
assert response_usage.input_tokens_details is not None
assert response_usage.input_tokens_details.cache_write_tokens == 5429
def test_transform_usage_with_image_tokens(self):
"""Test that image_tokens from Vertex AI/Gemini are properly transformed to output_tokens_details"""
# Setup: Simulate Vertex AI/Gemini usage with image_tokens in completion_tokens_details

View file

@ -15,7 +15,11 @@ import litellm
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.llms.openai import (
InputTokensDetails,
ResponseAPIUsage,
ResponsesAPIOptionalRequestParams,
)
from litellm.types.utils import Usage
@ -461,6 +465,61 @@ class TestResponseAPILoggingUtils:
assert result.completion_tokens_details.text_tokens == 20
assert result.completion_tokens_details.audio_tokens is None
def test_transform_response_api_usage_maps_cache_write_tokens(self):
"""cache_write_tokens (GPT-5.6 paid cache writes) maps to cache_creation_input_tokens."""
usage = {
"input_tokens": 6429,
"output_tokens": 100,
"total_tokens": 6529,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 5429},
}
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
usage
)
assert result.prompt_tokens_details is not None
assert result.prompt_tokens_details.cache_creation_tokens == 5429
assert result._cache_creation_input_tokens == 5429
assert result.cache_creation_input_tokens == 5429
def test_transform_response_api_usage_object_input_maps_cache_write_tokens(self):
"""cache_write_tokens maps when usage is a ResponseAPIUsage object, not a dict."""
usage = ResponseAPIUsage(
input_tokens=6429,
output_tokens=100,
total_tokens=6529,
input_tokens_details=InputTokensDetails(
cached_tokens=0, cache_write_tokens=5429
),
)
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
usage
)
assert result.prompt_tokens_details is not None
assert result.prompt_tokens_details.cache_creation_tokens == 5429
assert result._cache_creation_input_tokens == 5429
def test_transform_response_api_usage_without_cache_write_tokens_unchanged(self):
"""Usage without cache_write_tokens stays identical to pre-fix output."""
usage = {
"input_tokens": 10,
"output_tokens": 20,
"total_tokens": 30,
"input_tokens_details": {"cached_tokens": 2},
}
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
usage
)
assert result.prompt_tokens_details is not None
assert result.prompt_tokens_details.cached_tokens == 2
assert getattr(result.prompt_tokens_details, "cache_creation_tokens", None) is None
assert result._cache_creation_input_tokens == 0
class TestResponsesAPIProviderSpecificParams:
"""

View file

@ -3122,6 +3122,92 @@ def test_custom_pricing_applies_cache_creation_input_cost_via_cache_write_tokens
assert completion_cost == pytest.approx(expected_completion)
def test_responses_usage_cache_write_tokens_billed_at_cache_creation_rate():
"""
GPT-5.6 reports paid cache writes as input_tokens_details.cache_write_tokens
on the Responses API. The Responses->Chat usage transform must map them to
cache_creation_tokens so cost calc bills them at
cache_creation_input_token_cost (1.25x input) instead of the base rate.
"""
from litellm.responses.utils import ResponseAPILoggingUtils
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
{
"input_tokens": 6429,
"output_tokens": 100,
"total_tokens": 6529,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 5429},
}
)
response = ModelResponse(
id="test-id",
created=1234567890,
model="gpt-5.6-luna",
object="chat.completion",
choices=[],
usage=usage,
)
cost = litellm.completion_cost(
completion_response=response,
model="gpt-5.6-luna",
custom_llm_provider="openai",
)
# gpt-5.6-luna: input 1e-06, cache write 1.25e-06, output 6e-06
expected = (6429 - 5429) * 1e-06 + 5429 * 1.25e-06 + 100 * 6e-06
assert cost == pytest.approx(expected)
def test_cache_creation_cost_falls_back_to_input_rate_when_unset():
"""
When a model's pricing has no cache_creation_input_token_cost (custom/DB
pricing, azure gpt-5.6), cache-write tokens must be billed at the base
input rate, not $0.
"""
pt_details = PromptTokensDetailsWrapper(cached_tokens=1000, audio_tokens=0)
pt_details.cache_creation_tokens = 500
usage = Usage(
prompt_tokens=4000,
completion_tokens=100,
total_tokens=4100,
prompt_tokens_details=pt_details,
)
response = ModelResponse(
id="test-id",
created=1234567890,
model="openai/gpt-5.4",
object="chat.completion",
choices=[],
usage=usage,
)
cost = litellm.completion_cost(
completion_response=response,
model="openai/gpt-5.4",
custom_llm_provider="openai",
custom_cost_per_token={
"input_cost_per_token": 0.0000025,
"output_cost_per_token": 0.000015,
"cache_read_input_token_cost": 0.00000025,
},
)
expected = (
(4000 - 1000 - 500) * 0.0000025
+ 1000 * 0.00000025
+ 500 * 0.0000025 # write tokens fall back to base input rate
+ 100 * 0.000015
)
assert cost == pytest.approx(expected)
# ---------------------------------------------------------------------------
# Bug 2 — db_spend_update_writer cache token extraction helpers.
# ---------------------------------------------------------------------------