mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-15 23:31:29 +00:00
fix(responses): map cache_write_tokens to cache_creation_input_tokens
This commit is contained in:
parent
f1f0a0bacd
commit
14537d20ae
7 changed files with 197 additions and 2 deletions
|
|
@ -225,7 +225,8 @@ def _get_token_base_cost(
|
|||
output_image_cost = _get_cost_per_unit(model_info, "output_cost_per_image_token", None)
|
||||
if output_image_cost is not None:
|
||||
completion_base_cost = cast(float, output_image_cost)
|
||||
cache_creation_cost = cast(float, _get_cost_per_unit(model_info, cache_creation_cost_key))
|
||||
cache_creation_cost_from_map = _get_cost_per_unit(model_info, cache_creation_cost_key, None)
|
||||
cache_creation_cost = cache_creation_cost_from_map if cache_creation_cost_from_map is not None else prompt_base_cost
|
||||
cache_creation_cost_above_1hr = cast(
|
||||
float,
|
||||
_get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"),
|
||||
|
|
|
|||
|
|
@ -2011,6 +2011,9 @@ class LiteLLMCompletionResponsesConfig:
|
|||
if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None:
|
||||
input_details_dict["audio_tokens"] = prompt_details.audio_tokens
|
||||
|
||||
if hasattr(prompt_details, "cache_creation_tokens") and prompt_details.cache_creation_tokens is not None:
|
||||
input_details_dict["cache_write_tokens"] = prompt_details.cache_creation_tokens
|
||||
|
||||
if input_details_dict:
|
||||
response_usage.input_tokens_details = InputTokensDetails(**input_details_dict)
|
||||
|
||||
|
|
|
|||
|
|
@ -987,10 +987,15 @@ class ResponseAPILoggingUtils:
|
|||
prompt_tokens: int = response_api_usage.input_tokens or 0
|
||||
completion_tokens: int = response_api_usage.output_tokens or 0
|
||||
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
|
||||
cache_creation_input_tokens: int | None = None
|
||||
if response_api_usage.input_tokens_details:
|
||||
if isinstance(response_api_usage.input_tokens_details, dict):
|
||||
cache_creation_input_tokens = response_api_usage.input_tokens_details.get("cache_write_tokens")
|
||||
prompt_tokens_details = PromptTokensDetailsWrapper(**response_api_usage.input_tokens_details)
|
||||
else:
|
||||
cache_creation_input_tokens = getattr(
|
||||
response_api_usage.input_tokens_details, "cache_write_tokens", None
|
||||
)
|
||||
prompt_tokens_details = PromptTokensDetailsWrapper(
|
||||
cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None),
|
||||
audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None),
|
||||
|
|
@ -1013,6 +1018,11 @@ class ResponseAPILoggingUtils:
|
|||
total_tokens=prompt_tokens + completion_tokens,
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
completion_tokens_details=completion_tokens_details,
|
||||
**(
|
||||
{"cache_creation_input_tokens": cache_creation_input_tokens}
|
||||
if cache_creation_input_tokens is not None
|
||||
else {}
|
||||
),
|
||||
)
|
||||
|
||||
# Preserve cost attribute if it exists on ResponseAPIUsage
|
||||
|
|
|
|||
|
|
@ -1196,6 +1196,7 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject):
|
|||
class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
|
||||
audio_tokens: Optional[int] = None
|
||||
cached_tokens: int = 0
|
||||
cache_write_tokens: Optional[int] = None
|
||||
text_tokens: Optional[int] = None
|
||||
|
||||
model_config = {"extra": "allow"}
|
||||
|
|
|
|||
|
|
@ -1916,6 +1916,41 @@ class TestUsageTransformation:
|
|||
assert response_usage.input_tokens_details is None
|
||||
assert response_usage.output_tokens_details is None
|
||||
|
||||
def test_transform_chat_usage_maps_cache_creation_to_cache_write_tokens(self):
|
||||
"""Test that cache_creation_tokens (Anthropic cache writes) map to input_tokens_details.cache_write_tokens"""
|
||||
# Setup: Usage with cache_creation_input_tokens (Anthropic convention)
|
||||
usage = Usage(
|
||||
prompt_tokens=6429,
|
||||
completion_tokens=100,
|
||||
total_tokens=6529,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=0),
|
||||
cache_creation_input_tokens=5429,
|
||||
)
|
||||
|
||||
chat_completion_response = ModelResponse(
|
||||
id="test-response-id",
|
||||
created=1234567890,
|
||||
model="claude-sonnet-4",
|
||||
object="chat.completion",
|
||||
usage=usage,
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
message=Message(content="Hello!", role="assistant"),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
# Execute
|
||||
response_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(
|
||||
chat_completion_response=chat_completion_response
|
||||
)
|
||||
|
||||
# Assert
|
||||
assert response_usage.input_tokens_details is not None
|
||||
assert response_usage.input_tokens_details.cache_write_tokens == 5429
|
||||
|
||||
def test_transform_usage_with_image_tokens(self):
|
||||
"""Test that image_tokens from Vertex AI/Gemini are properly transformed to output_tokens_details"""
|
||||
# Setup: Simulate Vertex AI/Gemini usage with image_tokens in completion_tokens_details
|
||||
|
|
|
|||
|
|
@ -15,7 +15,11 @@ import litellm
|
|||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
|
||||
from litellm.types.llms.openai import (
|
||||
InputTokensDetails,
|
||||
ResponseAPIUsage,
|
||||
ResponsesAPIOptionalRequestParams,
|
||||
)
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
|
|
@ -461,6 +465,61 @@ class TestResponseAPILoggingUtils:
|
|||
assert result.completion_tokens_details.text_tokens == 20
|
||||
assert result.completion_tokens_details.audio_tokens is None
|
||||
|
||||
def test_transform_response_api_usage_maps_cache_write_tokens(self):
|
||||
"""cache_write_tokens (GPT-5.6 paid cache writes) maps to cache_creation_input_tokens."""
|
||||
usage = {
|
||||
"input_tokens": 6429,
|
||||
"output_tokens": 100,
|
||||
"total_tokens": 6529,
|
||||
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 5429},
|
||||
}
|
||||
|
||||
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
|
||||
usage
|
||||
)
|
||||
|
||||
assert result.prompt_tokens_details is not None
|
||||
assert result.prompt_tokens_details.cache_creation_tokens == 5429
|
||||
assert result._cache_creation_input_tokens == 5429
|
||||
assert result.cache_creation_input_tokens == 5429
|
||||
|
||||
def test_transform_response_api_usage_object_input_maps_cache_write_tokens(self):
|
||||
"""cache_write_tokens maps when usage is a ResponseAPIUsage object, not a dict."""
|
||||
usage = ResponseAPIUsage(
|
||||
input_tokens=6429,
|
||||
output_tokens=100,
|
||||
total_tokens=6529,
|
||||
input_tokens_details=InputTokensDetails(
|
||||
cached_tokens=0, cache_write_tokens=5429
|
||||
),
|
||||
)
|
||||
|
||||
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
|
||||
usage
|
||||
)
|
||||
|
||||
assert result.prompt_tokens_details is not None
|
||||
assert result.prompt_tokens_details.cache_creation_tokens == 5429
|
||||
assert result._cache_creation_input_tokens == 5429
|
||||
|
||||
def test_transform_response_api_usage_without_cache_write_tokens_unchanged(self):
|
||||
"""Usage without cache_write_tokens stays identical to pre-fix output."""
|
||||
usage = {
|
||||
"input_tokens": 10,
|
||||
"output_tokens": 20,
|
||||
"total_tokens": 30,
|
||||
"input_tokens_details": {"cached_tokens": 2},
|
||||
}
|
||||
|
||||
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
|
||||
usage
|
||||
)
|
||||
|
||||
assert result.prompt_tokens_details is not None
|
||||
assert result.prompt_tokens_details.cached_tokens == 2
|
||||
assert getattr(result.prompt_tokens_details, "cache_creation_tokens", None) is None
|
||||
assert result._cache_creation_input_tokens == 0
|
||||
|
||||
|
||||
class TestResponsesAPIProviderSpecificParams:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -3122,6 +3122,92 @@ def test_custom_pricing_applies_cache_creation_input_cost_via_cache_write_tokens
|
|||
assert completion_cost == pytest.approx(expected_completion)
|
||||
|
||||
|
||||
def test_responses_usage_cache_write_tokens_billed_at_cache_creation_rate():
|
||||
"""
|
||||
GPT-5.6 reports paid cache writes as input_tokens_details.cache_write_tokens
|
||||
on the Responses API. The Responses->Chat usage transform must map them to
|
||||
cache_creation_tokens so cost calc bills them at
|
||||
cache_creation_input_token_cost (1.25x input) instead of the base rate.
|
||||
"""
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
|
||||
{
|
||||
"input_tokens": 6429,
|
||||
"output_tokens": 100,
|
||||
"total_tokens": 6529,
|
||||
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 5429},
|
||||
}
|
||||
)
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="gpt-5.6-luna",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="gpt-5.6-luna",
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
# gpt-5.6-luna: input 1e-06, cache write 1.25e-06, output 6e-06
|
||||
expected = (6429 - 5429) * 1e-06 + 5429 * 1.25e-06 + 100 * 6e-06
|
||||
|
||||
assert cost == pytest.approx(expected)
|
||||
|
||||
|
||||
def test_cache_creation_cost_falls_back_to_input_rate_when_unset():
|
||||
"""
|
||||
When a model's pricing has no cache_creation_input_token_cost (custom/DB
|
||||
pricing, azure gpt-5.6), cache-write tokens must be billed at the base
|
||||
input rate, not $0.
|
||||
"""
|
||||
pt_details = PromptTokensDetailsWrapper(cached_tokens=1000, audio_tokens=0)
|
||||
pt_details.cache_creation_tokens = 500
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=4000,
|
||||
completion_tokens=100,
|
||||
total_tokens=4100,
|
||||
prompt_tokens_details=pt_details,
|
||||
)
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="openai/gpt-5.4",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="openai/gpt-5.4",
|
||||
custom_llm_provider="openai",
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": 0.0000025,
|
||||
"output_cost_per_token": 0.000015,
|
||||
"cache_read_input_token_cost": 0.00000025,
|
||||
},
|
||||
)
|
||||
|
||||
expected = (
|
||||
(4000 - 1000 - 500) * 0.0000025
|
||||
+ 1000 * 0.00000025
|
||||
+ 500 * 0.0000025 # write tokens fall back to base input rate
|
||||
+ 100 * 0.000015
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(expected)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bug 2 — db_spend_update_writer cache token extraction helpers.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue