From 0d04e12bc0c32324a2ac8330bff59b913a086d22 Mon Sep 17 00:00:00 2001 From: milan Date: Thu, 23 Jul 2026 18:46:52 +0000 Subject: [PATCH] fix(cost): account for OpenAI cache_write_tokens in cost tracking Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/llm_cost_calc/utils.py | 3 +- litellm/responses/utils.py | 10 +- litellm/types/utils.py | 6 + litellm/utils.py | 7 ++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 116 ++++++++++++++++++ .../responses/test_responses_utils.py | 53 ++++++++ 6 files changed, 193 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 33bf546c239..e4b91dcf453 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -457,7 +457,8 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: cache_creation_tokens = ( cast( Optional[int], - getattr(usage.prompt_tokens_details, "cache_creation_tokens", 0), + getattr(usage.prompt_tokens_details, "cache_creation_tokens", 0) + or getattr(usage.prompt_tokens_details, "cache_write_tokens", 0), ) or 0 ) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index ac92e5d6dcc..67a0ed75fdc 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1042,13 +1042,21 @@ class ResponseAPILoggingUtils: prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None if response_api_usage.input_tokens_details: if isinstance(response_api_usage.input_tokens_details, dict): - prompt_tokens_details = PromptTokensDetailsWrapper(**response_api_usage.input_tokens_details) + input_tokens_details = dict(response_api_usage.input_tokens_details) + cache_write_tokens = input_tokens_details.pop("cache_write_tokens", None) + if input_tokens_details.get("cache_creation_tokens") is None and cache_write_tokens is not None: + input_tokens_details["cache_creation_tokens"] = cache_write_tokens + prompt_tokens_details = PromptTokensDetailsWrapper(**input_tokens_details) else: prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None), audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None), text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None), image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None), + cache_creation_tokens=getattr( + response_api_usage.input_tokens_details, "cache_creation_tokens", None + ) + or getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), ) completion_tokens_details: Optional[CompletionTokensDetailsWrapper] = None output_tokens_details = getattr(response_api_usage, "output_tokens_details", None) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 714ad372a5f..c6c835ae0c2 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -198,7 +198,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing input_cost_per_token_priority: Optional[float] # OpenAI priority service tier pricing cache_creation_input_token_cost: Optional[float] + cache_creation_input_token_cost_flex: Optional[float] # OpenAI flex service tier pricing + cache_creation_input_token_cost_priority: Optional[float] # OpenAI priority service tier pricing cache_creation_input_token_cost_above_200k_tokens: Optional[float] + cache_creation_input_token_cost_above_272k_tokens: Optional[float] cache_creation_input_token_cost_above_1hr: Optional[float] cache_read_input_token_cost: Optional[float] cache_read_input_token_cost_flex: Optional[float] # OpenAI flex service tier pricing @@ -3072,8 +3075,11 @@ class CustomPricingLiteLLMParams(BaseModel): input_cost_per_token_flex: Optional[float] = None input_cost_per_token_priority: Optional[float] = None cache_creation_input_token_cost: Optional[float] = None + cache_creation_input_token_cost_flex: Optional[float] = None + cache_creation_input_token_cost_priority: Optional[float] = None cache_creation_input_token_cost_above_1hr: Optional[float] = None cache_creation_input_token_cost_above_200k_tokens: Optional[float] = None + cache_creation_input_token_cost_above_272k_tokens: Optional[float] = None cache_creation_input_audio_token_cost: Optional[float] = None cache_read_input_token_cost: Optional[float] = None cache_read_input_token_cost_flex: Optional[float] = None diff --git a/litellm/utils.py b/litellm/utils.py index a11c5500503..53a6a4fbbcf 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5407,9 +5407,16 @@ def _get_model_info_helper( input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None), input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None), cache_creation_input_token_cost=_model_info.get("cache_creation_input_token_cost", None), + cache_creation_input_token_cost_flex=_model_info.get("cache_creation_input_token_cost_flex", None), + cache_creation_input_token_cost_priority=_model_info.get( + "cache_creation_input_token_cost_priority", None + ), cache_creation_input_token_cost_above_200k_tokens=_model_info.get( "cache_creation_input_token_cost_above_200k_tokens", None ), + cache_creation_input_token_cost_above_272k_tokens=_model_info.get( + "cache_creation_input_token_cost_above_272k_tokens", None + ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 9ff67a82f40..a2eba8f27a2 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -900,6 +900,122 @@ def test_generic_cost_per_token_anthropic_prompt_caching_with_cache_creation(): assert round(prompt_cost, 3) == 0.029 +def test_generic_cost_per_token_prices_openai_cache_write_tokens(): + """Regression for LIT-4725 / #33772: OpenAI-style models (e.g. GPT-5.6 on Bedrock) + report cache-write tokens under prompt_tokens_details.cache_write_tokens, not the + Anthropic-style cache_creation_tokens. Those tokens must be billed at + cache_creation_input_token_cost, not silently dropped to the plain input rate.""" + model = "litellm-test-openai-cache-write" + custom_llm_provider = "bedrock" + litellm.register_model( + { + model: { + "litellm_provider": custom_llm_provider, + "mode": "chat", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 4e-6, + "cache_read_input_token_cost": 1e-7, + "cache_creation_input_token_cost": 1.25e-6, + } + } + ) + + plain_tokens, cache_read, cache_write = 1_000, 5_000, 10_000 + prompt_tokens_details = PromptTokensDetailsWrapper(cached_tokens=cache_read) + prompt_tokens_details.cache_write_tokens = cache_write + usage = Usage( + prompt_tokens=plain_tokens + cache_read + cache_write, + completion_tokens=100, + total_tokens=plain_tokens + cache_read + cache_write + 100, + prompt_tokens_details=prompt_tokens_details, + ) + + try: + prompt_cost, _ = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + ) + finally: + litellm.model_cost.pop(model, None) + + expected_prompt_cost = plain_tokens * 1e-6 + cache_read * 1e-7 + cache_write * 1.25e-6 + assert prompt_cost == pytest.approx(expected_prompt_cost, rel=1e-9) + + +def test_get_model_info_registers_tiered_cache_creation_keys(): + """Regression for LIT-4725 / #33772: get_model_info must copy the tiered + cache-creation cost keys (_flex, _priority, _above_272k_tokens) onto ModelInfo, + so cache-write cost can vary by service/context tier instead of being discarded.""" + model = "litellm-test-tiered-cache-creation" + custom_llm_provider = "openai" + litellm.register_model( + { + model: { + "litellm_provider": custom_llm_provider, + "mode": "chat", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 4e-6, + "cache_creation_input_token_cost": 1.25e-6, + "cache_creation_input_token_cost_flex": 6e-7, + "cache_creation_input_token_cost_priority": 2.5e-6, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-6, + } + } + ) + + try: + model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + assert model_info["cache_creation_input_token_cost_flex"] == 6e-7 + assert model_info["cache_creation_input_token_cost_priority"] == 2.5e-6 + assert model_info["cache_creation_input_token_cost_above_272k_tokens"] == 2.5e-6 + finally: + litellm.model_cost.pop(model, None) + + +def test_generic_cost_per_token_prices_cache_write_at_above_272k_tier(): + """Regression for LIT-4725 / #33772: past the 272k threshold, OpenAI cache-write + tokens must bill at cache_creation_input_token_cost_above_272k_tokens once that key + is registered on ModelInfo, rather than the base cache-creation rate.""" + model = "litellm-test-cache-write-272k" + custom_llm_provider = "openai" + litellm.register_model( + { + model: { + "litellm_provider": custom_llm_provider, + "mode": "chat", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 4e-6, + "input_cost_per_token_above_272k_tokens": 2e-6, + "cache_creation_input_token_cost": 1.25e-6, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-6, + } + } + ) + + plain_tokens, cache_write = 200_000, 100_000 + prompt_tokens_details = PromptTokensDetailsWrapper() + prompt_tokens_details.cache_write_tokens = cache_write + usage = Usage( + prompt_tokens=plain_tokens + cache_write, + completion_tokens=100, + total_tokens=plain_tokens + cache_write + 100, + prompt_tokens_details=prompt_tokens_details, + ) + + try: + prompt_cost, _ = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + ) + finally: + litellm.model_cost.pop(model, None) + + expected_prompt_cost = plain_tokens * 2e-6 + cache_write * 2.5e-6 + assert prompt_cost == pytest.approx(expected_prompt_cost, rel=1e-9) + + def test_string_cost_values(): """Test that cost values defined as strings are properly converted to floats.""" from unittest.mock import patch diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 3a75a33fdc7..bc2a4a26104 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -461,6 +461,59 @@ class TestResponseAPILoggingUtils: assert result.completion_tokens_details.text_tokens == 20 assert result.completion_tokens_details.audio_tokens is None + def test_transform_response_api_usage_maps_cache_write_tokens_dict(self): + """Regression for LIT-4725 / #33772: the Responses API reports cache writes under + input_tokens_details.cache_write_tokens. The chat-shaped usage must carry them on + cache_creation_tokens so cost is computed identically to /chat/completions.""" + usage = { + "input_tokens": 10_000, + "output_tokens": 20, + "total_tokens": 10_020, + "input_tokens_details": {"cached_tokens": 2_000, "cache_write_tokens": 8_000}, + } + + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage) + + assert result.prompt_tokens_details is not None + assert result.prompt_tokens_details.cached_tokens == 2_000 + assert result.prompt_tokens_details.cache_creation_tokens == 8_000 + assert getattr(result.prompt_tokens_details, "cache_write_tokens", None) is None + + def test_transform_response_api_usage_cache_creation_tokens_precedence_dict(self): + """When both cache_creation_tokens and cache_write_tokens are present, the explicit + cache_creation_tokens wins (they describe the same tokens under different names).""" + usage = { + "input_tokens": 10_000, + "output_tokens": 20, + "total_tokens": 10_020, + "input_tokens_details": {"cache_creation_tokens": 5_000, "cache_write_tokens": 8_000}, + } + + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage) + + assert result.prompt_tokens_details is not None + assert result.prompt_tokens_details.cache_creation_tokens == 5_000 + + def test_transform_response_api_usage_maps_cache_write_tokens_object(self): + """Object-path counterpart: a ResponseAPIUsage whose input_tokens_details object + carries cache_write_tokens must still land on cache_creation_tokens.""" + from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage + + input_tokens_details = InputTokensDetails(cached_tokens=2_000) + input_tokens_details.cache_write_tokens = 8_000 + usage = ResponseAPIUsage( + input_tokens=10_000, + output_tokens=20, + total_tokens=10_020, + input_tokens_details=input_tokens_details, + ) + + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage) + + assert result.prompt_tokens_details is not None + assert result.prompt_tokens_details.cached_tokens == 2_000 + assert result.prompt_tokens_details.cache_creation_tokens == 8_000 + class TestResponsesAPIProviderSpecificParams: """