From 909f402ed4964cea7a9265abb357d7038ff75de3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 21:11:43 +0000 Subject: [PATCH] fix(cost): price OpenAI cache_write_tokens and register tiered cache-creation costs --- .../litellm_core_utils/llm_cost_calc/utils.py | 2 + litellm/responses/utils.py | 4 + litellm/types/utils.py | 6 + litellm/utils.py | 10 ++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 127 ++++++++++++++++++ .../responses/test_responses_utils.py | 49 +++++++ 6 files changed, 198 insertions(+) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 33bf546c239..2f1f9c89f42 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -461,6 +461,8 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: ) or 0 ) + if not cache_creation_tokens: + cache_creation_tokens = _coerce_token_count(getattr(usage.prompt_tokens_details, "cache_write_tokens", 0)) cache_creation_token_details = ( cast( Optional[CacheCreationTokenDetails], diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 234eb777aca..0a131041f8a 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -993,6 +993,10 @@ class ResponseAPILoggingUtils: audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None), text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None), image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None), + cache_creation_tokens=( + getattr(response_api_usage.input_tokens_details, "cache_creation_tokens", None) + or getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None) + ), ) completion_tokens_details: Optional[CompletionTokensDetailsWrapper] = None output_tokens_details = getattr(response_api_usage, "output_tokens_details", None) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ec8a9336ca7..790acc51eed 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -187,7 +187,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing input_cost_per_token_priority: Optional[float] # OpenAI priority service tier pricing cache_creation_input_token_cost: Optional[float] + cache_creation_input_token_cost_flex: Optional[float] # OpenAI flex service tier pricing + cache_creation_input_token_cost_priority: Optional[float] # OpenAI priority service tier pricing cache_creation_input_token_cost_above_200k_tokens: Optional[float] + cache_creation_input_token_cost_above_272k_tokens: Optional[float] cache_creation_input_token_cost_above_1hr: Optional[float] cache_read_input_token_cost: Optional[float] cache_read_input_token_cost_flex: Optional[float] # OpenAI flex service tier pricing @@ -3038,8 +3041,11 @@ class CustomPricingLiteLLMParams(BaseModel): input_cost_per_token_flex: Optional[float] = None input_cost_per_token_priority: Optional[float] = None cache_creation_input_token_cost: Optional[float] = None + cache_creation_input_token_cost_flex: Optional[float] = None + cache_creation_input_token_cost_priority: Optional[float] = None cache_creation_input_token_cost_above_1hr: Optional[float] = None cache_creation_input_token_cost_above_200k_tokens: Optional[float] = None + cache_creation_input_token_cost_above_272k_tokens: Optional[float] = None cache_creation_input_audio_token_cost: Optional[float] = None cache_read_input_token_cost: Optional[float] = None cache_read_input_token_cost_flex: Optional[float] = None diff --git a/litellm/utils.py b/litellm/utils.py index e19d2b36a52..ee55bc94329 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2611,8 +2611,11 @@ _BEDROCK_REGION_PREFIXES = ( _CACHE_PRICING_FIELDS = ( "cache_creation_input_token_cost", + "cache_creation_input_token_cost_flex", + "cache_creation_input_token_cost_priority", "cache_creation_input_token_cost_above_1hr", "cache_creation_input_token_cost_above_200k_tokens", + "cache_creation_input_token_cost_above_272k_tokens", "cache_read_input_token_cost", "cache_read_input_token_cost_above_200k_tokens", ) @@ -5399,9 +5402,16 @@ def _get_model_info_helper( input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None), input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None), cache_creation_input_token_cost=_model_info.get("cache_creation_input_token_cost", None), + cache_creation_input_token_cost_flex=_model_info.get("cache_creation_input_token_cost_flex", None), + cache_creation_input_token_cost_priority=_model_info.get( + "cache_creation_input_token_cost_priority", None + ), cache_creation_input_token_cost_above_200k_tokens=_model_info.get( "cache_creation_input_token_cost_above_200k_tokens", None ), + cache_creation_input_token_cost_above_272k_tokens=_model_info.get( + "cache_creation_input_token_cost_above_272k_tokens", None + ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index b156faf3ea6..2b566e48a15 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2237,3 +2237,130 @@ def test_token_type_cost_breakdown_applies_regional_uplift(): text_input_cost = 600 * model_info["input_cost_per_token"] * uplift assert text_output_cost + eu.reasoning_cost == pytest.approx(completion_cost) assert text_input_cost + eu.cache_read_cost == pytest.approx(prompt_cost) + + +def test_generic_cost_per_token_prices_cache_write_tokens_at_cache_creation_rate(): + """ + Regression for https://github.com/BerriAI/litellm/issues/33772. + + OpenAI(-compatible) providers report cache-write tokens under + `prompt_tokens_details.cache_write_tokens`, not the Anthropic-style + `cache_creation_tokens`. The total-cost path (generic_cost_per_token -> + _parse_prompt_tokens_details) previously ignored `cache_write_tokens`, so those + tokens were billed at the plain input rate instead of the cache-creation rate. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gpt-5.6" + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, cache_write_tokens=400 + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider="openai" + ) + model_info = litellm.get_model_info(model=model, custom_llm_provider="openai") + + expected = ( + 600 * model_info["input_cost_per_token"] + + 400 * model_info["cache_creation_input_token_cost"] + ) + assert prompt_cost == pytest.approx(expected) + # Guard against regressing to the plain-input-rate mispricing the bug described. + assert prompt_cost != pytest.approx(1000 * model_info["input_cost_per_token"]) + + +def test_get_model_info_registers_tiered_cache_creation_cost_keys(): + """ + Regression for https://github.com/BerriAI/litellm/issues/33772. + + get_model_info previously dropped the tiered cache-creation cost keys, so even + when a model config defined them they never reached ModelInfo and cache-write + cost could not vary by service/context tier. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gpt-5.6" + model_info = litellm.get_model_info(model=model, custom_llm_provider="openai") + raw = litellm.model_cost[model] + + for key in ( + "cache_creation_input_token_cost_flex", + "cache_creation_input_token_cost_priority", + "cache_creation_input_token_cost_above_272k_tokens", + ): + assert model_info[key] == raw[key] + + +def test_generic_cost_per_token_applies_priority_cache_creation_rate(): + """ + Regression for https://github.com/BerriAI/litellm/issues/33772. + + With the tiered keys registered, a `priority` service tier must price cache-write + tokens at cache_creation_input_token_cost_priority, not the standard rate. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gpt-5.6" + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, cache_write_tokens=400 + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider="openai", service_tier="priority" + ) + model_info = litellm.get_model_info(model=model, custom_llm_provider="openai") + + expected = ( + 600 * model_info["input_cost_per_token_priority"] + + 400 * model_info["cache_creation_input_token_cost_priority"] + ) + assert prompt_cost == pytest.approx(expected) + + +def test_generic_cost_per_token_applies_above_272k_cache_creation_rate(): + """ + Regression for https://github.com/BerriAI/litellm/issues/33772. + + Prompts above the 272k threshold must price cache-write tokens at + cache_creation_input_token_cost_above_272k_tokens. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gpt-5.6" + cache_write = 400 + prompt_tokens = 300_000 + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=0, + total_tokens=prompt_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, cache_write_tokens=cache_write + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider="openai" + ) + model_info = litellm.get_model_info(model=model, custom_llm_provider="openai") + + text_tokens = prompt_tokens - cache_write + expected = ( + text_tokens * model_info["input_cost_per_token_above_272k_tokens"] + + cache_write * model_info["cache_creation_input_token_cost_above_272k_tokens"] + ) + assert prompt_cost == pytest.approx(expected) diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index bbc137b959f..6c0b45e40b4 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -240,6 +240,55 @@ class TestResponseAPILoggingUtils: and result.prompt_tokens_details.cached_tokens == 2 ) + def test_transform_response_api_usage_carries_cache_write_tokens_from_object(self): + """ + Regression for https://github.com/BerriAI/litellm/issues/33772. + + When input_tokens_details arrives as a ResponseAPIUsage object (not a dict), + the OpenAI(-compatible) `cache_write_tokens` split was dropped, so cost was + computed without any cache-write tokens. It must be carried through onto the + chat-shaped `cache_creation_tokens`. + """ + from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage + + usage = ResponseAPIUsage( + input_tokens=1000, + output_tokens=0, + total_tokens=1000, + input_tokens_details=InputTokensDetails( + cached_tokens=0, cache_write_tokens=400 + ), + ) + + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + usage + ) + + assert result.prompt_tokens_details is not None + assert result.prompt_tokens_details.cache_creation_tokens == 400 + + def test_transform_response_api_usage_carries_cache_write_tokens_from_dict(self): + """ + Regression for https://github.com/BerriAI/litellm/issues/33772. + + The dict-shaped usage path must also carry the `cache_write_tokens` split + through onto `cache_creation_tokens` so the downstream cost path can price it + at the cache-creation rate. + """ + usage = { + "input_tokens": 1000, + "output_tokens": 0, + "total_tokens": 1000, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 400}, + } + + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + usage + ) + + assert result.prompt_tokens_details is not None + assert result.prompt_tokens_details.cache_creation_tokens == 400 + def test_transform_response_api_usage_with_none_values(self): """Test transformation handles None values properly""" # Setup