fix(proxy): hold the cache-write rate only for a prompt long enough to be cached

OpenAI and Azure write a prompt into their own cache from 1024 tokens up and
nothing below that, so a shorter prompt bills the plain input rate. Reserving
the cache-creation rate for every prompt held 25% more than those requests can
spend, and a cancelled request is reconciled to that inflated input cost as
real spend rather than released.

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -783,13 +783,18 @@ class PromptTokensDetailsResult(TypedDict):
 
 
 AUTO_CACHE_WRITING_PROVIDERS: Final = frozenset({"openai", "azure"})
+AUTO_CACHE_WRITING_MINIMUM_PROMPT_TOKENS: Final = 1024
 
 
-def provider_writes_prompt_to_cache_unasked(custom_llm_provider: object) -> bool:
+def provider_writes_prompt_to_cache_unasked(custom_llm_provider: object, prompt_tokens: int) -> bool:
     """OpenAI and Azure write a long prompt into their own cache without the request asking for it
-    and bill every written token at the cache-creation rate, which runs above the input rate.
-    Every other provider writes only the blocks a request marks with cache_control."""
-    return custom_llm_provider in AUTO_CACHE_WRITING_PROVIDERS
+    and bill every written token at the cache-creation rate, which runs above the input rate. Both
+    of them cache from 1024 tokens up and nothing below that, so a shorter prompt bills the plain
+    input rate. Every other provider writes only the blocks a request marks with cache_control."""
+    return (
+        custom_llm_provider in AUTO_CACHE_WRITING_PROVIDERS
+        and prompt_tokens >= AUTO_CACHE_WRITING_MINIMUM_PROMPT_TOKENS
+    )
 
 
 def normalize_service_tier(service_tier: object) -> str | None:

litellm/proxy/spend_tracking/budget_reservation.py

@@ -1138,6 +1138,7 @@ def _input_cost_for_cost_info(
         rates=rates,
         model_info=pricing.model_info,
         request_body=request_body,
+        input_tokens=estimated_input_tokens,
     )
 
 
@@ -1145,19 +1146,24 @@ def _billed_input_rate(
     rates: TokenRates,
     model_info: Mapping[str, object],
     request_body: Mapping[str, object],
+    input_tokens: int,
 ) -> float:
     """An input token costs the higher of the input and cache-creation rates wherever the prompt
     lands in the provider's cache, since every written token bills at the write rate. Holding the
     write rate for a request that marks nothing, on a provider that writes nothing unasked, would
     reserve above anything the request can bill."""
-    if _writes_the_prompt_to_cache(model_info=model_info, request_body=request_body):
+    if _writes_the_prompt_to_cache(model_info=model_info, request_body=request_body, input_tokens=input_tokens):
         return max(rates.input_rate, rates.cache_creation_rate)
     return rates.input_rate
 
 
-def _writes_the_prompt_to_cache(model_info: Mapping[str, object], request_body: Mapping[str, object]) -> bool:
+def _writes_the_prompt_to_cache(
+    model_info: Mapping[str, object],
+    request_body: Mapping[str, object],
+    input_tokens: int,
+) -> bool:
     return provider_writes_prompt_to_cache_unasked(
-        model_info.get("litellm_provider")
+        model_info.get("litellm_provider"), prompt_tokens=input_tokens
     ) or AnthropicCacheControlHook.body_has_cache_control(request_body)
 
 
@@ -1253,6 +1259,7 @@ def _max_cost_for_cost_info(
         rates=rates,
         model_info=pricing.model_info,
         request_body=request_body,
+        input_tokens=estimated_input_tokens,
     )
     output_rate: Final = _billed_output_rate(rates=rates, model_info=pricing.model_info)
     return (estimated_input_tokens * input_rate) + (output_tokens * output_multiplier * output_rate)

tests/test_litellm/proxy/test_budget_reservation.py

@@ -1513,6 +1513,31 @@ def test_cancelled_request_keeps_the_cache_write_cost_it_already_incurred():
     assert estimated_input == pytest.approx(input_tokens * cache_write_rate)
 
 
+def test_a_prompt_too_short_to_cache_reserves_the_plain_input_rate():
+    """OpenAI and Azure write a prompt into their own cache from 1024 tokens up and nothing below
+    that, so a shorter prompt bills the plain input rate. Reserving the 25% dearer write rate for
+    it holds budget the request cannot spend, and a cancelled request is reconciled to that
+    inflated number as real spend rather than as a hold that gets released."""
+    model_info = litellm.get_model_info("gpt-5.6-luna")
+    input_rate = model_info.get("input_cost_per_token")
+    cache_write_rate = model_info.get("cache_creation_input_token_cost")
+    assert input_rate is not None
+    assert cache_write_rate is not None and cache_write_rate > input_rate
+
+    below_threshold, at_threshold = [
+        estimate_request_input_cost(
+            request_body=_long_prompt_request(model="gpt-5.6-luna", max_tokens=100),
+            route="/chat/completions",
+            llm_router=None,
+            input_token_counts={"gpt-5.6-luna": input_tokens},
+        )
+        for input_tokens in (1_023, 1_024)
+    ]
+
+    assert below_threshold == pytest.approx(1_023 * input_rate)
+    assert at_threshold == pytest.approx(1_024 * cache_write_rate)
+
+
 def test_reservation_holds_the_cache_write_rate_only_when_the_request_asks_for_it():
     """Anthropic writes a prompt into its cache only where cache_control marks a block, so a
     request that marks nothing is billed the plain input rate. Holding the 25% dearer write rate