diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index b4899d4a7e9..105994b827e 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -783,13 +783,18 @@ class PromptTokensDetailsResult(TypedDict): AUTO_CACHE_WRITING_PROVIDERS: Final = frozenset({"openai", "azure"}) +AUTO_CACHE_WRITING_MINIMUM_PROMPT_TOKENS: Final = 1024 -def provider_writes_prompt_to_cache_unasked(custom_llm_provider: object) -> bool: +def provider_writes_prompt_to_cache_unasked(custom_llm_provider: object, prompt_tokens: int) -> bool: """OpenAI and Azure write a long prompt into their own cache without the request asking for it - and bill every written token at the cache-creation rate, which runs above the input rate. - Every other provider writes only the blocks a request marks with cache_control.""" - return custom_llm_provider in AUTO_CACHE_WRITING_PROVIDERS + and bill every written token at the cache-creation rate, which runs above the input rate. Both + of them cache from 1024 tokens up and nothing below that, so a shorter prompt bills the plain + input rate. Every other provider writes only the blocks a request marks with cache_control.""" + return ( + custom_llm_provider in AUTO_CACHE_WRITING_PROVIDERS + and prompt_tokens >= AUTO_CACHE_WRITING_MINIMUM_PROMPT_TOKENS + ) def normalize_service_tier(service_tier: object) -> str | None: diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 0c8a44106a8..7628695d8fe 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -1138,6 +1138,7 @@ def _input_cost_for_cost_info( rates=rates, model_info=pricing.model_info, request_body=request_body, + input_tokens=estimated_input_tokens, ) @@ -1145,19 +1146,24 @@ def _billed_input_rate( rates: TokenRates, model_info: Mapping[str, object], request_body: Mapping[str, object], + input_tokens: int, ) -> float: """An input token costs the higher of the input and cache-creation rates wherever the prompt lands in the provider's cache, since every written token bills at the write rate. Holding the write rate for a request that marks nothing, on a provider that writes nothing unasked, would reserve above anything the request can bill.""" - if _writes_the_prompt_to_cache(model_info=model_info, request_body=request_body): + if _writes_the_prompt_to_cache(model_info=model_info, request_body=request_body, input_tokens=input_tokens): return max(rates.input_rate, rates.cache_creation_rate) return rates.input_rate -def _writes_the_prompt_to_cache(model_info: Mapping[str, object], request_body: Mapping[str, object]) -> bool: +def _writes_the_prompt_to_cache( + model_info: Mapping[str, object], + request_body: Mapping[str, object], + input_tokens: int, +) -> bool: return provider_writes_prompt_to_cache_unasked( - model_info.get("litellm_provider") + model_info.get("litellm_provider"), prompt_tokens=input_tokens ) or AnthropicCacheControlHook.body_has_cache_control(request_body) @@ -1253,6 +1259,7 @@ def _max_cost_for_cost_info( rates=rates, model_info=pricing.model_info, request_body=request_body, + input_tokens=estimated_input_tokens, ) output_rate: Final = _billed_output_rate(rates=rates, model_info=pricing.model_info) return (estimated_input_tokens * input_rate) + (output_tokens * output_multiplier * output_rate) diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 15056de6699..39c39ca6213 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -1513,6 +1513,31 @@ def test_cancelled_request_keeps_the_cache_write_cost_it_already_incurred(): assert estimated_input == pytest.approx(input_tokens * cache_write_rate) +def test_a_prompt_too_short_to_cache_reserves_the_plain_input_rate(): + """OpenAI and Azure write a prompt into their own cache from 1024 tokens up and nothing below + that, so a shorter prompt bills the plain input rate. Reserving the 25% dearer write rate for + it holds budget the request cannot spend, and a cancelled request is reconciled to that + inflated number as real spend rather than as a hold that gets released.""" + model_info = litellm.get_model_info("gpt-5.6-luna") + input_rate = model_info.get("input_cost_per_token") + cache_write_rate = model_info.get("cache_creation_input_token_cost") + assert input_rate is not None + assert cache_write_rate is not None and cache_write_rate > input_rate + + below_threshold, at_threshold = [ + estimate_request_input_cost( + request_body=_long_prompt_request(model="gpt-5.6-luna", max_tokens=100), + route="/chat/completions", + llm_router=None, + input_token_counts={"gpt-5.6-luna": input_tokens}, + ) + for input_tokens in (1_023, 1_024) + ] + + assert below_threshold == pytest.approx(1_023 * input_rate) + assert at_threshold == pytest.approx(1_024 * cache_write_rate) + + def test_reservation_holds_the_cache_write_rate_only_when_the_request_asks_for_it(): """Anthropic writes a prompt into its cache only where cache_control marks a block, so a request that marks nothing is billed the plain input rate. Holding the 25% dearer write rate