diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 9432fefc368..68dc27ec25e 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -514,6 +514,7 @@ def _get_token_base_cost( current_time: datetime | None = None, *, threshold_is_inclusive: bool = False, + missing_cache_read_uses_input: bool = False, ) -> tuple[float, float, float, float, float]: """ Return prompt cost, completion cost, and cache costs for a given model and usage. @@ -524,6 +525,9 @@ def _get_token_base_cost( `threshold_is_inclusive` switches that comparison to >=, for providers such as xAI that bill the higher tier once the prompt reaches the threshold. + `missing_cache_read_uses_input` resolves an absent cache-read rate to the resolved + input rate instead of 0.0; an explicit 0.0 rate stays a real price either way. + Returns: Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost) """ @@ -551,29 +555,16 @@ def _get_token_base_cost( float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"), ) - cache_read_cost = cast(float, _get_cost_per_unit(model_info, cache_read_cost_key)) + cache_read_cost = _get_cost_per_unit(model_info, cache_read_cost_key, default_value=None) ## CHECK IF ABOVE THRESHOLD # Optimization: collect threshold keys first to avoid sorting all model_info keys. - # Most models don't have threshold pricing, so we can return early. # Exclude service_tier-specific variants (e.g. input_cost_per_token_above_200k_tokens_priority) # so that the threshold detection loop only processes standard keys. The # service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key. threshold_keys: Final = [ k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES) ] - if not threshold_keys: - return _apply_off_peak_to_base_costs( - model_info, - current_time, - ( - prompt_base_cost, - completion_base_cost, - cache_creation_cost, - cache_creation_cost_above_1hr, - cache_read_cost, - ), - ) # Only sort the threshold keys (typically 1-2 keys instead of 66+) threshold: float | None = None @@ -662,10 +653,7 @@ def _get_token_base_cost( ), ) - cache_read_cost = cast( - float, - _get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost), - ) + cache_read_cost = _get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost) break except (IndexError, ValueError): @@ -673,6 +661,17 @@ def _get_token_base_cost( except Exception: continue + if cache_read_cost is None: + cache_read_cost = ( + _off_peak_rate( + _open_off_peak_block(model_info, current_time) or MappingProxyType({}), + "input_cost_per_token", + prompt_base_cost, + ) + if missing_cache_read_uses_input + else 0.0 + ) + return _apply_off_peak_to_base_costs( model_info, current_time, @@ -1416,6 +1415,57 @@ def get_token_type_cost_breakdown( ) +def calculate_prompt_caching_savings( + model_info: ModelInfo, + usage: Usage, + custom_llm_provider: str | None, + service_tier: str | None = None, + data_residency: str | None = None, + vertex_location: str | None = None, + billed_at: datetime | None = None, +) -> float: + """Read discount minus write premium, using the biller's rate and TTL resolution. + + Missing reads and unpublished (missing/zero) writes claim no saving or premium; + explicit zero reads remain free. An unpublished 1h price uses the ordinary write rate. + ``billed_at`` is the request's completion time, so off-peak windows resolve as the + biller saw them rather than at the later spend write. + """ + prompt_base_cost, _, cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier=service_tier, + current_time=billed_at, + threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), + missing_cache_read_uses_input=True, + ) + write_rate: Final = cache_creation_cost or prompt_base_cost + write_rate_1h: Final = cache_creation_cost_above_1hr or write_rate + prompt_tokens_details: Final = parse_prompt_tokens_details(usage) + cache_read_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"], 0) + cache_creation_tokens: Final = max(prompt_tokens_details["cache_creation_tokens"], 0) + details: Final = prompt_tokens_details["cache_creation_token_details"] + cache_creation_details: Final = ( + CacheCreationTokenDetails( + ephemeral_5m_input_tokens=max(details.ephemeral_5m_input_tokens or 0, 0), + ephemeral_1h_input_tokens=max(details.ephemeral_1h_input_tokens or 0, 0), + ) + if details is not None + else None + ) + read_discount: Final = cache_read_tokens * max(prompt_base_cost - cache_read_cost, 0.0) + write_premium: Final = calculate_cache_writing_cost( + cache_creation_tokens=cache_creation_tokens, + cache_creation_token_details=cache_creation_details, + cache_creation_cost_above_1hr=write_rate_1h - prompt_base_cost, + cache_creation_cost=write_rate - prompt_base_cost, + ) + uplift: Final = _get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift( + model_info, vertex_location + ) + return (read_discount - write_premium) * uplift + + def calculate_image_response_cost_from_usage( model: str, image_response: ImageResponse, diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 9230be8055e..914c961b145 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -502,6 +502,7 @@ class DBSpendUpdateWriter: llm_router=get_llm_router, cost_breakdown=metadata.get("cost_breakdown"), recorded_autorouter_savings=metadata.get("autorouter_savings"), + billed_at=payload.get("endTime"), ) transaction: Final = build_autorouter_turn_transaction( payload=payload, @@ -2188,6 +2189,7 @@ class DBSpendUpdateWriter: usage_object=usage_obj, cost_breakdown=_metadata.get("cost_breakdown"), recorded_autorouter_savings=_metadata.get("autorouter_savings"), + billed_at=payload.get("endTime"), ) daily_transaction: Final = BaseDailySpendTransaction( diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 1d0eb12da75..c541b9b40e5 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -9,12 +9,17 @@ have been aggregated across models. """ from collections.abc import Callable, Mapping +from datetime import datetime from typing import TYPE_CHECKING, Final, NamedTuple import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY -from litellm.litellm_core_utils.llm_cost_calc.utils import _get_cost_per_unit, generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + _get_cost_per_unit, + calculate_prompt_caching_savings, + generic_cost_per_token, +) from litellm.types.integrations.anthropic_cache_control_hook import ( GATEWAY_INJECTED_CACHE_METADATA_KEY, GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT, @@ -32,42 +37,13 @@ class SavingsSpend(NamedTuple): gateway_injected_caching: float = 0.0 -def _input_cache_read_and_write_cost(info: ModelInfo | None) -> tuple[float, float, float]: - """ - Return ``(input_cost, cache_read_cost, cache_write_cost)`` per token. - - ``info`` is whatever pricing the caller resolved -- deployment rates when the - request came through a router deployment, public rates otherwise -- so a - negotiated price is honoured here rather than silently replaced by the list rate. - ``None`` falls open to ``(0.0, 0.0, 0.0)`` so savings degrade to zero rather than - raising inside the spend writer. - - Prices are read through ``_get_cost_per_unit``, the same accessor the cost - calculator uses, which coerces the string prices a ``config.yaml`` can produce - (``"3e-7"``) and resolves service-tier suffixes. - - An absent cache price mirrors the input cost, which yields a zero discount on the - read leg and a zero premium on the write leg. Mirroring rather than taking - ``_get_cost_per_unit``'s 0.0 default is load-bearing on the write leg: a zero write - price would make the premium ``0 - input_cost``, turning a model that simply has no - write pricing into a spurious extra saving. - - The two legs then differ on an explicit ``0.0``, and the asymmetry is deliberate. A - free cache *write* does not exist -- entries carrying a literal zero (``deepseek-chat`` - does) mean "no separate price", so a falsy write price also mirrors input. A free - cache *read* is real: 15 models charge for input and serve reads for nothing, which - is the largest discount available, so the read leg keeps its literal zero. - """ - if info is None: - return 0.0, 0.0, 0.0 - input_cost: Final = _get_cost_per_unit(info, "input_cost_per_token") or 0.0 - cache_read_cost: Final = _get_cost_per_unit(info, "cache_read_input_token_cost", default_value=None) - cache_write_cost: Final = _get_cost_per_unit(info, "cache_creation_input_token_cost", default_value=None) - return ( - input_cost, - input_cost if cache_read_cost is None else cache_read_cost, - cache_write_cost if cache_write_cost else input_cost, - ) +def _coerce_billed_at(value: datetime | str | None) -> datetime | None: + if isinstance(value, datetime) or value is None: + return value + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None class _ModelIdentity(NamedTuple): @@ -586,6 +562,7 @@ def compute_savings_spend( llm_router: "Callable[[], Router | None] | None" = None, cost_breakdown: Mapping[str, object] | None = None, recorded_autorouter_savings: object = None, + billed_at: datetime | str | None = None, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -595,24 +572,10 @@ def compute_savings_spend( premium paid to write those entries, both derived here from ``usage_object`` so no caller can hand in a count that disagrees with the usage record. - The net form follows from what the request would have cost with caching off. The - provider reports ``prompt_tokens`` as the inclusive total of three disjoint - partitions (uncached text, cache reads, cache writes), so an uncached counterfactual - bills every one of those tokens at the flat input rate:: - - would_have_cost = (text + reads + writes) * input - actually_cost = text * input + reads * read_rate + writes * write_rate - savings = reads * (input - read_rate) - writes * (write_rate - input) - - So the write leg subtracts the write PREMIUM, not the whole write cost: those tokens - had to be sent either way, and the counterfactual already pays the input rate for - them. The premium stays signed, because a handful of models price writes below their - input rate and there the write is a genuine extra saving. - - A request that only writes cache and gets no hits therefore reports negative savings, - which is accurate: it really did cost more than the uncached call would have. The - daily rollup increments arithmetically, so those rows offset positive ones in the - same bucket. + The uncached counterfactual pays the ordinary input rate for the same prompt size + and tier. Cache writes subtract only the premium over that rate, split by TTL. + Savings stay signed: a write-only request can lose money, and daily rollups net + those losses against read savings. Caching is reported twice. ``prompt_caching`` is every net dollar caching saved, whoever caused it, which is what a customer means by "what did caching save me". @@ -638,12 +601,9 @@ def compute_savings_spend( calls this and only auto-routed ones need one, so looking it up eagerly at the call site would fetch and discard it on the rest. - ``cost_breakdown`` is what the cost calculator recorded for this request, and it - carries both what the request really cost and the tier and region it was priced on. - Only the auto-router driver reads it. Compression and prompt caching price a - hypothetical token delta off flat rate keys, so they are blind to tiered pricing in - the same way; that is pre-existing behaviour on two shipped drivers rather than - something introduced here, and moving those numbers is its own change. + ``cost_breakdown`` supplies the biller's tier and region to caching and auto-router + savings. Caching also uses the logged prompt size and TTL split. Compression retains + its flat input-rate estimate; changing that counterfactual is a separate concern. ``recorded_autorouter_savings`` is the figure the logging path stamped on the spend log's metadata, honoured over recomputation so the rollup, the turn table and the @@ -658,13 +618,24 @@ def compute_savings_spend( pricing: Final = _effective_model_info(router_instance, model_id, model or "") or ( _model_info(identity) if identity else None ) - input_cost, cache_read_cost, cache_write_cost = _input_cache_read_and_write_cost(pricing) + input_cost: Final = (_get_cost_per_unit(pricing, "input_cost_per_token") or 0.0) if pricing else 0.0 compression: Final = max(compression_saved_tokens, 0) * input_cost - cache_read_input_tokens: Final = extract_cache_read_tokens(usage_object) - cache_creation_input_tokens: Final = extract_cache_creation_tokens(usage_object) - read_discount: Final = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0) - write_premium: Final = max(cache_creation_input_tokens, 0) * (cache_write_cost - input_cost) - prompt_caching: Final = read_discount - write_premium + usage: Final = _usage_from_spend_log(usage_object) + basis: Final = _pricing_basis(cost_breakdown) + billed_at_datetime: Final = _coerce_billed_at(billed_at) + prompt_caching: Final = ( + calculate_prompt_caching_savings( + model_info=pricing, + usage=usage, + custom_llm_provider=identity.provider if identity else custom_llm_provider, + service_tier=basis.service_tier, + data_residency=basis.data_residency, + vertex_location=basis.vertex_location, + billed_at=billed_at_datetime, + ) + if pricing is not None and usage is not None + else 0.0 + ) gateway_injected_caching: Final = prompt_caching if gateway_injected_cache else 0.0 # The figure the logging path recorded wins, before the usage gate on purpose: a row diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index e1bd20ece6f..59f0938e338 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -49,6 +49,44 @@ def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) +@pytest.mark.parametrize("prompt_tokens", [100, 200000, 200001]) +@pytest.mark.parametrize("read_rate", [None, 0.0, 0.25e-6]) +@pytest.mark.parametrize("service_tier", [None, "priority"]) +def test_missing_cache_read_policy_preserves_billing(prompt_tokens, read_rate, service_tier): + info = { + "input_cost_per_token": 3e-6, + "input_cost_per_token_priority": 4e-6, + "input_cost_per_token_above_200k_tokens": 6e-6, + "input_cost_per_token_above_200k_tokens_priority": 8e-6, + "output_cost_per_token": 1e-6, + "cache_read_input_token_cost": read_rate, + } + usage = Usage(prompt_tokens=prompt_tokens, prompt_tokens_details={"cached_tokens": 100}) + billed = _get_token_base_cost(info, usage, service_tier=service_tier) + savings = _get_token_base_cost(info, usage, service_tier=service_tier, missing_cache_read_uses_input=True) + prompt_cost, _ = generic_cost_per_token("policy-fixture", usage, "openai", service_tier=service_tier, model_info=info) + assert billed[4] == pytest.approx(read_rate or 0.0) + assert savings[:4] == billed[:4] + assert savings[4] == pytest.approx(billed[0] if read_rate is None else read_rate) + assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed[0] + 100 * billed[4]) + + +def test_missing_cache_read_uses_off_peak_input_rate(): + from datetime import datetime, timezone + + info = { + "input_cost_per_token": 3e-6, + "off_peak_pricing": {"hours_utc": "00:00-23:59", "input_cost_per_token": 5e-6}, + } + when = datetime(2026, 9, 7, 12, tzinfo=timezone.utc) + billed = _get_token_base_cost(info, Usage(prompt_tokens=100), current_time=when) + savings = _get_token_base_cost( + info, Usage(prompt_tokens=100), current_time=when, missing_cache_read_uses_input=True + ) + assert billed[4] == 0.0 + assert savings[0] == savings[4] == 5e-6 + + def test_reasoning_tokens_no_price_set(_local_model_cost_map): # Use o1 - o1-mini was deprecated/renamed; o1 has same reasoning-token semantics # (no separate output_cost_per_reasoning_token, so all completion tokens use output_cost_per_token) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 3f775d82b7f..cc8fdeb0160 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -1,4 +1,4 @@ - +from typing import Final import pytest @@ -121,6 +121,147 @@ def _caching_usage(read: int, written: int, text: int = 10, out: int = 100) -> d } +@pytest.mark.parametrize( + "model,provider,prompt,reads,writes_5m,writes_1h,tier,region,location", + [ + pytest.param("claude-sonnet-4-5", "anthropic", 50000, 20000, 20000, 0, None, None, None, id="5m"), + pytest.param("claude-sonnet-4-5", "anthropic", 50000, 20000, 0, 20000, None, None, None, id="1h"), + pytest.param("claude-sonnet-4-5", "anthropic", 50000, 20000, 12000, 8000, None, None, None, id="mixed-ttl"), + pytest.param("claude-sonnet-4-5", "anthropic", 199999, 80000, 20000, 0, None, None, None, id="below-200k"), + pytest.param("claude-sonnet-4-5", "anthropic", 200000, 80000, 20000, 0, None, None, None, id="exactly-200k"), + pytest.param("claude-sonnet-4-5", "anthropic", 200001, 80000, 20000, 0, None, None, None, id="above-200k"), + pytest.param("claude-sonnet-4-5", "anthropic", 250000, 80000, 12000, 8000, None, None, None, id="ttl-and-200k"), + pytest.param( + "claude-sonnet-4-5", "anthropic", 250000, 80000, 0, 20000, "priority", None, None, id="absent-tier" + ), + pytest.param("gpt-5.5", "openai", 100000, 80000, 0, 0, "priority", None, None, id="priority"), + pytest.param("gpt-5.5", "openai", 100000, 80000, 0, 0, "flex", None, None, id="flex"), + pytest.param("gpt-5.5", "openai", 100000, 80000, 0, 0, "batch", None, None, id="batch-resolver-fallback"), + pytest.param("gpt-5.5", "openai", 300000, 80000, 0, 0, "flex", None, None, id="flex-and-272k"), + pytest.param("gpt-5.5", "openai", 100000, 80000, 0, 0, "priority", "eu", None, id="priority-and-eu"), + pytest.param("gemini-2.5-pro", "vertex_ai", 250000, 80000, 20000, 0, None, None, None, id="variant-only-write"), + pytest.param("gemini-3.5-flash", "vertex_ai", 100000, 80000, 0, 0, None, None, "us-east5", id="vertex-region"), + pytest.param("gpt-5.5", "openai", 300000, 0, 0, 0, "priority", "eu", None, id="no-cache"), + ], +) +def test_caching_savings_agree_with_biller_on_the_request_pricing_basis( + model: str, + provider: str, + prompt: int, + reads: int, + writes_5m: int, + writes_1h: int, + tier: str | None, + region: str | None, + location: str | None, +) -> None: + pricing: Final = litellm.get_model_info(model=model, custom_llm_provider=provider) + usage: Final = Usage( + prompt_tokens=prompt, + completion_tokens=100, + total_tokens=prompt + 100, + prompt_tokens_details={ + "cached_tokens": reads, + "cache_creation_tokens": writes_5m + writes_1h, + "text_tokens": prompt - reads - writes_5m - writes_1h, + "cache_creation_token_details": { + "ephemeral_5m_input_tokens": writes_5m, + "ephemeral_1h_input_tokens": writes_1h, + }, + }, + ) + uncached: Final = Usage( + prompt_tokens=prompt, + completion_tokens=100, + total_tokens=prompt + 100, + prompt_tokens_details={"text_tokens": prompt, "cached_tokens": 0, "cache_creation_tokens": 0}, + ) + costs: Final = tuple( + sum( + generic_cost_per_token( + model=model, + usage=arm, + custom_llm_provider=provider, + model_info=pricing, + service_tier=tier, + data_residency=region, + vertex_location=location, + ) + ) + for arm in (uncached, usage) + ) + expected: Final = costs[0] - costs[1] + for attributed in (False, True): + result: Final = compute_savings_spend( + model=model, + custom_llm_provider=provider, + compression_saved_tokens=4389, + gateway_injected_cache=attributed, + usage_object=usage.model_dump(), + cost_breakdown={"service_tier": tier, "data_residency": region, "vertex_location": location}, + billed_at="2026-09-07T12:00:00+00:00", + ) + assert result.prompt_caching == pytest.approx(expected) + assert result.gateway_injected_caching == pytest.approx(expected if attributed else 0.0) + assert result.compression == pytest.approx(4389 * (pricing["input_cost_per_token"] or 0.0)) + assert result.autorouter == 0.0 + if reads + writes_5m + writes_1h == 0: + assert expected == 0.0 + + +def test_negative_ttl_counts_do_not_become_cache_write_credits() -> None: + results: Final = tuple( + compute_savings_spend( + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + gateway_injected_cache=True, + usage_object={ + "prompt_tokens": 6000, + "completion_tokens": 100, + "prompt_tokens_details": { + "text_tokens": 1000, + "cache_creation_tokens": 5000, + "cache_creation_token_details": { + "ephemeral_5m_input_tokens": short_count, + "ephemeral_1h_input_tokens": 5000, + }, + }, + }, + ) + for short_count in (-5000, 0) + ) + assert results[0] == results[1] + assert results[0].prompt_caching < 0 + + +def test_unpublished_one_hour_price_uses_the_ordinary_write_price() -> None: + model: Final = "claude-4-opus-20250514" + pricing: Final = litellm.get_model_info(model=model, custom_llm_provider="anthropic") + assert pricing.get("cache_creation_input_token_cost_above_1hr") is None + assert pricing["cache_creation_input_token_cost"] > pricing["input_cost_per_token"] + results: Final = tuple( + compute_savings_spend( + model=model, + custom_llm_provider="anthropic", + compression_saved_tokens=0, + gateway_injected_cache=True, + usage_object={ + "prompt_tokens": 6000, + "completion_tokens": 100, + "prompt_tokens_details": { + "text_tokens": 1000, + "cache_creation_tokens": 5000, + "cache_creation_token_details": ttl, + }, + }, + ) + for ttl in (None, {"ephemeral_1h_input_tokens": 5000}) + ) + assert results[0] == results[1] + assert results[0].prompt_caching < 0 + + def test_prompt_caching_savings_nets_out_the_cache_write_premium(): """A cache-writing request is only credited the read discount minus the write premium.""" input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5")