scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 87

fix(spend): price caching savings on the billed request basis (#40160)

Browse source 
Resolves LIT-7137

Co-authored-by: Claude Code <noreply@anthropic.com>
This commit is contained in:
tin-berri 2026-09-07 14:41:05 -07:00 • committed by GitHub
parent 038025ba5e
commit c5ec2eedc1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 288 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -514,6 +514,7 @@ def _get_token_base_cost(
current_time: datetime | None = None,
*,
threshold_is_inclusive: bool = False,
missing_cache_read_uses_input: bool = False,
) -> tuple[float, float, float, float, float]:
"""
Return prompt cost, completion cost, and cache costs for a given model and usage.
@ -524,6 +525,9 @@ def _get_token_base_cost(
`threshold_is_inclusive` switches that comparison to >=, for providers such as xAI
that bill the higher tier once the prompt reaches the threshold.
`missing_cache_read_uses_input` resolves an absent cache-read rate to the resolved
input rate instead of 0.0; an explicit 0.0 rate stays a real price either way.
Returns:
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
"""
@ -551,29 +555,16 @@ def _get_token_base_cost(
float,
_get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"),
)
cache_read_cost = cast(float, _get_cost_per_unit(model_info, cache_read_cost_key))
cache_read_cost = _get_cost_per_unit(model_info, cache_read_cost_key, default_value=None)
## CHECK IF ABOVE THRESHOLD
# Optimization: collect threshold keys first to avoid sorting all model_info keys.
# Most models don't have threshold pricing, so we can return early.
# Exclude service_tier-specific variants (e.g. input_cost_per_token_above_200k_tokens_priority)
# so that the threshold detection loop only processes standard keys. The
# service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key.
threshold_keys: Final = [
k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES)
]
if not threshold_keys:
return _apply_off_peak_to_base_costs(
model_info,
current_time,
(
prompt_base_cost,
completion_base_cost,
cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
),
)
# Only sort the threshold keys (typically 1-2 keys instead of 66+)
threshold: float | None = None
@ -662,10 +653,7 @@ def _get_token_base_cost(
),
)
cache_read_cost = cast(
float,
_get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost),
)
cache_read_cost = _get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost)
break
except (IndexError, ValueError):
@ -673,6 +661,17 @@ def _get_token_base_cost(
except Exception:
continue
if cache_read_cost is None:
cache_read_cost = (
_off_peak_rate(
_open_off_peak_block(model_info, current_time) or MappingProxyType({}),
"input_cost_per_token",
prompt_base_cost,
)
if missing_cache_read_uses_input
else 0.0
)
return _apply_off_peak_to_base_costs(
model_info,
current_time,
@ -1416,6 +1415,57 @@ def get_token_type_cost_breakdown(
)
def calculate_prompt_caching_savings(
model_info: ModelInfo,
usage: Usage,
custom_llm_provider: str | None,
service_tier: str | None = None,
data_residency: str | None = None,
vertex_location: str | None = None,
billed_at: datetime | None = None,
) -> float:
"""Read discount minus write premium, using the biller's rate and TTL resolution.
Missing reads and unpublished (missing/zero) writes claim no saving or premium;
explicit zero reads remain free. An unpublished 1h price uses the ordinary write rate.
``billed_at`` is the request's completion time, so off-peak windows resolve as the
biller saw them rather than at the later spend write.
"""
prompt_base_cost, _, cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost = _get_token_base_cost(
model_info=model_info,
usage=usage,
service_tier=service_tier,
current_time=billed_at,
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
missing_cache_read_uses_input=True,
)
write_rate: Final = cache_creation_cost or prompt_base_cost
write_rate_1h: Final = cache_creation_cost_above_1hr or write_rate
prompt_tokens_details: Final = parse_prompt_tokens_details(usage)
cache_read_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"], 0)
cache_creation_tokens: Final = max(prompt_tokens_details["cache_creation_tokens"], 0)
details: Final = prompt_tokens_details["cache_creation_token_details"]
cache_creation_details: Final = (
CacheCreationTokenDetails(
ephemeral_5m_input_tokens=max(details.ephemeral_5m_input_tokens or 0, 0),
ephemeral_1h_input_tokens=max(details.ephemeral_1h_input_tokens or 0, 0),
)
if details is not None
else None
)
read_discount: Final = cache_read_tokens * max(prompt_base_cost - cache_read_cost, 0.0)
write_premium: Final = calculate_cache_writing_cost(
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_details,
cache_creation_cost_above_1hr=write_rate_1h - prompt_base_cost,
cache_creation_cost=write_rate - prompt_base_cost,
)
uplift: Final = _get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift(
model_info, vertex_location
)
return (read_discount - write_premium) * uplift
def calculate_image_response_cost_from_usage(
model: str,
image_response: ImageResponse,

2
litellm/proxy/db/db_spend_update_writer.py
View file

@ -502,6 +502,7 @@ class DBSpendUpdateWriter:
llm_router=get_llm_router,
cost_breakdown=metadata.get("cost_breakdown"),
recorded_autorouter_savings=metadata.get("autorouter_savings"),
billed_at=payload.get("endTime"),
)
transaction: Final = build_autorouter_turn_transaction(
payload=payload,
@ -2188,6 +2189,7 @@ class DBSpendUpdateWriter:
usage_object=usage_obj,
cost_breakdown=_metadata.get("cost_breakdown"),
recorded_autorouter_savings=_metadata.get("autorouter_savings"),
billed_at=payload.get("endTime"),
)
daily_transaction: Final = BaseDailySpendTransaction(

105
litellm/proxy/spend_tracking/savings.py
View file

@ -9,12 +9,17 @@ have been aggregated across models.
"""
from collections.abc import Callable, Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Final, NamedTuple
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.litellm_core_utils.llm_cost_calc.utils import _get_cost_per_unit, generic_cost_per_token
from litellm.litellm_core_utils.llm_cost_calc.utils import (
_get_cost_per_unit,
calculate_prompt_caching_savings,
generic_cost_per_token,
)
from litellm.types.integrations.anthropic_cache_control_hook import (
GATEWAY_INJECTED_CACHE_METADATA_KEY,
GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT,
@ -32,42 +37,13 @@ class SavingsSpend(NamedTuple):
gateway_injected_caching: float = 0.0
def _input_cache_read_and_write_cost(info: ModelInfo | None) -> tuple[float, float, float]:
"""
Return ``(input_cost, cache_read_cost, cache_write_cost)`` per token.
``info`` is whatever pricing the caller resolved -- deployment rates when the
request came through a router deployment, public rates otherwise -- so a
negotiated price is honoured here rather than silently replaced by the list rate.
``None`` falls open to ``(0.0, 0.0, 0.0)`` so savings degrade to zero rather than
raising inside the spend writer.
Prices are read through ``_get_cost_per_unit``, the same accessor the cost
calculator uses, which coerces the string prices a ``config.yaml`` can produce
(``"3e-7"``) and resolves service-tier suffixes.
An absent cache price mirrors the input cost, which yields a zero discount on the
read leg and a zero premium on the write leg. Mirroring rather than taking
``_get_cost_per_unit``'s 0.0 default is load-bearing on the write leg: a zero write
price would make the premium ``0 - input_cost``, turning a model that simply has no
write pricing into a spurious extra saving.
The two legs then differ on an explicit ``0.0``, and the asymmetry is deliberate. A
free cache *write* does not exist -- entries carrying a literal zero (``deepseek-chat``
does) mean "no separate price", so a falsy write price also mirrors input. A free
cache *read* is real: 15 models charge for input and serve reads for nothing, which
is the largest discount available, so the read leg keeps its literal zero.
"""
if info is None:
return 0.0, 0.0, 0.0
input_cost: Final = _get_cost_per_unit(info, "input_cost_per_token") or 0.0
cache_read_cost: Final = _get_cost_per_unit(info, "cache_read_input_token_cost", default_value=None)
cache_write_cost: Final = _get_cost_per_unit(info, "cache_creation_input_token_cost", default_value=None)
return (
input_cost,
input_cost if cache_read_cost is None else cache_read_cost,
cache_write_cost if cache_write_cost else input_cost,
)
def _coerce_billed_at(value: datetime | str | None) -> datetime | None:
if isinstance(value, datetime) or value is None:
return value
try:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None
class _ModelIdentity(NamedTuple):
@ -586,6 +562,7 @@ def compute_savings_spend(
llm_router: "Callable[[], Router | None] | None" = None,
cost_breakdown: Mapping[str, object] | None = None,
recorded_autorouter_savings: object = None,
billed_at: datetime | str | None = None,
) -> SavingsSpend:
"""
Dollar savings for one request, split by optimization driver.
@ -595,24 +572,10 @@ def compute_savings_spend(
premium paid to write those entries, both derived here from ``usage_object`` so no
caller can hand in a count that disagrees with the usage record.
The net form follows from what the request would have cost with caching off. The
provider reports ``prompt_tokens`` as the inclusive total of three disjoint
partitions (uncached text, cache reads, cache writes), so an uncached counterfactual
bills every one of those tokens at the flat input rate::
would_have_cost = (text + reads + writes) * input
actually_cost = text * input + reads * read_rate + writes * write_rate
savings = reads * (input - read_rate) - writes * (write_rate - input)
So the write leg subtracts the write PREMIUM, not the whole write cost: those tokens
had to be sent either way, and the counterfactual already pays the input rate for
them. The premium stays signed, because a handful of models price writes below their
input rate and there the write is a genuine extra saving.
A request that only writes cache and gets no hits therefore reports negative savings,
which is accurate: it really did cost more than the uncached call would have. The
daily rollup increments arithmetically, so those rows offset positive ones in the
same bucket.
The uncached counterfactual pays the ordinary input rate for the same prompt size
and tier. Cache writes subtract only the premium over that rate, split by TTL.
Savings stay signed: a write-only request can lose money, and daily rollups net
those losses against read savings.
Caching is reported twice. ``prompt_caching`` is every net dollar caching saved,
whoever caused it, which is what a customer means by "what did caching save me".
@ -638,12 +601,9 @@ def compute_savings_spend(
calls this and only auto-routed ones need one, so looking it up eagerly at the call
site would fetch and discard it on the rest.
``cost_breakdown`` is what the cost calculator recorded for this request, and it
carries both what the request really cost and the tier and region it was priced on.
Only the auto-router driver reads it. Compression and prompt caching price a
hypothetical token delta off flat rate keys, so they are blind to tiered pricing in
the same way; that is pre-existing behaviour on two shipped drivers rather than
something introduced here, and moving those numbers is its own change.
``cost_breakdown`` supplies the biller's tier and region to caching and auto-router
savings. Caching also uses the logged prompt size and TTL split. Compression retains
its flat input-rate estimate; changing that counterfactual is a separate concern.
``recorded_autorouter_savings`` is the figure the logging path stamped on the spend
log's metadata, honoured over recomputation so the rollup, the turn table and the
@ -658,13 +618,24 @@ def compute_savings_spend(
pricing: Final = _effective_model_info(router_instance, model_id, model or "") or (
_model_info(identity) if identity else None
)
input_cost, cache_read_cost, cache_write_cost = _input_cache_read_and_write_cost(pricing)
input_cost: Final = (_get_cost_per_unit(pricing, "input_cost_per_token") or 0.0) if pricing else 0.0
compression: Final = max(compression_saved_tokens, 0) * input_cost
cache_read_input_tokens: Final = extract_cache_read_tokens(usage_object)
cache_creation_input_tokens: Final = extract_cache_creation_tokens(usage_object)
read_discount: Final = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0)
write_premium: Final = max(cache_creation_input_tokens, 0) * (cache_write_cost - input_cost)
prompt_caching: Final = read_discount - write_premium
usage: Final = _usage_from_spend_log(usage_object)
basis: Final = _pricing_basis(cost_breakdown)
billed_at_datetime: Final = _coerce_billed_at(billed_at)
prompt_caching: Final = (
calculate_prompt_caching_savings(
model_info=pricing,
usage=usage,
custom_llm_provider=identity.provider if identity else custom_llm_provider,
service_tier=basis.service_tier,
data_residency=basis.data_residency,
vertex_location=basis.vertex_location,
billed_at=billed_at_datetime,
)
if pricing is not None and usage is not None
else 0.0
)
gateway_injected_caching: Final = prompt_caching if gateway_injected_cache else 0.0
# The figure the logging path recorded wins, before the usage gate on purpose: a row

38
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -49,6 +49,44 @@ def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
@pytest.mark.parametrize("prompt_tokens", [100, 200000, 200001])
@pytest.mark.parametrize("read_rate", [None, 0.0, 0.25e-6])
@pytest.mark.parametrize("service_tier", [None, "priority"])
def test_missing_cache_read_policy_preserves_billing(prompt_tokens, read_rate, service_tier):
info = {
"input_cost_per_token": 3e-6,
"input_cost_per_token_priority": 4e-6,
"input_cost_per_token_above_200k_tokens": 6e-6,
"input_cost_per_token_above_200k_tokens_priority": 8e-6,
"output_cost_per_token": 1e-6,
"cache_read_input_token_cost": read_rate,
}
usage = Usage(prompt_tokens=prompt_tokens, prompt_tokens_details={"cached_tokens": 100})
billed = _get_token_base_cost(info, usage, service_tier=service_tier)
savings = _get_token_base_cost(info, usage, service_tier=service_tier, missing_cache_read_uses_input=True)
prompt_cost, _ = generic_cost_per_token("policy-fixture", usage, "openai", service_tier=service_tier, model_info=info)
assert billed[4] == pytest.approx(read_rate or 0.0)
assert savings[:4] == billed[:4]
assert savings[4] == pytest.approx(billed[0] if read_rate is None else read_rate)
assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed[0] + 100 * billed[4])
def test_missing_cache_read_uses_off_peak_input_rate():
from datetime import datetime, timezone
info = {
"input_cost_per_token": 3e-6,
"off_peak_pricing": {"hours_utc": "00:00-23:59", "input_cost_per_token": 5e-6},
}
when = datetime(2026, 9, 7, 12, tzinfo=timezone.utc)
billed = _get_token_base_cost(info, Usage(prompt_tokens=100), current_time=when)
savings = _get_token_base_cost(
info, Usage(prompt_tokens=100), current_time=when, missing_cache_read_uses_input=True
)
assert billed[4] == 0.0
assert savings[0] == savings[4] == 5e-6
def test_reasoning_tokens_no_price_set(_local_model_cost_map):
# Use o1 - o1-mini was deprecated/renamed; o1 has same reasoning-token semantics
# (no separate output_cost_per_reasoning_token, so all completion tokens use output_cost_per_token)

143
tests/test_litellm/proxy/spend_tracking/test_savings.py
View file

@ -1,4 +1,4 @@
from typing import Final
import pytest
@ -121,6 +121,147 @@ def _caching_usage(read: int, written: int, text: int = 10, out: int = 100) -> d
}
@pytest.mark.parametrize(
"model,provider,prompt,reads,writes_5m,writes_1h,tier,region,location",
[
pytest.param("claude-sonnet-4-5", "anthropic", 50000, 20000, 20000, 0, None, None, None, id="5m"),
pytest.param("claude-sonnet-4-5", "anthropic", 50000, 20000, 0, 20000, None, None, None, id="1h"),
pytest.param("claude-sonnet-4-5", "anthropic", 50000, 20000, 12000, 8000, None, None, None, id="mixed-ttl"),
pytest.param("claude-sonnet-4-5", "anthropic", 199999, 80000, 20000, 0, None, None, None, id="below-200k"),
pytest.param("claude-sonnet-4-5", "anthropic", 200000, 80000, 20000, 0, None, None, None, id="exactly-200k"),
pytest.param("claude-sonnet-4-5", "anthropic", 200001, 80000, 20000, 0, None, None, None, id="above-200k"),
pytest.param("claude-sonnet-4-5", "anthropic", 250000, 80000, 12000, 8000, None, None, None, id="ttl-and-200k"),
pytest.param(
"claude-sonnet-4-5", "anthropic", 250000, 80000, 0, 20000, "priority", None, None, id="absent-tier"
),
pytest.param("gpt-5.5", "openai", 100000, 80000, 0, 0, "priority", None, None, id="priority"),
pytest.param("gpt-5.5", "openai", 100000, 80000, 0, 0, "flex", None, None, id="flex"),
pytest.param("gpt-5.5", "openai", 100000, 80000, 0, 0, "batch", None, None, id="batch-resolver-fallback"),
pytest.param("gpt-5.5", "openai", 300000, 80000, 0, 0, "flex", None, None, id="flex-and-272k"),
pytest.param("gpt-5.5", "openai", 100000, 80000, 0, 0, "priority", "eu", None, id="priority-and-eu"),
pytest.param("gemini-2.5-pro", "vertex_ai", 250000, 80000, 20000, 0, None, None, None, id="variant-only-write"),
pytest.param("gemini-3.5-flash", "vertex_ai", 100000, 80000, 0, 0, None, None, "us-east5", id="vertex-region"),
pytest.param("gpt-5.5", "openai", 300000, 0, 0, 0, "priority", "eu", None, id="no-cache"),
],
)
def test_caching_savings_agree_with_biller_on_the_request_pricing_basis(
model: str,
provider: str,
prompt: int,
reads: int,
writes_5m: int,
writes_1h: int,
tier: str | None,
region: str | None,
location: str | None,
) -> None:
pricing: Final = litellm.get_model_info(model=model, custom_llm_provider=provider)
usage: Final = Usage(
prompt_tokens=prompt,
completion_tokens=100,
total_tokens=prompt + 100,
prompt_tokens_details={
"cached_tokens": reads,
"cache_creation_tokens": writes_5m + writes_1h,
"text_tokens": prompt - reads - writes_5m - writes_1h,
"cache_creation_token_details": {
"ephemeral_5m_input_tokens": writes_5m,
"ephemeral_1h_input_tokens": writes_1h,
},
},
)
uncached: Final = Usage(
prompt_tokens=prompt,
completion_tokens=100,
total_tokens=prompt + 100,
prompt_tokens_details={"text_tokens": prompt, "cached_tokens": 0, "cache_creation_tokens": 0},
)
costs: Final = tuple(
sum(
generic_cost_per_token(
model=model,
usage=arm,
custom_llm_provider=provider,
model_info=pricing,
service_tier=tier,
data_residency=region,
vertex_location=location,
)
)
for arm in (uncached, usage)
)
expected: Final = costs[0] - costs[1]
for attributed in (False, True):
result: Final = compute_savings_spend(
model=model,
custom_llm_provider=provider,
compression_saved_tokens=4389,
gateway_injected_cache=attributed,
usage_object=usage.model_dump(),
cost_breakdown={"service_tier": tier, "data_residency": region, "vertex_location": location},
billed_at="2026-09-07T12:00:00+00:00",
)
assert result.prompt_caching == pytest.approx(expected)
assert result.gateway_injected_caching == pytest.approx(expected if attributed else 0.0)
assert result.compression == pytest.approx(4389 * (pricing["input_cost_per_token"] or 0.0))
assert result.autorouter == 0.0
if reads + writes_5m + writes_1h == 0:
assert expected == 0.0
def test_negative_ttl_counts_do_not_become_cache_write_credits() -> None:
results: Final = tuple(
compute_savings_spend(
model="claude-sonnet-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object={
"prompt_tokens": 6000,
"completion_tokens": 100,
"prompt_tokens_details": {
"text_tokens": 1000,
"cache_creation_tokens": 5000,
"cache_creation_token_details": {
"ephemeral_5m_input_tokens": short_count,
"ephemeral_1h_input_tokens": 5000,
},
},
},
)
for short_count in (-5000, 0)
)
assert results[0] == results[1]
assert results[0].prompt_caching < 0
def test_unpublished_one_hour_price_uses_the_ordinary_write_price() -> None:
model: Final = "claude-4-opus-20250514"
pricing: Final = litellm.get_model_info(model=model, custom_llm_provider="anthropic")
assert pricing.get("cache_creation_input_token_cost_above_1hr") is None
assert pricing["cache_creation_input_token_cost"] > pricing["input_cost_per_token"]
results: Final = tuple(
compute_savings_spend(
model=model,
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object={
"prompt_tokens": 6000,
"completion_tokens": 100,
"prompt_tokens_details": {
"text_tokens": 1000,
"cache_creation_tokens": 5000,
"cache_creation_token_details": ttl,
},
},
)
for ttl in (None, {"ephemeral_1h_input_tokens": 5000})
)
assert results[0] == results[1]
assert results[0].prompt_caching < 0
def test_prompt_caching_savings_nets_out_the_cache_write_premium():
"""A cache-writing request is only credited the read discount minus the write premium."""
input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5")