fix(spend): price the baseline on the whole request, not just its cache split

`_baseline_usage` rebuilt `PromptTokensDetailsWrapper` from three fields, so
everything else the request was priced on came back `None`. Audio, image and
video counts, the character and image counts Vertex prices multimodal
embeddings by, and the audio and video durations all vanished from the
baseline arm, which then priced a text-only request that never ran. On a 20k
prompt carrying 4k image tokens the baseline came out 36% light, and since it
is the arm being subtracted from, every one of those requests reported less
saving than it earned.

Dump the details and override only the cache buckets, so a field added to the
wrapper later is carried without anyone remembering to add it here. The 5m/1h
creation breakdown is dropped along with the creation count: `generic_cost_per_token`
charges a cache-write whenever that breakdown is present, even against a zeroed
count, which would have put the phantom write back on the baseline for every
long-cache request.

The exclude set is a frozenset fed straight to `model_dump`, so the override
is a call signature rather than a dict literal the reader has to trust.
This commit is contained in:
Tin Chi Lo 2026-08-01 14:08:36 -07:00
parent 5457640776
commit d0b59af3aa
2 changed files with 67 additions and 1 deletions

View file

@ -98,6 +98,11 @@ def _cache_token_split(usage: Usage) -> tuple[int, int]:
return int(read), int(created)
_CACHE_SPLIT_FIELDS = frozenset(
("cached_tokens", "cache_creation_tokens", "cache_write_tokens", "cache_creation_token_details", "text_tokens")
)
def _baseline_usage(usage: Usage) -> Usage:
"""The same request as a single-model baseline would have met it.
@ -108,9 +113,16 @@ def _baseline_usage(usage: Usage) -> Usage:
penalty exists for. Gating on a read instead would charge the baseline a write it
would never repeat, and a cold switch would then report a larger saving than the
same traffic with caching turned off.
Only the cache buckets move. Every other field the request was priced on travels
through untouched, audio and image and video counts among them, because the baseline
is this same request served by a model that happened to be warm; naming the fields to
keep instead would price the baseline on a request that never ran, and would go stale
the next time a priced field is added.
"""
cache_read, cache_creation = _cache_token_split(usage)
if cache_creation <= 0:
details = usage.prompt_tokens_details
if details is None or cache_creation <= 0:
return usage
return Usage(
prompt_tokens=usage.prompt_tokens,
@ -118,11 +130,15 @@ def _baseline_usage(usage: Usage) -> Usage:
total_tokens=usage.total_tokens,
completion_tokens_details=usage.completion_tokens_details,
prompt_tokens_details=PromptTokensDetailsWrapper(
**details.model_dump(exclude=_CACHE_SPLIT_FIELDS),
# The tokens this request paid to write are moved into the cached count and
# the creation charge is dropped: on one model that cache was already warm,
# so the baseline would have read them rather than paying to create them.
# The 5m/1h breakdown goes with it; left behind it re-charges the write.
cached_tokens=cache_read + cache_creation,
cache_creation_tokens=0,
cache_write_tokens=0,
cache_creation_token_details=None,
text_tokens=max(usage.prompt_tokens - cache_read - cache_creation, 0),
),
)

View file

@ -8,6 +8,7 @@ import pytest
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.proxy.spend_tracking.savings import (
_baseline_usage,
compute_autorouter_savings,
compute_savings_spend,
)
@ -202,6 +203,55 @@ def test_moving_one_token_between_cache_buckets_does_not_move_the_answer():
assert reads_one == pytest.approx(reads_nothing, abs=1e-4)
def test_multimodal_prompts_are_priced_on_the_baseline_too():
"""The baseline is this same request met by a warm cache, so every field it was
priced on has to survive. Rebuilding the details from the cache buckets alone
dropped the image and audio counts, which priced the baseline as a text-only
request that never ran and shrank the reported saving on multimodal traffic.
"""
details = {"cached_tokens": 0, "cache_creation_tokens": 16_000, "text_tokens": 0, "image_tokens": 4_000}
with_images = Usage(
prompt_tokens=20_000,
completion_tokens=1_000,
total_tokens=21_000,
prompt_tokens_details=details,
)
baseline = _baseline_usage(with_images)
assert baseline.prompt_tokens_details.image_tokens == 4_000, "image tokens must survive into the baseline"
opus = litellm.get_model_info("claude-opus-5", "anthropic")
priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic")
text_only = 20_000 * opus["cache_read_input_token_cost"]
assert priced > text_only, "dropping the image tokens undercharges the baseline and hides the saving"
def test_the_baseline_is_never_charged_a_cache_write():
"""Carrying the details through must not carry the 5m/1h creation breakdown with
them. `generic_cost_per_token` charges a creation cost whenever that breakdown is
present, even against a zeroed creation count, which would put the phantom write
back on the baseline for every long-cache request.
"""
long_cache = Usage(
prompt_tokens=20_000,
completion_tokens=1_000,
total_tokens=21_000,
prompt_tokens_details={
"cached_tokens": 0,
"cache_creation_tokens": 20_000,
"text_tokens": 0,
"cache_creation_token_details": {"ephemeral_1h_input_tokens": 20_000},
},
)
baseline = _baseline_usage(long_cache)
opus = litellm.get_model_info("claude-opus-5", "anthropic")
priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic")
assert priced == pytest.approx(20_000 * opus["cache_read_input_token_cost"]), (
"the baseline reads a warm cache; it never pays to create one"
)
def test_uncached_request_is_the_plain_rate_difference():
usage = _usage(fresh=2000, cached=0, written=0, out=500)
sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic")