From d0b59af3aa31a636ec2c821e838b6f2bc311f8e3 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 1 Aug 2026 14:08:36 -0700 Subject: [PATCH] fix(spend): price the baseline on the whole request, not just its cache split `_baseline_usage` rebuilt `PromptTokensDetailsWrapper` from three fields, so everything else the request was priced on came back `None`. Audio, image and video counts, the character and image counts Vertex prices multimodal embeddings by, and the audio and video durations all vanished from the baseline arm, which then priced a text-only request that never ran. On a 20k prompt carrying 4k image tokens the baseline came out 36% light, and since it is the arm being subtracted from, every one of those requests reported less saving than it earned. Dump the details and override only the cache buckets, so a field added to the wrapper later is carried without anyone remembering to add it here. The 5m/1h creation breakdown is dropped along with the creation count: `generic_cost_per_token` charges a cache-write whenever that breakdown is present, even against a zeroed count, which would have put the phantom write back on the baseline for every long-cache request. The exclude set is a frozenset fed straight to `model_dump`, so the override is a call signature rather than a dict literal the reader has to trust. --- litellm/proxy/spend_tracking/savings.py | 18 ++++++- .../proxy/spend_tracking/test_savings.py | 50 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 7c81c313a2f..07563e494dc 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -98,6 +98,11 @@ def _cache_token_split(usage: Usage) -> tuple[int, int]: return int(read), int(created) +_CACHE_SPLIT_FIELDS = frozenset( + ("cached_tokens", "cache_creation_tokens", "cache_write_tokens", "cache_creation_token_details", "text_tokens") +) + + def _baseline_usage(usage: Usage) -> Usage: """The same request as a single-model baseline would have met it. @@ -108,9 +113,16 @@ def _baseline_usage(usage: Usage) -> Usage: penalty exists for. Gating on a read instead would charge the baseline a write it would never repeat, and a cold switch would then report a larger saving than the same traffic with caching turned off. + + Only the cache buckets move. Every other field the request was priced on travels + through untouched, audio and image and video counts among them, because the baseline + is this same request served by a model that happened to be warm; naming the fields to + keep instead would price the baseline on a request that never ran, and would go stale + the next time a priced field is added. """ cache_read, cache_creation = _cache_token_split(usage) - if cache_creation <= 0: + details = usage.prompt_tokens_details + if details is None or cache_creation <= 0: return usage return Usage( prompt_tokens=usage.prompt_tokens, @@ -118,11 +130,15 @@ def _baseline_usage(usage: Usage) -> Usage: total_tokens=usage.total_tokens, completion_tokens_details=usage.completion_tokens_details, prompt_tokens_details=PromptTokensDetailsWrapper( + **details.model_dump(exclude=_CACHE_SPLIT_FIELDS), # The tokens this request paid to write are moved into the cached count and # the creation charge is dropped: on one model that cache was already warm, # so the baseline would have read them rather than paying to create them. + # The 5m/1h breakdown goes with it; left behind it re-charges the write. cached_tokens=cache_read + cache_creation, cache_creation_tokens=0, + cache_write_tokens=0, + cache_creation_token_details=None, text_tokens=max(usage.prompt_tokens - cache_read - cache_creation, 0), ), ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 6e7c86fe7da..a5205b65bca 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -8,6 +8,7 @@ import pytest import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.proxy.spend_tracking.savings import ( + _baseline_usage, compute_autorouter_savings, compute_savings_spend, ) @@ -202,6 +203,55 @@ def test_moving_one_token_between_cache_buckets_does_not_move_the_answer(): assert reads_one == pytest.approx(reads_nothing, abs=1e-4) +def test_multimodal_prompts_are_priced_on_the_baseline_too(): + """The baseline is this same request met by a warm cache, so every field it was + priced on has to survive. Rebuilding the details from the cache buckets alone + dropped the image and audio counts, which priced the baseline as a text-only + request that never ran and shrank the reported saving on multimodal traffic. + """ + details = {"cached_tokens": 0, "cache_creation_tokens": 16_000, "text_tokens": 0, "image_tokens": 4_000} + with_images = Usage( + prompt_tokens=20_000, + completion_tokens=1_000, + total_tokens=21_000, + prompt_tokens_details=details, + ) + baseline = _baseline_usage(with_images) + + assert baseline.prompt_tokens_details.image_tokens == 4_000, "image tokens must survive into the baseline" + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic") + text_only = 20_000 * opus["cache_read_input_token_cost"] + assert priced > text_only, "dropping the image tokens undercharges the baseline and hides the saving" + + +def test_the_baseline_is_never_charged_a_cache_write(): + """Carrying the details through must not carry the 5m/1h creation breakdown with + them. `generic_cost_per_token` charges a creation cost whenever that breakdown is + present, even against a zeroed creation count, which would put the phantom write + back on the baseline for every long-cache request. + """ + long_cache = Usage( + prompt_tokens=20_000, + completion_tokens=1_000, + total_tokens=21_000, + prompt_tokens_details={ + "cached_tokens": 0, + "cache_creation_tokens": 20_000, + "text_tokens": 0, + "cache_creation_token_details": {"ephemeral_1h_input_tokens": 20_000}, + }, + ) + baseline = _baseline_usage(long_cache) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic") + assert priced == pytest.approx(20_000 * opus["cache_read_input_token_cost"]), ( + "the baseline reads a warm cache; it never pays to create one" + ) + + def test_uncached_request_is_the_plain_rate_difference(): usage = _usage(fresh=2000, cached=0, written=0, out=500) sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic")