mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(spend): price the baseline on the whole request, not just its cache split
`_baseline_usage` rebuilt `PromptTokensDetailsWrapper` from three fields, so everything else the request was priced on came back `None`. Audio, image and video counts, the character and image counts Vertex prices multimodal embeddings by, and the audio and video durations all vanished from the baseline arm, which then priced a text-only request that never ran. On a 20k prompt carrying 4k image tokens the baseline came out 36% light, and since it is the arm being subtracted from, every one of those requests reported less saving than it earned. Dump the details and override only the cache buckets, so a field added to the wrapper later is carried without anyone remembering to add it here. The 5m/1h creation breakdown is dropped along with the creation count: `generic_cost_per_token` charges a cache-write whenever that breakdown is present, even against a zeroed count, which would have put the phantom write back on the baseline for every long-cache request. The exclude set is a frozenset fed straight to `model_dump`, so the override is a call signature rather than a dict literal the reader has to trust.
This commit is contained in:
parent
5457640776
commit
d0b59af3aa
2 changed files with 67 additions and 1 deletions
|
|
@ -98,6 +98,11 @@ def _cache_token_split(usage: Usage) -> tuple[int, int]:
|
|||
return int(read), int(created)
|
||||
|
||||
|
||||
_CACHE_SPLIT_FIELDS = frozenset(
|
||||
("cached_tokens", "cache_creation_tokens", "cache_write_tokens", "cache_creation_token_details", "text_tokens")
|
||||
)
|
||||
|
||||
|
||||
def _baseline_usage(usage: Usage) -> Usage:
|
||||
"""The same request as a single-model baseline would have met it.
|
||||
|
||||
|
|
@ -108,9 +113,16 @@ def _baseline_usage(usage: Usage) -> Usage:
|
|||
penalty exists for. Gating on a read instead would charge the baseline a write it
|
||||
would never repeat, and a cold switch would then report a larger saving than the
|
||||
same traffic with caching turned off.
|
||||
|
||||
Only the cache buckets move. Every other field the request was priced on travels
|
||||
through untouched, audio and image and video counts among them, because the baseline
|
||||
is this same request served by a model that happened to be warm; naming the fields to
|
||||
keep instead would price the baseline on a request that never ran, and would go stale
|
||||
the next time a priced field is added.
|
||||
"""
|
||||
cache_read, cache_creation = _cache_token_split(usage)
|
||||
if cache_creation <= 0:
|
||||
details = usage.prompt_tokens_details
|
||||
if details is None or cache_creation <= 0:
|
||||
return usage
|
||||
return Usage(
|
||||
prompt_tokens=usage.prompt_tokens,
|
||||
|
|
@ -118,11 +130,15 @@ def _baseline_usage(usage: Usage) -> Usage:
|
|||
total_tokens=usage.total_tokens,
|
||||
completion_tokens_details=usage.completion_tokens_details,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
**details.model_dump(exclude=_CACHE_SPLIT_FIELDS),
|
||||
# The tokens this request paid to write are moved into the cached count and
|
||||
# the creation charge is dropped: on one model that cache was already warm,
|
||||
# so the baseline would have read them rather than paying to create them.
|
||||
# The 5m/1h breakdown goes with it; left behind it re-charges the write.
|
||||
cached_tokens=cache_read + cache_creation,
|
||||
cache_creation_tokens=0,
|
||||
cache_write_tokens=0,
|
||||
cache_creation_token_details=None,
|
||||
text_tokens=max(usage.prompt_tokens - cache_read - cache_creation, 0),
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import pytest
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.proxy.spend_tracking.savings import (
|
||||
_baseline_usage,
|
||||
compute_autorouter_savings,
|
||||
compute_savings_spend,
|
||||
)
|
||||
|
|
@ -202,6 +203,55 @@ def test_moving_one_token_between_cache_buckets_does_not_move_the_answer():
|
|||
assert reads_one == pytest.approx(reads_nothing, abs=1e-4)
|
||||
|
||||
|
||||
def test_multimodal_prompts_are_priced_on_the_baseline_too():
|
||||
"""The baseline is this same request met by a warm cache, so every field it was
|
||||
priced on has to survive. Rebuilding the details from the cache buckets alone
|
||||
dropped the image and audio counts, which priced the baseline as a text-only
|
||||
request that never ran and shrank the reported saving on multimodal traffic.
|
||||
"""
|
||||
details = {"cached_tokens": 0, "cache_creation_tokens": 16_000, "text_tokens": 0, "image_tokens": 4_000}
|
||||
with_images = Usage(
|
||||
prompt_tokens=20_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=21_000,
|
||||
prompt_tokens_details=details,
|
||||
)
|
||||
baseline = _baseline_usage(with_images)
|
||||
|
||||
assert baseline.prompt_tokens_details.image_tokens == 4_000, "image tokens must survive into the baseline"
|
||||
|
||||
opus = litellm.get_model_info("claude-opus-5", "anthropic")
|
||||
priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic")
|
||||
text_only = 20_000 * opus["cache_read_input_token_cost"]
|
||||
assert priced > text_only, "dropping the image tokens undercharges the baseline and hides the saving"
|
||||
|
||||
|
||||
def test_the_baseline_is_never_charged_a_cache_write():
|
||||
"""Carrying the details through must not carry the 5m/1h creation breakdown with
|
||||
them. `generic_cost_per_token` charges a creation cost whenever that breakdown is
|
||||
present, even against a zeroed creation count, which would put the phantom write
|
||||
back on the baseline for every long-cache request.
|
||||
"""
|
||||
long_cache = Usage(
|
||||
prompt_tokens=20_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=21_000,
|
||||
prompt_tokens_details={
|
||||
"cached_tokens": 0,
|
||||
"cache_creation_tokens": 20_000,
|
||||
"text_tokens": 0,
|
||||
"cache_creation_token_details": {"ephemeral_1h_input_tokens": 20_000},
|
||||
},
|
||||
)
|
||||
baseline = _baseline_usage(long_cache)
|
||||
|
||||
opus = litellm.get_model_info("claude-opus-5", "anthropic")
|
||||
priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic")
|
||||
assert priced == pytest.approx(20_000 * opus["cache_read_input_token_cost"]), (
|
||||
"the baseline reads a warm cache; it never pays to create one"
|
||||
)
|
||||
|
||||
|
||||
def test_uncached_request_is_the_plain_rate_difference():
|
||||
usage = _usage(fresh=2000, cached=0, written=0, out=500)
|
||||
sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue