scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-15 09:12:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 311

fix(savings): refresh baseline cache pricing

Browse source
This commit is contained in:
qdivan 2026-09-08 17:02:46 +08:00
parent 1af7a403c6
commit b0b60bfac2
5 changed files with 213 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
litellm/litellm_core_utils/cache_pricing.py Normal file
View file

@ -0,0 +1,45 @@
from types import MappingProxyType
from typing import Final
from litellm.types.utils import ModelInfo, Usage
def fallback_missing_cache_rates_to_input(usage: Usage, model_info: ModelInfo | None) -> Usage:
"""Move cache buckets without configured rates into ordinary input."""
details: Final = usage.prompt_tokens_details
if details is None or model_info is None:
return usage
cached_tokens: Final = getattr(details, "cached_tokens", 0) or 0
cache_creation_tokens: Final = (getattr(details, "cache_creation_tokens", 0) or 0) or (
getattr(details, "cache_write_tokens", 0) or 0
)
prices_reads: Final = model_info.get("cache_read_input_token_cost") is not None
prices_writes: Final = model_info.get("cache_creation_input_token_cost") is not None
reads: Final = cached_tokens if prices_reads else 0
writes: Final = cache_creation_tokens if prices_writes else 0
if (reads, writes) == (cached_tokens, cache_creation_tokens):
return usage
other_modalities: Final = sum(
(getattr(details, field, 0) or 0) for field in ("audio_tokens", "image_tokens", "video_tokens")
)
return Usage(
prompt_tokens=usage.prompt_tokens,
completion_tokens=usage.completion_tokens,
total_tokens=usage.total_tokens,
completion_tokens_details=usage.completion_tokens_details,
prompt_tokens_details=details.model_copy(
update=MappingProxyType(
{
"cached_tokens": reads,
"cache_creation_tokens": writes,
"cache_write_tokens": writes,
"cache_creation_token_details": getattr(details, "cache_creation_token_details", None)
if writes
else None,
"text_tokens": max(usage.prompt_tokens - reads - writes - other_modalities, 0),
}
)
),
)

62
litellm/proxy/spend_tracking/savings.py
View file

@ -10,11 +10,13 @@ have been aggregated across models.
from collections.abc import Callable, Mapping
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, NamedTuple
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.litellm_core_utils.cache_pricing import fallback_missing_cache_rates_to_input
from litellm.litellm_core_utils.llm_cost_calc.utils import (
_get_cost_per_unit,
calculate_prompt_caching_savings,
@ -27,7 +29,7 @@ from litellm.types.integrations.anthropic_cache_control_hook import (
if TYPE_CHECKING:
from litellm.router import Router
from litellm.types.utils import ModelInfo, PromptTokensDetailsWrapper, Usage
from litellm.types.utils import ModelInfo, Usage
class SavingsSpend(NamedTuple):
@ -198,28 +200,6 @@ def _cache_token_split(usage: Usage) -> tuple[int, int]:
return int(read), int(created)
_CACHE_SPLIT_FIELDS: Final = frozenset(
("cached_tokens", "cache_creation_tokens", "cache_write_tokens", "cache_creation_token_details", "text_tokens")
)
def _baseline_cache_rate_keys(baseline_info: ModelInfo | None) -> tuple[bool, bool]:
"""Whether the baseline model has a ``(cache read, cache write)`` rate of its own.
A missing rate is not a free bucket. `_get_token_base_cost` resolves an absent
`cache_read_input_token_cost` or `cache_creation_input_token_cost` to 0.0, so a
baseline whose provider prices caching implicitly, which is every OpenAI, Azure and
Gemini entry for cache writes, would carry the whole prompt for nothing and turn a
profitable route into a reported loss. Such a model pays its plain input rate for
those tokens, so the buckets it cannot price become ordinary input below.
"""
if baseline_info is None:
return True, True
return bool(baseline_info.get("cache_read_input_token_cost")), bool(
baseline_info.get("cache_creation_input_token_cost")
)
def _baseline_usage(usage: Usage, conversation_continuing: bool, baseline_info: ModelInfo | None = None) -> Usage:
"""The same request as a single-model baseline would have met it.
@ -264,29 +244,31 @@ def _baseline_usage(usage: Usage, conversation_continuing: bool, baseline_info:
reads = cache_read + cache_creation if warm else cache_read
writes = 0 if warm else cache_creation
prices_reads, prices_writes = _baseline_cache_rate_keys(baseline_info)
reads = reads if prices_reads else 0
writes = writes if prices_writes else 0
if (reads, writes) == (cache_read, cache_creation):
return usage
return fallback_missing_cache_rates_to_input(usage, baseline_info)
other_modalities: Final = sum(
(getattr(details, field, 0) or 0) for field in ("audio_tokens", "image_tokens", "video_tokens")
)
return Usage(
prompt_tokens=usage.prompt_tokens,
completion_tokens=usage.completion_tokens,
total_tokens=usage.total_tokens,
completion_tokens_details=usage.completion_tokens_details,
prompt_tokens_details=PromptTokensDetailsWrapper(
**details.model_dump(exclude=_CACHE_SPLIT_FIELDS),
cached_tokens=reads,
cache_creation_tokens=writes,
cache_write_tokens=writes,
cache_creation_token_details=details.cache_creation_token_details if writes else None,
# Whatever no longer sits in a cache bucket is plain input on the baseline.
text_tokens=max(usage.prompt_tokens - reads - writes - other_modalities, 0),
return fallback_missing_cache_rates_to_input(
Usage(
prompt_tokens=usage.prompt_tokens,
completion_tokens=usage.completion_tokens,
total_tokens=usage.total_tokens,
completion_tokens_details=usage.completion_tokens_details,
prompt_tokens_details=details.model_copy(
update=MappingProxyType(
{
"cached_tokens": reads,
"cache_creation_tokens": writes,
"cache_write_tokens": writes,
"cache_creation_token_details": details.cache_creation_token_details if writes else None,
"text_tokens": max(usage.prompt_tokens - reads - writes - other_modalities, 0),
}
)
),
),
baseline_info,
)

3
litellm/router_strategy/savings_baseline.py
View file

@ -18,6 +18,7 @@ from collections.abc import Iterable
from typing import TYPE_CHECKING, Final, NamedTuple
from litellm._logging import verbose_router_logger
from litellm.litellm_core_utils.cache_pricing import fallback_missing_cache_rates_to_input
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
if TYPE_CHECKING:
@ -120,7 +121,7 @@ def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | N
return None
prompt_cost, completion_cost = generic_cost_per_token(
model=model_name or candidate.model,
usage=_REFERENCE_REQUEST,
usage=fallback_missing_cache_rates_to_input(_REFERENCE_REQUEST, info),
custom_llm_provider=provider,
model_info=info,
)

42
tests/test_litellm/proxy/spend_tracking/test_savings.py
View file

@ -3,6 +3,7 @@ from typing import Final
import pytest
import litellm
from litellm.litellm_core_utils.cache_pricing import fallback_missing_cache_rates_to_input
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.proxy.spend_tracking.savings import (
_baseline_usage,
@ -40,6 +41,47 @@ def _cached_usage_object() -> dict:
}
def test_autorouter_savings_prices_missing_baseline_cache_rates_as_input():
usage = Usage(
prompt_tokens=100,
completion_tokens=1,
total_tokens=101,
prompt_tokens_details={"cached_tokens": 80, "cache_creation_tokens": 10, "text_tokens": 10},
)
baseline_info = {
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.0001,
}
selected_info = {
"input_cost_per_token": 0.0001,
"output_cost_per_token": 0.00001,
"cache_read_input_token_cost": 0.00001,
"cache_creation_input_token_cost": 0.00001,
}
baseline_prompt, baseline_completion = generic_cost_per_token(
model="missing-baseline-cache-rates",
usage=fallback_missing_cache_rates_to_input(usage, baseline_info),
custom_llm_provider="openai",
model_info=baseline_info,
)
selected_prompt, selected_completion = generic_cost_per_token(
model="explicit-selected-cache-rates",
usage=usage,
custom_llm_provider="openai",
model_info=selected_info,
)
assert compute_autorouter_savings(
baseline_model="openai/missing-baseline-cache-rates",
selected_model="explicit-selected-cache-rates",
selected_provider="openai",
usage=usage,
conversation_continuing=False,
selected_info=selected_info,
baseline_info=baseline_info,
) == pytest.approx(baseline_prompt + baseline_completion - selected_prompt - selected_completion)
def _cost_on(model: str, usage_object: dict) -> float:
prompt_cost, completion_cost = generic_cost_per_token(
model=model, usage=Usage(**usage_object), custom_llm_provider="anthropic"

103
tests/test_litellm/router_strategy/test_savings_baseline.py
View file

@ -1,13 +1,28 @@
import pytest
from litellm.litellm_core_utils.cache_pricing import fallback_missing_cache_rates_to_input
from litellm.router import Router
from litellm.router_strategy.savings_baseline import (
Baseline,
canonical_model,
_models_in,
_most_expensive,
canonical_model,
resolve_baseline,
)
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
def _cached_reference_usage() -> Usage:
return Usage(
prompt_tokens=100,
completion_tokens=1,
total_tokens=101,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=80,
cache_creation_tokens=10,
text_tokens=10,
),
)
@pytest.fixture
@ -97,6 +112,92 @@ class TestMostExpensive:
def test_returns_none_for_an_empty_candidate_set(self, parent):
assert _most_expensive(parent, []) is None
def test_missing_cache_rates_fall_back_to_input_pricing_for_reference_ranking(self):
router = Router(
model_list=[
{
"model_name": "missing-cache-rates",
"litellm_params": {
"model": "openai/missing-cache-rates",
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.000001,
},
},
{
"model_name": "priced-cache-rates",
"litellm_params": {
"model": "openai/priced-cache-rates",
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00001,
"cache_read_input_token_cost": 0.00001,
"cache_creation_input_token_cost": 0.00001,
},
},
]
)
assert (
resolve_baseline(router, ["missing-cache-rates", "priced-cache-rates"]).model
== "openai/missing-cache-rates"
)
def test_equal_reference_costs_choose_a_deterministic_candidate(self):
router = Router(
model_list=[
{
"model_name": "tie-a",
"litellm_params": {
"model": "openai/tie-a",
"input_cost_per_token": 0.0001,
"output_cost_per_token": 0.0001,
"cache_read_input_token_cost": 0.0001,
"cache_creation_input_token_cost": 0.0001,
},
},
{
"model_name": "tie-b",
"litellm_params": {
"model": "openai/tie-b",
"input_cost_per_token": 0.0001,
"output_cost_per_token": 0.0001,
"cache_read_input_token_cost": 0.0001,
"cache_creation_input_token_cost": 0.0001,
},
},
]
)
assert resolve_baseline(router, ["tie-a", "tie-b"]).model == "openai/tie-b"
class TestCacheRateFallback:
@pytest.mark.parametrize(
"model_info, expected",
[
({}, (0, 0, 100)),
(
{
"cache_read_input_token_cost": 0.0,
"cache_creation_input_token_cost": 0.0,
},
(80, 10, 10),
),
(
{
"cache_read_input_token_cost": 0.00001,
"cache_creation_input_token_cost": 0.00002,
},
(80, 10, 10),
),
({"cache_read_input_token_cost": 0.00001}, (80, 0, 20)),
({"cache_creation_input_token_cost": 0.00002}, (0, 10, 90)),
],
)
def test_preserves_only_cache_buckets_with_explicit_rates(self, model_info, expected):
details = fallback_missing_cache_rates_to_input(_cached_reference_usage(), model_info).prompt_tokens_details
assert (details.cached_tokens, details.cache_creation_tokens, details.text_tokens) == expected
class TestResolveBaseline:
def test_derives_the_priciest_candidate(self, parent):