diff --git a/litellm/litellm_core_utils/cache_pricing.py b/litellm/litellm_core_utils/cache_pricing.py new file mode 100644 index 00000000000..b500f618268 --- /dev/null +++ b/litellm/litellm_core_utils/cache_pricing.py @@ -0,0 +1,45 @@ +from types import MappingProxyType +from typing import Final + +from litellm.types.utils import ModelInfo, Usage + + +def fallback_missing_cache_rates_to_input(usage: Usage, model_info: ModelInfo | None) -> Usage: + """Move cache buckets without configured rates into ordinary input.""" + details: Final = usage.prompt_tokens_details + if details is None or model_info is None: + return usage + + cached_tokens: Final = getattr(details, "cached_tokens", 0) or 0 + cache_creation_tokens: Final = (getattr(details, "cache_creation_tokens", 0) or 0) or ( + getattr(details, "cache_write_tokens", 0) or 0 + ) + prices_reads: Final = model_info.get("cache_read_input_token_cost") is not None + prices_writes: Final = model_info.get("cache_creation_input_token_cost") is not None + reads: Final = cached_tokens if prices_reads else 0 + writes: Final = cache_creation_tokens if prices_writes else 0 + if (reads, writes) == (cached_tokens, cache_creation_tokens): + return usage + + other_modalities: Final = sum( + (getattr(details, field, 0) or 0) for field in ("audio_tokens", "image_tokens", "video_tokens") + ) + return Usage( + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + total_tokens=usage.total_tokens, + completion_tokens_details=usage.completion_tokens_details, + prompt_tokens_details=details.model_copy( + update=MappingProxyType( + { + "cached_tokens": reads, + "cache_creation_tokens": writes, + "cache_write_tokens": writes, + "cache_creation_token_details": getattr(details, "cache_creation_token_details", None) + if writes + else None, + "text_tokens": max(usage.prompt_tokens - reads - writes - other_modalities, 0), + } + ) + ), + ) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 950fcca2039..84d3f942da8 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -10,11 +10,13 @@ have been aggregated across models. from collections.abc import Callable, Mapping from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Final, NamedTuple import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.litellm_core_utils.cache_pricing import fallback_missing_cache_rates_to_input from litellm.litellm_core_utils.llm_cost_calc.utils import ( _get_cost_per_unit, calculate_prompt_caching_savings, @@ -27,7 +29,7 @@ from litellm.types.integrations.anthropic_cache_control_hook import ( if TYPE_CHECKING: from litellm.router import Router -from litellm.types.utils import ModelInfo, PromptTokensDetailsWrapper, Usage +from litellm.types.utils import ModelInfo, Usage class SavingsSpend(NamedTuple): @@ -198,28 +200,6 @@ def _cache_token_split(usage: Usage) -> tuple[int, int]: return int(read), int(created) -_CACHE_SPLIT_FIELDS: Final = frozenset( - ("cached_tokens", "cache_creation_tokens", "cache_write_tokens", "cache_creation_token_details", "text_tokens") -) - - -def _baseline_cache_rate_keys(baseline_info: ModelInfo | None) -> tuple[bool, bool]: - """Whether the baseline model has a ``(cache read, cache write)`` rate of its own. - - A missing rate is not a free bucket. `_get_token_base_cost` resolves an absent - `cache_read_input_token_cost` or `cache_creation_input_token_cost` to 0.0, so a - baseline whose provider prices caching implicitly, which is every OpenAI, Azure and - Gemini entry for cache writes, would carry the whole prompt for nothing and turn a - profitable route into a reported loss. Such a model pays its plain input rate for - those tokens, so the buckets it cannot price become ordinary input below. - """ - if baseline_info is None: - return True, True - return bool(baseline_info.get("cache_read_input_token_cost")), bool( - baseline_info.get("cache_creation_input_token_cost") - ) - - def _baseline_usage(usage: Usage, conversation_continuing: bool, baseline_info: ModelInfo | None = None) -> Usage: """The same request as a single-model baseline would have met it. @@ -264,29 +244,31 @@ def _baseline_usage(usage: Usage, conversation_continuing: bool, baseline_info: reads = cache_read + cache_creation if warm else cache_read writes = 0 if warm else cache_creation - prices_reads, prices_writes = _baseline_cache_rate_keys(baseline_info) - reads = reads if prices_reads else 0 - writes = writes if prices_writes else 0 if (reads, writes) == (cache_read, cache_creation): - return usage + return fallback_missing_cache_rates_to_input(usage, baseline_info) other_modalities: Final = sum( (getattr(details, field, 0) or 0) for field in ("audio_tokens", "image_tokens", "video_tokens") ) - return Usage( - prompt_tokens=usage.prompt_tokens, - completion_tokens=usage.completion_tokens, - total_tokens=usage.total_tokens, - completion_tokens_details=usage.completion_tokens_details, - prompt_tokens_details=PromptTokensDetailsWrapper( - **details.model_dump(exclude=_CACHE_SPLIT_FIELDS), - cached_tokens=reads, - cache_creation_tokens=writes, - cache_write_tokens=writes, - cache_creation_token_details=details.cache_creation_token_details if writes else None, - # Whatever no longer sits in a cache bucket is plain input on the baseline. - text_tokens=max(usage.prompt_tokens - reads - writes - other_modalities, 0), + return fallback_missing_cache_rates_to_input( + Usage( + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + total_tokens=usage.total_tokens, + completion_tokens_details=usage.completion_tokens_details, + prompt_tokens_details=details.model_copy( + update=MappingProxyType( + { + "cached_tokens": reads, + "cache_creation_tokens": writes, + "cache_write_tokens": writes, + "cache_creation_token_details": details.cache_creation_token_details if writes else None, + "text_tokens": max(usage.prompt_tokens - reads - writes - other_modalities, 0), + } + ) + ), ), + baseline_info, ) diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index a2e983a8369..c99d4863932 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -18,6 +18,7 @@ from collections.abc import Iterable from typing import TYPE_CHECKING, Final, NamedTuple from litellm._logging import verbose_router_logger +from litellm.litellm_core_utils.cache_pricing import fallback_missing_cache_rates_to_input from litellm.types.utils import PromptTokensDetailsWrapper, Usage if TYPE_CHECKING: @@ -120,7 +121,7 @@ def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | N return None prompt_cost, completion_cost = generic_cost_per_token( model=model_name or candidate.model, - usage=_REFERENCE_REQUEST, + usage=fallback_missing_cache_rates_to_input(_REFERENCE_REQUEST, info), custom_llm_provider=provider, model_info=info, ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index e466edab131..0503301f036 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -3,6 +3,7 @@ from typing import Final import pytest import litellm +from litellm.litellm_core_utils.cache_pricing import fallback_missing_cache_rates_to_input from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.proxy.spend_tracking.savings import ( _baseline_usage, @@ -40,6 +41,47 @@ def _cached_usage_object() -> dict: } +def test_autorouter_savings_prices_missing_baseline_cache_rates_as_input(): + usage = Usage( + prompt_tokens=100, + completion_tokens=1, + total_tokens=101, + prompt_tokens_details={"cached_tokens": 80, "cache_creation_tokens": 10, "text_tokens": 10}, + ) + baseline_info = { + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.0001, + } + selected_info = { + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 0.00001, + "cache_creation_input_token_cost": 0.00001, + } + baseline_prompt, baseline_completion = generic_cost_per_token( + model="missing-baseline-cache-rates", + usage=fallback_missing_cache_rates_to_input(usage, baseline_info), + custom_llm_provider="openai", + model_info=baseline_info, + ) + selected_prompt, selected_completion = generic_cost_per_token( + model="explicit-selected-cache-rates", + usage=usage, + custom_llm_provider="openai", + model_info=selected_info, + ) + + assert compute_autorouter_savings( + baseline_model="openai/missing-baseline-cache-rates", + selected_model="explicit-selected-cache-rates", + selected_provider="openai", + usage=usage, + conversation_continuing=False, + selected_info=selected_info, + baseline_info=baseline_info, + ) == pytest.approx(baseline_prompt + baseline_completion - selected_prompt - selected_completion) + + def _cost_on(model: str, usage_object: dict) -> float: prompt_cost, completion_cost = generic_cost_per_token( model=model, usage=Usage(**usage_object), custom_llm_provider="anthropic" diff --git a/tests/test_litellm/router_strategy/test_savings_baseline.py b/tests/test_litellm/router_strategy/test_savings_baseline.py index 5efc73d2dcb..4d1ae80b442 100644 --- a/tests/test_litellm/router_strategy/test_savings_baseline.py +++ b/tests/test_litellm/router_strategy/test_savings_baseline.py @@ -1,13 +1,28 @@ import pytest +from litellm.litellm_core_utils.cache_pricing import fallback_missing_cache_rates_to_input from litellm.router import Router from litellm.router_strategy.savings_baseline import ( Baseline, - canonical_model, _models_in, _most_expensive, + canonical_model, resolve_baseline, ) +from litellm.types.utils import PromptTokensDetailsWrapper, Usage + + +def _cached_reference_usage() -> Usage: + return Usage( + prompt_tokens=100, + completion_tokens=1, + total_tokens=101, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=80, + cache_creation_tokens=10, + text_tokens=10, + ), + ) @pytest.fixture @@ -97,6 +112,92 @@ class TestMostExpensive: def test_returns_none_for_an_empty_candidate_set(self, parent): assert _most_expensive(parent, []) is None + def test_missing_cache_rates_fall_back_to_input_pricing_for_reference_ranking(self): + router = Router( + model_list=[ + { + "model_name": "missing-cache-rates", + "litellm_params": { + "model": "openai/missing-cache-rates", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.000001, + }, + }, + { + "model_name": "priced-cache-rates", + "litellm_params": { + "model": "openai/priced-cache-rates", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00001, + "cache_read_input_token_cost": 0.00001, + "cache_creation_input_token_cost": 0.00001, + }, + }, + ] + ) + + assert ( + resolve_baseline(router, ["missing-cache-rates", "priced-cache-rates"]).model + == "openai/missing-cache-rates" + ) + + def test_equal_reference_costs_choose_a_deterministic_candidate(self): + router = Router( + model_list=[ + { + "model_name": "tie-a", + "litellm_params": { + "model": "openai/tie-a", + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.0001, + "cache_read_input_token_cost": 0.0001, + "cache_creation_input_token_cost": 0.0001, + }, + }, + { + "model_name": "tie-b", + "litellm_params": { + "model": "openai/tie-b", + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.0001, + "cache_read_input_token_cost": 0.0001, + "cache_creation_input_token_cost": 0.0001, + }, + }, + ] + ) + + assert resolve_baseline(router, ["tie-a", "tie-b"]).model == "openai/tie-b" + + +class TestCacheRateFallback: + @pytest.mark.parametrize( + "model_info, expected", + [ + ({}, (0, 0, 100)), + ( + { + "cache_read_input_token_cost": 0.0, + "cache_creation_input_token_cost": 0.0, + }, + (80, 10, 10), + ), + ( + { + "cache_read_input_token_cost": 0.00001, + "cache_creation_input_token_cost": 0.00002, + }, + (80, 10, 10), + ), + ({"cache_read_input_token_cost": 0.00001}, (80, 0, 20)), + ({"cache_creation_input_token_cost": 0.00002}, (0, 10, 90)), + ], + ) + def test_preserves_only_cache_buckets_with_explicit_rates(self, model_info, expected): + details = fallback_missing_cache_rates_to_input(_cached_reference_usage(), model_info).prompt_tokens_details + + assert (details.cached_tokens, details.cache_creation_tokens, details.text_tokens) == expected + class TestResolveBaseline: def test_derives_the_priciest_candidate(self, parent):