diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index a611dc4301d..00d6e82a79b 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -128,7 +128,6 @@ def _baseline_usage(usage: Usage) -> Usage: def compute_autorouter_savings( baseline_model: str | None, selected_model: str | None, - baseline_provider: str | None, selected_provider: str | None, usage: Usage, ) -> float: @@ -140,7 +139,10 @@ def compute_autorouter_savings( dashboard has to be able to say so. Zero when both sides resolve to the same deployment, or when either cannot be resolved or priced. """ - baseline = _resolve_model(baseline_model, baseline_provider) + # No provider argument for the baseline on purpose: it arrives from the routing + # metadata as a single self-describing string, already qualified by the auto-router, + # so there is no second field that could disagree with it. + baseline = _resolve_model(baseline_model, None) selected = _resolve_model(selected_model, selected_provider) if baseline is None or selected is None or baseline == selected: return 0.0 @@ -168,7 +170,6 @@ def compute_savings_spend( compression_saved_tokens: int, cache_read_input_tokens: int, baseline_model: str | None = None, - baseline_provider: str | None = None, usage_object: dict | None = None, ) -> SavingsSpend: """ @@ -191,7 +192,6 @@ def compute_savings_spend( autorouter = compute_autorouter_savings( baseline_model=baseline_model, selected_model=model, - baseline_provider=baseline_provider, selected_provider=custom_llm_provider, usage=usage, ) diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index 2fd31dca541..7ef2b5a3562 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -153,10 +153,18 @@ class AutoRouter(CustomLogger): honest: a fixed flagship credits savings against a model the operator would never have run, and drifts the moment the routes change. + Always provider-qualified, whether derived or configured, because it travels to + the spend writer as a bare string with no provider beside it; an operator who + writes `deepseek-r1` meaning Azure would otherwise be priced against whoever + owns that name. + ``None`` when nothing can be priced, which zeroes the driver rather than inventing a baseline. """ - return self.configured_savings_baseline_model or self._derived_savings_baseline_model + configured = self.configured_savings_baseline_model + if configured: + return self._canonical_model(configured, None) + return self._derived_savings_baseline_model def _load_semantic_routing_routes(self) -> List[Route]: from semantic_router.routers import SemanticRouter diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 18991f53e6f..5229ec69e04 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3401,6 +3401,7 @@ all_litellm_params = ( "auto_router_config", "auto_router_default_model", "auto_router_embedding_model", + "auto_router_savings_baseline_model", "complexity_router_config", "complexity_router_default_model", "adaptive_router_config", diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 49e0273dd15..112d96934ae 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -132,7 +132,6 @@ def _savings(baseline: str, selected: str, usage: Usage) -> float: return compute_autorouter_savings( baseline_model=baseline, selected_model=selected, - baseline_provider="anthropic", selected_provider="anthropic", usage=usage, ) @@ -231,7 +230,6 @@ def test_compute_savings_spend_carries_a_losing_switch_through(): compression_saved_tokens=0, cache_read_input_tokens=0, baseline_model="claude-sonnet-5", - baseline_provider="anthropic", usage_object=_cached_usage_object(), ) assert result.autorouter < 0 @@ -288,14 +286,12 @@ def test_baseline_is_priced_under_its_own_provider(): azure = compute_autorouter_savings( baseline_model="azure_ai/deepseek-r1", selected_model="claude-haiku-4-5", - baseline_provider=None, selected_provider="anthropic", usage=usage, ) deepseek = compute_autorouter_savings( baseline_model="deepseek/deepseek-r1", selected_model="claude-haiku-4-5", - baseline_provider=None, selected_provider="anthropic", usage=usage, ) diff --git a/tests/test_litellm/router_strategy/test_auto_router.py b/tests/test_litellm/router_strategy/test_auto_router.py index 72ecee53bc5..ce8406a44e0 100644 --- a/tests/test_litellm/router_strategy/test_auto_router.py +++ b/tests/test_litellm/router_strategy/test_auto_router.py @@ -459,13 +459,25 @@ class TestSavingsBaselineModel: assert auto_router.savings_baseline_model == "anthropic/claude-opus-5" def test_an_explicit_baseline_overrides_the_derived_one(self): + """And is qualified like a derived one: the baseline reaches the spend writer as + a bare string with no provider beside it, so an operator who writes a name that + another vendor also owns would otherwise be priced against that vendor.""" auto_router = self._auto_router( {"cheap-tier": "anthropic/claude-haiku-4-5"}, ["cheap-tier"], "cheap-tier", configured="claude-opus-5", ) - assert auto_router.savings_baseline_model == "claude-opus-5" + assert auto_router.savings_baseline_model == "anthropic/claude-opus-5" + + def test_an_unresolvable_explicit_baseline_disables_the_driver(self): + auto_router = self._auto_router( + {"cheap-tier": "anthropic/claude-haiku-4-5"}, + ["cheap-tier"], + "cheap-tier", + configured="no-such-provider-xyz/no-such-model", + ) + assert auto_router.savings_baseline_model is None def test_nothing_priceable_disables_the_driver_rather_than_inventing_a_baseline(self): """A missing number beats a fabricated one.""" diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 78cf64aa9a8..c776e5bb072 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1733,20 +1733,24 @@ class TestRouterPreRoutingAliasOverrides: ships raw to the real provider as extra_body - verified live via litellm.completion(..., complexity_router_config={...}) landing in extra_body before this list included it.""" + import re + + from litellm.types.router import GenericLiteLLMParams, LiteLLM_Params from litellm.types.utils import all_litellm_params - router_init_only_params = ( - "auto_router_config_path", - "auto_router_config", - "auto_router_default_model", - "auto_router_embedding_model", - "complexity_router_config", - "complexity_router_default_model", - "adaptive_router_config", - "adaptive_router_default_model", - "quality_router_config", - "quality_router_default_model", + # Derived, not hand-listed: a hard-coded tuple can only catch a field being + # REMOVED from the strip list, never a newly added router param that was never + # registered in the first place, which is the way this actually goes wrong. + router_config_param = re.compile(r"^(auto|complexity|adaptive|quality)_router_") + router_init_only_params = tuple( + sorted( + field + for field in set(LiteLLM_Params.model_fields) | set(GenericLiteLLMParams.model_fields) + if router_config_param.match(field) + ) ) + assert len(router_init_only_params) >= 11, "expected the known router-strategy config params" + for param in router_init_only_params: assert param in all_litellm_params, ( f"{param} must stay in litellm.types.utils.all_litellm_params - "