mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(router): keep the savings baseline out of the outbound provider request
`auto_router_savings_baseline_model` was missing from `all_litellm_params`, so unlike the four auto-router fields beside it, it was classified as a provider-specific parameter and rode along in extra_body where a downstream provider could read it. The guard that exists for exactly this was a hand-written tuple of eleven params, which can only catch a field being removed from the strip list, never a new one that was never registered. That is the way this actually goes wrong, and it is how this field slipped through. The test now derives the list from the params model itself, so any future router-strategy field is covered the moment it is declared. The baseline is also qualified when it comes from configuration, not only when derived. It travels to the spend writer as a bare string with no provider beside it, so an operator writing `deepseek-r1` meaning Azure would otherwise be priced against whoever owns that name. With every baseline qualified at the source, `compute_savings_spend` no longer takes a `baseline_provider` it could never be given: the parameter existing at all was the implicit contract, and removing it is what makes the invariant explicit.
This commit is contained in:
parent
52ca688c9d
commit
c5072834a3
6 changed files with 42 additions and 21 deletions
|
|
@ -128,7 +128,6 @@ def _baseline_usage(usage: Usage) -> Usage:
|
|||
def compute_autorouter_savings(
|
||||
baseline_model: str | None,
|
||||
selected_model: str | None,
|
||||
baseline_provider: str | None,
|
||||
selected_provider: str | None,
|
||||
usage: Usage,
|
||||
) -> float:
|
||||
|
|
@ -140,7 +139,10 @@ def compute_autorouter_savings(
|
|||
dashboard has to be able to say so. Zero when both sides resolve to the same
|
||||
deployment, or when either cannot be resolved or priced.
|
||||
"""
|
||||
baseline = _resolve_model(baseline_model, baseline_provider)
|
||||
# No provider argument for the baseline on purpose: it arrives from the routing
|
||||
# metadata as a single self-describing string, already qualified by the auto-router,
|
||||
# so there is no second field that could disagree with it.
|
||||
baseline = _resolve_model(baseline_model, None)
|
||||
selected = _resolve_model(selected_model, selected_provider)
|
||||
if baseline is None or selected is None or baseline == selected:
|
||||
return 0.0
|
||||
|
|
@ -168,7 +170,6 @@ def compute_savings_spend(
|
|||
compression_saved_tokens: int,
|
||||
cache_read_input_tokens: int,
|
||||
baseline_model: str | None = None,
|
||||
baseline_provider: str | None = None,
|
||||
usage_object: dict | None = None,
|
||||
) -> SavingsSpend:
|
||||
"""
|
||||
|
|
@ -191,7 +192,6 @@ def compute_savings_spend(
|
|||
autorouter = compute_autorouter_savings(
|
||||
baseline_model=baseline_model,
|
||||
selected_model=model,
|
||||
baseline_provider=baseline_provider,
|
||||
selected_provider=custom_llm_provider,
|
||||
usage=usage,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -153,10 +153,18 @@ class AutoRouter(CustomLogger):
|
|||
honest: a fixed flagship credits savings against a model the operator would
|
||||
never have run, and drifts the moment the routes change.
|
||||
|
||||
Always provider-qualified, whether derived or configured, because it travels to
|
||||
the spend writer as a bare string with no provider beside it; an operator who
|
||||
writes `deepseek-r1` meaning Azure would otherwise be priced against whoever
|
||||
owns that name.
|
||||
|
||||
``None`` when nothing can be priced, which zeroes the driver rather than
|
||||
inventing a baseline.
|
||||
"""
|
||||
return self.configured_savings_baseline_model or self._derived_savings_baseline_model
|
||||
configured = self.configured_savings_baseline_model
|
||||
if configured:
|
||||
return self._canonical_model(configured, None)
|
||||
return self._derived_savings_baseline_model
|
||||
|
||||
def _load_semantic_routing_routes(self) -> List[Route]:
|
||||
from semantic_router.routers import SemanticRouter
|
||||
|
|
|
|||
|
|
@ -3401,6 +3401,7 @@ all_litellm_params = (
|
|||
"auto_router_config",
|
||||
"auto_router_default_model",
|
||||
"auto_router_embedding_model",
|
||||
"auto_router_savings_baseline_model",
|
||||
"complexity_router_config",
|
||||
"complexity_router_default_model",
|
||||
"adaptive_router_config",
|
||||
|
|
|
|||
|
|
@ -132,7 +132,6 @@ def _savings(baseline: str, selected: str, usage: Usage) -> float:
|
|||
return compute_autorouter_savings(
|
||||
baseline_model=baseline,
|
||||
selected_model=selected,
|
||||
baseline_provider="anthropic",
|
||||
selected_provider="anthropic",
|
||||
usage=usage,
|
||||
)
|
||||
|
|
@ -231,7 +230,6 @@ def test_compute_savings_spend_carries_a_losing_switch_through():
|
|||
compression_saved_tokens=0,
|
||||
cache_read_input_tokens=0,
|
||||
baseline_model="claude-sonnet-5",
|
||||
baseline_provider="anthropic",
|
||||
usage_object=_cached_usage_object(),
|
||||
)
|
||||
assert result.autorouter < 0
|
||||
|
|
@ -288,14 +286,12 @@ def test_baseline_is_priced_under_its_own_provider():
|
|||
azure = compute_autorouter_savings(
|
||||
baseline_model="azure_ai/deepseek-r1",
|
||||
selected_model="claude-haiku-4-5",
|
||||
baseline_provider=None,
|
||||
selected_provider="anthropic",
|
||||
usage=usage,
|
||||
)
|
||||
deepseek = compute_autorouter_savings(
|
||||
baseline_model="deepseek/deepseek-r1",
|
||||
selected_model="claude-haiku-4-5",
|
||||
baseline_provider=None,
|
||||
selected_provider="anthropic",
|
||||
usage=usage,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -459,13 +459,25 @@ class TestSavingsBaselineModel:
|
|||
assert auto_router.savings_baseline_model == "anthropic/claude-opus-5"
|
||||
|
||||
def test_an_explicit_baseline_overrides_the_derived_one(self):
|
||||
"""And is qualified like a derived one: the baseline reaches the spend writer as
|
||||
a bare string with no provider beside it, so an operator who writes a name that
|
||||
another vendor also owns would otherwise be priced against that vendor."""
|
||||
auto_router = self._auto_router(
|
||||
{"cheap-tier": "anthropic/claude-haiku-4-5"},
|
||||
["cheap-tier"],
|
||||
"cheap-tier",
|
||||
configured="claude-opus-5",
|
||||
)
|
||||
assert auto_router.savings_baseline_model == "claude-opus-5"
|
||||
assert auto_router.savings_baseline_model == "anthropic/claude-opus-5"
|
||||
|
||||
def test_an_unresolvable_explicit_baseline_disables_the_driver(self):
|
||||
auto_router = self._auto_router(
|
||||
{"cheap-tier": "anthropic/claude-haiku-4-5"},
|
||||
["cheap-tier"],
|
||||
"cheap-tier",
|
||||
configured="no-such-provider-xyz/no-such-model",
|
||||
)
|
||||
assert auto_router.savings_baseline_model is None
|
||||
|
||||
def test_nothing_priceable_disables_the_driver_rather_than_inventing_a_baseline(self):
|
||||
"""A missing number beats a fabricated one."""
|
||||
|
|
|
|||
|
|
@ -1733,20 +1733,24 @@ class TestRouterPreRoutingAliasOverrides:
|
|||
ships raw to the real provider as extra_body - verified live via
|
||||
litellm.completion(..., complexity_router_config={...}) landing in
|
||||
extra_body before this list included it."""
|
||||
import re
|
||||
|
||||
from litellm.types.router import GenericLiteLLMParams, LiteLLM_Params
|
||||
from litellm.types.utils import all_litellm_params
|
||||
|
||||
router_init_only_params = (
|
||||
"auto_router_config_path",
|
||||
"auto_router_config",
|
||||
"auto_router_default_model",
|
||||
"auto_router_embedding_model",
|
||||
"complexity_router_config",
|
||||
"complexity_router_default_model",
|
||||
"adaptive_router_config",
|
||||
"adaptive_router_default_model",
|
||||
"quality_router_config",
|
||||
"quality_router_default_model",
|
||||
# Derived, not hand-listed: a hard-coded tuple can only catch a field being
|
||||
# REMOVED from the strip list, never a newly added router param that was never
|
||||
# registered in the first place, which is the way this actually goes wrong.
|
||||
router_config_param = re.compile(r"^(auto|complexity|adaptive|quality)_router_")
|
||||
router_init_only_params = tuple(
|
||||
sorted(
|
||||
field
|
||||
for field in set(LiteLLM_Params.model_fields) | set(GenericLiteLLMParams.model_fields)
|
||||
if router_config_param.match(field)
|
||||
)
|
||||
)
|
||||
assert len(router_init_only_params) >= 11, "expected the known router-strategy config params"
|
||||
|
||||
for param in router_init_only_params:
|
||||
assert param in all_litellm_params, (
|
||||
f"{param} must stay in litellm.types.utils.all_litellm_params - "
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue