mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-15 23:31:29 +00:00
feat(spend): give the complexity router a savings baseline from its hardest tier
The savings driver only ever worked for the semantic auto-router. It was the one strategy router that declared a baseline model, so every complexity, quality and adaptive router fell through to no baseline, compute_autorouter_savings short-circuited, and autorouter_savings_spend was structurally zero. A deployment routing exclusively through complexity routers saw $0.00 against real spend, which reads as "routing saved nothing" rather than "nothing was measured". A complexity router's tier ladder already names the model an operator would have had to run to serve the hardest request, so the counterfactual is the priciest model in the REASONING tier, falling back to the highest-severity tier configured when REASONING is absent. Deliberately not the priciest model the router can reach: a pricey model sitting in a low tier is a choice the router made, not a ceiling it was bounded by, and crediting savings against it would overstate them. The pricing and resolution both routers need is now one module rather than two copies. Deployments resolve through model_info.base_model before litellm_params .model, because on Azure the latter is a deployment name that is absent from the cost map; without that hop an Azure candidate never prices, and if it was the priciest the baseline silently drops to the second priciest and understates every saving. resolve_baseline can no longer raise. It is read on the routing path while decorating a request that is about to be served, and a dashboard counterfactual must not be able to take a live request down; an unresolvable baseline zeroes the driver instead.
This commit is contained in:
Tin Chi Lo
parent
d539fa621d
commit
68c56fe1f6
7 changed files with 395 additions and 106 deletions
|
|
@ -7725,6 +7725,7 @@ class Router:
|
|||
default_model=default_model,
|
||||
litellm_router_instance=self,
|
||||
complexity_router_config=complexity_router_config,
|
||||
savings_baseline_model=deployment.litellm_params.auto_router_savings_baseline_model,
|
||||
)
|
||||
self._register_pre_routing_strategy(
|
||||
registry=self.complexity_routers,
|
||||
|
|
|
|||
|
|
@ -55,113 +55,19 @@ class AutoRouter(CustomLogger):
|
|||
self.litellm_router_instance: "Router" = litellm_router_instance
|
||||
self.configured_savings_baseline_model: str | None = savings_baseline_model
|
||||
|
||||
@staticmethod
|
||||
def _canonical_model(model: str, custom_llm_provider: str | None) -> str | None:
|
||||
"""``provider/model``, or ``None`` when the pair names no known provider.
|
||||
|
||||
A deployment may name its vendor either in the model prefix or in a separate
|
||||
`custom_llm_provider`, and the bare name alone is not enough to price: it can
|
||||
resolve to a different vendor's rates, or to nothing at all. Qualifying it here
|
||||
means the baseline that reaches the spend writer resolves back to the same
|
||||
vendor that served it.
|
||||
"""
|
||||
import litellm
|
||||
|
||||
try:
|
||||
resolved_model, provider, _, _ = litellm.get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # an unroutable candidate cannot be the baseline
|
||||
verbose_router_logger.debug("auto-router savings: cannot resolve candidate %s (%s)", model, e)
|
||||
return None
|
||||
return f"{provider}/{resolved_model}"
|
||||
|
||||
def _deployment_model(self, index: int) -> str | None:
|
||||
"""The model a deployment calls, qualified by the provider it declares."""
|
||||
params = self.litellm_router_instance.model_list[index].get("litellm_params")
|
||||
if not isinstance(params, dict):
|
||||
return None
|
||||
model = params.get("model")
|
||||
return self._canonical_model(model, params.get("custom_llm_provider")) if model else None
|
||||
|
||||
def _models_for_group(self, group_name: str) -> tuple[str, ...]:
|
||||
"""The models a route's model group actually calls, or the name itself when the
|
||||
parent router has no deployment under it."""
|
||||
indices = self.litellm_router_instance.model_name_to_deployment_indices.get(group_name)
|
||||
if not indices:
|
||||
canonical = self._canonical_model(group_name, None)
|
||||
return (canonical,) if canonical else ()
|
||||
return tuple(model for index in indices if (model := self._deployment_model(index)))
|
||||
|
||||
def _candidate_models(self) -> tuple[str, ...]:
|
||||
"""Every model this router can route to, as pricable model names.
|
||||
|
||||
Routes name the router's own model groups rather than models, so each is
|
||||
resolved through the parent router's deployments before anything is priced.
|
||||
"""
|
||||
group_names = frozenset(
|
||||
name for name in (*(route.name for route in self.loaded_routes), self.default_model) if name
|
||||
)
|
||||
return tuple(model for group_name in group_names for model in self._models_for_group(group_name))
|
||||
|
||||
@staticmethod
|
||||
def _priced_candidate(model: str) -> tuple[float, float, str] | None:
|
||||
"""``(output_rate, input_rate, model)``, or ``None`` when the model has no pricing."""
|
||||
import litellm
|
||||
|
||||
try:
|
||||
info = litellm.get_model_info(model=model)
|
||||
except Exception as e: # noqa: BLE001 # unmapped candidates simply cannot be the baseline
|
||||
verbose_router_logger.debug("auto-router savings: no pricing for candidate %s (%s)", model, e)
|
||||
return None
|
||||
output_rate = info.get("output_cost_per_token") or 0.0
|
||||
input_rate = info.get("input_cost_per_token") or 0.0
|
||||
if output_rate <= 0.0 and input_rate <= 0.0:
|
||||
# A model that costs nothing per token cannot stand in for what the traffic
|
||||
# would otherwise have cost, and as a baseline it would report the whole
|
||||
# real spend as a loss.
|
||||
verbose_router_logger.debug("auto-router savings: candidate %s has no per-token price", model)
|
||||
return None
|
||||
return (output_rate, input_rate, model)
|
||||
|
||||
def _most_expensive_candidate(self) -> str | None:
|
||||
"""The priciest candidate by output rate, input rate breaking the tie."""
|
||||
priced = tuple(
|
||||
candidate for model in self._candidate_models() if (candidate := self._priced_candidate(model)) is not None
|
||||
)
|
||||
if not priced:
|
||||
verbose_router_logger.debug("auto-router savings: no priceable candidates; savings driver disabled")
|
||||
return None
|
||||
return max(priced)[2]
|
||||
|
||||
@property
|
||||
def savings_baseline_model(self) -> str | None:
|
||||
"""The model this router's savings are measured against.
|
||||
|
||||
Without the router a deployment has to pick one model, and it has to be one that
|
||||
can carry the hardest request, so the counterfactual is the priciest model this
|
||||
router could have chosen. Deriving it from the router's own candidates keeps it
|
||||
honest: a fixed flagship credits savings against a model the operator would
|
||||
never have run, and drifts the moment the routes change.
|
||||
|
||||
Always provider-qualified, whether derived or configured, because it travels to
|
||||
the spend writer as a bare string with no provider beside it; an operator who
|
||||
writes `deepseek-r1` meaning Azure would otherwise be priced against whoever
|
||||
owns that name.
|
||||
|
||||
Derived per call rather than cached: the parent router adds and removes
|
||||
deployments while it runs, so a baseline pinned on first use would keep naming a
|
||||
model the router no longer has, and a pricier one added later could never become
|
||||
the baseline. Resolving costs tens of microseconds against a network call, which
|
||||
is not worth trading correctness for.
|
||||
|
||||
``None`` when nothing can be priced, which zeroes the driver rather than
|
||||
inventing a baseline.
|
||||
Every model group its routes can reach is a candidate, because any of them
|
||||
could have been the one model a deployment picked without the router.
|
||||
"""
|
||||
configured = self.configured_savings_baseline_model
|
||||
if configured:
|
||||
return self._canonical_model(configured, None)
|
||||
return self._most_expensive_candidate()
|
||||
from litellm.router_strategy.savings_baseline import resolve_baseline
|
||||
|
||||
group_names = frozenset(
|
||||
name for name in (*(route.name for route in self.loaded_routes), self.default_model) if name
|
||||
)
|
||||
return resolve_baseline(self.configured_savings_baseline_model, self.litellm_router_instance, group_names)
|
||||
|
||||
def _load_semantic_routing_routes(self) -> List[Route]:
|
||||
from semantic_router.routers import SemanticRouter
|
||||
|
|
|
|||
|
|
@ -314,6 +314,7 @@ class ComplexityRouter(CustomLogger):
|
|||
litellm_router_instance: Router,
|
||||
complexity_router_config: dict[str, Any] | None = None,
|
||||
default_model: str | None = None,
|
||||
savings_baseline_model: str | None = None,
|
||||
):
|
||||
"""
|
||||
Initialize ComplexityRouter.
|
||||
|
|
@ -323,9 +324,11 @@ class ComplexityRouter(CustomLogger):
|
|||
litellm_router_instance: The LiteLLM Router instance.
|
||||
complexity_router_config: Optional configuration dict from proxy config.
|
||||
default_model: Optional default model to use if tier cannot be determined.
|
||||
savings_baseline_model: Overrides the counterfactual model the dashboard measures savings against; derived from the hardest configured tier when unset.
|
||||
"""
|
||||
self.model_name = model_name
|
||||
self.litellm_router_instance = litellm_router_instance
|
||||
self.configured_savings_baseline_model: str | None = savings_baseline_model
|
||||
|
||||
# Parse config - always create a new instance to avoid singleton mutation
|
||||
if complexity_router_config:
|
||||
|
|
@ -373,6 +376,36 @@ class ComplexityRouter(CustomLogger):
|
|||
|
||||
verbose_router_logger.debug(f"ComplexityRouter initialized for {model_name} with tiers: {self.config.tiers}")
|
||||
|
||||
def _hardest_tier_models(self) -> tuple[str, ...]:
|
||||
"""The model or pool serving the hardest tier this router configures.
|
||||
|
||||
REASONING when it is configured, since that is the tier a request has to be
|
||||
hard enough to reach; otherwise the highest-severity tier present, so a
|
||||
deployment that only defines SIMPLE and MEDIUM is still measured against
|
||||
the best it could actually have picked.
|
||||
"""
|
||||
for tier in reversed(TIER_SEVERITY_ORDER):
|
||||
models = self.config.tiers.get(tier.value)
|
||||
if models:
|
||||
return tuple(models) if isinstance(models, list) else (models,)
|
||||
return ()
|
||||
|
||||
@property
|
||||
def savings_baseline_model(self) -> str | None:
|
||||
"""The model this router's savings are measured against.
|
||||
|
||||
A complexity router's tier ladder already names the model an operator
|
||||
would have had to run to serve the hardest request, so the counterfactual
|
||||
is the priciest model in that tier rather than the priciest model the
|
||||
router can reach; a cheap tier is a choice the router made, not a ceiling
|
||||
it was bounded by.
|
||||
"""
|
||||
from litellm.router_strategy.savings_baseline import resolve_baseline
|
||||
|
||||
return resolve_baseline(
|
||||
self.configured_savings_baseline_model, self.litellm_router_instance, self._hardest_tier_models()
|
||||
)
|
||||
|
||||
def _estimate_tokens(self, text: str) -> int:
|
||||
"""
|
||||
Estimate token count from text.
|
||||
|
|
@ -1367,6 +1400,7 @@ class ComplexityRouter(CustomLogger):
|
|||
return PreRoutingHookResponse(
|
||||
model=routed_model,
|
||||
messages=messages if has_original_messages else None,
|
||||
savings_baseline_model=self.savings_baseline_model,
|
||||
routing_decision=self._build_routing_decision(
|
||||
routed_model=routed_model,
|
||||
cause=cause,
|
||||
|
|
@ -1443,6 +1477,7 @@ class ComplexityRouter(CustomLogger):
|
|||
return PreRoutingHookResponse(
|
||||
model=routed_model,
|
||||
messages=messages if has_original_messages else None,
|
||||
savings_baseline_model=self.savings_baseline_model,
|
||||
routing_decision=self._build_routing_decision(routed_model=routed_model, cause="default_fallback"),
|
||||
)
|
||||
|
||||
|
|
@ -1464,6 +1499,7 @@ class ComplexityRouter(CustomLogger):
|
|||
return PreRoutingHookResponse(
|
||||
model=routed_model,
|
||||
messages=messages if has_original_messages else None,
|
||||
savings_baseline_model=self.savings_baseline_model,
|
||||
routing_decision=self._build_routing_decision(
|
||||
routed_model=routed_model,
|
||||
cause=keyword_cause,
|
||||
|
|
@ -1511,6 +1547,7 @@ class ComplexityRouter(CustomLogger):
|
|||
return PreRoutingHookResponse(
|
||||
model=routed_model,
|
||||
messages=messages if has_original_messages else None,
|
||||
savings_baseline_model=self.savings_baseline_model,
|
||||
routing_decision=self._build_routing_decision(
|
||||
routed_model=routed_model,
|
||||
cause=outcome.cause,
|
||||
|
|
|
|||
127
litellm/router_strategy/savings_baseline.py
Normal file
127
litellm/router_strategy/savings_baseline.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
"""Resolving the counterfactual model a strategy router's savings are measured against.
|
||||
|
||||
Without the router a deployment has to pick one model, and it has to be one that
|
||||
can carry the hardest request it will see. That model is the baseline: what the
|
||||
traffic would have cost had nobody routed it.
|
||||
|
||||
Every strategy router answers the same two questions differently, so the shared
|
||||
part is here and the per-router part is the candidate set it supplies. A semantic
|
||||
auto-router offers every model group its routes can reach; a complexity router
|
||||
offers the models in its hardest tier.
|
||||
|
||||
Baselines are always provider-qualified, whether derived or configured, because
|
||||
they travel to the spend writer as a bare string with no provider beside them; an
|
||||
operator who writes ``deepseek-r1`` meaning Azure would otherwise be priced
|
||||
against whoever else owns that name.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
|
||||
def canonical_model(model: str, custom_llm_provider: str | None = None) -> str | None:
|
||||
"""``provider/model``, or ``None`` when the pair names no known provider.
|
||||
|
||||
A deployment may name its vendor in the model prefix or in a separate
|
||||
``custom_llm_provider``, and the bare name alone is not enough to price: it
|
||||
can resolve to a different vendor's rates, or to nothing at all.
|
||||
"""
|
||||
import litellm
|
||||
|
||||
try:
|
||||
resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception as e: # noqa: BLE001 # an unroutable candidate cannot be the baseline
|
||||
verbose_router_logger.debug("savings baseline: cannot resolve candidate %s (%s)", model, e)
|
||||
return None
|
||||
return f"{provider}/{resolved_model}"
|
||||
|
||||
|
||||
def _deployment_model(router: "Router", index: int) -> str | None:
|
||||
"""The model a deployment is priced as, qualified by the provider it declares.
|
||||
|
||||
`litellm_params.model` is not always a model. On Azure it is the deployment name,
|
||||
which is absent from the cost map, and `model_info.base_model` is what names the
|
||||
real model; the same holds for wildcard and aliased deployments. Router.py resolves
|
||||
pricing through the same base_model, base_model, model chain.
|
||||
"""
|
||||
deployment = router.model_list[index]
|
||||
params = deployment.get("litellm_params")
|
||||
if not isinstance(params, dict):
|
||||
return None
|
||||
model_info = deployment.get("model_info")
|
||||
base_model = model_info.get("base_model") if isinstance(model_info, dict) else None
|
||||
model = base_model or params.get("base_model") or params.get("model")
|
||||
return canonical_model(model, params.get("custom_llm_provider")) if model else None
|
||||
|
||||
|
||||
def models_for_group(router: "Router", group_name: str) -> tuple[str, ...]:
|
||||
"""The models a model group actually calls.
|
||||
|
||||
Falls back to treating the name as a model itself, which is what a tier
|
||||
pointing straight at a provider model rather than at a configured group does.
|
||||
"""
|
||||
indices = router.model_name_to_deployment_indices.get(group_name)
|
||||
if not indices:
|
||||
canonical = canonical_model(group_name)
|
||||
return (canonical,) if canonical else ()
|
||||
return tuple(model for index in indices if (model := _deployment_model(router, index)))
|
||||
|
||||
|
||||
def _priced(model: str) -> tuple[float, float, str] | None:
|
||||
"""``(output_rate, input_rate, model)``, or ``None`` when the model has no pricing."""
|
||||
import litellm
|
||||
|
||||
try:
|
||||
info = litellm.get_model_info(model=model)
|
||||
except Exception as e: # noqa: BLE001 # unmapped candidates simply cannot be the baseline
|
||||
verbose_router_logger.debug("savings baseline: no pricing for candidate %s (%s)", model, e)
|
||||
return None
|
||||
output_rate = info.get("output_cost_per_token") or 0.0
|
||||
input_rate = info.get("input_cost_per_token") or 0.0
|
||||
if output_rate <= 0.0 and input_rate <= 0.0:
|
||||
# A model that costs nothing per token cannot stand in for what the traffic
|
||||
# would otherwise have cost, and as a baseline it would report the whole
|
||||
# real spend as a loss.
|
||||
verbose_router_logger.debug("savings baseline: candidate %s has no per-token price", model)
|
||||
return None
|
||||
return (output_rate, input_rate, model)
|
||||
|
||||
|
||||
def most_expensive(models: Iterable[str]) -> str | None:
|
||||
"""The priciest model by output rate, input rate breaking the tie."""
|
||||
priced = tuple(candidate for model in models if (candidate := _priced(model)) is not None)
|
||||
if not priced:
|
||||
verbose_router_logger.debug("savings baseline: no priceable candidates; savings driver disabled")
|
||||
return None
|
||||
return max(priced)[2]
|
||||
|
||||
|
||||
def resolve_baseline(configured: str | None, router: "Router", group_names: Iterable[str]) -> str | None:
|
||||
"""The baseline for a router offering ``group_names`` as its candidates.
|
||||
|
||||
A configured override wins and is only qualified, never re-derived. Otherwise
|
||||
the groups are resolved through the parent router's deployments and the
|
||||
priciest result is taken.
|
||||
|
||||
Derived per call rather than cached: the parent router adds and removes
|
||||
deployments while it runs, so a baseline pinned on first use would keep naming
|
||||
a model the router no longer has, and a pricier one added later could never
|
||||
become the baseline. Resolving costs tens of microseconds against a network
|
||||
call, which is not worth trading correctness for.
|
||||
|
||||
Never raises. This is read on the routing path to decorate a request that is
|
||||
about to be served, and a dashboard's counterfactual is not worth failing a
|
||||
live request over; an unresolvable baseline zeroes the savings driver instead.
|
||||
"""
|
||||
try:
|
||||
if configured:
|
||||
return canonical_model(configured)
|
||||
return most_expensive(model for group_name in group_names for model in models_for_group(router, group_name))
|
||||
except Exception as e: # noqa: BLE001 # see docstring: routing must not fail for a metric
|
||||
verbose_router_logger.warning("savings baseline: could not resolve, savings will read zero (%s)", e)
|
||||
return None
|
||||
|
|
@ -422,10 +422,13 @@ class TestSavingsBaselineModel:
|
|||
["cheap-tier", "mid-tier"],
|
||||
"cheap-tier",
|
||||
)
|
||||
assert sorted(auto_router._candidate_models()) == [
|
||||
"anthropic/claude-haiku-4-5",
|
||||
"anthropic/claude-sonnet-5",
|
||||
]
|
||||
from litellm.router_strategy.savings_baseline import models_for_group
|
||||
|
||||
parent = auto_router.litellm_router_instance
|
||||
resolved = sorted(
|
||||
model for group in ("cheap-tier", "mid-tier") for model in models_for_group(parent, group)
|
||||
)
|
||||
assert resolved == ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
|
||||
|
||||
def test_baseline_is_the_priciest_model_this_router_could_have_picked(self):
|
||||
"""Without the router a deployment picks one model that can carry the hardest
|
||||
|
|
|
|||
|
|
@ -2185,6 +2185,8 @@ class FakeEmbeddingRouter:
|
|||
_CLUSTER_MARKERS = ("k8s", "kube", "container", "cluster", "orchestrat")
|
||||
|
||||
def __init__(self):
|
||||
self.model_name_to_deployment_indices: Dict[str, List[int]] = {}
|
||||
self.model_list: List[Dict] = []
|
||||
self.async_embedding_calls: List[List[str]] = []
|
||||
self.async_embedding_kwargs: List[Dict] = []
|
||||
# Every embedded batch (sync route-index build AND async query), so tests can count
|
||||
|
|
@ -4782,3 +4784,61 @@ class TestClassifierTrustBoundary:
|
|||
assert system_message["content"] == _CLASSIFICATION_SYSTEM_RUBRIC
|
||||
assert hostile not in system_message["content"]
|
||||
assert hostile in user_message["content"]
|
||||
|
||||
|
||||
class TestSavingsBaselineModel:
|
||||
"""The counterfactual model a complexity router's savings are measured against."""
|
||||
|
||||
@staticmethod
|
||||
def _router_with_tiers(tiers: dict, **kwargs) -> ComplexityRouter:
|
||||
from litellm.router import Router
|
||||
|
||||
parent = Router(
|
||||
model_list=[
|
||||
{"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}},
|
||||
{"model_name": "mid", "litellm_params": {"model": "anthropic/claude-sonnet-4-5"}},
|
||||
{"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-4-5"}},
|
||||
]
|
||||
)
|
||||
return ComplexityRouter(
|
||||
model_name="bench",
|
||||
litellm_router_instance=parent,
|
||||
complexity_router_config={"tiers": tiers},
|
||||
default_model="mid",
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def test_baseline_is_the_reasoning_tier_not_the_priciest_reachable_model(self):
|
||||
"""The ladder names what an operator would have had to run for the hardest
|
||||
request; a pricier model sitting in a lower tier is a choice, not a ceiling."""
|
||||
router = self._router_with_tiers({"SIMPLE": "top", "MEDIUM": "cheap", "REASONING": "mid"})
|
||||
assert router.savings_baseline_model == "anthropic/claude-sonnet-4-5"
|
||||
|
||||
def test_baseline_is_the_priciest_model_when_the_reasoning_tier_is_a_pool(self):
|
||||
router = self._router_with_tiers({"SIMPLE": "cheap", "REASONING": ["cheap", "top", "mid"]})
|
||||
assert router.savings_baseline_model == "anthropic/claude-opus-4-5"
|
||||
|
||||
def test_falls_back_to_the_hardest_configured_tier_when_reasoning_is_absent(self):
|
||||
router = self._router_with_tiers({"SIMPLE": "cheap", "COMPLEX": "top"})
|
||||
assert router.savings_baseline_model == "anthropic/claude-opus-4-5"
|
||||
|
||||
def test_a_configured_baseline_wins_and_is_provider_qualified(self):
|
||||
router = self._router_with_tiers(
|
||||
{"SIMPLE": "cheap", "REASONING": "mid"}, savings_baseline_model="claude-opus-4-5"
|
||||
)
|
||||
assert router.savings_baseline_model == "anthropic/claude-opus-4-5"
|
||||
|
||||
def test_an_unpriceable_tier_disables_the_driver_rather_than_inventing_a_baseline(self):
|
||||
router = self._router_with_tiers({"REASONING": "not-a-real-model-anywhere"})
|
||||
assert router.savings_baseline_model is None
|
||||
|
||||
def test_the_baseline_travels_on_every_pre_routing_response(self):
|
||||
"""A response without it silently zeroes the savings driver for that path."""
|
||||
import inspect
|
||||
|
||||
from litellm.router_strategy.complexity_router import complexity_router as module
|
||||
|
||||
source = inspect.getsource(module.ComplexityRouter.async_pre_routing_hook)
|
||||
returns = source.count("return PreRoutingHookResponse(")
|
||||
assert returns > 0
|
||||
assert source.count("savings_baseline_model=self.savings_baseline_model") == returns
|
||||
|
|
|
|||
155
tests/test_litellm/router_strategy/test_savings_baseline.py
Normal file
155
tests/test_litellm/router_strategy/test_savings_baseline.py
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
import pytest
|
||||
|
||||
from litellm.router import Router
|
||||
from litellm.router_strategy.savings_baseline import (
|
||||
canonical_model,
|
||||
models_for_group,
|
||||
most_expensive,
|
||||
resolve_baseline,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parent() -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}},
|
||||
{"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-4-5"}},
|
||||
{"model_name": "pool", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}},
|
||||
{"model_name": "pool", "litellm_params": {"model": "anthropic/claude-opus-4-5"}},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class TestCanonicalModel:
|
||||
def test_qualifies_a_bare_name_with_the_provider_that_owns_it(self):
|
||||
assert canonical_model("claude-opus-4-5") == "anthropic/claude-opus-4-5"
|
||||
|
||||
def test_keeps_an_already_qualified_name_qualified(self):
|
||||
assert canonical_model("anthropic/claude-opus-4-5") == "anthropic/claude-opus-4-5"
|
||||
|
||||
def test_honours_a_separately_declared_provider(self):
|
||||
assert canonical_model("claude-opus-4-5", "openai") == "openai/claude-opus-4-5"
|
||||
|
||||
def test_returns_none_for_a_name_no_provider_claims(self):
|
||||
assert canonical_model("") is None
|
||||
|
||||
|
||||
class TestModelsForGroup:
|
||||
def test_resolves_a_group_to_the_models_its_deployments_call(self, parent):
|
||||
assert models_for_group(parent, "cheap") == ("anthropic/claude-haiku-4-5",)
|
||||
|
||||
def test_returns_every_deployment_in_a_pooled_group(self, parent):
|
||||
assert sorted(models_for_group(parent, "pool")) == [
|
||||
"anthropic/claude-haiku-4-5",
|
||||
"anthropic/claude-opus-4-5",
|
||||
]
|
||||
|
||||
def test_treats_an_unknown_group_as_a_model_name(self, parent):
|
||||
"""A tier can point straight at a provider model rather than a configured group."""
|
||||
assert models_for_group(parent, "claude-opus-4-5") == ("anthropic/claude-opus-4-5",)
|
||||
|
||||
|
||||
class TestMostExpensive:
|
||||
def test_picks_by_output_rate(self):
|
||||
assert (
|
||||
most_expensive(["anthropic/claude-haiku-4-5", "anthropic/claude-opus-4-5"])
|
||||
== "anthropic/claude-opus-4-5"
|
||||
)
|
||||
|
||||
def test_ignores_models_with_no_per_token_price(self):
|
||||
"""A free model as baseline would report the whole real spend as a loss."""
|
||||
assert most_expensive(["not-a-real-model-anywhere", "anthropic/claude-haiku-4-5"]) == (
|
||||
"anthropic/claude-haiku-4-5"
|
||||
)
|
||||
|
||||
def test_returns_none_when_nothing_can_be_priced(self):
|
||||
assert most_expensive(["not-a-real-model-anywhere"]) is None
|
||||
|
||||
def test_returns_none_for_an_empty_candidate_set(self):
|
||||
assert most_expensive([]) is None
|
||||
|
||||
|
||||
class TestResolveBaseline:
|
||||
def test_a_configured_baseline_wins_over_the_candidates(self, parent):
|
||||
assert resolve_baseline("claude-haiku-4-5", parent, ["top"]) == "anthropic/claude-haiku-4-5"
|
||||
|
||||
def test_derives_the_priciest_candidate_when_unconfigured(self, parent):
|
||||
assert resolve_baseline(None, parent, ["cheap", "top"]) == "anthropic/claude-opus-4-5"
|
||||
|
||||
def test_never_raises_so_a_metric_cannot_fail_a_live_request(self):
|
||||
"""Read on the routing path while decorating a request that is about to be
|
||||
served; a dashboard counterfactual must not be able to take routing down."""
|
||||
|
||||
class Exploding:
|
||||
@property
|
||||
def model_name_to_deployment_indices(self):
|
||||
raise RuntimeError("router is mid-reload")
|
||||
|
||||
assert resolve_baseline(None, Exploding(), ["anything"]) is None
|
||||
|
||||
def test_an_empty_candidate_set_zeroes_the_driver_rather_than_inventing_one(self, parent):
|
||||
assert resolve_baseline(None, parent, []) is None
|
||||
|
||||
|
||||
class TestDeploymentsPricedByBaseModel:
|
||||
"""`litellm_params.model` is not always a model.
|
||||
|
||||
On Azure it is the deployment name, which is absent from the cost map, so pricing it
|
||||
directly drops the candidate. If that candidate was the priciest, the baseline quietly
|
||||
becomes the second priciest and every saving is understated; if the whole pool is
|
||||
Azure, nothing prices and the driver reports zero with nothing at default log level
|
||||
saying why. `model_info.base_model` is what names the real model, which is the chain
|
||||
router.py already resolves pricing through.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _router(*deployments: dict) -> Router:
|
||||
return Router(model_list=list(deployments))
|
||||
|
||||
def test_model_info_base_model_is_preferred_over_the_deployment_name(self):
|
||||
router = self._router(
|
||||
{
|
||||
"model_name": "big",
|
||||
"litellm_params": {"model": "azure/my-gpt5-deployment"},
|
||||
"model_info": {"base_model": "azure/gpt-4.1"},
|
||||
},
|
||||
)
|
||||
assert models_for_group(router, "big") == ("azure/gpt-4.1",)
|
||||
|
||||
def test_litellm_params_base_model_is_the_other_accepted_spelling(self):
|
||||
router = self._router(
|
||||
{
|
||||
"model_name": "big",
|
||||
"litellm_params": {"model": "azure/my-gpt5-deployment", "base_model": "azure/gpt-4.1"},
|
||||
},
|
||||
)
|
||||
assert models_for_group(router, "big") == ("azure/gpt-4.1",)
|
||||
|
||||
def test_a_deployment_without_a_base_model_still_prices_by_its_model(self):
|
||||
router = self._router({"model_name": "big", "litellm_params": {"model": "anthropic/claude-opus-4-5"}})
|
||||
assert models_for_group(router, "big") == ("anthropic/claude-opus-4-5",)
|
||||
|
||||
def test_an_azure_deployment_can_win_the_priciest_candidate(self):
|
||||
"""Without the base_model hop the Azure candidate never prices, so the cheaper
|
||||
model wins by default and the reported saving shrinks."""
|
||||
router = self._router(
|
||||
{"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}},
|
||||
{
|
||||
"model_name": "big",
|
||||
"litellm_params": {"model": "azure/my-gpt5-deployment"},
|
||||
"model_info": {"base_model": "azure/gpt-4.1"},
|
||||
},
|
||||
)
|
||||
assert resolve_baseline(None, router, ["cheap", "big"]) == "azure/gpt-4.1"
|
||||
|
||||
def test_an_all_azure_pool_still_has_a_baseline(self):
|
||||
"""Otherwise nothing prices, the driver is disabled and the card reads $0.00."""
|
||||
router = self._router(
|
||||
{
|
||||
"model_name": "big",
|
||||
"litellm_params": {"model": "azure/my-gpt5-deployment"},
|
||||
"model_info": {"base_model": "azure/gpt-4.1"},
|
||||
},
|
||||
)
|
||||
assert resolve_baseline(None, router, ["big"]) == "azure/gpt-4.1"
|
||||
Loading…
Add table
Reference in a new issue