scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

feat(spend): derive the savings baseline from the router's own candidates

Browse source 
Without an auto-router a deployment has to pick one model, and it has to be one
that can carry the hardest request, so the counterfactual is the priciest model
that router could have chosen.

A fixed flagship default measured savings against a model the operator may never
have run. A router choosing only between sonnet and haiku saved nobody the price
of opus, so every such deployment would have opened the dashboard to savings it
was never going to make, and the figure drifted the moment the routes changed.

The baseline is now the priciest candidate the router itself can reach, by output
rate with input breaking the tie. Routes name model groups rather than models, so
each is resolved through the parent router's deployments before being priced, and
the default model counts as a candidate. Resolution is lazy and cached, because
the parent router's deployments are still being assembled while the auto-router
is constructed.

`auto_router_savings_baseline_model` still overrides it for operators who would
genuinely have run something else. When nothing can be priced the baseline is
None and the driver reports zero, since a missing number beats a fabricated one.

Total saved needs no change: it sums the drivers, so the derived baseline flows
into it. Compression and prompt caching stay priced at the served model's rates,
which answers what each optimization saved on the request that actually ran.
This commit is contained in:
Tin Chi Lo 2026-07-31 20:52:29 -07:00
parent c46e96a6a9
commit 59d66e48e6
2 changed files with 189 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
litellm/router_strategy/auto_router/auto_router.py
View file

@ -20,7 +20,6 @@ else:
class AutoRouter(CustomLogger):
DEFAULT_AUTO_SYNC_VALUE = "local"
DEFAULT_SAVINGS_BASELINE_MODEL = "claude-opus-5"
def __init__(
self,
@ -42,7 +41,7 @@ class AutoRouter(CustomLogger):
default_model: The default model to use if no route is found.
embedding_model: The embedding model to use for the auto-router.
litellm_router_instance: The instance of the LiteLLM Router.
savings_baseline_model: The counterfactual model the dashboard measures savings against; falls back to DEFAULT_SAVINGS_BASELINE_MODEL.
savings_baseline_model: Overrides the counterfactual model the dashboard measures savings against; derived from this router's own candidates when unset.
"""
from semantic_router.routers import SemanticRouter
@ -54,7 +53,76 @@ class AutoRouter(CustomLogger):
self.default_model = default_model
self.embedding_model: str = embedding_model
self.litellm_router_instance: "Router" = litellm_router_instance
self.savings_baseline_model: str = savings_baseline_model or self.DEFAULT_SAVINGS_BASELINE_MODEL
self.configured_savings_baseline_model: str | None = savings_baseline_model
self._derived_savings_baseline_model: str | None = None
self._savings_baseline_derived = False
def _deployment_model(self, index: int) -> str | None:
"""The model a deployment actually calls."""
params = self.litellm_router_instance.model_list[index].get("litellm_params")
return params.get("model") if isinstance(params, dict) else None
def _models_for_group(self, group_name: str) -> tuple[str, ...]:
"""The models a route's model group actually calls, or the name itself when the
parent router has no deployment under it."""
indices = self.litellm_router_instance.model_name_to_deployment_indices.get(group_name)
if not indices:
return (group_name,)
return tuple(model for index in indices if (model := self._deployment_model(index)))
def _candidate_models(self) -> tuple[str, ...]:
"""Every model this router can route to, as pricable model names.
Routes name the router's own model groups rather than models, so each is
resolved through the parent router's deployments before anything is priced.
"""
group_names = frozenset(
name for name in (*(route.name for route in self.loaded_routes), self.default_model) if name
)
return tuple(model for group_name in group_names for model in self._models_for_group(group_name))
@staticmethod
def _priced_candidate(model: str) -> tuple[float, float, str] | None:
"""``(output_rate, input_rate, model)``, or ``None`` when the model has no pricing."""
import litellm
try:
info = litellm.get_model_info(model=model)
except Exception as e: # noqa: BLE001 # unmapped candidates simply cannot be the baseline
verbose_router_logger.debug("auto-router savings: no pricing for candidate %s (%s)", model, e)
return None
return (info.get("output_cost_per_token") or 0.0, info.get("input_cost_per_token") or 0.0, model)
def _most_expensive_candidate(self) -> str | None:
"""The priciest candidate by output rate, input rate breaking the tie."""
priced = tuple(
candidate for model in self._candidate_models() if (candidate := self._priced_candidate(model)) is not None
)
if not priced:
verbose_router_logger.debug("auto-router savings: no priceable candidates; savings driver disabled")
return None
return max(priced)[2]
@property
def savings_baseline_model(self) -> str | None:
"""The model this router's savings are measured against.
Without the router a deployment has to pick one model, and it has to be one that
can carry the hardest request, so the counterfactual is the priciest model this
router could have chosen. Deriving it from the router's own candidates keeps it
honest: a fixed flagship credits savings against a model the operator would
never have run, and drifts the moment the routes change.
Resolved lazily and cached, because the parent router's deployments are still
being assembled while this router is constructed. ``None`` when nothing can be
priced, which zeroes the driver rather than inventing a baseline.
"""
if self.configured_savings_baseline_model:
return self.configured_savings_baseline_model
if not self._savings_baseline_derived:
self._derived_savings_baseline_model = self._most_expensive_candidate()
self._savings_baseline_derived = True
return self._derived_savings_baseline_model
def _load_semantic_routing_routes(self) -> List[Route]:
from semantic_router.routers import SemanticRouter

133
tests/test_litellm/router_strategy/test_auto_router.py
View file

@ -156,6 +156,22 @@ class TestExtractTextFromMessages:
assert result == ""
def _routes(*names):
routes = []
for name in names:
route = MagicMock()
route.name = name
routes.append(route)
return routes
def _configure_candidates(router, group_to_model):
router.model_list = [
{"model_name": group, "litellm_params": {"model": model}} for group, model in group_to_model.items()
]
router.model_name_to_deployment_indices = {group: [i] for i, group in enumerate(group_to_model)}
@pytest.fixture
def mock_router_instance():
"""Create a mock LiteLLM Router instance."""
@ -317,21 +333,6 @@ class TestAutoRouter:
# Assert
assert result is None
@patch("semantic_router.routers.SemanticRouter")
def test_init_defaults_savings_baseline_model(self, mock_semantic_router_class, mock_router_instance):
"""Unconfigured deployments fall back to the flagship default, not an empty baseline."""
mock_semantic_router_class.from_json.return_value = mock_semantic_router_class
auto_router = AutoRouter(
model_name="test-auto-router",
auto_router_config_path="test/path/router.json",
default_model="gpt-4o-mini",
embedding_model="text-embedding-model",
litellm_router_instance=mock_router_instance,
)
assert auto_router.savings_baseline_model == AutoRouter.DEFAULT_SAVINGS_BASELINE_MODEL
@patch("semantic_router.routers.SemanticRouter")
def test_init_honors_configured_savings_baseline_model(self, mock_semantic_router_class, mock_router_instance):
"""An operator-configured baseline overrides the flagship default."""
@ -383,3 +384,105 @@ class TestAutoRouter:
assert result is not None
assert result.savings_baseline_model == "claude-opus-5"
class TestSavingsBaselineModel:
"""The counterfactual the cost dashboard measures auto-router savings against.
Constructed without __init__ on purpose: resolving the baseline touches only the
router's own deployments, never semantic_router, so this runs wherever the rest of
the beta suite is skipped.
"""
@staticmethod
def _auto_router(group_to_model: dict, route_names: list, default_model: str, configured=None) -> AutoRouter:
parent = MagicMock()
parent.model_list = [
{"model_name": group, "litellm_params": {"model": model}} for group, model in group_to_model.items()
]
parent.model_name_to_deployment_indices = {group: [i] for i, group in enumerate(group_to_model)}
auto_router = AutoRouter.__new__(AutoRouter)
auto_router.loaded_routes = []
for name in route_names:
route = MagicMock()
route.name = name
auto_router.loaded_routes.append(route)
auto_router.default_model = default_model
auto_router.litellm_router_instance = parent
auto_router.configured_savings_baseline_model = configured
auto_router._derived_savings_baseline_model = None
auto_router._savings_baseline_derived = False
return auto_router
def test_route_names_resolve_through_the_parent_router_to_pricable_models(self):
"""Routes name the router's own model groups, not models, so a group has to be
resolved to the model it actually calls before anything can be priced."""
auto_router = self._auto_router(
{"cheap-tier": "anthropic/claude-haiku-4-5", "mid-tier": "anthropic/claude-sonnet-5"},
["cheap-tier", "mid-tier"],
"cheap-tier",
)
assert sorted(auto_router._candidate_models()) == [
"anthropic/claude-haiku-4-5",
"anthropic/claude-sonnet-5",
]
def test_baseline_is_the_priciest_model_this_router_could_have_picked(self):
"""Without the router a deployment picks one model that can carry the hardest
request, so the counterfactual is this router's priciest candidate. A fixed
flagship credits savings against a model the operator would never have run: a
router choosing only between sonnet and haiku saved nobody the price of opus."""
sonnet_only = self._auto_router(
{"cheap-tier": "anthropic/claude-haiku-4-5", "mid-tier": "anthropic/claude-sonnet-5"},
["cheap-tier", "mid-tier"],
"cheap-tier",
)
assert sonnet_only.savings_baseline_model == "anthropic/claude-sonnet-5"
with_flagship = self._auto_router(
{
"cheap-tier": "anthropic/claude-haiku-4-5",
"mid-tier": "anthropic/claude-sonnet-5",
"big-tier": "anthropic/claude-opus-5",
},
["cheap-tier", "mid-tier", "big-tier"],
"cheap-tier",
)
assert with_flagship.savings_baseline_model == "anthropic/claude-opus-5"
def test_the_default_model_counts_as_a_candidate(self):
auto_router = self._auto_router(
{"cheap-tier": "anthropic/claude-haiku-4-5", "fallback": "anthropic/claude-opus-5"},
["cheap-tier"],
"fallback",
)
assert auto_router.savings_baseline_model == "anthropic/claude-opus-5"
def test_an_explicit_baseline_overrides_the_derived_one(self):
auto_router = self._auto_router(
{"cheap-tier": "anthropic/claude-haiku-4-5"},
["cheap-tier"],
"cheap-tier",
configured="claude-opus-5",
)
assert auto_router.savings_baseline_model == "claude-opus-5"
def test_nothing_priceable_disables_the_driver_rather_than_inventing_a_baseline(self):
"""A missing number beats a fabricated one."""
auto_router = self._auto_router({}, ["not-a-real-model"], "also-not-real")
assert auto_router.savings_baseline_model is None
def test_the_derived_baseline_is_resolved_once(self):
"""The parent router's deployments are still being assembled while this router
is constructed, so resolution is lazy; it must not re-derive per request."""
auto_router = self._auto_router(
{"cheap-tier": "anthropic/claude-haiku-4-5", "mid-tier": "anthropic/claude-sonnet-5"},
["cheap-tier", "mid-tier"],
"cheap-tier",
)
assert auto_router.savings_baseline_model == "anthropic/claude-sonnet-5"
auto_router.litellm_router_instance.model_list = []
auto_router.litellm_router_instance.model_name_to_deployment_indices = {}
assert auto_router.savings_baseline_model == "anthropic/claude-sonnet-5"