diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index 3b8d4d134b0..ba59b711234 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -20,7 +20,6 @@ else: class AutoRouter(CustomLogger): DEFAULT_AUTO_SYNC_VALUE = "local" - DEFAULT_SAVINGS_BASELINE_MODEL = "claude-opus-5" def __init__( self, @@ -42,7 +41,7 @@ class AutoRouter(CustomLogger): default_model: The default model to use if no route is found. embedding_model: The embedding model to use for the auto-router. litellm_router_instance: The instance of the LiteLLM Router. - savings_baseline_model: The counterfactual model the dashboard measures savings against; falls back to DEFAULT_SAVINGS_BASELINE_MODEL. + savings_baseline_model: Overrides the counterfactual model the dashboard measures savings against; derived from this router's own candidates when unset. """ from semantic_router.routers import SemanticRouter @@ -54,7 +53,76 @@ class AutoRouter(CustomLogger): self.default_model = default_model self.embedding_model: str = embedding_model self.litellm_router_instance: "Router" = litellm_router_instance - self.savings_baseline_model: str = savings_baseline_model or self.DEFAULT_SAVINGS_BASELINE_MODEL + self.configured_savings_baseline_model: str | None = savings_baseline_model + self._derived_savings_baseline_model: str | None = None + self._savings_baseline_derived = False + + def _deployment_model(self, index: int) -> str | None: + """The model a deployment actually calls.""" + params = self.litellm_router_instance.model_list[index].get("litellm_params") + return params.get("model") if isinstance(params, dict) else None + + def _models_for_group(self, group_name: str) -> tuple[str, ...]: + """The models a route's model group actually calls, or the name itself when the + parent router has no deployment under it.""" + indices = self.litellm_router_instance.model_name_to_deployment_indices.get(group_name) + if not indices: + return (group_name,) + return tuple(model for index in indices if (model := self._deployment_model(index))) + + def _candidate_models(self) -> tuple[str, ...]: + """Every model this router can route to, as pricable model names. + + Routes name the router's own model groups rather than models, so each is + resolved through the parent router's deployments before anything is priced. + """ + group_names = frozenset( + name for name in (*(route.name for route in self.loaded_routes), self.default_model) if name + ) + return tuple(model for group_name in group_names for model in self._models_for_group(group_name)) + + @staticmethod + def _priced_candidate(model: str) -> tuple[float, float, str] | None: + """``(output_rate, input_rate, model)``, or ``None`` when the model has no pricing.""" + import litellm + + try: + info = litellm.get_model_info(model=model) + except Exception as e: # noqa: BLE001 # unmapped candidates simply cannot be the baseline + verbose_router_logger.debug("auto-router savings: no pricing for candidate %s (%s)", model, e) + return None + return (info.get("output_cost_per_token") or 0.0, info.get("input_cost_per_token") or 0.0, model) + + def _most_expensive_candidate(self) -> str | None: + """The priciest candidate by output rate, input rate breaking the tie.""" + priced = tuple( + candidate for model in self._candidate_models() if (candidate := self._priced_candidate(model)) is not None + ) + if not priced: + verbose_router_logger.debug("auto-router savings: no priceable candidates; savings driver disabled") + return None + return max(priced)[2] + + @property + def savings_baseline_model(self) -> str | None: + """The model this router's savings are measured against. + + Without the router a deployment has to pick one model, and it has to be one that + can carry the hardest request, so the counterfactual is the priciest model this + router could have chosen. Deriving it from the router's own candidates keeps it + honest: a fixed flagship credits savings against a model the operator would + never have run, and drifts the moment the routes change. + + Resolved lazily and cached, because the parent router's deployments are still + being assembled while this router is constructed. ``None`` when nothing can be + priced, which zeroes the driver rather than inventing a baseline. + """ + if self.configured_savings_baseline_model: + return self.configured_savings_baseline_model + if not self._savings_baseline_derived: + self._derived_savings_baseline_model = self._most_expensive_candidate() + self._savings_baseline_derived = True + return self._derived_savings_baseline_model def _load_semantic_routing_routes(self) -> List[Route]: from semantic_router.routers import SemanticRouter diff --git a/tests/test_litellm/router_strategy/test_auto_router.py b/tests/test_litellm/router_strategy/test_auto_router.py index 3cc23cc6f9e..3d4d7931ae6 100644 --- a/tests/test_litellm/router_strategy/test_auto_router.py +++ b/tests/test_litellm/router_strategy/test_auto_router.py @@ -156,6 +156,22 @@ class TestExtractTextFromMessages: assert result == "" +def _routes(*names): + routes = [] + for name in names: + route = MagicMock() + route.name = name + routes.append(route) + return routes + + +def _configure_candidates(router, group_to_model): + router.model_list = [ + {"model_name": group, "litellm_params": {"model": model}} for group, model in group_to_model.items() + ] + router.model_name_to_deployment_indices = {group: [i] for i, group in enumerate(group_to_model)} + + @pytest.fixture def mock_router_instance(): """Create a mock LiteLLM Router instance.""" @@ -317,21 +333,6 @@ class TestAutoRouter: # Assert assert result is None - @patch("semantic_router.routers.SemanticRouter") - def test_init_defaults_savings_baseline_model(self, mock_semantic_router_class, mock_router_instance): - """Unconfigured deployments fall back to the flagship default, not an empty baseline.""" - mock_semantic_router_class.from_json.return_value = mock_semantic_router_class - - auto_router = AutoRouter( - model_name="test-auto-router", - auto_router_config_path="test/path/router.json", - default_model="gpt-4o-mini", - embedding_model="text-embedding-model", - litellm_router_instance=mock_router_instance, - ) - - assert auto_router.savings_baseline_model == AutoRouter.DEFAULT_SAVINGS_BASELINE_MODEL - @patch("semantic_router.routers.SemanticRouter") def test_init_honors_configured_savings_baseline_model(self, mock_semantic_router_class, mock_router_instance): """An operator-configured baseline overrides the flagship default.""" @@ -383,3 +384,105 @@ class TestAutoRouter: assert result is not None assert result.savings_baseline_model == "claude-opus-5" + + +class TestSavingsBaselineModel: + """The counterfactual the cost dashboard measures auto-router savings against. + + Constructed without __init__ on purpose: resolving the baseline touches only the + router's own deployments, never semantic_router, so this runs wherever the rest of + the beta suite is skipped. + """ + + @staticmethod + def _auto_router(group_to_model: dict, route_names: list, default_model: str, configured=None) -> AutoRouter: + parent = MagicMock() + parent.model_list = [ + {"model_name": group, "litellm_params": {"model": model}} for group, model in group_to_model.items() + ] + parent.model_name_to_deployment_indices = {group: [i] for i, group in enumerate(group_to_model)} + + auto_router = AutoRouter.__new__(AutoRouter) + auto_router.loaded_routes = [] + for name in route_names: + route = MagicMock() + route.name = name + auto_router.loaded_routes.append(route) + auto_router.default_model = default_model + auto_router.litellm_router_instance = parent + auto_router.configured_savings_baseline_model = configured + auto_router._derived_savings_baseline_model = None + auto_router._savings_baseline_derived = False + return auto_router + + def test_route_names_resolve_through_the_parent_router_to_pricable_models(self): + """Routes name the router's own model groups, not models, so a group has to be + resolved to the model it actually calls before anything can be priced.""" + auto_router = self._auto_router( + {"cheap-tier": "anthropic/claude-haiku-4-5", "mid-tier": "anthropic/claude-sonnet-5"}, + ["cheap-tier", "mid-tier"], + "cheap-tier", + ) + assert sorted(auto_router._candidate_models()) == [ + "anthropic/claude-haiku-4-5", + "anthropic/claude-sonnet-5", + ] + + def test_baseline_is_the_priciest_model_this_router_could_have_picked(self): + """Without the router a deployment picks one model that can carry the hardest + request, so the counterfactual is this router's priciest candidate. A fixed + flagship credits savings against a model the operator would never have run: a + router choosing only between sonnet and haiku saved nobody the price of opus.""" + sonnet_only = self._auto_router( + {"cheap-tier": "anthropic/claude-haiku-4-5", "mid-tier": "anthropic/claude-sonnet-5"}, + ["cheap-tier", "mid-tier"], + "cheap-tier", + ) + assert sonnet_only.savings_baseline_model == "anthropic/claude-sonnet-5" + + with_flagship = self._auto_router( + { + "cheap-tier": "anthropic/claude-haiku-4-5", + "mid-tier": "anthropic/claude-sonnet-5", + "big-tier": "anthropic/claude-opus-5", + }, + ["cheap-tier", "mid-tier", "big-tier"], + "cheap-tier", + ) + assert with_flagship.savings_baseline_model == "anthropic/claude-opus-5" + + def test_the_default_model_counts_as_a_candidate(self): + auto_router = self._auto_router( + {"cheap-tier": "anthropic/claude-haiku-4-5", "fallback": "anthropic/claude-opus-5"}, + ["cheap-tier"], + "fallback", + ) + assert auto_router.savings_baseline_model == "anthropic/claude-opus-5" + + def test_an_explicit_baseline_overrides_the_derived_one(self): + auto_router = self._auto_router( + {"cheap-tier": "anthropic/claude-haiku-4-5"}, + ["cheap-tier"], + "cheap-tier", + configured="claude-opus-5", + ) + assert auto_router.savings_baseline_model == "claude-opus-5" + + def test_nothing_priceable_disables_the_driver_rather_than_inventing_a_baseline(self): + """A missing number beats a fabricated one.""" + auto_router = self._auto_router({}, ["not-a-real-model"], "also-not-real") + assert auto_router.savings_baseline_model is None + + def test_the_derived_baseline_is_resolved_once(self): + """The parent router's deployments are still being assembled while this router + is constructed, so resolution is lazy; it must not re-derive per request.""" + auto_router = self._auto_router( + {"cheap-tier": "anthropic/claude-haiku-4-5", "mid-tier": "anthropic/claude-sonnet-5"}, + ["cheap-tier", "mid-tier"], + "cheap-tier", + ) + assert auto_router.savings_baseline_model == "anthropic/claude-sonnet-5" + + auto_router.litellm_router_instance.model_list = [] + auto_router.litellm_router_instance.model_name_to_deployment_indices = {} + assert auto_router.savings_baseline_model == "anthropic/claude-sonnet-5"