diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index ba59b711234..737cbeedb50 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -57,17 +57,42 @@ class AutoRouter(CustomLogger): self._derived_savings_baseline_model: str | None = None self._savings_baseline_derived = False + @staticmethod + def _canonical_model(model: str, custom_llm_provider: str | None) -> str | None: + """``provider/model``, or ``None`` when the pair names no known provider. + + A deployment may name its vendor either in the model prefix or in a separate + `custom_llm_provider`, and the bare name alone is not enough to price: it can + resolve to a different vendor's rates, or to nothing at all. Qualifying it here + means the baseline that reaches the spend writer resolves back to the same + vendor that served it. + """ + import litellm + + try: + resolved_model, provider, _, _ = litellm.get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider + ) + except Exception as e: # noqa: BLE001 # an unroutable candidate cannot be the baseline + verbose_router_logger.debug("auto-router savings: cannot resolve candidate %s (%s)", model, e) + return None + return f"{provider}/{resolved_model}" + def _deployment_model(self, index: int) -> str | None: - """The model a deployment actually calls.""" + """The model a deployment calls, qualified by the provider it declares.""" params = self.litellm_router_instance.model_list[index].get("litellm_params") - return params.get("model") if isinstance(params, dict) else None + if not isinstance(params, dict): + return None + model = params.get("model") + return self._canonical_model(model, params.get("custom_llm_provider")) if model else None def _models_for_group(self, group_name: str) -> tuple[str, ...]: """The models a route's model group actually calls, or the name itself when the parent router has no deployment under it.""" indices = self.litellm_router_instance.model_name_to_deployment_indices.get(group_name) if not indices: - return (group_name,) + canonical = self._canonical_model(group_name, None) + return (canonical,) if canonical else () return tuple(model for index in indices if (model := self._deployment_model(index))) def _candidate_models(self) -> tuple[str, ...]: @@ -91,7 +116,15 @@ class AutoRouter(CustomLogger): except Exception as e: # noqa: BLE001 # unmapped candidates simply cannot be the baseline verbose_router_logger.debug("auto-router savings: no pricing for candidate %s (%s)", model, e) return None - return (info.get("output_cost_per_token") or 0.0, info.get("input_cost_per_token") or 0.0, model) + output_rate = info.get("output_cost_per_token") or 0.0 + input_rate = info.get("input_cost_per_token") or 0.0 + if output_rate <= 0.0 and input_rate <= 0.0: + # A model that costs nothing per token cannot stand in for what the traffic + # would otherwise have cost, and as a baseline it would report the whole + # real spend as a loss. + verbose_router_logger.debug("auto-router savings: candidate %s has no per-token price", model) + return None + return (output_rate, input_rate, model) def _most_expensive_candidate(self) -> str | None: """The priciest candidate by output rate, input rate breaking the tie.""" diff --git a/tests/test_litellm/router_strategy/test_auto_router.py b/tests/test_litellm/router_strategy/test_auto_router.py index 3d4d7931ae6..52c39fff6c3 100644 --- a/tests/test_litellm/router_strategy/test_auto_router.py +++ b/tests/test_litellm/router_strategy/test_auto_router.py @@ -398,7 +398,8 @@ class TestSavingsBaselineModel: def _auto_router(group_to_model: dict, route_names: list, default_model: str, configured=None) -> AutoRouter: parent = MagicMock() parent.model_list = [ - {"model_name": group, "litellm_params": {"model": model}} for group, model in group_to_model.items() + {"model_name": group, "litellm_params": dict(params) if isinstance(params, dict) else {"model": params}} + for group, params in group_to_model.items() ] parent.model_name_to_deployment_indices = {group: [i] for i, group in enumerate(group_to_model)} @@ -486,3 +487,55 @@ class TestSavingsBaselineModel: auto_router.litellm_router_instance.model_list = [] auto_router.litellm_router_instance.model_name_to_deployment_indices = {} assert auto_router.savings_baseline_model == "anthropic/claude-sonnet-5" + + def test_a_deployment_naming_its_provider_separately_is_still_priced(self): + """A deployment may name its vendor in `custom_llm_provider` rather than in the + model prefix. Pricing the bare name then resolves to a different vendor's rates + or to nothing at all, so the candidate is mispriced or silently dropped and the + derived baseline is wrong. Vertex prices this model at $0 without its provider + and azure_ai raises outright, so neither could ever win as the priciest.""" + auto_router = self._auto_router( + { + "cheap": {"model": "claude-haiku-4-5", "custom_llm_provider": "anthropic"}, + "vertex-tier": {"model": "claude-sonnet-4@20250514", "custom_llm_provider": "vertex_ai"}, + }, + ["cheap", "vertex-tier"], + "cheap", + ) + assert auto_router.savings_baseline_model == "vertex_ai/claude-sonnet-4@20250514" + + def test_candidates_are_qualified_so_the_spend_writer_resolves_the_same_vendor(self): + """The baseline travels to the spend writer as a bare string, so it has to carry + its provider or the writer prices it under whichever vendor owns the bare name.""" + from litellm.proxy.spend_tracking.savings import _resolve_model + + auto_router = self._auto_router( + {"azure-tier": {"model": "deepseek-r1", "custom_llm_provider": "azure_ai"}}, + ["azure-tier"], + "azure-tier", + ) + baseline = auto_router.savings_baseline_model + assert baseline == "azure_ai/deepseek-r1" + assert _resolve_model(baseline, None) == ("deepseek-r1", "azure_ai") + + def test_a_candidate_with_no_per_token_price_cannot_be_the_baseline(self): + """A model that costs nothing per token cannot stand in for what the traffic + would otherwise have cost. Left in, it would report the whole real spend as a + loss the moment it won the priciest-candidate contest.""" + auto_router = self._auto_router( + {"images": {"model": "dall-e-2", "custom_llm_provider": "openai"}}, + ["images"], + "images", + ) + assert auto_router.savings_baseline_model is None + + def test_a_priced_candidate_still_wins_over_an_unpriced_one(self): + auto_router = self._auto_router( + { + "images": {"model": "dall-e-2", "custom_llm_provider": "openai"}, + "chat": {"model": "claude-haiku-4-5", "custom_llm_provider": "anthropic"}, + }, + ["images", "chat"], + "chat", + ) + assert auto_router.savings_baseline_model == "anthropic/claude-haiku-4-5"