fix(router): price baseline candidates under the provider their deployment declares

A deployment can name its vendor in `custom_llm_provider` rather than in the
model prefix, which is the normal shape for Azure, Bedrock and Vertex. Pricing
the bare name alone resolves it to whichever vendor owns that name, or to
nothing: `claude-sonnet-4@20250514` prices at $0 without vertex_ai, and
`deepseek-r1` raises without azure_ai. Either way the candidate lost the
priciest-candidate contest, so the derived baseline silently became a cheaper
model and the driver under-reported.

Candidates now resolve through `get_llm_provider` and are carried as
`provider/model`, which is also what reaches the spend writer, so the baseline
resolves back to the vendor that served it rather than to whoever owns the bare
name.

A candidate with no per-token price is no longer eligible. Nothing that costs
nothing can stand in for what the traffic would otherwise have cost, and as a
baseline it would report the whole real spend as a loss. That outcome was
already unreachable, but only because the served model is drawn from the same
candidate set and fallbacks clear the baseline; the driver should not depend on
that chain holding.

litellm/router_strategy/auto_router/auto_router.py

@@ -57,17 +57,42 @@ class AutoRouter(CustomLogger):
         self._derived_savings_baseline_model: str | None = None
         self._savings_baseline_derived = False
 
+    @staticmethod
+    def _canonical_model(model: str, custom_llm_provider: str | None) -> str | None:
+        """``provider/model``, or ``None`` when the pair names no known provider.
+
+        A deployment may name its vendor either in the model prefix or in a separate
+        `custom_llm_provider`, and the bare name alone is not enough to price: it can
+        resolve to a different vendor's rates, or to nothing at all. Qualifying it here
+        means the baseline that reaches the spend writer resolves back to the same
+        vendor that served it.
+        """
+        import litellm
+
+        try:
+            resolved_model, provider, _, _ = litellm.get_llm_provider(
+                model=model, custom_llm_provider=custom_llm_provider
+            )
+        except Exception as e:  # noqa: BLE001  # an unroutable candidate cannot be the baseline
+            verbose_router_logger.debug("auto-router savings: cannot resolve candidate %s (%s)", model, e)
+            return None
+        return f"{provider}/{resolved_model}"
+
     def _deployment_model(self, index: int) -> str | None:
-        """The model a deployment actually calls."""
+        """The model a deployment calls, qualified by the provider it declares."""
         params = self.litellm_router_instance.model_list[index].get("litellm_params")
-        return params.get("model") if isinstance(params, dict) else None
+        if not isinstance(params, dict):
+            return None
+        model = params.get("model")
+        return self._canonical_model(model, params.get("custom_llm_provider")) if model else None
 
     def _models_for_group(self, group_name: str) -> tuple[str, ...]:
         """The models a route's model group actually calls, or the name itself when the
         parent router has no deployment under it."""
         indices = self.litellm_router_instance.model_name_to_deployment_indices.get(group_name)
         if not indices:
-            return (group_name,)
+            canonical = self._canonical_model(group_name, None)
+            return (canonical,) if canonical else ()
         return tuple(model for index in indices if (model := self._deployment_model(index)))
 
     def _candidate_models(self) -> tuple[str, ...]:
@@ -91,7 +116,15 @@ class AutoRouter(CustomLogger):
         except Exception as e:  # noqa: BLE001  # unmapped candidates simply cannot be the baseline
             verbose_router_logger.debug("auto-router savings: no pricing for candidate %s (%s)", model, e)
             return None
-        return (info.get("output_cost_per_token") or 0.0, info.get("input_cost_per_token") or 0.0, model)
+        output_rate = info.get("output_cost_per_token") or 0.0
+        input_rate = info.get("input_cost_per_token") or 0.0
+        if output_rate <= 0.0 and input_rate <= 0.0:
+            # A model that costs nothing per token cannot stand in for what the traffic
+            # would otherwise have cost, and as a baseline it would report the whole
+            # real spend as a loss.
+            verbose_router_logger.debug("auto-router savings: candidate %s has no per-token price", model)
+            return None
+        return (output_rate, input_rate, model)
 
     def _most_expensive_candidate(self) -> str | None:
         """The priciest candidate by output rate, input rate breaking the tie."""

tests/test_litellm/router_strategy/test_auto_router.py

@@ -398,7 +398,8 @@ class TestSavingsBaselineModel:
     def _auto_router(group_to_model: dict, route_names: list, default_model: str, configured=None) -> AutoRouter:
         parent = MagicMock()
         parent.model_list = [
-            {"model_name": group, "litellm_params": {"model": model}} for group, model in group_to_model.items()
+            {"model_name": group, "litellm_params": dict(params) if isinstance(params, dict) else {"model": params}}
+            for group, params in group_to_model.items()
         ]
         parent.model_name_to_deployment_indices = {group: [i] for i, group in enumerate(group_to_model)}
 
@@ -486,3 +487,55 @@ class TestSavingsBaselineModel:
         auto_router.litellm_router_instance.model_list = []
         auto_router.litellm_router_instance.model_name_to_deployment_indices = {}
         assert auto_router.savings_baseline_model == "anthropic/claude-sonnet-5"
+
+    def test_a_deployment_naming_its_provider_separately_is_still_priced(self):
+        """A deployment may name its vendor in `custom_llm_provider` rather than in the
+        model prefix. Pricing the bare name then resolves to a different vendor's rates
+        or to nothing at all, so the candidate is mispriced or silently dropped and the
+        derived baseline is wrong. Vertex prices this model at $0 without its provider
+        and azure_ai raises outright, so neither could ever win as the priciest."""
+        auto_router = self._auto_router(
+            {
+                "cheap": {"model": "claude-haiku-4-5", "custom_llm_provider": "anthropic"},
+                "vertex-tier": {"model": "claude-sonnet-4@20250514", "custom_llm_provider": "vertex_ai"},
+            },
+            ["cheap", "vertex-tier"],
+            "cheap",
+        )
+        assert auto_router.savings_baseline_model == "vertex_ai/claude-sonnet-4@20250514"
+
+    def test_candidates_are_qualified_so_the_spend_writer_resolves_the_same_vendor(self):
+        """The baseline travels to the spend writer as a bare string, so it has to carry
+        its provider or the writer prices it under whichever vendor owns the bare name."""
+        from litellm.proxy.spend_tracking.savings import _resolve_model
+
+        auto_router = self._auto_router(
+            {"azure-tier": {"model": "deepseek-r1", "custom_llm_provider": "azure_ai"}},
+            ["azure-tier"],
+            "azure-tier",
+        )
+        baseline = auto_router.savings_baseline_model
+        assert baseline == "azure_ai/deepseek-r1"
+        assert _resolve_model(baseline, None) == ("deepseek-r1", "azure_ai")
+
+    def test_a_candidate_with_no_per_token_price_cannot_be_the_baseline(self):
+        """A model that costs nothing per token cannot stand in for what the traffic
+        would otherwise have cost. Left in, it would report the whole real spend as a
+        loss the moment it won the priciest-candidate contest."""
+        auto_router = self._auto_router(
+            {"images": {"model": "dall-e-2", "custom_llm_provider": "openai"}},
+            ["images"],
+            "images",
+        )
+        assert auto_router.savings_baseline_model is None
+
+    def test_a_priced_candidate_still_wins_over_an_unpriced_one(self):
+        auto_router = self._auto_router(
+            {
+                "images": {"model": "dall-e-2", "custom_llm_provider": "openai"},
+                "chat": {"model": "claude-haiku-4-5", "custom_llm_provider": "anthropic"},
+            },
+            ["images", "chat"],
+            "chat",
+        )
+        assert auto_router.savings_baseline_model == "anthropic/claude-haiku-4-5"