fix(complexity_router): make tier resolution report how it resolved, not just to what

Resolution returned (model, served_tier), a shape that cannot express two of its own
outcomes. When default_model answered, served_tier was still the classified tier, so it
was indistinguishable from an ordinary pick out of that tier; likewise when nothing
reported a live deployment and the tier was served on a best effort. Callers inferred
"no fallback happened" from served_tier == classified_tier, which was true in both of
those cases and wrong

_pick_model_for_tier now returns a TierResolution naming the model, the classified tier,
the tier that served, and how the model was obtained. `resolved_by` is `default_model`
when the ladder was exhausted and `best_effort` when nothing confirmed a live
deployment, and it is omitted for the ordinary path. The climb keeps its own field. The
two are independent facts, since an exhausted ladder can both climb to a configured tier
and be serving something unconfirmed, so neither is derived from the other

_build_routing_decision takes the resolution and derives both facts itself, instead of
each of the three call sites recomputing the comparison and only one of them being able
to spot the fallback at all

Also drops a comment that described the pre-ladder fallback (default_model reached
"after the MEDIUM tier"), which stopped being true when the ladder replaced the fixed
MEDIUM jump

litellm/router_strategy/complexity_router/complexity_router.py

@@ -32,6 +32,7 @@ from litellm.types.utils import (
     AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
     ModelResponse,
     RoutingDecisionCause,
+    RoutingDecisionTierResolution,
     StandardLoggingRoutingDecision,
     StandardLoggingRoutingDecisionTierBoundaries,
 )
@@ -279,6 +280,22 @@ class KeywordOverride(NamedTuple):
     matched_keyword: str | None
 
 
+class TierResolution(NamedTuple):
+    """How a classified tier became a model, not just which model came out.
+
+    Returning the served tier alone cannot express that `default_model` answered, or that
+    nothing reported a live deployment and the tier was served anyway: both look identical
+    to an ordinary pick from the classified tier, so a decision record could not say which
+    happened. `resolved_by` carries that, and `served_tier` differing from `classified_tier`
+    carries the climb. The two are independent; an exhausted ladder can do both.
+    """
+
+    model: str
+    classified_tier: ComplexityTier
+    served_tier: ComplexityTier
+    resolved_by: RoutingDecisionTierResolution
+
+
 class ClassificationOutcome(NamedTuple):
     """What the classifier decided and which mechanism actually produced it.
 
@@ -596,7 +613,7 @@ class ComplexityRouter(CustomLogger):
         escalation_keyword: str | None = None,
         escalated: bool = False,
         classifier_model: str | None = None,
-        tier_fallback_from: ComplexityTier | None = None,
+        resolution: TierResolution | None = None,
     ) -> StandardLoggingRoutingDecision:
         """Assemble the per-request provenance record for this router's decision.
 
@@ -633,11 +650,16 @@ class ComplexityRouter(CustomLogger):
             decision["escalated"] = escalated
         if classifier_model is not None:
             decision["classifier_model"] = classifier_model
-        if tier_fallback_from is not None:
-            # `tier` is what served the request. When the ladder had to climb, the tier the
-            # request was actually classified into is the fact that explains why a cheap
-            # request is on an expensive model, so it rides along rather than being lost.
-            decision["tier_fallback_from"] = tier_fallback_from.value
+        if resolution is not None:
+            # Two independent facts, both invisible from the routed model alone. A served
+            # tier above the classified one explains why a cheap request is on an expensive
+            # model; resolved_by says the model came from somewhere other than a tier pool
+            # at all. An exhausted ladder can produce both, so neither is derived from the
+            # other, and the ordinary case adds nothing.
+            if resolution.served_tier != resolution.classified_tier:
+                decision["tier_fallback_from"] = resolution.classified_tier.value
+            if resolution.resolved_by != "tier_pool":
+                decision["resolved_by"] = resolution.resolved_by
         return decision
 
     async def aclassify(
@@ -818,28 +840,38 @@ class ComplexityRouter(CustomLogger):
         Returns:
             The model name configured for that tier.
         """
-        model, _served_tier = self._resolve_configured_tier(tier)
-        return model
+        return self._resolve_configured_tier(tier).model
 
-    def _resolve_configured_tier(self, tier: ComplexityTier) -> tuple[str, ComplexityTier]:
+    def _resolve_configured_tier(
+        self, tier: ComplexityTier, resolved_by: RoutingDecisionTierResolution = "tier_pool"
+    ) -> TierResolution:
         """Structural half of tier resolution: the first tier at or above `tier` that has
         models configured, then `default_model` as the last resort.
 
         Falling upward rather than to a fixed tier keeps an under-configured deployment from
-        quietly serving a request from a cheaper model than it classified.
+        quietly serving a request from a cheaper model than it classified. `resolved_by` is
+        what the caller already knows about how it got here, since reaching this without a
+        live deployment anywhere is a best effort rather than an ordinary pick.
         """
         for candidate_tier in self._tier_ladder_from(tier):
             if candidate_tier.value not in self.config.tiers:
                 continue
             # A tier configured as an empty pool is a config error, not a gap to route
             # around, so _pick_from_tier_value raises rather than climbing past it.
-            return (
-                self._pick_from_tier_value(self.config.tiers[candidate_tier.value], candidate_tier.value),
-                candidate_tier,
+            return TierResolution(
+                model=self._pick_from_tier_value(self.config.tiers[candidate_tier.value], candidate_tier.value),
+                classified_tier=tier,
+                served_tier=candidate_tier,
+                resolved_by=resolved_by,
             )
 
         if self.config.default_model:
-            return self.config.default_model, tier
+            return TierResolution(
+                model=self.config.default_model,
+                classified_tier=tier,
+                served_tier=tier,
+                resolved_by="default_model",
+            )
 
         raise ValueError(
             f"No model configured for tier {tier.value} or any tier above it "
@@ -895,8 +927,8 @@ class ComplexityRouter(CustomLogger):
         raw_messages: list[dict[str, Any]] | None,
         resolved_messages: list[dict[str, Any]] | None,
         request_kwargs: dict,
-    ) -> tuple[str, ComplexityTier]:
-        """Resolve a classified tier to a servable model, returning the tier it came from.
+    ) -> TierResolution:
+        """Resolve a classified tier to a servable model, reporting how it got there.
 
         Walks the tier ladder: the classified tier's live models first, then the next tier
         up, and so on. `default_model` stays the last resort, and is skipped entirely when
@@ -912,12 +944,19 @@ class ComplexityRouter(CustomLogger):
             live = await self._live_models(pool)
             if not live:
                 continue
-            if not self.config.plugins:
-                return random.choice(live), candidate_tier
-            narrowed = await self._run_routing_plugins(
-                live, candidate_tier, raw_messages, resolved_messages, request_kwargs
+            candidates = (
+                live
+                if not self.config.plugins
+                else await self._run_routing_plugins(
+                    live, candidate_tier, raw_messages, resolved_messages, request_kwargs
+                )
+            )
+            return TierResolution(
+                model=random.choice(candidates),
+                classified_tier=tier,
+                served_tier=candidate_tier,
+                resolved_by="tier_pool",
             )
-            return random.choice(narrowed), candidate_tier
 
         if self.config.plugins:
             # default_model was never checked against the plugins, so serving it here would
@@ -927,12 +966,18 @@ class ComplexityRouter(CustomLogger):
                 f"No servable models for tier {tier.value} or any tier above it, with routing plugins configured"
             )
         if self.config.default_model:
-            return self.config.default_model, tier
+            return TierResolution(
+                model=self.config.default_model,
+                classified_tier=tier,
+                served_tier=tier,
+                resolved_by="default_model",
+            )
         # Nothing on the ladder reported a live deployment and there is no declared
         # fallback. Cooldowns expire and the health view is a snapshot, so serve the
         # classified tier anyway and let the router surface the real error, rather than
-        # failing a request that may well succeed.
-        return self._resolve_configured_tier(tier)
+        # failing a request that may well succeed. It is recorded as a best effort, since
+        # the model that answers is one nothing confirmed was up.
+        return self._resolve_configured_tier(tier, resolved_by="best_effort")
 
     async def _run_routing_plugins(
         self,
@@ -1527,12 +1572,12 @@ class ComplexityRouter(CustomLogger):
                 # every non-plugin user, not just a security fix).
                 routed_model = self.config.default_model
             else:
-                # Plugins configured: default_model must never bypass them, so it's not
-                # checked here at all -- _pick_model_for_tier -> get_model_for_tier still
-                # falls back to it (after the MEDIUM tier) once the plugin pipeline runs.
-                routed_model, _ = await self._pick_model_for_tier(
-                    ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs
-                )
+                # Plugins configured: default_model must never bypass them, so it is not
+                # checked here at all. The tier ladder still reaches it as its last resort,
+                # but only once the plugin pipeline has run.
+                routed_model = (
+                    await self._pick_model_for_tier(ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs)
+                ).model
             return PreRoutingHookResponse(
                 model=routed_model,
                 messages=messages if has_original_messages else None,
@@ -1546,9 +1591,10 @@ class ComplexityRouter(CustomLogger):
         if override is not None:
             routed_tier = self._escalate_tier(override.tier) if escalation_keyword is not None else override.tier
             keyword_escalated = routed_tier != override.tier
-            routed_model, keyword_served_tier = await self._pick_model_for_tier(
+            keyword_resolution = await self._pick_model_for_tier(
                 routed_tier, messages, resolved_messages, request_kwargs
             )
+            routed_model = keyword_resolution.model
             keyword_cause: RoutingDecisionCause = (
                 "semantic_keyword_match" if self.config.semantic_keyword_matching else "literal_keyword_match"
             )
@@ -1562,11 +1608,11 @@ class ComplexityRouter(CustomLogger):
                 routing_decision=self._build_routing_decision(
                     routed_model=routed_model,
                     cause=keyword_cause,
-                    tier=keyword_served_tier,
+                    tier=keyword_resolution.served_tier,
                     matched_keyword=override.matched_keyword,
                     escalation_keyword=escalation_keyword,
                     escalated=keyword_escalated,
-                    tier_fallback_from=routed_tier if keyword_served_tier != routed_tier else None,
+                    resolution=keyword_resolution,
                 ),
             )
 
@@ -1579,6 +1625,7 @@ class ComplexityRouter(CustomLogger):
         if escalated:
             signals = (*signals, "escalation")
         score_repr = f"{score:.3f}" if score is not None else "n/a"
+        resolution: TierResolution | None = None
         served_tier = tier
         if self.config.adaptive:
             routed_model = self._soft_floor_pick(tier, user_message, request_kwargs)
@@ -1594,13 +1641,13 @@ class ComplexityRouter(CustomLogger):
                 f"signals={signals}, routed_model={routed_model}"
             )
         else:
-            routed_model, served_tier = await self._pick_model_for_tier(
-                tier, messages, resolved_messages, request_kwargs
-            )
+            resolution = await self._pick_model_for_tier(tier, messages, resolved_messages, request_kwargs)
+            routed_model, served_tier = resolution.model, resolution.served_tier
             verbose_router_logger.info(
                 f"ComplexityRouter: routing decision cause={outcome.cause}, tier={served_tier.value}, "
                 f"score={score_repr}, signals={signals}, routed_model={routed_model}"
                 + (f", tier_fallback_from={tier.value}" if served_tier != tier else "")
+                + (f", resolved_by={resolution.resolved_by}" if resolution.resolved_by != "tier_pool" else "")
             )
 
         classifier_model = (
@@ -1620,6 +1667,6 @@ class ComplexityRouter(CustomLogger):
                 escalation_keyword=escalation_keyword,
                 escalated=escalated,
                 classifier_model=classifier_model,
-                tier_fallback_from=tier if served_tier != tier else None,
+                resolution=resolution,
             ),
         )

litellm/types/utils.py

@@ -2718,6 +2718,17 @@ records that it is not traffic the caller sent."""
 AUTOROUTER_CLASSIFIER_CALL_ORIGIN: InternalCallOrigin = "autorouter_classifier"
 
 
+RoutingDecisionTierResolution = Literal[
+    "tier_pool",
+    "default_model",
+    "best_effort",
+]
+"""How a classified tier became a model. `tier_pool` is the ordinary path and is omitted
+from the record; `default_model` means no tier on the ladder could serve, so the declared
+fallback did; `best_effort` means nothing reported a live deployment and the tier was
+served anyway rather than failing a request that may still succeed."""
+
+
 class StandardLoggingRoutingDecision(TypedDict, total=False):
     """Per-request provenance for a pre-routing strategy (auto-router) decision."""
 
@@ -2734,6 +2745,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
     classifier_model: str
     escalated: bool
     tier_fallback_from: str
+    resolved_by: RoutingDecisionTierResolution
     tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries
 
 
@@ -2754,6 +2766,7 @@ DERIVED_ROUTING_DECISION_FIELDS: FrozenSet[str] = frozenset(
         "classifier_model",
         "escalated",
         "tier_fallback_from",
+        "resolved_by",
         "tier_boundaries",
     }
 )

tests/test_litellm/router_strategy/test_complexity_router.py

@@ -4834,10 +4834,52 @@ class TestTierFallbackLadder:
         assert response.routing_decision["tier_fallback_from"] == "SIMPLE"
 
     @pytest.mark.asyncio
-    async def test_default_model_serves_when_the_ladder_is_exhausted(self):
+    async def test_default_model_serves_when_the_ladder_is_exhausted_and_says_so(self):
+        """The routed model alone cannot be told apart from an ordinary pick out of the
+        classified tier, so the record has to name default_model as what answered."""
         router = _ladder_router(live=set(), default_model="fallback-model")
         response = await _route(router, "What is 2+2?")
         assert response.model == "fallback-model"
+        decision = response.routing_decision
+        assert decision["resolved_by"] == "default_model"
+        assert decision["tier"] == "SIMPLE"
+        assert "tier_fallback_from" not in decision
+
+    @pytest.mark.asyncio
+    async def test_serving_an_unconfirmed_tier_is_recorded_as_best_effort(self):
+        """Nothing reported a live deployment and there is no default_model, so the tier is
+        served anyway rather than failing a request a stale cooldown may not block. That is
+        a different fact from an ordinary pick and the record keeps them apart."""
+        router = _ladder_router(live=set())
+        response = await _route(router, "What is 2+2?")
+        assert response.model in {"simple-a", "simple-b"}
+        decision = response.routing_decision
+        assert decision["resolved_by"] == "best_effort"
+        assert decision["tier"] == "SIMPLE"
+
+    @pytest.mark.asyncio
+    async def test_an_ordinary_pick_records_no_resolution_fact(self):
+        router = _ladder_router(live={"simple-a", "simple-b"})
+        decision = (await _route(router, "What is 2+2?")).routing_decision
+        assert "resolved_by" not in decision
+        assert "tier_fallback_from" not in decision
+
+    @pytest.mark.asyncio
+    async def test_a_best_effort_climb_records_both_facts(self):
+        """The two facts are independent: an exhausted ladder can both climb to a
+        configured tier and be serving something nothing confirmed was up."""
+        router = ComplexityRouter(
+            model_name="test-ladder-router",
+            litellm_router_instance=StubRouter(live=set()),
+            complexity_router_config={
+                "tiers": {"MEDIUM": "medium-model", "COMPLEX": "complex-model"},
+                "session_affinity": False,
+            },
+        )
+        decision = (await _route(router, "What is 2+2?")).routing_decision
+        assert decision["tier"] == "MEDIUM"
+        assert decision["tier_fallback_from"] == "SIMPLE"
+        assert decision["resolved_by"] == "best_effort"
 
     @pytest.mark.asyncio
     async def test_a_healthy_tier_is_probed_and_no_higher_tier_is(self):