diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 3f73d225730..f7b9f84bf9d 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -32,6 +32,7 @@ from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ModelResponse, RoutingDecisionCause, + RoutingDecisionTierResolution, StandardLoggingRoutingDecision, StandardLoggingRoutingDecisionTierBoundaries, ) @@ -279,6 +280,22 @@ class KeywordOverride(NamedTuple): matched_keyword: str | None +class TierResolution(NamedTuple): + """How a classified tier became a model, not just which model came out. + + Returning the served tier alone cannot express that `default_model` answered, or that + nothing reported a live deployment and the tier was served anyway: both look identical + to an ordinary pick from the classified tier, so a decision record could not say which + happened. `resolved_by` carries that, and `served_tier` differing from `classified_tier` + carries the climb. The two are independent; an exhausted ladder can do both. + """ + + model: str + classified_tier: ComplexityTier + served_tier: ComplexityTier + resolved_by: RoutingDecisionTierResolution + + class ClassificationOutcome(NamedTuple): """What the classifier decided and which mechanism actually produced it. @@ -596,7 +613,7 @@ class ComplexityRouter(CustomLogger): escalation_keyword: str | None = None, escalated: bool = False, classifier_model: str | None = None, - tier_fallback_from: ComplexityTier | None = None, + resolution: TierResolution | None = None, ) -> StandardLoggingRoutingDecision: """Assemble the per-request provenance record for this router's decision. @@ -633,11 +650,16 @@ class ComplexityRouter(CustomLogger): decision["escalated"] = escalated if classifier_model is not None: decision["classifier_model"] = classifier_model - if tier_fallback_from is not None: - # `tier` is what served the request. When the ladder had to climb, the tier the - # request was actually classified into is the fact that explains why a cheap - # request is on an expensive model, so it rides along rather than being lost. - decision["tier_fallback_from"] = tier_fallback_from.value + if resolution is not None: + # Two independent facts, both invisible from the routed model alone. A served + # tier above the classified one explains why a cheap request is on an expensive + # model; resolved_by says the model came from somewhere other than a tier pool + # at all. An exhausted ladder can produce both, so neither is derived from the + # other, and the ordinary case adds nothing. + if resolution.served_tier != resolution.classified_tier: + decision["tier_fallback_from"] = resolution.classified_tier.value + if resolution.resolved_by != "tier_pool": + decision["resolved_by"] = resolution.resolved_by return decision async def aclassify( @@ -818,28 +840,38 @@ class ComplexityRouter(CustomLogger): Returns: The model name configured for that tier. """ - model, _served_tier = self._resolve_configured_tier(tier) - return model + return self._resolve_configured_tier(tier).model - def _resolve_configured_tier(self, tier: ComplexityTier) -> tuple[str, ComplexityTier]: + def _resolve_configured_tier( + self, tier: ComplexityTier, resolved_by: RoutingDecisionTierResolution = "tier_pool" + ) -> TierResolution: """Structural half of tier resolution: the first tier at or above `tier` that has models configured, then `default_model` as the last resort. Falling upward rather than to a fixed tier keeps an under-configured deployment from - quietly serving a request from a cheaper model than it classified. + quietly serving a request from a cheaper model than it classified. `resolved_by` is + what the caller already knows about how it got here, since reaching this without a + live deployment anywhere is a best effort rather than an ordinary pick. """ for candidate_tier in self._tier_ladder_from(tier): if candidate_tier.value not in self.config.tiers: continue # A tier configured as an empty pool is a config error, not a gap to route # around, so _pick_from_tier_value raises rather than climbing past it. - return ( - self._pick_from_tier_value(self.config.tiers[candidate_tier.value], candidate_tier.value), - candidate_tier, + return TierResolution( + model=self._pick_from_tier_value(self.config.tiers[candidate_tier.value], candidate_tier.value), + classified_tier=tier, + served_tier=candidate_tier, + resolved_by=resolved_by, ) if self.config.default_model: - return self.config.default_model, tier + return TierResolution( + model=self.config.default_model, + classified_tier=tier, + served_tier=tier, + resolved_by="default_model", + ) raise ValueError( f"No model configured for tier {tier.value} or any tier above it " @@ -895,8 +927,8 @@ class ComplexityRouter(CustomLogger): raw_messages: list[dict[str, Any]] | None, resolved_messages: list[dict[str, Any]] | None, request_kwargs: dict, - ) -> tuple[str, ComplexityTier]: - """Resolve a classified tier to a servable model, returning the tier it came from. + ) -> TierResolution: + """Resolve a classified tier to a servable model, reporting how it got there. Walks the tier ladder: the classified tier's live models first, then the next tier up, and so on. `default_model` stays the last resort, and is skipped entirely when @@ -912,12 +944,19 @@ class ComplexityRouter(CustomLogger): live = await self._live_models(pool) if not live: continue - if not self.config.plugins: - return random.choice(live), candidate_tier - narrowed = await self._run_routing_plugins( - live, candidate_tier, raw_messages, resolved_messages, request_kwargs + candidates = ( + live + if not self.config.plugins + else await self._run_routing_plugins( + live, candidate_tier, raw_messages, resolved_messages, request_kwargs + ) + ) + return TierResolution( + model=random.choice(candidates), + classified_tier=tier, + served_tier=candidate_tier, + resolved_by="tier_pool", ) - return random.choice(narrowed), candidate_tier if self.config.plugins: # default_model was never checked against the plugins, so serving it here would @@ -927,12 +966,18 @@ class ComplexityRouter(CustomLogger): f"No servable models for tier {tier.value} or any tier above it, with routing plugins configured" ) if self.config.default_model: - return self.config.default_model, tier + return TierResolution( + model=self.config.default_model, + classified_tier=tier, + served_tier=tier, + resolved_by="default_model", + ) # Nothing on the ladder reported a live deployment and there is no declared # fallback. Cooldowns expire and the health view is a snapshot, so serve the # classified tier anyway and let the router surface the real error, rather than - # failing a request that may well succeed. - return self._resolve_configured_tier(tier) + # failing a request that may well succeed. It is recorded as a best effort, since + # the model that answers is one nothing confirmed was up. + return self._resolve_configured_tier(tier, resolved_by="best_effort") async def _run_routing_plugins( self, @@ -1527,12 +1572,12 @@ class ComplexityRouter(CustomLogger): # every non-plugin user, not just a security fix). routed_model = self.config.default_model else: - # Plugins configured: default_model must never bypass them, so it's not - # checked here at all -- _pick_model_for_tier -> get_model_for_tier still - # falls back to it (after the MEDIUM tier) once the plugin pipeline runs. - routed_model, _ = await self._pick_model_for_tier( - ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs - ) + # Plugins configured: default_model must never bypass them, so it is not + # checked here at all. The tier ladder still reaches it as its last resort, + # but only once the plugin pipeline has run. + routed_model = ( + await self._pick_model_for_tier(ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs) + ).model return PreRoutingHookResponse( model=routed_model, messages=messages if has_original_messages else None, @@ -1546,9 +1591,10 @@ class ComplexityRouter(CustomLogger): if override is not None: routed_tier = self._escalate_tier(override.tier) if escalation_keyword is not None else override.tier keyword_escalated = routed_tier != override.tier - routed_model, keyword_served_tier = await self._pick_model_for_tier( + keyword_resolution = await self._pick_model_for_tier( routed_tier, messages, resolved_messages, request_kwargs ) + routed_model = keyword_resolution.model keyword_cause: RoutingDecisionCause = ( "semantic_keyword_match" if self.config.semantic_keyword_matching else "literal_keyword_match" ) @@ -1562,11 +1608,11 @@ class ComplexityRouter(CustomLogger): routing_decision=self._build_routing_decision( routed_model=routed_model, cause=keyword_cause, - tier=keyword_served_tier, + tier=keyword_resolution.served_tier, matched_keyword=override.matched_keyword, escalation_keyword=escalation_keyword, escalated=keyword_escalated, - tier_fallback_from=routed_tier if keyword_served_tier != routed_tier else None, + resolution=keyword_resolution, ), ) @@ -1579,6 +1625,7 @@ class ComplexityRouter(CustomLogger): if escalated: signals = (*signals, "escalation") score_repr = f"{score:.3f}" if score is not None else "n/a" + resolution: TierResolution | None = None served_tier = tier if self.config.adaptive: routed_model = self._soft_floor_pick(tier, user_message, request_kwargs) @@ -1594,13 +1641,13 @@ class ComplexityRouter(CustomLogger): f"signals={signals}, routed_model={routed_model}" ) else: - routed_model, served_tier = await self._pick_model_for_tier( - tier, messages, resolved_messages, request_kwargs - ) + resolution = await self._pick_model_for_tier(tier, messages, resolved_messages, request_kwargs) + routed_model, served_tier = resolution.model, resolution.served_tier verbose_router_logger.info( f"ComplexityRouter: routing decision cause={outcome.cause}, tier={served_tier.value}, " f"score={score_repr}, signals={signals}, routed_model={routed_model}" + (f", tier_fallback_from={tier.value}" if served_tier != tier else "") + + (f", resolved_by={resolution.resolved_by}" if resolution.resolved_by != "tier_pool" else "") ) classifier_model = ( @@ -1620,6 +1667,6 @@ class ComplexityRouter(CustomLogger): escalation_keyword=escalation_keyword, escalated=escalated, classifier_model=classifier_model, - tier_fallback_from=tier if served_tier != tier else None, + resolution=resolution, ), ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 221aa247875..ce88e331724 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2718,6 +2718,17 @@ records that it is not traffic the caller sent.""" AUTOROUTER_CLASSIFIER_CALL_ORIGIN: InternalCallOrigin = "autorouter_classifier" +RoutingDecisionTierResolution = Literal[ + "tier_pool", + "default_model", + "best_effort", +] +"""How a classified tier became a model. `tier_pool` is the ordinary path and is omitted +from the record; `default_model` means no tier on the ladder could serve, so the declared +fallback did; `best_effort` means nothing reported a live deployment and the tier was +served anyway rather than failing a request that may still succeed.""" + + class StandardLoggingRoutingDecision(TypedDict, total=False): """Per-request provenance for a pre-routing strategy (auto-router) decision.""" @@ -2734,6 +2745,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): classifier_model: str escalated: bool tier_fallback_from: str + resolved_by: RoutingDecisionTierResolution tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries @@ -2754,6 +2766,7 @@ DERIVED_ROUTING_DECISION_FIELDS: FrozenSet[str] = frozenset( "classifier_model", "escalated", "tier_fallback_from", + "resolved_by", "tier_boundaries", } ) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 71d029f42d9..0e0183f02fb 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -4834,10 +4834,52 @@ class TestTierFallbackLadder: assert response.routing_decision["tier_fallback_from"] == "SIMPLE" @pytest.mark.asyncio - async def test_default_model_serves_when_the_ladder_is_exhausted(self): + async def test_default_model_serves_when_the_ladder_is_exhausted_and_says_so(self): + """The routed model alone cannot be told apart from an ordinary pick out of the + classified tier, so the record has to name default_model as what answered.""" router = _ladder_router(live=set(), default_model="fallback-model") response = await _route(router, "What is 2+2?") assert response.model == "fallback-model" + decision = response.routing_decision + assert decision["resolved_by"] == "default_model" + assert decision["tier"] == "SIMPLE" + assert "tier_fallback_from" not in decision + + @pytest.mark.asyncio + async def test_serving_an_unconfirmed_tier_is_recorded_as_best_effort(self): + """Nothing reported a live deployment and there is no default_model, so the tier is + served anyway rather than failing a request a stale cooldown may not block. That is + a different fact from an ordinary pick and the record keeps them apart.""" + router = _ladder_router(live=set()) + response = await _route(router, "What is 2+2?") + assert response.model in {"simple-a", "simple-b"} + decision = response.routing_decision + assert decision["resolved_by"] == "best_effort" + assert decision["tier"] == "SIMPLE" + + @pytest.mark.asyncio + async def test_an_ordinary_pick_records_no_resolution_fact(self): + router = _ladder_router(live={"simple-a", "simple-b"}) + decision = (await _route(router, "What is 2+2?")).routing_decision + assert "resolved_by" not in decision + assert "tier_fallback_from" not in decision + + @pytest.mark.asyncio + async def test_a_best_effort_climb_records_both_facts(self): + """The two facts are independent: an exhausted ladder can both climb to a + configured tier and be serving something nothing confirmed was up.""" + router = ComplexityRouter( + model_name="test-ladder-router", + litellm_router_instance=StubRouter(live=set()), + complexity_router_config={ + "tiers": {"MEDIUM": "medium-model", "COMPLEX": "complex-model"}, + "session_affinity": False, + }, + ) + decision = (await _route(router, "What is 2+2?")).routing_decision + assert decision["tier"] == "MEDIUM" + assert decision["tier_fallback_from"] == "SIMPLE" + assert decision["resolved_by"] == "best_effort" @pytest.mark.asyncio async def test_a_healthy_tier_is_probed_and_no_higher_tier_is(self):