diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 2c1097b7af3..7dbb2ddc544 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -313,6 +313,8 @@ _TRUNCATION_MARKER: Final = "..." _TRUNCATION_HEAD_FRACTION: Final = 0.3 _MIN_QUOTED_TURN_CHARS: Final = 120 +_CLASSIFIER_CIRCUIT_OPEN_SIGNAL: Final = "classifier-circuit-open" + _CJK_CHARACTER: Final = re.compile("[぀-ヿㇰ-ㇿ㐀-䶿一-鿿豈-﫿ｦ-ﾝ\U00020000-\U0003ffff]") @@ -757,6 +759,12 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo image), not what the session's traffic looks like, and pinning it would hold every following text turn on the vision-capable model the image forced. A modality pin override is the same fact on a session that already holds a pin, so it must not overwrite the pin it displaced. + + An open classifier circuit is the shortest-lived state of all: the fallback ran because the + breaker skipped the classifier, not because the request got classified, and the cooldown is + seconds against a TTL of an hour that every later turn refreshes. Its cause is whatever the + fallback path reports, so the circuit signal is what marks the decision, and leaving it + unpinned lets the session classify again as soon as the breaker closes. """ return decision is None or ( decision.get("cause") @@ -768,6 +776,7 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo "modality_pin_override", ) and not decision.get("context_escalated") + and _CLASSIFIER_CIRCUIT_OPEN_SIGNAL not in (decision.get("signals") or ()) ) @@ -818,6 +827,10 @@ class ClassificationOutcome(NamedTuple): classifier_cost: float | None = None +def _with_signal(outcome: ClassificationOutcome, signal: str | None) -> ClassificationOutcome: + return outcome if signal is None else outcome._replace(signals=(*outcome.signals, signal)) + + class _ClassifierCircuitBreaker: """Process-local timeout breaker for one complexity-router classifier. @@ -1564,7 +1577,7 @@ class ComplexityRouter(CustomLogger): prompt, system_prompt, scored, - signal="classifier-circuit-open", + signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL, ) try: tier, classifier_cost = await self._classify_with_llm(prompt, system_prompt, request_kwargs, messages) @@ -1602,28 +1615,24 @@ class ComplexityRouter(CustomLogger): fallback_tier: Final = self.config.fallback_tier if fallback_tier is not None: verbose_router_logger.warning("ComplexityRouter: %s, routing to fallback_tier %s", reason, fallback_tier) - outcome: Final = ClassificationOutcome( - tier=fallback_tier, - score=None, - signals=(f"classifier-fallback:{fallback_tier}",), - cause="classifier_fallback", + return _with_signal( + ClassificationOutcome( + tier=fallback_tier, + score=None, + signals=(f"classifier-fallback:{fallback_tier}",), + cause="classifier_fallback", + ), + signal, ) - return outcome if signal is None else outcome._replace(signals=(*outcome.signals, signal)) verbose_router_logger.warning( "ComplexityRouter: %s, falling back to %s", reason, self.config.classifier_fallback ) if self.config.classifier_fallback == "default_model": - outcome = self._default_model_fallback_outcome() - return outcome if signal is None else outcome._replace(signals=(*outcome.signals, signal)) + return _with_signal(self._default_model_fallback_outcome(), signal) if scored is not None: - return scored if signal is None else scored._replace(signals=(*scored.signals, signal)) + return _with_signal(scored, signal) tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) - return ClassificationOutcome( - tier=tier, - score=score, - signals=signals if signal is None else (*signals, signal), - cause=cause, - ) + return _with_signal(ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause), signal) async def _classify_with_plugin( self, diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 130a6f5a488..b4c48b53376 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -4750,6 +4750,45 @@ class TestSessionAffinity: # Pinned to the first turn's model, not re-classified down to SIMPLE. assert second.model == "o1-preview" + @pytest.mark.asyncio + async def test_circuit_open_fallback_does_not_pin_the_session(self, mock_router_instance, session_affinity_config): + """Regression: the classifier circuit cools down in seconds while a pin lasts for the whole + TTL, so a session whose only turn landed on the cooldown fallback must classify again once + the breaker closes instead of holding that fallback's model.""" + now = 100.0 + mock_router_instance.cache = DualCache() + mock_router_instance.acompletion = AsyncMock( + side_effect=[TimeoutError("classifier timed out"), _llm_response('{"tier": "REASONING"}')] + ) + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + **session_affinity_config, + "classifier_type": "llm", + "classifier_llm_config": {"model": "haiku-classifier", "timeout_ms": 400}, + }, + ) + router._classifier_circuit_breaker = _ClassifierCircuitBreaker(30.0, clock=lambda: now) + + await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._request_kwargs("outage-session"), + messages=self.SIMPLE_MESSAGE, + ) + cooled_down_kwargs = self._request_kwargs("cooldown-session") + during_cooldown = await router.async_pre_routing_hook( + model="test-model", request_kwargs=cooled_down_kwargs, messages=self.SIMPLE_MESSAGE + ) + now = 130.0 + after_cooldown = await router.async_pre_routing_hook( + model="test-model", request_kwargs=cooled_down_kwargs, messages=self.SIMPLE_MESSAGE + ) + + assert during_cooldown.model == "gpt-4o-mini" + assert after_cooldown.model == "o1-preview" + assert mock_router_instance.acompletion.await_count == 2 + @pytest.mark.asyncio async def test_a_pinned_turn_reports_the_tier_that_serves_it(self, mock_router_instance, session_affinity_config): mock_router_instance.cache = DualCache()