feat(complexity-router): add classification_mode to skip classifier on continuation turns (#38861)

litellm/router_strategy/complexity_router/complexity_router.py

@@ -479,6 +479,33 @@ def _last_human_ask_index(
     )
 
 
+def _newest_turn_is_human_ask(
+    messages: Sequence[Mapping[str, object]] | None,
+    marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS,
+) -> bool:
+    """Whether the request's newest turn carries a real human ask, i.e. this is a new ask rather
+    than an agent loop's continuation traffic.
+
+    Anchored on `_last_human_ask_index` so every surface's plumbing reads as a continuation:
+    chat-completions tool turns are role=tool, Messages-surface tool_result turns flatten to empty
+    human text, and a hybrid turn carrying an ask alongside a tool_result still counts as an ask.
+    Compared against the newest non-system message rather than the raw tail, because Claude Code
+    appends a system-role reminder after the human turn; that trailing plumbing is neither an ask
+    nor loop traffic and must not turn a fresh ask into a continuation. An unreadable request (no
+    messages) is treated as a continuation: there is no ask to classify, which is the same reading
+    `_extract_current_ask_and_system_prompt` gives it downstream.
+    """
+    if not messages:
+        return False
+    newest_non_system: Final = next(
+        (index for index in range(len(messages) - 1, -1, -1) if messages[index].get("role") != "system"),
+        None,
+    )
+    if newest_non_system is None:
+        return False
+    return _last_human_ask_index(messages, marker_pairs) == newest_non_system
+
+
 def _iter_system_scope_texts(
     body_system: object,
     messages: Sequence[Mapping[str, object]],
@@ -2247,14 +2274,18 @@ class ComplexityRouter(CustomLogger):
 
     @property
     def _uses_tier_pin(self) -> bool:
-        return bool(self.config.session_affinity and not self.config.plugins)
+        """classification_mode 'user_turn' implies the tier pin machinery: the pin write after each
+        pinnable classification is what gives a continuation a held decision to replay."""
+        return bool(
+            (self.config.session_affinity or self.config.classification_mode == "user_turn") and not self.config.plugins
+        )
 
     @property
     def _uses_deployment_pin(self) -> bool:
-        """session_affinity implies the deployment pin: a session frozen onto one model
+        """The tier pin implies the deployment pin: a session frozen onto one model
         group but load-balanced across its deployments would still go cache-cold, which
         is the exact failure both flags exist to prevent."""
-        return bool((self.config.deployment_affinity or self.config.session_affinity) and not self.config.plugins)
+        return bool(self.config.deployment_affinity and not self.config.plugins) or self._uses_tier_pin
 
     def _with_session_deployment_affinity(
         self, response: PreRoutingHookResponse | None
@@ -2282,6 +2313,11 @@ class ComplexityRouter(CustomLogger):
         pins the model chosen on the session's first turn and reuses it for every later
         turn, skipping classification entirely. Otherwise delegates to `_classify_and_route`.
 
+        When `classification_mode` is 'user_turn', the same pin is replayed only on
+        continuation turns (an agent loop's tool traffic); a new human ask always falls
+        through to classification, so the session can still move tiers between asks.
+        With both knobs on, session_affinity's pin-first behavior wins.
+
         Skipped entirely when `plugins` are configured: reusing a stale pin would bypass
         the plugin pipeline on every turn after the first, since a pinned model was never
         re-checked against a policy plugin whose decision can change between turns (e.g. a
@@ -2305,7 +2341,13 @@ class ComplexityRouter(CustomLogger):
         session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None
         cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None
 
-        if cache_key is not None:
+        # In 'user_turn' mode a held pin is replayed only on continuation turns; a new human
+        # ask falls through and re-classifies. session_affinity restores pin-first for asks too.
+        pin_replay_allowed: Final = bool(self.config.session_affinity) or not _newest_turn_is_human_ask(
+            resolved_messages, self._reminder_markers
+        )
+
+        if cache_key is not None and pin_replay_allowed:
             pinned_value: Final = await self.litellm_router_instance.cache.async_get_cache(key=cache_key)
             pinned_pin: Final = _parse_session_affinity_pin(pinned_value)
             if pinned_pin is not None:
@@ -2354,10 +2396,11 @@ class ComplexityRouter(CustomLogger):
                         kwargs_metadata: Final = request_kwargs.setdefault("metadata", {})
                         if isinstance(kwargs_metadata, dict):
                             kwargs_metadata[ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY] = routed_model
+                    replay_cause: Final[RoutingDecisionCause] = (
+                        "session_affinity_pin" if self.config.session_affinity else "user_turn_continuation"
+                    )
                     cause: RoutingDecisionCause = (
-                        "plan_mode"
-                        if plan_floored
-                        else ("session_affinity_escalation" if escalated else "session_affinity_pin")
+                        "plan_mode" if plan_floored else ("session_affinity_escalation" if escalated else replay_cause)
                     )
                     verbose_router_logger.info(
                         "ComplexityRouter: routing decision cause=%s, routed_model=%s", cause, routed_model

litellm/router_strategy/complexity_router/config.py

@@ -839,6 +839,21 @@ class ComplexityRouterConfig(BaseModel):
         description="Minimum cosine similarity for a semantic keyword match",
     )
 
+    classification_mode: Literal["every_request", "user_turn"] = Field(
+        default="every_request",
+        description=(
+            "When to run the complexity classifier. 'every_request' (the default) classifies every "
+            "inference request, including the tool-result continuation turns of an agentic loop. "
+            "'user_turn' classifies only requests whose newest turn is a new human ask and replays "
+            "the session's held routing decision on continuation turns, which cuts classifier "
+            "spend and eliminates mid-loop model switches. Continuations with no held decision to "
+            "replay (no resolvable session_id, expired pin, fresh restart) still classify. Unlike "
+            "session_affinity, a new human ask always re-classifies, so a session can still move "
+            "tiers between asks. Suppressed when plugins are configured, for the same reason "
+            "session_affinity is: a replayed decision would bypass the plugin pipeline."
+        ),
+    )
+
     # Session affinity: pin the first turn's routed model for the rest of the session
     session_affinity: bool = Field(
         default=False,

litellm/types/utils.py

@@ -2842,6 +2842,11 @@ RoutingDecisionCause = Literal[
     "housekeeping",
     "session_affinity_pin",
     "session_affinity_escalation",
+    # classification_mode 'user_turn': the request is an agent loop's continuation turn (no new
+    # human ask), so the session's held routing decision was replayed and the classifier was never
+    # called. Distinct from "session_affinity_pin", which reports the session_affinity flag pinning
+    # every turn including new asks; this cause only appears when session_affinity is off.
+    "user_turn_continuation",
     "default_fallback",
     "keyword",
     "quality_tier",

tests/test_litellm/router_strategy/test_complexity_router.py

@@ -4425,6 +4425,265 @@ class _DummyPlugin:
         return context
 
 
+class TestClassificationMode:
+    """Test classification_mode='user_turn': classify only requests whose newest turn is a new
+    human ask; tool-loop continuation turns replay the session's held routing decision."""
+
+    REASONING_ASK = {
+        "role": "user",
+        "content": "Let's think step by step and reason through this problem carefully.",
+    }
+    SIMPLE_ASK = {"role": "user", "content": "Hello!"}
+    ASSISTANT_ANSWER = {"role": "assistant", "content": "the answer"}
+    TOOL_CALL_1 = {
+        "role": "assistant",
+        "content": None,
+        "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read_file", "arguments": "{}"}}],
+    }
+    TOOL_RESULT_1 = {"role": "tool", "tool_call_id": "call_1", "content": "file contents"}
+    TOOL_CALL_2 = {
+        "role": "assistant",
+        "content": None,
+        "tool_calls": [{"id": "call_2", "type": "function", "function": {"name": "run_tests", "arguments": "{}"}}],
+    }
+    TOOL_RESULT_2 = {"role": "tool", "tool_call_id": "call_2", "content": "3 passed"}
+
+    @pytest.fixture
+    def user_turn_config(self, basic_config) -> dict:
+        return {**basic_config, "classification_mode": "user_turn"}
+
+    @staticmethod
+    def _request_kwargs(session_id: str) -> dict:
+        return {"metadata": {"session_id": session_id}}
+
+    def _router(self, mock_router_instance, config: dict) -> ComplexityRouter:
+        mock_router_instance.cache = DualCache()
+        return ComplexityRouter(
+            model_name="test-router",
+            litellm_router_instance=mock_router_instance,
+            complexity_router_config=config,
+        )
+
+    def _tool_loop_turns(self) -> list[list[dict]]:
+        return [
+            [self.REASONING_ASK],
+            [self.REASONING_ASK, self.TOOL_CALL_1, self.TOOL_RESULT_1],
+            [self.REASONING_ASK, self.TOOL_CALL_1, self.TOOL_RESULT_1, self.TOOL_CALL_2, self.TOOL_RESULT_2],
+        ]
+
+    def test_default_mode_is_every_request(self, complexity_router):
+        assert complexity_router.config.classification_mode == "every_request"
+
+    def test_invalid_classification_mode_rejected(self, mock_router_instance, basic_config):
+        with pytest.raises(ValidationError):
+            ComplexityRouter(
+                model_name="test-router",
+                litellm_router_instance=mock_router_instance,
+                complexity_router_config={**basic_config, "classification_mode": "sometimes"},
+            )
+
+    @pytest.mark.asyncio
+    async def test_user_turn_mode_classifies_tool_loop_once(self, mock_router_instance, user_turn_config):
+        """The mutation check: a 3-request tool loop drives exactly one classification, and both
+        continuation turns hold the classified model under the user_turn_continuation cause."""
+        router = self._router(mock_router_instance, user_turn_config)
+        with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy:
+            responses = [
+                await router.async_pre_routing_hook(
+                    model="test-model", request_kwargs=self._request_kwargs("loop-1"), messages=turn
+                )
+                for turn in self._tool_loop_turns()
+            ]
+        assert spy.call_count == 1
+        assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"]
+        assert [r.routing_decision["cause"] for r in responses[1:]] == [
+            "user_turn_continuation",
+            "user_turn_continuation",
+        ]
+
+    @pytest.mark.asyncio
+    async def test_every_request_default_classifies_every_tool_loop_turn(self, mock_router_instance, basic_config):
+        """Pins today's default: every request classifies, including tool-loop continuations."""
+        router = self._router(mock_router_instance, basic_config)
+        with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy:
+            responses = [
+                await router.async_pre_routing_hook(
+                    model="test-model", request_kwargs=self._request_kwargs("loop-2"), messages=turn
+                )
+                for turn in self._tool_loop_turns()
+            ]
+        assert spy.call_count == 3
+        assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"]
+        assert all(r.routing_decision["cause"] != "user_turn_continuation" for r in responses)
+
+    @pytest.mark.asyncio
+    async def test_continuation_without_session_id_still_classifies(self, mock_router_instance, user_turn_config):
+        """No resolvable session id means no held decision to replay, so every request classifies."""
+        router = self._router(mock_router_instance, user_turn_config)
+        with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy:
+            responses = [
+                await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=turn)
+                for turn in self._tool_loop_turns()
+            ]
+        assert spy.call_count == 3
+        assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"]
+        assert all(r.routing_decision["cause"] != "user_turn_continuation" for r in responses)
+
+    @pytest.mark.asyncio
+    async def test_plugins_suppress_user_turn_gate(self, mock_router_instance, basic_config):
+        """A replayed decision would bypass the plugin pipeline, so plugins force every request
+        through _classify_and_route, exactly as they do for session_affinity."""
+        router = self._router(
+            mock_router_instance,
+            {**basic_config, "classification_mode": "user_turn", "plugins": [_DummyPlugin()]},
+        )
+        with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy:
+            responses = [
+                await router.async_pre_routing_hook(
+                    model="test-model", request_kwargs=self._request_kwargs("loop-3"), messages=turn
+                )
+                for turn in self._tool_loop_turns()
+            ]
+        assert spy.call_count == 3
+        assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"]
+        assert all(r.routing_decision["cause"] != "user_turn_continuation" for r in responses)
+
+    @pytest.mark.asyncio
+    async def test_new_human_ask_reclassifies_and_repins(self, mock_router_instance, user_turn_config):
+        """Unlike session_affinity, a new human ask never short-circuits on the pin: the session
+        re-classifies, moves tier, and the moved decision becomes the next held decision."""
+        router = self._router(mock_router_instance, user_turn_config)
+        first = await router.async_pre_routing_hook(
+            model="test-model", request_kwargs=self._request_kwargs("s-repin"), messages=[self.REASONING_ASK]
+        )
+        second = await router.async_pre_routing_hook(
+            model="test-model",
+            request_kwargs=self._request_kwargs("s-repin"),
+            messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.SIMPLE_ASK],
+        )
+        third = await router.async_pre_routing_hook(
+            model="test-model",
+            request_kwargs=self._request_kwargs("s-repin"),
+            messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.SIMPLE_ASK, self.TOOL_CALL_1, self.TOOL_RESULT_1],
+        )
+        assert first.model == "o1-preview"
+        assert second.model == "gpt-4o-mini"
+        assert third.model == "gpt-4o-mini"
+        assert third.routing_decision["cause"] == "user_turn_continuation"
+
+    @pytest.mark.asyncio
+    async def test_new_ask_with_trailing_system_reminder_reclassifies(self, mock_router_instance, user_turn_config):
+        """Claude Code appends a system-role reminder after the human turn; that trailing plumbing
+        must not turn a new ask into a continuation, and a continuation turn carrying the same
+        trailing reminder stays a continuation."""
+        router = self._router(mock_router_instance, user_turn_config)
+        reminder = {"role": "system", "content": "<total_tokens>100 tokens left</total_tokens>"}
+        first = await router.async_pre_routing_hook(
+            model="test-model", request_kwargs=self._request_kwargs("s-reminder"), messages=[self.REASONING_ASK]
+        )
+        second = await router.async_pre_routing_hook(
+            model="test-model",
+            request_kwargs=self._request_kwargs("s-reminder"),
+            messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.SIMPLE_ASK, reminder],
+        )
+        third = await router.async_pre_routing_hook(
+            model="test-model",
+            request_kwargs=self._request_kwargs("s-reminder"),
+            messages=[
+                self.REASONING_ASK,
+                self.ASSISTANT_ANSWER,
+                self.SIMPLE_ASK,
+                reminder,
+                self.TOOL_CALL_1,
+                self.TOOL_RESULT_1,
+                reminder,
+            ],
+        )
+        assert first.model == "o1-preview"
+        assert second.model == "gpt-4o-mini"
+        assert second.routing_decision["cause"] != "user_turn_continuation"
+        assert third.model == "gpt-4o-mini"
+        assert third.routing_decision["cause"] == "user_turn_continuation"
+
+    @pytest.mark.asyncio
+    async def test_escalation_keyword_turn_is_a_new_ask(self, mock_router_instance, user_turn_config):
+        """An escalation keyword arrives as human text, so the turn classifies and escalates
+        instead of replaying the held decision."""
+        router = self._router(mock_router_instance, user_turn_config)
+        first = await router.async_pre_routing_hook(
+            model="test-model", request_kwargs=self._request_kwargs("s-esc"), messages=[self.SIMPLE_ASK]
+        )
+        second = await router.async_pre_routing_hook(
+            model="test-model",
+            request_kwargs=self._request_kwargs("s-esc"),
+            messages=[self.SIMPLE_ASK, self.ASSISTANT_ANSWER, {"role": "user", "content": "LITELLM ESCALATE"}],
+        )
+        assert first.model == "gpt-4o-mini"
+        assert second.model == "gpt-4o"
+        assert second.routing_decision["escalated"] is True
+
+    @pytest.mark.asyncio
+    async def test_messages_surface_tool_result_shapes(self, mock_router_instance, user_turn_config):
+        """Messages-surface shapes: a tool_result-only user turn is a continuation, while an ask
+        riding alongside a tool_result in the same turn is a new ask."""
+        router = self._router(mock_router_instance, user_turn_config)
+        tool_use = {"role": "assistant", "content": [{"type": "tool_use", "id": "x", "name": "t", "input": {}}]}
+        tool_result = {"type": "tool_result", "tool_use_id": "x", "content": "ok"}
+        first = await router.async_pre_routing_hook(
+            model="test-model", request_kwargs=self._request_kwargs("s-msgs"), messages=[self.REASONING_ASK]
+        )
+        pure = await router.async_pre_routing_hook(
+            model="test-model",
+            request_kwargs=self._request_kwargs("s-msgs"),
+            messages=[self.REASONING_ASK, tool_use, {"role": "user", "content": [tool_result]}],
+        )
+        hybrid = await router.async_pre_routing_hook(
+            model="test-model",
+            request_kwargs=self._request_kwargs("s-msgs"),
+            messages=[
+                self.REASONING_ASK,
+                tool_use,
+                {"role": "user", "content": [tool_result, {"type": "text", "text": "Hello!"}]},
+            ],
+        )
+        assert first.model == "o1-preview"
+        assert pure.model == "o1-preview"
+        assert pure.routing_decision["cause"] == "user_turn_continuation"
+        assert hybrid.model == "gpt-4o-mini"
+
+    @pytest.mark.asyncio
+    async def test_session_affinity_wins_when_both_knobs_are_on(self, mock_router_instance, user_turn_config):
+        """With session_affinity also on, the pin short-circuits new asks too and keeps its own
+        cause, so the session stays on turn 1's model."""
+        router = self._router(mock_router_instance, {**user_turn_config, "session_affinity": True})
+        first = await router.async_pre_routing_hook(
+            model="test-model", request_kwargs=self._request_kwargs("s-both"), messages=[self.REASONING_ASK]
+        )
+        second = await router.async_pre_routing_hook(
+            model="test-model",
+            request_kwargs=self._request_kwargs("s-both"),
+            messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.SIMPLE_ASK],
+        )
+        assert first.model == "o1-preview"
+        assert second.model == "o1-preview"
+        assert second.routing_decision["cause"] == "session_affinity_pin"
+
+    def test_user_turn_mode_enables_tier_and_deployment_pins(self, mock_router_instance, basic_config):
+        """user_turn implies the tier pin machinery (the pin write is what gives a continuation
+        a held decision) and the tier pin implies the deployment pin; plugins suppress both."""
+        default = self._router(mock_router_instance, basic_config)
+        enabled = self._router(mock_router_instance, {**basic_config, "classification_mode": "user_turn"})
+        suppressed = self._router(
+            mock_router_instance,
+            {**basic_config, "classification_mode": "user_turn", "plugins": [_DummyPlugin()]},
+        )
+        assert default._uses_tier_pin is False
+        assert enabled._uses_tier_pin is True
+        assert enabled._uses_deployment_pin is True
+        assert suppressed._uses_tier_pin is False
+        assert suppressed._uses_deployment_pin is False
+
+
 class TestRoutingPlugins:
     """Test the `complexity_router_config.plugins` field: narrows the classified
     tier's candidate pool before a model is picked. Discussion:

ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx

@@ -89,6 +89,7 @@ const CONSTANT_CAUSE_LABELS: Record<string, string> = {
   semantic_keyword_match: "Semantic keyword match",
   session_affinity_pin: "Pinned to session",
   session_affinity_escalation: "Escalated from session pin",
+  user_turn_continuation: "Continuation turn, classifier skipped",
   quality_tier: "Quality tier mapping",
   bandit: "Adaptive bandit",
   default_fallback: "Default model, no route matched",

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -34318,6 +34318,13 @@ export interface components {
             adaptive_eligible: "all" | "classified_tier";
             /** @description Quality vs cost weights for adaptive selection (used when adaptive=True) */
             adaptive_weights?: components["schemas"]["AdaptiveRouterWeights"];
+            /**
+             * Classification Mode
+             * @description When to run the complexity classifier. 'every_request' (the default) classifies every inference request, including the tool-result continuation turns of an agentic loop. 'user_turn' classifies only requests whose newest turn is a new human ask and replays the session's held routing decision on continuation turns, which cuts classifier spend and eliminates mid-loop model switches. Continuations with no held decision to replay (no resolvable session_id, expired pin, fresh restart) still classify. Unlike session_affinity, a new human ask always re-classifies, so a session can still move tiers between asks. Suppressed when plugins are configured, for the same reason session_affinity is: a replayed decision would bypass the plugin pipeline.
+             * @default every_request
+             * @enum {string}
+             */
+            classification_mode: "every_request" | "user_turn";
             /**
              * Classification Prompt
              * @description Replaces the opening instructions of the LLM classifier rubric (the judging-criteria prose) for a custom tier set. The per-tier bullets and the trust-boundary paragraph telling the classifier to ignore tier requests embedded in quoted caller text are always appended after it and cannot be overridden. Requires tier_definitions; a built-in-tier router customizes its prompt via classifier_llm_config.system_prompt or classification_rubric instead.
@@ -35590,7 +35597,7 @@ export interface components {
              * Cause
              * @enum {string}
              */
-            cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "session_affinity_pin" | "session_affinity_escalation" | "default_fallback" | "keyword" | "quality_tier" | "bandit";
+            cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit";
             /** Classifier Cost */
             classifier_cost?: number;
             /** Classifier Model */