diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py
index 2f4305756e9..1aeedaa97b2 100644
--- a/litellm/router_strategy/complexity_router/complexity_router.py
+++ b/litellm/router_strategy/complexity_router/complexity_router.py
@@ -479,6 +479,33 @@ def _last_human_ask_index(
)
+def _newest_turn_is_human_ask(
+ messages: Sequence[Mapping[str, object]] | None,
+ marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS,
+) -> bool:
+ """Whether the request's newest turn carries a real human ask, i.e. this is a new ask rather
+ than an agent loop's continuation traffic.
+
+ Anchored on `_last_human_ask_index` so every surface's plumbing reads as a continuation:
+ chat-completions tool turns are role=tool, Messages-surface tool_result turns flatten to empty
+ human text, and a hybrid turn carrying an ask alongside a tool_result still counts as an ask.
+ Compared against the newest non-system message rather than the raw tail, because Claude Code
+ appends a system-role reminder after the human turn; that trailing plumbing is neither an ask
+ nor loop traffic and must not turn a fresh ask into a continuation. An unreadable request (no
+ messages) is treated as a continuation: there is no ask to classify, which is the same reading
+ `_extract_current_ask_and_system_prompt` gives it downstream.
+ """
+ if not messages:
+ return False
+ newest_non_system: Final = next(
+ (index for index in range(len(messages) - 1, -1, -1) if messages[index].get("role") != "system"),
+ None,
+ )
+ if newest_non_system is None:
+ return False
+ return _last_human_ask_index(messages, marker_pairs) == newest_non_system
+
+
def _iter_system_scope_texts(
body_system: object,
messages: Sequence[Mapping[str, object]],
@@ -2247,14 +2274,18 @@ class ComplexityRouter(CustomLogger):
@property
def _uses_tier_pin(self) -> bool:
- return bool(self.config.session_affinity and not self.config.plugins)
+ """classification_mode 'user_turn' implies the tier pin machinery: the pin write after each
+ pinnable classification is what gives a continuation a held decision to replay."""
+ return bool(
+ (self.config.session_affinity or self.config.classification_mode == "user_turn") and not self.config.plugins
+ )
@property
def _uses_deployment_pin(self) -> bool:
- """session_affinity implies the deployment pin: a session frozen onto one model
+ """The tier pin implies the deployment pin: a session frozen onto one model
group but load-balanced across its deployments would still go cache-cold, which
is the exact failure both flags exist to prevent."""
- return bool((self.config.deployment_affinity or self.config.session_affinity) and not self.config.plugins)
+ return bool(self.config.deployment_affinity and not self.config.plugins) or self._uses_tier_pin
def _with_session_deployment_affinity(
self, response: PreRoutingHookResponse | None
@@ -2282,6 +2313,11 @@ class ComplexityRouter(CustomLogger):
pins the model chosen on the session's first turn and reuses it for every later
turn, skipping classification entirely. Otherwise delegates to `_classify_and_route`.
+ When `classification_mode` is 'user_turn', the same pin is replayed only on
+ continuation turns (an agent loop's tool traffic); a new human ask always falls
+ through to classification, so the session can still move tiers between asks.
+ With both knobs on, session_affinity's pin-first behavior wins.
+
Skipped entirely when `plugins` are configured: reusing a stale pin would bypass
the plugin pipeline on every turn after the first, since a pinned model was never
re-checked against a policy plugin whose decision can change between turns (e.g. a
@@ -2305,7 +2341,13 @@ class ComplexityRouter(CustomLogger):
session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None
cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None
- if cache_key is not None:
+ # In 'user_turn' mode a held pin is replayed only on continuation turns; a new human
+ # ask falls through and re-classifies. session_affinity restores pin-first for asks too.
+ pin_replay_allowed: Final = bool(self.config.session_affinity) or not _newest_turn_is_human_ask(
+ resolved_messages, self._reminder_markers
+ )
+
+ if cache_key is not None and pin_replay_allowed:
pinned_value: Final = await self.litellm_router_instance.cache.async_get_cache(key=cache_key)
pinned_pin: Final = _parse_session_affinity_pin(pinned_value)
if pinned_pin is not None:
@@ -2354,10 +2396,11 @@ class ComplexityRouter(CustomLogger):
kwargs_metadata: Final = request_kwargs.setdefault("metadata", {})
if isinstance(kwargs_metadata, dict):
kwargs_metadata[ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY] = routed_model
+ replay_cause: Final[RoutingDecisionCause] = (
+ "session_affinity_pin" if self.config.session_affinity else "user_turn_continuation"
+ )
cause: RoutingDecisionCause = (
- "plan_mode"
- if plan_floored
- else ("session_affinity_escalation" if escalated else "session_affinity_pin")
+ "plan_mode" if plan_floored else ("session_affinity_escalation" if escalated else replay_cause)
)
verbose_router_logger.info(
"ComplexityRouter: routing decision cause=%s, routed_model=%s", cause, routed_model
diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py
index 335de11e669..0abe962edf1 100644
--- a/litellm/router_strategy/complexity_router/config.py
+++ b/litellm/router_strategy/complexity_router/config.py
@@ -839,6 +839,21 @@ class ComplexityRouterConfig(BaseModel):
description="Minimum cosine similarity for a semantic keyword match",
)
+ classification_mode: Literal["every_request", "user_turn"] = Field(
+ default="every_request",
+ description=(
+ "When to run the complexity classifier. 'every_request' (the default) classifies every "
+ "inference request, including the tool-result continuation turns of an agentic loop. "
+ "'user_turn' classifies only requests whose newest turn is a new human ask and replays "
+ "the session's held routing decision on continuation turns, which cuts classifier "
+ "spend and eliminates mid-loop model switches. Continuations with no held decision to "
+ "replay (no resolvable session_id, expired pin, fresh restart) still classify. Unlike "
+ "session_affinity, a new human ask always re-classifies, so a session can still move "
+ "tiers between asks. Suppressed when plugins are configured, for the same reason "
+ "session_affinity is: a replayed decision would bypass the plugin pipeline."
+ ),
+ )
+
# Session affinity: pin the first turn's routed model for the rest of the session
session_affinity: bool = Field(
default=False,
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index 4bf8289d725..14749ecde6a 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -2842,6 +2842,11 @@ RoutingDecisionCause = Literal[
"housekeeping",
"session_affinity_pin",
"session_affinity_escalation",
+ # classification_mode 'user_turn': the request is an agent loop's continuation turn (no new
+ # human ask), so the session's held routing decision was replayed and the classifier was never
+ # called. Distinct from "session_affinity_pin", which reports the session_affinity flag pinning
+ # every turn including new asks; this cause only appears when session_affinity is off.
+ "user_turn_continuation",
"default_fallback",
"keyword",
"quality_tier",
diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py
index eee4e9aa185..97f60ec8e57 100644
--- a/tests/test_litellm/router_strategy/test_complexity_router.py
+++ b/tests/test_litellm/router_strategy/test_complexity_router.py
@@ -4425,6 +4425,265 @@ class _DummyPlugin:
return context
+class TestClassificationMode:
+ """Test classification_mode='user_turn': classify only requests whose newest turn is a new
+ human ask; tool-loop continuation turns replay the session's held routing decision."""
+
+ REASONING_ASK = {
+ "role": "user",
+ "content": "Let's think step by step and reason through this problem carefully.",
+ }
+ SIMPLE_ASK = {"role": "user", "content": "Hello!"}
+ ASSISTANT_ANSWER = {"role": "assistant", "content": "the answer"}
+ TOOL_CALL_1 = {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read_file", "arguments": "{}"}}],
+ }
+ TOOL_RESULT_1 = {"role": "tool", "tool_call_id": "call_1", "content": "file contents"}
+ TOOL_CALL_2 = {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [{"id": "call_2", "type": "function", "function": {"name": "run_tests", "arguments": "{}"}}],
+ }
+ TOOL_RESULT_2 = {"role": "tool", "tool_call_id": "call_2", "content": "3 passed"}
+
+ @pytest.fixture
+ def user_turn_config(self, basic_config) -> dict:
+ return {**basic_config, "classification_mode": "user_turn"}
+
+ @staticmethod
+ def _request_kwargs(session_id: str) -> dict:
+ return {"metadata": {"session_id": session_id}}
+
+ def _router(self, mock_router_instance, config: dict) -> ComplexityRouter:
+ mock_router_instance.cache = DualCache()
+ return ComplexityRouter(
+ model_name="test-router",
+ litellm_router_instance=mock_router_instance,
+ complexity_router_config=config,
+ )
+
+ def _tool_loop_turns(self) -> list[list[dict]]:
+ return [
+ [self.REASONING_ASK],
+ [self.REASONING_ASK, self.TOOL_CALL_1, self.TOOL_RESULT_1],
+ [self.REASONING_ASK, self.TOOL_CALL_1, self.TOOL_RESULT_1, self.TOOL_CALL_2, self.TOOL_RESULT_2],
+ ]
+
+ def test_default_mode_is_every_request(self, complexity_router):
+ assert complexity_router.config.classification_mode == "every_request"
+
+ def test_invalid_classification_mode_rejected(self, mock_router_instance, basic_config):
+ with pytest.raises(ValidationError):
+ ComplexityRouter(
+ model_name="test-router",
+ litellm_router_instance=mock_router_instance,
+ complexity_router_config={**basic_config, "classification_mode": "sometimes"},
+ )
+
+ @pytest.mark.asyncio
+ async def test_user_turn_mode_classifies_tool_loop_once(self, mock_router_instance, user_turn_config):
+ """The mutation check: a 3-request tool loop drives exactly one classification, and both
+ continuation turns hold the classified model under the user_turn_continuation cause."""
+ router = self._router(mock_router_instance, user_turn_config)
+ with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy:
+ responses = [
+ await router.async_pre_routing_hook(
+ model="test-model", request_kwargs=self._request_kwargs("loop-1"), messages=turn
+ )
+ for turn in self._tool_loop_turns()
+ ]
+ assert spy.call_count == 1
+ assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"]
+ assert [r.routing_decision["cause"] for r in responses[1:]] == [
+ "user_turn_continuation",
+ "user_turn_continuation",
+ ]
+
+ @pytest.mark.asyncio
+ async def test_every_request_default_classifies_every_tool_loop_turn(self, mock_router_instance, basic_config):
+ """Pins today's default: every request classifies, including tool-loop continuations."""
+ router = self._router(mock_router_instance, basic_config)
+ with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy:
+ responses = [
+ await router.async_pre_routing_hook(
+ model="test-model", request_kwargs=self._request_kwargs("loop-2"), messages=turn
+ )
+ for turn in self._tool_loop_turns()
+ ]
+ assert spy.call_count == 3
+ assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"]
+ assert all(r.routing_decision["cause"] != "user_turn_continuation" for r in responses)
+
+ @pytest.mark.asyncio
+ async def test_continuation_without_session_id_still_classifies(self, mock_router_instance, user_turn_config):
+ """No resolvable session id means no held decision to replay, so every request classifies."""
+ router = self._router(mock_router_instance, user_turn_config)
+ with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy:
+ responses = [
+ await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=turn)
+ for turn in self._tool_loop_turns()
+ ]
+ assert spy.call_count == 3
+ assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"]
+ assert all(r.routing_decision["cause"] != "user_turn_continuation" for r in responses)
+
+ @pytest.mark.asyncio
+ async def test_plugins_suppress_user_turn_gate(self, mock_router_instance, basic_config):
+ """A replayed decision would bypass the plugin pipeline, so plugins force every request
+ through _classify_and_route, exactly as they do for session_affinity."""
+ router = self._router(
+ mock_router_instance,
+ {**basic_config, "classification_mode": "user_turn", "plugins": [_DummyPlugin()]},
+ )
+ with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy:
+ responses = [
+ await router.async_pre_routing_hook(
+ model="test-model", request_kwargs=self._request_kwargs("loop-3"), messages=turn
+ )
+ for turn in self._tool_loop_turns()
+ ]
+ assert spy.call_count == 3
+ assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"]
+ assert all(r.routing_decision["cause"] != "user_turn_continuation" for r in responses)
+
+ @pytest.mark.asyncio
+ async def test_new_human_ask_reclassifies_and_repins(self, mock_router_instance, user_turn_config):
+ """Unlike session_affinity, a new human ask never short-circuits on the pin: the session
+ re-classifies, moves tier, and the moved decision becomes the next held decision."""
+ router = self._router(mock_router_instance, user_turn_config)
+ first = await router.async_pre_routing_hook(
+ model="test-model", request_kwargs=self._request_kwargs("s-repin"), messages=[self.REASONING_ASK]
+ )
+ second = await router.async_pre_routing_hook(
+ model="test-model",
+ request_kwargs=self._request_kwargs("s-repin"),
+ messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.SIMPLE_ASK],
+ )
+ third = await router.async_pre_routing_hook(
+ model="test-model",
+ request_kwargs=self._request_kwargs("s-repin"),
+ messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.SIMPLE_ASK, self.TOOL_CALL_1, self.TOOL_RESULT_1],
+ )
+ assert first.model == "o1-preview"
+ assert second.model == "gpt-4o-mini"
+ assert third.model == "gpt-4o-mini"
+ assert third.routing_decision["cause"] == "user_turn_continuation"
+
+ @pytest.mark.asyncio
+ async def test_new_ask_with_trailing_system_reminder_reclassifies(self, mock_router_instance, user_turn_config):
+ """Claude Code appends a system-role reminder after the human turn; that trailing plumbing
+ must not turn a new ask into a continuation, and a continuation turn carrying the same
+ trailing reminder stays a continuation."""
+ router = self._router(mock_router_instance, user_turn_config)
+ reminder = {"role": "system", "content": "100 tokens left"}
+ first = await router.async_pre_routing_hook(
+ model="test-model", request_kwargs=self._request_kwargs("s-reminder"), messages=[self.REASONING_ASK]
+ )
+ second = await router.async_pre_routing_hook(
+ model="test-model",
+ request_kwargs=self._request_kwargs("s-reminder"),
+ messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.SIMPLE_ASK, reminder],
+ )
+ third = await router.async_pre_routing_hook(
+ model="test-model",
+ request_kwargs=self._request_kwargs("s-reminder"),
+ messages=[
+ self.REASONING_ASK,
+ self.ASSISTANT_ANSWER,
+ self.SIMPLE_ASK,
+ reminder,
+ self.TOOL_CALL_1,
+ self.TOOL_RESULT_1,
+ reminder,
+ ],
+ )
+ assert first.model == "o1-preview"
+ assert second.model == "gpt-4o-mini"
+ assert second.routing_decision["cause"] != "user_turn_continuation"
+ assert third.model == "gpt-4o-mini"
+ assert third.routing_decision["cause"] == "user_turn_continuation"
+
+ @pytest.mark.asyncio
+ async def test_escalation_keyword_turn_is_a_new_ask(self, mock_router_instance, user_turn_config):
+ """An escalation keyword arrives as human text, so the turn classifies and escalates
+ instead of replaying the held decision."""
+ router = self._router(mock_router_instance, user_turn_config)
+ first = await router.async_pre_routing_hook(
+ model="test-model", request_kwargs=self._request_kwargs("s-esc"), messages=[self.SIMPLE_ASK]
+ )
+ second = await router.async_pre_routing_hook(
+ model="test-model",
+ request_kwargs=self._request_kwargs("s-esc"),
+ messages=[self.SIMPLE_ASK, self.ASSISTANT_ANSWER, {"role": "user", "content": "LITELLM ESCALATE"}],
+ )
+ assert first.model == "gpt-4o-mini"
+ assert second.model == "gpt-4o"
+ assert second.routing_decision["escalated"] is True
+
+ @pytest.mark.asyncio
+ async def test_messages_surface_tool_result_shapes(self, mock_router_instance, user_turn_config):
+ """Messages-surface shapes: a tool_result-only user turn is a continuation, while an ask
+ riding alongside a tool_result in the same turn is a new ask."""
+ router = self._router(mock_router_instance, user_turn_config)
+ tool_use = {"role": "assistant", "content": [{"type": "tool_use", "id": "x", "name": "t", "input": {}}]}
+ tool_result = {"type": "tool_result", "tool_use_id": "x", "content": "ok"}
+ first = await router.async_pre_routing_hook(
+ model="test-model", request_kwargs=self._request_kwargs("s-msgs"), messages=[self.REASONING_ASK]
+ )
+ pure = await router.async_pre_routing_hook(
+ model="test-model",
+ request_kwargs=self._request_kwargs("s-msgs"),
+ messages=[self.REASONING_ASK, tool_use, {"role": "user", "content": [tool_result]}],
+ )
+ hybrid = await router.async_pre_routing_hook(
+ model="test-model",
+ request_kwargs=self._request_kwargs("s-msgs"),
+ messages=[
+ self.REASONING_ASK,
+ tool_use,
+ {"role": "user", "content": [tool_result, {"type": "text", "text": "Hello!"}]},
+ ],
+ )
+ assert first.model == "o1-preview"
+ assert pure.model == "o1-preview"
+ assert pure.routing_decision["cause"] == "user_turn_continuation"
+ assert hybrid.model == "gpt-4o-mini"
+
+ @pytest.mark.asyncio
+ async def test_session_affinity_wins_when_both_knobs_are_on(self, mock_router_instance, user_turn_config):
+ """With session_affinity also on, the pin short-circuits new asks too and keeps its own
+ cause, so the session stays on turn 1's model."""
+ router = self._router(mock_router_instance, {**user_turn_config, "session_affinity": True})
+ first = await router.async_pre_routing_hook(
+ model="test-model", request_kwargs=self._request_kwargs("s-both"), messages=[self.REASONING_ASK]
+ )
+ second = await router.async_pre_routing_hook(
+ model="test-model",
+ request_kwargs=self._request_kwargs("s-both"),
+ messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.SIMPLE_ASK],
+ )
+ assert first.model == "o1-preview"
+ assert second.model == "o1-preview"
+ assert second.routing_decision["cause"] == "session_affinity_pin"
+
+ def test_user_turn_mode_enables_tier_and_deployment_pins(self, mock_router_instance, basic_config):
+ """user_turn implies the tier pin machinery (the pin write is what gives a continuation
+ a held decision) and the tier pin implies the deployment pin; plugins suppress both."""
+ default = self._router(mock_router_instance, basic_config)
+ enabled = self._router(mock_router_instance, {**basic_config, "classification_mode": "user_turn"})
+ suppressed = self._router(
+ mock_router_instance,
+ {**basic_config, "classification_mode": "user_turn", "plugins": [_DummyPlugin()]},
+ )
+ assert default._uses_tier_pin is False
+ assert enabled._uses_tier_pin is True
+ assert enabled._uses_deployment_pin is True
+ assert suppressed._uses_tier_pin is False
+ assert suppressed._uses_deployment_pin is False
+
+
class TestRoutingPlugins:
"""Test the `complexity_router_config.plugins` field: narrows the classified
tier's candidate pool before a model is picked. Discussion:
diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx
index 8c77b2db630..d2aa20901f5 100644
--- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx
@@ -89,6 +89,7 @@ const CONSTANT_CAUSE_LABELS: Record = {
semantic_keyword_match: "Semantic keyword match",
session_affinity_pin: "Pinned to session",
session_affinity_escalation: "Escalated from session pin",
+ user_turn_continuation: "Continuation turn, classifier skipped",
quality_tier: "Quality tier mapping",
bandit: "Adaptive bandit",
default_fallback: "Default model, no route matched",
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index cc56483ac85..140130adc4b 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -34318,6 +34318,13 @@ export interface components {
adaptive_eligible: "all" | "classified_tier";
/** @description Quality vs cost weights for adaptive selection (used when adaptive=True) */
adaptive_weights?: components["schemas"]["AdaptiveRouterWeights"];
+ /**
+ * Classification Mode
+ * @description When to run the complexity classifier. 'every_request' (the default) classifies every inference request, including the tool-result continuation turns of an agentic loop. 'user_turn' classifies only requests whose newest turn is a new human ask and replays the session's held routing decision on continuation turns, which cuts classifier spend and eliminates mid-loop model switches. Continuations with no held decision to replay (no resolvable session_id, expired pin, fresh restart) still classify. Unlike session_affinity, a new human ask always re-classifies, so a session can still move tiers between asks. Suppressed when plugins are configured, for the same reason session_affinity is: a replayed decision would bypass the plugin pipeline.
+ * @default every_request
+ * @enum {string}
+ */
+ classification_mode: "every_request" | "user_turn";
/**
* Classification Prompt
* @description Replaces the opening instructions of the LLM classifier rubric (the judging-criteria prose) for a custom tier set. The per-tier bullets and the trust-boundary paragraph telling the classifier to ignore tier requests embedded in quoted caller text are always appended after it and cannot be overridden. Requires tier_definitions; a built-in-tier router customizes its prompt via classifier_llm_config.system_prompt or classification_rubric instead.
@@ -35590,7 +35597,7 @@ export interface components {
* Cause
* @enum {string}
*/
- cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "session_affinity_pin" | "session_affinity_escalation" | "default_fallback" | "keyword" | "quality_tier" | "bandit";
+ cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit";
/** Classifier Cost */
classifier_cost?: number;
/** Classifier Model */