fix(router): reuse previous response routing for Responses API continuations

litellm/router_strategy/complexity_router/complexity_router.py

@@ -662,6 +662,16 @@ def _conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None)
     return any(message.get("role") == "assistant" for message in messages)
 
 
+def _previous_response_id_from_request_kwargs(request_kwargs: Mapping[str, object]) -> str | None:
+    """The Responses API `previous_response_id` on this request, or None when absent.
+
+    Read from the top-level kwargs, where both the proxy and direct Router calls
+    surface it (the same place DeploymentAffinityCheck reads it).
+    """
+    value: Final = request_kwargs.get("previous_response_id")
+    return value if isinstance(value, str) and value else None
+
+
 def _newest_turn_ask(
     messages: Sequence[Mapping[str, object]],
     marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS,
@@ -3410,6 +3420,43 @@ class ComplexityRouter(CustomLogger):
             return None
         return limit if isinstance(limit, int) else None
 
+    def _previous_response_model(
+        self, request_kwargs: Mapping[str, object]
+    ) -> tuple[str, ComplexityTier | None] | None:
+        """The model group that produced the response a continuation request references.
+
+        A Responses API tool continuation carries no human ask, because its context
+        lives server-side behind `previous_response_id`. The LiteLLM response id
+        encodes the deployment that served that response, so the decision that
+        produced it can be replayed instead of re-derived from text that is not
+        there. Returns None when the id is not a LiteLLM encoded id, names a
+        deployment outside this router's tier pools and default model, or plugins
+        are configured (a replayed decision would bypass the plugin pipeline, the
+        same reason session pins are skipped then).
+        """
+        if self.config.plugins:
+            return None
+        previous_response_id: Final = _previous_response_id_from_request_kwargs(request_kwargs)
+        if previous_response_id is None:
+            return None
+        from litellm.responses.utils import ResponsesAPIRequestUtils
+
+        model_id: Final = ResponsesAPIRequestUtils.get_model_id_from_response_id(previous_response_id)
+        if model_id is None:
+            return None
+        candidate_groups: list[str] = []
+        for pool in self._tier_pools().values():
+            candidate_groups.extend(pool)
+        if self.config.default_model and self.config.default_model not in candidate_groups:
+            candidate_groups.append(self.config.default_model)
+        for group in candidate_groups:
+            for deployment in self._group_deployments(group):
+                model_info: Final = deployment.get("model_info")
+                deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None
+                if deployment_id is not None and str(deployment_id) == model_id:
+                    return group, self._tier_for_model(group)
+        return None
+
     def _group_deployments(self, group: str) -> Sequence[Mapping[str, object]]:
         list_models: Final = getattr(self.litellm_router_instance, "get_model_list", None)
         deployments: Final = list_models(model_name=group) if callable(list_models) else None
@@ -4366,7 +4413,10 @@ class ComplexityRouter(CustomLogger):
         resolved_messages: Final = self._resolve_messages(messages, request_kwargs)
         context_fit: Final = await self._request_context_fit(resolved_messages, request_kwargs)
         marker_pairs: Final = self._reminder_markers_for_request(request_kwargs)
-        conversation_continuing: Final = _conversation_is_continuing(resolved_messages)
+        conversation_continuing: Final = (
+            _conversation_is_continuing(resolved_messages)
+            or _previous_response_id_from_request_kwargs(request_kwargs) is not None
+        )
 
         use_session_affinity: Final = self._uses_tier_pin
         session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None
@@ -4678,6 +4728,26 @@ class ComplexityRouter(CustomLogger):
         classifier_images: Final = self._classifier_image_parts(resolved_messages)
 
         if user_message is None and not classifier_images:
+            previous_response: Final = self._previous_response_model(request_kwargs)
+            if previous_response is not None:
+                reused_model, reused_tier = previous_response
+                reused_params: Final = self._litellm_params_for_model(reused_tier, reused_model)
+                verbose_router_logger.info(
+                    "ComplexityRouter: routing decision cause=previous_response_reuse, routed_model=%s",
+                    reused_model,
+                )
+                return PreRoutingHookResponse(
+                    model=reused_model,
+                    messages=messages if has_original_messages else None,
+                    routing_decision=self._build_routing_decision(
+                        routed_model=reused_model,
+                        cause="previous_response_reuse",
+                        tier=reused_tier,
+                        conversation_continuing=conversation_continuing,
+                        tier_litellm_params=reused_params,
+                    ),
+                    litellm_params=reused_params,
+                )
             verbose_router_logger.debug("ComplexityRouter: No user message found, routing to default model")
             default_model_first: Final = not self.config.plugins and self.config.default_model
             if default_model_first:

litellm/types/utils.py

@@ -3166,6 +3166,10 @@ RoutingDecisionCause = Literal[
     # called. Distinct from "session_affinity_pin", which reports the session_affinity flag pinning
     # every turn including new asks; this cause only appears when session_affinity is off.
     "user_turn_continuation",
+    # A Responses API continuation (previous_response_id set, no human ask) reuses the routing
+    # decision that produced the referenced response, decoded from the LiteLLM response id.
+    # Distinct from "user_turn_continuation", which replays a decision held in the session cache.
+    "previous_response_reuse",
     "default_fallback",
     "keyword",
     "quality_tier",

tests/unit/router_strategy/test_complexity_router.py

@@ -5,6 +5,7 @@ Tests the rule-based complexity scoring and tier assignment logic.
 """
 
 import asyncio
+import base64
 import json
 import logging
 import math
@@ -17345,3 +17346,129 @@ class TestNonReasoningTier:
             "complex",
             "reasoning",
         )
+
+
+class TestPreviousResponseReuse:
+    """Regression tests for Responses API tool continuations (issue #46048).
+
+    A continuation request carries only tool outputs plus `previous_response_id`,
+    so it has no human ask to classify. Before the fix it always fell to the
+    default model, even though the LiteLLM response id names the deployment that
+    produced the referenced response.
+    """
+
+    @staticmethod
+    def _encoded_response_id(model_id: str) -> str:
+        payload = f"litellm:custom_llm_provider:azure_ai;model_id:{model_id};response_id:resp_original"
+        return "resp_" + base64.b64encode(payload.encode("utf-8")).decode("utf-8")
+
+    @staticmethod
+    def _router(mock_router_instance, basic_config, **overrides) -> ComplexityRouter:
+        deployments = {
+            group: [
+                {
+                    "model_name": group,
+                    "litellm_params": {"model": group},
+                    "model_info": {"id": deployment_id},
+                }
+            ]
+            for group, deployment_id in {
+                "gpt-4o-mini": "deployment-mini",
+                "gpt-4o": "deployment-4o",
+                "claude-sonnet-4-20250514": "deployment-sonnet",
+                "o1-preview": "deployment-o1",
+            }.items()
+        }
+        mock_router_instance.get_model_list.side_effect = lambda model_name=None: deployments.get(model_name, [])
+        return ComplexityRouter(
+            model_name="test-complexity-router",
+            litellm_router_instance=mock_router_instance,
+            complexity_router_config={**basic_config, **overrides},
+        )
+
+    @pytest.mark.asyncio
+    async def test_tool_continuation_reuses_the_model_that_produced_the_previous_response(
+        self, mock_router_instance, basic_config
+    ):
+        router = self._router(mock_router_instance, basic_config, default_model="gpt-4o-mini")
+        result = await router.async_pre_routing_hook(
+            model="test-complexity-router",
+            request_kwargs={"previous_response_id": self._encoded_response_id("deployment-sonnet")},
+            messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}],
+        )
+        assert result is not None
+        assert result.model == "claude-sonnet-4-20250514", result.routing_decision
+        decision = result.routing_decision
+        assert decision is not None
+        assert decision["cause"] == "previous_response_reuse", decision
+        assert decision["tier"] == "COMPLEX", decision
+        assert decision["conversation_continuing"] is True, decision
+
+    @pytest.mark.asyncio
+    async def test_unencoded_previous_response_id_still_falls_back_to_default(
+        self, mock_router_instance, basic_config
+    ):
+        router = self._router(mock_router_instance, basic_config, default_model="gpt-4o-mini")
+        result = await router.async_pre_routing_hook(
+            model="test-complexity-router",
+            request_kwargs={"previous_response_id": "resp_plain_provider_id"},
+            messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}],
+        )
+        assert result is not None
+        assert result.model == "gpt-4o-mini", result.routing_decision
+        decision = result.routing_decision
+        assert decision is not None
+        assert decision["cause"] == "default_fallback", decision
+
+    @pytest.mark.asyncio
+    async def test_previous_response_from_an_unserved_deployment_still_falls_back_to_default(
+        self, mock_router_instance, basic_config
+    ):
+        router = self._router(mock_router_instance, basic_config, default_model="gpt-4o-mini")
+        result = await router.async_pre_routing_hook(
+            model="test-complexity-router",
+            request_kwargs={"previous_response_id": self._encoded_response_id("deployment-elsewhere")},
+            messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}],
+        )
+        assert result is not None
+        assert result.model == "gpt-4o-mini", result.routing_decision
+        decision = result.routing_decision
+        assert decision is not None
+        assert decision["cause"] == "default_fallback", decision
+
+    @pytest.mark.asyncio
+    async def test_plugins_are_not_bypassed_by_previous_response_reuse(self, mock_router_instance, basic_config):
+        class AllowAll:
+            async def run(self, context):
+                return context
+
+        router = self._router(
+            mock_router_instance,
+            basic_config,
+            default_model="gpt-4o-mini",
+            plugins=[AllowAll()],
+        )
+        result = await router.async_pre_routing_hook(
+            model="test-complexity-router",
+            request_kwargs={"previous_response_id": self._encoded_response_id("deployment-sonnet")},
+            messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}],
+        )
+        assert result is not None
+        assert result.model == "gpt-4o", result.routing_decision
+        decision = result.routing_decision
+        assert decision is not None
+        assert decision["cause"] == "default_fallback", decision
+
+    @pytest.mark.asyncio
+    async def test_previous_response_id_marks_conversation_as_continuing(self, mock_router_instance, basic_config):
+        router = self._router(mock_router_instance, basic_config)
+        result = await router.async_pre_routing_hook(
+            model="test-complexity-router",
+            request_kwargs={"previous_response_id": self._encoded_response_id("deployment-4o")},
+            messages=[{"role": "user", "content": "Hello!"}],
+        )
+        assert result is not None
+        assert result.model == "gpt-4o-mini", result.routing_decision
+        decision = result.routing_decision
+        assert decision is not None
+        assert decision["conversation_continuing"] is True, decision