Merge 6f0ea1be5f into e0982d6248

litellm/router_strategy/complexity_router/complexity_router.py

@@ -673,6 +673,16 @@ def _conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None)
     return any(message.get("role") == "assistant" for message in messages)
 
 
+def _previous_response_id_from_request_kwargs(request_kwargs: Mapping[str, object]) -> str | None:
+    """The Responses API `previous_response_id` on this request, or None when absent.
+
+    Read from the top-level kwargs, where both the proxy and direct Router calls
+    surface it (the same place DeploymentAffinityCheck reads it).
+    """
+    value: Final = request_kwargs.get("previous_response_id")
+    return value if isinstance(value, str) and value else None
+
+
 def _newest_turn_ask(
     messages: Sequence[Mapping[str, object]],
     marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS,
@@ -3421,6 +3431,57 @@ class ComplexityRouter(CustomLogger):
             return None
         return limit if isinstance(limit, int) else None
 
+    def _previous_response_model(
+        self, request_kwargs: Mapping[str, object]
+    ) -> tuple[str, ComplexityTier | None] | None:
+        """The model group that produced the response a continuation request references.
+
+        A Responses API tool continuation carries no human ask, because its context
+        lives server-side behind `previous_response_id`. The LiteLLM response id
+        encodes the deployment that served that response, so the decision that
+        produced it can be replayed instead of re-derived from text that is not
+        there. Returns None when the id is not a LiteLLM encoded id, names a
+        deployment outside this router's tier pools and default model, or plugins
+        are configured (a replayed decision would bypass the plugin pipeline, the
+        same reason session pins are skipped then).
+        """
+        if self.config.plugins:
+            return None
+        previous_response_id: Final = _previous_response_id_from_request_kwargs(request_kwargs)
+        if previous_response_id is None:
+            return None
+        from litellm.responses.utils import ResponsesAPIRequestUtils
+
+        model_id: Final = ResponsesAPIRequestUtils.get_model_id_from_response_id(previous_response_id)
+        if model_id is None:
+            return None
+        for pool in self._tier_pools().values():
+            for group in pool:
+                if self._group_has_deployment_id(group, model_id):
+                    return group, self._tier_for_model(group)
+        default_model: Final = self.config.default_model
+        if default_model and self._group_has_deployment_id(default_model, model_id):
+            return default_model, self._tier_for_model(default_model)
+        return None
+
+    @staticmethod
+    def _deployment_model_id(deployment: Mapping[str, object]) -> str | None:
+        model_info: Final = deployment.get("model_info")
+        deployment_id: Final = (
+            cast(  # cast-ok: the isinstance check in the condition narrows model_info to a Mapping with str keys
+                "Mapping[str, object]", model_info
+            ).get("id")
+            if isinstance(model_info, Mapping)
+            else None
+        )
+        return None if deployment_id is None else str(deployment_id)
+
+    def _group_has_deployment_id(self, group: str, model_id: str) -> bool:
+        for deployment in self._group_deployments(group):
+            if self._deployment_model_id(deployment) == model_id:
+                return True
+        return False
+
     def _group_deployments(self, group: str) -> Sequence[Mapping[str, object]]:
         list_models: Final = getattr(self.litellm_router_instance, "get_model_list", None)
         deployments: Final = list_models(model_name=group) if callable(list_models) else None
@@ -4377,7 +4438,15 @@ class ComplexityRouter(CustomLogger):
         resolved_messages: Final = self._resolve_messages(messages, request_kwargs)
         context_fit: Final = await self._request_context_fit(resolved_messages, request_kwargs)
         marker_pairs: Final = self._reminder_markers_for_request(request_kwargs)
-        conversation_continuing: Final = _conversation_is_continuing(resolved_messages)
+        conversation_continuing: Final = (
+            _conversation_is_continuing(resolved_messages)
+            or _previous_response_id_from_request_kwargs(
+                cast(  # cast-ok: the hook's bare kwargs dict is the str-keyed request mapping
+                    Mapping[str, object], request_kwargs
+                )
+            )
+            is not None
+        )
 
         use_session_affinity: Final = self._uses_tier_pin
         session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None
@@ -4689,6 +4758,30 @@ class ComplexityRouter(CustomLogger):
         classifier_images: Final = self._classifier_image_parts(resolved_messages)
 
         if user_message is None and not classifier_images:
+            previous_response: Final = self._previous_response_model(
+                cast(  # cast-ok: the hook's bare kwargs dict is the str-keyed request mapping
+                    Mapping[str, object], request_kwargs
+                )
+            )
+            if previous_response is not None:
+                reused_model, reused_tier = previous_response
+                reused_params: Final = self._litellm_params_for_model(reused_tier, reused_model)
+                verbose_router_logger.info(
+                    "ComplexityRouter: routing decision cause=previous_response_reuse, routed_model=%s",
+                    reused_model,
+                )
+                return PreRoutingHookResponse(
+                    model=reused_model,
+                    messages=messages if has_original_messages else None,
+                    routing_decision=self._build_routing_decision(
+                        routed_model=reused_model,
+                        cause="previous_response_reuse",
+                        tier=reused_tier,
+                        conversation_continuing=conversation_continuing,
+                        tier_litellm_params=reused_params,
+                    ),
+                    litellm_params=reused_params,
+                )
             verbose_router_logger.debug("ComplexityRouter: No user message found, routing to default model")
             default_model_first: Final = not self.config.plugins and self.config.default_model
             if default_model_first:

litellm/types/utils.py

@@ -3202,6 +3202,10 @@ RoutingDecisionCause = Literal[
     # called. Distinct from "session_affinity_pin", which reports the session_affinity flag pinning
     # every turn including new asks; this cause only appears when session_affinity is off.
     "user_turn_continuation",
+    # A Responses API continuation (previous_response_id set, no human ask) reuses the routing
+    # decision that produced the referenced response, decoded from the LiteLLM response id.
+    # Distinct from "user_turn_continuation", which replays a decision held in the session cache.
+    "previous_response_reuse",
     "default_fallback",
     "keyword",
     "quality_tier",

tests/unit/router_strategy/test_complexity_router.py

@@ -5,6 +5,7 @@ Tests the rule-based complexity scoring and tier assignment logic.
 """
 
 import asyncio
+import base64
 import json
 import logging
 import math
@@ -17350,3 +17351,176 @@ class TestNonReasoningTier:
             "complex",
             "reasoning",
         )
+
+
+class TestPreviousResponseReuse:
+    """Regression tests for Responses API tool continuations (issue #46048).
+
+    A continuation request carries only tool outputs plus `previous_response_id`,
+    so it has no human ask to classify. Before the fix it always fell to the
+    default model, even though the LiteLLM response id names the deployment that
+    produced the referenced response.
+    """
+
+    @staticmethod
+    def _encoded_response_id(model_id: str) -> str:
+        payload = f"litellm:custom_llm_provider:azure_ai;model_id:{model_id};response_id:resp_original"
+        return "resp_" + base64.b64encode(payload.encode("utf-8")).decode("utf-8")
+
+    @staticmethod
+    def _router(mock_router_instance, basic_config, **overrides) -> ComplexityRouter:
+        deployments = {
+            group: [
+                {
+                    "model_name": group,
+                    "litellm_params": {"model": group},
+                    "model_info": {"id": deployment_id},
+                }
+            ]
+            for group, deployment_id in {
+                "gpt-4o-mini": "deployment-mini",
+                "gpt-4o": "deployment-4o",
+                "claude-sonnet-4-20250514": "deployment-sonnet",
+                "o1-preview": "deployment-o1",
+            }.items()
+        }
+        mock_router_instance.get_model_list.side_effect = lambda model_name=None: deployments.get(model_name, [])
+        return ComplexityRouter(
+            model_name="test-complexity-router",
+            litellm_router_instance=mock_router_instance,
+            complexity_router_config={**basic_config, **overrides},
+        )
+
+    @pytest.mark.asyncio
+    async def test_tool_continuation_reuses_the_model_that_produced_the_previous_response(
+        self, mock_router_instance, basic_config
+    ):
+        router = self._router(mock_router_instance, basic_config, default_model="gpt-4o-mini")
+        result = await router.async_pre_routing_hook(
+            model="test-complexity-router",
+            request_kwargs={"previous_response_id": self._encoded_response_id("deployment-sonnet")},
+            messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}],
+        )
+        assert result is not None
+        assert result.model == "claude-sonnet-4-20250514", result.routing_decision
+        decision = result.routing_decision
+        assert decision is not None
+        assert decision["cause"] == "previous_response_reuse", decision
+        assert decision["tier"] == "COMPLEX", decision
+        assert decision["conversation_continuing"] is True, decision
+
+    @pytest.mark.asyncio
+    async def test_tool_continuation_reuses_default_model_when_only_it_served_the_previous_response(
+        self, mock_router_instance, basic_config
+    ):
+        router = self._router(
+            mock_router_instance,
+            basic_config,
+            tiers={"SIMPLE": "gpt-4o-mini"},
+            default_model="gpt-4o",
+        )
+        result = await router.async_pre_routing_hook(
+            model="test-complexity-router",
+            request_kwargs={"previous_response_id": self._encoded_response_id("deployment-4o")},
+            messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}],
+        )
+        assert result is not None
+        assert result.model == "gpt-4o", result.routing_decision
+        decision = result.routing_decision
+        assert decision is not None
+        assert decision["cause"] == "previous_response_reuse", decision
+        assert decision["conversation_continuing"] is True, decision
+
+    @pytest.mark.asyncio
+    async def test_tool_continuation_skips_deployments_without_model_info(self, mock_router_instance, basic_config):
+        router = self._router(mock_router_instance, basic_config, default_model="gpt-4o")
+        deployments = {
+            "gpt-4o-mini": [
+                {"model_name": "gpt-4o-mini", "litellm_params": {"model": "gpt-4o-mini"}},
+                {
+                    "model_name": "gpt-4o-mini",
+                    "litellm_params": {"model": "gpt-4o-mini"},
+                    "model_info": {"id": "deployment-mini"},
+                },
+            ]
+        }
+        mock_router_instance.get_model_list.side_effect = lambda model_name=None: deployments.get(model_name, [])
+        result = await router.async_pre_routing_hook(
+            model="test-complexity-router",
+            request_kwargs={"previous_response_id": self._encoded_response_id("deployment-mini")},
+            messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}],
+        )
+        assert result is not None
+        assert result.model == "gpt-4o-mini", result.routing_decision
+        decision = result.routing_decision
+        assert decision is not None
+        assert decision["cause"] == "previous_response_reuse", decision
+
+    @pytest.mark.asyncio
+    async def test_unencoded_previous_response_id_still_falls_back_to_default(
+        self, mock_router_instance, basic_config
+    ):
+        router = self._router(mock_router_instance, basic_config, default_model="gpt-4o-mini")
+        result = await router.async_pre_routing_hook(
+            model="test-complexity-router",
+            request_kwargs={"previous_response_id": "resp_plain_provider_id"},
+            messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}],
+        )
+        assert result is not None
+        assert result.model == "gpt-4o-mini", result.routing_decision
+        decision = result.routing_decision
+        assert decision is not None
+        assert decision["cause"] == "default_fallback", decision
+
+    @pytest.mark.asyncio
+    async def test_previous_response_from_an_unserved_deployment_still_falls_back_to_default(
+        self, mock_router_instance, basic_config
+    ):
+        router = self._router(mock_router_instance, basic_config, default_model="gpt-4o-mini")
+        result = await router.async_pre_routing_hook(
+            model="test-complexity-router",
+            request_kwargs={"previous_response_id": self._encoded_response_id("deployment-elsewhere")},
+            messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}],
+        )
+        assert result is not None
+        assert result.model == "gpt-4o-mini", result.routing_decision
+        decision = result.routing_decision
+        assert decision is not None
+        assert decision["cause"] == "default_fallback", decision
+
+    @pytest.mark.asyncio
+    async def test_plugins_are_not_bypassed_by_previous_response_reuse(self, mock_router_instance, basic_config):
+        class AllowAll:
+            async def run(self, context):
+                return context
+
+        router = self._router(
+            mock_router_instance,
+            basic_config,
+            default_model="gpt-4o-mini",
+            plugins=[AllowAll()],
+        )
+        result = await router.async_pre_routing_hook(
+            model="test-complexity-router",
+            request_kwargs={"previous_response_id": self._encoded_response_id("deployment-sonnet")},
+            messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}],
+        )
+        assert result is not None
+        assert result.model == "gpt-4o", result.routing_decision
+        decision = result.routing_decision
+        assert decision is not None
+        assert decision["cause"] == "default_fallback", decision
+
+    @pytest.mark.asyncio
+    async def test_previous_response_id_marks_conversation_as_continuing(self, mock_router_instance, basic_config):
+        router = self._router(mock_router_instance, basic_config)
+        result = await router.async_pre_routing_hook(
+            model="test-complexity-router",
+            request_kwargs={"previous_response_id": self._encoded_response_id("deployment-4o")},
+            messages=[{"role": "user", "content": "Hello!"}],
+        )
+        assert result is not None
+        assert result.model == "gpt-4o-mini", result.routing_decision
+        decision = result.routing_decision
+        assert decision is not None
+        assert decision["conversation_continuing"] is True, decision

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -45828,7 +45828,7 @@ export interface components {
              * Cause
              * @enum {string}
              */
-            cause?: "prompt_cache_cost" | "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "capability_classifier" | "jev_classifier" | "llm_v2_classifier" | "llm_v2_fallback" | "heuristic_first_short_circuit" | "hybrid_short_circuit" | "classifier_plugin" | "classifier_fallback" | "capability_classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "modality_pin_override" | "health_failover" | "health_default_fallback" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit" | "semantic_match" | "semantic_no_match" | "semantic_error";
+            cause?: "prompt_cache_cost" | "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "capability_classifier" | "jev_classifier" | "llm_v2_classifier" | "llm_v2_fallback" | "heuristic_first_short_circuit" | "hybrid_short_circuit" | "classifier_plugin" | "classifier_fallback" | "capability_classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "modality_pin_override" | "health_failover" | "health_default_fallback" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "previous_response_reuse" | "default_fallback" | "keyword" | "quality_tier" | "bandit" | "semantic_match" | "semantic_no_match" | "semantic_error";
             /** Classifier Calibrated Capable P Solve */
             classifier_calibrated_capable_p_solve?: number;
             /** Classifier Calibrated Efficient P Solve */