From d4c6831733601bbde85fa7a23c1fbee16d16b402 Mon Sep 17 00:00:00 2001 From: Azmi Muwahid Date: Tue, 20 Oct 2026 14:56:18 +0700 Subject: [PATCH 1/4] fix(router): reuse previous response routing for Responses API continuations --- .../complexity_router/complexity_router.py | 72 +++++++++- litellm/types/utils.py | 4 + .../router_strategy/test_complexity_router.py | 127 ++++++++++++++++++ 3 files changed, 202 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 1638f20c8e3..b66b8f03fbd 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -662,6 +662,16 @@ def _conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None) return any(message.get("role") == "assistant" for message in messages) +def _previous_response_id_from_request_kwargs(request_kwargs: Mapping[str, object]) -> str | None: + """The Responses API `previous_response_id` on this request, or None when absent. + + Read from the top-level kwargs, where both the proxy and direct Router calls + surface it (the same place DeploymentAffinityCheck reads it). + """ + value: Final = request_kwargs.get("previous_response_id") + return value if isinstance(value, str) and value else None + + def _newest_turn_ask( messages: Sequence[Mapping[str, object]], marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, @@ -3410,6 +3420,43 @@ class ComplexityRouter(CustomLogger): return None return limit if isinstance(limit, int) else None + def _previous_response_model( + self, request_kwargs: Mapping[str, object] + ) -> tuple[str, ComplexityTier | None] | None: + """The model group that produced the response a continuation request references. + + A Responses API tool continuation carries no human ask, because its context + lives server-side behind `previous_response_id`. The LiteLLM response id + encodes the deployment that served that response, so the decision that + produced it can be replayed instead of re-derived from text that is not + there. Returns None when the id is not a LiteLLM encoded id, names a + deployment outside this router's tier pools and default model, or plugins + are configured (a replayed decision would bypass the plugin pipeline, the + same reason session pins are skipped then). + """ + if self.config.plugins: + return None + previous_response_id: Final = _previous_response_id_from_request_kwargs(request_kwargs) + if previous_response_id is None: + return None + from litellm.responses.utils import ResponsesAPIRequestUtils + + model_id: Final = ResponsesAPIRequestUtils.get_model_id_from_response_id(previous_response_id) + if model_id is None: + return None + candidate_groups: list[str] = [] + for pool in self._tier_pools().values(): + candidate_groups.extend(pool) + if self.config.default_model and self.config.default_model not in candidate_groups: + candidate_groups.append(self.config.default_model) + for group in candidate_groups: + for deployment in self._group_deployments(group): + model_info: Final = deployment.get("model_info") + deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None + if deployment_id is not None and str(deployment_id) == model_id: + return group, self._tier_for_model(group) + return None + def _group_deployments(self, group: str) -> Sequence[Mapping[str, object]]: list_models: Final = getattr(self.litellm_router_instance, "get_model_list", None) deployments: Final = list_models(model_name=group) if callable(list_models) else None @@ -4366,7 +4413,10 @@ class ComplexityRouter(CustomLogger): resolved_messages: Final = self._resolve_messages(messages, request_kwargs) context_fit: Final = await self._request_context_fit(resolved_messages, request_kwargs) marker_pairs: Final = self._reminder_markers_for_request(request_kwargs) - conversation_continuing: Final = _conversation_is_continuing(resolved_messages) + conversation_continuing: Final = ( + _conversation_is_continuing(resolved_messages) + or _previous_response_id_from_request_kwargs(request_kwargs) is not None + ) use_session_affinity: Final = self._uses_tier_pin session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None @@ -4678,6 +4728,26 @@ class ComplexityRouter(CustomLogger): classifier_images: Final = self._classifier_image_parts(resolved_messages) if user_message is None and not classifier_images: + previous_response: Final = self._previous_response_model(request_kwargs) + if previous_response is not None: + reused_model, reused_tier = previous_response + reused_params: Final = self._litellm_params_for_model(reused_tier, reused_model) + verbose_router_logger.info( + "ComplexityRouter: routing decision cause=previous_response_reuse, routed_model=%s", + reused_model, + ) + return PreRoutingHookResponse( + model=reused_model, + messages=messages if has_original_messages else None, + routing_decision=self._build_routing_decision( + routed_model=reused_model, + cause="previous_response_reuse", + tier=reused_tier, + conversation_continuing=conversation_continuing, + tier_litellm_params=reused_params, + ), + litellm_params=reused_params, + ) verbose_router_logger.debug("ComplexityRouter: No user message found, routing to default model") default_model_first: Final = not self.config.plugins and self.config.default_model if default_model_first: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 576c4f093d3..e9f6ca88680 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3166,6 +3166,10 @@ RoutingDecisionCause = Literal[ # called. Distinct from "session_affinity_pin", which reports the session_affinity flag pinning # every turn including new asks; this cause only appears when session_affinity is off. "user_turn_continuation", + # A Responses API continuation (previous_response_id set, no human ask) reuses the routing + # decision that produced the referenced response, decoded from the LiteLLM response id. + # Distinct from "user_turn_continuation", which replays a decision held in the session cache. + "previous_response_reuse", "default_fallback", "keyword", "quality_tier", diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index f956bb66e6c..11ac3148557 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -5,6 +5,7 @@ Tests the rule-based complexity scoring and tier assignment logic. """ import asyncio +import base64 import json import logging import math @@ -17345,3 +17346,129 @@ class TestNonReasoningTier: "complex", "reasoning", ) + + +class TestPreviousResponseReuse: + """Regression tests for Responses API tool continuations (issue #46048). + + A continuation request carries only tool outputs plus `previous_response_id`, + so it has no human ask to classify. Before the fix it always fell to the + default model, even though the LiteLLM response id names the deployment that + produced the referenced response. + """ + + @staticmethod + def _encoded_response_id(model_id: str) -> str: + payload = f"litellm:custom_llm_provider:azure_ai;model_id:{model_id};response_id:resp_original" + return "resp_" + base64.b64encode(payload.encode("utf-8")).decode("utf-8") + + @staticmethod + def _router(mock_router_instance, basic_config, **overrides) -> ComplexityRouter: + deployments = { + group: [ + { + "model_name": group, + "litellm_params": {"model": group}, + "model_info": {"id": deployment_id}, + } + ] + for group, deployment_id in { + "gpt-4o-mini": "deployment-mini", + "gpt-4o": "deployment-4o", + "claude-sonnet-4-20250514": "deployment-sonnet", + "o1-preview": "deployment-o1", + }.items() + } + mock_router_instance.get_model_list.side_effect = lambda model_name=None: deployments.get(model_name, []) + return ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, **overrides}, + ) + + @pytest.mark.asyncio + async def test_tool_continuation_reuses_the_model_that_produced_the_previous_response( + self, mock_router_instance, basic_config + ): + router = self._router(mock_router_instance, basic_config, default_model="gpt-4o-mini") + result = await router.async_pre_routing_hook( + model="test-complexity-router", + request_kwargs={"previous_response_id": self._encoded_response_id("deployment-sonnet")}, + messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}], + ) + assert result is not None + assert result.model == "claude-sonnet-4-20250514", result.routing_decision + decision = result.routing_decision + assert decision is not None + assert decision["cause"] == "previous_response_reuse", decision + assert decision["tier"] == "COMPLEX", decision + assert decision["conversation_continuing"] is True, decision + + @pytest.mark.asyncio + async def test_unencoded_previous_response_id_still_falls_back_to_default( + self, mock_router_instance, basic_config + ): + router = self._router(mock_router_instance, basic_config, default_model="gpt-4o-mini") + result = await router.async_pre_routing_hook( + model="test-complexity-router", + request_kwargs={"previous_response_id": "resp_plain_provider_id"}, + messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}], + ) + assert result is not None + assert result.model == "gpt-4o-mini", result.routing_decision + decision = result.routing_decision + assert decision is not None + assert decision["cause"] == "default_fallback", decision + + @pytest.mark.asyncio + async def test_previous_response_from_an_unserved_deployment_still_falls_back_to_default( + self, mock_router_instance, basic_config + ): + router = self._router(mock_router_instance, basic_config, default_model="gpt-4o-mini") + result = await router.async_pre_routing_hook( + model="test-complexity-router", + request_kwargs={"previous_response_id": self._encoded_response_id("deployment-elsewhere")}, + messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}], + ) + assert result is not None + assert result.model == "gpt-4o-mini", result.routing_decision + decision = result.routing_decision + assert decision is not None + assert decision["cause"] == "default_fallback", decision + + @pytest.mark.asyncio + async def test_plugins_are_not_bypassed_by_previous_response_reuse(self, mock_router_instance, basic_config): + class AllowAll: + async def run(self, context): + return context + + router = self._router( + mock_router_instance, + basic_config, + default_model="gpt-4o-mini", + plugins=[AllowAll()], + ) + result = await router.async_pre_routing_hook( + model="test-complexity-router", + request_kwargs={"previous_response_id": self._encoded_response_id("deployment-sonnet")}, + messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}], + ) + assert result is not None + assert result.model == "gpt-4o", result.routing_decision + decision = result.routing_decision + assert decision is not None + assert decision["cause"] == "default_fallback", decision + + @pytest.mark.asyncio + async def test_previous_response_id_marks_conversation_as_continuing(self, mock_router_instance, basic_config): + router = self._router(mock_router_instance, basic_config) + result = await router.async_pre_routing_hook( + model="test-complexity-router", + request_kwargs={"previous_response_id": self._encoded_response_id("deployment-4o")}, + messages=[{"role": "user", "content": "Hello!"}], + ) + assert result is not None + assert result.model == "gpt-4o-mini", result.routing_decision + decision = result.routing_decision + assert decision is not None + assert decision["conversation_continuing"] is True, decision From daf924c0ef998523cd9a53a264cf0c351f3d4e98 Mon Sep 17 00:00:00 2001 From: Azmi Muwahid Date: Tue, 20 Oct 2026 16:36:31 +0700 Subject: [PATCH 2/4] fix(router): satisfy type discipline gate and sync generated API types --- .../complexity_router/complexity_router.py | 22 +++++++++++-------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index b66b8f03fbd..2a1bcad68e7 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -3444,19 +3444,23 @@ class ComplexityRouter(CustomLogger): model_id: Final = ResponsesAPIRequestUtils.get_model_id_from_response_id(previous_response_id) if model_id is None: return None - candidate_groups: list[str] = [] for pool in self._tier_pools().values(): - candidate_groups.extend(pool) - if self.config.default_model and self.config.default_model not in candidate_groups: - candidate_groups.append(self.config.default_model) - for group in candidate_groups: - for deployment in self._group_deployments(group): - model_info: Final = deployment.get("model_info") - deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None - if deployment_id is not None and str(deployment_id) == model_id: + for group in pool: + if self._group_has_deployment_id(group, model_id): return group, self._tier_for_model(group) + default_model: Final = self.config.default_model + if default_model and self._group_has_deployment_id(default_model, model_id): + return default_model, self._tier_for_model(default_model) return None + def _group_has_deployment_id(self, group: str, model_id: str) -> bool: + for deployment in self._group_deployments(group): + model_info: Final = deployment.get("model_info") + deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None + if deployment_id is not None and str(deployment_id) == model_id: + return True + return False + def _group_deployments(self, group: str) -> Sequence[Mapping[str, object]]: list_models: Final = getattr(self.litellm_router_instance, "get_model_list", None) deployments: Final = list_models(model_name=group) if callable(list_models) else None diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bf2fe4300a3..5451380520b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -45617,7 +45617,7 @@ export interface components { * Cause * @enum {string} */ - cause?: "prompt_cache_cost" | "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "capability_classifier" | "jev_classifier" | "llm_v2_classifier" | "llm_v2_fallback" | "heuristic_first_short_circuit" | "hybrid_short_circuit" | "classifier_plugin" | "classifier_fallback" | "capability_classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "modality_pin_override" | "health_failover" | "health_default_fallback" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit" | "semantic_match" | "semantic_no_match" | "semantic_error"; + cause?: "prompt_cache_cost" | "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "capability_classifier" | "jev_classifier" | "llm_v2_classifier" | "llm_v2_fallback" | "heuristic_first_short_circuit" | "hybrid_short_circuit" | "classifier_plugin" | "classifier_fallback" | "capability_classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "modality_pin_override" | "health_failover" | "health_default_fallback" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "previous_response_reuse" | "default_fallback" | "keyword" | "quality_tier" | "bandit" | "semantic_match" | "semantic_no_match" | "semantic_error"; /** Classifier Calibrated Capable P Solve */ classifier_calibrated_capable_p_solve?: number; /** Classifier Calibrated Efficient P Solve */ From 12db2c6b0d86fadb394f819e79613dd772d7e338 Mon Sep 17 00:00:00 2001 From: Azmi Muwahid <92735014+azmi2409@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:33:48 +0700 Subject: [PATCH 3/4] fix(router): clear type-check gate violations from previous-response reuse --- .../complexity_router/complexity_router.py | 25 +++++++++++++++---- .../router_strategy/test_complexity_router.py | 22 ++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 2a1bcad68e7..736bb6cc59c 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -3453,11 +3453,17 @@ class ComplexityRouter(CustomLogger): return default_model, self._tier_for_model(default_model) return None + @staticmethod + def _deployment_model_id(deployment: Mapping[str, object]) -> str | None: + model_info: Final = deployment.get("model_info") + deployment_id: Final = cast( # cast-ok: the isinstance check in the condition narrows model_info to a Mapping with str keys + "Mapping[str, object]", model_info + ).get("id") if isinstance(model_info, Mapping) else None + return None if deployment_id is None else str(deployment_id) + def _group_has_deployment_id(self, group: str, model_id: str) -> bool: for deployment in self._group_deployments(group): - model_info: Final = deployment.get("model_info") - deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None - if deployment_id is not None and str(deployment_id) == model_id: + if self._deployment_model_id(deployment) == model_id: return True return False @@ -4419,7 +4425,12 @@ class ComplexityRouter(CustomLogger): marker_pairs: Final = self._reminder_markers_for_request(request_kwargs) conversation_continuing: Final = ( _conversation_is_continuing(resolved_messages) - or _previous_response_id_from_request_kwargs(request_kwargs) is not None + or _previous_response_id_from_request_kwargs( + cast( # cast-ok: the hook's bare kwargs dict is the str-keyed request mapping + Mapping[str, object], request_kwargs + ) + ) + is not None ) use_session_affinity: Final = self._uses_tier_pin @@ -4732,7 +4743,11 @@ class ComplexityRouter(CustomLogger): classifier_images: Final = self._classifier_image_parts(resolved_messages) if user_message is None and not classifier_images: - previous_response: Final = self._previous_response_model(request_kwargs) + previous_response: Final = self._previous_response_model( + cast( # cast-ok: the hook's bare kwargs dict is the str-keyed request mapping + Mapping[str, object], request_kwargs + ) + ) if previous_response is not None: reused_model, reused_tier = previous_response reused_params: Final = self._litellm_params_for_model(reused_tier, reused_model) diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index 11ac3148557..a5c39e616b5 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -17404,6 +17404,28 @@ class TestPreviousResponseReuse: assert decision["tier"] == "COMPLEX", decision assert decision["conversation_continuing"] is True, decision + @pytest.mark.asyncio + async def test_tool_continuation_reuses_default_model_when_only_it_served_the_previous_response( + self, mock_router_instance, basic_config + ): + router = self._router( + mock_router_instance, + basic_config, + tiers={"SIMPLE": "gpt-4o-mini"}, + default_model="gpt-4o", + ) + result = await router.async_pre_routing_hook( + model="test-complexity-router", + request_kwargs={"previous_response_id": self._encoded_response_id("deployment-4o")}, + messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}], + ) + assert result is not None + assert result.model == "gpt-4o", result.routing_decision + decision = result.routing_decision + assert decision is not None + assert decision["cause"] == "previous_response_reuse", decision + assert decision["conversation_continuing"] is True, decision + @pytest.mark.asyncio async def test_unencoded_previous_response_id_still_falls_back_to_default( self, mock_router_instance, basic_config From 6f0ea1be5f3327b482b5f57f58c968886b345b32 Mon Sep 17 00:00:00 2001 From: Azmi Muwahid <92735014+azmi2409@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:46:07 +0700 Subject: [PATCH 4/4] fix(router): satisfy ruff format check and cover deployments without model info --- .../complexity_router/complexity_router.py | 10 +++++--- .../router_strategy/test_complexity_router.py | 25 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 736bb6cc59c..e139206b874 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -3456,9 +3456,13 @@ class ComplexityRouter(CustomLogger): @staticmethod def _deployment_model_id(deployment: Mapping[str, object]) -> str | None: model_info: Final = deployment.get("model_info") - deployment_id: Final = cast( # cast-ok: the isinstance check in the condition narrows model_info to a Mapping with str keys - "Mapping[str, object]", model_info - ).get("id") if isinstance(model_info, Mapping) else None + deployment_id: Final = ( + cast( # cast-ok: the isinstance check in the condition narrows model_info to a Mapping with str keys + "Mapping[str, object]", model_info + ).get("id") + if isinstance(model_info, Mapping) + else None + ) return None if deployment_id is None else str(deployment_id) def _group_has_deployment_id(self, group: str, model_id: str) -> bool: diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index a5c39e616b5..fc54cb120c1 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -17426,6 +17426,31 @@ class TestPreviousResponseReuse: assert decision["cause"] == "previous_response_reuse", decision assert decision["conversation_continuing"] is True, decision + @pytest.mark.asyncio + async def test_tool_continuation_skips_deployments_without_model_info(self, mock_router_instance, basic_config): + router = self._router(mock_router_instance, basic_config, default_model="gpt-4o") + deployments = { + "gpt-4o-mini": [ + {"model_name": "gpt-4o-mini", "litellm_params": {"model": "gpt-4o-mini"}}, + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "gpt-4o-mini"}, + "model_info": {"id": "deployment-mini"}, + }, + ] + } + mock_router_instance.get_model_list.side_effect = lambda model_name=None: deployments.get(model_name, []) + result = await router.async_pre_routing_hook( + model="test-complexity-router", + request_kwargs={"previous_response_id": self._encoded_response_id("deployment-mini")}, + messages=[{"role": "tool", "content": "search results", "tool_call_id": "call_1"}], + ) + assert result is not None + assert result.model == "gpt-4o-mini", result.routing_decision + decision = result.routing_decision + assert decision is not None + assert decision["cause"] == "previous_response_reuse", decision + @pytest.mark.asyncio async def test_unencoded_previous_response_id_still_falls_back_to_default( self, mock_router_instance, basic_config