From abb1d8f8a32487bd673b2a89d8f3b05421360769 Mon Sep 17 00:00:00 2001 From: Mohammad Javad Naderi Date: Thu, 23 Jul 2026 06:16:40 +0330 Subject: [PATCH 1/6] Apply order fallback before deployment affinity - Treat an explicit target order as a retry constraint before affinity callbacks - Cover session affinity pinned to an earlier deployment order --- litellm/router.py | 18 +++++-- .../test_router_order_fallback.py | 53 +++++++++++++++++++ 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 0fd3cf6af1b..96ef9dc11f3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11052,6 +11052,14 @@ class Router: healthy_deployments = self._filter_blocked_deployments(healthy_deployments) + # An explicit order fallback is a hard retry constraint. Apply it before + # affinity so a stale pin cannot send the retry back to an earlier order. + _target_order: Final = (request_kwargs or {}).pop("_target_order", None) + if _target_order is not None: + healthy_deployments = litellm.utils._get_order_filtered_deployments( + cast(list[dict], healthy_deployments), target_order=_target_order + ) + healthy_deployments = await self.async_callback_filter_deployments( model=model, healthy_deployments=healthy_deployments, @@ -11083,11 +11091,11 @@ class Router: request_kwargs=request_kwargs, ) - ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) - _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - healthy_deployments = litellm.utils._get_order_filtered_deployments( - cast(list[dict], healthy_deployments), target_order=_target_order - ) + # Without a fallback target, return deployments with the lowest configured order. + if _target_order is None: + healthy_deployments = litellm.utils._get_order_filtered_deployments( + cast(list[dict], healthy_deployments) + ) ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in ## this request via weighted-failover. Always honored, regardless of the diff --git a/tests/test_litellm/test_router_order_fallback.py b/tests/test_litellm/test_router_order_fallback.py index 083f35456a3..140bf0613c5 100644 --- a/tests/test_litellm/test_router_order_fallback.py +++ b/tests/test_litellm/test_router_order_fallback.py @@ -11,6 +11,9 @@ from typing import Optional import pytest from litellm import Router +from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( + DeploymentAffinityCheck, +) from litellm.utils import _get_order_filtered_deployments # --------------------------------------------------------------------------- @@ -192,6 +195,56 @@ async def test_router_order_fallback_on_failure(): assert response._hidden_params["model_id"] == "2" +@pytest.mark.asyncio +async def test_order_fallback_overrides_session_affinity(): + """An order fallback must not reuse a deployment pinned at an earlier order.""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": {"model": "gpt-4o", "api_key": "key", "order": 1}, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": {"model": "gpt-4o", "api_key": "key", "order": 2}, + "model_info": {"id": "2"}, + }, + ], + optional_pre_call_checks=["session_affinity"], + ) + callback = next( + callback + for callback in router.optional_callbacks + if isinstance(callback, DeploymentAffinityCheck) + ) + session_id = "test-session" + await callback.cache.async_set_cache( + key=DeploymentAffinityCheck.get_session_affinity_cache_key( + "test-model", session_id + ), + value={"model_id": "1"}, + ) + + try: + pinned = await router.async_get_healthy_deployments( + model="test-model", + request_kwargs={"metadata": {"session_id": session_id}}, + ) + fallback = await router.async_get_healthy_deployments( + model="test-model", + request_kwargs={ + "_target_order": 2, + "metadata": {"session_id": session_id}, + }, + ) + + assert [deployment["model_info"]["id"] for deployment in pinned] == ["1"] + assert [deployment["model_info"]["id"] for deployment in fallback] == ["2"] + finally: + router.discard() + + @pytest.mark.asyncio async def test_router_order_fallback_three_levels(): """When order=1 and order=2 both fail, order=3 should be tried.""" From 0c820202530c65a90982305dabf98276750647e7 Mon Sep 17 00:00:00 2001 From: Mohammad Javad Naderi Date: Thu, 23 Jul 2026 06:29:24 +0330 Subject: [PATCH 2/6] Format default order filter --- litellm/router.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 96ef9dc11f3..97cbbabe8f9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11093,9 +11093,7 @@ class Router: # Without a fallback target, return deployments with the lowest configured order. if _target_order is None: - healthy_deployments = litellm.utils._get_order_filtered_deployments( - cast(list[dict], healthy_deployments) - ) + healthy_deployments = litellm.utils._get_order_filtered_deployments(cast(list[dict], healthy_deployments)) ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in ## this request via weighted-failover. Always honored, regardless of the From dbb7b456088a282997138c377f03ca0a6bd6e9ef Mon Sep 17 00:00:00 2001 From: Mohammad Javad Naderi Date: Thu, 23 Jul 2026 06:40:07 +0330 Subject: [PATCH 3/6] Remove unnecessary order fallback cast --- litellm/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 97cbbabe8f9..fd53042d4d6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11057,7 +11057,7 @@ class Router: _target_order: Final = (request_kwargs or {}).pop("_target_order", None) if _target_order is not None: healthy_deployments = litellm.utils._get_order_filtered_deployments( - cast(list[dict], healthy_deployments), target_order=_target_order + healthy_deployments, target_order=_target_order ) healthy_deployments = await self.async_callback_filter_deployments( From 772ecfb3ffa09c2aecd2f09278fd6f84c6cf42a7 Mon Sep 17 00:00:00 2001 From: Mohammad Javad Naderi Date: Sun, 16 Aug 2026 14:55:06 +0330 Subject: [PATCH 4/6] test(router): update affinity cache key fixture --- tests/test_litellm/test_router_order_fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/test_router_order_fallback.py b/tests/test_litellm/test_router_order_fallback.py index 140bf0613c5..e54d7e87d38 100644 --- a/tests/test_litellm/test_router_order_fallback.py +++ b/tests/test_litellm/test_router_order_fallback.py @@ -221,7 +221,7 @@ async def test_order_fallback_overrides_session_affinity(): session_id = "test-session" await callback.cache.async_set_cache( key=DeploymentAffinityCheck.get_session_affinity_cache_key( - "test-model", session_id + "test-model", session_id, None ), value={"model_id": "1"}, ) From e4847869de1192a163582b7608b6f0167f221ab7 Mon Sep 17 00:00:00 2001 From: Mohammad Javad Naderi Date: Sun, 16 Aug 2026 15:08:20 +0330 Subject: [PATCH 5/6] fix(router): satisfy private usage type check --- litellm/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index fd53042d4d6..41560ada552 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11056,7 +11056,7 @@ class Router: # affinity so a stale pin cannot send the retry back to an earlier order. _target_order: Final = (request_kwargs or {}).pop("_target_order", None) if _target_order is not None: - healthy_deployments = litellm.utils._get_order_filtered_deployments( + healthy_deployments = litellm.utils._get_order_filtered_deployments( # pyright: ignore[reportPrivateUsage] healthy_deployments, target_order=_target_order ) From ec93858dbd20a9566fdabc70433575b3f61e9c82 Mon Sep 17 00:00:00 2001 From: Mohammad Javad Naderi Date: Sun, 16 Aug 2026 15:28:22 +0330 Subject: [PATCH 6/6] fix(router): document type-check suppression --- litellm/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 41560ada552..ed188f3f8b2 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11056,7 +11056,7 @@ class Router: # affinity so a stale pin cannot send the retry back to an earlier order. _target_order: Final = (request_kwargs or {}).pop("_target_order", None) if _target_order is not None: - healthy_deployments = litellm.utils._get_order_filtered_deployments( # pyright: ignore[reportPrivateUsage] + healthy_deployments = litellm.utils._get_order_filtered_deployments( # pyright: ignore[reportPrivateUsage] # Router's internal order helper healthy_deployments, target_order=_target_order )