Merge ec93858dbd into 49affa7c01

litellm/router.py

@@ -11689,6 +11689,14 @@ class Router:
 
         healthy_deployments = self._filter_blocked_deployments(healthy_deployments)
 
+        # An explicit order fallback is a hard retry constraint. Apply it before
+        # affinity so a stale pin cannot send the retry back to an earlier order.
+        _target_order: Final = (request_kwargs or {}).pop("_target_order", None)
+        if _target_order is not None:
+            healthy_deployments = litellm.utils._get_order_filtered_deployments(  # pyright: ignore[reportPrivateUsage]  # Router's internal order helper
+                healthy_deployments, target_order=_target_order
+            )
+
         healthy_deployments = await self.async_callback_filter_deployments(
             model=model,
             healthy_deployments=healthy_deployments,
@@ -11729,11 +11737,9 @@ class Router:
             request_kwargs=request_kwargs,
         )
 
-        ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
-        _target_order: Final = (request_kwargs or {}).pop("_target_order", None)
-        healthy_deployments = litellm.utils._get_order_filtered_deployments(
-            cast(list[dict], healthy_deployments), target_order=_target_order
-        )
+        # Without a fallback target, return deployments with the lowest configured order.
+        if _target_order is None:
+            healthy_deployments = litellm.utils._get_order_filtered_deployments(cast(list[dict], healthy_deployments))
 
         ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in
         ## this request via weighted-failover. Always honored, regardless of the

tests/test_litellm/test_router_order_fallback.py

@@ -11,6 +11,9 @@ from typing import Optional
 import pytest
 
 from litellm import Router
+from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
+    DeploymentAffinityCheck,
+)
 from litellm.utils import _get_order_filtered_deployments
 
 # ---------------------------------------------------------------------------
@@ -192,6 +195,56 @@ async def test_router_order_fallback_on_failure():
     assert response._hidden_params["model_id"] == "2"
 
 
+@pytest.mark.asyncio
+async def test_order_fallback_overrides_session_affinity():
+    """An order fallback must not reuse a deployment pinned at an earlier order."""
+    router = Router(
+        model_list=[
+            {
+                "model_name": "test-model",
+                "litellm_params": {"model": "gpt-4o", "api_key": "key", "order": 1},
+                "model_info": {"id": "1"},
+            },
+            {
+                "model_name": "test-model",
+                "litellm_params": {"model": "gpt-4o", "api_key": "key", "order": 2},
+                "model_info": {"id": "2"},
+            },
+        ],
+        optional_pre_call_checks=["session_affinity"],
+    )
+    callback = next(
+        callback
+        for callback in router.optional_callbacks
+        if isinstance(callback, DeploymentAffinityCheck)
+    )
+    session_id = "test-session"
+    await callback.cache.async_set_cache(
+        key=DeploymentAffinityCheck.get_session_affinity_cache_key(
+            "test-model", session_id, None
+        ),
+        value={"model_id": "1"},
+    )
+
+    try:
+        pinned = await router.async_get_healthy_deployments(
+            model="test-model",
+            request_kwargs={"metadata": {"session_id": session_id}},
+        )
+        fallback = await router.async_get_healthy_deployments(
+            model="test-model",
+            request_kwargs={
+                "_target_order": 2,
+                "metadata": {"session_id": session_id},
+            },
+        )
+
+        assert [deployment["model_info"]["id"] for deployment in pinned] == ["1"]
+        assert [deployment["model_info"]["id"] for deployment in fallback] == ["2"]
+    finally:
+        router.discard()
+
+
 @pytest.mark.asyncio
 async def test_router_order_fallback_three_levels():
     """When order=1 and order=2 both fail, order=3 should be tried."""