diff --git a/litellm/router.py b/litellm/router.py index 021dafa9791..417f3e87fb6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11689,6 +11689,14 @@ class Router: healthy_deployments = self._filter_blocked_deployments(healthy_deployments) + # An explicit order fallback is a hard retry constraint. Apply it before + # affinity so a stale pin cannot send the retry back to an earlier order. + _target_order: Final = (request_kwargs or {}).pop("_target_order", None) + if _target_order is not None: + healthy_deployments = litellm.utils._get_order_filtered_deployments( # pyright: ignore[reportPrivateUsage] # Router's internal order helper + healthy_deployments, target_order=_target_order + ) + healthy_deployments = await self.async_callback_filter_deployments( model=model, healthy_deployments=healthy_deployments, @@ -11729,11 +11737,9 @@ class Router: request_kwargs=request_kwargs, ) - ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) - _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - healthy_deployments = litellm.utils._get_order_filtered_deployments( - cast(list[dict], healthy_deployments), target_order=_target_order - ) + # Without a fallback target, return deployments with the lowest configured order. + if _target_order is None: + healthy_deployments = litellm.utils._get_order_filtered_deployments(cast(list[dict], healthy_deployments)) ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in ## this request via weighted-failover. Always honored, regardless of the diff --git a/tests/test_litellm/test_router_order_fallback.py b/tests/test_litellm/test_router_order_fallback.py index 7743cb005d0..5a56a49f96f 100644 --- a/tests/test_litellm/test_router_order_fallback.py +++ b/tests/test_litellm/test_router_order_fallback.py @@ -11,6 +11,9 @@ from typing import Optional import pytest from litellm import Router +from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( + DeploymentAffinityCheck, +) from litellm.utils import _get_order_filtered_deployments # --------------------------------------------------------------------------- @@ -192,6 +195,56 @@ async def test_router_order_fallback_on_failure(): assert response._hidden_params["model_id"] == "2" +@pytest.mark.asyncio +async def test_order_fallback_overrides_session_affinity(): + """An order fallback must not reuse a deployment pinned at an earlier order.""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": {"model": "gpt-4o", "api_key": "key", "order": 1}, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": {"model": "gpt-4o", "api_key": "key", "order": 2}, + "model_info": {"id": "2"}, + }, + ], + optional_pre_call_checks=["session_affinity"], + ) + callback = next( + callback + for callback in router.optional_callbacks + if isinstance(callback, DeploymentAffinityCheck) + ) + session_id = "test-session" + await callback.cache.async_set_cache( + key=DeploymentAffinityCheck.get_session_affinity_cache_key( + "test-model", session_id, None + ), + value={"model_id": "1"}, + ) + + try: + pinned = await router.async_get_healthy_deployments( + model="test-model", + request_kwargs={"metadata": {"session_id": session_id}}, + ) + fallback = await router.async_get_healthy_deployments( + model="test-model", + request_kwargs={ + "_target_order": 2, + "metadata": {"session_id": session_id}, + }, + ) + + assert [deployment["model_info"]["id"] for deployment in pinned] == ["1"] + assert [deployment["model_info"]["id"] for deployment in fallback] == ["2"] + finally: + router.discard() + + @pytest.mark.asyncio async def test_router_order_fallback_three_levels(): """When order=1 and order=2 both fail, order=3 should be tried."""