diff --git a/litellm/router.py b/litellm/router.py index 705accbcf8d..93523fe6ad9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8856,8 +8856,6 @@ class Router: status_code: Final = getattr(exception, "status_code", None) if not failed_deployment_id or not isinstance(status_code, int): return () - if litellm.should_retry(status_code): - return () already_skipped_ids: Final = _as_retry_skipped_deployment_ids(already_skipped) skipped: Final = tuple(sorted(frozenset((*already_skipped_ids, failed_deployment_id)))) verbose_router_logger.debug( diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index 88fb8876175..7a7b79413ac 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -369,6 +369,16 @@ class DeploymentAffinityCheck(CustomLogger): typed_healthy_deployments: Final = cast(list[dict], healthy_deployments) if request_kwargs.get("_target_order") is not None: return typed_healthy_deployments + raw_retry_skipped_ids: Final[object] = cast( # cast-ok: request kwargs use string parameter names + dict[str, object], request_kwargs + ).get("_retry_skipped_deployment_ids") + retry_skipped_ids: Final[tuple[object, ...]] = ( + cast( # cast-ok: isinstance validates the tuple container before membership checks + tuple[object, ...], raw_retry_skipped_ids + ) + if isinstance(raw_retry_skipped_ids, tuple) + else () + ) ( enable_user_key, @@ -386,7 +396,7 @@ class DeploymentAffinityCheck(CustomLogger): healthy_deployments=typed_healthy_deployments, model_id=responses_model_id, ) - if deployment is not None: + if deployment is not None and responses_model_id not in retry_skipped_ids: verbose_router_logger.debug( "DeploymentAffinityCheck: previous_response_id pinning -> deployment=%s", responses_model_id, @@ -431,7 +441,7 @@ class DeploymentAffinityCheck(CustomLogger): healthy_deployments=typed_healthy_deployments, model_id=session_model_id, ) - if session_deployment is not None: + if session_deployment is not None and session_model_id not in retry_skipped_ids: verbose_router_logger.debug( "DeploymentAffinityCheck: session-id affinity hit -> deployment=%s session_id=%s", session_model_id, @@ -468,7 +478,7 @@ class DeploymentAffinityCheck(CustomLogger): healthy_deployments=typed_healthy_deployments, model_id=model_id, ) - if deployment is None: + if deployment is None or model_id in retry_skipped_ids: verbose_router_logger.debug( "DeploymentAffinityCheck: pinned deployment=%s not found in healthy_deployments", model_id, diff --git a/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py b/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py index 9bbaed0ae1b..d7e05b2e602 100644 --- a/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py +++ b/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py @@ -3,7 +3,9 @@ import json from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest +import respx import litellm from litellm.caching.affinity_cache import claim_affinity_pin @@ -26,6 +28,121 @@ class MockResponse: return self._json_data +@pytest.mark.asyncio +async def test_retry_skips_failed_session_affinity_deployment( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "affinity-model", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-fake", + "api_base": "https://failed.local/v1", + }, + "model_info": {"id": "failed"}, + }, + { + "model_name": "affinity-model", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-fake", + "api_base": "https://healthy.local/v1", + }, + "model_info": {"id": "healthy"}, + }, + ], + optional_pre_call_checks=["session_affinity"], + num_retries=2, + allowed_fails=1, + ) + affinity: Final = next( + callback for callback in router.optional_callbacks or () if isinstance(callback, DeploymentAffinityCheck) + ) + session_id: Final = "pinned-session" + await affinity.cache.async_set_cache( + DeploymentAffinityCheck.get_session_affinity_cache_key( + model_group="affinity-model", session_id=session_id, user_key=None + ), + {"model_id": "failed"}, + ) + failed: Final = respx_mock.post("https://failed.local/v1/chat/completions").mock( + return_value=httpx.Response( + 500, + json={"error": {"message": "deployment unavailable", "type": "server_error", "code": "internal_error"}}, + ) + ) + healthy: Final = respx_mock.post("https://healthy.local/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-affinity-retry", + "object": "chat.completion", + "created": 1, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + + response: Final = await router.acompletion( + model="affinity-model", + messages=[{"role": "user", "content": "hi"}], + metadata={"session_id": session_id}, + ) + + assert response.choices[0].message.content == "ok" + assert failed.call_count == 1 + assert healthy.call_count == 1 + + +@pytest.mark.asyncio +async def test_retry_with_only_session_affinity_deployment_stays_bounded( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "affinity-model", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-fake", + "api_base": "https://only.local/v1", + }, + "model_info": {"id": "only"}, + } + ], + optional_pre_call_checks=["session_affinity"], + num_retries=2, + disable_cooldowns=True, + ) + only: Final = respx_mock.post("https://only.local/v1/chat/completions").mock( + return_value=httpx.Response( + 500, + json={"error": {"message": "deployment unavailable", "type": "server_error", "code": "internal_error"}}, + ) + ) + + with pytest.raises(litellm.InternalServerError): + await router.acompletion( + model="affinity-model", + messages=[{"role": "user", "content": "hi"}], + metadata={"session_id": "pinned-session"}, + ) + + assert only.call_count == 3 + + @pytest.mark.asyncio async def test_async_session_id_affinity_routes_to_same_deployment(): """ diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index f06f9e08f7e..a17c7ab65f6 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -20991,9 +20991,9 @@ def _retry_skip_deployment(deployment_id, host, litellm_params=None, model_info= (403, "rejecting", None, ("rejecting",)), (400, "second", ("first",), ("first", "second")), (400, "first", ("first",), ("first",)), - (429, "rejecting", None, ()), - (503, "rejecting", None, ()), - (408, "rejecting", None, ()), + (429, "rejecting", None, ("rejecting",)), + (503, "rejecting", None, ("rejecting",)), + (408, "rejecting", None, ("rejecting",)), (400, None, None, ()), (None, "rejecting", None, ()), ("400", "rejecting", None, ()),