From 93b9812980a38188c3c17c4eb6e5f61770a24fcb Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 20 Oct 2026 15:12:07 -0400 Subject: [PATCH] fix(router): fail over encrypted affinity retries --- litellm/router.py | 2 - .../encrypted_content_affinity_check.py | 36 ++++++++- .../test_encrypted_content_affinity_check.py | 76 +++++++++++++++++++ tests/unit/test_router/test_router.py | 6 +- 4 files changed, 112 insertions(+), 8 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index ac0933e158b..11e19e5ec29 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8702,8 +8702,6 @@ class Router: status_code: Final = getattr(exception, "status_code", None) if not failed_deployment_id or not isinstance(status_code, int): return () - if litellm.should_retry(status_code): - return () already_skipped_ids: Final = _as_retry_skipped_deployment_ids(already_skipped) skipped: Final = tuple(sorted(frozenset((*already_skipped_ids, failed_deployment_id)))) verbose_router_logger.debug( diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 5d47bf63a38..6cd6601390b 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -40,7 +40,7 @@ from collections.abc import Iterator, Mapping, Sequence from functools import cache from typing import TYPE_CHECKING, Final, Optional, cast -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger, Span @@ -59,6 +59,7 @@ if TYPE_CHECKING: from litellm.router import Router _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_RETRY_SKIPPED_DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...]) class EncryptedContentAffinityCheck(CustomLogger): @@ -88,6 +89,22 @@ class EncryptedContentAffinityCheck(CustomLogger): # Helpers # ------------------------------------------------------------------ + @staticmethod + def _retry_skipped_deployment_ids(value: object) -> frozenset[str]: + try: + return frozenset(_RETRY_SKIPPED_DEPLOYMENT_IDS_ADAPTER.validate_python(value)) + except ValidationError: + return frozenset() + + @staticmethod + def _deployment_id(deployment: Mapping[str, object]) -> str | None: + try: + model_info: Final = _REQUEST_KWARGS_ADAPTER.validate_python(deployment.get("model_info")) + except ValidationError: + return None + model_id: Final = model_info.get("id") + return model_id if isinstance(model_id, str) else None + @staticmethod def has_model_group_affinity_enabled( model_group_affinity_config: dict[str, list[str]] | None, @@ -345,12 +362,25 @@ class EncryptedContentAffinityCheck(CustomLogger): typed_healthy_deployments: Final = cast(list[dict[str, object]], healthy_deployments) if not self._is_enabled_for_model_group(model): return typed_healthy_deployments - target_order: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs).get("_target_order") - candidates: Final = ( + validated_request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs) + target_order: Final = validated_request_kwargs.get("_target_order") + order_candidates: Final = ( get_order_filtered_deployments(typed_healthy_deployments, target_order=target_order) if isinstance(target_order, int) else typed_healthy_deployments ) + retry_skipped_value: Final = validated_request_kwargs.get("_retry_skipped_deployment_ids") + retry_skipped_deployment_ids: Final = self._retry_skipped_deployment_ids(retry_skipped_value) + untried_candidates: Final = ( + [ + deployment + for deployment in order_candidates + if self._deployment_id(deployment) not in retry_skipped_deployment_ids + ] + if retry_skipped_deployment_ids + else order_candidates + ) + candidates: Final = untried_candidates or order_candidates # Signal to the response post-processor that encrypted item IDs should be # encoded in the output of this request. Only set the flag when diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 87fbf4433a1..a1818159b42 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -16,9 +16,12 @@ The mechanism works without any cache and supports two encoding strategies: """ import time +from typing import Final from unittest.mock import AsyncMock, patch +import httpx import pytest +import respx import litellm from litellm.models.credentials import CredentialItem @@ -26,6 +29,79 @@ from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ResponsesAPIResponse +@pytest.mark.asyncio +async def test_retry_leaves_failed_encrypted_content_affinity_deployment( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_group: Final = "gpt-5.6" + router: Final = litellm.Router( + model_list=[ + { + "model_name": model_group, + "litellm_params": { + "model": "openai/gpt-5.6", + "api_base": f"https://region-{region}.example.invalid/v1", + "api_key": "sk-one-org", + }, + "model_info": {"id": f"region-{region}"}, + } + for region in range(1, 4) + ], + disable_cooldowns=True, + num_retries=1, + retry_after=0, + optional_pre_call_checks=["encrypted_content_affinity"], + ) + failed_route: Final = respx_mock.post("https://region-1.example.invalid/v1/responses").mock( + return_value=httpx.Response( + 500, + json={"error": {"message": "upstream failed", "type": "server_error", "code": "server_error"}}, + ) + ) + completed_response: Final[dict[str, object]] = { + "id": "resp_retry_peer", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "id": "msg_retry_peer", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + second_route: Final = respx_mock.post("https://region-2.example.invalid/v1/responses").mock( + return_value=httpx.Response(200, json=completed_response) + ) + third_route: Final = respx_mock.post("https://region-3.example.invalid/v1/responses").mock( + return_value=httpx.Response(200, json=completed_response) + ) + followup: Final[list[dict[str, object]]] = [ + { + "type": "reasoning", + "summary": [], + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + "gAAAA-opaque", "region-1" + ), + }, + {"role": "user", "content": "follow-up"}, + ] + + try: + await router.aresponses(model=model_group, input=followup, litellm_metadata={}) + + assert failed_route.call_count == 1 + assert second_route.call_count + third_route.call_count == 1 + finally: + router.discard() + + @pytest.fixture(autouse=True) def isolate_litellm_router_state(): saved = { diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 4a9dd199c08..0b38698f70c 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -20920,9 +20920,9 @@ def _retry_skip_deployment(deployment_id, host, litellm_params=None, model_info= (403, "rejecting", None, ("rejecting",)), (400, "second", ("first",), ("first", "second")), (400, "first", ("first",), ("first",)), - (429, "rejecting", None, ()), - (503, "rejecting", None, ()), - (408, "rejecting", None, ()), + (429, "rejecting", None, ("rejecting",)), + (503, "rejecting", None, ("rejecting",)), + (408, "rejecting", None, ("rejecting",)), (400, None, None, ()), (None, "rejecting", None, ()), ("400", "rejecting", None, ()),