This commit is contained in:
Aditya Singhal 2026-10-27 02:25:10 +00:00 • committed by GitHub
commit c26197d44f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 112 additions and 8 deletions

View file

@ -8917,8 +8917,6 @@ class Router:
status_code: Final = getattr(exception, "status_code", None)
if not failed_deployment_id or not isinstance(status_code, int):
return ()
if litellm.should_retry(status_code):
return ()
already_skipped_ids: Final = _as_retry_skipped_deployment_ids(already_skipped)
skipped: Final = tuple(sorted(frozenset((*already_skipped_ids, failed_deployment_id))))
verbose_router_logger.debug(

View file

@ -40,7 +40,7 @@ from collections.abc import Iterator, Mapping, Sequence
from functools import cache
from typing import TYPE_CHECKING, Final, Optional, cast
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_router_logger
from litellm.integrations.custom_logger import CustomLogger, Span
@ -59,6 +59,7 @@ if TYPE_CHECKING:
from litellm.router import Router
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_RETRY_SKIPPED_DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...])
class EncryptedContentAffinityCheck(CustomLogger):
@ -88,6 +89,22 @@ class EncryptedContentAffinityCheck(CustomLogger):
# Helpers
# ------------------------------------------------------------------
@staticmethod
def _retry_skipped_deployment_ids(value: object) -> frozenset[str]:
try:
return frozenset(_RETRY_SKIPPED_DEPLOYMENT_IDS_ADAPTER.validate_python(value))
except ValidationError:
return frozenset()
@staticmethod
def _deployment_id(deployment: Mapping[str, object]) -> str | None:
try:
model_info: Final = _REQUEST_KWARGS_ADAPTER.validate_python(deployment.get("model_info"))
except ValidationError:
return None
model_id: Final = model_info.get("id")
return model_id if isinstance(model_id, str) else None
@staticmethod
def has_model_group_affinity_enabled(
model_group_affinity_config: dict[str, list[str]] | None,
@ -345,12 +362,25 @@ class EncryptedContentAffinityCheck(CustomLogger):
typed_healthy_deployments: Final = cast(list[dict[str, object]], healthy_deployments)
if not self._is_enabled_for_model_group(model):
return typed_healthy_deployments
target_order: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs).get("_target_order")
candidates: Final = (
validated_request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs)
target_order: Final = validated_request_kwargs.get("_target_order")
order_candidates: Final = (
get_order_filtered_deployments(typed_healthy_deployments, target_order=target_order)
if isinstance(target_order, int)
else typed_healthy_deployments
)
retry_skipped_value: Final = validated_request_kwargs.get("_retry_skipped_deployment_ids")
retry_skipped_deployment_ids: Final = self._retry_skipped_deployment_ids(retry_skipped_value)
untried_candidates: Final = (
[
deployment
for deployment in order_candidates
if self._deployment_id(deployment) not in retry_skipped_deployment_ids
]
if retry_skipped_deployment_ids
else order_candidates
)
candidates: Final = untried_candidates or order_candidates
# Signal to the response post-processor that encrypted item IDs should be
# encoded in the output of this request. Only set the flag when

View file

@ -16,9 +16,12 @@ The mechanism works without any cache and supports two encoding strategies:
"""
import time
from typing import Final
from unittest.mock import AsyncMock, patch
import httpx
import pytest
import respx
import litellm
from litellm.models.credentials import CredentialItem
@ -26,6 +29,79 @@ from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import ResponsesAPIResponse
@pytest.mark.asyncio
async def test_retry_leaves_failed_encrypted_content_affinity_deployment(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
model_group: Final = "gpt-5.6"
router: Final = litellm.Router(
model_list=[
{
"model_name": model_group,
"litellm_params": {
"model": "openai/gpt-5.6",
"api_base": f"https://region-{region}.example.invalid/v1",
"api_key": "sk-one-org",
},
"model_info": {"id": f"region-{region}"},
}
for region in range(1, 4)
],
disable_cooldowns=True,
num_retries=1,
retry_after=0,
optional_pre_call_checks=["encrypted_content_affinity"],
)
failed_route: Final = respx_mock.post("https://region-1.example.invalid/v1/responses").mock(
return_value=httpx.Response(
500,
json={"error": {"message": "upstream failed", "type": "server_error", "code": "server_error"}},
)
)
completed_response: Final[dict[str, object]] = {
"id": "resp_retry_peer",
"object": "response",
"created_at": 0,
"status": "completed",
"model": "gpt-5.6",
"output": [
{
"id": "msg_retry_peer",
"type": "message",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
}
],
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
}
second_route: Final = respx_mock.post("https://region-2.example.invalid/v1/responses").mock(
return_value=httpx.Response(200, json=completed_response)
)
third_route: Final = respx_mock.post("https://region-3.example.invalid/v1/responses").mock(
return_value=httpx.Response(200, json=completed_response)
)
followup: Final[list[dict[str, object]]] = [
{
"type": "reasoning",
"summary": [],
"encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id(
"gAAAA-opaque", "region-1"
),
},
{"role": "user", "content": "follow-up"},
]
try:
await router.aresponses(model=model_group, input=followup, litellm_metadata={})
assert failed_route.call_count == 1
assert second_route.call_count + third_route.call_count == 1
finally:
router.discard()
@pytest.fixture(autouse=True)
def isolate_litellm_router_state():
saved = {

View file

@ -21169,9 +21169,9 @@ def _retry_skip_deployment(deployment_id, host, litellm_params=None, model_info=
(403, "rejecting", None, ("rejecting",)),
(400, "second", ("first",), ("first", "second")),
(400, "first", ("first",), ("first",)),
(429, "rejecting", None, ()),
(503, "rejecting", None, ()),
(408, "rejecting", None, ()),
(429, "rejecting", None, ("rejecting",)),
(503, "rejecting", None, ("rejecting",)),
(408, "rejecting", None, ("rejecting",)),
(400, None, None, ()),
(None, "rejecting", None, ()),
("400", "rejecting", None, ()),