mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 93b9812980 into e16d79d635
This commit is contained in:
commit
c26197d44f
4 changed files with 112 additions and 8 deletions
|
|
@ -8917,8 +8917,6 @@ class Router:
|
|||
status_code: Final = getattr(exception, "status_code", None)
|
||||
if not failed_deployment_id or not isinstance(status_code, int):
|
||||
return ()
|
||||
if litellm.should_retry(status_code):
|
||||
return ()
|
||||
already_skipped_ids: Final = _as_retry_skipped_deployment_ids(already_skipped)
|
||||
skipped: Final = tuple(sorted(frozenset((*already_skipped_ids, failed_deployment_id))))
|
||||
verbose_router_logger.debug(
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ from collections.abc import Iterator, Mapping, Sequence
|
|||
from functools import cache
|
||||
from typing import TYPE_CHECKING, Final, Optional, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger, Span
|
||||
|
|
@ -59,6 +59,7 @@ if TYPE_CHECKING:
|
|||
from litellm.router import Router
|
||||
|
||||
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_RETRY_SKIPPED_DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
class EncryptedContentAffinityCheck(CustomLogger):
|
||||
|
|
@ -88,6 +89,22 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
# Helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _retry_skipped_deployment_ids(value: object) -> frozenset[str]:
|
||||
try:
|
||||
return frozenset(_RETRY_SKIPPED_DEPLOYMENT_IDS_ADAPTER.validate_python(value))
|
||||
except ValidationError:
|
||||
return frozenset()
|
||||
|
||||
@staticmethod
|
||||
def _deployment_id(deployment: Mapping[str, object]) -> str | None:
|
||||
try:
|
||||
model_info: Final = _REQUEST_KWARGS_ADAPTER.validate_python(deployment.get("model_info"))
|
||||
except ValidationError:
|
||||
return None
|
||||
model_id: Final = model_info.get("id")
|
||||
return model_id if isinstance(model_id, str) else None
|
||||
|
||||
@staticmethod
|
||||
def has_model_group_affinity_enabled(
|
||||
model_group_affinity_config: dict[str, list[str]] | None,
|
||||
|
|
@ -345,12 +362,25 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
typed_healthy_deployments: Final = cast(list[dict[str, object]], healthy_deployments)
|
||||
if not self._is_enabled_for_model_group(model):
|
||||
return typed_healthy_deployments
|
||||
target_order: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs).get("_target_order")
|
||||
candidates: Final = (
|
||||
validated_request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs)
|
||||
target_order: Final = validated_request_kwargs.get("_target_order")
|
||||
order_candidates: Final = (
|
||||
get_order_filtered_deployments(typed_healthy_deployments, target_order=target_order)
|
||||
if isinstance(target_order, int)
|
||||
else typed_healthy_deployments
|
||||
)
|
||||
retry_skipped_value: Final = validated_request_kwargs.get("_retry_skipped_deployment_ids")
|
||||
retry_skipped_deployment_ids: Final = self._retry_skipped_deployment_ids(retry_skipped_value)
|
||||
untried_candidates: Final = (
|
||||
[
|
||||
deployment
|
||||
for deployment in order_candidates
|
||||
if self._deployment_id(deployment) not in retry_skipped_deployment_ids
|
||||
]
|
||||
if retry_skipped_deployment_ids
|
||||
else order_candidates
|
||||
)
|
||||
candidates: Final = untried_candidates or order_candidates
|
||||
|
||||
# Signal to the response post-processor that encrypted item IDs should be
|
||||
# encoded in the output of this request. Only set the flag when
|
||||
|
|
|
|||
|
|
@ -16,9 +16,12 @@ The mechanism works without any cache and supports two encoding strategies:
|
|||
"""
|
||||
|
||||
import time
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.models.credentials import CredentialItem
|
||||
|
|
@ -26,6 +29,79 @@ from litellm.responses.utils import ResponsesAPIRequestUtils
|
|||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_leaves_failed_encrypted_content_affinity_deployment(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
model_group: Final = "gpt-5.6"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": model_group,
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.6",
|
||||
"api_base": f"https://region-{region}.example.invalid/v1",
|
||||
"api_key": "sk-one-org",
|
||||
},
|
||||
"model_info": {"id": f"region-{region}"},
|
||||
}
|
||||
for region in range(1, 4)
|
||||
],
|
||||
disable_cooldowns=True,
|
||||
num_retries=1,
|
||||
retry_after=0,
|
||||
optional_pre_call_checks=["encrypted_content_affinity"],
|
||||
)
|
||||
failed_route: Final = respx_mock.post("https://region-1.example.invalid/v1/responses").mock(
|
||||
return_value=httpx.Response(
|
||||
500,
|
||||
json={"error": {"message": "upstream failed", "type": "server_error", "code": "server_error"}},
|
||||
)
|
||||
)
|
||||
completed_response: Final[dict[str, object]] = {
|
||||
"id": "resp_retry_peer",
|
||||
"object": "response",
|
||||
"created_at": 0,
|
||||
"status": "completed",
|
||||
"model": "gpt-5.6",
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_retry_peer",
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
second_route: Final = respx_mock.post("https://region-2.example.invalid/v1/responses").mock(
|
||||
return_value=httpx.Response(200, json=completed_response)
|
||||
)
|
||||
third_route: Final = respx_mock.post("https://region-3.example.invalid/v1/responses").mock(
|
||||
return_value=httpx.Response(200, json=completed_response)
|
||||
)
|
||||
followup: Final[list[dict[str, object]]] = [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"summary": [],
|
||||
"encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id(
|
||||
"gAAAA-opaque", "region-1"
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": "follow-up"},
|
||||
]
|
||||
|
||||
try:
|
||||
await router.aresponses(model=model_group, input=followup, litellm_metadata={})
|
||||
|
||||
assert failed_route.call_count == 1
|
||||
assert second_route.call_count + third_route.call_count == 1
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolate_litellm_router_state():
|
||||
saved = {
|
||||
|
|
|
|||
|
|
@ -21169,9 +21169,9 @@ def _retry_skip_deployment(deployment_id, host, litellm_params=None, model_info=
|
|||
(403, "rejecting", None, ("rejecting",)),
|
||||
(400, "second", ("first",), ("first", "second")),
|
||||
(400, "first", ("first",), ("first",)),
|
||||
(429, "rejecting", None, ()),
|
||||
(503, "rejecting", None, ()),
|
||||
(408, "rejecting", None, ()),
|
||||
(429, "rejecting", None, ("rejecting",)),
|
||||
(503, "rejecting", None, ("rejecting",)),
|
||||
(408, "rejecting", None, ("rejecting",)),
|
||||
(400, None, None, ()),
|
||||
(None, "rejecting", None, ()),
|
||||
("400", "rejecting", None, ()),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue