This commit is contained in:
hahahahahayesyeseys 2026-10-24 07:26:12 +00:00 • committed by GitHub
commit b3c8104dc7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 133 additions and 8 deletions

View file

@ -8856,8 +8856,6 @@ class Router:
status_code: Final = getattr(exception, "status_code", None)
if not failed_deployment_id or not isinstance(status_code, int):
return ()
if litellm.should_retry(status_code):
return ()
already_skipped_ids: Final = _as_retry_skipped_deployment_ids(already_skipped)
skipped: Final = tuple(sorted(frozenset((*already_skipped_ids, failed_deployment_id))))
verbose_router_logger.debug(

View file

@ -369,6 +369,16 @@ class DeploymentAffinityCheck(CustomLogger):
typed_healthy_deployments: Final = cast(list[dict], healthy_deployments)
if request_kwargs.get("_target_order") is not None:
return typed_healthy_deployments
raw_retry_skipped_ids: Final[object] = cast( # cast-ok: request kwargs use string parameter names
dict[str, object], request_kwargs
).get("_retry_skipped_deployment_ids")
retry_skipped_ids: Final[tuple[object, ...]] = (
cast( # cast-ok: isinstance validates the tuple container before membership checks
tuple[object, ...], raw_retry_skipped_ids
)
if isinstance(raw_retry_skipped_ids, tuple)
else ()
)
(
enable_user_key,
@ -386,7 +396,7 @@ class DeploymentAffinityCheck(CustomLogger):
healthy_deployments=typed_healthy_deployments,
model_id=responses_model_id,
)
if deployment is not None:
if deployment is not None and responses_model_id not in retry_skipped_ids:
verbose_router_logger.debug(
"DeploymentAffinityCheck: previous_response_id pinning -> deployment=%s",
responses_model_id,
@ -431,7 +441,7 @@ class DeploymentAffinityCheck(CustomLogger):
healthy_deployments=typed_healthy_deployments,
model_id=session_model_id,
)
if session_deployment is not None:
if session_deployment is not None and session_model_id not in retry_skipped_ids:
verbose_router_logger.debug(
"DeploymentAffinityCheck: session-id affinity hit -> deployment=%s session_id=%s",
session_model_id,
@ -468,7 +478,7 @@ class DeploymentAffinityCheck(CustomLogger):
healthy_deployments=typed_healthy_deployments,
model_id=model_id,
)
if deployment is None:
if deployment is None or model_id in retry_skipped_ids:
verbose_router_logger.debug(
"DeploymentAffinityCheck: pinned deployment=%s not found in healthy_deployments",
model_id,

View file

@ -3,7 +3,9 @@ import json
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
import litellm
from litellm.caching.affinity_cache import claim_affinity_pin
@ -26,6 +28,121 @@ class MockResponse:
return self._json_data
@pytest.mark.asyncio
async def test_retry_skips_failed_session_affinity_deployment(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router: Final = litellm.Router(
model_list=[
{
"model_name": "affinity-model",
"litellm_params": {
"model": "openai/gpt-5.6",
"api_key": "sk-fake",
"api_base": "https://failed.local/v1",
},
"model_info": {"id": "failed"},
},
{
"model_name": "affinity-model",
"litellm_params": {
"model": "openai/gpt-5.6",
"api_key": "sk-fake",
"api_base": "https://healthy.local/v1",
},
"model_info": {"id": "healthy"},
},
],
optional_pre_call_checks=["session_affinity"],
num_retries=2,
allowed_fails=1,
)
affinity: Final = next(
callback for callback in router.optional_callbacks or () if isinstance(callback, DeploymentAffinityCheck)
)
session_id: Final = "pinned-session"
await affinity.cache.async_set_cache(
DeploymentAffinityCheck.get_session_affinity_cache_key(
model_group="affinity-model", session_id=session_id, user_key=None
),
{"model_id": "failed"},
)
failed: Final = respx_mock.post("https://failed.local/v1/chat/completions").mock(
return_value=httpx.Response(
500,
json={"error": {"message": "deployment unavailable", "type": "server_error", "code": "internal_error"}},
)
)
healthy: Final = respx_mock.post("https://healthy.local/v1/chat/completions").mock(
return_value=httpx.Response(
200,
json={
"id": "chatcmpl-affinity-retry",
"object": "chat.completion",
"created": 1,
"model": "gpt-5.6",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
)
)
response: Final = await router.acompletion(
model="affinity-model",
messages=[{"role": "user", "content": "hi"}],
metadata={"session_id": session_id},
)
assert response.choices[0].message.content == "ok"
assert failed.call_count == 1
assert healthy.call_count == 1
@pytest.mark.asyncio
async def test_retry_with_only_session_affinity_deployment_stays_bounded(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router: Final = litellm.Router(
model_list=[
{
"model_name": "affinity-model",
"litellm_params": {
"model": "openai/gpt-5.6",
"api_key": "sk-fake",
"api_base": "https://only.local/v1",
},
"model_info": {"id": "only"},
}
],
optional_pre_call_checks=["session_affinity"],
num_retries=2,
disable_cooldowns=True,
)
only: Final = respx_mock.post("https://only.local/v1/chat/completions").mock(
return_value=httpx.Response(
500,
json={"error": {"message": "deployment unavailable", "type": "server_error", "code": "internal_error"}},
)
)
with pytest.raises(litellm.InternalServerError):
await router.acompletion(
model="affinity-model",
messages=[{"role": "user", "content": "hi"}],
metadata={"session_id": "pinned-session"},
)
assert only.call_count == 3
@pytest.mark.asyncio
async def test_async_session_id_affinity_routes_to_same_deployment():
"""

View file

@ -20991,9 +20991,9 @@ def _retry_skip_deployment(deployment_id, host, litellm_params=None, model_info=
(403, "rejecting", None, ("rejecting",)),
(400, "second", ("first",), ("first", "second")),
(400, "first", ("first",), ("first",)),
(429, "rejecting", None, ()),
(503, "rejecting", None, ()),
(408, "rejecting", None, ()),
(429, "rejecting", None, ("rejecting",)),
(503, "rejecting", None, ("rejecting",)),
(408, "rejecting", None, ("rejecting",)),
(400, None, None, ()),
(None, "rejecting", None, ()),
("400", "rejecting", None, ()),