mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
Merge fb66f8f553 into e04e89c701
This commit is contained in:
commit
b3c8104dc7
4 changed files with 133 additions and 8 deletions
|
|
@ -8856,8 +8856,6 @@ class Router:
|
|||
status_code: Final = getattr(exception, "status_code", None)
|
||||
if not failed_deployment_id or not isinstance(status_code, int):
|
||||
return ()
|
||||
if litellm.should_retry(status_code):
|
||||
return ()
|
||||
already_skipped_ids: Final = _as_retry_skipped_deployment_ids(already_skipped)
|
||||
skipped: Final = tuple(sorted(frozenset((*already_skipped_ids, failed_deployment_id))))
|
||||
verbose_router_logger.debug(
|
||||
|
|
|
|||
|
|
@ -369,6 +369,16 @@ class DeploymentAffinityCheck(CustomLogger):
|
|||
typed_healthy_deployments: Final = cast(list[dict], healthy_deployments)
|
||||
if request_kwargs.get("_target_order") is not None:
|
||||
return typed_healthy_deployments
|
||||
raw_retry_skipped_ids: Final[object] = cast( # cast-ok: request kwargs use string parameter names
|
||||
dict[str, object], request_kwargs
|
||||
).get("_retry_skipped_deployment_ids")
|
||||
retry_skipped_ids: Final[tuple[object, ...]] = (
|
||||
cast( # cast-ok: isinstance validates the tuple container before membership checks
|
||||
tuple[object, ...], raw_retry_skipped_ids
|
||||
)
|
||||
if isinstance(raw_retry_skipped_ids, tuple)
|
||||
else ()
|
||||
)
|
||||
|
||||
(
|
||||
enable_user_key,
|
||||
|
|
@ -386,7 +396,7 @@ class DeploymentAffinityCheck(CustomLogger):
|
|||
healthy_deployments=typed_healthy_deployments,
|
||||
model_id=responses_model_id,
|
||||
)
|
||||
if deployment is not None:
|
||||
if deployment is not None and responses_model_id not in retry_skipped_ids:
|
||||
verbose_router_logger.debug(
|
||||
"DeploymentAffinityCheck: previous_response_id pinning -> deployment=%s",
|
||||
responses_model_id,
|
||||
|
|
@ -431,7 +441,7 @@ class DeploymentAffinityCheck(CustomLogger):
|
|||
healthy_deployments=typed_healthy_deployments,
|
||||
model_id=session_model_id,
|
||||
)
|
||||
if session_deployment is not None:
|
||||
if session_deployment is not None and session_model_id not in retry_skipped_ids:
|
||||
verbose_router_logger.debug(
|
||||
"DeploymentAffinityCheck: session-id affinity hit -> deployment=%s session_id=%s",
|
||||
session_model_id,
|
||||
|
|
@ -468,7 +478,7 @@ class DeploymentAffinityCheck(CustomLogger):
|
|||
healthy_deployments=typed_healthy_deployments,
|
||||
model_id=model_id,
|
||||
)
|
||||
if deployment is None:
|
||||
if deployment is None or model_id in retry_skipped_ids:
|
||||
verbose_router_logger.debug(
|
||||
"DeploymentAffinityCheck: pinned deployment=%s not found in healthy_deployments",
|
||||
model_id,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ import json
|
|||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.caching.affinity_cache import claim_affinity_pin
|
||||
|
|
@ -26,6 +28,121 @@ class MockResponse:
|
|||
return self._json_data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_skips_failed_session_affinity_deployment(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "affinity-model",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.6",
|
||||
"api_key": "sk-fake",
|
||||
"api_base": "https://failed.local/v1",
|
||||
},
|
||||
"model_info": {"id": "failed"},
|
||||
},
|
||||
{
|
||||
"model_name": "affinity-model",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.6",
|
||||
"api_key": "sk-fake",
|
||||
"api_base": "https://healthy.local/v1",
|
||||
},
|
||||
"model_info": {"id": "healthy"},
|
||||
},
|
||||
],
|
||||
optional_pre_call_checks=["session_affinity"],
|
||||
num_retries=2,
|
||||
allowed_fails=1,
|
||||
)
|
||||
affinity: Final = next(
|
||||
callback for callback in router.optional_callbacks or () if isinstance(callback, DeploymentAffinityCheck)
|
||||
)
|
||||
session_id: Final = "pinned-session"
|
||||
await affinity.cache.async_set_cache(
|
||||
DeploymentAffinityCheck.get_session_affinity_cache_key(
|
||||
model_group="affinity-model", session_id=session_id, user_key=None
|
||||
),
|
||||
{"model_id": "failed"},
|
||||
)
|
||||
failed: Final = respx_mock.post("https://failed.local/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
500,
|
||||
json={"error": {"message": "deployment unavailable", "type": "server_error", "code": "internal_error"}},
|
||||
)
|
||||
)
|
||||
healthy: Final = respx_mock.post("https://healthy.local/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-affinity-retry",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-5.6",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
response: Final = await router.acompletion(
|
||||
model="affinity-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
metadata={"session_id": session_id},
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "ok"
|
||||
assert failed.call_count == 1
|
||||
assert healthy.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_with_only_session_affinity_deployment_stays_bounded(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "affinity-model",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.6",
|
||||
"api_key": "sk-fake",
|
||||
"api_base": "https://only.local/v1",
|
||||
},
|
||||
"model_info": {"id": "only"},
|
||||
}
|
||||
],
|
||||
optional_pre_call_checks=["session_affinity"],
|
||||
num_retries=2,
|
||||
disable_cooldowns=True,
|
||||
)
|
||||
only: Final = respx_mock.post("https://only.local/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
500,
|
||||
json={"error": {"message": "deployment unavailable", "type": "server_error", "code": "internal_error"}},
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.InternalServerError):
|
||||
await router.acompletion(
|
||||
model="affinity-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
metadata={"session_id": "pinned-session"},
|
||||
)
|
||||
|
||||
assert only.call_count == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_session_id_affinity_routes_to_same_deployment():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -20991,9 +20991,9 @@ def _retry_skip_deployment(deployment_id, host, litellm_params=None, model_info=
|
|||
(403, "rejecting", None, ("rejecting",)),
|
||||
(400, "second", ("first",), ("first", "second")),
|
||||
(400, "first", ("first",), ("first",)),
|
||||
(429, "rejecting", None, ()),
|
||||
(503, "rejecting", None, ()),
|
||||
(408, "rejecting", None, ()),
|
||||
(429, "rejecting", None, ("rejecting",)),
|
||||
(503, "rejecting", None, ("rejecting",)),
|
||||
(408, "rejecting", None, ("rejecting",)),
|
||||
(400, None, None, ()),
|
||||
(None, "rejecting", None, ()),
|
||||
("400", "rejecting", None, ()),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue