fix(router): isolate session affinity retry state

This commit is contained in:
ishaan-j-berri 2026-10-23 18:10:52 -07:00
parent ef25a34428
commit 71fad367b4
7 changed files with 153 additions and 99 deletions

View file

@ -312,6 +312,7 @@ from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
)
from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
DeploymentAffinityCheck,
DeploymentAffinityRetryState,
warn_on_unknown_model_group_affinity_flags,
)
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
@ -8893,11 +8894,10 @@ class Router:
verbose_router_logger.debug(
"async function w/ retries: original_function - %s, num_retries - %s", original_function, num_retries
)
affinity_retry_state: Final = DeploymentAffinityRetryState()
kwargs["_deployment_affinity_retry_state"] = affinity_retry_state # rebind-ok: kwargs is local to this loop
## ADD RETRY TRACKING TO METADATA - used for spend logs retry tracking
retry_metadata_values: Final[tuple[object, object]] = (kwargs.get("metadata"), kwargs.get("litellm_metadata"))
for metadata_value in retry_metadata_values:
if isinstance(metadata_value, dict):
metadata_value.pop("retry_failed_deployment_ids", None)
_metadata["attempted_retries"] = 0
_metadata["max_retries"] = num_retries # Updated after overrides in exception handler
try:
@ -8911,6 +8911,7 @@ class Router:
raise
current_attempt = None
original_exception = e
affinity_retry_state.record_failure(e)
deployment_num_retries: Final = getattr(e, "num_retries", None)
if (
@ -9012,6 +9013,7 @@ class Router:
# Always track the latest error so we raise the most
# recent exception instead of the first one.
original_exception = e
affinity_retry_state.record_failure(e)
## LOGGING
kwargs = self.log_retry(kwargs=kwargs, e=e)
@ -9633,12 +9635,10 @@ class Router:
from litellm.types.router import RetryAttemptRecord
_metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
request_metadata: Final = cast( # cast-ok: router-owned metadata dict
"dict[str, object]", kwargs[_metadata_var]
)
request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs
model_group: Final = kwargs.get("model")
model_info: Final = request_metadata.get("model_info")
deployment_id: Final[object] = model_info.get("id") if isinstance(model_info, Mapping) else None
deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None
base_model: Final = get_base_model_from_litellm_call_metadata(
metadata=request_metadata
) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None)
@ -9660,16 +9660,6 @@ class Router:
else ()
)
breadcrumbs: Final = (*kept_breadcrumbs, attempt_record)
failed_deployment_id: Final = (
getattr(e, "retry_skip_deployment_id", None) or getattr(e, "failed_deployment_id", None) or deployment_id
)
if isinstance(failed_deployment_id, str):
earlier_failed_ids: Final = _as_retry_skipped_deployment_ids(
request_metadata.get("retry_failed_deployment_ids")
)
request_metadata["retry_failed_deployment_ids"] = tuple(
sorted(frozenset((*earlier_failed_ids, failed_deployment_id)))
)
earlier: Final = request_metadata.get("request_retry_count")
request_retry_count: Final = (earlier if type(earlier) is int and 0 <= earlier else 0) + 1
decision: Final = request_metadata.get("routing_decision")
@ -9685,8 +9675,8 @@ class Router:
**({"litellm.deployment.id": deployment_id} if isinstance(deployment_id, str) else {}),
},
)
request_metadata["previous_models"] = breadcrumbs
request_metadata["request_retry_count"] = request_retry_count
kwargs[_metadata_var]["previous_models"] = breadcrumbs # rebind-ok: the logging object already holds this dict
kwargs[_metadata_var]["request_retry_count"] = request_retry_count # rebind-ok: same dict, read by the cap
return kwargs
def _deployment_params_base_model(self, deployment_id: str | None) -> str | None:

View file

@ -14,10 +14,12 @@ where routing to a consistent deployment is still beneficial.
import hashlib
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import Final, cast
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.affinity_cache import (
@ -66,6 +68,18 @@ def warn_on_unknown_model_group_affinity_flags(model_group_affinity_config: Mapp
)
@dataclass(slots=True)
class DeploymentAffinityRetryState:
failed_deployment_ids: frozenset[str] = frozenset()
def record_failure(self, exception: Exception) -> None:
deployment_id: Final[object] = getattr(exception, "retry_skip_deployment_id", None) or getattr(
exception, "failed_deployment_id", None
)
if isinstance(deployment_id, str) and deployment_id:
self.failed_deployment_ids = self.failed_deployment_ids | frozenset((deployment_id,))
class DeploymentAffinityCheck(CustomLogger):
"""
Router deployment affinity callback.
@ -341,26 +355,12 @@ class DeploymentAffinityCheck(CustomLogger):
return self._pinned_model_id(winner)
@staticmethod
def _failed_deployment_ids(metadata: Mapping[object, object]) -> frozenset[str]:
failed_ids: Final = metadata.get("retry_failed_deployment_ids")
if not isinstance(failed_ids, tuple):
return frozenset()
values: Final = cast(tuple[object, ...], failed_ids) # cast-ok: checked tuple, opaque elements
return frozenset(deployment_id for deployment_id in values if isinstance(deployment_id, str))
@staticmethod
def _deployment_model_id(deployment: Mapping[str, object]) -> object:
model_info: Final = deployment.get("model_info")
if not isinstance(model_info, dict):
return None
return cast(Mapping[object, object], model_info).get("id") # cast-ok: dict checked; values stay opaque
@classmethod
def _find_deployment_by_model_id(
cls, healthy_deployments: Sequence[dict[str, object]], model_id: str
) -> dict[str, object] | None:
def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None:
for deployment in healthy_deployments:
deployment_model_id = cls._deployment_model_id(deployment)
model_info = deployment.get("model_info")
if not isinstance(model_info, dict):
continue
deployment_model_id = model_info.get("id")
if deployment_model_id is not None and str(deployment_model_id) == str(model_id):
return deployment
return None
@ -380,7 +380,7 @@ class DeploymentAffinityCheck(CustomLogger):
2. cached API-key deployment affinity
"""
request_kwargs = request_kwargs or {}
typed_healthy_deployments: Final = cast(list[dict[str, object]], healthy_deployments)
typed_healthy_deployments: Final = cast(list[dict], healthy_deployments)
if request_kwargs.get("_target_order") is not None:
return typed_healthy_deployments
@ -424,22 +424,25 @@ class DeploymentAffinityCheck(CustomLogger):
if (session_affinity_active or enable_user_key)
else None
)
session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs=request_kwargs)
session_id: Final = (
self._get_session_id_from_request_kwargs(request_kwargs=request_kwargs) if session_affinity_active else None
)
retry_state: Final[object] = request_kwargs.get("_deployment_affinity_retry_state")
failed_deployment_ids: Final = (
frozenset[str]().union(
*(self._failed_deployment_ids(metadata) for metadata in self._iter_metadata_dicts(request_kwargs))
)
if (session_affinity_active and session_id is not None) or (enable_user_key and user_key is not None)
retry_state.failed_deployment_ids
if isinstance(retry_state, DeploymentAffinityRetryState)
and (session_id is not None or (enable_user_key and user_key is not None))
else frozenset[str]()
)
affinity_deployments: Final = [
deployment
for deployment in typed_healthy_deployments
if self._deployment_model_id(deployment) not in failed_deployment_ids
] or typed_healthy_deployments
affinity_deployments: Final = cast( # cast-ok: filtering preserves the router deployment dictionaries
list[dict[str, object]],
litellm.utils.get_excluded_filtered_deployments(typed_healthy_deployments, failed_deployment_ids)
or typed_healthy_deployments,
)
# 2) Session-id -> deployment affinity
if session_affinity_active and session_id is not None:
if session_id is not None:
session_cache_key: Final = self.get_session_affinity_cache_key(
model_group=stable_model_map_key, session_id=session_id, user_key=user_key
)

View file

@ -19,6 +19,7 @@ if TYPE_CHECKING:
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.router_strategy.complexity_router.context_compaction import CompactionState
from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets
from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityRetryState
from litellm.types.caching import DynamicCacheControl
from litellm.types.llms.openai import ChatCompletionAssistantMessage, ChatCompletionUserMessage
from litellm.types.proxy.litellm_pre_call_utils import SecretFields
@ -392,6 +393,9 @@ class RouterState:
fallback_depth: int | None = None
max_fallbacks: int | None = None
attempted_targets: "AttemptedFallbackTargets | None" = None
deployment_affinity_retry_state: "DeploymentAffinityRetryState | None" = field(
default=None, metadata=wire("_deployment_affinity_retry_state")
)
@dataclass(frozen=True, slots=True, kw_only=True)

View file

@ -1726,10 +1726,8 @@ async def async_post_call_failure_deployment_hook(
Callbacks receive a same-class snapshot of ``exception``, not the live
object that's about to be re-raised, so a callback setting an attribute
on it (e.g. ``status_code``) cannot change what the real caller sees.
``request_data`` omits ``attempted_targets``: unlike the rest of this
attempt's own kwargs, it's the *same* object shared by reference across
every hop of the live fallback walk, so a callback calling ``.record()``
on it would make the router skip a deployment it hasn't actually tried.
``request_data`` omits live routing state shared across retries and fallbacks.
Callbacks must not be able to mark an untried deployment as failed.
"""
try:
typed_call_type = CallTypes(call_type)
@ -1738,7 +1736,9 @@ async def async_post_call_failure_deployment_hook(
_raw_fallback_depth: Final = request_data.get("fallback_depth")
fallback_depth: Final = _raw_fallback_depth if isinstance(_raw_fallback_depth, int) else None
safe_request_data: Final = MappingProxyType({k: v for k, v in request_data.items() if k != "attempted_targets"})
safe_request_data: Final = MappingProxyType(
{k: v for k, v in request_data.items() if k not in ("attempted_targets", "_deployment_affinity_retry_state")}
)
safe_exception: Final = _snapshot_exception_for_hook(exception)
custom_logger_class: Final = _get_cached_custom_logger()

View file

@ -1,7 +1,7 @@
import asyncio
import json
from collections.abc import Mapping
from typing import Final
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import httpx
@ -15,6 +15,7 @@ from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESS
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
DeploymentAffinityCheck,
DeploymentAffinityRetryState,
)
@ -29,7 +30,14 @@ class MockResponse:
return self._json_data
def _session_retry_router(deployment_ids: tuple[str, ...], allowed_fails: int) -> litellm.Router:
def _session_retry_router(
deployment_ids: tuple[str, ...],
allowed_fails: int,
checks: tuple[Literal["session_affinity", "deployment_affinity", "responses_api_deployment_check"], ...] | None = (
"session_affinity",
),
group_checks: tuple[str, ...] | None = None,
) -> litellm.Router:
return litellm.Router(
model_list=[
{
@ -38,12 +46,14 @@ def _session_retry_router(deployment_ids: tuple[str, ...], allowed_fails: int) -
"model": "openai/affinity-test-model",
"api_key": "test-key",
"api_base": f"https://{deployment_id}.test/v1",
"weight": 1 if deployment_id == "b1" else 0,
},
"model_info": {"id": deployment_id},
}
for deployment_id in deployment_ids
],
optional_pre_call_checks=["session_affinity"],
optional_pre_call_checks=list(checks) if checks is not None else None,
model_group_affinity_config={"session-retry-model": list(group_checks)} if group_checks is not None else None,
num_retries=2,
allowed_fails=allowed_fails,
retry_after=0,
@ -140,6 +150,10 @@ async def test_session_affinity_retry_exclusion_does_not_leak_to_next_request(
statuses: Final = iter((500, 200, 200))
upstream: Final = Mock(side_effect=lambda request: _session_retry_response(request, next(statuses)))
metadata: Final = {"session_id": "retry-session"}
request_kwargs: Final = {
"metadata": metadata,
**({"litellm_metadata": {"session_id": "retry-session"}} if both_metadata_buckets else {}),
}
async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as client:
monkeypatch.setattr(litellm, "aclient_session", client)
router: Final = _session_retry_router(("b1", "b2"), allowed_fails=100)
@ -148,13 +162,12 @@ async def test_session_affinity_retry_exclusion_does_not_leak_to_next_request(
recovered: Final = await router.acompletion(
model="session-retry-model",
messages=[{"role": "user", "content": "Hello"}],
metadata=metadata,
**request_kwargs,
)
next_request: Final = await router.acompletion(
model="session-retry-model",
messages=[{"role": "user", "content": "Hello again"}],
metadata={**metadata, "retry_failed_deployment_ids": ("b1",)} if both_metadata_buckets else metadata,
**({"litellm_metadata": {"session_id": "retry-session"}} if both_metadata_buckets else {}),
**request_kwargs,
)
assert recovered.choices[0].message.content == "b2.test"
@ -189,41 +202,49 @@ async def test_session_affinity_remembers_failed_backends_beyond_retry_log_limit
@pytest.mark.asyncio
@pytest.mark.parametrize(
("enable_session", "group_checks", "metadata", "expected_ids"),
("checks", "group_checks", "metadata"),
[
(True, None, {"session_id": "retry-session"}, ("b2",)),
(False, ("session_affinity",), {"session_id": "retry-session"}, ("b2",)),
(
False,
None,
{"session_id": "retry-session", SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: 600},
("b2",),
),
(True, None, {}, ("b1", "b2")),
(False, None, {"session_id": "retry-session"}, ("b1", "b2")),
(
True,
None,
{"session_id": "retry-session", SESSION_ID_GENERATED_METADATA_KEY: True},
("b1", "b2"),
),
(None, ("session_affinity",), {"session_id": "retry-session"}),
(("deployment_affinity",), None, {"user_api_key_hash": "test-user-key"}),
],
ids=["global", "per-group", "auto-router-marker", "absent-session", "disabled-affinity", "generated-session"],
ids=["per-group-session", "api-key"],
)
async def test_retry_filter_requires_active_session_affinity(
enable_session: bool,
async def test_configured_affinity_retries_another_backend(
checks: tuple[Literal["session_affinity", "deployment_affinity", "responses_api_deployment_check"], ...] | None,
group_checks: tuple[str, ...] | None,
metadata: Mapping[str, object],
expected_ids: tuple[str, ...],
monkeypatch: pytest.MonkeyPatch,
) -> None:
upstream: Final = Mock(
side_effect=lambda request: _session_retry_response(request, 500 if request.url.host == "b1.test" else 200)
)
async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as client:
monkeypatch.setattr(litellm, "aclient_session", client)
router: Final = _session_retry_router(("b1", "b2"), allowed_fails=100, checks=checks, group_checks=group_checks)
await _pin_session_retry_router(router)
await router.cache.async_set_cache(
key=DeploymentAffinityCheck.get_affinity_cache_key("session-retry-model", "test-user-key"),
value={"model_id": "b1"},
)
response: Final = await router.acompletion(
model="session-retry-model",
messages=[{"role": "user", "content": "Hello"}],
metadata=dict(metadata),
)
assert response.choices[0].message.content == "b2.test"
assert tuple(call.args[0].url.host for call in upstream.call_args_list) == ("b1.test", "b2.test")
@pytest.mark.asyncio
async def test_auto_router_marker_skips_a_failed_pinned_backend() -> None:
cache: Final = DualCache()
callback: Final = DeploymentAffinityCheck(
cache=cache,
ttl_seconds=600,
enable_user_key_affinity=False,
enable_responses_api_affinity=False,
enable_session_id_affinity=enable_session,
model_group_affinity_config={"session-retry-model": list(group_checks)} if group_checks is not None else None,
)
deployments: Final = [
{"model_name": "session-retry-model", "model_info": {"id": deployment_id}} for deployment_id in ("b1", "b2")
@ -237,10 +258,47 @@ async def test_retry_filter_requires_active_session_affinity(
model="session-retry-model",
healthy_deployments=deployments,
messages=[],
request_kwargs={"litellm_metadata": {**metadata, "retry_failed_deployment_ids": ("b1",)}},
request_kwargs={
"_deployment_affinity_retry_state": DeploymentAffinityRetryState(frozenset({"b1"})),
"litellm_metadata": {
"session_id": "retry-session",
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: 600,
},
},
)
assert tuple(deployment["model_info"]["id"] for deployment in available) == expected_ids
assert available == [deployments[1]]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("checks", "metadata"),
[
(None, {"session_id": "retry-session"}),
(("session_affinity",), {}),
(("session_affinity",), {"session_id": "retry-session", SESSION_ID_GENERATED_METADATA_KEY: True}),
],
ids=["default-disabled", "absent-session", "generated-session"],
)
async def test_inactive_affinity_preserves_configured_backend_weights_on_retry(
checks: tuple[Literal["session_affinity", "deployment_affinity", "responses_api_deployment_check"], ...] | None,
metadata: Mapping[str, object],
monkeypatch: pytest.MonkeyPatch,
) -> None:
upstream: Final = Mock(side_effect=lambda request: _session_retry_response(request, 500))
async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as client:
monkeypatch.setattr(litellm, "aclient_session", client)
router: Final = _session_retry_router(("b1", "b2"), allowed_fails=100, checks=checks)
await _pin_session_retry_router(router)
with pytest.raises(litellm.InternalServerError, match="Unavailable backend"):
await router.acompletion(
model="session-retry-model",
messages=[{"role": "user", "content": "Hello"}],
metadata=dict(metadata),
)
assert tuple(call.args[0].url.host for call in upstream.call_args_list) == ("b1.test", "b1.test", "b1.test")
@pytest.mark.asyncio
@ -265,7 +323,8 @@ async def test_response_continuity_keeps_its_backend_after_a_session_affinity_fa
messages=[],
request_kwargs={
"previous_response_id": previous_response_id,
"litellm_metadata": {"session_id": "retry-session", "retry_failed_deployment_ids": ("b1",)},
"_deployment_affinity_retry_state": DeploymentAffinityRetryState(frozenset({"b1"})),
"litellm_metadata": {"session_id": "retry-session"},
},
)

View file

@ -6525,25 +6525,20 @@ async def test_wrapper_async_failure_hook_exception_mutation_does_not_change_rai
@pytest.mark.asyncio
async def test_async_post_call_failure_deployment_hook_omits_attempted_targets_from_request_data(
monkeypatch: pytest.MonkeyPatch,
@pytest.mark.parametrize("private_field", ("attempted_targets", "_deployment_affinity_retry_state"))
async def test_async_post_call_failure_deployment_hook_omits_live_routing_state_from_request_data(
monkeypatch: pytest.MonkeyPatch, private_field: str
) -> None:
"""Regression: attempted_targets is the router's own live fallback-walk bookkeeping,
shared by reference across every hop of a single request - unlike the rest of
request_data, it is not this attempt's own isolated copy. A callback calling .record()
on it would make the router skip a deployment it hasn't actually tried, so the
dispatcher must never hand it to a callback."""
recorder = _RecordingDeploymentFailureLogger()
recorder: Final = _RecordingDeploymentFailureLogger()
monkeypatch.setattr(litellm, "callbacks", [recorder])
sentinel_targets = object()
await async_post_call_failure_deployment_hook(
request_data={"model": "gpt-4o-mini", "attempted_targets": sentinel_targets},
request_data={"model": "gpt-4o-mini", private_field: object()},
exception=ValueError("x"),
call_type="acompletion",
)
assert recorder.calls[0][0].get("attempted_targets") is None
assert private_field not in recorder.calls[0][0]
@pytest.mark.asyncio

View file

@ -20,6 +20,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.router_strategy.complexity_router.context_compaction import CompactionState
from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets
from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityRetryState
from litellm.types import litellm_params
from litellm.types import utils as types_utils
from litellm.types.caching import DynamicCacheControl
@ -254,6 +255,7 @@ INTERNAL_STATE_NAMES: Final = (
"_context_compaction_state",
*AGENTIC_LOOP_STATE_NAMES,
"_router_weights",
"_deployment_affinity_retry_state",
"_encrypted_content_affinity_pinned",
"fallback_depth",
"max_fallbacks",
@ -527,6 +529,7 @@ TYPE_HINT_NAMESPACE: Final[Mapping[str, object]] = {
"CompactionState": CompactionState,
"RouterWeights": RouterWeights,
"AttemptedFallbackTargets": AttemptedFallbackTargets,
"DeploymentAffinityRetryState": DeploymentAffinityRetryState,
}
LEAF_SAMPLES: Final[Mapping[type, Mapping[str, object]]] = {