From 71fad367b4566d5676307ba74e332765f8bbd97e Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Fri, 23 Oct 2026 18:10:52 -0700 Subject: [PATCH] fix(router): isolate session affinity retry state --- litellm/router.py | 30 ++--- .../deployment_affinity_check.py | 65 ++++----- litellm/types/litellm_params.py | 4 + litellm/utils.py | 10 +- .../test_session_id_affinity.py | 123 +++++++++++++----- tests/unit/test_utils.py | 17 +-- tests/unit/types/test_litellm_params.py | 3 + 7 files changed, 153 insertions(+), 99 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 4f89a943f87..ed1cc8ca9d9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -312,6 +312,7 @@ from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( ) from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, + DeploymentAffinityRetryState, warn_on_unknown_model_group_affinity_flags, ) from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( @@ -8893,11 +8894,10 @@ class Router: verbose_router_logger.debug( "async function w/ retries: original_function - %s, num_retries - %s", original_function, num_retries ) + affinity_retry_state: Final = DeploymentAffinityRetryState() + kwargs["_deployment_affinity_retry_state"] = affinity_retry_state # rebind-ok: kwargs is local to this loop + ## ADD RETRY TRACKING TO METADATA - used for spend logs retry tracking - retry_metadata_values: Final[tuple[object, object]] = (kwargs.get("metadata"), kwargs.get("litellm_metadata")) - for metadata_value in retry_metadata_values: - if isinstance(metadata_value, dict): - metadata_value.pop("retry_failed_deployment_ids", None) _metadata["attempted_retries"] = 0 _metadata["max_retries"] = num_retries # Updated after overrides in exception handler try: @@ -8911,6 +8911,7 @@ class Router: raise current_attempt = None original_exception = e + affinity_retry_state.record_failure(e) deployment_num_retries: Final = getattr(e, "num_retries", None) if ( @@ -9012,6 +9013,7 @@ class Router: # Always track the latest error so we raise the most # recent exception instead of the first one. original_exception = e + affinity_retry_state.record_failure(e) ## LOGGING kwargs = self.log_retry(kwargs=kwargs, e=e) @@ -9633,12 +9635,10 @@ class Router: from litellm.types.router import RetryAttemptRecord _metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" - request_metadata: Final = cast( # cast-ok: router-owned metadata dict - "dict[str, object]", kwargs[_metadata_var] - ) + request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs model_group: Final = kwargs.get("model") model_info: Final = request_metadata.get("model_info") - deployment_id: Final[object] = model_info.get("id") if isinstance(model_info, Mapping) else None + deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None base_model: Final = get_base_model_from_litellm_call_metadata( metadata=request_metadata ) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None) @@ -9660,16 +9660,6 @@ class Router: else () ) breadcrumbs: Final = (*kept_breadcrumbs, attempt_record) - failed_deployment_id: Final = ( - getattr(e, "retry_skip_deployment_id", None) or getattr(e, "failed_deployment_id", None) or deployment_id - ) - if isinstance(failed_deployment_id, str): - earlier_failed_ids: Final = _as_retry_skipped_deployment_ids( - request_metadata.get("retry_failed_deployment_ids") - ) - request_metadata["retry_failed_deployment_ids"] = tuple( - sorted(frozenset((*earlier_failed_ids, failed_deployment_id))) - ) earlier: Final = request_metadata.get("request_retry_count") request_retry_count: Final = (earlier if type(earlier) is int and 0 <= earlier else 0) + 1 decision: Final = request_metadata.get("routing_decision") @@ -9685,8 +9675,8 @@ class Router: **({"litellm.deployment.id": deployment_id} if isinstance(deployment_id, str) else {}), }, ) - request_metadata["previous_models"] = breadcrumbs - request_metadata["request_retry_count"] = request_retry_count + kwargs[_metadata_var]["previous_models"] = breadcrumbs # rebind-ok: the logging object already holds this dict + kwargs[_metadata_var]["request_retry_count"] = request_retry_count # rebind-ok: same dict, read by the cap return kwargs def _deployment_params_base_model(self, deployment_id: str | None) -> str | None: diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index 1c869b6a551..e52b481d7d7 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -14,10 +14,12 @@ where routing to a consistent deployment is still beneficial. import hashlib from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass from typing import Final, cast from typing_extensions import ReadOnly, TypedDict +import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.affinity_cache import ( @@ -66,6 +68,18 @@ def warn_on_unknown_model_group_affinity_flags(model_group_affinity_config: Mapp ) +@dataclass(slots=True) +class DeploymentAffinityRetryState: + failed_deployment_ids: frozenset[str] = frozenset() + + def record_failure(self, exception: Exception) -> None: + deployment_id: Final[object] = getattr(exception, "retry_skip_deployment_id", None) or getattr( + exception, "failed_deployment_id", None + ) + if isinstance(deployment_id, str) and deployment_id: + self.failed_deployment_ids = self.failed_deployment_ids | frozenset((deployment_id,)) + + class DeploymentAffinityCheck(CustomLogger): """ Router deployment affinity callback. @@ -341,26 +355,12 @@ class DeploymentAffinityCheck(CustomLogger): return self._pinned_model_id(winner) @staticmethod - def _failed_deployment_ids(metadata: Mapping[object, object]) -> frozenset[str]: - failed_ids: Final = metadata.get("retry_failed_deployment_ids") - if not isinstance(failed_ids, tuple): - return frozenset() - values: Final = cast(tuple[object, ...], failed_ids) # cast-ok: checked tuple, opaque elements - return frozenset(deployment_id for deployment_id in values if isinstance(deployment_id, str)) - - @staticmethod - def _deployment_model_id(deployment: Mapping[str, object]) -> object: - model_info: Final = deployment.get("model_info") - if not isinstance(model_info, dict): - return None - return cast(Mapping[object, object], model_info).get("id") # cast-ok: dict checked; values stay opaque - - @classmethod - def _find_deployment_by_model_id( - cls, healthy_deployments: Sequence[dict[str, object]], model_id: str - ) -> dict[str, object] | None: + def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None: for deployment in healthy_deployments: - deployment_model_id = cls._deployment_model_id(deployment) + model_info = deployment.get("model_info") + if not isinstance(model_info, dict): + continue + deployment_model_id = model_info.get("id") if deployment_model_id is not None and str(deployment_model_id) == str(model_id): return deployment return None @@ -380,7 +380,7 @@ class DeploymentAffinityCheck(CustomLogger): 2. cached API-key deployment affinity """ request_kwargs = request_kwargs or {} - typed_healthy_deployments: Final = cast(list[dict[str, object]], healthy_deployments) + typed_healthy_deployments: Final = cast(list[dict], healthy_deployments) if request_kwargs.get("_target_order") is not None: return typed_healthy_deployments @@ -424,22 +424,25 @@ class DeploymentAffinityCheck(CustomLogger): if (session_affinity_active or enable_user_key) else None ) - session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs=request_kwargs) + + session_id: Final = ( + self._get_session_id_from_request_kwargs(request_kwargs=request_kwargs) if session_affinity_active else None + ) + retry_state: Final[object] = request_kwargs.get("_deployment_affinity_retry_state") failed_deployment_ids: Final = ( - frozenset[str]().union( - *(self._failed_deployment_ids(metadata) for metadata in self._iter_metadata_dicts(request_kwargs)) - ) - if (session_affinity_active and session_id is not None) or (enable_user_key and user_key is not None) + retry_state.failed_deployment_ids + if isinstance(retry_state, DeploymentAffinityRetryState) + and (session_id is not None or (enable_user_key and user_key is not None)) else frozenset[str]() ) - affinity_deployments: Final = [ - deployment - for deployment in typed_healthy_deployments - if self._deployment_model_id(deployment) not in failed_deployment_ids - ] or typed_healthy_deployments + affinity_deployments: Final = cast( # cast-ok: filtering preserves the router deployment dictionaries + list[dict[str, object]], + litellm.utils.get_excluded_filtered_deployments(typed_healthy_deployments, failed_deployment_ids) + or typed_healthy_deployments, + ) # 2) Session-id -> deployment affinity - if session_affinity_active and session_id is not None: + if session_id is not None: session_cache_key: Final = self.get_session_affinity_cache_key( model_group=stable_model_map_key, session_id=session_id, user_key=user_key ) diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 089f21a33c1..835fdf7d92e 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -19,6 +19,7 @@ if TYPE_CHECKING: from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.router_strategy.complexity_router.context_compaction import CompactionState from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets + from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityRetryState from litellm.types.caching import DynamicCacheControl from litellm.types.llms.openai import ChatCompletionAssistantMessage, ChatCompletionUserMessage from litellm.types.proxy.litellm_pre_call_utils import SecretFields @@ -392,6 +393,9 @@ class RouterState: fallback_depth: int | None = None max_fallbacks: int | None = None attempted_targets: "AttemptedFallbackTargets | None" = None + deployment_affinity_retry_state: "DeploymentAffinityRetryState | None" = field( + default=None, metadata=wire("_deployment_affinity_retry_state") + ) @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/litellm/utils.py b/litellm/utils.py index 73057f140a8..bbc410ae9f2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1726,10 +1726,8 @@ async def async_post_call_failure_deployment_hook( Callbacks receive a same-class snapshot of ``exception``, not the live object that's about to be re-raised, so a callback setting an attribute on it (e.g. ``status_code``) cannot change what the real caller sees. - ``request_data`` omits ``attempted_targets``: unlike the rest of this - attempt's own kwargs, it's the *same* object shared by reference across - every hop of the live fallback walk, so a callback calling ``.record()`` - on it would make the router skip a deployment it hasn't actually tried. + ``request_data`` omits live routing state shared across retries and fallbacks. + Callbacks must not be able to mark an untried deployment as failed. """ try: typed_call_type = CallTypes(call_type) @@ -1738,7 +1736,9 @@ async def async_post_call_failure_deployment_hook( _raw_fallback_depth: Final = request_data.get("fallback_depth") fallback_depth: Final = _raw_fallback_depth if isinstance(_raw_fallback_depth, int) else None - safe_request_data: Final = MappingProxyType({k: v for k, v in request_data.items() if k != "attempted_targets"}) + safe_request_data: Final = MappingProxyType( + {k: v for k, v in request_data.items() if k not in ("attempted_targets", "_deployment_affinity_retry_state")} + ) safe_exception: Final = _snapshot_exception_for_hook(exception) custom_logger_class: Final = _get_cached_custom_logger() diff --git a/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py b/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py index 29a1ca0ab12..3c5d11c00dd 100644 --- a/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py +++ b/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py @@ -1,7 +1,7 @@ import asyncio import json from collections.abc import Mapping -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx @@ -15,6 +15,7 @@ from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESS from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, + DeploymentAffinityRetryState, ) @@ -29,7 +30,14 @@ class MockResponse: return self._json_data -def _session_retry_router(deployment_ids: tuple[str, ...], allowed_fails: int) -> litellm.Router: +def _session_retry_router( + deployment_ids: tuple[str, ...], + allowed_fails: int, + checks: tuple[Literal["session_affinity", "deployment_affinity", "responses_api_deployment_check"], ...] | None = ( + "session_affinity", + ), + group_checks: tuple[str, ...] | None = None, +) -> litellm.Router: return litellm.Router( model_list=[ { @@ -38,12 +46,14 @@ def _session_retry_router(deployment_ids: tuple[str, ...], allowed_fails: int) - "model": "openai/affinity-test-model", "api_key": "test-key", "api_base": f"https://{deployment_id}.test/v1", + "weight": 1 if deployment_id == "b1" else 0, }, "model_info": {"id": deployment_id}, } for deployment_id in deployment_ids ], - optional_pre_call_checks=["session_affinity"], + optional_pre_call_checks=list(checks) if checks is not None else None, + model_group_affinity_config={"session-retry-model": list(group_checks)} if group_checks is not None else None, num_retries=2, allowed_fails=allowed_fails, retry_after=0, @@ -140,6 +150,10 @@ async def test_session_affinity_retry_exclusion_does_not_leak_to_next_request( statuses: Final = iter((500, 200, 200)) upstream: Final = Mock(side_effect=lambda request: _session_retry_response(request, next(statuses))) metadata: Final = {"session_id": "retry-session"} + request_kwargs: Final = { + "metadata": metadata, + **({"litellm_metadata": {"session_id": "retry-session"}} if both_metadata_buckets else {}), + } async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as client: monkeypatch.setattr(litellm, "aclient_session", client) router: Final = _session_retry_router(("b1", "b2"), allowed_fails=100) @@ -148,13 +162,12 @@ async def test_session_affinity_retry_exclusion_does_not_leak_to_next_request( recovered: Final = await router.acompletion( model="session-retry-model", messages=[{"role": "user", "content": "Hello"}], - metadata=metadata, + **request_kwargs, ) next_request: Final = await router.acompletion( model="session-retry-model", messages=[{"role": "user", "content": "Hello again"}], - metadata={**metadata, "retry_failed_deployment_ids": ("b1",)} if both_metadata_buckets else metadata, - **({"litellm_metadata": {"session_id": "retry-session"}} if both_metadata_buckets else {}), + **request_kwargs, ) assert recovered.choices[0].message.content == "b2.test" @@ -189,41 +202,49 @@ async def test_session_affinity_remembers_failed_backends_beyond_retry_log_limit @pytest.mark.asyncio @pytest.mark.parametrize( - ("enable_session", "group_checks", "metadata", "expected_ids"), + ("checks", "group_checks", "metadata"), [ - (True, None, {"session_id": "retry-session"}, ("b2",)), - (False, ("session_affinity",), {"session_id": "retry-session"}, ("b2",)), - ( - False, - None, - {"session_id": "retry-session", SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: 600}, - ("b2",), - ), - (True, None, {}, ("b1", "b2")), - (False, None, {"session_id": "retry-session"}, ("b1", "b2")), - ( - True, - None, - {"session_id": "retry-session", SESSION_ID_GENERATED_METADATA_KEY: True}, - ("b1", "b2"), - ), + (None, ("session_affinity",), {"session_id": "retry-session"}), + (("deployment_affinity",), None, {"user_api_key_hash": "test-user-key"}), ], - ids=["global", "per-group", "auto-router-marker", "absent-session", "disabled-affinity", "generated-session"], + ids=["per-group-session", "api-key"], ) -async def test_retry_filter_requires_active_session_affinity( - enable_session: bool, +async def test_configured_affinity_retries_another_backend( + checks: tuple[Literal["session_affinity", "deployment_affinity", "responses_api_deployment_check"], ...] | None, group_checks: tuple[str, ...] | None, metadata: Mapping[str, object], - expected_ids: tuple[str, ...], + monkeypatch: pytest.MonkeyPatch, ) -> None: + upstream: Final = Mock( + side_effect=lambda request: _session_retry_response(request, 500 if request.url.host == "b1.test" else 200) + ) + async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as client: + monkeypatch.setattr(litellm, "aclient_session", client) + router: Final = _session_retry_router(("b1", "b2"), allowed_fails=100, checks=checks, group_checks=group_checks) + await _pin_session_retry_router(router) + await router.cache.async_set_cache( + key=DeploymentAffinityCheck.get_affinity_cache_key("session-retry-model", "test-user-key"), + value={"model_id": "b1"}, + ) + + response: Final = await router.acompletion( + model="session-retry-model", + messages=[{"role": "user", "content": "Hello"}], + metadata=dict(metadata), + ) + + assert response.choices[0].message.content == "b2.test" + assert tuple(call.args[0].url.host for call in upstream.call_args_list) == ("b1.test", "b2.test") + + +@pytest.mark.asyncio +async def test_auto_router_marker_skips_a_failed_pinned_backend() -> None: cache: Final = DualCache() callback: Final = DeploymentAffinityCheck( cache=cache, ttl_seconds=600, enable_user_key_affinity=False, enable_responses_api_affinity=False, - enable_session_id_affinity=enable_session, - model_group_affinity_config={"session-retry-model": list(group_checks)} if group_checks is not None else None, ) deployments: Final = [ {"model_name": "session-retry-model", "model_info": {"id": deployment_id}} for deployment_id in ("b1", "b2") @@ -237,10 +258,47 @@ async def test_retry_filter_requires_active_session_affinity( model="session-retry-model", healthy_deployments=deployments, messages=[], - request_kwargs={"litellm_metadata": {**metadata, "retry_failed_deployment_ids": ("b1",)}}, + request_kwargs={ + "_deployment_affinity_retry_state": DeploymentAffinityRetryState(frozenset({"b1"})), + "litellm_metadata": { + "session_id": "retry-session", + SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: 600, + }, + }, ) - assert tuple(deployment["model_info"]["id"] for deployment in available) == expected_ids + assert available == [deployments[1]] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("checks", "metadata"), + [ + (None, {"session_id": "retry-session"}), + (("session_affinity",), {}), + (("session_affinity",), {"session_id": "retry-session", SESSION_ID_GENERATED_METADATA_KEY: True}), + ], + ids=["default-disabled", "absent-session", "generated-session"], +) +async def test_inactive_affinity_preserves_configured_backend_weights_on_retry( + checks: tuple[Literal["session_affinity", "deployment_affinity", "responses_api_deployment_check"], ...] | None, + metadata: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, +) -> None: + upstream: Final = Mock(side_effect=lambda request: _session_retry_response(request, 500)) + async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as client: + monkeypatch.setattr(litellm, "aclient_session", client) + router: Final = _session_retry_router(("b1", "b2"), allowed_fails=100, checks=checks) + await _pin_session_retry_router(router) + + with pytest.raises(litellm.InternalServerError, match="Unavailable backend"): + await router.acompletion( + model="session-retry-model", + messages=[{"role": "user", "content": "Hello"}], + metadata=dict(metadata), + ) + + assert tuple(call.args[0].url.host for call in upstream.call_args_list) == ("b1.test", "b1.test", "b1.test") @pytest.mark.asyncio @@ -265,7 +323,8 @@ async def test_response_continuity_keeps_its_backend_after_a_session_affinity_fa messages=[], request_kwargs={ "previous_response_id": previous_response_id, - "litellm_metadata": {"session_id": "retry-session", "retry_failed_deployment_ids": ("b1",)}, + "_deployment_affinity_retry_state": DeploymentAffinityRetryState(frozenset({"b1"})), + "litellm_metadata": {"session_id": "retry-session"}, }, ) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ad069aa987b..66d3a04dbee 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -6525,25 +6525,20 @@ async def test_wrapper_async_failure_hook_exception_mutation_does_not_change_rai @pytest.mark.asyncio -async def test_async_post_call_failure_deployment_hook_omits_attempted_targets_from_request_data( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize("private_field", ("attempted_targets", "_deployment_affinity_retry_state")) +async def test_async_post_call_failure_deployment_hook_omits_live_routing_state_from_request_data( + monkeypatch: pytest.MonkeyPatch, private_field: str ) -> None: - """Regression: attempted_targets is the router's own live fallback-walk bookkeeping, - shared by reference across every hop of a single request - unlike the rest of - request_data, it is not this attempt's own isolated copy. A callback calling .record() - on it would make the router skip a deployment it hasn't actually tried, so the - dispatcher must never hand it to a callback.""" - recorder = _RecordingDeploymentFailureLogger() + recorder: Final = _RecordingDeploymentFailureLogger() monkeypatch.setattr(litellm, "callbacks", [recorder]) - sentinel_targets = object() await async_post_call_failure_deployment_hook( - request_data={"model": "gpt-4o-mini", "attempted_targets": sentinel_targets}, + request_data={"model": "gpt-4o-mini", private_field: object()}, exception=ValueError("x"), call_type="acompletion", ) - assert recorder.calls[0][0].get("attempted_targets") is None + assert private_field not in recorder.calls[0][0] @pytest.mark.asyncio diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index b0086662f1c..e6ed2345f9a 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.router_strategy.complexity_router.context_compaction import CompactionState from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets +from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityRetryState from litellm.types import litellm_params from litellm.types import utils as types_utils from litellm.types.caching import DynamicCacheControl @@ -254,6 +255,7 @@ INTERNAL_STATE_NAMES: Final = ( "_context_compaction_state", *AGENTIC_LOOP_STATE_NAMES, "_router_weights", + "_deployment_affinity_retry_state", "_encrypted_content_affinity_pinned", "fallback_depth", "max_fallbacks", @@ -527,6 +529,7 @@ TYPE_HINT_NAMESPACE: Final[Mapping[str, object]] = { "CompactionState": CompactionState, "RouterWeights": RouterWeights, "AttemptedFallbackTargets": AttemptedFallbackTargets, + "DeploymentAffinityRetryState": DeploymentAffinityRetryState, } LEAF_SAMPLES: Final[Mapping[type, Mapping[str, object]]] = {