mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
fix(router): isolate session affinity retry state
This commit is contained in:
parent
ef25a34428
commit
71fad367b4
7 changed files with 153 additions and 99 deletions
|
|
@ -312,6 +312,7 @@ from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
|
|||
)
|
||||
from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
|
||||
DeploymentAffinityCheck,
|
||||
DeploymentAffinityRetryState,
|
||||
warn_on_unknown_model_group_affinity_flags,
|
||||
)
|
||||
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
|
||||
|
|
@ -8893,11 +8894,10 @@ class Router:
|
|||
verbose_router_logger.debug(
|
||||
"async function w/ retries: original_function - %s, num_retries - %s", original_function, num_retries
|
||||
)
|
||||
affinity_retry_state: Final = DeploymentAffinityRetryState()
|
||||
kwargs["_deployment_affinity_retry_state"] = affinity_retry_state # rebind-ok: kwargs is local to this loop
|
||||
|
||||
## ADD RETRY TRACKING TO METADATA - used for spend logs retry tracking
|
||||
retry_metadata_values: Final[tuple[object, object]] = (kwargs.get("metadata"), kwargs.get("litellm_metadata"))
|
||||
for metadata_value in retry_metadata_values:
|
||||
if isinstance(metadata_value, dict):
|
||||
metadata_value.pop("retry_failed_deployment_ids", None)
|
||||
_metadata["attempted_retries"] = 0
|
||||
_metadata["max_retries"] = num_retries # Updated after overrides in exception handler
|
||||
try:
|
||||
|
|
@ -8911,6 +8911,7 @@ class Router:
|
|||
raise
|
||||
current_attempt = None
|
||||
original_exception = e
|
||||
affinity_retry_state.record_failure(e)
|
||||
deployment_num_retries: Final = getattr(e, "num_retries", None)
|
||||
|
||||
if (
|
||||
|
|
@ -9012,6 +9013,7 @@ class Router:
|
|||
# Always track the latest error so we raise the most
|
||||
# recent exception instead of the first one.
|
||||
original_exception = e
|
||||
affinity_retry_state.record_failure(e)
|
||||
|
||||
## LOGGING
|
||||
kwargs = self.log_retry(kwargs=kwargs, e=e)
|
||||
|
|
@ -9633,12 +9635,10 @@ class Router:
|
|||
from litellm.types.router import RetryAttemptRecord
|
||||
|
||||
_metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
|
||||
request_metadata: Final = cast( # cast-ok: router-owned metadata dict
|
||||
"dict[str, object]", kwargs[_metadata_var]
|
||||
)
|
||||
request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs
|
||||
model_group: Final = kwargs.get("model")
|
||||
model_info: Final = request_metadata.get("model_info")
|
||||
deployment_id: Final[object] = model_info.get("id") if isinstance(model_info, Mapping) else None
|
||||
deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None
|
||||
base_model: Final = get_base_model_from_litellm_call_metadata(
|
||||
metadata=request_metadata
|
||||
) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None)
|
||||
|
|
@ -9660,16 +9660,6 @@ class Router:
|
|||
else ()
|
||||
)
|
||||
breadcrumbs: Final = (*kept_breadcrumbs, attempt_record)
|
||||
failed_deployment_id: Final = (
|
||||
getattr(e, "retry_skip_deployment_id", None) or getattr(e, "failed_deployment_id", None) or deployment_id
|
||||
)
|
||||
if isinstance(failed_deployment_id, str):
|
||||
earlier_failed_ids: Final = _as_retry_skipped_deployment_ids(
|
||||
request_metadata.get("retry_failed_deployment_ids")
|
||||
)
|
||||
request_metadata["retry_failed_deployment_ids"] = tuple(
|
||||
sorted(frozenset((*earlier_failed_ids, failed_deployment_id)))
|
||||
)
|
||||
earlier: Final = request_metadata.get("request_retry_count")
|
||||
request_retry_count: Final = (earlier if type(earlier) is int and 0 <= earlier else 0) + 1
|
||||
decision: Final = request_metadata.get("routing_decision")
|
||||
|
|
@ -9685,8 +9675,8 @@ class Router:
|
|||
**({"litellm.deployment.id": deployment_id} if isinstance(deployment_id, str) else {}),
|
||||
},
|
||||
)
|
||||
request_metadata["previous_models"] = breadcrumbs
|
||||
request_metadata["request_retry_count"] = request_retry_count
|
||||
kwargs[_metadata_var]["previous_models"] = breadcrumbs # rebind-ok: the logging object already holds this dict
|
||||
kwargs[_metadata_var]["request_retry_count"] = request_retry_count # rebind-ok: same dict, read by the cap
|
||||
return kwargs
|
||||
|
||||
def _deployment_params_base_model(self, deployment_id: str | None) -> str | None:
|
||||
|
|
|
|||
|
|
@ -14,10 +14,12 @@ where routing to a consistent deployment is still beneficial.
|
|||
|
||||
import hashlib
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, cast
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.affinity_cache import (
|
||||
|
|
@ -66,6 +68,18 @@ def warn_on_unknown_model_group_affinity_flags(model_group_affinity_config: Mapp
|
|||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class DeploymentAffinityRetryState:
|
||||
failed_deployment_ids: frozenset[str] = frozenset()
|
||||
|
||||
def record_failure(self, exception: Exception) -> None:
|
||||
deployment_id: Final[object] = getattr(exception, "retry_skip_deployment_id", None) or getattr(
|
||||
exception, "failed_deployment_id", None
|
||||
)
|
||||
if isinstance(deployment_id, str) and deployment_id:
|
||||
self.failed_deployment_ids = self.failed_deployment_ids | frozenset((deployment_id,))
|
||||
|
||||
|
||||
class DeploymentAffinityCheck(CustomLogger):
|
||||
"""
|
||||
Router deployment affinity callback.
|
||||
|
|
@ -341,26 +355,12 @@ class DeploymentAffinityCheck(CustomLogger):
|
|||
return self._pinned_model_id(winner)
|
||||
|
||||
@staticmethod
|
||||
def _failed_deployment_ids(metadata: Mapping[object, object]) -> frozenset[str]:
|
||||
failed_ids: Final = metadata.get("retry_failed_deployment_ids")
|
||||
if not isinstance(failed_ids, tuple):
|
||||
return frozenset()
|
||||
values: Final = cast(tuple[object, ...], failed_ids) # cast-ok: checked tuple, opaque elements
|
||||
return frozenset(deployment_id for deployment_id in values if isinstance(deployment_id, str))
|
||||
|
||||
@staticmethod
|
||||
def _deployment_model_id(deployment: Mapping[str, object]) -> object:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
if not isinstance(model_info, dict):
|
||||
return None
|
||||
return cast(Mapping[object, object], model_info).get("id") # cast-ok: dict checked; values stay opaque
|
||||
|
||||
@classmethod
|
||||
def _find_deployment_by_model_id(
|
||||
cls, healthy_deployments: Sequence[dict[str, object]], model_id: str
|
||||
) -> dict[str, object] | None:
|
||||
def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None:
|
||||
for deployment in healthy_deployments:
|
||||
deployment_model_id = cls._deployment_model_id(deployment)
|
||||
model_info = deployment.get("model_info")
|
||||
if not isinstance(model_info, dict):
|
||||
continue
|
||||
deployment_model_id = model_info.get("id")
|
||||
if deployment_model_id is not None and str(deployment_model_id) == str(model_id):
|
||||
return deployment
|
||||
return None
|
||||
|
|
@ -380,7 +380,7 @@ class DeploymentAffinityCheck(CustomLogger):
|
|||
2. cached API-key deployment affinity
|
||||
"""
|
||||
request_kwargs = request_kwargs or {}
|
||||
typed_healthy_deployments: Final = cast(list[dict[str, object]], healthy_deployments)
|
||||
typed_healthy_deployments: Final = cast(list[dict], healthy_deployments)
|
||||
if request_kwargs.get("_target_order") is not None:
|
||||
return typed_healthy_deployments
|
||||
|
||||
|
|
@ -424,22 +424,25 @@ class DeploymentAffinityCheck(CustomLogger):
|
|||
if (session_affinity_active or enable_user_key)
|
||||
else None
|
||||
)
|
||||
session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs=request_kwargs)
|
||||
|
||||
session_id: Final = (
|
||||
self._get_session_id_from_request_kwargs(request_kwargs=request_kwargs) if session_affinity_active else None
|
||||
)
|
||||
retry_state: Final[object] = request_kwargs.get("_deployment_affinity_retry_state")
|
||||
failed_deployment_ids: Final = (
|
||||
frozenset[str]().union(
|
||||
*(self._failed_deployment_ids(metadata) for metadata in self._iter_metadata_dicts(request_kwargs))
|
||||
)
|
||||
if (session_affinity_active and session_id is not None) or (enable_user_key and user_key is not None)
|
||||
retry_state.failed_deployment_ids
|
||||
if isinstance(retry_state, DeploymentAffinityRetryState)
|
||||
and (session_id is not None or (enable_user_key and user_key is not None))
|
||||
else frozenset[str]()
|
||||
)
|
||||
affinity_deployments: Final = [
|
||||
deployment
|
||||
for deployment in typed_healthy_deployments
|
||||
if self._deployment_model_id(deployment) not in failed_deployment_ids
|
||||
] or typed_healthy_deployments
|
||||
affinity_deployments: Final = cast( # cast-ok: filtering preserves the router deployment dictionaries
|
||||
list[dict[str, object]],
|
||||
litellm.utils.get_excluded_filtered_deployments(typed_healthy_deployments, failed_deployment_ids)
|
||||
or typed_healthy_deployments,
|
||||
)
|
||||
|
||||
# 2) Session-id -> deployment affinity
|
||||
if session_affinity_active and session_id is not None:
|
||||
if session_id is not None:
|
||||
session_cache_key: Final = self.get_session_affinity_cache_key(
|
||||
model_group=stable_model_map_key, session_id=session_id, user_key=user_key
|
||||
)
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ if TYPE_CHECKING:
|
|||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.router_strategy.complexity_router.context_compaction import CompactionState
|
||||
from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets
|
||||
from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityRetryState
|
||||
from litellm.types.caching import DynamicCacheControl
|
||||
from litellm.types.llms.openai import ChatCompletionAssistantMessage, ChatCompletionUserMessage
|
||||
from litellm.types.proxy.litellm_pre_call_utils import SecretFields
|
||||
|
|
@ -392,6 +393,9 @@ class RouterState:
|
|||
fallback_depth: int | None = None
|
||||
max_fallbacks: int | None = None
|
||||
attempted_targets: "AttemptedFallbackTargets | None" = None
|
||||
deployment_affinity_retry_state: "DeploymentAffinityRetryState | None" = field(
|
||||
default=None, metadata=wire("_deployment_affinity_retry_state")
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
|
|
|
|||
|
|
@ -1726,10 +1726,8 @@ async def async_post_call_failure_deployment_hook(
|
|||
Callbacks receive a same-class snapshot of ``exception``, not the live
|
||||
object that's about to be re-raised, so a callback setting an attribute
|
||||
on it (e.g. ``status_code``) cannot change what the real caller sees.
|
||||
``request_data`` omits ``attempted_targets``: unlike the rest of this
|
||||
attempt's own kwargs, it's the *same* object shared by reference across
|
||||
every hop of the live fallback walk, so a callback calling ``.record()``
|
||||
on it would make the router skip a deployment it hasn't actually tried.
|
||||
``request_data`` omits live routing state shared across retries and fallbacks.
|
||||
Callbacks must not be able to mark an untried deployment as failed.
|
||||
"""
|
||||
try:
|
||||
typed_call_type = CallTypes(call_type)
|
||||
|
|
@ -1738,7 +1736,9 @@ async def async_post_call_failure_deployment_hook(
|
|||
|
||||
_raw_fallback_depth: Final = request_data.get("fallback_depth")
|
||||
fallback_depth: Final = _raw_fallback_depth if isinstance(_raw_fallback_depth, int) else None
|
||||
safe_request_data: Final = MappingProxyType({k: v for k, v in request_data.items() if k != "attempted_targets"})
|
||||
safe_request_data: Final = MappingProxyType(
|
||||
{k: v for k, v in request_data.items() if k not in ("attempted_targets", "_deployment_affinity_retry_state")}
|
||||
)
|
||||
safe_exception: Final = _snapshot_exception_for_hook(exception)
|
||||
|
||||
custom_logger_class: Final = _get_cached_custom_logger()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -15,6 +15,7 @@ from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESS
|
|||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
|
||||
DeploymentAffinityCheck,
|
||||
DeploymentAffinityRetryState,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -29,7 +30,14 @@ class MockResponse:
|
|||
return self._json_data
|
||||
|
||||
|
||||
def _session_retry_router(deployment_ids: tuple[str, ...], allowed_fails: int) -> litellm.Router:
|
||||
def _session_retry_router(
|
||||
deployment_ids: tuple[str, ...],
|
||||
allowed_fails: int,
|
||||
checks: tuple[Literal["session_affinity", "deployment_affinity", "responses_api_deployment_check"], ...] | None = (
|
||||
"session_affinity",
|
||||
),
|
||||
group_checks: tuple[str, ...] | None = None,
|
||||
) -> litellm.Router:
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -38,12 +46,14 @@ def _session_retry_router(deployment_ids: tuple[str, ...], allowed_fails: int) -
|
|||
"model": "openai/affinity-test-model",
|
||||
"api_key": "test-key",
|
||||
"api_base": f"https://{deployment_id}.test/v1",
|
||||
"weight": 1 if deployment_id == "b1" else 0,
|
||||
},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
for deployment_id in deployment_ids
|
||||
],
|
||||
optional_pre_call_checks=["session_affinity"],
|
||||
optional_pre_call_checks=list(checks) if checks is not None else None,
|
||||
model_group_affinity_config={"session-retry-model": list(group_checks)} if group_checks is not None else None,
|
||||
num_retries=2,
|
||||
allowed_fails=allowed_fails,
|
||||
retry_after=0,
|
||||
|
|
@ -140,6 +150,10 @@ async def test_session_affinity_retry_exclusion_does_not_leak_to_next_request(
|
|||
statuses: Final = iter((500, 200, 200))
|
||||
upstream: Final = Mock(side_effect=lambda request: _session_retry_response(request, next(statuses)))
|
||||
metadata: Final = {"session_id": "retry-session"}
|
||||
request_kwargs: Final = {
|
||||
"metadata": metadata,
|
||||
**({"litellm_metadata": {"session_id": "retry-session"}} if both_metadata_buckets else {}),
|
||||
}
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as client:
|
||||
monkeypatch.setattr(litellm, "aclient_session", client)
|
||||
router: Final = _session_retry_router(("b1", "b2"), allowed_fails=100)
|
||||
|
|
@ -148,13 +162,12 @@ async def test_session_affinity_retry_exclusion_does_not_leak_to_next_request(
|
|||
recovered: Final = await router.acompletion(
|
||||
model="session-retry-model",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
metadata=metadata,
|
||||
**request_kwargs,
|
||||
)
|
||||
next_request: Final = await router.acompletion(
|
||||
model="session-retry-model",
|
||||
messages=[{"role": "user", "content": "Hello again"}],
|
||||
metadata={**metadata, "retry_failed_deployment_ids": ("b1",)} if both_metadata_buckets else metadata,
|
||||
**({"litellm_metadata": {"session_id": "retry-session"}} if both_metadata_buckets else {}),
|
||||
**request_kwargs,
|
||||
)
|
||||
|
||||
assert recovered.choices[0].message.content == "b2.test"
|
||||
|
|
@ -189,41 +202,49 @@ async def test_session_affinity_remembers_failed_backends_beyond_retry_log_limit
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("enable_session", "group_checks", "metadata", "expected_ids"),
|
||||
("checks", "group_checks", "metadata"),
|
||||
[
|
||||
(True, None, {"session_id": "retry-session"}, ("b2",)),
|
||||
(False, ("session_affinity",), {"session_id": "retry-session"}, ("b2",)),
|
||||
(
|
||||
False,
|
||||
None,
|
||||
{"session_id": "retry-session", SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: 600},
|
||||
("b2",),
|
||||
),
|
||||
(True, None, {}, ("b1", "b2")),
|
||||
(False, None, {"session_id": "retry-session"}, ("b1", "b2")),
|
||||
(
|
||||
True,
|
||||
None,
|
||||
{"session_id": "retry-session", SESSION_ID_GENERATED_METADATA_KEY: True},
|
||||
("b1", "b2"),
|
||||
),
|
||||
(None, ("session_affinity",), {"session_id": "retry-session"}),
|
||||
(("deployment_affinity",), None, {"user_api_key_hash": "test-user-key"}),
|
||||
],
|
||||
ids=["global", "per-group", "auto-router-marker", "absent-session", "disabled-affinity", "generated-session"],
|
||||
ids=["per-group-session", "api-key"],
|
||||
)
|
||||
async def test_retry_filter_requires_active_session_affinity(
|
||||
enable_session: bool,
|
||||
async def test_configured_affinity_retries_another_backend(
|
||||
checks: tuple[Literal["session_affinity", "deployment_affinity", "responses_api_deployment_check"], ...] | None,
|
||||
group_checks: tuple[str, ...] | None,
|
||||
metadata: Mapping[str, object],
|
||||
expected_ids: tuple[str, ...],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
upstream: Final = Mock(
|
||||
side_effect=lambda request: _session_retry_response(request, 500 if request.url.host == "b1.test" else 200)
|
||||
)
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as client:
|
||||
monkeypatch.setattr(litellm, "aclient_session", client)
|
||||
router: Final = _session_retry_router(("b1", "b2"), allowed_fails=100, checks=checks, group_checks=group_checks)
|
||||
await _pin_session_retry_router(router)
|
||||
await router.cache.async_set_cache(
|
||||
key=DeploymentAffinityCheck.get_affinity_cache_key("session-retry-model", "test-user-key"),
|
||||
value={"model_id": "b1"},
|
||||
)
|
||||
|
||||
response: Final = await router.acompletion(
|
||||
model="session-retry-model",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
metadata=dict(metadata),
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "b2.test"
|
||||
assert tuple(call.args[0].url.host for call in upstream.call_args_list) == ("b1.test", "b2.test")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_router_marker_skips_a_failed_pinned_backend() -> None:
|
||||
cache: Final = DualCache()
|
||||
callback: Final = DeploymentAffinityCheck(
|
||||
cache=cache,
|
||||
ttl_seconds=600,
|
||||
enable_user_key_affinity=False,
|
||||
enable_responses_api_affinity=False,
|
||||
enable_session_id_affinity=enable_session,
|
||||
model_group_affinity_config={"session-retry-model": list(group_checks)} if group_checks is not None else None,
|
||||
)
|
||||
deployments: Final = [
|
||||
{"model_name": "session-retry-model", "model_info": {"id": deployment_id}} for deployment_id in ("b1", "b2")
|
||||
|
|
@ -237,10 +258,47 @@ async def test_retry_filter_requires_active_session_affinity(
|
|||
model="session-retry-model",
|
||||
healthy_deployments=deployments,
|
||||
messages=[],
|
||||
request_kwargs={"litellm_metadata": {**metadata, "retry_failed_deployment_ids": ("b1",)}},
|
||||
request_kwargs={
|
||||
"_deployment_affinity_retry_state": DeploymentAffinityRetryState(frozenset({"b1"})),
|
||||
"litellm_metadata": {
|
||||
"session_id": "retry-session",
|
||||
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: 600,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert tuple(deployment["model_info"]["id"] for deployment in available) == expected_ids
|
||||
assert available == [deployments[1]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("checks", "metadata"),
|
||||
[
|
||||
(None, {"session_id": "retry-session"}),
|
||||
(("session_affinity",), {}),
|
||||
(("session_affinity",), {"session_id": "retry-session", SESSION_ID_GENERATED_METADATA_KEY: True}),
|
||||
],
|
||||
ids=["default-disabled", "absent-session", "generated-session"],
|
||||
)
|
||||
async def test_inactive_affinity_preserves_configured_backend_weights_on_retry(
|
||||
checks: tuple[Literal["session_affinity", "deployment_affinity", "responses_api_deployment_check"], ...] | None,
|
||||
metadata: Mapping[str, object],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
upstream: Final = Mock(side_effect=lambda request: _session_retry_response(request, 500))
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as client:
|
||||
monkeypatch.setattr(litellm, "aclient_session", client)
|
||||
router: Final = _session_retry_router(("b1", "b2"), allowed_fails=100, checks=checks)
|
||||
await _pin_session_retry_router(router)
|
||||
|
||||
with pytest.raises(litellm.InternalServerError, match="Unavailable backend"):
|
||||
await router.acompletion(
|
||||
model="session-retry-model",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
metadata=dict(metadata),
|
||||
)
|
||||
|
||||
assert tuple(call.args[0].url.host for call in upstream.call_args_list) == ("b1.test", "b1.test", "b1.test")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -265,7 +323,8 @@ async def test_response_continuity_keeps_its_backend_after_a_session_affinity_fa
|
|||
messages=[],
|
||||
request_kwargs={
|
||||
"previous_response_id": previous_response_id,
|
||||
"litellm_metadata": {"session_id": "retry-session", "retry_failed_deployment_ids": ("b1",)},
|
||||
"_deployment_affinity_retry_state": DeploymentAffinityRetryState(frozenset({"b1"})),
|
||||
"litellm_metadata": {"session_id": "retry-session"},
|
||||
},
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -6525,25 +6525,20 @@ async def test_wrapper_async_failure_hook_exception_mutation_does_not_change_rai
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_post_call_failure_deployment_hook_omits_attempted_targets_from_request_data(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@pytest.mark.parametrize("private_field", ("attempted_targets", "_deployment_affinity_retry_state"))
|
||||
async def test_async_post_call_failure_deployment_hook_omits_live_routing_state_from_request_data(
|
||||
monkeypatch: pytest.MonkeyPatch, private_field: str
|
||||
) -> None:
|
||||
"""Regression: attempted_targets is the router's own live fallback-walk bookkeeping,
|
||||
shared by reference across every hop of a single request - unlike the rest of
|
||||
request_data, it is not this attempt's own isolated copy. A callback calling .record()
|
||||
on it would make the router skip a deployment it hasn't actually tried, so the
|
||||
dispatcher must never hand it to a callback."""
|
||||
recorder = _RecordingDeploymentFailureLogger()
|
||||
recorder: Final = _RecordingDeploymentFailureLogger()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
|
||||
sentinel_targets = object()
|
||||
await async_post_call_failure_deployment_hook(
|
||||
request_data={"model": "gpt-4o-mini", "attempted_targets": sentinel_targets},
|
||||
request_data={"model": "gpt-4o-mini", private_field: object()},
|
||||
exception=ValueError("x"),
|
||||
call_type="acompletion",
|
||||
)
|
||||
|
||||
assert recorder.calls[0][0].get("attempted_targets") is None
|
||||
assert private_field not in recorder.calls[0][0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging
|
|||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.router_strategy.complexity_router.context_compaction import CompactionState
|
||||
from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets
|
||||
from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityRetryState
|
||||
from litellm.types import litellm_params
|
||||
from litellm.types import utils as types_utils
|
||||
from litellm.types.caching import DynamicCacheControl
|
||||
|
|
@ -254,6 +255,7 @@ INTERNAL_STATE_NAMES: Final = (
|
|||
"_context_compaction_state",
|
||||
*AGENTIC_LOOP_STATE_NAMES,
|
||||
"_router_weights",
|
||||
"_deployment_affinity_retry_state",
|
||||
"_encrypted_content_affinity_pinned",
|
||||
"fallback_depth",
|
||||
"max_fallbacks",
|
||||
|
|
@ -527,6 +529,7 @@ TYPE_HINT_NAMESPACE: Final[Mapping[str, object]] = {
|
|||
"CompactionState": CompactionState,
|
||||
"RouterWeights": RouterWeights,
|
||||
"AttemptedFallbackTargets": AttemptedFallbackTargets,
|
||||
"DeploymentAffinityRetryState": DeploymentAffinityRetryState,
|
||||
}
|
||||
|
||||
LEAF_SAMPLES: Final[Mapping[type, Mapping[str, object]]] = {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue