diff --git a/litellm/router.py b/litellm/router.py index c8153b596d6..18c5c67abce 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -238,11 +238,12 @@ from litellm.router_utils.cooldown_handlers import ( async_get_cooldown_deployments_with_debug_info, get_cooldown_deployment_info, get_cooldown_deployments, - is_advisor_orchestration_failure, is_background_response_poll_not_found, is_caller_scoped_auth_failure, is_caller_timeout_408, + is_failure_outside_selected_deployment, is_video_lookup_not_found, + mark_deployment_selection_failure, set_cooldown_deployments, ) from litellm.router_utils.deployment_pricing import deployment_pricing_models @@ -1519,6 +1520,7 @@ class Router: self.allowed_fails = allowed_fails else: self.allowed_fails = litellm.allowed_fails + self.explicit_allowed_fails = allowed_fails self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time) self.disable_cooldowns = disable_cooldowns @@ -9579,10 +9581,10 @@ class Router: try: exception: Final = kwargs.get("exception", None) - if is_advisor_orchestration_failure(exception): + if is_failure_outside_selected_deployment(exception): verbose_router_logger.debug( "Router: Exiting 'deployment_callback_on_failure' without cooldown. " - "Failure originated from advisor orchestration, not the selected deployment." + "Failure came from advisor orchestration or deployment selection, not the selected deployment." ) return False @@ -15473,6 +15475,7 @@ class Router: phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes) return deployment except Exception as e: + mark_deployment_selection_failure(e) traceback_exception: Final = traceback.format_exc() # if router rejects call -> log to langfuse/otel/etc. if request_kwargs is not None: @@ -15641,6 +15644,7 @@ class Router: return deployment except Exception as e: + mark_deployment_selection_failure(e) traceback_exception: Final = traceback.format_exc() if request_kwargs is not None: logging_obj: Final = request_kwargs.get("litellm_logging_obj", None) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 330500789a4..e1baf66b201 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -57,6 +57,7 @@ else: Span = Any _ADVISOR_ORCHESTRATION_FAILURE_ATTR: Final = "_litellm_advisor_orchestration_failure" +_DEPLOYMENT_SELECTION_FAILURE_ATTR: Final = "_litellm_deployment_selection_failure" _CREDENTIAL_VALUES_ADAPTER: Final = TypeAdapter(Mapping[str, object]) @@ -79,6 +80,16 @@ def is_advisor_orchestration_failure(exception: BaseException | None) -> bool: return bool(getattr(exception, _ADVISOR_ORCHESTRATION_FAILURE_ATTR, False)) +def mark_deployment_selection_failure(exception: BaseException) -> None: + setattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, True) + + +def is_failure_outside_selected_deployment(exception: BaseException | None) -> bool: + return is_advisor_orchestration_failure(exception) or bool( + getattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, False) + ) + + _BACKGROUND_RESPONSE_POLL_CALL_ORIGINS: Final[frozenset[InternalCallOrigin]] = frozenset( {BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN} ) @@ -781,14 +792,17 @@ def _is_allowed_fails_set_on_router( litellm_router_instance: LitellmRouter, ) -> bool: """ - Check if Router.allowed_fails is set or is Non-default Value + Check if Router.allowed_fails was passed to the Router or is a Non-default Value Returns: - - True if Router.allowed_fails is set or is Non-default Value - - False if Router.allowed_fails is None or is Default Value + - True if allowed_fails was passed to the Router, even when it equals the default + - True if Router.allowed_fails is a Non-default Value + - False if Router.allowed_fails is None, or is the Default Value and was not passed """ if litellm_router_instance.allowed_fails is None: return False + if litellm_router_instance.explicit_allowed_fails is not None: + return True if litellm_router_instance.allowed_fails != litellm.allowed_fails: return True return False diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index bab6d257579..4299b61c703 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -1,10 +1,12 @@ import asyncio import copy +import datetime import importlib import itertools import random import time -from collections.abc import Mapping +import uuid +from collections.abc import Mapping, Sequence from typing import Final from unittest.mock import MagicMock, patch @@ -682,6 +684,45 @@ class TestDeploymentLevelAllowedFails: ) +def _gpt_4_group_with_a_sibling() -> Sequence[Mapping[str, object]]: + return ( + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}}, + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "secondary"}}, + ) + + +def _primary_cooldown_decisions_on_repeated_503s(router: Router) -> tuple[bool, ...]: + exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4") + return tuple( + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="primary", + exception_status=503, + original_exception=exception, + ) + for _ in range(litellm.allowed_fails + 1) + ) + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +class TestRouterLevelAllowedFailsEqualToDefault: + def test_allowed_fails_equal_to_the_default_cools_down_after_that_many_fails(self): + router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling()), allowed_fails=litellm.allowed_fails) + + assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * litellm.allowed_fails + (True,) + + def test_unset_allowed_fails_keeps_the_rate_based_default(self): + router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling())) + + assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * (litellm.allowed_fails + 1) + + def test_default_allowed_fails_saved_through_update_settings_keeps_the_rate_based_default(self): + router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling())) + router.update_settings(allowed_fails=litellm.allowed_fails) + + assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * (litellm.allowed_fails + 1) + + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestDeploymentLevelAllowedFailsPolicyByExceptionType: def test_rate_limit_error_triggers_cooldown_with_zero_threshold(self): @@ -2931,6 +2972,104 @@ async def test_bad_request_error_does_not_cool_down_a_deployment( assert tuple(cooldown_deployments) == () +async def _complete_like_the_proxy(router: Router, model: str) -> str: + logging_obj, request = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.datetime(2026, 1, 1), + model=model, + messages=[{"role": "user", "content": "hi"}], + litellm_call_id=str(uuid.uuid4()), + ) + response: Final = await router.acompletion(**request, litellm_logging_obj=logging_obj) + model_id: Final = response._hidden_params["model_id"] + assert isinstance(model_id, str) + return model_id + + +_CHAT_COMPLETION_OK: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +def _openai_deployment(model_name: str, host: str, deployment_id: str) -> Mapping[str, object]: + return { + "model_name": model_name, + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": f"https://{host}/v1"}, + "model_info": {"id": deployment_id}, + } + + +@pytest.mark.asyncio +async def test_fallback_deployment_stays_up_while_the_primary_is_in_cooldown( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://primary.test/v1/chat/completions").mock( + return_value=httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}}) + ) + backup_route: Final = respx_mock.post("https://backup.test/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK) + ) + router: Final = litellm.Router( + model_list=[ + _openai_deployment("primary", "primary.test", "primary-dep"), + _openai_deployment("backup", "backup.test", "backup-dep"), + ], + fallbacks=[{"primary": ["backup"]}], + allowed_fails=litellm.allowed_fails, + num_retries=0, + ) + + served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(20)] + cooldown_deployments: Final = await async_get_cooldown_deployments( + litellm_router_instance=router, + parent_otel_span=None, + ) + + assert served_by == ["backup-dep"] * 20 + assert len(backup_route.calls) == 20 + assert tuple(cooldown_deployments) == ("primary-dep",) + + +@pytest.mark.asyncio +async def test_fallback_group_stays_up_after_a_bad_key_cools_the_primary_under_the_rate_based_default( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://primary.test/v1/chat/completions").mock( + return_value=httpx.Response(401, json={"error": {"message": "bad key", "type": "invalid_request_error"}}) + ) + respx_mock.post("https://backup.test/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK) + ) + router: Final = litellm.Router( + model_list=[ + _openai_deployment("primary", "primary.test", "primary-dep"), + _openai_deployment("backup", "backup.test", "backup-1"), + _openai_deployment("backup", "backup.test", "backup-2"), + ], + fallbacks=[{"primary": ["backup"]}], + num_retries=0, + ) + + served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(12)] + cooldown_deployments: Final = await async_get_cooldown_deployments( + litellm_router_instance=router, + parent_otel_span=None, + ) + + assert set(served_by) <= {"backup-1", "backup-2"} + assert tuple(cooldown_deployments) == ("primary-dep",) + + TIMED_OUT_DEPLOYMENT: Final = { "model_name": "test-model", "litellm_params": {"model": "openai/test-model", "api_key": "test-key"},