diff --git a/litellm/router.py b/litellm/router.py index 74facf200d2..a94db3b1387 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -235,11 +235,12 @@ from litellm.router_utils.cooldown_handlers import ( async_get_cooldown_deployments_with_debug_info, get_cooldown_deployment_info, get_cooldown_deployments, - is_advisor_orchestration_failure, is_background_response_poll_not_found, is_caller_scoped_auth_failure, is_caller_timeout_408, + is_failure_outside_selected_deployment, is_video_lookup_not_found, + mark_deployment_selection_failure, set_cooldown_deployments, ) from litellm.router_utils.embedding_batch_size_check import ( @@ -1471,6 +1472,7 @@ class Router: self.allowed_fails = allowed_fails else: self.allowed_fails = litellm.allowed_fails + self.explicit_allowed_fails = allowed_fails self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time) self.disable_cooldowns = disable_cooldowns @@ -9333,10 +9335,10 @@ class Router: try: exception: Final = kwargs.get("exception", None) - if is_advisor_orchestration_failure(exception): + if is_failure_outside_selected_deployment(exception): verbose_router_logger.debug( "Router: Exiting 'deployment_callback_on_failure' without cooldown. " - "Failure originated from advisor orchestration, not the selected deployment." + "Failure came from advisor orchestration or deployment selection, not the selected deployment." ) return False @@ -15103,6 +15105,7 @@ class Router: phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes) return deployment except Exception as e: + mark_deployment_selection_failure(e) traceback_exception: Final = traceback.format_exc() # if router rejects call -> log to langfuse/otel/etc. if request_kwargs is not None: @@ -15273,6 +15276,7 @@ class Router: return deployment except Exception as e: + mark_deployment_selection_failure(e) traceback_exception: Final = traceback.format_exc() if request_kwargs is not None: logging_obj: Final = request_kwargs.get("litellm_logging_obj", None) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index c362c6899db..db6cdc9f8e2 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -57,6 +57,7 @@ else: Span = Any _ADVISOR_ORCHESTRATION_FAILURE_ATTR: Final = "_litellm_advisor_orchestration_failure" +_DEPLOYMENT_SELECTION_FAILURE_ATTR: Final = "_litellm_deployment_selection_failure" _CREDENTIAL_VALUES_ADAPTER: Final = TypeAdapter(Mapping[str, object]) @@ -79,6 +80,16 @@ def is_advisor_orchestration_failure(exception: BaseException | None) -> bool: return bool(getattr(exception, _ADVISOR_ORCHESTRATION_FAILURE_ATTR, False)) +def mark_deployment_selection_failure(exception: BaseException) -> None: + setattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, True) + + +def is_failure_outside_selected_deployment(exception: BaseException | None) -> bool: + return is_advisor_orchestration_failure(exception) or bool( + getattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, False) + ) + + _BACKGROUND_RESPONSE_POLL_CALL_ORIGINS: Final[frozenset[InternalCallOrigin]] = frozenset( {BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN} ) @@ -741,14 +752,17 @@ def _is_allowed_fails_set_on_router( litellm_router_instance: LitellmRouter, ) -> bool: """ - Check if Router.allowed_fails is set or is Non-default Value + Check if Router.allowed_fails was passed to the Router or is a Non-default Value Returns: - - True if Router.allowed_fails is set or is Non-default Value - - False if Router.allowed_fails is None or is Default Value + - True if allowed_fails was passed to the Router, even when it equals the default + - True if Router.allowed_fails is a Non-default Value + - False if Router.allowed_fails is None, or is the Default Value and was not passed """ if litellm_router_instance.allowed_fails is None: return False + if litellm_router_instance.explicit_allowed_fails is not None: + return True if litellm_router_instance.allowed_fails != litellm.allowed_fails: return True return False diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 0c9d7ab9f85..817aa7b1ec4 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -1,9 +1,11 @@ import asyncio import copy +import datetime import importlib import itertools import random import time +import uuid from collections.abc import Mapping from typing import Final from unittest.mock import MagicMock, patch @@ -682,6 +684,79 @@ class TestDeploymentLevelAllowedFails: ) +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +class TestRouterLevelAllowedFailsEqualToDefault: + def test_allowed_fails_equal_to_the_default_cools_down_after_that_many_fails(self): + """ + Passing allowed_fails equal to litellm.allowed_fails must switch on the + count-based cooldown like any other value, not count as "not set". + """ + router: Final = _make_router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}} + ], + allowed_fails=litellm.allowed_fails, + ) + _exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4") + + decisions: Final = [ + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="primary", + exception_status=503, + original_exception=_exception, + ) + for _ in range(litellm.allowed_fails + 1) + ] + + assert decisions == [False] * litellm.allowed_fails + [True] + + def test_unset_allowed_fails_keeps_the_rate_based_default(self): + """ + Without allowed_fails the router keeps its rate-based logic, which does not + cool a single-deployment group down on a handful of failures. + """ + router: Final = _make_router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}} + ], + ) + _exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4") + + decisions: Final = [ + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="primary", + exception_status=503, + original_exception=_exception, + ) + for _ in range(litellm.allowed_fails + 1) + ] + + assert decisions == [False] * (litellm.allowed_fails + 1) + + def test_default_allowed_fails_saved_through_update_settings_keeps_the_rate_based_default(self): + router: Final = _make_router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}} + ], + ) + router.update_settings(allowed_fails=litellm.allowed_fails) + _exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4") + + decisions: Final = [ + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="primary", + exception_status=503, + original_exception=_exception, + ) + for _ in range(litellm.allowed_fails + 1) + ] + + assert decisions == [False] * (litellm.allowed_fails + 1) + + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestDeploymentLevelAllowedFailsPolicyByExceptionType: def test_rate_limit_error_triggers_cooldown_with_zero_threshold(self): @@ -2931,6 +3006,72 @@ async def test_bad_request_error_does_not_cool_down_a_deployment( assert tuple(cooldown_deployments) == () +async def _complete_like_the_proxy(router: Router, model: str) -> str: + logging_obj, request = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.datetime(2026, 1, 1), + model=model, + messages=[{"role": "user", "content": "hi"}], + litellm_call_id=str(uuid.uuid4()), + ) + response: Final = await router.acompletion(**request, litellm_logging_obj=logging_obj) + model_id: Final = response._hidden_params["model_id"] + assert isinstance(model_id, str) + return model_id + + +@pytest.mark.asyncio +async def test_fallback_deployment_stays_up_while_the_primary_is_in_cooldown( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://primary.test/v1/chat/completions").mock( + return_value=httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}}) + ) + backup_route: Final = respx_mock.post("https://backup.test/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "https://primary.test/v1"}, + "model_info": {"id": "primary-dep"}, + }, + { + "model_name": "backup", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "https://backup.test/v1"}, + "model_info": {"id": "backup-dep"}, + }, + ], + fallbacks=[{"primary": ["backup"]}], + allowed_fails=litellm.allowed_fails, + num_retries=0, + ) + + served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(20)] + cooldown_deployments: Final = await async_get_cooldown_deployments( + litellm_router_instance=router, + parent_otel_span=None, + ) + + assert served_by == ["backup-dep"] * 20 + assert len(backup_route.calls) == 20 + assert tuple(cooldown_deployments) == ("primary-dep",) + + @pytest.mark.parametrize("use_policy", [False, True], ids=["allowed_fails", "allowed_fails_policy"]) @pytest.mark.asyncio async def test_allowed_fails_cools_a_single_deployment(