diff --git a/litellm/router.py b/litellm/router.py index 99adc6c5fa6..7ae246af10b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1398,6 +1398,7 @@ class Router: self.allowed_fails = allowed_fails else: self.allowed_fails = litellm.allowed_fails + self.explicit_allowed_fails = allowed_fails self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time) self.disable_cooldowns = disable_cooldowns diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index bb64d912c4c..10e70ac8166 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -712,14 +712,17 @@ def _is_allowed_fails_set_on_router( litellm_router_instance: LitellmRouter, ) -> bool: """ - Check if Router.allowed_fails is set or is Non-default Value + Check if Router.allowed_fails was passed to the Router or is a Non-default Value Returns: - - True if Router.allowed_fails is set or is Non-default Value - - False if Router.allowed_fails is None or is Default Value + - True if allowed_fails was passed to the Router, even when it equals the default + - True if Router.allowed_fails is a Non-default Value + - False if Router.allowed_fails is None, or is the Default Value and was not passed """ if litellm_router_instance.allowed_fails is None: return False + if litellm_router_instance.explicit_allowed_fails is not None: + return True if litellm_router_instance.allowed_fails != litellm.allowed_fails: return True return False diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 2376e0f4d0c..e459befaec9 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -678,6 +678,58 @@ class TestDeploymentLevelAllowedFails: ) +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +class TestRouterLevelAllowedFailsEqualToDefault: + def test_allowed_fails_equal_to_the_default_cools_down_after_that_many_fails(self): + """ + Passing allowed_fails equal to litellm.allowed_fails must switch on the + count-based cooldown like any other value, not count as "not set". + """ + router = _make_router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}} + ], + allowed_fails=litellm.allowed_fails, + ) + _exception = litellm.ServiceUnavailableError("down", "openai", "gpt-4") + + decisions = [ + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="primary", + exception_status=503, + original_exception=_exception, + ) + for _ in range(litellm.allowed_fails + 1) + ] + + assert decisions == [False] * litellm.allowed_fails + [True] + + def test_unset_allowed_fails_keeps_the_rate_based_default(self): + """ + Without allowed_fails the router keeps its rate-based logic, which does not + cool a single-deployment group down on a handful of failures. + """ + router = _make_router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}} + ], + ) + _exception = litellm.ServiceUnavailableError("down", "openai", "gpt-4") + + decisions = [ + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="primary", + exception_status=503, + original_exception=_exception, + ) + for _ in range(litellm.allowed_fails + 1) + ] + + assert decisions == [False] * (litellm.allowed_fails + 1) + + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestDeploymentLevelAllowedFailsPolicyByExceptionType: def test_rate_limit_error_triggers_cooldown_with_zero_threshold(self):