Merge 4e2d1c696f into 82ca10cc2d

litellm/router.py

@@ -1398,6 +1398,7 @@ class Router:
             self.allowed_fails = allowed_fails
         else:
             self.allowed_fails = litellm.allowed_fails
+        self.explicit_allowed_fails = allowed_fails
         self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS
         self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time)
         self.disable_cooldowns = disable_cooldowns

litellm/router_utils/cooldown_handlers.py

@@ -712,14 +712,17 @@ def _is_allowed_fails_set_on_router(
     litellm_router_instance: LitellmRouter,
 ) -> bool:
     """
-    Check if Router.allowed_fails is set or is Non-default Value
+    Check if Router.allowed_fails was passed to the Router or is a Non-default Value
 
     Returns:
-    - True if Router.allowed_fails is set or is Non-default Value
-    - False if Router.allowed_fails is None or is Default Value
+    - True if allowed_fails was passed to the Router, even when it equals the default
+    - True if Router.allowed_fails is a Non-default Value
+    - False if Router.allowed_fails is None, or is the Default Value and was not passed
     """
     if litellm_router_instance.allowed_fails is None:
         return False
+    if litellm_router_instance.explicit_allowed_fails is not None:
+        return True
     if litellm_router_instance.allowed_fails != litellm.allowed_fails:
         return True
     return False

tests/unit/router_utils/test_cooldown_handlers.py

@@ -678,6 +678,58 @@ class TestDeploymentLevelAllowedFails:
         )
 
 
+@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
+class TestRouterLevelAllowedFailsEqualToDefault:
+    def test_allowed_fails_equal_to_the_default_cools_down_after_that_many_fails(self):
+        """
+        Passing allowed_fails equal to litellm.allowed_fails must switch on the
+        count-based cooldown like any other value, not count as "not set".
+        """
+        router = _make_router(
+            model_list=[
+                {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}}
+            ],
+            allowed_fails=litellm.allowed_fails,
+        )
+        _exception = litellm.ServiceUnavailableError("down", "openai", "gpt-4")
+
+        decisions = [
+            _should_cooldown_deployment(
+                litellm_router_instance=router,
+                deployment="primary",
+                exception_status=503,
+                original_exception=_exception,
+            )
+            for _ in range(litellm.allowed_fails + 1)
+        ]
+
+        assert decisions == [False] * litellm.allowed_fails + [True]
+
+    def test_unset_allowed_fails_keeps_the_rate_based_default(self):
+        """
+        Without allowed_fails the router keeps its rate-based logic, which does not
+        cool a single-deployment group down on a handful of failures.
+        """
+        router = _make_router(
+            model_list=[
+                {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}}
+            ],
+        )
+        _exception = litellm.ServiceUnavailableError("down", "openai", "gpt-4")
+
+        decisions = [
+            _should_cooldown_deployment(
+                litellm_router_instance=router,
+                deployment="primary",
+                exception_status=503,
+                original_exception=_exception,
+            )
+            for _ in range(litellm.allowed_fails + 1)
+        ]
+
+        assert decisions == [False] * (litellm.allowed_fails + 1)
+
+
 @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
 class TestDeploymentLevelAllowedFailsPolicyByExceptionType:
     def test_rate_limit_error_triggers_cooldown_with_zero_threshold(self):