fix(router)!: count an explicit allowed_fails of 3 and keep fallback deployments out of cooldown (#46183)

* fix(router): count an explicit allowed_fails equal to the default as set

Router(allowed_fails=3) stored the same value as an unset Router, so
_is_allowed_fails_set_on_router treated it as "not set" and kept the
rate-based cooldown logic. Keep the passed value in explicit_allowed_fails,
like _explicit_timeout, and count it as set.

* chore(router): drop the comment on explicit_allowed_fails

* fix(router): keep the fallback deployment out of cooldown when no deployment can be picked

When every deployment of the requested model is in cooldown, the router raises
"No deployments available" and schedules the failure callbacks on the request's
logging object. By the time they run, the fallback has already written its own
deployment into that object, so the 429 counted as a failure of the healthy
fallback deployment and cooled it down too, leaving the model group with nothing
to serve.

Tag the selection error and skip cooldown accounting for it, the same way
advisor orchestration failures are skipped.

* test(router): annotate the new cooldown test variables with Final

* test(router): pin that a saved allowed_fails of 3 stays unset and type the fallback helper

* test(router): use a fixed start time in the fallback helper

* test(router): cover a fallback group staying up after the primary's selection fails

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>

litellm/router.py

@@ -238,11 +238,12 @@ from litellm.router_utils.cooldown_handlers import (
     async_get_cooldown_deployments_with_debug_info,
     get_cooldown_deployment_info,
     get_cooldown_deployments,
-    is_advisor_orchestration_failure,
     is_background_response_poll_not_found,
     is_caller_scoped_auth_failure,
     is_caller_timeout_408,
+    is_failure_outside_selected_deployment,
     is_video_lookup_not_found,
+    mark_deployment_selection_failure,
     set_cooldown_deployments,
 )
 from litellm.router_utils.deployment_pricing import deployment_pricing_models
@@ -1519,6 +1520,7 @@ class Router:
             self.allowed_fails = allowed_fails
         else:
             self.allowed_fails = litellm.allowed_fails
+        self.explicit_allowed_fails = allowed_fails
         self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS
         self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time)
         self.disable_cooldowns = disable_cooldowns
@@ -9579,10 +9581,10 @@ class Router:
         try:
             exception: Final = kwargs.get("exception", None)
 
-            if is_advisor_orchestration_failure(exception):
+            if is_failure_outside_selected_deployment(exception):
                 verbose_router_logger.debug(
                     "Router: Exiting 'deployment_callback_on_failure' without cooldown. "
-                    "Failure originated from advisor orchestration, not the selected deployment."
+                    "Failure came from advisor orchestration or deployment selection, not the selected deployment."
                 )
                 return False
 
@@ -15473,6 +15475,7 @@ class Router:
             phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes)
             return deployment
         except Exception as e:
+            mark_deployment_selection_failure(e)
             traceback_exception: Final = traceback.format_exc()
             # if router rejects call -> log to langfuse/otel/etc.
             if request_kwargs is not None:
@@ -15641,6 +15644,7 @@ class Router:
 
             return deployment
         except Exception as e:
+            mark_deployment_selection_failure(e)
             traceback_exception: Final = traceback.format_exc()
             if request_kwargs is not None:
                 logging_obj: Final = request_kwargs.get("litellm_logging_obj", None)

litellm/router_utils/cooldown_handlers.py

@@ -57,6 +57,7 @@ else:
     Span = Any
 
 _ADVISOR_ORCHESTRATION_FAILURE_ATTR: Final = "_litellm_advisor_orchestration_failure"
+_DEPLOYMENT_SELECTION_FAILURE_ATTR: Final = "_litellm_deployment_selection_failure"
 _CREDENTIAL_VALUES_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 
 
@@ -79,6 +80,16 @@ def is_advisor_orchestration_failure(exception: BaseException | None) -> bool:
     return bool(getattr(exception, _ADVISOR_ORCHESTRATION_FAILURE_ATTR, False))
 
 
+def mark_deployment_selection_failure(exception: BaseException) -> None:
+    setattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, True)
+
+
+def is_failure_outside_selected_deployment(exception: BaseException | None) -> bool:
+    return is_advisor_orchestration_failure(exception) or bool(
+        getattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, False)
+    )
+
+
 _BACKGROUND_RESPONSE_POLL_CALL_ORIGINS: Final[frozenset[InternalCallOrigin]] = frozenset(
     {BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN}
 )
@@ -781,14 +792,17 @@ def _is_allowed_fails_set_on_router(
     litellm_router_instance: LitellmRouter,
 ) -> bool:
     """
-    Check if Router.allowed_fails is set or is Non-default Value
+    Check if Router.allowed_fails was passed to the Router or is a Non-default Value
 
     Returns:
-    - True if Router.allowed_fails is set or is Non-default Value
-    - False if Router.allowed_fails is None or is Default Value
+    - True if allowed_fails was passed to the Router, even when it equals the default
+    - True if Router.allowed_fails is a Non-default Value
+    - False if Router.allowed_fails is None, or is the Default Value and was not passed
     """
     if litellm_router_instance.allowed_fails is None:
         return False
+    if litellm_router_instance.explicit_allowed_fails is not None:
+        return True
     if litellm_router_instance.allowed_fails != litellm.allowed_fails:
         return True
     return False

tests/unit/router_utils/test_cooldown_handlers.py

@@ -1,10 +1,12 @@
 import asyncio
 import copy
+import datetime
 import importlib
 import itertools
 import random
 import time
-from collections.abc import Mapping
+import uuid
+from collections.abc import Mapping, Sequence
 from typing import Final
 from unittest.mock import MagicMock, patch
 
@@ -682,6 +684,45 @@ class TestDeploymentLevelAllowedFails:
         )
 
 
+def _gpt_4_group_with_a_sibling() -> Sequence[Mapping[str, object]]:
+    return (
+        {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}},
+        {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "secondary"}},
+    )
+
+
+def _primary_cooldown_decisions_on_repeated_503s(router: Router) -> tuple[bool, ...]:
+    exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4")
+    return tuple(
+        _should_cooldown_deployment(
+            litellm_router_instance=router,
+            deployment="primary",
+            exception_status=503,
+            original_exception=exception,
+        )
+        for _ in range(litellm.allowed_fails + 1)
+    )
+
+
+@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
+class TestRouterLevelAllowedFailsEqualToDefault:
+    def test_allowed_fails_equal_to_the_default_cools_down_after_that_many_fails(self):
+        router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling()), allowed_fails=litellm.allowed_fails)
+
+        assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * litellm.allowed_fails + (True,)
+
+    def test_unset_allowed_fails_keeps_the_rate_based_default(self):
+        router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling()))
+
+        assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * (litellm.allowed_fails + 1)
+
+    def test_default_allowed_fails_saved_through_update_settings_keeps_the_rate_based_default(self):
+        router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling()))
+        router.update_settings(allowed_fails=litellm.allowed_fails)
+
+        assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * (litellm.allowed_fails + 1)
+
+
 @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
 class TestDeploymentLevelAllowedFailsPolicyByExceptionType:
     def test_rate_limit_error_triggers_cooldown_with_zero_threshold(self):
@@ -2931,6 +2972,104 @@ async def test_bad_request_error_does_not_cool_down_a_deployment(
     assert tuple(cooldown_deployments) == ()
 
 
+async def _complete_like_the_proxy(router: Router, model: str) -> str:
+    logging_obj, request = litellm.utils.function_setup(
+        original_function="acompletion",
+        rules_obj=litellm.utils.Rules(),
+        start_time=datetime.datetime(2026, 1, 1),
+        model=model,
+        messages=[{"role": "user", "content": "hi"}],
+        litellm_call_id=str(uuid.uuid4()),
+    )
+    response: Final = await router.acompletion(**request, litellm_logging_obj=logging_obj)
+    model_id: Final = response._hidden_params["model_id"]
+    assert isinstance(model_id, str)
+    return model_id
+
+
+_CHAT_COMPLETION_OK: Final = {
+    "id": "chatcmpl-1",
+    "object": "chat.completion",
+    "created": 1,
+    "model": "gpt-4o",
+    "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
+    "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+}
+
+
+def _openai_deployment(model_name: str, host: str, deployment_id: str) -> Mapping[str, object]:
+    return {
+        "model_name": model_name,
+        "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": f"https://{host}/v1"},
+        "model_info": {"id": deployment_id},
+    }
+
+
+@pytest.mark.asyncio
+async def test_fallback_deployment_stays_up_while_the_primary_is_in_cooldown(
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    respx_mock.post("https://primary.test/v1/chat/completions").mock(
+        return_value=httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}})
+    )
+    backup_route: Final = respx_mock.post("https://backup.test/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK)
+    )
+    router: Final = litellm.Router(
+        model_list=[
+            _openai_deployment("primary", "primary.test", "primary-dep"),
+            _openai_deployment("backup", "backup.test", "backup-dep"),
+        ],
+        fallbacks=[{"primary": ["backup"]}],
+        allowed_fails=litellm.allowed_fails,
+        num_retries=0,
+    )
+
+    served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(20)]
+    cooldown_deployments: Final = await async_get_cooldown_deployments(
+        litellm_router_instance=router,
+        parent_otel_span=None,
+    )
+
+    assert served_by == ["backup-dep"] * 20
+    assert len(backup_route.calls) == 20
+    assert tuple(cooldown_deployments) == ("primary-dep",)
+
+
+@pytest.mark.asyncio
+async def test_fallback_group_stays_up_after_a_bad_key_cools_the_primary_under_the_rate_based_default(
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    respx_mock.post("https://primary.test/v1/chat/completions").mock(
+        return_value=httpx.Response(401, json={"error": {"message": "bad key", "type": "invalid_request_error"}})
+    )
+    respx_mock.post("https://backup.test/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK)
+    )
+    router: Final = litellm.Router(
+        model_list=[
+            _openai_deployment("primary", "primary.test", "primary-dep"),
+            _openai_deployment("backup", "backup.test", "backup-1"),
+            _openai_deployment("backup", "backup.test", "backup-2"),
+        ],
+        fallbacks=[{"primary": ["backup"]}],
+        num_retries=0,
+    )
+
+    served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(12)]
+    cooldown_deployments: Final = await async_get_cooldown_deployments(
+        litellm_router_instance=router,
+        parent_otel_span=None,
+    )
+
+    assert set(served_by) <= {"backup-1", "backup-2"}
+    assert tuple(cooldown_deployments) == ("primary-dep",)
+
+
 TIMED_OUT_DEPLOYMENT: Final = {
     "model_name": "test-model",
     "litellm_params": {"model": "openai/test-model", "api_key": "test-key"},