Merge e88e8e1659 into 730db4d228

2026-10-28 06:23:07 +00:00 · 2026-10-20 15:07:48 -07:00 · 2026-10-20 15:07:48 -07:00 · bbe4215739
commit bbe4215739
parent 730db4d228 e88e8e1659
3 changed files with 165 additions and 6 deletions
--- a/litellm/router.py
+++ b/litellm/router.py
@ -235,11 +235,12 @@ from litellm.router_utils.cooldown_handlers import (
    async_get_cooldown_deployments_with_debug_info,
    get_cooldown_deployment_info,
    get_cooldown_deployments,
-    is_advisor_orchestration_failure,
    is_background_response_poll_not_found,
    is_caller_scoped_auth_failure,
    is_caller_timeout_408,
+    is_failure_outside_selected_deployment,
    is_video_lookup_not_found,
+    mark_deployment_selection_failure,
    set_cooldown_deployments,
 )
 from litellm.router_utils.embedding_batch_size_check import (
@ -1471,6 +1472,7 @@ class Router:
            self.allowed_fails = allowed_fails
        else:
            self.allowed_fails = litellm.allowed_fails
+        self.explicit_allowed_fails = allowed_fails
        self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS
        self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time)
        self.disable_cooldowns = disable_cooldowns
@ -9333,10 +9335,10 @@ class Router:
        try:
            exception: Final = kwargs.get("exception", None)

-            if is_advisor_orchestration_failure(exception):
+            if is_failure_outside_selected_deployment(exception):
                verbose_router_logger.debug(
                    "Router: Exiting 'deployment_callback_on_failure' without cooldown. "
-                    "Failure originated from advisor orchestration, not the selected deployment."
+                    "Failure came from advisor orchestration or deployment selection, not the selected deployment."
                )
                return False

@ -15103,6 +15105,7 @@ class Router:
            phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes)
            return deployment
        except Exception as e:
+            mark_deployment_selection_failure(e)
            traceback_exception: Final = traceback.format_exc()
            # if router rejects call -> log to langfuse/otel/etc.
            if request_kwargs is not None:
@ -15273,6 +15276,7 @@ class Router:

            return deployment
        except Exception as e:
+            mark_deployment_selection_failure(e)
            traceback_exception: Final = traceback.format_exc()
            if request_kwargs is not None:
                logging_obj: Final = request_kwargs.get("litellm_logging_obj", None)
--- a/litellm/router_utils/cooldown_handlers.py
+++ b/litellm/router_utils/cooldown_handlers.py
@ -57,6 +57,7 @@ else:
    Span = Any

 _ADVISOR_ORCHESTRATION_FAILURE_ATTR: Final = "_litellm_advisor_orchestration_failure"
+_DEPLOYMENT_SELECTION_FAILURE_ATTR: Final = "_litellm_deployment_selection_failure"
 _CREDENTIAL_VALUES_ADAPTER: Final = TypeAdapter(Mapping[str, object])


@ -79,6 +80,16 @@ def is_advisor_orchestration_failure(exception: BaseException | None) -> bool:
    return bool(getattr(exception, _ADVISOR_ORCHESTRATION_FAILURE_ATTR, False))


+def mark_deployment_selection_failure(exception: BaseException) -> None:
+    setattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, True)
+
+
+def is_failure_outside_selected_deployment(exception: BaseException | None) -> bool:
+    return is_advisor_orchestration_failure(exception) or bool(
+        getattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, False)
+    )
+
+
 _BACKGROUND_RESPONSE_POLL_CALL_ORIGINS: Final[frozenset[InternalCallOrigin]] = frozenset(
    {BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN}
 )
@ -741,14 +752,17 @@ def _is_allowed_fails_set_on_router(
    litellm_router_instance: LitellmRouter,
 ) -> bool:
    """
-    Check if Router.allowed_fails is set or is Non-default Value
+    Check if Router.allowed_fails was passed to the Router or is a Non-default Value

    Returns:
-    - True if Router.allowed_fails is set or is Non-default Value
-    - False if Router.allowed_fails is None or is Default Value
+    - True if allowed_fails was passed to the Router, even when it equals the default
+    - True if Router.allowed_fails is a Non-default Value
+    - False if Router.allowed_fails is None, or is the Default Value and was not passed
    """
    if litellm_router_instance.allowed_fails is None:
        return False
+    if litellm_router_instance.explicit_allowed_fails is not None:
+        return True
    if litellm_router_instance.allowed_fails != litellm.allowed_fails:
        return True
    return False
--- a/tests/unit/router_utils/test_cooldown_handlers.py
+++ b/tests/unit/router_utils/test_cooldown_handlers.py
@ -1,9 +1,11 @@
 import asyncio
 import copy
+import datetime
 import importlib
 import itertools
 import random
 import time
+import uuid
 from collections.abc import Mapping
 from typing import Final
 from unittest.mock import MagicMock, patch
@ -682,6 +684,79 @@ class TestDeploymentLevelAllowedFails:
        )


+@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
+class TestRouterLevelAllowedFailsEqualToDefault:
+    def test_allowed_fails_equal_to_the_default_cools_down_after_that_many_fails(self):
+        """
+        Passing allowed_fails equal to litellm.allowed_fails must switch on the
+        count-based cooldown like any other value, not count as "not set".
+        """
+        router: Final = _make_router(
+            model_list=[
+                {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}}
+            ],
+            allowed_fails=litellm.allowed_fails,
+        )
+        _exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4")
+
+        decisions: Final = [
+            _should_cooldown_deployment(
+                litellm_router_instance=router,
+                deployment="primary",
+                exception_status=503,
+                original_exception=_exception,
+            )
+            for _ in range(litellm.allowed_fails + 1)
+        ]
+
+        assert decisions == [False] * litellm.allowed_fails + [True]
+
+    def test_unset_allowed_fails_keeps_the_rate_based_default(self):
+        """
+        Without allowed_fails the router keeps its rate-based logic, which does not
+        cool a single-deployment group down on a handful of failures.
+        """
+        router: Final = _make_router(
+            model_list=[
+                {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}}
+            ],
+        )
+        _exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4")
+
+        decisions: Final = [
+            _should_cooldown_deployment(
+                litellm_router_instance=router,
+                deployment="primary",
+                exception_status=503,
+                original_exception=_exception,
+            )
+            for _ in range(litellm.allowed_fails + 1)
+        ]
+
+        assert decisions == [False] * (litellm.allowed_fails + 1)
+
+    def test_default_allowed_fails_saved_through_update_settings_keeps_the_rate_based_default(self):
+        router: Final = _make_router(
+            model_list=[
+                {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}}
+            ],
+        )
+        router.update_settings(allowed_fails=litellm.allowed_fails)
+        _exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4")
+
+        decisions: Final = [
+            _should_cooldown_deployment(
+                litellm_router_instance=router,
+                deployment="primary",
+                exception_status=503,
+                original_exception=_exception,
+            )
+            for _ in range(litellm.allowed_fails + 1)
+        ]
+
+        assert decisions == [False] * (litellm.allowed_fails + 1)
+
+
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
 class TestDeploymentLevelAllowedFailsPolicyByExceptionType:
    def test_rate_limit_error_triggers_cooldown_with_zero_threshold(self):
@ -2931,6 +3006,72 @@ async def test_bad_request_error_does_not_cool_down_a_deployment(
    assert tuple(cooldown_deployments) == ()


+async def _complete_like_the_proxy(router: Router, model: str) -> str:
+    logging_obj, request = litellm.utils.function_setup(
+        original_function="acompletion",
+        rules_obj=litellm.utils.Rules(),
+        start_time=datetime.datetime(2026, 1, 1),
+        model=model,
+        messages=[{"role": "user", "content": "hi"}],
+        litellm_call_id=str(uuid.uuid4()),
+    )
+    response: Final = await router.acompletion(**request, litellm_logging_obj=logging_obj)
+    model_id: Final = response._hidden_params["model_id"]
+    assert isinstance(model_id, str)
+    return model_id
+
+
+@pytest.mark.asyncio
+async def test_fallback_deployment_stays_up_while_the_primary_is_in_cooldown(
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    respx_mock.post("https://primary.test/v1/chat/completions").mock(
+        return_value=httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}})
+    )
+    backup_route: Final = respx_mock.post("https://backup.test/v1/chat/completions").mock(
+        return_value=httpx.Response(
+            200,
+            json={
+                "id": "chatcmpl-1",
+                "object": "chat.completion",
+                "created": 1,
+                "model": "gpt-4o",
+                "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
+                "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+            },
+        )
+    )
+    router: Final = litellm.Router(
+        model_list=[
+            {
+                "model_name": "primary",
+                "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "https://primary.test/v1"},
+                "model_info": {"id": "primary-dep"},
+            },
+            {
+                "model_name": "backup",
+                "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "https://backup.test/v1"},
+                "model_info": {"id": "backup-dep"},
+            },
+        ],
+        fallbacks=[{"primary": ["backup"]}],
+        allowed_fails=litellm.allowed_fails,
+        num_retries=0,
+    )
+
+    served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(20)]
+    cooldown_deployments: Final = await async_get_cooldown_deployments(
+        litellm_router_instance=router,
+        parent_otel_span=None,
+    )
+
+    assert served_by == ["backup-dep"] * 20
+    assert len(backup_route.calls) == 20
+    assert tuple(cooldown_deployments) == ("primary-dep",)
+
+
@pytest.mark.parametrize("use_policy", [False, True], ids=["allowed_fails", "allowed_fails_policy"])
@pytest.mark.asyncio
 async def test_allowed_fails_cools_a_single_deployment(