From af47f7580730c3fde84709755fe64d62ea82d1ae Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Tue, 13 Oct 2026 11:03:29 +0200 Subject: [PATCH 1/6] fix(router): count an explicit allowed_fails equal to the default as set Router(allowed_fails=3) stored the same value as an unset Router, so _is_allowed_fails_set_on_router treated it as "not set" and kept the rate-based cooldown logic. Keep the passed value in explicit_allowed_fails, like _explicit_timeout, and count it as set. --- litellm/router.py | 1 + litellm/router_utils/cooldown_handlers.py | 9 ++-- .../router_utils/test_cooldown_handlers.py | 52 +++++++++++++++++++ 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index cdede46ef5c..bc90f081036 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1196,6 +1196,7 @@ class Router: self.allowed_fails = allowed_fails else: self.allowed_fails = litellm.allowed_fails + self.explicit_allowed_fails = allowed_fails # None when user did not pass allowed_fails self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time) self.disable_cooldowns = disable_cooldowns diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index a5f6277d2d7..ee6e1d31227 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -697,14 +697,17 @@ def _is_allowed_fails_set_on_router( litellm_router_instance: LitellmRouter, ) -> bool: """ - Check if Router.allowed_fails is set or is Non-default Value + Check if Router.allowed_fails was passed to the Router or is a Non-default Value Returns: - - True if Router.allowed_fails is set or is Non-default Value - - False if Router.allowed_fails is None or is Default Value + - True if allowed_fails was passed to the Router, even when it equals the default + - True if Router.allowed_fails is a Non-default Value + - False if Router.allowed_fails is None, or is the Default Value and was not passed """ if litellm_router_instance.allowed_fails is None: return False + if litellm_router_instance.explicit_allowed_fails is not None: + return True if litellm_router_instance.allowed_fails != litellm.allowed_fails: return True return False diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 2376e0f4d0c..e459befaec9 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -678,6 +678,58 @@ class TestDeploymentLevelAllowedFails: ) +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +class TestRouterLevelAllowedFailsEqualToDefault: + def test_allowed_fails_equal_to_the_default_cools_down_after_that_many_fails(self): + """ + Passing allowed_fails equal to litellm.allowed_fails must switch on the + count-based cooldown like any other value, not count as "not set". + """ + router = _make_router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}} + ], + allowed_fails=litellm.allowed_fails, + ) + _exception = litellm.ServiceUnavailableError("down", "openai", "gpt-4") + + decisions = [ + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="primary", + exception_status=503, + original_exception=_exception, + ) + for _ in range(litellm.allowed_fails + 1) + ] + + assert decisions == [False] * litellm.allowed_fails + [True] + + def test_unset_allowed_fails_keeps_the_rate_based_default(self): + """ + Without allowed_fails the router keeps its rate-based logic, which does not + cool a single-deployment group down on a handful of failures. + """ + router = _make_router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}} + ], + ) + _exception = litellm.ServiceUnavailableError("down", "openai", "gpt-4") + + decisions = [ + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="primary", + exception_status=503, + original_exception=_exception, + ) + for _ in range(litellm.allowed_fails + 1) + ] + + assert decisions == [False] * (litellm.allowed_fails + 1) + + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestDeploymentLevelAllowedFailsPolicyByExceptionType: def test_rate_limit_error_triggers_cooldown_with_zero_threshold(self): From 4e2d1c696f9d5a4f95bcb385e07974c37fd5b03e Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Tue, 13 Oct 2026 11:26:53 +0200 Subject: [PATCH 2/6] chore(router): drop the comment on explicit_allowed_fails --- litellm/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index bc90f081036..ce56aed9ae1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1196,7 +1196,7 @@ class Router: self.allowed_fails = allowed_fails else: self.allowed_fails = litellm.allowed_fails - self.explicit_allowed_fails = allowed_fails # None when user did not pass allowed_fails + self.explicit_allowed_fails = allowed_fails self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time) self.disable_cooldowns = disable_cooldowns From 97a5fd71b838930e04fefed2c1a437eb441836b3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 23:24:30 -0700 Subject: [PATCH 3/6] fix(router): keep the fallback deployment out of cooldown when no deployment can be picked When every deployment of the requested model is in cooldown, the router raises "No deployments available" and schedules the failure callbacks on the request's logging object. By the time they run, the fallback has already written its own deployment into that object, so the 429 counted as a failure of the healthy fallback deployment and cooled it down too, leaving the model group with nothing to serve. Tag the selection error and skip cooldown accounting for it, the same way advisor orchestration failures are skipped. --- litellm/router.py | 9 ++- litellm/router_utils/cooldown_handlers.py | 11 ++++ .../router_utils/test_cooldown_handlers.py | 66 +++++++++++++++++++ 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index d8c28f11cbc..b8ae043848e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -226,10 +226,11 @@ from litellm.router_utils.cooldown_handlers import ( async_get_cooldown_deployments_with_debug_info, get_cooldown_deployment_info, get_cooldown_deployments, - is_advisor_orchestration_failure, is_background_response_poll_not_found, is_caller_scoped_auth_failure, is_caller_timeout_408, + is_failure_outside_selected_deployment, + mark_deployment_selection_failure, set_cooldown_deployments, ) from litellm.router_utils.fallback_event_handlers import ( @@ -9167,10 +9168,10 @@ class Router: try: exception: Final = kwargs.get("exception", None) - if is_advisor_orchestration_failure(exception): + if is_failure_outside_selected_deployment(exception): verbose_router_logger.debug( "Router: Exiting 'deployment_callback_on_failure' without cooldown. " - "Failure originated from advisor orchestration, not the selected deployment." + "Failure came from advisor orchestration or deployment selection, not the selected deployment." ) return False @@ -14852,6 +14853,7 @@ class Router: phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes) return deployment except Exception as e: + mark_deployment_selection_failure(e) traceback_exception: Final = traceback.format_exc() # if router rejects call -> log to langfuse/otel/etc. if request_kwargs is not None: @@ -15022,6 +15024,7 @@ class Router: return deployment except Exception as e: + mark_deployment_selection_failure(e) traceback_exception: Final = traceback.format_exc() if request_kwargs is not None: logging_obj: Final = request_kwargs.get("litellm_logging_obj", None) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index a07385459f0..201409241dc 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -56,6 +56,7 @@ else: Span = Any _ADVISOR_ORCHESTRATION_FAILURE_ATTR: Final = "_litellm_advisor_orchestration_failure" +_DEPLOYMENT_SELECTION_FAILURE_ATTR: Final = "_litellm_deployment_selection_failure" _CREDENTIAL_VALUES_ADAPTER: Final = TypeAdapter(Mapping[str, object]) @@ -78,6 +79,16 @@ def is_advisor_orchestration_failure(exception: BaseException | None) -> bool: return bool(getattr(exception, _ADVISOR_ORCHESTRATION_FAILURE_ATTR, False)) +def mark_deployment_selection_failure(exception: BaseException) -> None: + setattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, True) + + +def is_failure_outside_selected_deployment(exception: BaseException | None) -> bool: + return is_advisor_orchestration_failure(exception) or bool( + getattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, False) + ) + + _BACKGROUND_RESPONSE_POLL_CALL_ORIGINS: Final[frozenset[InternalCallOrigin]] = frozenset( {BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN} ) diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index f6bd794431c..de4fc53a6ce 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -1,9 +1,11 @@ import asyncio import copy +import datetime import importlib import itertools import random import time +import uuid from collections.abc import Mapping from typing import Final from unittest.mock import MagicMock, patch @@ -2983,6 +2985,70 @@ async def test_bad_request_error_does_not_cool_down_a_deployment( assert tuple(cooldown_deployments) == () +async def _complete_like_the_proxy(router: Router, model: str) -> object: + logging_obj, request = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.datetime.now(), + model=model, + messages=[{"role": "user", "content": "hi"}], + litellm_call_id=str(uuid.uuid4()), + ) + response: Final = await router.acompletion(**request, litellm_logging_obj=logging_obj) + return response._hidden_params.get("model_id") + + +@pytest.mark.asyncio +async def test_fallback_deployment_stays_up_while_the_primary_is_in_cooldown( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://primary.test/v1/chat/completions").mock( + return_value=httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}}) + ) + backup_route: Final = respx_mock.post("https://backup.test/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "https://primary.test/v1"}, + "model_info": {"id": "primary-dep"}, + }, + { + "model_name": "backup", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "https://backup.test/v1"}, + "model_info": {"id": "backup-dep"}, + }, + ], + fallbacks=[{"primary": ["backup"]}], + allowed_fails=litellm.allowed_fails, + num_retries=0, + ) + + served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(20)] + cooldown_deployments: Final = await async_get_cooldown_deployments( + litellm_router_instance=router, + parent_otel_span=None, + ) + + assert served_by == ["backup-dep"] * 20 + assert len(backup_route.calls) == 20 + assert tuple(cooldown_deployments) == ("primary-dep",) + + @pytest.mark.parametrize("use_policy", [False, True], ids=["allowed_fails", "allowed_fails_policy"]) @pytest.mark.asyncio async def test_allowed_fails_cools_a_single_deployment( From 7b432a99649af93d10af76143d94fc4a91a4c378 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 00:42:08 -0700 Subject: [PATCH 4/6] test(router): annotate the new cooldown test variables with Final --- tests/unit/router_utils/test_cooldown_handlers.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index de4fc53a6ce..29efc0f24a2 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -691,15 +691,15 @@ class TestRouterLevelAllowedFailsEqualToDefault: Passing allowed_fails equal to litellm.allowed_fails must switch on the count-based cooldown like any other value, not count as "not set". """ - router = _make_router( + router: Final = _make_router( model_list=[ {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}} ], allowed_fails=litellm.allowed_fails, ) - _exception = litellm.ServiceUnavailableError("down", "openai", "gpt-4") + _exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4") - decisions = [ + decisions: Final = [ _should_cooldown_deployment( litellm_router_instance=router, deployment="primary", @@ -716,14 +716,14 @@ class TestRouterLevelAllowedFailsEqualToDefault: Without allowed_fails the router keeps its rate-based logic, which does not cool a single-deployment group down on a handful of failures. """ - router = _make_router( + router: Final = _make_router( model_list=[ {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}} ], ) - _exception = litellm.ServiceUnavailableError("down", "openai", "gpt-4") + _exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4") - decisions = [ + decisions: Final = [ _should_cooldown_deployment( litellm_router_instance=router, deployment="primary", From 1cb46a9d79bf087956f57a7199d0e96fe0a0fa8e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:44:15 -0700 Subject: [PATCH 5/6] test(router): pin that a saved allowed_fails of 3 stays unset and type the fallback helper --- .../router_utils/test_cooldown_handlers.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 29efc0f24a2..2035a72d1a1 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -735,6 +735,27 @@ class TestRouterLevelAllowedFailsEqualToDefault: assert decisions == [False] * (litellm.allowed_fails + 1) + def test_default_allowed_fails_saved_through_update_settings_keeps_the_rate_based_default(self): + router: Final = _make_router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}} + ], + ) + router.update_settings(allowed_fails=litellm.allowed_fails) + _exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4") + + decisions: Final = [ + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="primary", + exception_status=503, + original_exception=_exception, + ) + for _ in range(litellm.allowed_fails + 1) + ] + + assert decisions == [False] * (litellm.allowed_fails + 1) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestDeploymentLevelAllowedFailsPolicyByExceptionType: @@ -2985,7 +3006,7 @@ async def test_bad_request_error_does_not_cool_down_a_deployment( assert tuple(cooldown_deployments) == () -async def _complete_like_the_proxy(router: Router, model: str) -> object: +async def _complete_like_the_proxy(router: Router, model: str) -> str: logging_obj, request = litellm.utils.function_setup( original_function="acompletion", rules_obj=litellm.utils.Rules(), @@ -2995,7 +3016,9 @@ async def _complete_like_the_proxy(router: Router, model: str) -> object: litellm_call_id=str(uuid.uuid4()), ) response: Final = await router.acompletion(**request, litellm_logging_obj=logging_obj) - return response._hidden_params.get("model_id") + model_id: Final = response._hidden_params["model_id"] + assert isinstance(model_id, str) + return model_id @pytest.mark.asyncio From e88e8e1659ff7c1b213e57e52441ad6cf15f5fdc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:59:50 -0700 Subject: [PATCH 6/6] test(router): use a fixed start time in the fallback helper --- tests/unit/router_utils/test_cooldown_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 2035a72d1a1..817aa7b1ec4 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -3010,7 +3010,7 @@ async def _complete_like_the_proxy(router: Router, model: str) -> str: logging_obj, request = litellm.utils.function_setup( original_function="acompletion", rules_obj=litellm.utils.Rules(), - start_time=datetime.datetime.now(), + start_time=datetime.datetime(2026, 1, 1), model=model, messages=[{"role": "user", "content": "hi"}], litellm_call_id=str(uuid.uuid4()),