Merge pull request #40995 from BerriAI/litellm_all_deployments_cooldown_error

fix(router): name the all-deployments-in-cooldown error on 429 responses

litellm/router.py

@@ -13878,6 +13878,7 @@ class Router:
                 cooldown_time=_cooldown_time,
                 enable_pre_call_checks=self.enable_pre_call_checks,
                 cooldown_list=_cooldown_list,
+                model_ids=model_ids,
             )
 
         if strategy == "simple-shuffle":
@@ -13910,6 +13911,7 @@ class Router:
                 cooldown_time=_cooldown_time,
                 enable_pre_call_checks=self.enable_pre_call_checks,
                 cooldown_list=_cooldown_list,
+                model_ids=model_ids,
             )
         self._override_selector_pre_call_check(strategy, strategy_selector, deployment)
         verbose_router_logger.info(
@@ -13987,6 +13989,11 @@ class Router:
                 model=model,
                 llm_provider="",
             )
+        pass_through_model_ids: Final = tuple(
+            deployment["model_info"]["id"]
+            for deployment in pass_through_deployments
+            if "id" in deployment.get("model_info", {})
+        )
 
         # 4. Apply health-check and cooldown filtering
         parent_otel_span: Final[Span | None] = _get_parent_otel_span_from_kwargs(request_kwargs)
@@ -14024,6 +14031,7 @@ class Router:
                 cooldown_time=_cooldown_time,
                 enable_pre_call_checks=self.enable_pre_call_checks,
                 cooldown_list=_cooldown_list,
+                model_ids=pass_through_model_ids,
             )
 
         # 6. Apply load balancing strategy
@@ -14057,6 +14065,7 @@ class Router:
                 cooldown_time=_cooldown_time,
                 enable_pre_call_checks=self.enable_pre_call_checks,
                 cooldown_list=_cooldown_list,
+                model_ids=model_ids,
             )
         self._override_selector_pre_call_check(strategy, strategy_selector, deployment)
 

litellm/router_utils/handle_error.py

@@ -93,4 +93,5 @@ async def async_raise_no_deployment_exception(
         cooldown_time=_cooldown_time,
         enable_pre_call_checks=litellm_router_instance.enable_pre_call_checks,
         cooldown_list=cooldown_list_ids,
+        model_ids=model_ids,
     )

litellm/types/router.py

@@ -645,6 +645,7 @@ class RouterErrors(enum.Enum):
 
     user_defined_ratelimit_error = "Deployment over user-defined ratelimit."
     no_deployments_available = "No deployments available for selected model"
+    all_deployments_in_cooldown = "All deployments for selected model are in cooldown"
     no_deployments_with_tag_routing = "Not allowed to access model due to tags configuration"
     no_deployments_with_provider_budget_routing = "No deployments available - crossed budget"
     no_healthy_deployments = "There are no healthy deployments for this model"
@@ -868,6 +869,11 @@ class RouterRateLimitErrorBasic(ValueError):
         super().__init__(_message)
 
 
+class RouterErrorTypes(str, enum.Enum):
+    rate_limit_error = "rate_limit_error"
+    all_deployments_in_cooldown = "all_deployments_in_cooldown"
+
+
 class RouterRateLimitError(ValueError):
     def __init__(
         self,
@@ -875,12 +881,25 @@ class RouterRateLimitError(ValueError):
         cooldown_time: float,
         enable_pre_call_checks: bool,
         cooldown_list: list,
+        model_ids: Sequence[str] = (),
     ) -> None:
         self.model = model
         self.cooldown_time = cooldown_time
         self.enable_pre_call_checks = enable_pre_call_checks
         self.cooldown_list = cooldown_list
-        _message = f"{RouterErrors.no_deployments_available.value}, Try again in {cooldown_time} seconds. Passed model={model}. pre-call-checks={enable_pre_call_checks}, cooldown_list={cooldown_list}"
+        self.all_deployments_in_cooldown = bool(model_ids) and frozenset(model_ids) <= frozenset(cooldown_list)
+        self.type = (
+            RouterErrorTypes.all_deployments_in_cooldown.value
+            if self.all_deployments_in_cooldown
+            else RouterErrorTypes.rate_limit_error.value
+        )
+        _reason: Final = (
+            f" {RouterErrors.all_deployments_in_cooldown.value}." if self.all_deployments_in_cooldown else ""
+        )
+        _message: Final = (
+            f"{RouterErrors.no_deployments_available.value}, Try again in {cooldown_time} seconds.{_reason} "
+            f"Passed model={model}. pre-call-checks={enable_pre_call_checks}, cooldown_list={cooldown_list}"
+        )
         super().__init__(_message)
 
 

tests/test_litellm/proxy/test_common_request_processing.py

@@ -3879,6 +3879,39 @@ class TestHandleLLMApiExceptionRetryAfter:
         assert proxy_exc.headers["retry-after"] == "43"
         assert proxy_exc.headers["x-custom"] == "1"
 
+    async def test_handle_llm_api_exception_names_cooldown_when_every_deployment_is_cooled_down(self):
+        from litellm.types.router import RouterRateLimitError
+
+        exc = RouterRateLimitError(
+            model="gpt-4",
+            cooldown_time=120,
+            enable_pre_call_checks=False,
+            cooldown_list=["dep-a", "dep-b"],
+            model_ids=["dep-a", "dep-b"],
+        )
+        proxy_exc = await self._invoke(exc)
+        body = proxy_exc.to_dict()
+        assert body["type"] == "all_deployments_in_cooldown"
+        assert body["code"] == "429"
+        assert "All deployments for selected model are in cooldown" in body["message"]
+        assert proxy_exc.headers["retry-after"] == "120"
+
+    async def test_handle_llm_api_exception_keeps_rate_limit_type_when_cooldown_is_partial(self):
+        from litellm.types.router import RouterRateLimitError
+
+        exc = RouterRateLimitError(
+            model="gpt-4",
+            cooldown_time=120,
+            enable_pre_call_checks=False,
+            cooldown_list=["dep-a"],
+            model_ids=["dep-a", "dep-b"],
+        )
+        proxy_exc = await self._invoke(exc)
+        body = proxy_exc.to_dict()
+        assert body["type"] == "rate_limit_error"
+        assert body["code"] == "429"
+        assert "All deployments for selected model are in cooldown" not in body["message"]
+
 
 class TestHandleLLMApiExceptionFramingHeaders:
     """HTTP-framing headers on the provider exception must be stripped before the

tests/test_litellm/test_router.py

@@ -7735,6 +7735,52 @@ def test_get_available_deployment_raises_when_addressed_dict_is_blocked():
         router.get_available_deployment(model="dep-0", request_kwargs={})
 
 
+def _cool_down(router: Router, *deployment_ids: str) -> None:
+    for deployment_id in deployment_ids:
+        router.cooldown_cache.add_deployment_to_cooldown(
+            model_id=deployment_id,
+            original_exception=litellm.RateLimitError(message="upstream 429", llm_provider="openai", model="gpt-4o"),
+            exception_status=429,
+            cooldown_time=60,
+        )
+
+
+async def _select_deployment(router: Router, use_async: bool) -> None:
+    if use_async:
+        await router.async_get_available_deployment(model="gpt-4o", request_kwargs={})
+        return
+    router.get_available_deployment(model="gpt-4o", request_kwargs={})
+
+
+@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"])
+@pytest.mark.asyncio
+async def test_get_available_deployment_names_cooldown_when_every_deployment_is_cooled_down(use_async: bool):
+    from litellm.types.router import RouterErrors, RouterRateLimitError
+
+    router: Final = _router_with_two_deployments([False, False])
+    _cool_down(router, "dep-0", "dep-1")
+    with pytest.raises(RouterRateLimitError) as exc_info:
+        await _select_deployment(router, use_async)
+    assert exc_info.value.all_deployments_in_cooldown is True
+    assert exc_info.value.type == "all_deployments_in_cooldown"
+    assert RouterErrors.all_deployments_in_cooldown.value in str(exc_info.value)
+    assert str(exc_info.value).startswith("No deployments available for selected model, Try again in ")
+
+
+@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"])
+@pytest.mark.asyncio
+async def test_get_available_deployment_keeps_generic_error_when_cooldown_is_partial(use_async: bool):
+    from litellm.types.router import RouterErrors, RouterRateLimitError
+
+    router: Final = _router_with_two_deployments([False, True])
+    _cool_down(router, "dep-0")
+    with pytest.raises(RouterRateLimitError) as exc_info:
+        await _select_deployment(router, use_async)
+    assert exc_info.value.all_deployments_in_cooldown is False
+    assert exc_info.value.type == "rate_limit_error"
+    assert RouterErrors.all_deployments_in_cooldown.value not in str(exc_info.value)
+
+
 def _router_with_two_pass_through_deployments(blocked_flags):
     import litellm
 
@@ -7772,6 +7818,24 @@ def test_get_available_deployment_for_pass_through_raises_when_dict_blocked():
         )
 
 
+def test_get_available_deployment_for_pass_through_names_cooldown_despite_healthy_non_pass_through():
+    from litellm.types.router import RouterRateLimitError
+
+    router: Final = _router_with_two_pass_through_deployments([False, False])
+    router.add_deployment(
+        Deployment(
+            model_name="gpt-4o",
+            litellm_params=LiteLLM_Params(model="openai/gpt-4o-plain", api_key="sk-fake-for-tests"),
+            model_info=ModelInfo(id="plain-0"),
+        )
+    )
+    _cool_down(router, "pt-0", "pt-1")
+    with pytest.raises(RouterRateLimitError) as exc_info:
+        router.get_available_deployment_for_pass_through(model="gpt-4o", request_kwargs={})
+    assert exc_info.value.all_deployments_in_cooldown is True
+    assert exc_info.value.type == "all_deployments_in_cooldown"
+
+
 def test_initialize_deployment_for_pass_through_keeps_bedrock_iam_deployment():
     """
     Bedrock deployments using IAM/OIDC auth have no api_key; pass-through