Merge 236a000d34 into 1c61c2606e

litellm/router.py

@@ -247,6 +247,7 @@ from litellm.types.router import (
     RouterErrors,
     RouterGeneralSettings,
     RouterModelGroupAliasItem,
+    RouterNoDeploymentsAvailableError,
     RouterRateLimitError,
     RouterRateLimitErrorBasic,
     RoutingContext,
@@ -8268,6 +8269,9 @@ class Router:
                 )
                 return False
 
+            if isinstance(exception, RouterNoDeploymentsAvailableError):
+                return False
+
             exception_status: Final = getattr(exception, "status_code", "")
 
             # Cache litellm_params to avoid repeated dict lookups

litellm/router_strategy/budget_limiter.py

@@ -39,7 +39,12 @@ from litellm.router_utils.cooldown_callbacks import (
     _get_prometheus_logger_from_callbacks,
 )
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, RouterErrors
+from litellm.types.router import (
+    DeploymentTypedDict,
+    LiteLLM_Params,
+    RouterErrors,
+    RouterNoDeploymentsAvailableError,
+)
 from litellm.types.utils import BudgetConfig, GenericBudgetConfigType, StandardLoggingPayload
 from litellm.types.utils import BudgetConfig as GenericBudgetInfo
 
@@ -189,7 +194,7 @@ class RouterBudgetLimiting(CustomLogger):
             )
 
             if len(potential_deployments) == 0:
-                raise ValueError(
+                raise RouterNoDeploymentsAvailableError(
                     f"{RouterErrors.no_deployments_with_provider_budget_routing.value}: {deployment_above_budget_info}"
                 )
 

litellm/router_strategy/lowest_tpm_rpm_v2.py

@@ -12,7 +12,7 @@ from litellm._logging import verbose_logger, verbose_router_logger
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
-from litellm.types.router import RouterErrors
+from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError
 from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload
 from litellm.utils import get_utc_datetime, print_verbose
 
@@ -620,6 +620,6 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
                         "current_rpm": current_rpm,
                         "rpm_limit": _deployment_rpm,
                     }
-            raise ValueError(
+            raise RouterNoDeploymentsAvailableError(
                 f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}"
             )

litellm/types/router.py

@@ -848,7 +848,11 @@ class RouterGeneralSettings(BaseModel):
     )  # if passed a model not llm_router model list, pass through the request to litellm.acompletion/embedding
 
 
-class RouterRateLimitErrorBasic(ValueError):
+class RouterNoDeploymentsAvailableError(ValueError):
+    status_code: int = 429
+
+
+class RouterRateLimitErrorBasic(RouterNoDeploymentsAvailableError):
     """
     Raise a basic error inside helper functions.
     """
@@ -862,7 +866,7 @@ class RouterRateLimitErrorBasic(ValueError):
         super().__init__(_message)
 
 
-class RouterRateLimitError(ValueError):
+class RouterRateLimitError(RouterNoDeploymentsAvailableError):
     def __init__(
         self,
         model: str,

tests/local_testing/test_tpm_rpm_routing_v2.py

@@ -770,3 +770,26 @@ async def test_tpm_rpm_routing_model_name_checks():
             standard_logging_payload["hidden_params"]["litellm_model_name"]
             == "azure/gpt-4.1-mini"
         )
+
+
+def test_every_deployment_over_its_tpm_limit_raises_a_429():
+    from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError
+
+    test_cache = DualCache()
+    lowest_tpm_logger = LowestTPMLoggingHandler(router_cache=test_cache)
+    deployment = {
+        "model_name": "gpt-4o-mini",
+        "litellm_params": {"model": "openai/gpt-4o-mini", "tpm": 10},
+        "model_info": {"id": "d1"},
+    }
+    minute = get_utc_datetime().strftime("%H-%M")
+    test_cache.set_cache(key=f"d1:openai/gpt-4o-mini:tpm:{minute}", value=100)
+
+    with pytest.raises(RouterNoDeploymentsAvailableError) as raised:
+        lowest_tpm_logger.get_available_deployments(
+            model_group="gpt-4o-mini",
+            healthy_deployments=[deployment],
+            messages=[{"role": "user", "content": "hi"}],
+        )
+    assert raised.value.status_code == 429
+    assert RouterErrors.no_deployments_available.value in str(raised.value)

tests/test_litellm/router_strategy/test_budget_limiter_hotpath.py

@@ -396,3 +396,23 @@ async def test_push_task_failure_is_logged_once_and_not_leaked(disable_budget_sy
         "Error syncing in-memory cache with Redis: Error 61 connecting to 127.0.0.1:6379"
     ]
     unretrieved.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_every_provider_over_budget_raises_a_429(disable_budget_sync):
+    from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError
+
+    cache = DualCache()
+    limiter = RouterBudgetLimiting(
+        dual_cache=cache,
+        provider_budget_config={"openai": BudgetConfig(budget_duration="1d", max_budget=1.0)},
+    )
+    await cache.async_set_cache(key="provider_spend:openai:1d", value=5.0)
+    deployment = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "d1"}}
+
+    with pytest.raises(RouterNoDeploymentsAvailableError) as raised:
+        await limiter.async_filter_deployments(
+            model="gpt-4o-mini", healthy_deployments=[deployment], messages=None, request_kwargs={}
+        )
+    assert raised.value.status_code == 429
+    assert RouterErrors.no_deployments_with_provider_budget_routing.value in str(raised.value)

tests/test_litellm/test_router.py

@@ -8274,6 +8274,54 @@ class TestAdvisorSubCallCooldown:
         assert "dep-1" not in self._cooled_down_ids(router)
 
 
+class TestPoolExhaustionStatus:
+    def _router(self):
+        return litellm.Router(
+            model_list=[
+                {
+                    "model_name": "claude-sonnet-5",
+                    "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "sk-fake"},
+                    "model_info": {"id": "dep-a"},
+                },
+                {
+                    "model_name": "claude-sonnet-5",
+                    "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "sk-fake"},
+                    "model_info": {"id": "dep-b"},
+                },
+            ],
+            cooldown_time=60,
+        )
+
+    def test_exhausted_pool_raises_a_429(self):
+        from litellm.router_utils.router_callbacks.track_deployment_metrics import (
+            get_deployment_failures_for_current_minute,
+        )
+        from litellm.types.router import RouterRateLimitError
+
+        router = self._router()
+        for dep_id in ("dep-a", "dep-b"):
+            router.cooldown_cache.add_deployment_to_cooldown(
+                model_id=dep_id,
+                original_exception=litellm.RateLimitError(message="slow down", llm_provider="anthropic", model="x"),
+                exception_status=429,
+                cooldown_time=60,
+            )
+
+        with pytest.raises(RouterRateLimitError) as raised:
+            router.get_available_deployment(model="claude-sonnet-5", messages=[{"role": "user", "content": "hi"}])
+        assert raised.value.status_code == 429
+        assert raised.value.cooldown_time > 0
+
+        cooled = router.deployment_callback_on_failure(
+            {"exception": raised.value, "litellm_params": {"model_info": {"id": "dep-a"}, "metadata": {}}},
+            None,
+            datetime.now(),
+            datetime.now(),
+        )
+        assert cooled is False
+        assert get_deployment_failures_for_current_minute(litellm_router_instance=router, deployment_id="dep-a") == 0
+
+
 def test_stream_chunks_have_generated_content_detects_text_and_non_text():
     from litellm.router import _stream_chunks_have_generated_content
     from litellm.types.utils import (