Merge be50ed3773 into 6f123b7083

litellm/router.py

@@ -8254,6 +8254,11 @@ class Router:
                 )
                 return False
 
+            from litellm.types.router import RouterNoDeploymentsAvailableError
+
+            if isinstance(exception, RouterNoDeploymentsAvailableError):
+                return False
+
             # Cache litellm_params to avoid repeated dict lookups
             litellm_params: Final = kwargs.get("litellm_params", {})
             _model_info: Final = litellm_params.get("model_info", {})

litellm/router_strategy/budget_limiter.py

@@ -41,7 +41,12 @@ from litellm.router_utils.cooldown_callbacks import (
     _get_prometheus_logger_from_callbacks,
 )
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, RouterErrors
+from litellm.types.router import (
+    DeploymentTypedDict,
+    LiteLLM_Params,
+    RouterErrors,
+    RouterNoDeploymentsAvailableError,
+)
 from litellm.types.utils import BudgetConfig, GenericBudgetConfigType, StandardLoggingPayload
 from litellm.types.utils import BudgetConfig as GenericBudgetInfo
 
@@ -195,7 +200,7 @@ class RouterBudgetLimiting(CustomLogger):
             )
 
             if len(potential_deployments) == 0:
-                raise ValueError(
+                raise RouterNoDeploymentsAvailableError(
                     f"{RouterErrors.no_deployments_with_provider_budget_routing.value}: {deployment_above_budget_info}"
                 )
 

litellm/router_strategy/lowest_tpm_rpm_v2.py

@@ -16,7 +16,7 @@ from litellm.caching.caching import DualCache
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
 from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
-from litellm.types.router import RouterErrors
+from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError
 from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload
 from litellm.utils import get_utc_datetime, print_verbose
 
@@ -667,6 +667,6 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
                         "current_rpm": current_rpm,
                         "rpm_limit": _deployment_rpm,
                     }
-            raise ValueError(
+            raise RouterNoDeploymentsAvailableError(
                 f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}"
             )

litellm/types/router.py

@@ -942,7 +942,11 @@ class RouterGeneralSettings(BaseModel):
     )  # if passed a model not llm_router model list, pass through the request to litellm.acompletion/embedding
 
 
-class RouterRateLimitErrorBasic(ValueError):
+class RouterNoDeploymentsAvailableError(ValueError):
+    status_code: int = 429
+
+
+class RouterRateLimitErrorBasic(RouterNoDeploymentsAvailableError):
     """
     Raise a basic error inside helper functions.
     """
@@ -961,7 +965,7 @@ class RouterErrorTypes(str, enum.Enum):
     all_deployments_in_cooldown = "all_deployments_in_cooldown"
 
 
-class RouterRateLimitError(ValueError):
+class RouterRateLimitError(RouterNoDeploymentsAvailableError):
     def __init__(
         self,
         model: str,

tests/local_testing/test_tpm_rpm_routing_v2.py

@@ -770,3 +770,26 @@ async def test_tpm_rpm_routing_model_name_checks():
             standard_logging_payload["hidden_params"]["litellm_model_name"]
             == "azure/gpt-4.1-mini"
         )
+
+
+def test_every_deployment_over_its_tpm_limit_raises_a_429():
+    from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError
+
+    test_cache = DualCache()
+    lowest_tpm_logger = LowestTPMLoggingHandler(router_cache=test_cache)
+    deployment = {
+        "model_name": "gpt-4o-mini",
+        "litellm_params": {"model": "openai/gpt-4o-mini", "tpm": 10},
+        "model_info": {"id": "d1"},
+    }
+    minute = get_utc_datetime().strftime("%H-%M")
+    test_cache.set_cache(key=f"d1:openai/gpt-4o-mini:tpm:{minute}", value=100)
+
+    with pytest.raises(RouterNoDeploymentsAvailableError) as raised:
+        lowest_tpm_logger.get_available_deployments(
+            model_group="gpt-4o-mini",
+            healthy_deployments=[deployment],
+            messages=[{"role": "user", "content": "hi"}],
+        )
+    assert raised.value.status_code == 429
+    assert RouterErrors.no_deployments_available.value in str(raised.value)

tests/unit/router_strategy/test_budget_limiter_hotpath.py

@@ -784,3 +784,23 @@ async def test_cancelled_flush_does_not_requeue_an_applied_batch(cancellations:
     await limiter._push_in_memory_increments_to_redis()
     assert redis_cache.values[_SPEND_KEY] == 10.0
     assert limiter.redis_increment_operation_queue == []
+
+
+@pytest.mark.asyncio
+async def test_every_provider_over_budget_raises_a_429(disable_budget_sync):
+    from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError
+
+    cache = DualCache()
+    limiter = RouterBudgetLimiting(
+        dual_cache=cache,
+        provider_budget_config={"openai": BudgetConfig(budget_duration="1d", max_budget=1.0)},
+    )
+    await cache.async_set_cache(key="provider_spend:openai:1d", value=5.0)
+    deployment = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "d1"}}
+
+    with pytest.raises(RouterNoDeploymentsAvailableError) as raised:
+        await limiter.async_filter_deployments(
+            model="gpt-4o-mini", healthy_deployments=[deployment], messages=None, request_kwargs={}
+        )
+    assert raised.value.status_code == 429
+    assert RouterErrors.no_deployments_with_provider_budget_routing.value in str(raised.value)

tests/unit/test_router/test_router.py

@@ -9821,6 +9821,54 @@ class TestCallerTimeoutCooldown:
         assert self._cooled_down_ids(router) == []
 
 
+class TestPoolExhaustionStatus:
+    def _router(self):
+        return litellm.Router(
+            model_list=[
+                {
+                    "model_name": "claude-sonnet-5",
+                    "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "sk-fake"},
+                    "model_info": {"id": "dep-a"},
+                },
+                {
+                    "model_name": "claude-sonnet-5",
+                    "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "sk-fake"},
+                    "model_info": {"id": "dep-b"},
+                },
+            ],
+            cooldown_time=60,
+        )
+
+    def test_exhausted_pool_raises_a_429(self):
+        from litellm.router_utils.router_callbacks.track_deployment_metrics import (
+            get_deployment_failures_for_current_minute,
+        )
+        from litellm.types.router import RouterRateLimitError
+
+        router = self._router()
+        for dep_id in ("dep-a", "dep-b"):
+            router.cooldown_cache.add_deployment_to_cooldown(
+                model_id=dep_id,
+                original_exception=litellm.RateLimitError(message="slow down", llm_provider="anthropic", model="x"),
+                exception_status=429,
+                cooldown_time=60,
+            )
+
+        with pytest.raises(RouterRateLimitError) as raised:
+            router.get_available_deployment(model="claude-sonnet-5", messages=[{"role": "user", "content": "hi"}])
+        assert raised.value.status_code == 429
+        assert raised.value.cooldown_time > 0
+
+        cooled = router.deployment_callback_on_failure(
+            {"exception": raised.value, "litellm_params": {"model_info": {"id": "dep-a"}, "metadata": {}}},
+            None,
+            datetime.now(),
+            datetime.now(),
+        )
+        assert cooled is False
+        assert get_deployment_failures_for_current_minute(litellm_router_instance=router, deployment_id="dep-a") == 0
+
+
 def test_stream_chunks_have_generated_content_detects_text_and_non_text():
     from litellm.router import _stream_chunks_have_generated_content
     from litellm.types.utils import (