diff --git a/litellm/router.py b/litellm/router.py index 8865543badd..15f03f50fc3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -247,6 +247,7 @@ from litellm.types.router import ( RouterErrors, RouterGeneralSettings, RouterModelGroupAliasItem, + RouterNoDeploymentsAvailableError, RouterRateLimitError, RouterRateLimitErrorBasic, RoutingContext, @@ -8268,6 +8269,9 @@ class Router: ) return False + if isinstance(exception, RouterNoDeploymentsAvailableError): + return False + exception_status: Final = getattr(exception, "status_code", "") # Cache litellm_params to avoid repeated dict lookups diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 3e094df7ac8..bd71a5ca94e 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -39,7 +39,12 @@ from litellm.router_utils.cooldown_callbacks import ( _get_prometheus_logger_from_callbacks, ) from litellm.types.llms.openai import AllMessageValues -from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, RouterErrors +from litellm.types.router import ( + DeploymentTypedDict, + LiteLLM_Params, + RouterErrors, + RouterNoDeploymentsAvailableError, +) from litellm.types.utils import BudgetConfig, GenericBudgetConfigType, StandardLoggingPayload from litellm.types.utils import BudgetConfig as GenericBudgetInfo @@ -189,7 +194,7 @@ class RouterBudgetLimiting(CustomLogger): ) if len(potential_deployments) == 0: - raise ValueError( + raise RouterNoDeploymentsAvailableError( f"{RouterErrors.no_deployments_with_provider_budget_routing.value}: {deployment_above_budget_info}" ) diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index 665ff69ab47..9f9ea5710c8 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -12,7 +12,7 @@ from litellm._logging import verbose_logger, verbose_router_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs -from litellm.types.router import RouterErrors +from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload from litellm.utils import get_utc_datetime, print_verbose @@ -620,6 +620,6 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): "current_rpm": current_rpm, "rpm_limit": _deployment_rpm, } - raise ValueError( + raise RouterNoDeploymentsAvailableError( f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}" ) diff --git a/litellm/types/router.py b/litellm/types/router.py index fc09c40fe08..40cb46da6bc 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -848,7 +848,11 @@ class RouterGeneralSettings(BaseModel): ) # if passed a model not llm_router model list, pass through the request to litellm.acompletion/embedding -class RouterRateLimitErrorBasic(ValueError): +class RouterNoDeploymentsAvailableError(ValueError): + status_code: int = 429 + + +class RouterRateLimitErrorBasic(RouterNoDeploymentsAvailableError): """ Raise a basic error inside helper functions. """ @@ -862,7 +866,7 @@ class RouterRateLimitErrorBasic(ValueError): super().__init__(_message) -class RouterRateLimitError(ValueError): +class RouterRateLimitError(RouterNoDeploymentsAvailableError): def __init__( self, model: str, diff --git a/tests/local_testing/test_tpm_rpm_routing_v2.py b/tests/local_testing/test_tpm_rpm_routing_v2.py index 7478bd253b6..228046e8cb8 100644 --- a/tests/local_testing/test_tpm_rpm_routing_v2.py +++ b/tests/local_testing/test_tpm_rpm_routing_v2.py @@ -770,3 +770,26 @@ async def test_tpm_rpm_routing_model_name_checks(): standard_logging_payload["hidden_params"]["litellm_model_name"] == "azure/gpt-4.1-mini" ) + + +def test_every_deployment_over_its_tpm_limit_raises_a_429(): + from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError + + test_cache = DualCache() + lowest_tpm_logger = LowestTPMLoggingHandler(router_cache=test_cache) + deployment = { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "tpm": 10}, + "model_info": {"id": "d1"}, + } + minute = get_utc_datetime().strftime("%H-%M") + test_cache.set_cache(key=f"d1:openai/gpt-4o-mini:tpm:{minute}", value=100) + + with pytest.raises(RouterNoDeploymentsAvailableError) as raised: + lowest_tpm_logger.get_available_deployments( + model_group="gpt-4o-mini", + healthy_deployments=[deployment], + messages=[{"role": "user", "content": "hi"}], + ) + assert raised.value.status_code == 429 + assert RouterErrors.no_deployments_available.value in str(raised.value) diff --git a/tests/test_litellm/router_strategy/test_budget_limiter_hotpath.py b/tests/test_litellm/router_strategy/test_budget_limiter_hotpath.py index 4cc8fe78811..14919514d8d 100644 --- a/tests/test_litellm/router_strategy/test_budget_limiter_hotpath.py +++ b/tests/test_litellm/router_strategy/test_budget_limiter_hotpath.py @@ -396,3 +396,23 @@ async def test_push_task_failure_is_logged_once_and_not_leaked(disable_budget_sy "Error syncing in-memory cache with Redis: Error 61 connecting to 127.0.0.1:6379" ] unretrieved.assert_not_called() + + +@pytest.mark.asyncio +async def test_every_provider_over_budget_raises_a_429(disable_budget_sync): + from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError + + cache = DualCache() + limiter = RouterBudgetLimiting( + dual_cache=cache, + provider_budget_config={"openai": BudgetConfig(budget_duration="1d", max_budget=1.0)}, + ) + await cache.async_set_cache(key="provider_spend:openai:1d", value=5.0) + deployment = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "d1"}} + + with pytest.raises(RouterNoDeploymentsAvailableError) as raised: + await limiter.async_filter_deployments( + model="gpt-4o-mini", healthy_deployments=[deployment], messages=None, request_kwargs={} + ) + assert raised.value.status_code == 429 + assert RouterErrors.no_deployments_with_provider_budget_routing.value in str(raised.value) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index f5e9b2091a0..e9d2684d6a1 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8274,6 +8274,54 @@ class TestAdvisorSubCallCooldown: assert "dep-1" not in self._cooled_down_ids(router) +class TestPoolExhaustionStatus: + def _router(self): + return litellm.Router( + model_list=[ + { + "model_name": "claude-sonnet-5", + "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "sk-fake"}, + "model_info": {"id": "dep-a"}, + }, + { + "model_name": "claude-sonnet-5", + "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "sk-fake"}, + "model_info": {"id": "dep-b"}, + }, + ], + cooldown_time=60, + ) + + def test_exhausted_pool_raises_a_429(self): + from litellm.router_utils.router_callbacks.track_deployment_metrics import ( + get_deployment_failures_for_current_minute, + ) + from litellm.types.router import RouterRateLimitError + + router = self._router() + for dep_id in ("dep-a", "dep-b"): + router.cooldown_cache.add_deployment_to_cooldown( + model_id=dep_id, + original_exception=litellm.RateLimitError(message="slow down", llm_provider="anthropic", model="x"), + exception_status=429, + cooldown_time=60, + ) + + with pytest.raises(RouterRateLimitError) as raised: + router.get_available_deployment(model="claude-sonnet-5", messages=[{"role": "user", "content": "hi"}]) + assert raised.value.status_code == 429 + assert raised.value.cooldown_time > 0 + + cooled = router.deployment_callback_on_failure( + {"exception": raised.value, "litellm_params": {"model_info": {"id": "dep-a"}, "metadata": {}}}, + None, + datetime.now(), + datetime.now(), + ) + assert cooled is False + assert get_deployment_failures_for_current_minute(litellm_router_instance=router, deployment_id="dep-a") == 0 + + def test_stream_chunks_have_generated_content_detects_text_and_non_text(): from litellm.router import _stream_chunks_have_generated_content from litellm.types.utils import (