From a1c6c6da4bb5b5f5d58e7dbe954164261095a8d6 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:16:54 +0000 Subject: [PATCH 1/4] fix(router): carry 429 on no-deployment errors, skip them in cooldown When every deployment of a model group is unavailable the router raises RouterRateLimitError or RouterRateLimitErrorBasic, the provider budget limiter raises a bare ValueError, and the sync usage-based-routing-v2 strategy raises a bare ValueError. None of them carry a status_code. A developer calling litellm.Router directly gets a plain ValueError with nothing to branch on, while the async strategy path already raises litellm.RateLimitError for the same condition. The proxy is not affected on the wire: ProxyException rewrites the code to 429 whenever the message says "No deployments available", and the async strategy path raises a typed 429 All four raises now share RouterNoDeploymentsAvailableError, a ValueError subclass that carries status_code 429 and the cooldown time, so Router users get one typed error for "nothing can serve this call" and code that maps exceptions by status_code no longer needs to sniff the message Router.deployment_callback_on_failure returns early for these errors. The callback counts a failure and runs the cooldown logic against whatever model_info sits in the kwargs it receives, and a routing error is never that deployment's own failure. Today the proxy does not reach this branch with a deployment id attached: the retry resets the logging kwargs before re-selection, and the non-streaming failure log is deduped. The early return keeps a future caller, or a 429 now present on the exception, from turning an exhausted pool into an extra failure or a refreshed cooldown on the last deployment that was tried Co-authored-by: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Co-authored-by: songkuan-zheng --- litellm/router.py | 4 ++ litellm/router_strategy/budget_limiter.py | 9 +++- litellm/router_strategy/lowest_tpm_rpm_v2.py | 4 +- litellm/types/router.py | 9 +++- .../test_budget_limiter_hotpath.py | 20 ++++++++ .../router_strategy/test_lowest_tpm_rpm_v2.py | 27 +++++++++++ tests/test_litellm/test_router.py | 48 +++++++++++++++++++ 7 files changed, 115 insertions(+), 6 deletions(-) create mode 100644 tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py diff --git a/litellm/router.py b/litellm/router.py index 9e6db66db83..7ae89ed58e8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -245,6 +245,7 @@ from litellm.types.router import ( RouterErrors, RouterGeneralSettings, RouterModelGroupAliasItem, + RouterNoDeploymentsAvailableError, RouterRateLimitError, RouterRateLimitErrorBasic, RoutingContext, @@ -8253,6 +8254,9 @@ class Router: ) return False + if isinstance(exception, RouterNoDeploymentsAvailableError): + return False + exception_status: Final = getattr(exception, "status_code", "") # Cache litellm_params to avoid repeated dict lookups diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 3e094df7ac8..bd71a5ca94e 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -39,7 +39,12 @@ from litellm.router_utils.cooldown_callbacks import ( _get_prometheus_logger_from_callbacks, ) from litellm.types.llms.openai import AllMessageValues -from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, RouterErrors +from litellm.types.router import ( + DeploymentTypedDict, + LiteLLM_Params, + RouterErrors, + RouterNoDeploymentsAvailableError, +) from litellm.types.utils import BudgetConfig, GenericBudgetConfigType, StandardLoggingPayload from litellm.types.utils import BudgetConfig as GenericBudgetInfo @@ -189,7 +194,7 @@ class RouterBudgetLimiting(CustomLogger): ) if len(potential_deployments) == 0: - raise ValueError( + raise RouterNoDeploymentsAvailableError( f"{RouterErrors.no_deployments_with_provider_budget_routing.value}: {deployment_above_budget_info}" ) diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index 665ff69ab47..9f9ea5710c8 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -12,7 +12,7 @@ from litellm._logging import verbose_logger, verbose_router_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs -from litellm.types.router import RouterErrors +from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload from litellm.utils import get_utc_datetime, print_verbose @@ -620,6 +620,6 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): "current_rpm": current_rpm, "rpm_limit": _deployment_rpm, } - raise ValueError( + raise RouterNoDeploymentsAvailableError( f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}" ) diff --git a/litellm/types/router.py b/litellm/types/router.py index fc09c40fe08..480522e42cd 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -848,7 +848,12 @@ class RouterGeneralSettings(BaseModel): ) # if passed a model not llm_router model list, pass through the request to litellm.acompletion/embedding -class RouterRateLimitErrorBasic(ValueError): +class RouterNoDeploymentsAvailableError(ValueError): + status_code: int = 429 + cooldown_time: float | None = None + + +class RouterRateLimitErrorBasic(RouterNoDeploymentsAvailableError): """ Raise a basic error inside helper functions. """ @@ -862,7 +867,7 @@ class RouterRateLimitErrorBasic(ValueError): super().__init__(_message) -class RouterRateLimitError(ValueError): +class RouterRateLimitError(RouterNoDeploymentsAvailableError): def __init__( self, model: str, diff --git a/tests/test_litellm/router_strategy/test_budget_limiter_hotpath.py b/tests/test_litellm/router_strategy/test_budget_limiter_hotpath.py index 4cc8fe78811..14919514d8d 100644 --- a/tests/test_litellm/router_strategy/test_budget_limiter_hotpath.py +++ b/tests/test_litellm/router_strategy/test_budget_limiter_hotpath.py @@ -396,3 +396,23 @@ async def test_push_task_failure_is_logged_once_and_not_leaked(disable_budget_sy "Error syncing in-memory cache with Redis: Error 61 connecting to 127.0.0.1:6379" ] unretrieved.assert_not_called() + + +@pytest.mark.asyncio +async def test_every_provider_over_budget_raises_a_429(disable_budget_sync): + from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError + + cache = DualCache() + limiter = RouterBudgetLimiting( + dual_cache=cache, + provider_budget_config={"openai": BudgetConfig(budget_duration="1d", max_budget=1.0)}, + ) + await cache.async_set_cache(key="provider_spend:openai:1d", value=5.0) + deployment = {"litellm_params": {"model": "openai/gpt-4o-mini"}, "model_info": {"id": "d1"}} + + with pytest.raises(RouterNoDeploymentsAvailableError) as raised: + await limiter.async_filter_deployments( + model="gpt-4o-mini", healthy_deployments=[deployment], messages=None, request_kwargs={} + ) + assert raised.value.status_code == 429 + assert RouterErrors.no_deployments_with_provider_budget_routing.value in str(raised.value) diff --git a/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py b/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py new file mode 100644 index 00000000000..2619278e6ad --- /dev/null +++ b/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py @@ -0,0 +1,27 @@ +import pytest + +from litellm.caching.caching import DualCache +from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2 +from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError +from litellm.utils import get_utc_datetime + + +def test_every_deployment_over_its_tpm_limit_raises_a_429(): + cache = DualCache() + handler = LowestTPMLoggingHandler_v2(router_cache=cache) + deployment = { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "tpm": 10}, + "model_info": {"id": "d1"}, + } + minute = get_utc_datetime().strftime("%H-%M") + cache.set_cache(key=f"d1:openai/gpt-4o-mini:tpm:{minute}", value=100) + + with pytest.raises(RouterNoDeploymentsAvailableError) as raised: + handler.get_available_deployments( + model_group="gpt-4o-mini", + healthy_deployments=[deployment], + messages=[{"role": "user", "content": "hi"}], + ) + assert raised.value.status_code == 429 + assert RouterErrors.no_deployments_available.value in str(raised.value) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index def67ccf88b..3747ae779fe 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8068,6 +8068,54 @@ class TestAdvisorSubCallCooldown: assert "dep-1" not in self._cooled_down_ids(router) +class TestPoolExhaustionStatus: + def _router(self): + return litellm.Router( + model_list=[ + { + "model_name": "claude-sonnet-5", + "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "sk-fake"}, + "model_info": {"id": "dep-a"}, + }, + { + "model_name": "claude-sonnet-5", + "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "sk-fake"}, + "model_info": {"id": "dep-b"}, + }, + ], + cooldown_time=60, + ) + + def test_exhausted_pool_raises_a_429(self): + from litellm.router_utils.router_callbacks.track_deployment_metrics import ( + get_deployment_failures_for_current_minute, + ) + from litellm.types.router import RouterRateLimitError + + router = self._router() + for dep_id in ("dep-a", "dep-b"): + router.cooldown_cache.add_deployment_to_cooldown( + model_id=dep_id, + original_exception=litellm.RateLimitError(message="slow down", llm_provider="anthropic", model="x"), + exception_status=429, + cooldown_time=60, + ) + + with pytest.raises(RouterRateLimitError) as raised: + router.get_available_deployment(model="claude-sonnet-5", messages=[{"role": "user", "content": "hi"}]) + assert raised.value.status_code == 429 + assert raised.value.cooldown_time > 0 + + cooled = router.deployment_callback_on_failure( + {"exception": raised.value, "litellm_params": {"model_info": {"id": "dep-a"}, "metadata": {}}}, + None, + datetime.now(), + datetime.now(), + ) + assert cooled is False + assert get_deployment_failures_for_current_minute(litellm_router_instance=router, deployment_id="dep-a") == 0 + + def test_stream_chunks_have_generated_content_detects_text_and_non_text(): from litellm.router import _stream_chunks_have_generated_content from litellm.types.utils import ( From 403a3774d905dd28b03c719efaa3d53c39253b9b Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:06:35 +0000 Subject: [PATCH 2/4] test(router): move the sync tpm regression into the existing tpm routing test file tests/local_testing/test_tpm_rpm_routing_v2.py is the test file that already covers LowestTPMLoggingHandler_v2, so the regression for the sync get_available_deployments raise lives there instead of in a new file Co-authored-by: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Co-authored-by: songkuan-zheng --- .../local_testing/test_tpm_rpm_routing_v2.py | 23 ++++++++++++++++ .../router_strategy/test_lowest_tpm_rpm_v2.py | 27 ------------------- 2 files changed, 23 insertions(+), 27 deletions(-) delete mode 100644 tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py diff --git a/tests/local_testing/test_tpm_rpm_routing_v2.py b/tests/local_testing/test_tpm_rpm_routing_v2.py index 7478bd253b6..228046e8cb8 100644 --- a/tests/local_testing/test_tpm_rpm_routing_v2.py +++ b/tests/local_testing/test_tpm_rpm_routing_v2.py @@ -770,3 +770,26 @@ async def test_tpm_rpm_routing_model_name_checks(): standard_logging_payload["hidden_params"]["litellm_model_name"] == "azure/gpt-4.1-mini" ) + + +def test_every_deployment_over_its_tpm_limit_raises_a_429(): + from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError + + test_cache = DualCache() + lowest_tpm_logger = LowestTPMLoggingHandler(router_cache=test_cache) + deployment = { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "tpm": 10}, + "model_info": {"id": "d1"}, + } + minute = get_utc_datetime().strftime("%H-%M") + test_cache.set_cache(key=f"d1:openai/gpt-4o-mini:tpm:{minute}", value=100) + + with pytest.raises(RouterNoDeploymentsAvailableError) as raised: + lowest_tpm_logger.get_available_deployments( + model_group="gpt-4o-mini", + healthy_deployments=[deployment], + messages=[{"role": "user", "content": "hi"}], + ) + assert raised.value.status_code == 429 + assert RouterErrors.no_deployments_available.value in str(raised.value) diff --git a/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py b/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py deleted file mode 100644 index 2619278e6ad..00000000000 --- a/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py +++ /dev/null @@ -1,27 +0,0 @@ -import pytest - -from litellm.caching.caching import DualCache -from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2 -from litellm.types.router import RouterErrors, RouterNoDeploymentsAvailableError -from litellm.utils import get_utc_datetime - - -def test_every_deployment_over_its_tpm_limit_raises_a_429(): - cache = DualCache() - handler = LowestTPMLoggingHandler_v2(router_cache=cache) - deployment = { - "model_name": "gpt-4o-mini", - "litellm_params": {"model": "openai/gpt-4o-mini", "tpm": 10}, - "model_info": {"id": "d1"}, - } - minute = get_utc_datetime().strftime("%H-%M") - cache.set_cache(key=f"d1:openai/gpt-4o-mini:tpm:{minute}", value=100) - - with pytest.raises(RouterNoDeploymentsAvailableError) as raised: - handler.get_available_deployments( - model_group="gpt-4o-mini", - healthy_deployments=[deployment], - messages=[{"role": "user", "content": "hi"}], - ) - assert raised.value.status_code == 429 - assert RouterErrors.no_deployments_available.value in str(raised.value) From 5ff0e13f5f372914f6a0228cdd25c0184da16d8d Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:19:36 +0000 Subject: [PATCH 3/4] fix(router): declare cooldown_time as float on RouterRateLimitError The shared base declares cooldown_time as float | None because the budget limiter and the sync tpm strategy have no window to report. RouterRateLimitError always receives one, so it redeclares the attribute as float and callers that compare it keep a narrowed type Co-authored-by: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Co-authored-by: songkuan-zheng --- litellm/types/router.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/types/router.py b/litellm/types/router.py index 480522e42cd..c87b2b8e3c4 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -868,6 +868,8 @@ class RouterRateLimitErrorBasic(RouterNoDeploymentsAvailableError): class RouterRateLimitError(RouterNoDeploymentsAvailableError): + cooldown_time: float + def __init__( self, model: str, From 236a000d34b71181cf6131a26f789405180809e3 Mon Sep 17 00:00:00 2001 From: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:29:43 +0000 Subject: [PATCH 4/4] fix(router): keep cooldown_time off the shared no-deployment base error Declaring cooldown_time as float | None on the base widened the attribute on RouterRateLimitError, whose callers compare it as a float, and redeclaring it as float on the subclass is an incompatible override. The base now carries only status_code; RouterRateLimitError keeps its own float cooldown_time exactly as before Co-authored-by: songkuan-zheng <252822057+songkuan-zheng@users.noreply.github.com> Co-authored-by: songkuan-zheng --- litellm/types/router.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/litellm/types/router.py b/litellm/types/router.py index c87b2b8e3c4..40cb46da6bc 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -850,7 +850,6 @@ class RouterGeneralSettings(BaseModel): class RouterNoDeploymentsAvailableError(ValueError): status_code: int = 429 - cooldown_time: float | None = None class RouterRateLimitErrorBasic(RouterNoDeploymentsAvailableError): @@ -868,8 +867,6 @@ class RouterRateLimitErrorBasic(RouterNoDeploymentsAvailableError): class RouterRateLimitError(RouterNoDeploymentsAvailableError): - cooldown_time: float - def __init__( self, model: str,