From b1d74b3e7ada5619bfc9656c2fa0082bf2877680 Mon Sep 17 00:00:00 2001 From: anay Date: Tue, 7 Jul 2026 18:56:18 -0700 Subject: [PATCH 1/2] fix(router): fail open in usage-based-routing-v2 when tpm/rpm cache read fails --- litellm/router_strategy/lowest_tpm_rpm_v2.py | 8 +- .../router_strategy/test_lowest_tpm_rpm_v2.py | 108 ++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index d63621a7d77..911d6bece65 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -383,7 +383,13 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): """ if tpm_values is None or rpm_values is None: - return None + verbose_router_logger.warning( + "usage-based-routing-v2: tpm/rpm cache read failed for model_group=%s - " + "falling back to routing without usage data for this request", + model_group, + ) + tpm_values = [None] * len(tpm_keys) if tpm_values is None else tpm_values + rpm_values = [None] * len(rpm_keys) if rpm_values is None else rpm_values tpm_dict = {} # {model_id: 1, ..} for idx, key in enumerate(tpm_keys): diff --git a/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py b/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py new file mode 100644 index 00000000000..ee5683091c9 --- /dev/null +++ b/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py @@ -0,0 +1,108 @@ +""" +Regression tests for https://github.com/BerriAI/litellm/issues/16060 + +When the tpm/rpm usage cache read fails (DualCache.[async_]batch_get_cache +returns None, e.g. on a transient Redis error), usage-based-routing-v2 must +fail open and still return a healthy deployment, instead of raising +"No deployments available" (RateLimitError / 429). +""" + +import os +import sys +from unittest.mock import AsyncMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path + +from litellm.caching.caching import DualCache +from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2 + +MODEL_GROUP = "gpt-5" + +HEALTHY_DEPLOYMENTS = [ + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": "azure/gpt-5", "tpm": 5_000_000}, + "model_info": {"id": "deployment-1"}, + }, + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": "azure/gpt-5-eu", "tpm": 5_000_000}, + "model_info": {"id": "deployment-2"}, + }, +] + + +def _handler() -> LowestTPMLoggingHandler_v2: + return LowestTPMLoggingHandler_v2(router_cache=DualCache()) + + +@pytest.mark.asyncio +async def test_async_cache_read_failure_fails_open(): + """Batch cache read returning None must not fail the request.""" + handler = _handler() + with patch.object( + handler.router_cache, + "async_batch_get_cache", + new=AsyncMock(return_value=None), + ): + deployment = await handler.async_get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=HEALTHY_DEPLOYMENTS, + messages=[{"role": "user", "content": "hey"}], + ) + assert deployment is not None + assert deployment["model_info"]["id"] in {"deployment-1", "deployment-2"} + + +def test_sync_cache_read_failure_fails_open(): + """Sync path: batch cache read returning None must not fail the request.""" + handler = _handler() + with patch.object(handler.router_cache, "batch_get_cache", return_value=None): + deployment = handler.get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=HEALTHY_DEPLOYMENTS, + messages=[{"role": "user", "content": "hey"}], + ) + assert deployment is not None + assert deployment["model_info"]["id"] in {"deployment-1", "deployment-2"} + + +@pytest.mark.asyncio +async def test_async_over_limit_deployments_still_excluded(): + """Fail-open must not weaken normal limit enforcement when cache reads work.""" + handler = _handler() + cache_values = [5_000_001, 100, 1, 1] + with patch.object( + handler.router_cache, + "async_batch_get_cache", + new=AsyncMock(return_value=cache_values), + ): + deployment = await handler.async_get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=HEALTHY_DEPLOYMENTS, + messages=[{"role": "user", "content": "hey"}], + ) + assert deployment is not None + assert deployment["model_info"]["id"] == "deployment-2" + + +@pytest.mark.asyncio +async def test_async_all_over_limit_still_raises(): + """When usage data is present and all deployments are over limit, keep raising.""" + import litellm + + handler = _handler() + cache_values = [5_000_001, 5_000_001, 1, 1] + with patch.object( + handler.router_cache, + "async_batch_get_cache", + new=AsyncMock(return_value=cache_values), + ): + with pytest.raises(litellm.RateLimitError): + await handler.async_get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=HEALTHY_DEPLOYMENTS, + messages=[{"role": "user", "content": "hey"}], + ) From 004b8be9bf6c85079f5f7655a402d1f4085f54f2 Mon Sep 17 00:00:00 2001 From: anay Date: Wed, 22 Jul 2026 02:30:00 -0700 Subject: [PATCH 2/2] fix(router): gate cache-read fail-open behavior --- litellm/router_strategy/lowest_tpm_rpm_v2.py | 3 ++ .../router_strategy/test_lowest_tpm_rpm_v2.py | 33 +++++++++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index 911d6bece65..44d04507540 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -27,6 +27,7 @@ else: class RoutingArgs(LiteLLMPydanticObjectBase): ttl: int = 1 * 60 # 1min (RPM/TPM expire key) + allow_routing_on_cache_read_failure: bool = False class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): @@ -383,6 +384,8 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): """ if tpm_values is None or rpm_values is None: + if not self.routing_args.allow_routing_on_cache_read_failure: + return None verbose_router_logger.warning( "usage-based-routing-v2: tpm/rpm cache read failed for model_group=%s - " "falling back to routing without usage data for this request", diff --git a/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py b/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py index ee5683091c9..5cf6c6ce3a6 100644 --- a/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py +++ b/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py @@ -2,8 +2,8 @@ Regression tests for https://github.com/BerriAI/litellm/issues/16060 When the tpm/rpm usage cache read fails (DualCache.[async_]batch_get_cache -returns None, e.g. on a transient Redis error), usage-based-routing-v2 must -fail open and still return a healthy deployment, instead of raising +returns None, e.g. on a transient Redis error), usage-based-routing-v2 can +explicitly opt into returning a healthy deployment instead of raising "No deployments available" (RateLimitError / 429). """ @@ -34,14 +34,35 @@ HEALTHY_DEPLOYMENTS = [ ] -def _handler() -> LowestTPMLoggingHandler_v2: - return LowestTPMLoggingHandler_v2(router_cache=DualCache()) +def _handler(*, allow_routing_on_cache_read_failure: bool = False) -> LowestTPMLoggingHandler_v2: + return LowestTPMLoggingHandler_v2( + router_cache=DualCache(), + routing_args={"allow_routing_on_cache_read_failure": allow_routing_on_cache_read_failure}, + ) + + +@pytest.mark.asyncio +async def test_async_cache_read_failure_fails_closed_by_default(): + import litellm + + handler = _handler() + with patch.object( + handler.router_cache, + "async_batch_get_cache", + new=AsyncMock(return_value=None), + ): + with pytest.raises(litellm.RateLimitError): + await handler.async_get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=HEALTHY_DEPLOYMENTS, + messages=[{"role": "user", "content": "hey"}], + ) @pytest.mark.asyncio async def test_async_cache_read_failure_fails_open(): """Batch cache read returning None must not fail the request.""" - handler = _handler() + handler = _handler(allow_routing_on_cache_read_failure=True) with patch.object( handler.router_cache, "async_batch_get_cache", @@ -58,7 +79,7 @@ async def test_async_cache_read_failure_fails_open(): def test_sync_cache_read_failure_fails_open(): """Sync path: batch cache read returning None must not fail the request.""" - handler = _handler() + handler = _handler(allow_routing_on_cache_read_failure=True) with patch.object(handler.router_cache, "batch_get_cache", return_value=None): deployment = handler.get_available_deployments( model_group=MODEL_GROUP,