diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index a2acce5fcb5..34319692f7e 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -29,6 +29,7 @@ else: class RoutingArgs(LiteLLMPydanticObjectBase): ttl: int = 1 * 60 # 1min (RPM/TPM expire key) + allow_routing_on_cache_read_failure: bool = False class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): @@ -367,7 +368,15 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): """ if tpm_values is None or rpm_values is None: - return None + if not self.routing_args.allow_routing_on_cache_read_failure: + return None + verbose_router_logger.warning( + "usage-based-routing-v2: tpm/rpm cache read failed for model_group=%s - " + "falling back to routing without usage data for this request", + model_group, + ) + tpm_values = [None] * len(tpm_keys) if tpm_values is None else tpm_values + rpm_values = [None] * len(rpm_keys) if rpm_values is None else rpm_values tpm_dict = {} # {model_id: 1, ..} for idx, key in enumerate(tpm_keys): diff --git a/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py b/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py new file mode 100644 index 00000000000..5cf6c6ce3a6 --- /dev/null +++ b/tests/test_litellm/router_strategy/test_lowest_tpm_rpm_v2.py @@ -0,0 +1,129 @@ +""" +Regression tests for https://github.com/BerriAI/litellm/issues/16060 + +When the tpm/rpm usage cache read fails (DualCache.[async_]batch_get_cache +returns None, e.g. on a transient Redis error), usage-based-routing-v2 can +explicitly opt into returning a healthy deployment instead of raising +"No deployments available" (RateLimitError / 429). +""" + +import os +import sys +from unittest.mock import AsyncMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path + +from litellm.caching.caching import DualCache +from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2 + +MODEL_GROUP = "gpt-5" + +HEALTHY_DEPLOYMENTS = [ + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": "azure/gpt-5", "tpm": 5_000_000}, + "model_info": {"id": "deployment-1"}, + }, + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": "azure/gpt-5-eu", "tpm": 5_000_000}, + "model_info": {"id": "deployment-2"}, + }, +] + + +def _handler(*, allow_routing_on_cache_read_failure: bool = False) -> LowestTPMLoggingHandler_v2: + return LowestTPMLoggingHandler_v2( + router_cache=DualCache(), + routing_args={"allow_routing_on_cache_read_failure": allow_routing_on_cache_read_failure}, + ) + + +@pytest.mark.asyncio +async def test_async_cache_read_failure_fails_closed_by_default(): + import litellm + + handler = _handler() + with patch.object( + handler.router_cache, + "async_batch_get_cache", + new=AsyncMock(return_value=None), + ): + with pytest.raises(litellm.RateLimitError): + await handler.async_get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=HEALTHY_DEPLOYMENTS, + messages=[{"role": "user", "content": "hey"}], + ) + + +@pytest.mark.asyncio +async def test_async_cache_read_failure_fails_open(): + """Batch cache read returning None must not fail the request.""" + handler = _handler(allow_routing_on_cache_read_failure=True) + with patch.object( + handler.router_cache, + "async_batch_get_cache", + new=AsyncMock(return_value=None), + ): + deployment = await handler.async_get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=HEALTHY_DEPLOYMENTS, + messages=[{"role": "user", "content": "hey"}], + ) + assert deployment is not None + assert deployment["model_info"]["id"] in {"deployment-1", "deployment-2"} + + +def test_sync_cache_read_failure_fails_open(): + """Sync path: batch cache read returning None must not fail the request.""" + handler = _handler(allow_routing_on_cache_read_failure=True) + with patch.object(handler.router_cache, "batch_get_cache", return_value=None): + deployment = handler.get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=HEALTHY_DEPLOYMENTS, + messages=[{"role": "user", "content": "hey"}], + ) + assert deployment is not None + assert deployment["model_info"]["id"] in {"deployment-1", "deployment-2"} + + +@pytest.mark.asyncio +async def test_async_over_limit_deployments_still_excluded(): + """Fail-open must not weaken normal limit enforcement when cache reads work.""" + handler = _handler() + cache_values = [5_000_001, 100, 1, 1] + with patch.object( + handler.router_cache, + "async_batch_get_cache", + new=AsyncMock(return_value=cache_values), + ): + deployment = await handler.async_get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=HEALTHY_DEPLOYMENTS, + messages=[{"role": "user", "content": "hey"}], + ) + assert deployment is not None + assert deployment["model_info"]["id"] == "deployment-2" + + +@pytest.mark.asyncio +async def test_async_all_over_limit_still_raises(): + """When usage data is present and all deployments are over limit, keep raising.""" + import litellm + + handler = _handler() + cache_values = [5_000_001, 5_000_001, 1, 1] + with patch.object( + handler.router_cache, + "async_batch_get_cache", + new=AsyncMock(return_value=cache_values), + ): + with pytest.raises(litellm.RateLimitError): + await handler.async_get_available_deployments( + model_group=MODEL_GROUP, + healthy_deployments=HEALTHY_DEPLOYMENTS, + messages=[{"role": "user", "content": "hey"}], + )