diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index eebe81ebba1..4906a628198 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -412,18 +412,30 @@ class LowestLatencyLoggingHandler(CustomLogger): if _deployment is None: continue # skip to next one - _deployment_tpm = ( - _deployment.get("tpm", None) - or _deployment.get("litellm_params", {}).get("tpm", None) - or _deployment.get("model_info", {}).get("tpm", None) - or float("inf") + _deployment_tpm: Final = next( + ( + limit + for limit in ( + _deployment.get("tpm", None), + _deployment.get("litellm_params", {}).get("tpm", None), + _deployment.get("model_info", {}).get("tpm", None), + ) + if limit is not None + ), + float("inf"), ) - _deployment_rpm = ( - _deployment.get("rpm", None) - or _deployment.get("litellm_params", {}).get("rpm", None) - or _deployment.get("model_info", {}).get("rpm", None) - or float("inf") + _deployment_rpm: Final = next( + ( + limit + for limit in ( + _deployment.get("rpm", None), + _deployment.get("litellm_params", {}).get("rpm", None), + _deployment.get("model_info", {}).get("rpm", None), + ) + if limit is not None + ), + float("inf"), ) item_latency = item_map.get("latency", []) item_ttft_latency = item_map.get("time_to_first_token", []) diff --git a/tests/test_litellm/router_strategy/test_lowest_latency.py b/tests/test_litellm/router_strategy/test_lowest_latency.py index 6701f4a7aa2..3516ee918cc 100644 --- a/tests/test_litellm/router_strategy/test_lowest_latency.py +++ b/tests/test_litellm/router_strategy/test_lowest_latency.py @@ -9,7 +9,6 @@ from datetime import datetime, timedelta import pytest - import litellm from litellm.caching.caching import DualCache from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler @@ -58,9 +57,9 @@ def test_sync_embedding_latency_is_json_serializable(): latencies = _recorded_latencies(cache) assert latencies, "expected a latency entry to be recorded" - assert all( - not isinstance(value, timedelta) for value in latencies - ), f"raw timedelta leaked into latency list: {latencies}" + assert all(not isinstance(value, timedelta) for value in latencies), ( + f"raw timedelta leaked into latency list: {latencies}" + ) assert latencies[-1] == pytest.approx(2.0) # the exact failure mode from production: redis cache sync json.dumps json.dumps({"latency": latencies}) @@ -84,9 +83,9 @@ async def test_async_embedding_latency_is_json_serializable(): latencies = _recorded_latencies(cache) assert latencies, "expected a latency entry to be recorded" - assert all( - not isinstance(value, timedelta) for value in latencies - ), f"raw timedelta leaked into latency list: {latencies}" + assert all(not isinstance(value, timedelta) for value in latencies), ( + f"raw timedelta leaked into latency list: {latencies}" + ) assert latencies[-1] == pytest.approx(3.0) json.dumps({"latency": latencies}) @@ -163,3 +162,23 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds(): assert latencies and latencies[-1] == pytest.approx(2.0) assert not isinstance(latencies[-1], timedelta) json.dumps({"latency": latencies}) + + +@pytest.mark.parametrize("limit_name", ["tpm", "rpm"]) +def test_zero_capacity_excludes_deployment(limit_name: str): + handler = LowestLatencyLoggingHandler(router_cache=DualCache()) + deployment_id = "zero-capacity" + deployment = { + "model_name": "test-model", + "litellm_params": {"model": "openai/test-model", limit_name: 0}, + "model_info": {"id": deployment_id}, + } + + selected = handler._get_available_deployments( + model_group="test-model", + healthy_deployments=[deployment], + input="hello", + request_count_dict={deployment_id: {"latency": [0.1]}}, + ) + + assert selected is None