From 163662406f12570398e487b1959304b497b0f10e Mon Sep 17 00:00:00 2001 From: justavibedev <213012880+justavibedev@users.noreply.github.com> Date: Fri, 4 Sep 2026 08:43:24 -0700 Subject: [PATCH 1/2] fix(router): preserve zero capacity limits --- litellm/router_strategy/lowest_latency.py | 32 ++++++++++++------ .../router_strategy/test_lowest_latency.py | 33 +++++++++++++++---- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index eebe81ebba1..4906a628198 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -412,18 +412,30 @@ class LowestLatencyLoggingHandler(CustomLogger): if _deployment is None: continue # skip to next one - _deployment_tpm = ( - _deployment.get("tpm", None) - or _deployment.get("litellm_params", {}).get("tpm", None) - or _deployment.get("model_info", {}).get("tpm", None) - or float("inf") + _deployment_tpm: Final = next( + ( + limit + for limit in ( + _deployment.get("tpm", None), + _deployment.get("litellm_params", {}).get("tpm", None), + _deployment.get("model_info", {}).get("tpm", None), + ) + if limit is not None + ), + float("inf"), ) - _deployment_rpm = ( - _deployment.get("rpm", None) - or _deployment.get("litellm_params", {}).get("rpm", None) - or _deployment.get("model_info", {}).get("rpm", None) - or float("inf") + _deployment_rpm: Final = next( + ( + limit + for limit in ( + _deployment.get("rpm", None), + _deployment.get("litellm_params", {}).get("rpm", None), + _deployment.get("model_info", {}).get("rpm", None), + ) + if limit is not None + ), + float("inf"), ) item_latency = item_map.get("latency", []) item_ttft_latency = item_map.get("time_to_first_token", []) diff --git a/tests/test_litellm/router_strategy/test_lowest_latency.py b/tests/test_litellm/router_strategy/test_lowest_latency.py index 6701f4a7aa2..3516ee918cc 100644 --- a/tests/test_litellm/router_strategy/test_lowest_latency.py +++ b/tests/test_litellm/router_strategy/test_lowest_latency.py @@ -9,7 +9,6 @@ from datetime import datetime, timedelta import pytest - import litellm from litellm.caching.caching import DualCache from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler @@ -58,9 +57,9 @@ def test_sync_embedding_latency_is_json_serializable(): latencies = _recorded_latencies(cache) assert latencies, "expected a latency entry to be recorded" - assert all( - not isinstance(value, timedelta) for value in latencies - ), f"raw timedelta leaked into latency list: {latencies}" + assert all(not isinstance(value, timedelta) for value in latencies), ( + f"raw timedelta leaked into latency list: {latencies}" + ) assert latencies[-1] == pytest.approx(2.0) # the exact failure mode from production: redis cache sync json.dumps json.dumps({"latency": latencies}) @@ -84,9 +83,9 @@ async def test_async_embedding_latency_is_json_serializable(): latencies = _recorded_latencies(cache) assert latencies, "expected a latency entry to be recorded" - assert all( - not isinstance(value, timedelta) for value in latencies - ), f"raw timedelta leaked into latency list: {latencies}" + assert all(not isinstance(value, timedelta) for value in latencies), ( + f"raw timedelta leaked into latency list: {latencies}" + ) assert latencies[-1] == pytest.approx(3.0) json.dumps({"latency": latencies}) @@ -163,3 +162,23 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds(): assert latencies and latencies[-1] == pytest.approx(2.0) assert not isinstance(latencies[-1], timedelta) json.dumps({"latency": latencies}) + + +@pytest.mark.parametrize("limit_name", ["tpm", "rpm"]) +def test_zero_capacity_excludes_deployment(limit_name: str): + handler = LowestLatencyLoggingHandler(router_cache=DualCache()) + deployment_id = "zero-capacity" + deployment = { + "model_name": "test-model", + "litellm_params": {"model": "openai/test-model", limit_name: 0}, + "model_info": {"id": deployment_id}, + } + + selected = handler._get_available_deployments( + model_group="test-model", + healthy_deployments=[deployment], + input="hello", + request_count_dict={deployment_id: {"latency": [0.1]}}, + ) + + assert selected is None From 02968c41320ec9bfbcd7dab3421c4b59ea3ac346 Mon Sep 17 00:00:00 2001 From: justavibedev <213012880+justavibedev@users.noreply.github.com> Date: Thu, 22 Oct 2026 10:50:40 -0700 Subject: [PATCH 2/2] fix(router): simplify capacity limit selection --- litellm/router_strategy/lowest_latency.py | 35 ++++++++++++----------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index b21e6c7be8e..7948a140694 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -451,23 +451,24 @@ class LowestLatencyLoggingHandler(CustomLogger): if _deployment is None: continue # skip to next one - for _deployment_tpm in ( - _deployment.get("tpm", None), - _deployment.get("litellm_params", {}).get("tpm", None), - _deployment.get("model_info", {}).get("tpm", None), - float("inf"), - ): - if _deployment_tpm is not None: - break - - for _deployment_rpm in ( - _deployment.get("rpm", None), - _deployment.get("litellm_params", {}).get("rpm", None), - _deployment.get("model_info", {}).get("rpm", None), - float("inf"), - ): - if _deployment_rpm is not None: - break + _deployment_tpm = ( + _deployment["tpm"] + if _deployment.get("tpm", None) is not None + else _deployment["litellm_params"]["tpm"] + if _deployment.get("litellm_params", {}).get("tpm", None) is not None + else _deployment["model_info"]["tpm"] + if _deployment.get("model_info", {}).get("tpm", None) is not None + else float("inf") + ) + _deployment_rpm = ( + _deployment["rpm"] + if _deployment.get("rpm", None) is not None + else _deployment["litellm_params"]["rpm"] + if _deployment.get("litellm_params", {}).get("rpm", None) is not None + else _deployment["model_info"]["rpm"] + if _deployment.get("model_info", {}).get("rpm", None) is not None + else float("inf") + ) item_latency = item_map.get("latency_normalized", []) item_ttft_latency = item_map.get("time_to_first_token_seconds", []) item_rpm = item_map.get(precise_minute, {}).get("rpm", 0)