diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index c352934db6c..1da880e1867 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -462,17 +462,22 @@ class LowestLatencyLoggingHandler(CustomLogger): continue # skip to next one _deployment_tpm = ( - _deployment.get("tpm", None) - or _deployment.get("litellm_params", {}).get("tpm", None) - or _deployment.get("model_info", {}).get("tpm", None) - or float("inf") + _deployment["tpm"] + if _deployment.get("tpm", None) is not None + else _deployment["litellm_params"]["tpm"] + if _deployment.get("litellm_params", {}).get("tpm", None) is not None + else _deployment["model_info"]["tpm"] + if _deployment.get("model_info", {}).get("tpm", None) is not None + else float("inf") ) - _deployment_rpm = ( - _deployment.get("rpm", None) - or _deployment.get("litellm_params", {}).get("rpm", None) - or _deployment.get("model_info", {}).get("rpm", None) - or float("inf") + _deployment["rpm"] + if _deployment.get("rpm", None) is not None + else _deployment["litellm_params"]["rpm"] + if _deployment.get("litellm_params", {}).get("rpm", None) is not None + else _deployment["model_info"]["rpm"] + if _deployment.get("model_info", {}).get("rpm", None) is not None + else float("inf") ) item_latency = item_map.get("latency_normalized", []) item_ttft_latency = item_map.get("time_to_first_token_seconds", []) diff --git a/tests/unit/router_strategy/test_lowest_latency.py b/tests/unit/router_strategy/test_lowest_latency.py index 706ec567710..737f08ff788 100644 --- a/tests/unit/router_strategy/test_lowest_latency.py +++ b/tests/unit/router_strategy/test_lowest_latency.py @@ -8,7 +8,7 @@ import copy import json import sys from datetime import datetime, timedelta -from typing import Final +from typing import Final, Literal from unittest.mock import Mock import pytest @@ -1716,3 +1716,41 @@ async def test_timeout_penalty_updates_normalized_latency_and_avoids_failed_depl healthy_deployments=[{"model_info": {"id": name}, "litellm_params": {}} for name in ("native", "bridge")], ) assert selected["model_info"]["id"] == "bridge" + + +@pytest.mark.parametrize("limit_name", ["tpm", "rpm"]) +@pytest.mark.parametrize( + ("deployment_limit", "params_limit", "info_limit", "available"), + [ + pytest.param(0, 100, 100, False, id="zero-top-level"), + pytest.param(None, 0, 100, False, id="zero-litellm-params"), + pytest.param(None, None, 0, False, id="zero-model-info"), + pytest.param(100, 0, 0, True, id="positive-top-level"), + pytest.param(None, 100, 0, True, id="positive-litellm-params"), + pytest.param(None, None, None, True, id="unset"), + ], +) +def test_capacity_limits_preserve_zero_and_precedence( + limit_name: Literal["tpm", "rpm"], + deployment_limit: int | None, + params_limit: int | None, + info_limit: int | None, + available: bool, +) -> None: + handler: Final = LowestLatencyLoggingHandler(router_cache=DualCache()) + deployment_id: Final = "capacity-limit" + deployment: Final = { + "model_name": "test-model", + limit_name: deployment_limit, + "litellm_params": {"model": "openai/test-model", limit_name: params_limit}, + "model_info": {"id": deployment_id, limit_name: info_limit}, + } + + selected: Final = handler._get_available_deployments( + model_group="test-model", + healthy_deployments=[deployment], + input="hello", + request_count_dict={deployment_id: {"latency_normalized": [0.1]}}, + ) + + assert selected == (deployment if available else None)