scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-09 08:18:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 66

Merge 02968c4132 into 10b3deae0d

Browse source
This commit is contained in:
justavibedev 2026-11-02 22:39:33 +00:00 • committed by GitHub
commit 2e4bd35305
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 53 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
litellm/router_strategy/lowest_latency.py
View file

@ -462,17 +462,22 @@ class LowestLatencyLoggingHandler(CustomLogger):
continue # skip to next one
_deployment_tpm = (
_deployment.get("tpm", None)
or _deployment.get("litellm_params", {}).get("tpm", None)
or _deployment.get("model_info", {}).get("tpm", None)
or float("inf")
_deployment["tpm"]
if _deployment.get("tpm", None) is not None
else _deployment["litellm_params"]["tpm"]
if _deployment.get("litellm_params", {}).get("tpm", None) is not None
else _deployment["model_info"]["tpm"]
if _deployment.get("model_info", {}).get("tpm", None) is not None
else float("inf")
)
_deployment_rpm = (
_deployment.get("rpm", None)
or _deployment.get("litellm_params", {}).get("rpm", None)
or _deployment.get("model_info", {}).get("rpm", None)
or float("inf")
_deployment["rpm"]
if _deployment.get("rpm", None) is not None
else _deployment["litellm_params"]["rpm"]
if _deployment.get("litellm_params", {}).get("rpm", None) is not None
else _deployment["model_info"]["rpm"]
if _deployment.get("model_info", {}).get("rpm", None) is not None
else float("inf")
)
item_latency = item_map.get("latency_normalized", [])
item_ttft_latency = item_map.get("time_to_first_token_seconds", [])

40
tests/unit/router_strategy/test_lowest_latency.py
View file

@ -8,7 +8,7 @@ import copy
import json
import sys
from datetime import datetime, timedelta
from typing import Final
from typing import Final, Literal
from unittest.mock import Mock
import pytest
@ -1716,3 +1716,41 @@ async def test_timeout_penalty_updates_normalized_latency_and_avoids_failed_depl
healthy_deployments=[{"model_info": {"id": name}, "litellm_params": {}} for name in ("native", "bridge")],
)
assert selected["model_info"]["id"] == "bridge"
@pytest.mark.parametrize("limit_name", ["tpm", "rpm"])
@pytest.mark.parametrize(
("deployment_limit", "params_limit", "info_limit", "available"),
[
pytest.param(0, 100, 100, False, id="zero-top-level"),
pytest.param(None, 0, 100, False, id="zero-litellm-params"),
pytest.param(None, None, 0, False, id="zero-model-info"),
pytest.param(100, 0, 0, True, id="positive-top-level"),
pytest.param(None, 100, 0, True, id="positive-litellm-params"),
pytest.param(None, None, None, True, id="unset"),
],
)
def test_capacity_limits_preserve_zero_and_precedence(
limit_name: Literal["tpm", "rpm"],
deployment_limit: int | None,
params_limit: int | None,
info_limit: int | None,
available: bool,
) -> None:
handler: Final = LowestLatencyLoggingHandler(router_cache=DualCache())
deployment_id: Final = "capacity-limit"
deployment: Final = {
"model_name": "test-model",
limit_name: deployment_limit,
"litellm_params": {"model": "openai/test-model", limit_name: params_limit},
"model_info": {"id": deployment_id, limit_name: info_limit},
}
selected: Final = handler._get_available_deployments(
model_group="test-model",
healthy_deployments=[deployment],
input="hello",
request_count_dict={deployment_id: {"latency_normalized": [0.1]}},
)
assert selected == (deployment if available else None)