mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
Merge pull request #39970 from BerriAI/litellm_fix_latency_routing_empty_latency_list
fix(router): treat a routing entry with no latency samples as zero latency
This commit is contained in:
Mateo Wang
GitHub
commit
54af2ec411
4 changed files with 98 additions and 23 deletions
|
|
@ -1,6 +1,6 @@
|
|||
#### What this does ####
|
||||
# picks based on response time (for streaming, this is time to first token)
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
|
|
@ -52,16 +52,12 @@ class LowestCostLoggingHandler(CustomLogger):
|
|||
precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}"
|
||||
cost_key: Final = f"{model_group}_map"
|
||||
|
||||
response_ms: Final[timedelta] = end_time - start_time
|
||||
|
||||
total_tokens = 0
|
||||
|
||||
if isinstance(response_obj, ModelResponse):
|
||||
_usage: Final = getattr(response_obj, "usage", None)
|
||||
if _usage is not None and isinstance(_usage, litellm.Usage):
|
||||
completion_tokens: Final = _usage.completion_tokens
|
||||
total_tokens = _usage.total_tokens
|
||||
float(response_ms.total_seconds() / completion_tokens)
|
||||
|
||||
# ------------
|
||||
# Update usage
|
||||
|
|
@ -131,18 +127,13 @@ class LowestCostLoggingHandler(CustomLogger):
|
|||
current_minute: Final = datetime.now().strftime("%M")
|
||||
precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}"
|
||||
|
||||
response_ms: Final[timedelta] = end_time - start_time
|
||||
|
||||
total_tokens = 0
|
||||
|
||||
if isinstance(response_obj, ModelResponse):
|
||||
_usage: Final = getattr(response_obj, "usage", None)
|
||||
if _usage is not None and isinstance(_usage, litellm.Usage):
|
||||
completion_tokens: Final = _usage.completion_tokens
|
||||
total_tokens = _usage.total_tokens
|
||||
|
||||
float(response_ms.total_seconds() / completion_tokens)
|
||||
|
||||
# ------------
|
||||
# Update usage
|
||||
# ------------
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#### What this does ####
|
||||
# picks based on response time (for streaming, this is time to first token)
|
||||
import random
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime, timedelta
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
|
|
@ -25,6 +26,12 @@ class RoutingArgs(LiteLLMPydanticObjectBase):
|
|||
max_latency_list_size: int = 10
|
||||
|
||||
|
||||
def _average_latency(samples: Sequence[float]) -> float:
|
||||
if not samples:
|
||||
return 0.0
|
||||
return sum(samples) / len(samples)
|
||||
|
||||
|
||||
class LowestLatencyLoggingHandler(CustomLogger):
|
||||
test_flag: bool = False
|
||||
logged_success: int = 0
|
||||
|
|
@ -431,23 +438,13 @@ class LowestLatencyLoggingHandler(CustomLogger):
|
|||
item_tpm = item_map.get(precise_minute, {}).get("tpm", 0)
|
||||
|
||||
# get average latency or average ttft (depending on streaming/non-streaming)
|
||||
total: float = 0.0
|
||||
use_ttft = (
|
||||
request_kwargs is not None
|
||||
and request_kwargs.get("stream", None) is not None
|
||||
and request_kwargs["stream"] is True
|
||||
and len(item_ttft_latency) > 0
|
||||
)
|
||||
if use_ttft:
|
||||
for _call_latency in item_ttft_latency:
|
||||
if isinstance(_call_latency, float):
|
||||
total += _call_latency
|
||||
item_latency = total / len(item_ttft_latency)
|
||||
else:
|
||||
for _call_latency in item_latency:
|
||||
if isinstance(_call_latency, float):
|
||||
total += _call_latency
|
||||
item_latency = total / len(item_latency)
|
||||
average_latency = _average_latency(item_ttft_latency if use_ttft else item_latency)
|
||||
|
||||
# -------------- #
|
||||
# Debugging Logic
|
||||
|
|
@ -456,7 +453,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
|
|||
# this helps a user to debug why the router picked a specfic deployment #
|
||||
_deployment_api_base = _deployment.get("litellm_params", {}).get("api_base", "")
|
||||
if _deployment_api_base is not None:
|
||||
_latency_per_deployment[_deployment_api_base] = item_latency
|
||||
_latency_per_deployment[_deployment_api_base] = average_latency
|
||||
# -------------- #
|
||||
# End of Debugging Logic
|
||||
# -------------- #
|
||||
|
|
@ -466,7 +463,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
|
|||
): # if user passed in tpm / rpm in the model_list
|
||||
continue
|
||||
else:
|
||||
potential_deployments.append((_deployment, item_latency))
|
||||
potential_deployments.append((_deployment, average_latency))
|
||||
|
||||
if len(potential_deployments) == 0:
|
||||
return None
|
||||
|
|
|
|||
59
tests/test_litellm/router_strategy/test_lowest_cost.py
Normal file
59
tests/test_litellm/router_strategy/test_lowest_cost.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
|
||||
|
||||
DEPLOYMENT_ID = "9876"
|
||||
KWARGS = {
|
||||
"litellm_params": {
|
||||
"metadata": {"model_group": "gpt-5.5-pool"},
|
||||
"model_info": {"id": DEPLOYMENT_ID},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _chat_response_with_no_completion_tokens() -> litellm.ModelResponse:
|
||||
return litellm.ModelResponse(
|
||||
model="gpt-5.5",
|
||||
choices=[{"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "length"}],
|
||||
usage=litellm.Usage(prompt_tokens=12, completion_tokens=0, total_tokens=12),
|
||||
)
|
||||
|
||||
|
||||
def _recorded_minute_counters(cache: DualCache) -> dict[str, int]:
|
||||
cached = cache.get_cache(key="gpt-5.5-pool_map") or {}
|
||||
minute_buckets = cached.get(DEPLOYMENT_ID, {})
|
||||
assert len(minute_buckets) == 1, f"expected one minute bucket, got {minute_buckets}"
|
||||
return next(iter(minute_buckets.values()))
|
||||
|
||||
|
||||
def test_log_success_event_counts_a_response_with_no_completion_tokens():
|
||||
cache = DualCache()
|
||||
handler = LowestCostLoggingHandler(router_cache=cache)
|
||||
|
||||
handler.log_success_event(
|
||||
kwargs=KWARGS,
|
||||
response_obj=_chat_response_with_no_completion_tokens(),
|
||||
start_time=datetime(2026, 1, 1, 12, 0, 0),
|
||||
end_time=datetime(2026, 1, 1, 12, 0, 2),
|
||||
)
|
||||
|
||||
assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_log_success_event_counts_a_response_with_no_completion_tokens():
|
||||
cache = DualCache()
|
||||
handler = LowestCostLoggingHandler(router_cache=cache)
|
||||
|
||||
await handler.async_log_success_event(
|
||||
kwargs=KWARGS,
|
||||
response_obj=_chat_response_with_no_completion_tokens(),
|
||||
start_time=datetime(2026, 1, 1, 12, 0, 0),
|
||||
end_time=datetime(2026, 1, 1, 12, 0, 2),
|
||||
)
|
||||
|
||||
assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1}
|
||||
|
|
@ -163,3 +163,31 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds():
|
|||
assert latencies and latencies[-1] == pytest.approx(2.0)
|
||||
assert not isinstance(latencies[-1], timedelta)
|
||||
json.dumps({"latency": latencies})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"cached_entry",
|
||||
[{"latency": []}, {"2026-09-05-15-39": {"tpm": 28, "rpm": 1}}],
|
||||
ids=["empty_latency_list", "minute_bucket_only_as_cost_based_routing_writes"],
|
||||
)
|
||||
async def test_async_get_available_deployments_treats_missing_samples_as_zero_latency(cached_entry):
|
||||
cache = DualCache()
|
||||
handler = LowestLatencyLoggingHandler(router_cache=cache)
|
||||
cache.set_cache(
|
||||
key="gemini-embedding-001_map",
|
||||
value={DEPLOYMENT_ID: cached_entry, "slower": {"latency": [0.5]}},
|
||||
)
|
||||
healthy_deployments = [
|
||||
{"model_info": {"id": DEPLOYMENT_ID}, "litellm_params": {}},
|
||||
{"model_info": {"id": "slower"}, "litellm_params": {}},
|
||||
]
|
||||
|
||||
picked = await handler.async_get_available_deployments(
|
||||
model_group="gemini-embedding-001",
|
||||
healthy_deployments=healthy_deployments,
|
||||
request_kwargs={"stream": False, "metadata": {}},
|
||||
)
|
||||
|
||||
assert picked is not None
|
||||
assert picked["model_info"]["id"] == DEPLOYMENT_ID
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue