scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-19 04:54:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 70

fix(router): serialize non-chat latency as seconds

Browse source
This commit is contained in:
Devin AI 2026-07-14 07:08:54 +00:00
parent b200d664ee
commit 6f8dc33acf
2 changed files with 46 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
litellm/router_strategy/lowest_latency.py
View file

@ -79,7 +79,10 @@ class LowestLatencyLoggingHandler(CustomLogger):
# only log ttft for streaming request
time_to_first_token_response_time = kwargs.get("completion_start_time", end_time) - start_time
final_value: Union[float, timedelta] = response_ms
response_seconds = (
response_ms.total_seconds() if isinstance(response_ms, timedelta) else float(response_ms)
)
final_value = response_seconds
time_to_first_token: Optional[float] = None
total_tokens = 0
@ -89,12 +92,6 @@ class LowestLatencyLoggingHandler(CustomLogger):
completion_tokens = _usage.completion_tokens
total_tokens = _usage.total_tokens
# Handle both timedelta and float response times
if isinstance(response_ms, timedelta):
response_seconds = response_ms.total_seconds()
else:
response_seconds = response_ms
final_value = safe_divide_seconds(response_seconds, completion_tokens)
if final_value is not None:
final_value = float(final_value)
@ -267,7 +264,10 @@ class LowestLatencyLoggingHandler(CustomLogger):
# only log ttft for streaming request
time_to_first_token_response_time = kwargs.get("completion_start_time", end_time) - start_time
final_value: Union[float, timedelta] = response_ms
response_seconds = (
response_ms.total_seconds() if isinstance(response_ms, timedelta) else float(response_ms)
)
final_value = response_seconds
total_tokens = 0
time_to_first_token: Optional[float] = None
@ -277,17 +277,11 @@ class LowestLatencyLoggingHandler(CustomLogger):
completion_tokens = _usage.completion_tokens
total_tokens = _usage.total_tokens
# Handle both timedelta and float response times
if isinstance(response_ms, timedelta):
response_seconds = response_ms.total_seconds()
else:
response_seconds = response_ms
final_value = safe_divide_seconds(response_seconds, completion_tokens)
if final_value is not None:
final_value = float(final_value)
else:
final_value = response_ms
final_value = response_seconds
if time_to_first_token_response_time is not None:
if isinstance(time_to_first_token_response_time, timedelta):

37
tests/test_litellm/router_strategy/test_lowest_latency.py Normal file
View file

@ -0,0 +1,37 @@
import json
from datetime import datetime, timedelta
import pytest
import litellm
from litellm.caching.caching import DualCache
from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler
@pytest.mark.asyncio
@pytest.mark.parametrize("sync_mode", [True, False])
async def test_non_chat_latency_is_json_serializable(sync_mode: bool) -> None:
cache = DualCache()
handler = LowestLatencyLoggingHandler(router_cache=cache)
model_group = "embedding-group"
deployment_id = "embedding-deployment"
kwargs = {
"litellm_params": {
"metadata": {"model_group": model_group},
"model_info": {"id": deployment_id},
}
}
start_time = datetime.now()
end_time = start_time + timedelta(seconds=1.25)
response = litellm.EmbeddingResponse()
if sync_mode:
handler.log_success_event(kwargs, response, start_time, end_time)
else:
await handler.async_log_success_event(kwargs, response, start_time, end_time)
cached_value = cache.get_cache(key=f"{model_group}_map")
latency = cached_value[deployment_id]["latency"][0]
assert latency == 1.25
assert isinstance(latency, float)
json.dumps(cached_value)