From 1cc832f53ef78c5ef630ff9242a7bfd5c9fff418 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 18 Mar 2025 14:58:55 -0500 Subject: [PATCH] Fix TTFT prioritization for streaming in LowestLatencyLoggingHandler --- litellm/router_strategy/lowest_latency.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 0ec71116e29..a48b5442d4f 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -469,8 +469,6 @@ class LowestLatencyLoggingHandler(CustomLogger): total: float = 0.0 if ( request_kwargs is not None - and len(item_latency) > 0 - and isinstance(item_latency[0], timedelta) and request_kwargs.get("stream", None) is not None and request_kwargs["stream"] is True and len(item_ttft_latency) > 0 @@ -478,6 +476,7 @@ class LowestLatencyLoggingHandler(CustomLogger): for _call_latency in item_ttft_latency: if isinstance(_call_latency, float): total += _call_latency + total = total / len(item_ttft_latency) if total > 0 else float('inf') elif len(item_latency) > 0 and isinstance(item_latency[0], timedelta): for _call_latency in item_latency: total += _call_latency.total_seconds() @@ -485,7 +484,7 @@ class LowestLatencyLoggingHandler(CustomLogger): for _call_latency in item_latency: if isinstance(_call_latency, float): total += _call_latency - item_latency = total / len(item_latency) + item_latency = total / len(item_latency) if total > 0 else float('inf') # -------------- # # Debugging Logic