From 3ce37e6c35f41d9e337e9db6c6e58b3616e4699b Mon Sep 17 00:00:00 2001 From: JiangNan <1394485448@qq.com> Date: Tue, 10 Mar 2026 10:49:31 +0800 Subject: [PATCH] fix: use correct list length when averaging TTFT latency for streaming requests (#23100) In _get_available_deployments, when streaming mode is active the code sums time-to-first-token values from item_ttft_latency but divides by len(item_latency) instead of len(item_ttft_latency). These lists can have different lengths, producing an incorrect average that skews lowest-latency routing decisions for streaming requests. Signed-off-by: JiangNan <1394485448@qq.com> --- litellm/router_strategy/lowest_latency.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 0449a843bd2..e09b5c1456d 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -490,20 +490,22 @@ class LowestLatencyLoggingHandler(CustomLogger): # get average latency or average ttft (depending on streaming/non-streaming) total: float = 0.0 - if ( + use_ttft = ( request_kwargs is not None and request_kwargs.get("stream", None) is not None and request_kwargs["stream"] is True and len(item_ttft_latency) > 0 - ): + ) + if use_ttft: for _call_latency in item_ttft_latency: if isinstance(_call_latency, float): total += _call_latency + item_latency = total / len(item_ttft_latency) else: for _call_latency in item_latency: if isinstance(_call_latency, float): total += _call_latency - item_latency = total / len(item_latency) + item_latency = total / len(item_latency) # -------------- # # Debugging Logic