From f19d7327cadd5931b44a8fb0a22d50609bfe6669 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 21 May 2024 18:42:15 -0700 Subject: [PATCH] fix(lowest_latency.py): set default none value for time_to_first_token in sync log success event --- litellm/router_strategy/lowest_latency.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index b47865c7a9d..1e4b151ada5 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -84,6 +84,7 @@ class LowestLatencyLoggingHandler(CustomLogger): response_ms: timedelta = end_time - start_time time_to_first_token_response_time: Optional[timedelta] = None + if kwargs.get("stream", None) is not None and kwargs["stream"] == True: # only log ttft for streaming request time_to_first_token_response_time = ( @@ -91,7 +92,7 @@ class LowestLatencyLoggingHandler(CustomLogger): ) final_value = response_ms - + time_to_first_token: Optional[float] = None total_tokens = 0 if isinstance(response_obj, ModelResponse):