From 4cb4a7f06def393849f88d90d42adc02ed09ff61 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Apr 2024 16:02:57 -0700 Subject: [PATCH] fix - lowest latency routing --- litellm/router_strategy/lowest_latency.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index cd5cea2e942..eecf5578cee 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -312,7 +312,9 @@ class LowestLatencyLoggingHandler(CustomLogger): except: input_tokens = 0 - all_deployments = random.sample(all_deployments.items(), len(all_deployments)) + # randomly sample from all_deployments, incase all deployments have latency=0.0 + _items = all_deployments.items() + all_deployments = random.sample(list(_items), len(_items)) all_deployments = dict(all_deployments) for item, item_map in all_deployments.items(): ## get the item from model list