scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 154

fix - lowest latency routing

Browse source
This commit is contained in:
Ishaan Jaff 2024-04-29 16:02:57 -07:00
parent 3b0aa05378
commit 4cb4a7f06d
1 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/router_strategy/lowest_latency.py
View file

@ -312,7 +312,9 @@ class LowestLatencyLoggingHandler(CustomLogger):
except:
input_tokens = 0
all_deployments = random.sample(all_deployments.items(), len(all_deployments))
# randomly sample from all_deployments, incase all deployments have latency=0.0
_items = all_deployments.items()
all_deployments = random.sample(list(_items), len(_items))
all_deployments = dict(all_deployments)
for item, item_map in all_deployments.items():
## get the item from model list