scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Handle empty valid_deployments in LowestLatencyLoggingHandler

Browse source
This commit is contained in:
Emerson Gomes 2025-03-19 19:56:57 -05:00
parent 1cc832f53e
commit c91aafeed8
1 changed files with 6 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/router_strategy/lowest_latency.py
View file

@ -511,6 +511,9 @@ class LowestLatencyLoggingHandler(CustomLogger):
if len(potential_deployments) == 0:
return None
if not healthy_deployments:
return None
# Sort potential deployments by latency
sorted_deployments = sorted(potential_deployments, key=lambda x: x[1])
@ -520,9 +523,10 @@ class LowestLatencyLoggingHandler(CustomLogger):
# Find deployments within buffer of lowest latency
buffer = self.routing_args.lowest_latency_buffer * lowest_latency
# If no deployments within buffer, fall back to all sorted deployments
valid_deployments = [
x for x in sorted_deployments if x[1] <= lowest_latency + buffer
]
x for x in sorted_deployments if x[1] <= lowest_latency + buffer
] or sorted_deployments
# Pick a random deployment from valid deployments
random_valid_deployment = random.choice(valid_deployments)