scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

fix: prevent negative request counts in least_busy router strategy

Browse source 
The least_busy routing strategy can cause traffic to some deployments to
drop to zero because request counts can go negative. When a race
condition causes a decrement callback to fire without a matching
increment (e.g., cache expiry between pre-call and completion), the
count goes below zero. Since the router always picks the deployment
with the lowest count, a deployment with a negative count becomes a
"black hole" that attracts all traffic.

This fix:
1. Clamps request counts to a minimum of 0 using max(value - 1, 0)
2. Adds random tie-breaking when multiple deployments share the same
   minimum traffic count, ensuring fair distribution

Fixes #25323

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
easonysliu 2026-04-09 12:01:35 +08:00
parent 233870d7b2
commit 27ab742d4c
1 changed files with 13 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
litellm/router_strategy/least_busy.py
View file

@ -76,7 +76,7 @@ class LeastBusyLoggingHandler(CustomLogger):
request_count_value: Optional[int] = request_count_dict.get(id, 0)
if request_count_value is None:
return
request_count_dict[id] = request_count_value - 1
request_count_dict[id] = max(request_count_value - 1, 0)
self.router_cache.set_cache(
key=request_count_api_key, value=request_count_dict
)
@ -109,7 +109,7 @@ class LeastBusyLoggingHandler(CustomLogger):
request_count_value: Optional[int] = request_count_dict.get(id, 0)
if request_count_value is None:
return
request_count_dict[id] = request_count_value - 1
request_count_dict[id] = max(request_count_value - 1, 0)
self.router_cache.set_cache(
key=request_count_api_key, value=request_count_dict
)
@ -144,7 +144,7 @@ class LeastBusyLoggingHandler(CustomLogger):
request_count_value: Optional[int] = request_count_dict.get(id, 0)
if request_count_value is None:
return
request_count_dict[id] = request_count_value - 1
request_count_dict[id] = max(request_count_value - 1, 0)
await self.router_cache.async_set_cache(
key=request_count_api_key, value=request_count_dict
)
@ -178,7 +178,7 @@ class LeastBusyLoggingHandler(CustomLogger):
request_count_value: Optional[int] = request_count_dict.get(id, 0)
if request_count_value is None:
return
request_count_dict[id] = request_count_value - 1
request_count_dict[id] = max(request_count_value - 1, 0)
await self.router_cache.async_set_cache(
key=request_count_api_key, value=request_count_dict
)
@ -202,22 +202,22 @@ class LeastBusyLoggingHandler(CustomLogger):
if d["model_info"]["id"] not in all_deployments:
all_deployments[d["model_info"]["id"]] = 0
# map deployment to id
# pick least busy deployment
# pick least busy deployment, with random jitter on ties
min_traffic = float("inf")
min_deployment = None
for k, v in all_deployments.items():
if v < min_traffic:
min_traffic = v
min_deployment = k
if min_deployment is not None:
## check if min deployment is a string, if so, cast it to int
# collect all deployments tied at the minimum
min_deployment_ids = [k for k, v in all_deployments.items() if v == min_traffic]
if min_deployment_ids:
chosen_id = random.choice(min_deployment_ids)
for m in healthy_deployments:
if m["model_info"]["id"] == min_deployment:
if m["model_info"]["id"] == chosen_id:
return m
min_deployment = random.choice(healthy_deployments)
# chosen_id not in healthy list, fall back
return random.choice(healthy_deployments)
else:
min_deployment = random.choice(healthy_deployments)
return min_deployment
return random.choice(healthy_deployments)
def get_available_deployments(
self,