scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 123

fix(router): clamp least_busy request counter to prevent negative drift

Browse source 
The least_busy router strategy decrements a per-deployment request
counter on success/failure callbacks. Under race conditions (callback
fires before pre-call, or fires twice), the counter can go negative.

A negative count is always less than the 0 assigned to unused
deployments, so the negative-count deployment attracts ALL traffic
while others starve to zero requests.

Fix: clamp the counter with max(value - 1, 0) on all 4 decrement
paths (sync success, sync failure, async success, async failure).

Fixes BerriAI/litellm#25323

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
rudra717 2026-04-07 23:24:40 -07:00
parent 39c1042258
commit 4c831bfca7
1 changed files with 4 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/router_strategy/least_busy.py
View file

@ -76,7 +76,7 @@ class LeastBusyLoggingHandler(CustomLogger):
request_count_value: Optional[int] = request_count_dict.get(id, 0)
if request_count_value is None:
return
request_count_dict[id] = request_count_value - 1
request_count_dict[id] = max(request_count_value - 1, 0)
self.router_cache.set_cache(
key=request_count_api_key, value=request_count_dict
)
@ -109,7 +109,7 @@ class LeastBusyLoggingHandler(CustomLogger):
request_count_value: Optional[int] = request_count_dict.get(id, 0)
if request_count_value is None:
return
request_count_dict[id] = request_count_value - 1
request_count_dict[id] = max(request_count_value - 1, 0)
self.router_cache.set_cache(
key=request_count_api_key, value=request_count_dict
)
@ -144,7 +144,7 @@ class LeastBusyLoggingHandler(CustomLogger):
request_count_value: Optional[int] = request_count_dict.get(id, 0)
if request_count_value is None:
return
request_count_dict[id] = request_count_value - 1
request_count_dict[id] = max(request_count_value - 1, 0)
await self.router_cache.async_set_cache(
key=request_count_api_key, value=request_count_dict
)
@ -178,7 +178,7 @@ class LeastBusyLoggingHandler(CustomLogger):
request_count_value: Optional[int] = request_count_dict.get(id, 0)
if request_count_value is None:
return
request_count_dict[id] = request_count_value - 1
request_count_dict[id] = max(request_count_value - 1, 0)
await self.router_cache.async_set_cache(
key=request_count_api_key, value=request_count_dict
)