scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

fix(least-busy): keep realtime sessions out of in-flight counts

Browse source
This commit is contained in:
mateo-berri 2026-11-04 15:59:27 -08:00
parent 6643fc2e56
commit 58600945d3
2 changed files with 16 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/router_strategy/least_busy.py
View file

@ -17,6 +17,7 @@ from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_cache import log_redis_failure
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.types.utils import CallTypes
IN_FLIGHT_COUNT_TTL_SECONDS: Final = 60 * 60
SELECTION_RESERVATION_TTL_SECONDS: Final = 30.0
@ -97,7 +98,8 @@ def _routed_deployment_id(metadata: _Metadata | None) -> str | int | None:
def _deployment_ref(kwargs: Mapping[str, object]) -> tuple[str, str] | None:
if is_batch_retrieve_call_type(kwargs.get("call_type")):
call_type: Final = kwargs.get("call_type")
if is_batch_retrieve_call_type(call_type) or call_type == CallTypes.arealtime.value:
return None
try:
call: Final = _CALL_KWARGS.validate_python(kwargs)

14
tests/unit/router_strategy/test_least_busy.py
View file

@ -30,7 +30,7 @@ from litellm.router_strategy.least_busy import (
picks_reserve,
release_unstarted_picks,
)
from litellm.types.utils import ModelResponse
from litellm.types.utils import CallTypes, ModelResponse
GROUP: Final = "least-busy-group"
DEPLOYMENT_A: Final[dict[str, object]] = {"model_info": {"id": "dep-a"}}
@ -166,6 +166,18 @@ def test_a_request_routed_through_litellm_metadata_counts_while_in_flight() -> N
assert calling_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
def test_a_closed_realtime_session_leaves_its_deployment_idle() -> None:
shared: Final = SharedRedisCounters()
worker: Final = _worker(shared)
session_event: Final = {**_routed_through_litellm_metadata("dep-a"), "call_type": CallTypes.arealtime.value}
for _ in range(5):
worker.log_pre_api_call(model="m", messages=[], kwargs=session_event)
worker.log_success_event(session_event, None, None, None)
assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
def test_the_handler_never_pushes_a_counters_ttl_forward() -> None:
"""A worker that dies mid-request leaves a +1 nobody will ever decrement. Redis expires that
stuck count an hour after the key was created, which only works while nothing writes the TTL