This commit is contained in:
Oliver Yu 2026-10-30 01:41:16 +00:00 • committed by GitHub
commit 58b4e7af7c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 702 additions and 38 deletions

View file

@ -219,7 +219,7 @@ from litellm.router_strategy.complexity_router.context_compaction import (
reject_recursive_compactor,
surface_for_call,
)
from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
from litellm.router_strategy.least_busy import SELECTION_RESERVATION_MAX_KEYS, LeastBusyLoggingHandler
from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler
from litellm.router_strategy.lowest_tpm_rpm import LowestTPMLoggingHandler
@ -1387,6 +1387,7 @@ class Router(DeprecatedRouterMembers):
messages=messages,
input=input,
specific_deployment=specific_deployment,
reserve=False,
)
if kind == "healthy":
return await self._core_async_get_healthy_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts
@ -1656,6 +1657,9 @@ class Router(DeprecatedRouterMembers):
self._core_cache = DualCache(
redis_cache=redis_cache, in_memory_cache=InMemoryCache()
) # use a dual cache (Redis+In-Memory) for tracking cooldowns, usage, etc.
self._core_least_busy_reservations: InMemoryCache = InMemoryCache(
max_size_in_memory=SELECTION_RESERVATION_MAX_KEYS
)
self._claude_code_session_router_cache: DualCache = DualCache(
redis_cache=redis_cache,
in_memory_cache=InMemoryCache(),
@ -2163,7 +2167,9 @@ class Router(DeprecatedRouterMembers):
selector: RouterStrategySelector | None = None
match self._normalize_strategy(strategy):
case RoutingStrategy.LEAST_BUSY.value:
selector = LeastBusyLoggingHandler(router_cache=self._core_cache)
selector = LeastBusyLoggingHandler(
router_cache=self._core_cache, reservation_store=self._core_least_busy_reservations
)
case RoutingStrategy.USAGE_BASED_ROUTING.value:
selector = LowestTPMLoggingHandler(
router_cache=self._core_cache,
@ -2629,6 +2635,7 @@ class Router(DeprecatedRouterMembers):
messages: list[dict[str, str]] | None,
input: str | list | None,
request_kwargs: dict | None,
reserve: bool,
) -> Any | None:
"""
Asks the strategy selector for a deployment. Caller handles
@ -2642,6 +2649,7 @@ class Router(DeprecatedRouterMembers):
return await selector.async_get_available_deployments(
model_group=model,
healthy_deployments=healthy_deployments,
reserve=reserve,
)
case "usage-based-routing":
# `LowestTPMLoggingHandler` (v1) only exposes the sync
@ -2682,6 +2690,7 @@ class Router(DeprecatedRouterMembers):
messages: list[dict[str, str]] | None,
input: str | list | None,
request_kwargs: dict | None,
reserve: bool,
) -> Any | None:
"""
Sync sibling of `_select_deployment_async`. Caller handles
@ -2698,6 +2707,7 @@ class Router(DeprecatedRouterMembers):
return selector.get_available_deployments(
model_group=model,
healthy_deployments=healthy_deployments,
reserve=reserve,
)
case "usage-based-routing" | "usage-based-routing-v2":
return selector.get_available_deployments(
@ -8054,6 +8064,7 @@ class Router(DeprecatedRouterMembers):
messages=[{"role": "user", "content": "batch-api-fake-text"}],
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
reserve=False,
)
selected_deployment_id: Final = (deployment.get("model_info") or {}).get("id")
@ -10489,6 +10500,7 @@ class Router(DeprecatedRouterMembers):
messages=None,
input=None,
request_kwargs=routing_kwargs,
reserve=False,
)
if selected is None:
no_deployment: Final = await async_raise_no_deployment_exception(
@ -15694,6 +15706,7 @@ class Router(DeprecatedRouterMembers):
messages: list[dict[str, str]] | None = None,
input: str | list | None = None,
specific_deployment: bool | None = False,
reserve: bool = True,
):
"""
Async implementation of 'get_available_deployments'.
@ -15707,6 +15720,7 @@ class Router(DeprecatedRouterMembers):
messages=messages,
input=input,
specific_deployment=specific_deployment,
reserve=reserve,
)
async def _async_get_available_deployment(
@ -15716,6 +15730,7 @@ class Router(DeprecatedRouterMembers):
messages: list[dict[str, str]] | None,
input: str | list | None,
specific_deployment: bool | None,
reserve: bool,
):
if (
self._core_routing_strategy != "usage-based-routing-v2"
@ -15812,6 +15827,7 @@ class Router(DeprecatedRouterMembers):
messages=messages,
input=input,
request_kwargs=request_kwargs,
reserve=reserve and request_kwargs not in (None, {}),
)
if deployment is None:
exception: Final = await async_raise_no_deployment_exception(
@ -15981,6 +15997,7 @@ class Router(DeprecatedRouterMembers):
messages=messages,
input=input,
request_kwargs=request_kwargs,
reserve=request_kwargs not in (None, {}),
)
if deployment is None:
@ -16856,6 +16873,7 @@ class Router(DeprecatedRouterMembers):
messages=messages,
input=input,
request_kwargs=request_kwargs,
reserve=request_kwargs not in (None, {}),
)
if deployment is None:
@ -17038,6 +17056,7 @@ class Router(DeprecatedRouterMembers):
messages=messages,
input=input,
request_kwargs=request_kwargs,
reserve=request_kwargs not in (None, {}),
)
if deployment is None:

View file

@ -1,6 +1,8 @@
import logging
from collections.abc import Mapping, Sequence
from typing import Final
import threading
import time
from collections.abc import Callable, Mapping, Sequence
from typing import Final, Protocol
from pydantic import TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
@ -8,11 +10,16 @@ from typing_extensions import ReadOnly, TypedDict
from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_cache import log_redis_failure
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
IN_FLIGHT_COUNT_TTL_SECONDS: Final = 60 * 60
SELECTION_RESERVATION_TTL_SECONDS: Final = 30.0
SELECTION_RESERVATION_MAX_KEYS: Final = 10_000
_RESERVATIONS_LOCK: Final = threading.Lock()
class _ModelInfo(TypedDict, total=False):
@ -21,10 +28,12 @@ class _ModelInfo(TypedDict, total=False):
class _Metadata(TypedDict, total=False):
model_group: ReadOnly[str | None]
model_info: ReadOnly[_ModelInfo | None]
class _LitellmParams(TypedDict, total=False):
metadata: ReadOnly[_Metadata | None]
litellm_metadata: ReadOnly[_Metadata | None]
model_info: ReadOnly[_ModelInfo | None]
@ -42,13 +51,26 @@ class _Deployment(TypedDict):
_CALL_KWARGS: Final = TypeAdapter(_CallKwargs)
_DEPLOYMENTS: Final = TypeAdapter(list[_Deployment])
_MEMORY_COUNTS: Final = TypeAdapter(tuple[float | None, ...] | None)
_MEMORY_COUNTS: Final[TypeAdapter[tuple[float | None, ...] | None]] = TypeAdapter(tuple[float | None, ...] | None)
_RESERVATION_TIMES: Final[TypeAdapter[tuple[float, ...] | None]] = TypeAdapter(tuple[float, ...] | None)
def _request_count_key(model_group: str, deployment_id: str) -> str:
return f"{model_group}_request_count:{deployment_id}"
def _model_group(metadata: _Metadata | None) -> str | None:
return metadata.get("model_group") if metadata else None
def _deployment_id(model_info: _ModelInfo | None) -> str | int | None:
return model_info.get("id") if model_info else None
def _routed_deployment_id(metadata: _Metadata | None) -> str | int | None:
return _deployment_id(metadata.get("model_info")) if metadata else None
def _deployment_ref(kwargs: Mapping[str, object]) -> tuple[str, str] | None:
if is_batch_retrieve_call_type(kwargs.get("call_type")):
return None
@ -56,16 +78,35 @@ def _deployment_ref(kwargs: Mapping[str, object]) -> tuple[str, str] | None:
call: Final = _CALL_KWARGS.validate_python(kwargs)
except ValidationError:
return None
litellm_params: Final = call.get("litellm_params")
metadata: Final = litellm_params.get("metadata") if litellm_params else None
model_info: Final = litellm_params.get("model_info") if litellm_params else None
model_group: Final = metadata.get("model_group") if metadata else None
deployment_id: Final = model_info.get("id") if model_info else None
litellm_params: Final = call.get("litellm_params") or _LitellmParams()
routing_metadata: Final = (litellm_params.get("litellm_metadata"), litellm_params.get("metadata"))
model_group: Final = next((group for group in map(_model_group, routing_metadata) if group is not None), None)
deployment_id: Final = next(
(
deployment_id
for deployment_id in (
_deployment_id(litellm_params.get("model_info")),
*map(_routed_deployment_id, routing_metadata),
)
if deployment_id is not None
),
None,
)
if model_group is None or deployment_id is None:
return None
return model_group, str(deployment_id)
def _counted_in_flight(kwargs: Mapping[str, object]) -> bool:
return kwargs.get("cache_hit") is not True and (
kwargs.get("api_call_start_time") is not None or LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL in kwargs
)
def _reservation_key(request_count_key: str) -> str:
return f"{request_count_key}:reservations"
def _request_count_keys(model_group: str, healthy_deployments: Sequence[Mapping[str, object]]) -> tuple[str, ...]:
return tuple(
_request_count_key(model_group, str(deployment["model_info"]["id"]))
@ -84,12 +125,12 @@ def _local_counts(raw: object, keys: tuple[str, ...]) -> tuple[int, ...]:
return _as_counts(values)
def _least_busy(
healthy_deployments: Sequence[Mapping[str, object]], counts: tuple[int, ...]
) -> Mapping[str, object] | None:
if not healthy_deployments:
return None
return healthy_deployments[min(range(len(healthy_deployments)), key=lambda index: counts[index])]
def _unexpired(times: Sequence[float], now: float) -> tuple[float, ...]:
return tuple(t for t in times if now - t < SELECTION_RESERVATION_TTL_SECONDS)
def _least_busy_index(counts: tuple[int, ...]) -> int:
return min(range(len(counts)), key=lambda index: counts[index])
def _warn_unreadable(model_group: str, error: Exception) -> None:
@ -111,24 +152,80 @@ def _warn_unwritable(key: str, error: Exception) -> None:
)
class _ReservationStore(Protocol):
def get_cache(self, key: str) -> object: ...
def set_cache(self, key: str, value: tuple[float, ...], *, ttl: float) -> None: ...
def delete_cache(self, key: str) -> None: ...
class LeastBusyLoggingHandler(CustomLogger):
test_flag: bool = False
logged_success: int = 0
logged_failure: int = 0
def __init__(self, router_cache: DualCache):
def __init__(
self,
router_cache: DualCache,
clock: Callable[[], float] = time.monotonic,
reservation_store: _ReservationStore | None = None,
):
self.router_cache = router_cache
self.router_cache_id = str(id(router_cache))
self._clock = clock
self._reservations: Final[_ReservationStore] = (
reservation_store
if reservation_store is not None
else InMemoryCache(max_size_in_memory=SELECTION_RESERVATION_MAX_KEYS, clock=clock)
)
def _live_reservations(self, key: str, now: float) -> tuple[float, ...]:
raw: Final = self._reservations.get_cache(_reservation_key(key))
return _unexpired(_RESERVATION_TIMES.validate_python(raw) or (), now)
def _store_reservations(self, key: str, times: tuple[float, ...]) -> None:
# InMemoryCache keeps a live key's first TTL, so drop the key to restart it
self._reservations.delete_cache(_reservation_key(key))
if times:
self._reservations.set_cache(_reservation_key(key), times, ttl=SELECTION_RESERVATION_TTL_SECONDS)
def _select(
self,
healthy_deployments: Sequence[Mapping[str, object]],
keys: tuple[str, ...],
counts: tuple[int, ...],
reserve: bool,
) -> Mapping[str, object] | None:
if not healthy_deployments:
return None
with _RESERVATIONS_LOCK:
now: Final = self._clock()
live: Final = tuple(self._live_reservations(key, now) for key in keys)
index: Final = _least_busy_index(tuple(count + len(r) for count, r in zip(counts, live, strict=True)))
if reserve:
self._store_reservations(keys[index], (*live[index], now))
return healthy_deployments[index]
def _release_reservation(self, kwargs: Mapping[str, object]) -> None:
ref: Final = _deployment_ref(kwargs)
if ref is None:
return
key: Final = _request_count_key(*ref)
with _RESERVATIONS_LOCK:
now: Final = self._clock()
self._store_reservations(key, self._live_reservations(key, now)[1:])
@with_service_target("router_usage")
def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None:
self._release_reservation(kwargs)
self._increment(kwargs, 1)
@with_service_target("router_usage")
def log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self._increment(kwargs, -1)
self._settle(kwargs)
if self.test_flag:
self.logged_success += 1
@ -136,7 +233,7 @@ class LeastBusyLoggingHandler(CustomLogger):
def log_failure_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self._increment(kwargs, -1)
self._settle(kwargs)
if self.test_flag:
self.logged_failure += 1
@ -144,7 +241,7 @@ class LeastBusyLoggingHandler(CustomLogger):
async def async_log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
await self._async_increment(kwargs, -1)
await self._async_settle(kwargs)
if self.test_flag:
self.logged_success += 1
@ -152,13 +249,13 @@ class LeastBusyLoggingHandler(CustomLogger):
async def async_log_failure_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
await self._async_increment(kwargs, -1)
await self._async_settle(kwargs)
if self.test_flag:
self.logged_failure += 1
@with_service_target("router_usage")
def get_available_deployments(
self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]]
self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]], reserve: bool = False
) -> Mapping[str, object] | None:
keys: Final = _request_count_keys(model_group, healthy_deployments)
redis_cache: Final = self.router_cache.redis_cache
@ -168,13 +265,12 @@ class LeastBusyLoggingHandler(CustomLogger):
except Exception as e:
_warn_unreadable(model_group, e)
else:
return _least_busy(healthy_deployments, shared)
local: Final = _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys)
return _least_busy(healthy_deployments, local)
return self._select(healthy_deployments, keys, self._at_least_this_workers(keys, shared), reserve)
return self._select(healthy_deployments, keys, self._this_workers_counts(keys), reserve)
@with_service_target("router_usage")
async def async_get_available_deployments(
self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]]
self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]], reserve: bool = False
) -> Mapping[str, object] | None:
keys: Final = _request_count_keys(model_group, healthy_deployments)
redis_cache: Final = self.router_cache.redis_cache
@ -184,9 +280,26 @@ class LeastBusyLoggingHandler(CustomLogger):
except Exception as e:
_warn_unreadable(model_group, e)
else:
return _least_busy(healthy_deployments, shared)
local: Final = _local_counts(await self.router_cache.async_batch_get_cache(list(keys), local_only=True), keys)
return _least_busy(healthy_deployments, local)
return self._select(healthy_deployments, keys, self._at_least_this_workers(keys, shared), reserve)
return self._select(healthy_deployments, keys, self._this_workers_counts(keys), reserve)
def _settle(self, kwargs: Mapping[str, object]) -> None:
if not _counted_in_flight(kwargs):
self._release_reservation(kwargs)
return
self._increment(kwargs, -1)
async def _async_settle(self, kwargs: Mapping[str, object]) -> None:
if not _counted_in_flight(kwargs):
self._release_reservation(kwargs)
return
await self._async_increment(kwargs, -1)
def _this_workers_counts(self, keys: tuple[str, ...]) -> tuple[int, ...]:
return _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys)
def _at_least_this_workers(self, keys: tuple[str, ...], shared: tuple[int, ...]) -> tuple[int, ...]:
return tuple(max(counts) for counts in zip(shared, self._this_workers_counts(keys), strict=True))
def _increment(self, kwargs: Mapping[str, object], delta: int) -> None:
ref: Final = _deployment_ref(kwargs)

View file

@ -0,0 +1,90 @@
import json
import threading
from collections import Counter
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Final
import httpx
import pytest
import yaml
from integration._support.client import Gateway, gateway_from_environment
from integration._support.process import owned_proxy
from integration._support.wire import Reply, Request, wire_server
pytestmark = pytest.mark.timeout(240)
_GROUP: Final = "least-busy-decider"
_DEPLOYMENTS: Final = ("least-busy-decider-a", "least-busy-decider-b")
_BURST: Final = 4
_REPLY: Final = json.dumps(
{
"model": "Qwen/Qwen3-0.6B",
"answers": {"billing": {"type": "noul", "noul": 0.91}},
"usage": {"input_tokens": 367, "output_tokens": 3},
}
).encode()
_DECISION: Final = {
"model": _GROUP,
"input": "I was charged twice for order 4417.",
"questions": [{"type": "predicate", "name": "billing", "instructions": "Is this a billing issue?"}],
}
def _held_until_the_burst_arrives(arrived: threading.Barrier) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
arrived.wait()
return Reply(body=_REPLY)
return respond
def _write_config(path: Path, provider_url: str) -> Path:
path.write_text(
yaml.safe_dump(
{
"model_list": [
{
"model_name": _GROUP,
"litellm_params": {"model": "hosted_vllm/Qwen/Qwen3-0.6B", "api_base": provider_url},
"model_info": {"id": deployment_id},
}
for deployment_id in _DEPLOYMENTS
],
"general_settings": {
"master_key": "os.environ/LITELLM_MASTER_KEY",
"database_url": "os.environ/DATABASE_URL",
"disable_model_info_refresh": True,
},
"router_settings": {"routing_strategy": "least-busy", "num_retries": 0, "disable_cooldowns": True},
}
)
)
return path
def _decide(proxy: Gateway) -> httpx.Response:
return proxy.client.post(
"/v1/decisions", json=_DECISION, headers={"Authorization": f"Bearer {proxy.key}"}, timeout=120
)
def test_least_busy_spreads_a_burst_of_decisions_calls_across_the_group(tmp_path: Path) -> None:
arrived: Final = threading.Barrier(_BURST, timeout=60)
with (
wire_server(_held_until_the_burst_arrives(arrived)) as upstream,
gateway_from_environment() as rig,
owned_proxy(
rig,
tmp_path,
{"STORE_MODEL_IN_DB": "False"},
config=_write_config(tmp_path / "least-busy.yaml", upstream.url),
) as proxy,
ThreadPoolExecutor(max_workers=_BURST) as pool,
):
responses: Final = tuple(pool.map(lambda _: _decide(proxy), range(_BURST)))
assert [response.status_code for response in responses] == [200] * _BURST, [r.text for r in responses]
assert Counter(response.headers["x-litellm-model-id"] for response in responses) == Counter(
{deployment_id: _BURST // len(_DEPLOYMENTS) for deployment_id in _DEPLOYMENTS}
)

View file

@ -6,6 +6,8 @@ import copy
import json
import re
import threading
import uuid
from datetime import datetime
from collections.abc import Mapping, Sequence
from concurrent.futures import wait
from types import MappingProxyType
@ -297,6 +299,43 @@ async def test_router_dispatches_typesafe_decisions_without_api_base(
assert response.answers["sentiment"].choice == "positive"
@pytest.mark.asyncio
async def test_least_busy_frees_a_decisions_deployment_once_a_proxy_logged_call_finishes(
respx_mock: respx.MockRouter,
) -> None:
respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE)
router: Final = litellm.Router(
model_list=[
{
"model_name": "jev",
"litellm_params": {"model": "typesafe/jev-latest", "api_key": "k"},
"model_info": {"id": deployment_id},
}
for deployment_id in ("jev-a", "jev-b")
],
routing_strategy="least-busy",
)
async def call_through_proxy_logging() -> object:
logging_obj, _ = litellm.utils.function_setup(
original_function="adecisions",
rules_obj=litellm.utils.Rules(),
start_time=datetime.now(),
model="jev",
litellm_call_id=str(uuid.uuid4()),
)
response: Final = await router.adecisions(
model="jev",
input="Is this a defect?",
questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
litellm_logging_obj=logging_obj,
)
await _drain_logging_worker()
return response._hidden_params["model_id"]
assert (await call_through_proxy_logging(), await call_through_proxy_logging()) == ("jev-a", "jev-a")
def test_decisions_uses_the_same_wire_contract_for_sync_calls(respx_mock: respx.MockRouter) -> None:
route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)

View file

@ -1,9 +1,15 @@
import asyncio
import contextlib
import datetime
import json
import logging
import threading
from collections.abc import Callable, Iterator
from concurrent.futures import ThreadPoolExecutor
from typing import Final
import httpx
import openai
import pytest
import respx
@ -11,22 +17,32 @@ import litellm
from litellm import Router
from litellm.caching.caching import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY, InMemoryCache
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.router_strategy.least_busy import IN_FLIGHT_COUNT_TTL_SECONDS, LeastBusyLoggingHandler
from litellm.router_strategy.least_busy import (
IN_FLIGHT_COUNT_TTL_SECONDS,
SELECTION_RESERVATION_TTL_SECONDS,
LeastBusyLoggingHandler,
)
from litellm.types.utils import ModelResponse
GROUP: Final = "least-busy-group"
CALL_STARTED_AT: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
DEPLOYMENT_A: Final[dict[str, object]] = {"model_info": {"id": "dep-a"}}
DEPLOYMENT_B: Final[dict[str, object]] = {"model_info": {"id": "dep-b"}}
HEALTHY: Final = [DEPLOYMENT_A, DEPLOYMENT_B]
def _call_kwargs(deployment_id: str) -> dict[str, object]:
def _routed_call_kwargs(deployment_id: str) -> dict[str, object]:
return {"litellm_params": {"metadata": {"model_group": GROUP}, "model_info": {"id": deployment_id}}}
def _call_kwargs(deployment_id: str) -> dict[str, object]:
return {**_routed_call_kwargs(deployment_id), "api_call_start_time": CALL_STARTED_AT}
class SharedRedisCounters:
"""Mirrors what Redis gives the handler: increments clamped at zero, a TTL set once when
the key is created, and ordered reads that raise rather than invent a value."""
@ -58,6 +74,20 @@ class SharedRedisCounters:
return self.batch_get_counts(key_list)
class SlowRedisRead(SharedRedisCounters):
"""Answers a count read with the counts from when the read started, after letting other work run."""
def __init__(self) -> None:
super().__init__()
self.while_reading: Callable[[], None] = lambda: None
async def async_batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]:
counts_when_asked: Final = self.batch_get_counts(key_list)
await asyncio.sleep(0)
self.while_reading()
return counts_when_asked
def _worker(shared: SharedRedisCounters | None) -> LeastBusyLoggingHandler:
cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=shared) # pyright: ignore[reportArgumentType] # duck-typed Redis double
return LeastBusyLoggingHandler(router_cache=cache)
@ -81,6 +111,16 @@ async def test_worker_routes_around_a_request_another_worker_started() -> None:
assert await picking_worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
@pytest.mark.asyncio
async def test_a_request_that_starts_while_redis_is_read_still_counts_on_its_deployment() -> None:
redis: Final = SlowRedisRead()
worker: Final = _worker(redis)
assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
redis.while_reading = lambda: worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
def test_sync_pick_reads_the_shared_counts() -> None:
shared: Final = SharedRedisCounters()
streaming_worker: Final = _worker(shared)
@ -98,6 +138,32 @@ def test_sync_pick_reads_the_shared_counts() -> None:
assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
def _routed_through_litellm_metadata(deployment_id: str) -> dict[str, object]:
return {
"litellm_params": {
"metadata": {"tags": ["caller-tag"]},
"litellm_metadata": {"model_group": GROUP},
"model_info": {"id": deployment_id},
},
"api_call_start_time": CALL_STARTED_AT,
}
def test_a_request_routed_through_litellm_metadata_counts_while_in_flight() -> None:
shared: Final = SharedRedisCounters()
calling_worker: Final = _worker(shared)
picking_worker: Final = _worker(shared)
assert calling_worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
calling_worker.log_pre_api_call(model="m", messages=[], kwargs=_routed_through_litellm_metadata("dep-a"))
assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
calling_worker.log_success_event(_routed_through_litellm_metadata("dep-a"), None, None, None)
assert calling_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
def test_the_handler_never_pushes_a_counters_ttl_forward() -> None:
"""A worker that dies mid-request leaves a +1 nobody will ever decrement. Redis expires that
stuck count an hour after the key was created, which only works while nothing writes the TTL
@ -391,3 +457,273 @@ async def test_router_picks_least_busy_deployment_and_completion_restores_counts
assert response._hidden_params["model_id"] == "1"
assert route.calls.last.request.headers["authorization"] == "Bearer key-1"
assert _router_counts(router) == seeded
@pytest.mark.asyncio
async def test_router_spreads_requests_that_start_together(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
burst: Final = 2 * len(ROUTER_DEPLOYMENT_IDS)
all_started: Final = asyncio.Barrier(burst)
async def completion_after_all_started(_: httpx.Request) -> httpx.Response:
await all_started.wait()
return httpx.Response(
200,
json={
"id": "chatcmpl-least-busy",
"object": "chat.completion",
"created": 1,
"model": "gpt-4.1-mini",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "done"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
)
route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
side_effect=completion_after_all_started
)
router: Final = _least_busy_router()
await asyncio.gather(
*(router.acompletion(model=ROUTER_GROUP, messages=[{"role": "user", "content": "hi"}]) for _ in range(burst))
)
one_per_deployment: Final = tuple(f"Bearer key-{deployment_id}" for deployment_id in ROUTER_DEPLOYMENT_IDS)
assert sorted(call.request.headers["authorization"] for call in route.calls) == sorted(one_per_deployment * 2)
class _ManualClock:
def __init__(self) -> None:
self.now = 0.0
def __call__(self) -> float:
return self.now
def test_a_reservation_that_never_reaches_the_api_call_expires() -> None:
clock: Final = _ManualClock()
worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), clock=clock)
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
clock.now = 1.0
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
clock.now = 2.0
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
clock.now = SELECTION_RESERVATION_TTL_SECONDS + 0.5
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
def test_a_fresh_reservation_outlives_the_ttl_of_the_first_one_on_its_deployment() -> None:
clock: Final = _ManualClock()
cache: Final = DualCache(in_memory_cache=InMemoryCache(clock=clock))
worker: Final = LeastBusyLoggingHandler(router_cache=cache, clock=clock)
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
worker.log_success_event(kwargs=_call_kwargs("dep-a"), response_obj=None, start_time=None, end_time=None)
clock.now = SELECTION_RESERVATION_TTL_SECONDS - 0.1
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
clock.now = SELECTION_RESERVATION_TTL_SECONDS + 0.1
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
def test_a_started_request_moves_its_reservation_into_the_in_flight_count() -> None:
worker: Final = _worker(None)
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
def test_a_request_started_through_another_selector_of_the_router_releases_the_reservation() -> None:
cache: Final = DualCache()
reservations: Final = InMemoryCache()
picker: Final = LeastBusyLoggingHandler(router_cache=cache, reservation_store=reservations)
registered: Final = LeastBusyLoggingHandler(router_cache=cache, reservation_store=reservations)
assert picker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
registered.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
assert picker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
assert picker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
class _StoreWhereThreadsReadBeforeWriting:
def __init__(self) -> None:
self._store: Final = InMemoryCache()
self._first_reads: Final = threading.Barrier(2, timeout=0.5)
self._thread_state: Final = threading.local()
self._meeting: Final = threading.Event()
@contextlib.contextmanager
def first_reads_meet(self) -> Iterator[None]:
self._meeting.set()
yield
self._meeting.clear()
def get_cache(self, key: str) -> object:
value: Final = self._store.get_cache(key)
if self._meeting.is_set() and not getattr(self._thread_state, "has_read", False):
self._thread_state.has_read = True
with contextlib.suppress(threading.BrokenBarrierError):
self._first_reads.wait()
return value
def set_cache(self, key: str, value: tuple[float, ...], *, ttl: float) -> None:
self._store.set_cache(key, value, ttl=ttl)
def delete_cache(self, key: str) -> None:
self._store.delete_cache(key)
def test_two_threads_picking_at_once_reserve_different_deployments() -> None:
store: Final = _StoreWhereThreadsReadBeforeWriting()
worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=store)
with store.first_reads_meet(), ThreadPoolExecutor(max_workers=2) as pool:
picks: Final = tuple(pool.map(lambda _: worker.get_available_deployments(GROUP, HEALTHY, reserve=True), range(2)))
assert sorted(pick["model_info"]["id"] for pick in picks if pick is not None) == ["dep-a", "dep-b"]
def test_a_reservation_and_a_release_on_one_deployment_at_once_both_count() -> None:
store: Final = _StoreWhereThreadsReadBeforeWriting()
worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=store)
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b"))
worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b"))
with store.first_reads_meet(), ThreadPoolExecutor(max_workers=2) as pool:
reserved: Final = pool.submit(worker.get_available_deployments, GROUP, HEALTHY, True)
started: Final = pool.submit(worker.log_pre_api_call, "m", [], _call_kwargs("dep-a"))
assert reserved.result() is DEPLOYMENT_A
started.result()
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
def _router_pick(router: Router) -> str:
deployment: Final = router.get_available_deployment(ROUTER_GROUP, request_kwargs={"metadata": {}})
return deployment["model_info"]["id"]
def test_a_pick_stays_reserved_while_other_router_keys_fill_its_cache() -> None:
router: Final = _least_busy_router()
first: Final = _router_pick(router)
for index in range(DEFAULT_MAX_SIZE_IN_MEMORY):
router.health().store.set_cache(f"deployment-usage-{index}", 1, local_only=True, ttl=60)
assert _router_pick(router) != first
@pytest.mark.asyncio
async def test_a_cache_hit_releases_the_pick_it_reserved() -> None:
worker: Final = _worker(None)
worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
await worker.async_log_success_event({**_call_kwargs("dep-b"), "cache_hit": True}, None, None, None)
assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
def test_a_cache_hit_leaves_requests_in_flight_on_its_deployment_counted() -> None:
worker: Final = _worker(None)
worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
worker.log_success_event({**_call_kwargs("dep-a"), "cache_hit": True}, None, None, None)
assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
@pytest.mark.parametrize("outcome", [LeastBusyLoggingHandler.log_success_event, LeastBusyLoggingHandler.log_failure_event])
def test_a_call_that_ends_before_reaching_the_provider_frees_its_pick(
outcome: Callable[[LeastBusyLoggingHandler, dict[str, object], None, None, None], None],
) -> None:
worker: Final = _worker(None)
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
outcome(worker, _routed_call_kwargs("dep-a"), None, None, None)
assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
def test_a_call_the_proxy_rejected_after_logging_its_start_leaves_no_count_behind() -> None:
worker: Final = _worker(None)
rejected: Final = {**_routed_call_kwargs("dep-a"), LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True}
worker.log_pre_api_call(model="m", messages=[], kwargs=rejected)
worker.log_failure_event(rejected, None, None, None)
assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
async def _drain_logging() -> None:
for _ in range(10):
await asyncio.sleep(0)
await GLOBAL_LOGGING_WORKER.flush()
def _batch_deployment(route: respx.Route) -> str:
return route.calls.last.request.headers["authorization"].removeprefix("Bearer key-")
_BATCH: Final = {
"id": "batch_least_busy",
"object": "batch",
"endpoint": "/v1/chat/completions",
"input_file_id": "file-least-busy",
"completion_window": "24h",
"status": "validating",
"created_at": 1,
}
_REJECTED_BATCH: Final = {"error": {"message": "Invalid input file", "type": "invalid_request_error"}}
@pytest.mark.asyncio
@pytest.mark.parametrize("provider_response", [httpx.Response(200, json=_BATCH), httpx.Response(400, json=_REJECTED_BATCH)])
async def test_a_batch_create_does_not_keep_its_deployment_busy(
provider_response: httpx.Response, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
route: Final = respx_mock.post("https://api.openai.com/v1/batches").mock(return_value=provider_response)
router: Final = _least_busy_router()
with contextlib.suppress(openai.BadRequestError):
await router.acreate_batch(
model=ROUTER_GROUP, completion_window="24h", endpoint="/v1/chat/completions", input_file_id="file-least-busy"
)
await _drain_logging()
assert _router_pick(router) == _batch_deployment(route)
@pytest.mark.asyncio
async def test_a_batch_cancel_does_not_keep_its_deployment_busy(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
route: Final = respx_mock.post("https://api.openai.com/v1/batches/batch_least_busy/cancel").mock(
return_value=httpx.Response(200, json={**_BATCH, "status": "cancelling"})
)
router: Final = _least_busy_router()
await router.acancel_batch(model=ROUTER_GROUP, batch_id="batch_least_busy")
assert _router_pick(router) == _batch_deployment(route)
@pytest.mark.asyncio
async def test_picking_a_deployment_without_calling_it_does_not_keep_it_busy() -> None:
router: Final = _least_busy_router()
picked: Final = await router.select(ROUTER_GROUP, {"metadata": {}})
assert _router_pick(router) == picked["model_info"]["id"]

View file

@ -56,6 +56,9 @@ def _model_list():
]
CALL_STARTED_AT: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
def _build_router(routing_strategy="simple-shuffle", routing_groups=None):
return Router(
model_list=_model_list(),
@ -468,7 +471,8 @@ def test_two_least_busy_groups_count_a_request_once(monkeypatch):
"litellm_params": {
"metadata": {"model_group": "filtered-model"},
"model_info": {"id": "deploy-1"},
}
},
"api_call_start_time": CALL_STARTED_AT,
}
for callback in litellm.input_callback:
@ -481,6 +485,36 @@ def test_two_least_busy_groups_count_a_request_once(monkeypatch):
assert router.cache.get_cache("filtered-model_request_count:deploy-1") == 0
def _filtered_model_pick(router: Router) -> str:
deployment: Final = router.get_available_deployment(
"filtered-model", request_kwargs=_proxy_shaped_request(model="filtered-model")
)
return deployment["model_info"]["id"]
def test_a_least_busy_group_pick_stops_counting_once_its_request_finishes(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "callbacks", [])
monkeypatch.setattr(litellm, "input_callback", [])
router: Final = _build_router(
routing_strategy="least-busy",
routing_groups=[{"group_name": "fast", "models": ["filtered-model"], "routing_strategy": "least-busy"}],
)
first: Final = _filtered_model_pick(router)
kwargs: Final = {
"litellm_params": {"metadata": {"model_group": "filtered-model"}, "model_info": {"id": first}},
"api_call_start_time": CALL_STARTED_AT,
}
for callback in litellm.input_callback:
if isinstance(callback, CustomLogger):
callback.log_pre_api_call(model="filtered-model", messages=[], kwargs=kwargs)
for callback in litellm.callbacks:
if isinstance(callback, CustomLogger):
callback.log_success_event(kwargs, None, None, None)
assert _filtered_model_pick(router) == first
def test_two_routers_in_one_process_each_count_their_own_requests(monkeypatch):
"""
Least-busy hangs its counting off litellm's global callback lists, and those lists keep one
@ -498,7 +532,8 @@ def test_two_routers_in_one_process_each_count_their_own_requests(monkeypatch):
"litellm_params": {
"metadata": {"model_group": "filtered-model"},
"model_info": {"id": "deploy-1"},
}
},
"api_call_start_time": CALL_STARTED_AT,
}
for callback in litellm.input_callback:
@ -613,6 +648,7 @@ async def test_select_deployment_async_returns_none_without_selector():
messages=None,
input=None,
request_kwargs=None,
reserve=False,
)
assert result is None
@ -627,6 +663,7 @@ def test_select_deployment_sync_returns_none_without_selector():
messages=None,
input=None,
request_kwargs=None,
reserve=False,
)
assert result is None
@ -661,6 +698,7 @@ def test_select_deployment_sync_does_not_dispatch_cost_based_routing():
messages=None,
input=None,
request_kwargs=None,
reserve=False,
)
assert result is None
@ -707,6 +745,7 @@ async def test_async_dispatch_falls_back_to_sync_for_usage_based_routing_v1():
messages=[{"role": "user", "content": "hi"}],
input=None,
request_kwargs={},
reserve=False,
)
assert v1_spy.called, "async dispatch must route v1 strategy through sync method"
@ -1202,12 +1241,12 @@ async def test_least_busy_override_sees_the_overriding_request_in_flight():
stream = await router.acompletion(**_proxy_shaped_request(model="grp", routing_strategy="least-busy", stream=True))
busy = stream._hidden_params["model_id"]
idle = "d2" if busy == "d1" else "d1"
assert [await _async_override_pick(router, "least-busy") for _ in range(3)] == [idle, idle, idle]
assert await _async_override_pick(router, "least-busy") == idle
async for _ in stream:
pass
await _async_wait_until(lambda: _in_flight(router, busy) == 0)
assert await _async_override_pick(router, "least-busy") == "d1"
assert await _async_override_pick(router, "least-busy") == busy
assert _selector_is_not_global(router._override_selectors["least-busy"])
@ -1217,12 +1256,12 @@ def test_sync_least_busy_override_sees_the_overriding_request_in_flight():
stream = router.completion(**_proxy_shaped_request(model="grp", routing_strategy="least-busy", stream=True))
busy = stream._hidden_params["model_id"]
idle = "d2" if busy == "d1" else "d1"
assert [_sync_override_pick(router, "least-busy") for _ in range(3)] == [idle, idle, idle]
assert _sync_override_pick(router, "least-busy") == idle
for _ in stream:
pass
_sync_wait_until(lambda: _in_flight(router, busy) == 0)
assert _sync_override_pick(router, "least-busy") == "d1"
assert _sync_override_pick(router, "least-busy") == busy
assert _selector_is_not_global(router._override_selectors["least-busy"])

View file

@ -597,6 +597,34 @@ async def test_responses_websocket_turn_is_refused_once_its_deployment_crosses_b
await admit_turn()
@pytest.mark.asyncio
async def test_responses_websocket_turns_do_not_make_their_deployment_look_busy_to_least_busy():
router: Final = Router(
model_list=[
{
"model_name": "ws-least-busy",
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake"},
"model_info": {"id": deployment_id},
}
for deployment_id in ("ws-least-busy-a", "ws-least-busy-b")
],
routing_strategy="least-busy",
)
admit_turn: Final = (await _kwargs_routed_to(router, "ws-least-busy", "_aresponses_websocket"))["turn_admission"]
assert callable(admit_turn)
for _ in range(3):
await admit_turn()
picks: Final = [
(await router.async_get_available_deployment(model="ws-least-busy", request_kwargs={"input": "hi"}))[
"model_info"
]["id"]
for _ in range(2)
]
assert sorted(picks) == ["ws-least-busy-a", "ws-least-busy-b"]
@pytest.mark.asyncio
async def test_responses_websocket_turn_after_the_first_counts_toward_deployment_rpm():
router = Router(