mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 15f8086684 into 19e538ab11
This commit is contained in:
commit
58b4e7af7c
7 changed files with 702 additions and 38 deletions
|
|
@ -219,7 +219,7 @@ from litellm.router_strategy.complexity_router.context_compaction import (
|
|||
reject_recursive_compactor,
|
||||
surface_for_call,
|
||||
)
|
||||
from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
|
||||
from litellm.router_strategy.least_busy import SELECTION_RESERVATION_MAX_KEYS, LeastBusyLoggingHandler
|
||||
from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
|
||||
from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler
|
||||
from litellm.router_strategy.lowest_tpm_rpm import LowestTPMLoggingHandler
|
||||
|
|
@ -1387,6 +1387,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
reserve=False,
|
||||
)
|
||||
if kind == "healthy":
|
||||
return await self._core_async_get_healthy_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts
|
||||
|
|
@ -1656,6 +1657,9 @@ class Router(DeprecatedRouterMembers):
|
|||
self._core_cache = DualCache(
|
||||
redis_cache=redis_cache, in_memory_cache=InMemoryCache()
|
||||
) # use a dual cache (Redis+In-Memory) for tracking cooldowns, usage, etc.
|
||||
self._core_least_busy_reservations: InMemoryCache = InMemoryCache(
|
||||
max_size_in_memory=SELECTION_RESERVATION_MAX_KEYS
|
||||
)
|
||||
self._claude_code_session_router_cache: DualCache = DualCache(
|
||||
redis_cache=redis_cache,
|
||||
in_memory_cache=InMemoryCache(),
|
||||
|
|
@ -2163,7 +2167,9 @@ class Router(DeprecatedRouterMembers):
|
|||
selector: RouterStrategySelector | None = None
|
||||
match self._normalize_strategy(strategy):
|
||||
case RoutingStrategy.LEAST_BUSY.value:
|
||||
selector = LeastBusyLoggingHandler(router_cache=self._core_cache)
|
||||
selector = LeastBusyLoggingHandler(
|
||||
router_cache=self._core_cache, reservation_store=self._core_least_busy_reservations
|
||||
)
|
||||
case RoutingStrategy.USAGE_BASED_ROUTING.value:
|
||||
selector = LowestTPMLoggingHandler(
|
||||
router_cache=self._core_cache,
|
||||
|
|
@ -2629,6 +2635,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages: list[dict[str, str]] | None,
|
||||
input: str | list | None,
|
||||
request_kwargs: dict | None,
|
||||
reserve: bool,
|
||||
) -> Any | None:
|
||||
"""
|
||||
Asks the strategy selector for a deployment. Caller handles
|
||||
|
|
@ -2642,6 +2649,7 @@ class Router(DeprecatedRouterMembers):
|
|||
return await selector.async_get_available_deployments(
|
||||
model_group=model,
|
||||
healthy_deployments=healthy_deployments,
|
||||
reserve=reserve,
|
||||
)
|
||||
case "usage-based-routing":
|
||||
# `LowestTPMLoggingHandler` (v1) only exposes the sync
|
||||
|
|
@ -2682,6 +2690,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages: list[dict[str, str]] | None,
|
||||
input: str | list | None,
|
||||
request_kwargs: dict | None,
|
||||
reserve: bool,
|
||||
) -> Any | None:
|
||||
"""
|
||||
Sync sibling of `_select_deployment_async`. Caller handles
|
||||
|
|
@ -2698,6 +2707,7 @@ class Router(DeprecatedRouterMembers):
|
|||
return selector.get_available_deployments(
|
||||
model_group=model,
|
||||
healthy_deployments=healthy_deployments,
|
||||
reserve=reserve,
|
||||
)
|
||||
case "usage-based-routing" | "usage-based-routing-v2":
|
||||
return selector.get_available_deployments(
|
||||
|
|
@ -8054,6 +8064,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages=[{"role": "user", "content": "batch-api-fake-text"}],
|
||||
specific_deployment=kwargs.pop("specific_deployment", None),
|
||||
request_kwargs=kwargs,
|
||||
reserve=False,
|
||||
)
|
||||
|
||||
selected_deployment_id: Final = (deployment.get("model_info") or {}).get("id")
|
||||
|
|
@ -10489,6 +10500,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages=None,
|
||||
input=None,
|
||||
request_kwargs=routing_kwargs,
|
||||
reserve=False,
|
||||
)
|
||||
if selected is None:
|
||||
no_deployment: Final = await async_raise_no_deployment_exception(
|
||||
|
|
@ -15694,6 +15706,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages: list[dict[str, str]] | None = None,
|
||||
input: str | list | None = None,
|
||||
specific_deployment: bool | None = False,
|
||||
reserve: bool = True,
|
||||
):
|
||||
"""
|
||||
Async implementation of 'get_available_deployments'.
|
||||
|
|
@ -15707,6 +15720,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
reserve=reserve,
|
||||
)
|
||||
|
||||
async def _async_get_available_deployment(
|
||||
|
|
@ -15716,6 +15730,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages: list[dict[str, str]] | None,
|
||||
input: str | list | None,
|
||||
specific_deployment: bool | None,
|
||||
reserve: bool,
|
||||
):
|
||||
if (
|
||||
self._core_routing_strategy != "usage-based-routing-v2"
|
||||
|
|
@ -15812,6 +15827,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
reserve=reserve and request_kwargs not in (None, {}),
|
||||
)
|
||||
if deployment is None:
|
||||
exception: Final = await async_raise_no_deployment_exception(
|
||||
|
|
@ -15981,6 +15997,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
reserve=request_kwargs not in (None, {}),
|
||||
)
|
||||
|
||||
if deployment is None:
|
||||
|
|
@ -16856,6 +16873,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
reserve=request_kwargs not in (None, {}),
|
||||
)
|
||||
|
||||
if deployment is None:
|
||||
|
|
@ -17038,6 +17056,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
reserve=request_kwargs not in (None, {}),
|
||||
)
|
||||
|
||||
if deployment is None:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import logging
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import Final, Protocol
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
|
@ -8,11 +10,16 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.caching.redis_cache import log_redis_failure
|
||||
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
|
||||
IN_FLIGHT_COUNT_TTL_SECONDS: Final = 60 * 60
|
||||
SELECTION_RESERVATION_TTL_SECONDS: Final = 30.0
|
||||
SELECTION_RESERVATION_MAX_KEYS: Final = 10_000
|
||||
_RESERVATIONS_LOCK: Final = threading.Lock()
|
||||
|
||||
|
||||
class _ModelInfo(TypedDict, total=False):
|
||||
|
|
@ -21,10 +28,12 @@ class _ModelInfo(TypedDict, total=False):
|
|||
|
||||
class _Metadata(TypedDict, total=False):
|
||||
model_group: ReadOnly[str | None]
|
||||
model_info: ReadOnly[_ModelInfo | None]
|
||||
|
||||
|
||||
class _LitellmParams(TypedDict, total=False):
|
||||
metadata: ReadOnly[_Metadata | None]
|
||||
litellm_metadata: ReadOnly[_Metadata | None]
|
||||
model_info: ReadOnly[_ModelInfo | None]
|
||||
|
||||
|
||||
|
|
@ -42,13 +51,26 @@ class _Deployment(TypedDict):
|
|||
|
||||
_CALL_KWARGS: Final = TypeAdapter(_CallKwargs)
|
||||
_DEPLOYMENTS: Final = TypeAdapter(list[_Deployment])
|
||||
_MEMORY_COUNTS: Final = TypeAdapter(tuple[float | None, ...] | None)
|
||||
_MEMORY_COUNTS: Final[TypeAdapter[tuple[float | None, ...] | None]] = TypeAdapter(tuple[float | None, ...] | None)
|
||||
_RESERVATION_TIMES: Final[TypeAdapter[tuple[float, ...] | None]] = TypeAdapter(tuple[float, ...] | None)
|
||||
|
||||
|
||||
def _request_count_key(model_group: str, deployment_id: str) -> str:
|
||||
return f"{model_group}_request_count:{deployment_id}"
|
||||
|
||||
|
||||
def _model_group(metadata: _Metadata | None) -> str | None:
|
||||
return metadata.get("model_group") if metadata else None
|
||||
|
||||
|
||||
def _deployment_id(model_info: _ModelInfo | None) -> str | int | None:
|
||||
return model_info.get("id") if model_info else None
|
||||
|
||||
|
||||
def _routed_deployment_id(metadata: _Metadata | None) -> str | int | None:
|
||||
return _deployment_id(metadata.get("model_info")) if metadata else None
|
||||
|
||||
|
||||
def _deployment_ref(kwargs: Mapping[str, object]) -> tuple[str, str] | None:
|
||||
if is_batch_retrieve_call_type(kwargs.get("call_type")):
|
||||
return None
|
||||
|
|
@ -56,16 +78,35 @@ def _deployment_ref(kwargs: Mapping[str, object]) -> tuple[str, str] | None:
|
|||
call: Final = _CALL_KWARGS.validate_python(kwargs)
|
||||
except ValidationError:
|
||||
return None
|
||||
litellm_params: Final = call.get("litellm_params")
|
||||
metadata: Final = litellm_params.get("metadata") if litellm_params else None
|
||||
model_info: Final = litellm_params.get("model_info") if litellm_params else None
|
||||
model_group: Final = metadata.get("model_group") if metadata else None
|
||||
deployment_id: Final = model_info.get("id") if model_info else None
|
||||
litellm_params: Final = call.get("litellm_params") or _LitellmParams()
|
||||
routing_metadata: Final = (litellm_params.get("litellm_metadata"), litellm_params.get("metadata"))
|
||||
model_group: Final = next((group for group in map(_model_group, routing_metadata) if group is not None), None)
|
||||
deployment_id: Final = next(
|
||||
(
|
||||
deployment_id
|
||||
for deployment_id in (
|
||||
_deployment_id(litellm_params.get("model_info")),
|
||||
*map(_routed_deployment_id, routing_metadata),
|
||||
)
|
||||
if deployment_id is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
if model_group is None or deployment_id is None:
|
||||
return None
|
||||
return model_group, str(deployment_id)
|
||||
|
||||
|
||||
def _counted_in_flight(kwargs: Mapping[str, object]) -> bool:
|
||||
return kwargs.get("cache_hit") is not True and (
|
||||
kwargs.get("api_call_start_time") is not None or LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL in kwargs
|
||||
)
|
||||
|
||||
|
||||
def _reservation_key(request_count_key: str) -> str:
|
||||
return f"{request_count_key}:reservations"
|
||||
|
||||
|
||||
def _request_count_keys(model_group: str, healthy_deployments: Sequence[Mapping[str, object]]) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
_request_count_key(model_group, str(deployment["model_info"]["id"]))
|
||||
|
|
@ -84,12 +125,12 @@ def _local_counts(raw: object, keys: tuple[str, ...]) -> tuple[int, ...]:
|
|||
return _as_counts(values)
|
||||
|
||||
|
||||
def _least_busy(
|
||||
healthy_deployments: Sequence[Mapping[str, object]], counts: tuple[int, ...]
|
||||
) -> Mapping[str, object] | None:
|
||||
if not healthy_deployments:
|
||||
return None
|
||||
return healthy_deployments[min(range(len(healthy_deployments)), key=lambda index: counts[index])]
|
||||
def _unexpired(times: Sequence[float], now: float) -> tuple[float, ...]:
|
||||
return tuple(t for t in times if now - t < SELECTION_RESERVATION_TTL_SECONDS)
|
||||
|
||||
|
||||
def _least_busy_index(counts: tuple[int, ...]) -> int:
|
||||
return min(range(len(counts)), key=lambda index: counts[index])
|
||||
|
||||
|
||||
def _warn_unreadable(model_group: str, error: Exception) -> None:
|
||||
|
|
@ -111,24 +152,80 @@ def _warn_unwritable(key: str, error: Exception) -> None:
|
|||
)
|
||||
|
||||
|
||||
class _ReservationStore(Protocol):
|
||||
def get_cache(self, key: str) -> object: ...
|
||||
|
||||
def set_cache(self, key: str, value: tuple[float, ...], *, ttl: float) -> None: ...
|
||||
|
||||
def delete_cache(self, key: str) -> None: ...
|
||||
|
||||
|
||||
class LeastBusyLoggingHandler(CustomLogger):
|
||||
test_flag: bool = False
|
||||
logged_success: int = 0
|
||||
logged_failure: int = 0
|
||||
|
||||
def __init__(self, router_cache: DualCache):
|
||||
def __init__(
|
||||
self,
|
||||
router_cache: DualCache,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
reservation_store: _ReservationStore | None = None,
|
||||
):
|
||||
self.router_cache = router_cache
|
||||
self.router_cache_id = str(id(router_cache))
|
||||
self._clock = clock
|
||||
self._reservations: Final[_ReservationStore] = (
|
||||
reservation_store
|
||||
if reservation_store is not None
|
||||
else InMemoryCache(max_size_in_memory=SELECTION_RESERVATION_MAX_KEYS, clock=clock)
|
||||
)
|
||||
|
||||
def _live_reservations(self, key: str, now: float) -> tuple[float, ...]:
|
||||
raw: Final = self._reservations.get_cache(_reservation_key(key))
|
||||
return _unexpired(_RESERVATION_TIMES.validate_python(raw) or (), now)
|
||||
|
||||
def _store_reservations(self, key: str, times: tuple[float, ...]) -> None:
|
||||
# InMemoryCache keeps a live key's first TTL, so drop the key to restart it
|
||||
self._reservations.delete_cache(_reservation_key(key))
|
||||
if times:
|
||||
self._reservations.set_cache(_reservation_key(key), times, ttl=SELECTION_RESERVATION_TTL_SECONDS)
|
||||
|
||||
def _select(
|
||||
self,
|
||||
healthy_deployments: Sequence[Mapping[str, object]],
|
||||
keys: tuple[str, ...],
|
||||
counts: tuple[int, ...],
|
||||
reserve: bool,
|
||||
) -> Mapping[str, object] | None:
|
||||
if not healthy_deployments:
|
||||
return None
|
||||
with _RESERVATIONS_LOCK:
|
||||
now: Final = self._clock()
|
||||
live: Final = tuple(self._live_reservations(key, now) for key in keys)
|
||||
index: Final = _least_busy_index(tuple(count + len(r) for count, r in zip(counts, live, strict=True)))
|
||||
if reserve:
|
||||
self._store_reservations(keys[index], (*live[index], now))
|
||||
return healthy_deployments[index]
|
||||
|
||||
def _release_reservation(self, kwargs: Mapping[str, object]) -> None:
|
||||
ref: Final = _deployment_ref(kwargs)
|
||||
if ref is None:
|
||||
return
|
||||
key: Final = _request_count_key(*ref)
|
||||
with _RESERVATIONS_LOCK:
|
||||
now: Final = self._clock()
|
||||
self._store_reservations(key, self._live_reservations(key, now)[1:])
|
||||
|
||||
@with_service_target("router_usage")
|
||||
def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None:
|
||||
self._release_reservation(kwargs)
|
||||
self._increment(kwargs, 1)
|
||||
|
||||
@with_service_target("router_usage")
|
||||
def log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self._increment(kwargs, -1)
|
||||
self._settle(kwargs)
|
||||
if self.test_flag:
|
||||
self.logged_success += 1
|
||||
|
||||
|
|
@ -136,7 +233,7 @@ class LeastBusyLoggingHandler(CustomLogger):
|
|||
def log_failure_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self._increment(kwargs, -1)
|
||||
self._settle(kwargs)
|
||||
if self.test_flag:
|
||||
self.logged_failure += 1
|
||||
|
||||
|
|
@ -144,7 +241,7 @@ class LeastBusyLoggingHandler(CustomLogger):
|
|||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
await self._async_increment(kwargs, -1)
|
||||
await self._async_settle(kwargs)
|
||||
if self.test_flag:
|
||||
self.logged_success += 1
|
||||
|
||||
|
|
@ -152,13 +249,13 @@ class LeastBusyLoggingHandler(CustomLogger):
|
|||
async def async_log_failure_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
await self._async_increment(kwargs, -1)
|
||||
await self._async_settle(kwargs)
|
||||
if self.test_flag:
|
||||
self.logged_failure += 1
|
||||
|
||||
@with_service_target("router_usage")
|
||||
def get_available_deployments(
|
||||
self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]]
|
||||
self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]], reserve: bool = False
|
||||
) -> Mapping[str, object] | None:
|
||||
keys: Final = _request_count_keys(model_group, healthy_deployments)
|
||||
redis_cache: Final = self.router_cache.redis_cache
|
||||
|
|
@ -168,13 +265,12 @@ class LeastBusyLoggingHandler(CustomLogger):
|
|||
except Exception as e:
|
||||
_warn_unreadable(model_group, e)
|
||||
else:
|
||||
return _least_busy(healthy_deployments, shared)
|
||||
local: Final = _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys)
|
||||
return _least_busy(healthy_deployments, local)
|
||||
return self._select(healthy_deployments, keys, self._at_least_this_workers(keys, shared), reserve)
|
||||
return self._select(healthy_deployments, keys, self._this_workers_counts(keys), reserve)
|
||||
|
||||
@with_service_target("router_usage")
|
||||
async def async_get_available_deployments(
|
||||
self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]]
|
||||
self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]], reserve: bool = False
|
||||
) -> Mapping[str, object] | None:
|
||||
keys: Final = _request_count_keys(model_group, healthy_deployments)
|
||||
redis_cache: Final = self.router_cache.redis_cache
|
||||
|
|
@ -184,9 +280,26 @@ class LeastBusyLoggingHandler(CustomLogger):
|
|||
except Exception as e:
|
||||
_warn_unreadable(model_group, e)
|
||||
else:
|
||||
return _least_busy(healthy_deployments, shared)
|
||||
local: Final = _local_counts(await self.router_cache.async_batch_get_cache(list(keys), local_only=True), keys)
|
||||
return _least_busy(healthy_deployments, local)
|
||||
return self._select(healthy_deployments, keys, self._at_least_this_workers(keys, shared), reserve)
|
||||
return self._select(healthy_deployments, keys, self._this_workers_counts(keys), reserve)
|
||||
|
||||
def _settle(self, kwargs: Mapping[str, object]) -> None:
|
||||
if not _counted_in_flight(kwargs):
|
||||
self._release_reservation(kwargs)
|
||||
return
|
||||
self._increment(kwargs, -1)
|
||||
|
||||
async def _async_settle(self, kwargs: Mapping[str, object]) -> None:
|
||||
if not _counted_in_flight(kwargs):
|
||||
self._release_reservation(kwargs)
|
||||
return
|
||||
await self._async_increment(kwargs, -1)
|
||||
|
||||
def _this_workers_counts(self, keys: tuple[str, ...]) -> tuple[int, ...]:
|
||||
return _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys)
|
||||
|
||||
def _at_least_this_workers(self, keys: tuple[str, ...], shared: tuple[int, ...]) -> tuple[int, ...]:
|
||||
return tuple(max(counts) for counts in zip(shared, self._this_workers_counts(keys), strict=True))
|
||||
|
||||
def _increment(self, kwargs: Mapping[str, object], delta: int) -> None:
|
||||
ref: Final = _deployment_ref(kwargs)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,90 @@
|
|||
import json
|
||||
import threading
|
||||
from collections import Counter
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, gateway_from_environment
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
pytestmark = pytest.mark.timeout(240)
|
||||
|
||||
_GROUP: Final = "least-busy-decider"
|
||||
_DEPLOYMENTS: Final = ("least-busy-decider-a", "least-busy-decider-b")
|
||||
_BURST: Final = 4
|
||||
_REPLY: Final = json.dumps(
|
||||
{
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"answers": {"billing": {"type": "noul", "noul": 0.91}},
|
||||
"usage": {"input_tokens": 367, "output_tokens": 3},
|
||||
}
|
||||
).encode()
|
||||
_DECISION: Final = {
|
||||
"model": _GROUP,
|
||||
"input": "I was charged twice for order 4417.",
|
||||
"questions": [{"type": "predicate", "name": "billing", "instructions": "Is this a billing issue?"}],
|
||||
}
|
||||
|
||||
|
||||
def _held_until_the_burst_arrives(arrived: threading.Barrier) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
arrived.wait()
|
||||
return Reply(body=_REPLY)
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def _write_config(path: Path, provider_url: str) -> Path:
|
||||
path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": _GROUP,
|
||||
"litellm_params": {"model": "hosted_vllm/Qwen/Qwen3-0.6B", "api_base": provider_url},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
for deployment_id in _DEPLOYMENTS
|
||||
],
|
||||
"general_settings": {
|
||||
"master_key": "os.environ/LITELLM_MASTER_KEY",
|
||||
"database_url": "os.environ/DATABASE_URL",
|
||||
"disable_model_info_refresh": True,
|
||||
},
|
||||
"router_settings": {"routing_strategy": "least-busy", "num_retries": 0, "disable_cooldowns": True},
|
||||
}
|
||||
)
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def _decide(proxy: Gateway) -> httpx.Response:
|
||||
return proxy.client.post(
|
||||
"/v1/decisions", json=_DECISION, headers={"Authorization": f"Bearer {proxy.key}"}, timeout=120
|
||||
)
|
||||
|
||||
|
||||
def test_least_busy_spreads_a_burst_of_decisions_calls_across_the_group(tmp_path: Path) -> None:
|
||||
arrived: Final = threading.Barrier(_BURST, timeout=60)
|
||||
with (
|
||||
wire_server(_held_until_the_burst_arrives(arrived)) as upstream,
|
||||
gateway_from_environment() as rig,
|
||||
owned_proxy(
|
||||
rig,
|
||||
tmp_path,
|
||||
{"STORE_MODEL_IN_DB": "False"},
|
||||
config=_write_config(tmp_path / "least-busy.yaml", upstream.url),
|
||||
) as proxy,
|
||||
ThreadPoolExecutor(max_workers=_BURST) as pool,
|
||||
):
|
||||
responses: Final = tuple(pool.map(lambda _: _decide(proxy), range(_BURST)))
|
||||
assert [response.status_code for response in responses] == [200] * _BURST, [r.text for r in responses]
|
||||
assert Counter(response.headers["x-litellm-model-id"] for response in responses) == Counter(
|
||||
{deployment_id: _BURST // len(_DEPLOYMENTS) for deployment_id in _DEPLOYMENTS}
|
||||
)
|
||||
|
|
@ -6,6 +6,8 @@ import copy
|
|||
import json
|
||||
import re
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from collections.abc import Mapping, Sequence
|
||||
from concurrent.futures import wait
|
||||
from types import MappingProxyType
|
||||
|
|
@ -297,6 +299,43 @@ async def test_router_dispatches_typesafe_decisions_without_api_base(
|
|||
assert response.answers["sentiment"].choice == "positive"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_least_busy_frees_a_decisions_deployment_once_a_proxy_logged_call_finishes(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "jev",
|
||||
"litellm_params": {"model": "typesafe/jev-latest", "api_key": "k"},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
for deployment_id in ("jev-a", "jev-b")
|
||||
],
|
||||
routing_strategy="least-busy",
|
||||
)
|
||||
|
||||
async def call_through_proxy_logging() -> object:
|
||||
logging_obj, _ = litellm.utils.function_setup(
|
||||
original_function="adecisions",
|
||||
rules_obj=litellm.utils.Rules(),
|
||||
start_time=datetime.now(),
|
||||
model="jev",
|
||||
litellm_call_id=str(uuid.uuid4()),
|
||||
)
|
||||
response: Final = await router.adecisions(
|
||||
model="jev",
|
||||
input="Is this a defect?",
|
||||
questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
await _drain_logging_worker()
|
||||
return response._hidden_params["model_id"]
|
||||
|
||||
assert (await call_through_proxy_logging(), await call_through_proxy_logging()) == ("jev-a", "jev-a")
|
||||
|
||||
|
||||
def test_decisions_uses_the_same_wire_contract_for_sync_calls(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,15 @@
|
|||
import asyncio
|
||||
import contextlib
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import Callable, Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
|
|
@ -11,22 +17,32 @@ import litellm
|
|||
from litellm import Router
|
||||
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.caching.in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY, InMemoryCache
|
||||
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
|
||||
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.router_strategy.least_busy import IN_FLIGHT_COUNT_TTL_SECONDS, LeastBusyLoggingHandler
|
||||
from litellm.router_strategy.least_busy import (
|
||||
IN_FLIGHT_COUNT_TTL_SECONDS,
|
||||
SELECTION_RESERVATION_TTL_SECONDS,
|
||||
LeastBusyLoggingHandler,
|
||||
)
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
GROUP: Final = "least-busy-group"
|
||||
CALL_STARTED_AT: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
|
||||
DEPLOYMENT_A: Final[dict[str, object]] = {"model_info": {"id": "dep-a"}}
|
||||
DEPLOYMENT_B: Final[dict[str, object]] = {"model_info": {"id": "dep-b"}}
|
||||
HEALTHY: Final = [DEPLOYMENT_A, DEPLOYMENT_B]
|
||||
|
||||
|
||||
def _call_kwargs(deployment_id: str) -> dict[str, object]:
|
||||
def _routed_call_kwargs(deployment_id: str) -> dict[str, object]:
|
||||
return {"litellm_params": {"metadata": {"model_group": GROUP}, "model_info": {"id": deployment_id}}}
|
||||
|
||||
|
||||
def _call_kwargs(deployment_id: str) -> dict[str, object]:
|
||||
return {**_routed_call_kwargs(deployment_id), "api_call_start_time": CALL_STARTED_AT}
|
||||
|
||||
|
||||
class SharedRedisCounters:
|
||||
"""Mirrors what Redis gives the handler: increments clamped at zero, a TTL set once when
|
||||
the key is created, and ordered reads that raise rather than invent a value."""
|
||||
|
|
@ -58,6 +74,20 @@ class SharedRedisCounters:
|
|||
return self.batch_get_counts(key_list)
|
||||
|
||||
|
||||
class SlowRedisRead(SharedRedisCounters):
|
||||
"""Answers a count read with the counts from when the read started, after letting other work run."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.while_reading: Callable[[], None] = lambda: None
|
||||
|
||||
async def async_batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]:
|
||||
counts_when_asked: Final = self.batch_get_counts(key_list)
|
||||
await asyncio.sleep(0)
|
||||
self.while_reading()
|
||||
return counts_when_asked
|
||||
|
||||
|
||||
def _worker(shared: SharedRedisCounters | None) -> LeastBusyLoggingHandler:
|
||||
cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=shared) # pyright: ignore[reportArgumentType] # duck-typed Redis double
|
||||
return LeastBusyLoggingHandler(router_cache=cache)
|
||||
|
|
@ -81,6 +111,16 @@ async def test_worker_routes_around_a_request_another_worker_started() -> None:
|
|||
assert await picking_worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_request_that_starts_while_redis_is_read_still_counts_on_its_deployment() -> None:
|
||||
redis: Final = SlowRedisRead()
|
||||
worker: Final = _worker(redis)
|
||||
assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
redis.while_reading = lambda: worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
|
||||
|
||||
assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
|
||||
|
||||
|
||||
def test_sync_pick_reads_the_shared_counts() -> None:
|
||||
shared: Final = SharedRedisCounters()
|
||||
streaming_worker: Final = _worker(shared)
|
||||
|
|
@ -98,6 +138,32 @@ def test_sync_pick_reads_the_shared_counts() -> None:
|
|||
assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
|
||||
|
||||
|
||||
def _routed_through_litellm_metadata(deployment_id: str) -> dict[str, object]:
|
||||
return {
|
||||
"litellm_params": {
|
||||
"metadata": {"tags": ["caller-tag"]},
|
||||
"litellm_metadata": {"model_group": GROUP},
|
||||
"model_info": {"id": deployment_id},
|
||||
},
|
||||
"api_call_start_time": CALL_STARTED_AT,
|
||||
}
|
||||
|
||||
|
||||
def test_a_request_routed_through_litellm_metadata_counts_while_in_flight() -> None:
|
||||
shared: Final = SharedRedisCounters()
|
||||
calling_worker: Final = _worker(shared)
|
||||
picking_worker: Final = _worker(shared)
|
||||
|
||||
assert calling_worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
calling_worker.log_pre_api_call(model="m", messages=[], kwargs=_routed_through_litellm_metadata("dep-a"))
|
||||
|
||||
assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
|
||||
|
||||
calling_worker.log_success_event(_routed_through_litellm_metadata("dep-a"), None, None, None)
|
||||
|
||||
assert calling_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
|
||||
|
||||
|
||||
def test_the_handler_never_pushes_a_counters_ttl_forward() -> None:
|
||||
"""A worker that dies mid-request leaves a +1 nobody will ever decrement. Redis expires that
|
||||
stuck count an hour after the key was created, which only works while nothing writes the TTL
|
||||
|
|
@ -391,3 +457,273 @@ async def test_router_picks_least_busy_deployment_and_completion_restores_counts
|
|||
assert response._hidden_params["model_id"] == "1"
|
||||
assert route.calls.last.request.headers["authorization"] == "Bearer key-1"
|
||||
assert _router_counts(router) == seeded
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_spreads_requests_that_start_together(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
burst: Final = 2 * len(ROUTER_DEPLOYMENT_IDS)
|
||||
all_started: Final = asyncio.Barrier(burst)
|
||||
|
||||
async def completion_after_all_started(_: httpx.Request) -> httpx.Response:
|
||||
await all_started.wait()
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-least-busy",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4.1-mini",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "done"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
|
||||
side_effect=completion_after_all_started
|
||||
)
|
||||
router: Final = _least_busy_router()
|
||||
|
||||
await asyncio.gather(
|
||||
*(router.acompletion(model=ROUTER_GROUP, messages=[{"role": "user", "content": "hi"}]) for _ in range(burst))
|
||||
)
|
||||
|
||||
one_per_deployment: Final = tuple(f"Bearer key-{deployment_id}" for deployment_id in ROUTER_DEPLOYMENT_IDS)
|
||||
assert sorted(call.request.headers["authorization"] for call in route.calls) == sorted(one_per_deployment * 2)
|
||||
|
||||
|
||||
class _ManualClock:
|
||||
def __init__(self) -> None:
|
||||
self.now = 0.0
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.now
|
||||
|
||||
|
||||
def test_a_reservation_that_never_reaches_the_api_call_expires() -> None:
|
||||
clock: Final = _ManualClock()
|
||||
worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), clock=clock)
|
||||
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
clock.now = 1.0
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
|
||||
clock.now = 2.0
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
clock.now = SELECTION_RESERVATION_TTL_SECONDS + 0.5
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
|
||||
|
||||
def test_a_fresh_reservation_outlives_the_ttl_of_the_first_one_on_its_deployment() -> None:
|
||||
clock: Final = _ManualClock()
|
||||
cache: Final = DualCache(in_memory_cache=InMemoryCache(clock=clock))
|
||||
worker: Final = LeastBusyLoggingHandler(router_cache=cache, clock=clock)
|
||||
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
|
||||
worker.log_success_event(kwargs=_call_kwargs("dep-a"), response_obj=None, start_time=None, end_time=None)
|
||||
clock.now = SELECTION_RESERVATION_TTL_SECONDS - 0.1
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
clock.now = SELECTION_RESERVATION_TTL_SECONDS + 0.1
|
||||
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
|
||||
|
||||
|
||||
def test_a_started_request_moves_its_reservation_into_the_in_flight_count() -> None:
|
||||
worker: Final = _worker(None)
|
||||
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
|
||||
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
|
||||
|
||||
def test_a_request_started_through_another_selector_of_the_router_releases_the_reservation() -> None:
|
||||
cache: Final = DualCache()
|
||||
reservations: Final = InMemoryCache()
|
||||
picker: Final = LeastBusyLoggingHandler(router_cache=cache, reservation_store=reservations)
|
||||
registered: Final = LeastBusyLoggingHandler(router_cache=cache, reservation_store=reservations)
|
||||
|
||||
assert picker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
registered.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
|
||||
|
||||
assert picker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
|
||||
assert picker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
|
||||
|
||||
class _StoreWhereThreadsReadBeforeWriting:
|
||||
def __init__(self) -> None:
|
||||
self._store: Final = InMemoryCache()
|
||||
self._first_reads: Final = threading.Barrier(2, timeout=0.5)
|
||||
self._thread_state: Final = threading.local()
|
||||
self._meeting: Final = threading.Event()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def first_reads_meet(self) -> Iterator[None]:
|
||||
self._meeting.set()
|
||||
yield
|
||||
self._meeting.clear()
|
||||
|
||||
def get_cache(self, key: str) -> object:
|
||||
value: Final = self._store.get_cache(key)
|
||||
if self._meeting.is_set() and not getattr(self._thread_state, "has_read", False):
|
||||
self._thread_state.has_read = True
|
||||
with contextlib.suppress(threading.BrokenBarrierError):
|
||||
self._first_reads.wait()
|
||||
return value
|
||||
|
||||
def set_cache(self, key: str, value: tuple[float, ...], *, ttl: float) -> None:
|
||||
self._store.set_cache(key, value, ttl=ttl)
|
||||
|
||||
def delete_cache(self, key: str) -> None:
|
||||
self._store.delete_cache(key)
|
||||
|
||||
|
||||
def test_two_threads_picking_at_once_reserve_different_deployments() -> None:
|
||||
store: Final = _StoreWhereThreadsReadBeforeWriting()
|
||||
worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=store)
|
||||
|
||||
with store.first_reads_meet(), ThreadPoolExecutor(max_workers=2) as pool:
|
||||
picks: Final = tuple(pool.map(lambda _: worker.get_available_deployments(GROUP, HEALTHY, reserve=True), range(2)))
|
||||
|
||||
assert sorted(pick["model_info"]["id"] for pick in picks if pick is not None) == ["dep-a", "dep-b"]
|
||||
|
||||
|
||||
def test_a_reservation_and_a_release_on_one_deployment_at_once_both_count() -> None:
|
||||
store: Final = _StoreWhereThreadsReadBeforeWriting()
|
||||
worker: Final = LeastBusyLoggingHandler(router_cache=DualCache(), reservation_store=store)
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b"))
|
||||
worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-b"))
|
||||
|
||||
with store.first_reads_meet(), ThreadPoolExecutor(max_workers=2) as pool:
|
||||
reserved: Final = pool.submit(worker.get_available_deployments, GROUP, HEALTHY, True)
|
||||
started: Final = pool.submit(worker.log_pre_api_call, "m", [], _call_kwargs("dep-a"))
|
||||
assert reserved.result() is DEPLOYMENT_A
|
||||
started.result()
|
||||
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
|
||||
|
||||
|
||||
def _router_pick(router: Router) -> str:
|
||||
deployment: Final = router.get_available_deployment(ROUTER_GROUP, request_kwargs={"metadata": {}})
|
||||
return deployment["model_info"]["id"]
|
||||
|
||||
|
||||
def test_a_pick_stays_reserved_while_other_router_keys_fill_its_cache() -> None:
|
||||
router: Final = _least_busy_router()
|
||||
|
||||
first: Final = _router_pick(router)
|
||||
for index in range(DEFAULT_MAX_SIZE_IN_MEMORY):
|
||||
router.health().store.set_cache(f"deployment-usage-{index}", 1, local_only=True, ttl=60)
|
||||
|
||||
assert _router_pick(router) != first
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_cache_hit_releases_the_pick_it_reserved() -> None:
|
||||
worker: Final = _worker(None)
|
||||
worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
|
||||
|
||||
assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
|
||||
await worker.async_log_success_event({**_call_kwargs("dep-b"), "cache_hit": True}, None, None, None)
|
||||
|
||||
assert await worker.async_get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_B
|
||||
|
||||
|
||||
def test_a_cache_hit_leaves_requests_in_flight_on_its_deployment_counted() -> None:
|
||||
worker: Final = _worker(None)
|
||||
worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
|
||||
|
||||
worker.log_success_event({**_call_kwargs("dep-a"), "cache_hit": True}, None, None, None)
|
||||
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
|
||||
|
||||
|
||||
@pytest.mark.parametrize("outcome", [LeastBusyLoggingHandler.log_success_event, LeastBusyLoggingHandler.log_failure_event])
|
||||
def test_a_call_that_ends_before_reaching_the_provider_frees_its_pick(
|
||||
outcome: Callable[[LeastBusyLoggingHandler, dict[str, object], None, None, None], None],
|
||||
) -> None:
|
||||
worker: Final = _worker(None)
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
|
||||
outcome(worker, _routed_call_kwargs("dep-a"), None, None, None)
|
||||
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
|
||||
|
||||
|
||||
def test_a_call_the_proxy_rejected_after_logging_its_start_leaves_no_count_behind() -> None:
|
||||
worker: Final = _worker(None)
|
||||
rejected: Final = {**_routed_call_kwargs("dep-a"), LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True}
|
||||
|
||||
worker.log_pre_api_call(model="m", messages=[], kwargs=rejected)
|
||||
worker.log_failure_event(rejected, None, None, None)
|
||||
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
|
||||
|
||||
|
||||
async def _drain_logging() -> None:
|
||||
for _ in range(10):
|
||||
await asyncio.sleep(0)
|
||||
await GLOBAL_LOGGING_WORKER.flush()
|
||||
|
||||
|
||||
def _batch_deployment(route: respx.Route) -> str:
|
||||
return route.calls.last.request.headers["authorization"].removeprefix("Bearer key-")
|
||||
|
||||
|
||||
_BATCH: Final = {
|
||||
"id": "batch_least_busy",
|
||||
"object": "batch",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"input_file_id": "file-least-busy",
|
||||
"completion_window": "24h",
|
||||
"status": "validating",
|
||||
"created_at": 1,
|
||||
}
|
||||
_REJECTED_BATCH: Final = {"error": {"message": "Invalid input file", "type": "invalid_request_error"}}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("provider_response", [httpx.Response(200, json=_BATCH), httpx.Response(400, json=_REJECTED_BATCH)])
|
||||
async def test_a_batch_create_does_not_keep_its_deployment_busy(
|
||||
provider_response: httpx.Response, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/batches").mock(return_value=provider_response)
|
||||
router: Final = _least_busy_router()
|
||||
|
||||
with contextlib.suppress(openai.BadRequestError):
|
||||
await router.acreate_batch(
|
||||
model=ROUTER_GROUP, completion_window="24h", endpoint="/v1/chat/completions", input_file_id="file-least-busy"
|
||||
)
|
||||
await _drain_logging()
|
||||
|
||||
assert _router_pick(router) == _batch_deployment(route)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_batch_cancel_does_not_keep_its_deployment_busy(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/batches/batch_least_busy/cancel").mock(
|
||||
return_value=httpx.Response(200, json={**_BATCH, "status": "cancelling"})
|
||||
)
|
||||
router: Final = _least_busy_router()
|
||||
|
||||
await router.acancel_batch(model=ROUTER_GROUP, batch_id="batch_least_busy")
|
||||
|
||||
assert _router_pick(router) == _batch_deployment(route)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_picking_a_deployment_without_calling_it_does_not_keep_it_busy() -> None:
|
||||
router: Final = _least_busy_router()
|
||||
|
||||
picked: Final = await router.select(ROUTER_GROUP, {"metadata": {}})
|
||||
|
||||
assert _router_pick(router) == picked["model_info"]["id"]
|
||||
|
|
|
|||
|
|
@ -56,6 +56,9 @@ def _model_list():
|
|||
]
|
||||
|
||||
|
||||
CALL_STARTED_AT: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
|
||||
|
||||
|
||||
def _build_router(routing_strategy="simple-shuffle", routing_groups=None):
|
||||
return Router(
|
||||
model_list=_model_list(),
|
||||
|
|
@ -468,7 +471,8 @@ def test_two_least_busy_groups_count_a_request_once(monkeypatch):
|
|||
"litellm_params": {
|
||||
"metadata": {"model_group": "filtered-model"},
|
||||
"model_info": {"id": "deploy-1"},
|
||||
}
|
||||
},
|
||||
"api_call_start_time": CALL_STARTED_AT,
|
||||
}
|
||||
|
||||
for callback in litellm.input_callback:
|
||||
|
|
@ -481,6 +485,36 @@ def test_two_least_busy_groups_count_a_request_once(monkeypatch):
|
|||
assert router.cache.get_cache("filtered-model_request_count:deploy-1") == 0
|
||||
|
||||
|
||||
|
||||
def _filtered_model_pick(router: Router) -> str:
|
||||
deployment: Final = router.get_available_deployment(
|
||||
"filtered-model", request_kwargs=_proxy_shaped_request(model="filtered-model")
|
||||
)
|
||||
return deployment["model_info"]["id"]
|
||||
|
||||
|
||||
def test_a_least_busy_group_pick_stops_counting_once_its_request_finishes(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
monkeypatch.setattr(litellm, "input_callback", [])
|
||||
router: Final = _build_router(
|
||||
routing_strategy="least-busy",
|
||||
routing_groups=[{"group_name": "fast", "models": ["filtered-model"], "routing_strategy": "least-busy"}],
|
||||
)
|
||||
|
||||
first: Final = _filtered_model_pick(router)
|
||||
kwargs: Final = {
|
||||
"litellm_params": {"metadata": {"model_group": "filtered-model"}, "model_info": {"id": first}},
|
||||
"api_call_start_time": CALL_STARTED_AT,
|
||||
}
|
||||
for callback in litellm.input_callback:
|
||||
if isinstance(callback, CustomLogger):
|
||||
callback.log_pre_api_call(model="filtered-model", messages=[], kwargs=kwargs)
|
||||
for callback in litellm.callbacks:
|
||||
if isinstance(callback, CustomLogger):
|
||||
callback.log_success_event(kwargs, None, None, None)
|
||||
|
||||
assert _filtered_model_pick(router) == first
|
||||
|
||||
def test_two_routers_in_one_process_each_count_their_own_requests(monkeypatch):
|
||||
"""
|
||||
Least-busy hangs its counting off litellm's global callback lists, and those lists keep one
|
||||
|
|
@ -498,7 +532,8 @@ def test_two_routers_in_one_process_each_count_their_own_requests(monkeypatch):
|
|||
"litellm_params": {
|
||||
"metadata": {"model_group": "filtered-model"},
|
||||
"model_info": {"id": "deploy-1"},
|
||||
}
|
||||
},
|
||||
"api_call_start_time": CALL_STARTED_AT,
|
||||
}
|
||||
|
||||
for callback in litellm.input_callback:
|
||||
|
|
@ -613,6 +648,7 @@ async def test_select_deployment_async_returns_none_without_selector():
|
|||
messages=None,
|
||||
input=None,
|
||||
request_kwargs=None,
|
||||
reserve=False,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
|
@ -627,6 +663,7 @@ def test_select_deployment_sync_returns_none_without_selector():
|
|||
messages=None,
|
||||
input=None,
|
||||
request_kwargs=None,
|
||||
reserve=False,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
|
@ -661,6 +698,7 @@ def test_select_deployment_sync_does_not_dispatch_cost_based_routing():
|
|||
messages=None,
|
||||
input=None,
|
||||
request_kwargs=None,
|
||||
reserve=False,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
|
@ -707,6 +745,7 @@ async def test_async_dispatch_falls_back_to_sync_for_usage_based_routing_v1():
|
|||
messages=[{"role": "user", "content": "hi"}],
|
||||
input=None,
|
||||
request_kwargs={},
|
||||
reserve=False,
|
||||
)
|
||||
|
||||
assert v1_spy.called, "async dispatch must route v1 strategy through sync method"
|
||||
|
|
@ -1202,12 +1241,12 @@ async def test_least_busy_override_sees_the_overriding_request_in_flight():
|
|||
stream = await router.acompletion(**_proxy_shaped_request(model="grp", routing_strategy="least-busy", stream=True))
|
||||
busy = stream._hidden_params["model_id"]
|
||||
idle = "d2" if busy == "d1" else "d1"
|
||||
assert [await _async_override_pick(router, "least-busy") for _ in range(3)] == [idle, idle, idle]
|
||||
assert await _async_override_pick(router, "least-busy") == idle
|
||||
|
||||
async for _ in stream:
|
||||
pass
|
||||
await _async_wait_until(lambda: _in_flight(router, busy) == 0)
|
||||
assert await _async_override_pick(router, "least-busy") == "d1"
|
||||
assert await _async_override_pick(router, "least-busy") == busy
|
||||
assert _selector_is_not_global(router._override_selectors["least-busy"])
|
||||
|
||||
|
||||
|
|
@ -1217,12 +1256,12 @@ def test_sync_least_busy_override_sees_the_overriding_request_in_flight():
|
|||
stream = router.completion(**_proxy_shaped_request(model="grp", routing_strategy="least-busy", stream=True))
|
||||
busy = stream._hidden_params["model_id"]
|
||||
idle = "d2" if busy == "d1" else "d1"
|
||||
assert [_sync_override_pick(router, "least-busy") for _ in range(3)] == [idle, idle, idle]
|
||||
assert _sync_override_pick(router, "least-busy") == idle
|
||||
|
||||
for _ in stream:
|
||||
pass
|
||||
_sync_wait_until(lambda: _in_flight(router, busy) == 0)
|
||||
assert _sync_override_pick(router, "least-busy") == "d1"
|
||||
assert _sync_override_pick(router, "least-busy") == busy
|
||||
assert _selector_is_not_global(router._override_selectors["least-busy"])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -597,6 +597,34 @@ async def test_responses_websocket_turn_is_refused_once_its_deployment_crosses_b
|
|||
await admit_turn()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_turns_do_not_make_their_deployment_look_busy_to_least_busy():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "ws-least-busy",
|
||||
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "sk-fake"},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
for deployment_id in ("ws-least-busy-a", "ws-least-busy-b")
|
||||
],
|
||||
routing_strategy="least-busy",
|
||||
)
|
||||
admit_turn: Final = (await _kwargs_routed_to(router, "ws-least-busy", "_aresponses_websocket"))["turn_admission"]
|
||||
assert callable(admit_turn)
|
||||
for _ in range(3):
|
||||
await admit_turn()
|
||||
|
||||
picks: Final = [
|
||||
(await router.async_get_available_deployment(model="ws-least-busy", request_kwargs={"input": "hi"}))[
|
||||
"model_info"
|
||||
]["id"]
|
||||
for _ in range(2)
|
||||
]
|
||||
|
||||
assert sorted(picks) == ["ws-least-busy-a", "ws-least-busy-b"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_turn_after_the_first_counts_toward_deployment_rpm():
|
||||
router = Router(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue