mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): charge fair-queue passes only on admission, keep stats failures out of admission, set active request for every route
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
fe3692453c
commit
87fae76bb6
8 changed files with 253 additions and 22 deletions
|
|
@ -84,7 +84,6 @@ from litellm.proxy.auth.auth_checks import (
|
|||
)
|
||||
from litellm.proxy.auth.auth_utils import check_response_size_is_safe, get_request_route
|
||||
from litellm.proxy.bug_report_config import build_proxy_bug_report
|
||||
from litellm.proxy.common_utils.active_request import active_request
|
||||
from litellm.proxy.common_utils.callback_utils import (
|
||||
get_logging_caching_headers,
|
||||
get_remaining_tokens_and_requests_from_request_data,
|
||||
|
|
@ -2185,16 +2184,12 @@ class ProxyBaseLLMRequestProcessing:
|
|||
|
||||
if self._tags_before_guardrails is None:
|
||||
self._tags_before_guardrails = frozenset(get_tags_from_request_body(request_body=self.data))
|
||||
active_request_token: Final = active_request.set(request)
|
||||
try:
|
||||
self.data = await proxy_logging_obj.pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
data=self.data,
|
||||
call_type=route_type,
|
||||
skip_guardrails=skip_guardrails,
|
||||
)
|
||||
finally:
|
||||
active_request.reset(active_request_token)
|
||||
self.data = await proxy_logging_obj.pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
data=self.data,
|
||||
call_type=route_type,
|
||||
skip_guardrails=skip_guardrails,
|
||||
)
|
||||
await _enforce_guardrail_added_tag_budgets(
|
||||
data=self.data,
|
||||
tags_before_guardrails=self._tags_before_guardrails,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import math
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
|
|
@ -18,8 +19,10 @@ class QueueTicket:
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class ClassQueueState:
|
||||
head_request_id: str | None
|
||||
head_enqueued_at: float
|
||||
depth: int
|
||||
pass_value: float
|
||||
yielded: bool
|
||||
|
||||
|
||||
_ClassQueue = tuple[tuple[float, str], ...]
|
||||
|
|
@ -29,6 +32,7 @@ _ClassQueue = tuple[tuple[float, str], ...]
|
|||
class _ModelQueues:
|
||||
queues: dict[str, _ClassQueue] = field(default_factory=dict)
|
||||
passes: dict[str, float] = field(default_factory=dict)
|
||||
yielded: frozenset[str] = frozenset()
|
||||
virtual_time: float = 0.0
|
||||
|
||||
|
||||
|
|
@ -63,8 +67,10 @@ class InMemoryFairQueueStore:
|
|||
{
|
||||
class_name: ClassQueueState(
|
||||
head_request_id=(queue[0][1] if (queue := state.queues.get(class_name)) else None),
|
||||
head_enqueued_at=(queue[0][0] if queue else math.inf),
|
||||
depth=len(state.queues.get(class_name, ())),
|
||||
pass_value=state.passes.get(class_name, 0.0),
|
||||
yielded=class_name in state.yielded,
|
||||
)
|
||||
for class_name in class_names
|
||||
}
|
||||
|
|
@ -75,6 +81,13 @@ class InMemoryFairQueueStore:
|
|||
new_pass: Final = state.passes.get(class_name, 0.0) + amount
|
||||
state.passes[class_name] = new_pass
|
||||
state.virtual_time = new_pass
|
||||
state.yielded = frozenset()
|
||||
|
||||
def yield_turn(self, model: str, class_name: str) -> None:
|
||||
state: Final = self._model(model)
|
||||
waiting: Final = frozenset(name for name, queue in state.queues.items() if queue)
|
||||
yielded: Final = state.yielded | {class_name}
|
||||
state.yielded = frozenset() if waiting <= yielded else yielded
|
||||
|
||||
def depths(self, model: str) -> Mapping[str, int]:
|
||||
state: Final = self._models.get(model)
|
||||
|
|
@ -150,10 +163,12 @@ class FairQueue:
|
|||
if elapsed >= max_wait_seconds:
|
||||
return QueueRejected(reason="queue_deadline_exceeded", waited_seconds=elapsed)
|
||||
if _has_turn(ticket, self._store.snapshot(ticket.model, tuple(weights))):
|
||||
admitted: Final = await try_admit()
|
||||
self._store.advance(ticket.model, ticket.class_name, 1.0 / max(weights.get(ticket.class_name, 0.0), 1e-6))
|
||||
if admitted:
|
||||
if await try_admit():
|
||||
self._store.advance(
|
||||
ticket.model, ticket.class_name, 1.0 / max(weights.get(ticket.class_name, 0.0), 1e-6)
|
||||
)
|
||||
return QueueAdmitted(waited_seconds=self._clock() - started)
|
||||
self._store.yield_turn(ticket.model, ticket.class_name)
|
||||
remaining: Final = max_wait_seconds - (self._clock() - started)
|
||||
await asyncio.sleep(max(0.0, min(poll_interval_seconds, remaining)))
|
||||
return None
|
||||
|
|
@ -163,5 +178,10 @@ def _has_turn(ticket: QueueTicket, snapshot: Mapping[str, ClassQueueState]) -> b
|
|||
own: Final = snapshot.get(ticket.class_name)
|
||||
if own is None or own.head_request_id != ticket.request_id:
|
||||
return False
|
||||
active: Final = tuple((state.pass_value, name) for name, state in snapshot.items() if state.depth > 0)
|
||||
return min(active) == (own.pass_value, ticket.class_name)
|
||||
active: Final = tuple(_rank(name, state) for name, state in snapshot.items() if state.depth > 0)
|
||||
eligible: Final = tuple(entry for entry in active if not snapshot[entry[2]].yielded) or active
|
||||
return min(eligible) == _rank(ticket.class_name, own)
|
||||
|
||||
|
||||
def _rank(class_name: str, state: ClassQueueState) -> tuple[float, float, str]:
|
||||
return (state.pass_value, state.head_enqueued_at, class_name)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from typing import Final, Literal
|
|||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.utils import InternalUsageCache
|
||||
|
||||
FairnessMetric = Literal[
|
||||
|
|
@ -81,12 +82,17 @@ class FairnessStats:
|
|||
return tuple(range(current - STATS_WINDOW_SECONDS // _BUCKET_SECONDS + 1, current + 1))
|
||||
|
||||
async def record(self, model: str, class_name: str, metric: FairnessMetric, value: float = 1.0) -> None:
|
||||
await self._cache.async_increment_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs untyped upstream
|
||||
key=_bucket_key(model, class_name, metric, self._current_bucket()),
|
||||
value=value,
|
||||
litellm_parent_otel_span=None,
|
||||
ttl=_BUCKET_TTL_SECONDS,
|
||||
)
|
||||
try:
|
||||
await self._cache.async_increment_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs untyped upstream
|
||||
key=_bucket_key(model, class_name, metric, self._current_bucket()),
|
||||
value=value,
|
||||
litellm_parent_otel_span=None,
|
||||
ttl=_BUCKET_TTL_SECONDS,
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 # a lost counter must never change the admission decision
|
||||
verbose_proxy_logger.warning(
|
||||
"fairness stats %s for %s/%s not recorded: %s", metric, model, class_name, error
|
||||
)
|
||||
|
||||
async def read(self, model: str, class_names: Sequence[str]) -> Mapping[str, ClassStats]:
|
||||
buckets: Final = self._window_buckets()
|
||||
|
|
|
|||
21
litellm/proxy/middleware/active_request_middleware.py
Normal file
21
litellm/proxy/middleware/active_request_middleware.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
from typing import Final
|
||||
|
||||
from starlette.requests import Request
|
||||
from starlette.types import ASGIApp, Receive, Scope, Send
|
||||
|
||||
from litellm.proxy.common_utils.active_request import active_request
|
||||
|
||||
|
||||
class ActiveRequestMiddleware:
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
if scope["type"] != "http":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
token: Final = active_request.set(Request(scope, receive))
|
||||
try:
|
||||
await self.app(scope, receive, send)
|
||||
finally:
|
||||
active_request.reset(token)
|
||||
|
|
@ -706,6 +706,7 @@ try:
|
|||
except ImportError:
|
||||
build_billing_metrics_recorder = None
|
||||
shutdown_billing_metrics_recorder = None
|
||||
from litellm.proxy.middleware.active_request_middleware import ActiveRequestMiddleware
|
||||
from litellm.proxy.middleware.admission_control_middleware import (
|
||||
AdmissionControlMiddleware,
|
||||
admission_control_state,
|
||||
|
|
@ -2418,6 +2419,7 @@ app.add_middleware(
|
|||
sink_factory=lambda: gateway_request_accumulator if prisma_client is not None else None,
|
||||
)
|
||||
app.add_middleware(BudgetReservationReleaseMiddleware, release=release_unbound_budget_reservation)
|
||||
app.add_middleware(ActiveRequestMiddleware)
|
||||
app.add_middleware(InFlightRequestsMiddleware)
|
||||
app.add_middleware(SecurityHeadersMiddleware)
|
||||
|
||||
|
|
|
|||
|
|
@ -2127,6 +2127,40 @@ async def test_disabled_fairness_settings_keep_model_capacity_enforced(monkeypat
|
|||
assert over_capacity.value.headers["x-litellm-fairness-reason"] == "capacity_exhausted"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_over_capacity_request_is_still_rejected_when_stats_counter_write_fails(monkeypatch):
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
from litellm.proxy.hooks.fairness_stats import FairnessStats
|
||||
from litellm.proxy.utils import InternalUsageCache
|
||||
from litellm.types.proxy.fairness import FairnessSettings
|
||||
|
||||
class BrokenCounterCache(InternalUsageCache):
|
||||
async def async_increment_cache(self, key: str, value: float, litellm_parent_otel_span, **kwargs):
|
||||
raise ConnectionError("redis unavailable")
|
||||
|
||||
model = "fairness-stats-outage-model"
|
||||
monkeypatch.setattr(litellm, "fairness_settings", FairnessSettings(enabled=False))
|
||||
monkeypatch.setattr(litellm, "priority_reservation", None)
|
||||
dual_cache = DualCache()
|
||||
handler = DynamicRateLimitHandler(
|
||||
internal_usage_cache=dual_cache, fairness_stats=FairnessStats(cache=BrokenCounterCache(DualCache()))
|
||||
)
|
||||
handler.update_variables(llm_router=_fairness_router(model, rpm=1))
|
||||
|
||||
async def call(call_id: str) -> None:
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
cache=dual_cache,
|
||||
data={"model": model, "litellm_call_id": call_id},
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
await asyncio.create_task(call("first"))
|
||||
with pytest.raises(ProxyRateLimitError) as over_capacity:
|
||||
await asyncio.create_task(call("second"))
|
||||
assert over_capacity.value.status_code == 429
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_stream_settles_reservation_at_recovered_partial_usage(monkeypatch):
|
||||
from litellm.types.proxy.fairness import FairnessSettings, WorkloadClass
|
||||
|
|
|
|||
|
|
@ -28,6 +28,11 @@ async def _never_cancelled() -> bool:
|
|||
return False
|
||||
|
||||
|
||||
async def _never_cancelled_after_probing_the_client() -> bool:
|
||||
await asyncio.sleep(0)
|
||||
return False
|
||||
|
||||
|
||||
async def _always_cancelled() -> bool:
|
||||
return True
|
||||
|
||||
|
|
@ -186,6 +191,69 @@ async def test_weighted_ordering_gives_reserved_share_of_turns_without_starving_
|
|||
assert first_four.count("batch") == 1
|
||||
|
||||
|
||||
class PerClassGate:
|
||||
"""Admits per class while that class has slots, recording the class of each admission in order."""
|
||||
|
||||
def __init__(self, **slots: int) -> None:
|
||||
self.slots = dict(slots)
|
||||
self.attempts: dict[str, int] = {}
|
||||
self.admitted: list[str] = []
|
||||
|
||||
def for_class(self, class_name: str):
|
||||
async def try_admit() -> bool:
|
||||
self.attempts[class_name] = self.attempts.get(class_name, 0) + 1
|
||||
if self.slots.get(class_name, 0) <= 0:
|
||||
return False
|
||||
self.slots[class_name] -= 1
|
||||
self.admitted.append(class_name)
|
||||
return True
|
||||
|
||||
return try_admit
|
||||
|
||||
|
||||
def _waiter(queue: FairQueue, gate: PerClassGate, class_name: str, request_id: str, max_wait: float = 30.0):
|
||||
return asyncio.create_task(
|
||||
queue.wait_for_admission(
|
||||
ticket=_ticket(class_name, request_id),
|
||||
weights=WEIGHTS,
|
||||
max_wait_seconds=max_wait,
|
||||
max_depth=100,
|
||||
poll_interval_seconds=0.001,
|
||||
try_admit=gate.for_class(class_name),
|
||||
is_cancelled=_never_cancelled_after_probing_the_client,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_newer_low_share_request_does_not_jump_a_class_that_waited_through_failed_admits():
|
||||
queue: Final = FairQueue()
|
||||
gate: Final = PerClassGate(prod=0, batch=0)
|
||||
prod: Final = _waiter(queue, gate, "prod", "prod-1")
|
||||
while gate.attempts.get("prod", 0) < 30:
|
||||
await asyncio.sleep(0.001)
|
||||
batch: Final = _waiter(queue, gate, "batch", "batch-1")
|
||||
while queue.depths(MODEL).get("batch", 0) < 1:
|
||||
await asyncio.sleep(0)
|
||||
gate.slots["prod"] = 1
|
||||
gate.slots["batch"] = 1
|
||||
outcomes: Final = await asyncio.gather(prod, batch)
|
||||
assert all(isinstance(outcome, QueueAdmitted) for outcome in outcomes)
|
||||
assert gate.admitted == ["prod", "batch"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_class_blocked_on_its_own_share_yields_the_slot_to_a_class_that_fits():
|
||||
queue: Final = FairQueue()
|
||||
gate: Final = PerClassGate(prod=0, batch=1)
|
||||
prod: Final = _waiter(queue, gate, "prod", "prod-1", max_wait=5.0)
|
||||
await asyncio.sleep(0.01)
|
||||
batch_outcome: Final = await _waiter(queue, gate, "batch", "batch-1", max_wait=1.0)
|
||||
assert isinstance(batch_outcome, QueueAdmitted)
|
||||
assert gate.admitted == ["batch"]
|
||||
prod.cancel()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_class_admits_in_arrival_order():
|
||||
queue: Final = FairQueue()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,85 @@
|
|||
import asyncio
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
from starlette.routing import Route
|
||||
from starlette.types import Message
|
||||
|
||||
from litellm.proxy.common_utils.active_request import active_request, active_request_disconnected
|
||||
from litellm.proxy.middleware.active_request_middleware import ActiveRequestMiddleware
|
||||
|
||||
BODY: Final = json.dumps({"prompt": "draw a cat"}).encode()
|
||||
|
||||
|
||||
class DisconnectProbe:
|
||||
"""A non-chat route that reads its body, then asks the fairness helper whether the client is still there."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.observed: list[bool] = []
|
||||
|
||||
async def route(self, request: Request) -> Response:
|
||||
await request.json()
|
||||
self.observed.append(await active_request_disconnected())
|
||||
return Response(status_code=204)
|
||||
|
||||
def app(self) -> ActiveRequestMiddleware:
|
||||
return ActiveRequestMiddleware(
|
||||
Starlette(routes=[Route("/v1/images/generations", self.route, methods=["POST"])])
|
||||
)
|
||||
|
||||
|
||||
def _scope() -> dict[str, object]:
|
||||
return {
|
||||
"type": "http",
|
||||
"asgi": {"version": "3.0"},
|
||||
"http_version": "1.1",
|
||||
"method": "POST",
|
||||
"scheme": "http",
|
||||
"path": "/v1/images/generations",
|
||||
"raw_path": b"/v1/images/generations",
|
||||
"root_path": "",
|
||||
"query_string": b"",
|
||||
"headers": [(b"content-type", b"application/json"), (b"content-length", str(len(BODY)).encode())],
|
||||
"client": ("127.0.0.1", 1234),
|
||||
"server": ("testserver", 80),
|
||||
}
|
||||
|
||||
|
||||
async def _run(app: ActiveRequestMiddleware, messages_after_body: tuple[Message, ...]) -> None:
|
||||
pending: Final = [{"type": "http.request", "body": BODY, "more_body": False}, *messages_after_body]
|
||||
forever: Final = asyncio.Event()
|
||||
|
||||
async def receive() -> Message:
|
||||
if pending:
|
||||
return pending.pop(0)
|
||||
await forever.wait()
|
||||
return {"type": "http.disconnect"}
|
||||
|
||||
async def send(message: Message) -> None:
|
||||
del message
|
||||
|
||||
await app(_scope(), receive, send)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_chat_route_sees_client_disconnect_through_active_request():
|
||||
probe: Final = DisconnectProbe()
|
||||
await _run(probe.app(), ({"type": "http.disconnect"},))
|
||||
assert probe.observed == [True]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connected_client_is_not_reported_as_disconnected():
|
||||
probe: Final = DisconnectProbe()
|
||||
await _run(probe.app(), ())
|
||||
assert probe.observed == [False]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_active_request_is_cleared_once_the_response_is_sent():
|
||||
await _run(DisconnectProbe().app(), ())
|
||||
assert active_request.get() is None
|
||||
Loading…
Add table
Reference in a new issue