fix(proxy): charge fair-queue passes only on admission, keep stats failures out of admission, set active request for every route

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin 2026-09-25 18:40:13 +00:00
parent fe3692453c
commit 87fae76bb6
8 changed files with 253 additions and 22 deletions

View file

@ -84,7 +84,6 @@ from litellm.proxy.auth.auth_checks import (
)
from litellm.proxy.auth.auth_utils import check_response_size_is_safe, get_request_route
from litellm.proxy.bug_report_config import build_proxy_bug_report
from litellm.proxy.common_utils.active_request import active_request
from litellm.proxy.common_utils.callback_utils import (
get_logging_caching_headers,
get_remaining_tokens_and_requests_from_request_data,
@ -2185,16 +2184,12 @@ class ProxyBaseLLMRequestProcessing:
if self._tags_before_guardrails is None:
self._tags_before_guardrails = frozenset(get_tags_from_request_body(request_body=self.data))
active_request_token: Final = active_request.set(request)
try:
self.data = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=user_api_key_dict,
data=self.data,
call_type=route_type,
skip_guardrails=skip_guardrails,
)
finally:
active_request.reset(active_request_token)
self.data = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=user_api_key_dict,
data=self.data,
call_type=route_type,
skip_guardrails=skip_guardrails,
)
await _enforce_guardrail_added_tag_budgets(
data=self.data,
tags_before_guardrails=self._tags_before_guardrails,

View file

@ -1,4 +1,5 @@
import asyncio
import math
import time
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass, field
@ -18,8 +19,10 @@ class QueueTicket:
@dataclass(frozen=True, slots=True)
class ClassQueueState:
head_request_id: str | None
head_enqueued_at: float
depth: int
pass_value: float
yielded: bool
_ClassQueue = tuple[tuple[float, str], ...]
@ -29,6 +32,7 @@ _ClassQueue = tuple[tuple[float, str], ...]
class _ModelQueues:
queues: dict[str, _ClassQueue] = field(default_factory=dict)
passes: dict[str, float] = field(default_factory=dict)
yielded: frozenset[str] = frozenset()
virtual_time: float = 0.0
@ -63,8 +67,10 @@ class InMemoryFairQueueStore:
{
class_name: ClassQueueState(
head_request_id=(queue[0][1] if (queue := state.queues.get(class_name)) else None),
head_enqueued_at=(queue[0][0] if queue else math.inf),
depth=len(state.queues.get(class_name, ())),
pass_value=state.passes.get(class_name, 0.0),
yielded=class_name in state.yielded,
)
for class_name in class_names
}
@ -75,6 +81,13 @@ class InMemoryFairQueueStore:
new_pass: Final = state.passes.get(class_name, 0.0) + amount
state.passes[class_name] = new_pass
state.virtual_time = new_pass
state.yielded = frozenset()
def yield_turn(self, model: str, class_name: str) -> None:
state: Final = self._model(model)
waiting: Final = frozenset(name for name, queue in state.queues.items() if queue)
yielded: Final = state.yielded | {class_name}
state.yielded = frozenset() if waiting <= yielded else yielded
def depths(self, model: str) -> Mapping[str, int]:
state: Final = self._models.get(model)
@ -150,10 +163,12 @@ class FairQueue:
if elapsed >= max_wait_seconds:
return QueueRejected(reason="queue_deadline_exceeded", waited_seconds=elapsed)
if _has_turn(ticket, self._store.snapshot(ticket.model, tuple(weights))):
admitted: Final = await try_admit()
self._store.advance(ticket.model, ticket.class_name, 1.0 / max(weights.get(ticket.class_name, 0.0), 1e-6))
if admitted:
if await try_admit():
self._store.advance(
ticket.model, ticket.class_name, 1.0 / max(weights.get(ticket.class_name, 0.0), 1e-6)
)
return QueueAdmitted(waited_seconds=self._clock() - started)
self._store.yield_turn(ticket.model, ticket.class_name)
remaining: Final = max_wait_seconds - (self._clock() - started)
await asyncio.sleep(max(0.0, min(poll_interval_seconds, remaining)))
return None
@ -163,5 +178,10 @@ def _has_turn(ticket: QueueTicket, snapshot: Mapping[str, ClassQueueState]) -> b
own: Final = snapshot.get(ticket.class_name)
if own is None or own.head_request_id != ticket.request_id:
return False
active: Final = tuple((state.pass_value, name) for name, state in snapshot.items() if state.depth > 0)
return min(active) == (own.pass_value, ticket.class_name)
active: Final = tuple(_rank(name, state) for name, state in snapshot.items() if state.depth > 0)
eligible: Final = tuple(entry for entry in active if not snapshot[entry[2]].yielded) or active
return min(eligible) == _rank(ticket.class_name, own)
def _rank(class_name: str, state: ClassQueueState) -> tuple[float, float, str]:
return (state.pass_value, state.head_enqueued_at, class_name)

View file

@ -9,6 +9,7 @@ from typing import Final, Literal
from pydantic import TypeAdapter
from litellm._logging import verbose_proxy_logger
from litellm.proxy.utils import InternalUsageCache
FairnessMetric = Literal[
@ -81,12 +82,17 @@ class FairnessStats:
return tuple(range(current - STATS_WINDOW_SECONDS // _BUCKET_SECONDS + 1, current + 1))
async def record(self, model: str, class_name: str, metric: FairnessMetric, value: float = 1.0) -> None:
await self._cache.async_increment_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs untyped upstream
key=_bucket_key(model, class_name, metric, self._current_bucket()),
value=value,
litellm_parent_otel_span=None,
ttl=_BUCKET_TTL_SECONDS,
)
try:
await self._cache.async_increment_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs untyped upstream
key=_bucket_key(model, class_name, metric, self._current_bucket()),
value=value,
litellm_parent_otel_span=None,
ttl=_BUCKET_TTL_SECONDS,
)
except Exception as error: # noqa: BLE001 # a lost counter must never change the admission decision
verbose_proxy_logger.warning(
"fairness stats %s for %s/%s not recorded: %s", metric, model, class_name, error
)
async def read(self, model: str, class_names: Sequence[str]) -> Mapping[str, ClassStats]:
buckets: Final = self._window_buckets()

View file

@ -0,0 +1,21 @@
from typing import Final
from starlette.requests import Request
from starlette.types import ASGIApp, Receive, Scope, Send
from litellm.proxy.common_utils.active_request import active_request
class ActiveRequestMiddleware:
def __init__(self, app: ASGIApp) -> None:
self.app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http":
await self.app(scope, receive, send)
return
token: Final = active_request.set(Request(scope, receive))
try:
await self.app(scope, receive, send)
finally:
active_request.reset(token)

View file

@ -706,6 +706,7 @@ try:
except ImportError:
build_billing_metrics_recorder = None
shutdown_billing_metrics_recorder = None
from litellm.proxy.middleware.active_request_middleware import ActiveRequestMiddleware
from litellm.proxy.middleware.admission_control_middleware import (
AdmissionControlMiddleware,
admission_control_state,
@ -2418,6 +2419,7 @@ app.add_middleware(
sink_factory=lambda: gateway_request_accumulator if prisma_client is not None else None,
)
app.add_middleware(BudgetReservationReleaseMiddleware, release=release_unbound_budget_reservation)
app.add_middleware(ActiveRequestMiddleware)
app.add_middleware(InFlightRequestsMiddleware)
app.add_middleware(SecurityHeadersMiddleware)

View file

@ -2127,6 +2127,40 @@ async def test_disabled_fairness_settings_keep_model_capacity_enforced(monkeypat
assert over_capacity.value.headers["x-litellm-fairness-reason"] == "capacity_exhausted"
@pytest.mark.asyncio
async def test_over_capacity_request_is_still_rejected_when_stats_counter_write_fails(monkeypatch):
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.fairness_stats import FairnessStats
from litellm.proxy.utils import InternalUsageCache
from litellm.types.proxy.fairness import FairnessSettings
class BrokenCounterCache(InternalUsageCache):
async def async_increment_cache(self, key: str, value: float, litellm_parent_otel_span, **kwargs):
raise ConnectionError("redis unavailable")
model = "fairness-stats-outage-model"
monkeypatch.setattr(litellm, "fairness_settings", FairnessSettings(enabled=False))
monkeypatch.setattr(litellm, "priority_reservation", None)
dual_cache = DualCache()
handler = DynamicRateLimitHandler(
internal_usage_cache=dual_cache, fairness_stats=FairnessStats(cache=BrokenCounterCache(DualCache()))
)
handler.update_variables(llm_router=_fairness_router(model, rpm=1))
async def call(call_id: str) -> None:
await handler.async_pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(),
cache=dual_cache,
data={"model": model, "litellm_call_id": call_id},
call_type="completion",
)
await asyncio.create_task(call("first"))
with pytest.raises(ProxyRateLimitError) as over_capacity:
await asyncio.create_task(call("second"))
assert over_capacity.value.status_code == 429
@pytest.mark.asyncio
async def test_failed_stream_settles_reservation_at_recovered_partial_usage(monkeypatch):
from litellm.types.proxy.fairness import FairnessSettings, WorkloadClass

View file

@ -28,6 +28,11 @@ async def _never_cancelled() -> bool:
return False
async def _never_cancelled_after_probing_the_client() -> bool:
await asyncio.sleep(0)
return False
async def _always_cancelled() -> bool:
return True
@ -186,6 +191,69 @@ async def test_weighted_ordering_gives_reserved_share_of_turns_without_starving_
assert first_four.count("batch") == 1
class PerClassGate:
"""Admits per class while that class has slots, recording the class of each admission in order."""
def __init__(self, **slots: int) -> None:
self.slots = dict(slots)
self.attempts: dict[str, int] = {}
self.admitted: list[str] = []
def for_class(self, class_name: str):
async def try_admit() -> bool:
self.attempts[class_name] = self.attempts.get(class_name, 0) + 1
if self.slots.get(class_name, 0) <= 0:
return False
self.slots[class_name] -= 1
self.admitted.append(class_name)
return True
return try_admit
def _waiter(queue: FairQueue, gate: PerClassGate, class_name: str, request_id: str, max_wait: float = 30.0):
return asyncio.create_task(
queue.wait_for_admission(
ticket=_ticket(class_name, request_id),
weights=WEIGHTS,
max_wait_seconds=max_wait,
max_depth=100,
poll_interval_seconds=0.001,
try_admit=gate.for_class(class_name),
is_cancelled=_never_cancelled_after_probing_the_client,
)
)
@pytest.mark.asyncio
async def test_newer_low_share_request_does_not_jump_a_class_that_waited_through_failed_admits():
queue: Final = FairQueue()
gate: Final = PerClassGate(prod=0, batch=0)
prod: Final = _waiter(queue, gate, "prod", "prod-1")
while gate.attempts.get("prod", 0) < 30:
await asyncio.sleep(0.001)
batch: Final = _waiter(queue, gate, "batch", "batch-1")
while queue.depths(MODEL).get("batch", 0) < 1:
await asyncio.sleep(0)
gate.slots["prod"] = 1
gate.slots["batch"] = 1
outcomes: Final = await asyncio.gather(prod, batch)
assert all(isinstance(outcome, QueueAdmitted) for outcome in outcomes)
assert gate.admitted == ["prod", "batch"]
@pytest.mark.asyncio
async def test_class_blocked_on_its_own_share_yields_the_slot_to_a_class_that_fits():
queue: Final = FairQueue()
gate: Final = PerClassGate(prod=0, batch=1)
prod: Final = _waiter(queue, gate, "prod", "prod-1", max_wait=5.0)
await asyncio.sleep(0.01)
batch_outcome: Final = await _waiter(queue, gate, "batch", "batch-1", max_wait=1.0)
assert isinstance(batch_outcome, QueueAdmitted)
assert gate.admitted == ["batch"]
prod.cancel()
@pytest.mark.asyncio
async def test_same_class_admits_in_arrival_order():
queue: Final = FairQueue()

View file

@ -0,0 +1,85 @@
import asyncio
import json
from typing import Final
import pytest
from starlette.applications import Starlette
from starlette.requests import Request
from starlette.responses import Response
from starlette.routing import Route
from starlette.types import Message
from litellm.proxy.common_utils.active_request import active_request, active_request_disconnected
from litellm.proxy.middleware.active_request_middleware import ActiveRequestMiddleware
BODY: Final = json.dumps({"prompt": "draw a cat"}).encode()
class DisconnectProbe:
"""A non-chat route that reads its body, then asks the fairness helper whether the client is still there."""
def __init__(self) -> None:
self.observed: list[bool] = []
async def route(self, request: Request) -> Response:
await request.json()
self.observed.append(await active_request_disconnected())
return Response(status_code=204)
def app(self) -> ActiveRequestMiddleware:
return ActiveRequestMiddleware(
Starlette(routes=[Route("/v1/images/generations", self.route, methods=["POST"])])
)
def _scope() -> dict[str, object]:
return {
"type": "http",
"asgi": {"version": "3.0"},
"http_version": "1.1",
"method": "POST",
"scheme": "http",
"path": "/v1/images/generations",
"raw_path": b"/v1/images/generations",
"root_path": "",
"query_string": b"",
"headers": [(b"content-type", b"application/json"), (b"content-length", str(len(BODY)).encode())],
"client": ("127.0.0.1", 1234),
"server": ("testserver", 80),
}
async def _run(app: ActiveRequestMiddleware, messages_after_body: tuple[Message, ...]) -> None:
pending: Final = [{"type": "http.request", "body": BODY, "more_body": False}, *messages_after_body]
forever: Final = asyncio.Event()
async def receive() -> Message:
if pending:
return pending.pop(0)
await forever.wait()
return {"type": "http.disconnect"}
async def send(message: Message) -> None:
del message
await app(_scope(), receive, send)
@pytest.mark.asyncio
async def test_non_chat_route_sees_client_disconnect_through_active_request():
probe: Final = DisconnectProbe()
await _run(probe.app(), ({"type": "http.disconnect"},))
assert probe.observed == [True]
@pytest.mark.asyncio
async def test_connected_client_is_not_reported_as_disconnected():
probe: Final = DisconnectProbe()
await _run(probe.app(), ())
assert probe.observed == [False]
@pytest.mark.asyncio
async def test_active_request_is_cleared_once_the_response_is_sent():
await _run(DisconnectProbe().app(), ())
assert active_request.get() is None