From 87fae76bb6c0a7e96ed23c978581911d76d18d8f Mon Sep 17 00:00:00 2001 From: yassin Date: Fri, 25 Sep 2026 18:40:13 +0000 Subject: [PATCH] fix(proxy): charge fair-queue passes only on admission, keep stats failures out of admission, set active request for every route Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 17 ++-- litellm/proxy/hooks/fairness_queue.py | 30 +++++-- litellm/proxy/hooks/fairness_stats.py | 18 ++-- .../middleware/active_request_middleware.py | 21 +++++ litellm/proxy/proxy_server.py | 2 + .../hooks/test_dynamic_rate_limiter_v3.py | 34 ++++++++ .../proxy/hooks/test_fairness_queue.py | 68 +++++++++++++++ .../test_active_request_middleware.py | 85 +++++++++++++++++++ 8 files changed, 253 insertions(+), 22 deletions(-) create mode 100644 litellm/proxy/middleware/active_request_middleware.py create mode 100644 tests/test_litellm/proxy/middleware/test_active_request_middleware.py diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 6ea481b3d14..7e1989b0aab 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -84,7 +84,6 @@ from litellm.proxy.auth.auth_checks import ( ) from litellm.proxy.auth.auth_utils import check_response_size_is_safe, get_request_route from litellm.proxy.bug_report_config import build_proxy_bug_report -from litellm.proxy.common_utils.active_request import active_request from litellm.proxy.common_utils.callback_utils import ( get_logging_caching_headers, get_remaining_tokens_and_requests_from_request_data, @@ -2185,16 +2184,12 @@ class ProxyBaseLLMRequestProcessing: if self._tags_before_guardrails is None: self._tags_before_guardrails = frozenset(get_tags_from_request_body(request_body=self.data)) - active_request_token: Final = active_request.set(request) - try: - self.data = await proxy_logging_obj.pre_call_hook( - user_api_key_dict=user_api_key_dict, - data=self.data, - call_type=route_type, - skip_guardrails=skip_guardrails, - ) - finally: - active_request.reset(active_request_token) + self.data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_dict, + data=self.data, + call_type=route_type, + skip_guardrails=skip_guardrails, + ) await _enforce_guardrail_added_tag_budgets( data=self.data, tags_before_guardrails=self._tags_before_guardrails, diff --git a/litellm/proxy/hooks/fairness_queue.py b/litellm/proxy/hooks/fairness_queue.py index 41559307951..c9c34753281 100644 --- a/litellm/proxy/hooks/fairness_queue.py +++ b/litellm/proxy/hooks/fairness_queue.py @@ -1,4 +1,5 @@ import asyncio +import math import time from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass, field @@ -18,8 +19,10 @@ class QueueTicket: @dataclass(frozen=True, slots=True) class ClassQueueState: head_request_id: str | None + head_enqueued_at: float depth: int pass_value: float + yielded: bool _ClassQueue = tuple[tuple[float, str], ...] @@ -29,6 +32,7 @@ _ClassQueue = tuple[tuple[float, str], ...] class _ModelQueues: queues: dict[str, _ClassQueue] = field(default_factory=dict) passes: dict[str, float] = field(default_factory=dict) + yielded: frozenset[str] = frozenset() virtual_time: float = 0.0 @@ -63,8 +67,10 @@ class InMemoryFairQueueStore: { class_name: ClassQueueState( head_request_id=(queue[0][1] if (queue := state.queues.get(class_name)) else None), + head_enqueued_at=(queue[0][0] if queue else math.inf), depth=len(state.queues.get(class_name, ())), pass_value=state.passes.get(class_name, 0.0), + yielded=class_name in state.yielded, ) for class_name in class_names } @@ -75,6 +81,13 @@ class InMemoryFairQueueStore: new_pass: Final = state.passes.get(class_name, 0.0) + amount state.passes[class_name] = new_pass state.virtual_time = new_pass + state.yielded = frozenset() + + def yield_turn(self, model: str, class_name: str) -> None: + state: Final = self._model(model) + waiting: Final = frozenset(name for name, queue in state.queues.items() if queue) + yielded: Final = state.yielded | {class_name} + state.yielded = frozenset() if waiting <= yielded else yielded def depths(self, model: str) -> Mapping[str, int]: state: Final = self._models.get(model) @@ -150,10 +163,12 @@ class FairQueue: if elapsed >= max_wait_seconds: return QueueRejected(reason="queue_deadline_exceeded", waited_seconds=elapsed) if _has_turn(ticket, self._store.snapshot(ticket.model, tuple(weights))): - admitted: Final = await try_admit() - self._store.advance(ticket.model, ticket.class_name, 1.0 / max(weights.get(ticket.class_name, 0.0), 1e-6)) - if admitted: + if await try_admit(): + self._store.advance( + ticket.model, ticket.class_name, 1.0 / max(weights.get(ticket.class_name, 0.0), 1e-6) + ) return QueueAdmitted(waited_seconds=self._clock() - started) + self._store.yield_turn(ticket.model, ticket.class_name) remaining: Final = max_wait_seconds - (self._clock() - started) await asyncio.sleep(max(0.0, min(poll_interval_seconds, remaining))) return None @@ -163,5 +178,10 @@ def _has_turn(ticket: QueueTicket, snapshot: Mapping[str, ClassQueueState]) -> b own: Final = snapshot.get(ticket.class_name) if own is None or own.head_request_id != ticket.request_id: return False - active: Final = tuple((state.pass_value, name) for name, state in snapshot.items() if state.depth > 0) - return min(active) == (own.pass_value, ticket.class_name) + active: Final = tuple(_rank(name, state) for name, state in snapshot.items() if state.depth > 0) + eligible: Final = tuple(entry for entry in active if not snapshot[entry[2]].yielded) or active + return min(eligible) == _rank(ticket.class_name, own) + + +def _rank(class_name: str, state: ClassQueueState) -> tuple[float, float, str]: + return (state.pass_value, state.head_enqueued_at, class_name) diff --git a/litellm/proxy/hooks/fairness_stats.py b/litellm/proxy/hooks/fairness_stats.py index 81764a8d82c..f04084ac3bd 100644 --- a/litellm/proxy/hooks/fairness_stats.py +++ b/litellm/proxy/hooks/fairness_stats.py @@ -9,6 +9,7 @@ from typing import Final, Literal from pydantic import TypeAdapter +from litellm._logging import verbose_proxy_logger from litellm.proxy.utils import InternalUsageCache FairnessMetric = Literal[ @@ -81,12 +82,17 @@ class FairnessStats: return tuple(range(current - STATS_WINDOW_SECONDS // _BUCKET_SECONDS + 1, current + 1)) async def record(self, model: str, class_name: str, metric: FairnessMetric, value: float = 1.0) -> None: - await self._cache.async_increment_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs untyped upstream - key=_bucket_key(model, class_name, metric, self._current_bucket()), - value=value, - litellm_parent_otel_span=None, - ttl=_BUCKET_TTL_SECONDS, - ) + try: + await self._cache.async_increment_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs untyped upstream + key=_bucket_key(model, class_name, metric, self._current_bucket()), + value=value, + litellm_parent_otel_span=None, + ttl=_BUCKET_TTL_SECONDS, + ) + except Exception as error: # noqa: BLE001 # a lost counter must never change the admission decision + verbose_proxy_logger.warning( + "fairness stats %s for %s/%s not recorded: %s", metric, model, class_name, error + ) async def read(self, model: str, class_names: Sequence[str]) -> Mapping[str, ClassStats]: buckets: Final = self._window_buckets() diff --git a/litellm/proxy/middleware/active_request_middleware.py b/litellm/proxy/middleware/active_request_middleware.py new file mode 100644 index 00000000000..f4d533ba5e4 --- /dev/null +++ b/litellm/proxy/middleware/active_request_middleware.py @@ -0,0 +1,21 @@ +from typing import Final + +from starlette.requests import Request +from starlette.types import ASGIApp, Receive, Scope, Send + +from litellm.proxy.common_utils.active_request import active_request + + +class ActiveRequestMiddleware: + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + token: Final = active_request.set(Request(scope, receive)) + try: + await self.app(scope, receive, send) + finally: + active_request.reset(token) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6c81a5d8e72..4a14acf22df 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -706,6 +706,7 @@ try: except ImportError: build_billing_metrics_recorder = None shutdown_billing_metrics_recorder = None +from litellm.proxy.middleware.active_request_middleware import ActiveRequestMiddleware from litellm.proxy.middleware.admission_control_middleware import ( AdmissionControlMiddleware, admission_control_state, @@ -2418,6 +2419,7 @@ app.add_middleware( sink_factory=lambda: gateway_request_accumulator if prisma_client is not None else None, ) app.add_middleware(BudgetReservationReleaseMiddleware, release=release_unbound_budget_reservation) +app.add_middleware(ActiveRequestMiddleware) app.add_middleware(InFlightRequestsMiddleware) app.add_middleware(SecurityHeadersMiddleware) diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index 3bfe69078e2..cbdac50b22b 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -2127,6 +2127,40 @@ async def test_disabled_fairness_settings_keep_model_capacity_enforced(monkeypat assert over_capacity.value.headers["x-litellm-fairness-reason"] == "capacity_exhausted" +@pytest.mark.asyncio +async def test_over_capacity_request_is_still_rejected_when_stats_counter_write_fails(monkeypatch): + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + from litellm.proxy.hooks.fairness_stats import FairnessStats + from litellm.proxy.utils import InternalUsageCache + from litellm.types.proxy.fairness import FairnessSettings + + class BrokenCounterCache(InternalUsageCache): + async def async_increment_cache(self, key: str, value: float, litellm_parent_otel_span, **kwargs): + raise ConnectionError("redis unavailable") + + model = "fairness-stats-outage-model" + monkeypatch.setattr(litellm, "fairness_settings", FairnessSettings(enabled=False)) + monkeypatch.setattr(litellm, "priority_reservation", None) + dual_cache = DualCache() + handler = DynamicRateLimitHandler( + internal_usage_cache=dual_cache, fairness_stats=FairnessStats(cache=BrokenCounterCache(DualCache())) + ) + handler.update_variables(llm_router=_fairness_router(model, rpm=1)) + + async def call(call_id: str) -> None: + await handler.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=dual_cache, + data={"model": model, "litellm_call_id": call_id}, + call_type="completion", + ) + + await asyncio.create_task(call("first")) + with pytest.raises(ProxyRateLimitError) as over_capacity: + await asyncio.create_task(call("second")) + assert over_capacity.value.status_code == 429 + + @pytest.mark.asyncio async def test_failed_stream_settles_reservation_at_recovered_partial_usage(monkeypatch): from litellm.types.proxy.fairness import FairnessSettings, WorkloadClass diff --git a/tests/test_litellm/proxy/hooks/test_fairness_queue.py b/tests/test_litellm/proxy/hooks/test_fairness_queue.py index 5f853f97390..e7aa8a3eeb4 100644 --- a/tests/test_litellm/proxy/hooks/test_fairness_queue.py +++ b/tests/test_litellm/proxy/hooks/test_fairness_queue.py @@ -28,6 +28,11 @@ async def _never_cancelled() -> bool: return False +async def _never_cancelled_after_probing_the_client() -> bool: + await asyncio.sleep(0) + return False + + async def _always_cancelled() -> bool: return True @@ -186,6 +191,69 @@ async def test_weighted_ordering_gives_reserved_share_of_turns_without_starving_ assert first_four.count("batch") == 1 +class PerClassGate: + """Admits per class while that class has slots, recording the class of each admission in order.""" + + def __init__(self, **slots: int) -> None: + self.slots = dict(slots) + self.attempts: dict[str, int] = {} + self.admitted: list[str] = [] + + def for_class(self, class_name: str): + async def try_admit() -> bool: + self.attempts[class_name] = self.attempts.get(class_name, 0) + 1 + if self.slots.get(class_name, 0) <= 0: + return False + self.slots[class_name] -= 1 + self.admitted.append(class_name) + return True + + return try_admit + + +def _waiter(queue: FairQueue, gate: PerClassGate, class_name: str, request_id: str, max_wait: float = 30.0): + return asyncio.create_task( + queue.wait_for_admission( + ticket=_ticket(class_name, request_id), + weights=WEIGHTS, + max_wait_seconds=max_wait, + max_depth=100, + poll_interval_seconds=0.001, + try_admit=gate.for_class(class_name), + is_cancelled=_never_cancelled_after_probing_the_client, + ) + ) + + +@pytest.mark.asyncio +async def test_newer_low_share_request_does_not_jump_a_class_that_waited_through_failed_admits(): + queue: Final = FairQueue() + gate: Final = PerClassGate(prod=0, batch=0) + prod: Final = _waiter(queue, gate, "prod", "prod-1") + while gate.attempts.get("prod", 0) < 30: + await asyncio.sleep(0.001) + batch: Final = _waiter(queue, gate, "batch", "batch-1") + while queue.depths(MODEL).get("batch", 0) < 1: + await asyncio.sleep(0) + gate.slots["prod"] = 1 + gate.slots["batch"] = 1 + outcomes: Final = await asyncio.gather(prod, batch) + assert all(isinstance(outcome, QueueAdmitted) for outcome in outcomes) + assert gate.admitted == ["prod", "batch"] + + +@pytest.mark.asyncio +async def test_class_blocked_on_its_own_share_yields_the_slot_to_a_class_that_fits(): + queue: Final = FairQueue() + gate: Final = PerClassGate(prod=0, batch=1) + prod: Final = _waiter(queue, gate, "prod", "prod-1", max_wait=5.0) + await asyncio.sleep(0.01) + batch_outcome: Final = await _waiter(queue, gate, "batch", "batch-1", max_wait=1.0) + assert isinstance(batch_outcome, QueueAdmitted) + assert gate.admitted == ["batch"] + prod.cancel() + + @pytest.mark.asyncio async def test_same_class_admits_in_arrival_order(): queue: Final = FairQueue() diff --git a/tests/test_litellm/proxy/middleware/test_active_request_middleware.py b/tests/test_litellm/proxy/middleware/test_active_request_middleware.py new file mode 100644 index 00000000000..d06a1817314 --- /dev/null +++ b/tests/test_litellm/proxy/middleware/test_active_request_middleware.py @@ -0,0 +1,85 @@ +import asyncio +import json +from typing import Final + +import pytest +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import Response +from starlette.routing import Route +from starlette.types import Message + +from litellm.proxy.common_utils.active_request import active_request, active_request_disconnected +from litellm.proxy.middleware.active_request_middleware import ActiveRequestMiddleware + +BODY: Final = json.dumps({"prompt": "draw a cat"}).encode() + + +class DisconnectProbe: + """A non-chat route that reads its body, then asks the fairness helper whether the client is still there.""" + + def __init__(self) -> None: + self.observed: list[bool] = [] + + async def route(self, request: Request) -> Response: + await request.json() + self.observed.append(await active_request_disconnected()) + return Response(status_code=204) + + def app(self) -> ActiveRequestMiddleware: + return ActiveRequestMiddleware( + Starlette(routes=[Route("/v1/images/generations", self.route, methods=["POST"])]) + ) + + +def _scope() -> dict[str, object]: + return { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/v1/images/generations", + "raw_path": b"/v1/images/generations", + "root_path": "", + "query_string": b"", + "headers": [(b"content-type", b"application/json"), (b"content-length", str(len(BODY)).encode())], + "client": ("127.0.0.1", 1234), + "server": ("testserver", 80), + } + + +async def _run(app: ActiveRequestMiddleware, messages_after_body: tuple[Message, ...]) -> None: + pending: Final = [{"type": "http.request", "body": BODY, "more_body": False}, *messages_after_body] + forever: Final = asyncio.Event() + + async def receive() -> Message: + if pending: + return pending.pop(0) + await forever.wait() + return {"type": "http.disconnect"} + + async def send(message: Message) -> None: + del message + + await app(_scope(), receive, send) + + +@pytest.mark.asyncio +async def test_non_chat_route_sees_client_disconnect_through_active_request(): + probe: Final = DisconnectProbe() + await _run(probe.app(), ({"type": "http.disconnect"},)) + assert probe.observed == [True] + + +@pytest.mark.asyncio +async def test_connected_client_is_not_reported_as_disconnected(): + probe: Final = DisconnectProbe() + await _run(probe.app(), ()) + assert probe.observed == [False] + + +@pytest.mark.asyncio +async def test_active_request_is_cleared_once_the_response_is_sent(): + await _run(DisconnectProbe().app(), ()) + assert active_request.get() is None