From 5c63a56bce18c732d8fed072b30bc8a39c53a726 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Wed, 12 Aug 2026 22:50:02 -0700 Subject: [PATCH] fix(observability): publish the ceiling without a database, and count only shedding this proxy did Three defects, each reproduced on a live proxy before the fix. The ceiling gauge was published inside the database-gated startup branch, so a prometheus proxy with no DATABASE_URL never published and the registered gauge rendered as 0, which reads as "no requests allowed" on a proxy serving every request. Publishing is not conditional on a database: concurrency is bounded per worker either way. The shed counter counted any 429 or 503 seen at the ASGI layer, but litellm forwards upstream rate limits with the same 429 it uses for its own, so a provider throttling us was recorded as this pod shedding load, inverting the throttle-or-scale decision. Requests the proxy itself declines are now marked at ProxyRateLimitError, the one class litellm raises for that, and only marked responses count. The ceiling gauge is also republished when general_settings is reloaded, since it previously went stale for the life of the process. --- .../common_utils/proxy_rate_limit_error.py | 4 ++ .../common_utils/request_pressure_metrics.py | 16 +++++++ .../in_flight_requests_middleware.py | 9 +++- litellm/proxy/proxy_server.py | 6 ++- .../test_in_flight_requests_middleware.py | 47 +++++++++++++------ 5 files changed, 65 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/common_utils/proxy_rate_limit_error.py b/litellm/proxy/common_utils/proxy_rate_limit_error.py index c109da6f571..e77da101156 100644 --- a/litellm/proxy/common_utils/proxy_rate_limit_error.py +++ b/litellm/proxy/common_utils/proxy_rate_limit_error.py @@ -98,6 +98,9 @@ def _coerce_message(detail: Any) -> str: # Both narrowings are intentional and handled at construction time — every # instance always has status_code == 429 and a Dict-typed headers — so we # silence the ATTR-overlap check rather than relax the annotations. +from litellm.proxy.common_utils.request_pressure_metrics import mark_request_shed_by_proxy + + class ProxyRateLimitError(HTTPException, RateLimitError): """ A 429 raised by litellm's proxy-side rate limiting hooks. @@ -155,6 +158,7 @@ class ProxyRateLimitError(HTTPException, RateLimitError): # instance whose `.llm_provider` attribute is `None` — that would # break Prometheus' `_get_exception_class_name` (it calls # `.capitalize()` on the provider string). + mark_request_shed_by_proxy() model = model or "" llm_provider = llm_provider or "litellm_proxy" message: Final = _coerce_message(detail) diff --git a/litellm/proxy/common_utils/request_pressure_metrics.py b/litellm/proxy/common_utils/request_pressure_metrics.py index d7623d62c34..c9b872db751 100644 --- a/litellm/proxy/common_utils/request_pressure_metrics.py +++ b/litellm/proxy/common_utils/request_pressure_metrics.py @@ -15,12 +15,28 @@ See LIT-5460 for the enforcement gap itself. from __future__ import annotations +from contextvars import ContextVar from typing import Final from litellm._logging import verbose_proxy_logger UNBOUNDED: Final = float("inf") +# Set where the proxy itself declines a request. litellm forwards upstream +# rate limits with the same 429 the proxy uses for its own, so response status +# alone cannot tell "this pod shed load" from "the provider throttled us", and +# only the former should count toward a decision to throttle or scale out. +proxy_shed_request: Final[ContextVar[bool]] = ContextVar("litellm_proxy_shed_request", default=False) + + +def mark_request_shed_by_proxy() -> None: + """Record that this proxy, not an upstream, declined the current request.""" + proxy_shed_request.set(True) + + +def was_request_shed_by_proxy() -> bool: + return proxy_shed_request.get() + def is_global_limit_enforced() -> bool: """Whether the registered limiter reads ``global_max_parallel_requests``.""" diff --git a/litellm/proxy/middleware/in_flight_requests_middleware.py b/litellm/proxy/middleware/in_flight_requests_middleware.py index e414a85c946..0172c93511d 100644 --- a/litellm/proxy/middleware/in_flight_requests_middleware.py +++ b/litellm/proxy/middleware/in_flight_requests_middleware.py @@ -83,14 +83,19 @@ class InFlightRequestsMiddleware: return InFlightRequestsMiddleware._gauge -# Statuses that mean the proxy declined to serve rather than failed to. Kept to -# a fixed set so the metric label cannot grow. +# Statuses the proxy uses when it declines to serve. A response only counts once +# the request is also marked as shed by this proxy, since litellm forwards +# upstream 429s with the same status. _SHED_STATUSES: Final = frozenset({429, 503}) def _record_shed_response(status: int) -> None: if status not in _SHED_STATUSES: return + from litellm.proxy.common_utils.request_pressure_metrics import was_request_shed_by_proxy + + if not was_request_shed_by_proxy(): + return try: from litellm.integrations.prometheus import PrometheusLogger diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5c9b9ef8fd2..0ef67722078 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1217,6 +1217,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: prisma_client=prisma_client, ) + # Not gated on the database: concurrency is bounded per worker regardless, and + # a registered gauge always exposes a value, so skipping this would publish 0. + publish_global_max_parallel_requests(general_settings.get("global_max_parallel_requests")) + ### START BATCH WRITING DB + CHECKING NEW MODELS### worker_heartbeat: Final = ( await ProxyStartupEvent.initialize_scheduled_background_jobs( @@ -6326,6 +6330,7 @@ class ProxyConfig: if "global_max_parallel_requests" in _general_settings: general_settings["global_max_parallel_requests"] = _general_settings["global_max_parallel_requests"] + publish_global_max_parallel_requests(general_settings["global_max_parallel_requests"]) if "max_batch_file_size_mb" not in self._yaml_general_settings_keys: general_settings["max_batch_file_size_mb"] = _general_settings.get("max_batch_file_size_mb") @@ -9138,7 +9143,6 @@ class ProxyStartupEvent: ) # Registered before start so the first run of every job is observed. ScheduledJobMetricsListener().register(scheduler) - publish_global_max_parallel_requests(general_settings.get("global_max_parallel_requests")) # Start the scheduler immediately without processing backlogs scheduler.start(paused=False) diff --git a/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py index 544ebb08251..0ea0f53762f 100644 --- a/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py +++ b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py @@ -99,11 +99,12 @@ def test_non_http_scopes_not_counted(): @pytest.mark.asyncio @pytest.mark.parametrize("status, expected_calls", [(429, 1), (503, 1), (200, 0), (400, 0), (500, 0)]) -async def test_only_shed_responses_are_counted(status, expected_calls): - """A 500 is the proxy failing, not declining. Counting it here would blur the +async def test_only_shed_responses_the_proxy_itself_produced_are_counted(status, expected_calls): + """A 500 is the proxy failing, not declining. Counting it would blur the signal an operator uses to decide between throttling and scaling out.""" from unittest.mock import MagicMock, patch + from litellm.proxy.common_utils.request_pressure_metrics import mark_request_shed_by_proxy from litellm.proxy.middleware.in_flight_requests_middleware import ( InFlightRequestsMiddleware, ) @@ -117,6 +118,7 @@ async def test_only_shed_responses_are_counted(status, expected_calls): return {"type": "http.request"} async def app(scope, receive, send): + mark_request_shed_by_proxy() await send({"type": "http.response.start", "status": status, "headers": []}) await send({"type": "http.response.body", "body": b""}) @@ -125,36 +127,53 @@ async def test_only_shed_responses_are_counted(status, expected_calls): await InFlightRequestsMiddleware(app)({"type": "http"}, receive, send) assert logger.record_request_shed.call_count == expected_calls - if expected_calls: - logger.record_request_shed.assert_called_once_with(status) assert [m["type"] for m in sent] == ["http.response.start", "http.response.body"], ( "the wrapped send must still forward every message downstream" ) @pytest.mark.asyncio -async def test_a_broken_metric_never_breaks_the_response(): - from unittest.mock import patch +async def test_a_provider_rate_limit_is_not_counted_as_this_pod_shedding(): + """litellm forwards an upstream 429 with the same status the proxy uses for + its own limits. Counting it would tell an operator to scale out when the + bottleneck is the provider.""" + from unittest.mock import MagicMock, patch from litellm.proxy.middleware.in_flight_requests_middleware import ( InFlightRequestsMiddleware, ) - sent = [] - async def send(message): - sent.append(message) + return None async def receive(): return {"type": "http.request"} async def app(scope, receive, send): + # no mark: the 429 came back from the provider, not from a proxy limiter await send({"type": "http.response.start", "status": 429, "headers": []}) - with patch( - "litellm.integrations.prometheus.PrometheusLogger.get_instance", - side_effect=RuntimeError("metrics down"), - ): + logger = MagicMock() + with patch("litellm.integrations.prometheus.PrometheusLogger.get_instance", return_value=logger): await InFlightRequestsMiddleware(app)({"type": "http"}, receive, send) - assert len(sent) == 1 + logger.record_request_shed.assert_not_called() + + +@pytest.mark.asyncio +async def test_the_proxys_own_rate_limit_error_marks_the_request(): + """ProxyRateLimitError is the one class litellm raises for its own 429s, so + constructing it is what distinguishes the two cases.""" + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + from litellm.proxy.common_utils.request_pressure_metrics import ( + proxy_shed_request, + was_request_shed_by_proxy, + ) + + token = proxy_shed_request.set(False) + try: + assert was_request_shed_by_proxy() is False + ProxyRateLimitError(detail={"error": "limit"}) + assert was_request_shed_by_proxy() is True + finally: + proxy_shed_request.reset(token)