diff --git a/litellm/proxy/common_utils/proxy_rate_limit_error.py b/litellm/proxy/common_utils/proxy_rate_limit_error.py index c109da6f571..e77da101156 100644 --- a/litellm/proxy/common_utils/proxy_rate_limit_error.py +++ b/litellm/proxy/common_utils/proxy_rate_limit_error.py @@ -98,6 +98,9 @@ def _coerce_message(detail: Any) -> str: # Both narrowings are intentional and handled at construction time — every # instance always has status_code == 429 and a Dict-typed headers — so we # silence the ATTR-overlap check rather than relax the annotations. +from litellm.proxy.common_utils.request_pressure_metrics import mark_request_shed_by_proxy + + class ProxyRateLimitError(HTTPException, RateLimitError): """ A 429 raised by litellm's proxy-side rate limiting hooks. @@ -155,6 +158,7 @@ class ProxyRateLimitError(HTTPException, RateLimitError): # instance whose `.llm_provider` attribute is `None` — that would # break Prometheus' `_get_exception_class_name` (it calls # `.capitalize()` on the provider string). + mark_request_shed_by_proxy() model = model or "" llm_provider = llm_provider or "litellm_proxy" message: Final = _coerce_message(detail) diff --git a/litellm/proxy/common_utils/request_pressure_metrics.py b/litellm/proxy/common_utils/request_pressure_metrics.py index d7623d62c34..c9b872db751 100644 --- a/litellm/proxy/common_utils/request_pressure_metrics.py +++ b/litellm/proxy/common_utils/request_pressure_metrics.py @@ -15,12 +15,28 @@ See LIT-5460 for the enforcement gap itself. from __future__ import annotations +from contextvars import ContextVar from typing import Final from litellm._logging import verbose_proxy_logger UNBOUNDED: Final = float("inf") +# Set where the proxy itself declines a request. litellm forwards upstream +# rate limits with the same 429 the proxy uses for its own, so response status +# alone cannot tell "this pod shed load" from "the provider throttled us", and +# only the former should count toward a decision to throttle or scale out. +proxy_shed_request: Final[ContextVar[bool]] = ContextVar("litellm_proxy_shed_request", default=False) + + +def mark_request_shed_by_proxy() -> None: + """Record that this proxy, not an upstream, declined the current request.""" + proxy_shed_request.set(True) + + +def was_request_shed_by_proxy() -> bool: + return proxy_shed_request.get() + def is_global_limit_enforced() -> bool: """Whether the registered limiter reads ``global_max_parallel_requests``.""" diff --git a/litellm/proxy/middleware/in_flight_requests_middleware.py b/litellm/proxy/middleware/in_flight_requests_middleware.py index e414a85c946..0172c93511d 100644 --- a/litellm/proxy/middleware/in_flight_requests_middleware.py +++ b/litellm/proxy/middleware/in_flight_requests_middleware.py @@ -83,14 +83,19 @@ class InFlightRequestsMiddleware: return InFlightRequestsMiddleware._gauge -# Statuses that mean the proxy declined to serve rather than failed to. Kept to -# a fixed set so the metric label cannot grow. +# Statuses the proxy uses when it declines to serve. A response only counts once +# the request is also marked as shed by this proxy, since litellm forwards +# upstream 429s with the same status. _SHED_STATUSES: Final = frozenset({429, 503}) def _record_shed_response(status: int) -> None: if status not in _SHED_STATUSES: return + from litellm.proxy.common_utils.request_pressure_metrics import was_request_shed_by_proxy + + if not was_request_shed_by_proxy(): + return try: from litellm.integrations.prometheus import PrometheusLogger diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5c9b9ef8fd2..0ef67722078 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1217,6 +1217,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: prisma_client=prisma_client, ) + # Not gated on the database: concurrency is bounded per worker regardless, and + # a registered gauge always exposes a value, so skipping this would publish 0. + publish_global_max_parallel_requests(general_settings.get("global_max_parallel_requests")) + ### START BATCH WRITING DB + CHECKING NEW MODELS### worker_heartbeat: Final = ( await ProxyStartupEvent.initialize_scheduled_background_jobs( @@ -6326,6 +6330,7 @@ class ProxyConfig: if "global_max_parallel_requests" in _general_settings: general_settings["global_max_parallel_requests"] = _general_settings["global_max_parallel_requests"] + publish_global_max_parallel_requests(general_settings["global_max_parallel_requests"]) if "max_batch_file_size_mb" not in self._yaml_general_settings_keys: general_settings["max_batch_file_size_mb"] = _general_settings.get("max_batch_file_size_mb") @@ -9138,7 +9143,6 @@ class ProxyStartupEvent: ) # Registered before start so the first run of every job is observed. ScheduledJobMetricsListener().register(scheduler) - publish_global_max_parallel_requests(general_settings.get("global_max_parallel_requests")) # Start the scheduler immediately without processing backlogs scheduler.start(paused=False) diff --git a/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py index 544ebb08251..0ea0f53762f 100644 --- a/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py +++ b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py @@ -99,11 +99,12 @@ def test_non_http_scopes_not_counted(): @pytest.mark.asyncio @pytest.mark.parametrize("status, expected_calls", [(429, 1), (503, 1), (200, 0), (400, 0), (500, 0)]) -async def test_only_shed_responses_are_counted(status, expected_calls): - """A 500 is the proxy failing, not declining. Counting it here would blur the +async def test_only_shed_responses_the_proxy_itself_produced_are_counted(status, expected_calls): + """A 500 is the proxy failing, not declining. Counting it would blur the signal an operator uses to decide between throttling and scaling out.""" from unittest.mock import MagicMock, patch + from litellm.proxy.common_utils.request_pressure_metrics import mark_request_shed_by_proxy from litellm.proxy.middleware.in_flight_requests_middleware import ( InFlightRequestsMiddleware, ) @@ -117,6 +118,7 @@ async def test_only_shed_responses_are_counted(status, expected_calls): return {"type": "http.request"} async def app(scope, receive, send): + mark_request_shed_by_proxy() await send({"type": "http.response.start", "status": status, "headers": []}) await send({"type": "http.response.body", "body": b""}) @@ -125,36 +127,53 @@ async def test_only_shed_responses_are_counted(status, expected_calls): await InFlightRequestsMiddleware(app)({"type": "http"}, receive, send) assert logger.record_request_shed.call_count == expected_calls - if expected_calls: - logger.record_request_shed.assert_called_once_with(status) assert [m["type"] for m in sent] == ["http.response.start", "http.response.body"], ( "the wrapped send must still forward every message downstream" ) @pytest.mark.asyncio -async def test_a_broken_metric_never_breaks_the_response(): - from unittest.mock import patch +async def test_a_provider_rate_limit_is_not_counted_as_this_pod_shedding(): + """litellm forwards an upstream 429 with the same status the proxy uses for + its own limits. Counting it would tell an operator to scale out when the + bottleneck is the provider.""" + from unittest.mock import MagicMock, patch from litellm.proxy.middleware.in_flight_requests_middleware import ( InFlightRequestsMiddleware, ) - sent = [] - async def send(message): - sent.append(message) + return None async def receive(): return {"type": "http.request"} async def app(scope, receive, send): + # no mark: the 429 came back from the provider, not from a proxy limiter await send({"type": "http.response.start", "status": 429, "headers": []}) - with patch( - "litellm.integrations.prometheus.PrometheusLogger.get_instance", - side_effect=RuntimeError("metrics down"), - ): + logger = MagicMock() + with patch("litellm.integrations.prometheus.PrometheusLogger.get_instance", return_value=logger): await InFlightRequestsMiddleware(app)({"type": "http"}, receive, send) - assert len(sent) == 1 + logger.record_request_shed.assert_not_called() + + +@pytest.mark.asyncio +async def test_the_proxys_own_rate_limit_error_marks_the_request(): + """ProxyRateLimitError is the one class litellm raises for its own 429s, so + constructing it is what distinguishes the two cases.""" + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + from litellm.proxy.common_utils.request_pressure_metrics import ( + proxy_shed_request, + was_request_shed_by_proxy, + ) + + token = proxy_shed_request.set(False) + try: + assert was_request_shed_by_proxy() is False + ProxyRateLimitError(detail={"error": "limit"}) + assert was_request_shed_by_proxy() is True + finally: + proxy_shed_request.reset(token)