From 7c8c07532fd840005c0903a3183a9980a283b647 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Wed, 19 Aug 2026 13:33:42 -0700 Subject: [PATCH] fix(observability): count only the status this proxy sheds load with `_SHED_STATUSES` carried 503 alongside 429, but nothing ever marks a 503 as shed by this proxy, so the entry was unreachable and read as though those responses were counted. Leaving it and marking the 503 paths would be wrong. Both of them are fail-closed budget rejections, raised when spend cannot be verified against Redis or the database. That is a dependency being unreachable, not this pod running out of capacity, and the two call for opposite responses. Folding them in would rebuild exactly the conflation that dropping provider 429s removed. The status label stays, so a future rejection kind can be added deliberately along with whatever marks it. --- .../proxy/middleware/in_flight_requests_middleware.py | 10 ++++++---- .../middleware/test_in_flight_requests_middleware.py | 7 ++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/middleware/in_flight_requests_middleware.py b/litellm/proxy/middleware/in_flight_requests_middleware.py index 0172c93511d..278a585d086 100644 --- a/litellm/proxy/middleware/in_flight_requests_middleware.py +++ b/litellm/proxy/middleware/in_flight_requests_middleware.py @@ -83,10 +83,12 @@ class InFlightRequestsMiddleware: return InFlightRequestsMiddleware._gauge -# Statuses the proxy uses when it declines to serve. A response only counts once -# the request is also marked as shed by this proxy, since litellm forwards -# upstream 429s with the same status. -_SHED_STATUSES: Final = frozenset({429, 503}) +# The status this proxy uses when it declines load, and only counted once the +# request is also marked as shed here, since litellm forwards upstream 429s with +# the same status. The proxy's 503s are fail-closed budget rejections, a +# dependency being unreachable rather than this pod at capacity, so counting +# them would blur the throttle-vs-scale signal the metric exists to give. +_SHED_STATUSES: Final = frozenset({429}) def _record_shed_response(status: int) -> None: diff --git a/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py index 0ea0f53762f..9687a838c81 100644 --- a/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py +++ b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py @@ -98,10 +98,11 @@ def test_non_http_scopes_not_counted(): @pytest.mark.asyncio -@pytest.mark.parametrize("status, expected_calls", [(429, 1), (503, 1), (200, 0), (400, 0), (500, 0)]) +@pytest.mark.parametrize("status, expected_calls", [(429, 1), (503, 0), (200, 0), (400, 0), (500, 0)]) async def test_only_shed_responses_the_proxy_itself_produced_are_counted(status, expected_calls): - """A 500 is the proxy failing, not declining. Counting it would blur the - signal an operator uses to decide between throttling and scaling out.""" + """A 500 is the proxy failing, not declining, and the proxy's 503s are + fail-closed budget rejections raised when a dependency is unreachable. + Counting either would blur the throttle-vs-scale signal.""" from unittest.mock import MagicMock, patch from litellm.proxy.common_utils.request_pressure_metrics import mark_request_shed_by_proxy