diff --git a/litellm/proxy/middleware/in_flight_requests_middleware.py b/litellm/proxy/middleware/in_flight_requests_middleware.py index 0172c93511d..278a585d086 100644 --- a/litellm/proxy/middleware/in_flight_requests_middleware.py +++ b/litellm/proxy/middleware/in_flight_requests_middleware.py @@ -83,10 +83,12 @@ class InFlightRequestsMiddleware: return InFlightRequestsMiddleware._gauge -# Statuses the proxy uses when it declines to serve. A response only counts once -# the request is also marked as shed by this proxy, since litellm forwards -# upstream 429s with the same status. -_SHED_STATUSES: Final = frozenset({429, 503}) +# The status this proxy uses when it declines load, and only counted once the +# request is also marked as shed here, since litellm forwards upstream 429s with +# the same status. The proxy's 503s are fail-closed budget rejections, a +# dependency being unreachable rather than this pod at capacity, so counting +# them would blur the throttle-vs-scale signal the metric exists to give. +_SHED_STATUSES: Final = frozenset({429}) def _record_shed_response(status: int) -> None: diff --git a/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py index 0ea0f53762f..9687a838c81 100644 --- a/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py +++ b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py @@ -98,10 +98,11 @@ def test_non_http_scopes_not_counted(): @pytest.mark.asyncio -@pytest.mark.parametrize("status, expected_calls", [(429, 1), (503, 1), (200, 0), (400, 0), (500, 0)]) +@pytest.mark.parametrize("status, expected_calls", [(429, 1), (503, 0), (200, 0), (400, 0), (500, 0)]) async def test_only_shed_responses_the_proxy_itself_produced_are_counted(status, expected_calls): - """A 500 is the proxy failing, not declining. Counting it would blur the - signal an operator uses to decide between throttling and scaling out.""" + """A 500 is the proxy failing, not declining, and the proxy's 503s are + fail-closed budget rejections raised when a dependency is unreachable. + Counting either would blur the throttle-vs-scale signal.""" from unittest.mock import MagicMock, patch from litellm.proxy.common_utils.request_pressure_metrics import mark_request_shed_by_proxy