From 6d54a4eedb42c9cac79dd1d08f49d1d80b29cd1e Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Wed, 19 Aug 2026 14:11:37 -0700 Subject: [PATCH] fix(observability): document exactly the statuses the shed counter counts Removing 503 from the counted set left the metric still advertising it, so a consumer would have kept alerting on a status that can no longer appear. The documentation now names the counted status and says why the two excluded kinds are excluded: upstream rate limits arrive as 429 as well, and the proxy's 503s mean a budget could not be verified, so a dependency is unreachable rather than this pod being at capacity. The test parses the advertised list rather than substring-matching it, and clears the collector registry first, since a second PrometheusLogger in one process otherwise trips the duplicate-collector guard depending on test order. --- litellm/integrations/prometheus.py | 6 +++-- .../test_in_flight_requests_middleware.py | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 7e592b3bb0f..c09b91a713d 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -723,8 +723,10 @@ class PrometheusLogger(CustomLogger): self.litellm_requests_shed_total = self._counter_factory( name="litellm_requests_shed_total", documentation=( - "Responses where the proxy declined to serve rather than failed to, by status. " - "429 is a rate or concurrency limit, 503 is the database being unavailable" + "Responses where this proxy declined to serve rather than failed to, by status. " + "status is one of 429; upstream rate limits are forwarded with the same status and " + "are excluded, as are the 503s raised when a budget cannot be verified, which mean " + "a dependency is unreachable rather than this pod being at capacity" ), labelnames=("status",), ) diff --git a/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py index 9687a838c81..8b42dbc1414 100644 --- a/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py +++ b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py @@ -178,3 +178,30 @@ async def test_the_proxys_own_rate_limit_error_marks_the_request(): assert was_request_shed_by_proxy() is True finally: proxy_shed_request.reset(token) + +def test_the_shed_metric_documents_exactly_the_statuses_it_counts(monkeypatch): + """The advertised status set is what a consumer writes alerts against, so a + status the metric names but never counts inflates their view of shedding.""" + import re + + import litellm + from litellm.integrations.prometheus import PrometheusLogger + from litellm.proxy.middleware.in_flight_requests_middleware import _SHED_STATUSES + + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "success_callback", []) + from prometheus_client import REGISTRY + + for collector in list(REGISTRY._collector_to_names.keys()): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + documentation = PrometheusLogger().litellm_requests_shed_total._documentation + + advertised = re.search(r"status is one of ([^;]+);", documentation) + assert advertised is not None, f"no advertised status list in: {documentation}" + documented = {int(value.strip()) for value in advertised.group(1).split(",")} + + assert documented == set(_SHED_STATUSES)