diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 7e592b3bb0f..c09b91a713d 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -723,8 +723,10 @@ class PrometheusLogger(CustomLogger): self.litellm_requests_shed_total = self._counter_factory( name="litellm_requests_shed_total", documentation=( - "Responses where the proxy declined to serve rather than failed to, by status. " - "429 is a rate or concurrency limit, 503 is the database being unavailable" + "Responses where this proxy declined to serve rather than failed to, by status. " + "status is one of 429; upstream rate limits are forwarded with the same status and " + "are excluded, as are the 503s raised when a budget cannot be verified, which mean " + "a dependency is unreachable rather than this pod being at capacity" ), labelnames=("status",), ) diff --git a/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py index 9687a838c81..8b42dbc1414 100644 --- a/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py +++ b/tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py @@ -178,3 +178,30 @@ async def test_the_proxys_own_rate_limit_error_marks_the_request(): assert was_request_shed_by_proxy() is True finally: proxy_shed_request.reset(token) + +def test_the_shed_metric_documents_exactly_the_statuses_it_counts(monkeypatch): + """The advertised status set is what a consumer writes alerts against, so a + status the metric names but never counts inflates their view of shedding.""" + import re + + import litellm + from litellm.integrations.prometheus import PrometheusLogger + from litellm.proxy.middleware.in_flight_requests_middleware import _SHED_STATUSES + + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "success_callback", []) + from prometheus_client import REGISTRY + + for collector in list(REGISTRY._collector_to_names.keys()): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + documentation = PrometheusLogger().litellm_requests_shed_total._documentation + + advertised = re.search(r"status is one of ([^;]+);", documentation) + assert advertised is not None, f"no advertised status list in: {documentation}" + documented = {int(value.strip()) for value in advertised.group(1).split(",")} + + assert documented == set(_SHED_STATUSES)