fix(observability): publish the ceiling without a database, and count only shedding this proxy did

Three defects, each reproduced on a live proxy before the fix.

The ceiling gauge was published inside the database-gated startup branch, so a
prometheus proxy with no DATABASE_URL never published and the registered gauge
rendered as 0, which reads as "no requests allowed" on a proxy serving every
request. Publishing is not conditional on a database: concurrency is bounded per
worker either way.

The shed counter counted any 429 or 503 seen at the ASGI layer, but litellm
forwards upstream rate limits with the same 429 it uses for its own, so a
provider throttling us was recorded as this pod shedding load, inverting the
throttle-or-scale decision. Requests the proxy itself declines are now marked at
ProxyRateLimitError, the one class litellm raises for that, and only marked
responses count.

The ceiling gauge is also republished when general_settings is reloaded, since
it previously went stale for the life of the process.

litellm/proxy/common_utils/proxy_rate_limit_error.py

@@ -98,6 +98,9 @@ def _coerce_message(detail: Any) -> str:
 # Both narrowings are intentional and handled at construction time — every
 # instance always has status_code == 429 and a Dict-typed headers — so we
 # silence the ATTR-overlap check rather than relax the annotations.
+from litellm.proxy.common_utils.request_pressure_metrics import mark_request_shed_by_proxy
+
+
 class ProxyRateLimitError(HTTPException, RateLimitError):
     """
     A 429 raised by litellm's proxy-side rate limiting hooks.
@@ -155,6 +158,7 @@ class ProxyRateLimitError(HTTPException, RateLimitError):
         # instance whose `.llm_provider` attribute is `None` — that would
         # break Prometheus' `_get_exception_class_name` (it calls
         # `.capitalize()` on the provider string).
+        mark_request_shed_by_proxy()
         model = model or ""
         llm_provider = llm_provider or "litellm_proxy"
         message: Final = _coerce_message(detail)

litellm/proxy/common_utils/request_pressure_metrics.py

@@ -15,12 +15,28 @@ See LIT-5460 for the enforcement gap itself.
 
 from __future__ import annotations
 
+from contextvars import ContextVar
 from typing import Final
 
 from litellm._logging import verbose_proxy_logger
 
 UNBOUNDED: Final = float("inf")
 
+# Set where the proxy itself declines a request. litellm forwards upstream
+# rate limits with the same 429 the proxy uses for its own, so response status
+# alone cannot tell "this pod shed load" from "the provider throttled us", and
+# only the former should count toward a decision to throttle or scale out.
+proxy_shed_request: Final[ContextVar[bool]] = ContextVar("litellm_proxy_shed_request", default=False)
+
+
+def mark_request_shed_by_proxy() -> None:
+    """Record that this proxy, not an upstream, declined the current request."""
+    proxy_shed_request.set(True)
+
+
+def was_request_shed_by_proxy() -> bool:
+    return proxy_shed_request.get()
+
 
 def is_global_limit_enforced() -> bool:
     """Whether the registered limiter reads ``global_max_parallel_requests``."""

litellm/proxy/middleware/in_flight_requests_middleware.py

@@ -83,14 +83,19 @@ class InFlightRequestsMiddleware:
         return InFlightRequestsMiddleware._gauge
 
 
-# Statuses that mean the proxy declined to serve rather than failed to. Kept to
-# a fixed set so the metric label cannot grow.
+# Statuses the proxy uses when it declines to serve. A response only counts once
+# the request is also marked as shed by this proxy, since litellm forwards
+# upstream 429s with the same status.
 _SHED_STATUSES: Final = frozenset({429, 503})
 
 
 def _record_shed_response(status: int) -> None:
     if status not in _SHED_STATUSES:
         return
+    from litellm.proxy.common_utils.request_pressure_metrics import was_request_shed_by_proxy
+
+    if not was_request_shed_by_proxy():
+        return
     try:
         from litellm.integrations.prometheus import PrometheusLogger
 

litellm/proxy/proxy_server.py

@@ -1217,6 +1217,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
         prisma_client=prisma_client,
     )
 
+    # Not gated on the database: concurrency is bounded per worker regardless, and
+    # a registered gauge always exposes a value, so skipping this would publish 0.
+    publish_global_max_parallel_requests(general_settings.get("global_max_parallel_requests"))
+
     ### START BATCH WRITING DB + CHECKING NEW MODELS###
     worker_heartbeat: Final = (
         await ProxyStartupEvent.initialize_scheduled_background_jobs(
@@ -6326,6 +6330,7 @@ class ProxyConfig:
 
         if "global_max_parallel_requests" in _general_settings:
             general_settings["global_max_parallel_requests"] = _general_settings["global_max_parallel_requests"]
+            publish_global_max_parallel_requests(general_settings["global_max_parallel_requests"])
 
         if "max_batch_file_size_mb" not in self._yaml_general_settings_keys:
             general_settings["max_batch_file_size_mb"] = _general_settings.get("max_batch_file_size_mb")
@@ -9138,7 +9143,6 @@ class ProxyStartupEvent:
         )
         # Registered before start so the first run of every job is observed.
         ScheduledJobMetricsListener().register(scheduler)
-        publish_global_max_parallel_requests(general_settings.get("global_max_parallel_requests"))
 
         # Start the scheduler immediately without processing backlogs
         scheduler.start(paused=False)

tests/test_litellm/proxy/middleware/test_in_flight_requests_middleware.py

@@ -99,11 +99,12 @@ def test_non_http_scopes_not_counted():
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("status, expected_calls", [(429, 1), (503, 1), (200, 0), (400, 0), (500, 0)])
-async def test_only_shed_responses_are_counted(status, expected_calls):
-    """A 500 is the proxy failing, not declining. Counting it here would blur the
+async def test_only_shed_responses_the_proxy_itself_produced_are_counted(status, expected_calls):
+    """A 500 is the proxy failing, not declining. Counting it would blur the
     signal an operator uses to decide between throttling and scaling out."""
     from unittest.mock import MagicMock, patch
 
+    from litellm.proxy.common_utils.request_pressure_metrics import mark_request_shed_by_proxy
     from litellm.proxy.middleware.in_flight_requests_middleware import (
         InFlightRequestsMiddleware,
     )
@@ -117,6 +118,7 @@ async def test_only_shed_responses_are_counted(status, expected_calls):
         return {"type": "http.request"}
 
     async def app(scope, receive, send):
+        mark_request_shed_by_proxy()
         await send({"type": "http.response.start", "status": status, "headers": []})
         await send({"type": "http.response.body", "body": b""})
 
@@ -125,36 +127,53 @@ async def test_only_shed_responses_are_counted(status, expected_calls):
         await InFlightRequestsMiddleware(app)({"type": "http"}, receive, send)
 
     assert logger.record_request_shed.call_count == expected_calls
-    if expected_calls:
-        logger.record_request_shed.assert_called_once_with(status)
     assert [m["type"] for m in sent] == ["http.response.start", "http.response.body"], (
         "the wrapped send must still forward every message downstream"
     )
 
 
 @pytest.mark.asyncio
-async def test_a_broken_metric_never_breaks_the_response():
-    from unittest.mock import patch
+async def test_a_provider_rate_limit_is_not_counted_as_this_pod_shedding():
+    """litellm forwards an upstream 429 with the same status the proxy uses for
+    its own limits. Counting it would tell an operator to scale out when the
+    bottleneck is the provider."""
+    from unittest.mock import MagicMock, patch
 
     from litellm.proxy.middleware.in_flight_requests_middleware import (
         InFlightRequestsMiddleware,
     )
 
-    sent = []
-
     async def send(message):
-        sent.append(message)
+        return None
 
     async def receive():
         return {"type": "http.request"}
 
     async def app(scope, receive, send):
+        # no mark: the 429 came back from the provider, not from a proxy limiter
         await send({"type": "http.response.start", "status": 429, "headers": []})
 
-    with patch(
-        "litellm.integrations.prometheus.PrometheusLogger.get_instance",
-        side_effect=RuntimeError("metrics down"),
-    ):
+    logger = MagicMock()
+    with patch("litellm.integrations.prometheus.PrometheusLogger.get_instance", return_value=logger):
         await InFlightRequestsMiddleware(app)({"type": "http"}, receive, send)
 
-    assert len(sent) == 1
+    logger.record_request_shed.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_the_proxys_own_rate_limit_error_marks_the_request():
+    """ProxyRateLimitError is the one class litellm raises for its own 429s, so
+    constructing it is what distinguishes the two cases."""
+    from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
+    from litellm.proxy.common_utils.request_pressure_metrics import (
+        proxy_shed_request,
+        was_request_shed_by_proxy,
+    )
+
+    token = proxy_shed_request.set(False)
+    try:
+        assert was_request_shed_by_proxy() is False
+        ProxyRateLimitError(detail={"error": "limit"})
+        assert was_request_shed_by_proxy() is True
+    finally:
+        proxy_shed_request.reset(token)