diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index c09b91a713d..08a7bfa49b3 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -885,6 +885,30 @@ class PrometheusLogger(CustomLogger): print_verbose(f"Got exception on init prometheus client {e}") raise e + self._restore_effective_concurrency_ceiling() + + def _restore_effective_concurrency_ceiling(self) -> None: + """Set the ceiling gauge to the value already in force. + + Startup publishes the ceiling, but ``success_callback: ["prometheus"]`` + builds this logger lazily on the first request, long after that ran with + no logger to publish to. The gauge would then register at its zero + default and stay there, reading as "no requests allowed" on a proxy + serving everything, which is the reading ``+Inf`` exists to prevent. + + Set directly rather than through the module helper, because during + ``__init__`` this instance is not yet reachable by the logger lookup. + """ + try: + from litellm.proxy.common_utils.request_pressure_metrics import effective_global_limit + from litellm.proxy.proxy_server import general_settings + + self.set_global_max_parallel_requests_limit( + effective_global_limit(general_settings.get("global_max_parallel_requests")) + ) + except Exception as e: # noqa: BLE001 # a logger that cannot read proxy config must still start + print_verbose(f"could not publish the concurrency ceiling on init: {e}") + def _parse_prometheus_config(self) -> dict[str, list[str]]: """Parse prometheus metrics configuration for label filtering and enabled metrics""" import litellm diff --git a/tests/test_litellm/proxy/common_utils/test_request_pressure_metrics.py b/tests/test_litellm/proxy/common_utils/test_request_pressure_metrics.py index 7d2c3d663ac..c9d2887f80b 100644 --- a/tests/test_litellm/proxy/common_utils/test_request_pressure_metrics.py +++ b/tests/test_litellm/proxy/common_utils/test_request_pressure_metrics.py @@ -104,3 +104,32 @@ def test_publishing_never_blocks_startup(): ), ): publish_global_max_parallel_requests(3) + +def test_a_lazily_built_logger_publishes_the_ceiling_already_in_force(monkeypatch): + """success_callback: ["prometheus"] builds the logger on the first request, + long after startup published the ceiling with no logger to publish to. The + gauge would register at its zero default and stay there, reading as "no + requests allowed" on a proxy serving everything.""" + from prometheus_client import REGISTRY + + import litellm + import litellm.proxy.proxy_server as proxy_server + from litellm.integrations.prometheus import PrometheusLogger + + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setitem(proxy_server.general_settings, "global_max_parallel_requests", 7) + monkeypatch.setattr( + "litellm.proxy.common_utils.request_pressure_metrics.is_global_limit_enforced", + lambda: True, + ) + for collector in list(REGISTRY._collector_to_names.keys()): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + PrometheusLogger() + + assert REGISTRY.get_sample_value("litellm_global_max_parallel_requests_limit") == 7.0 +